Tensor: Use TensorCostModel overhead constants in numThreadsInnerDim libeigen/eigen!3382 Co-authored-by: Antonio Sanchez <cantonios@google.com>
diff --git a/contrib/Eigen/src/Tensor/TensorContractionThreadPool.h b/contrib/Eigen/src/Tensor/TensorContractionThreadPool.h index 733c020..2b40248 100644 --- a/contrib/Eigen/src/Tensor/TensorContractionThreadPool.h +++ b/contrib/Eigen/src/Tensor/TensorContractionThreadPool.h
@@ -1520,8 +1520,8 @@ TensorCostModel<ThreadPoolDevice>::totalCost(m * n, TensorOpCost(2, 1, 1, true, output_packet_size)); int num_threads = 1; double min_cost = total_parallel_cost; - double kPerThreadOverHead = 3000; - double kFixedOverHead = 20000; + double kPerThreadOverHead = TensorCostModel<ThreadPoolDevice>::kPerThreadCycles; + double kFixedOverHead = TensorCostModel<ThreadPoolDevice>::kStartupCycles; for (int nt = 2; nt <= this->m_device.numThreads(); nt += 2) { double sequential_cost = kFixedOverHead + nt * (reduction_cost + kPerThreadOverHead); double parallel_cost = total_parallel_cost / nt + sequential_cost;
diff --git a/contrib/test/tensor_thread_pool.cpp b/contrib/test/tensor_thread_pool.cpp index aa6a406..83b5d49 100644 --- a/contrib/test/tensor_thread_pool.cpp +++ b/contrib/test/tensor_thread_pool.cpp
@@ -558,14 +558,15 @@ // We are triggering 'evalShardedByInnerDim' optimization. template <int DataLayout> static void test_sharded_by_inner_dim_contraction() { - typedef Tensor<float, 1>::DimensionPair DimPair; + using DimPair = typename Tensor<float, 1>::DimensionPair; const int num_threads = internal::random<int>(4, 16); ThreadPool threads(num_threads); Eigen::ThreadPoolDevice device(&threads, num_threads); - Tensor<float, 2, DataLayout> t_left(2, 10000); - Tensor<float, 2, DataLayout> t_right(10000, 10); + const Index k = 30000 * PacketType<float, Eigen::ThreadPoolDevice>::size; + Tensor<float, 2, DataLayout> t_left(2, k); + Tensor<float, 2, DataLayout> t_right(k, 10); Tensor<float, 2, DataLayout> t_result(2, 10); t_left.setRandom(); @@ -577,16 +578,23 @@ t_left += t_left.constant(1.0f); t_right += t_right.constant(1.0f); - typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf; - MapXf m_left(t_left.data(), 2, 10000); - MapXf m_right(t_right.data(), 10000, 10); + using MapXf = Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>>; + MapXf m_left(t_left.data(), 2, k); + MapXf m_right(t_right.data(), k, 10); Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10); // this contraction should be equivalent to a single matrix multiplication Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}}; + auto contraction = t_left.contract(t_right, dims); + using ContractionEval = TensorEvaluator<const decltype(contraction), Eigen::ThreadPoolDevice>; + ContractionEval eval(contraction, device); + const int nt_k = eval.numThreadsInnerDim(2, 10, k); + VERIFY(nt_k > 1); + VERIFY(ContractionEval::shardByInnerDim(2, 10, k, /*num_threads=*/1, nt_k)); + // compute results by separate methods - t_result.device(device) = t_left.contract(t_right, dims); + t_result.device(device) = contraction; m_result = m_left * m_right; for (Index i = 0; i < t_result.dimensions().TotalSize(); i++) { @@ -597,14 +605,15 @@ // We are triggering 'evalShardedByInnerDim' optimization with output kernel. template <int DataLayout> static void test_sharded_by_inner_dim_contraction_with_output_kernel() { - typedef Tensor<float, 1>::DimensionPair DimPair; + using DimPair = typename Tensor<float, 1>::DimensionPair; const int num_threads = internal::random<int>(4, 16); ThreadPool threads(num_threads); Eigen::ThreadPoolDevice device(&threads, num_threads); - Tensor<float, 2, DataLayout> t_left(2, 10000); - Tensor<float, 2, DataLayout> t_right(10000, 10); + const Index k = 30000 * PacketType<float, Eigen::ThreadPoolDevice>::size; + Tensor<float, 2, DataLayout> t_left(2, k); + Tensor<float, 2, DataLayout> t_right(k, 10); Tensor<float, 2, DataLayout> t_result(2, 10); t_left.setRandom(); @@ -616,16 +625,23 @@ t_left += t_left.constant(1.0f); t_right += t_right.constant(1.0f); - typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf; - MapXf m_left(t_left.data(), 2, 10000); - MapXf m_right(t_right.data(), 10000, 10); + using MapXf = Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>>; + MapXf m_left(t_left.data(), 2, k); + MapXf m_right(t_right.data(), k, 10); Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10); // this contraction should be equivalent to a single matrix multiplication Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}}; + auto contraction = t_left.contract(t_right, dims, SqrtOutputKernel()); + using ContractionEval = TensorEvaluator<const decltype(contraction), Eigen::ThreadPoolDevice>; + ContractionEval eval(contraction, device); + const int nt_k = eval.numThreadsInnerDim(2, 10, k); + VERIFY(nt_k > 1); + VERIFY(ContractionEval::shardByInnerDim(2, 10, k, /*num_threads=*/1, nt_k)); + // compute results by separate methods - t_result.device(device) = t_left.contract(t_right, dims, SqrtOutputKernel()); + t_result.device(device) = contraction; m_result = m_left * m_right; for (Index i = 0; i < t_result.dimensions().TotalSize(); i++) { @@ -636,14 +652,15 @@ // We are triggering 'evalShardedByInnerDim' optimization. template <int DataLayout> static void test_async_sharded_by_inner_dim_contraction() { - typedef Tensor<float, 1>::DimensionPair DimPair; + using DimPair = typename Tensor<float, 1>::DimensionPair; const int num_threads = internal::random<int>(4, 16); ThreadPool threads(num_threads); Eigen::ThreadPoolDevice device(&threads, num_threads); - Tensor<float, 2, DataLayout> t_left(2, 10000); - Tensor<float, 2, DataLayout> t_right(10000, 10); + const Index k = 30000 * PacketType<float, Eigen::ThreadPoolDevice>::size; + Tensor<float, 2, DataLayout> t_left(2, k); + Tensor<float, 2, DataLayout> t_right(k, 10); Tensor<float, 2, DataLayout> t_result(2, 10); t_left.setRandom(); @@ -655,17 +672,24 @@ t_left += t_left.constant(1.0f); t_right += t_right.constant(1.0f); - typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf; - MapXf m_left(t_left.data(), 2, 10000); - MapXf m_right(t_right.data(), 10000, 10); + using MapXf = Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>>; + MapXf m_left(t_left.data(), 2, k); + MapXf m_right(t_right.data(), k, 10); Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10); // this contraction should be equivalent to a single matrix multiplication Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}}; + auto contraction = t_left.contract(t_right, dims); + using ContractionEval = TensorEvaluator<const decltype(contraction), Eigen::ThreadPoolDevice>; + ContractionEval eval(contraction, device); + const int nt_k = eval.numThreadsInnerDim(2, 10, k); + VERIFY(nt_k > 1); + VERIFY(ContractionEval::shardByInnerDim(2, 10, k, /*num_threads=*/1, nt_k)); + // compute results by separate methods Eigen::Barrier barrier(1); - t_result.device(device, [&barrier]() { barrier.Notify(); }) = t_left.contract(t_right, dims); + t_result.device(device, [&barrier]() { barrier.Notify(); }) = contraction; barrier.Wait(); m_result = m_left * m_right; @@ -678,14 +702,15 @@ // We are triggering 'evalShardedByInnerDim' optimization with output kernel. template <int DataLayout> static void test_async_sharded_by_inner_dim_contraction_with_output_kernel() { - typedef Tensor<float, 1>::DimensionPair DimPair; + using DimPair = typename Tensor<float, 1>::DimensionPair; const int num_threads = internal::random<int>(4, 16); ThreadPool threads(num_threads); Eigen::ThreadPoolDevice device(&threads, num_threads); - Tensor<float, 2, DataLayout> t_left(2, 10000); - Tensor<float, 2, DataLayout> t_right(10000, 10); + const Index k = 30000 * PacketType<float, Eigen::ThreadPoolDevice>::size; + Tensor<float, 2, DataLayout> t_left(2, k); + Tensor<float, 2, DataLayout> t_right(k, 10); Tensor<float, 2, DataLayout> t_result(2, 10); t_left.setRandom(); @@ -697,17 +722,24 @@ t_left += t_left.constant(1.0f); t_right += t_right.constant(1.0f); - typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf; - MapXf m_left(t_left.data(), 2, 10000); - MapXf m_right(t_right.data(), 10000, 10); + using MapXf = Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>>; + MapXf m_left(t_left.data(), 2, k); + MapXf m_right(t_right.data(), k, 10); Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10); // this contraction should be equivalent to a single matrix multiplication Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}}; + auto contraction = t_left.contract(t_right, dims, SqrtOutputKernel()); + using ContractionEval = TensorEvaluator<const decltype(contraction), Eigen::ThreadPoolDevice>; + ContractionEval eval(contraction, device); + const int nt_k = eval.numThreadsInnerDim(2, 10, k); + VERIFY(nt_k > 1); + VERIFY(ContractionEval::shardByInnerDim(2, 10, k, /*num_threads=*/1, nt_k)); + // compute results by separate methods Eigen::Barrier barrier(1); - t_result.device(device, [&barrier]() { barrier.Notify(); }) = t_left.contract(t_right, dims, SqrtOutputKernel()); + t_result.device(device, [&barrier]() { barrier.Notify(); }) = contraction; barrier.Wait(); m_result = m_left * m_right; @@ -717,6 +749,48 @@ } template <int DataLayout> +static void test_sharded_by_inner_dim_cost_model() { + using DimPair = typename Tensor<float, 1>::DimensionPair; + Eigen::array<DimPair, 1> dims{{DimPair(1, 0)}}; + + ThreadPool threads(8); + Eigen::ThreadPoolDevice device(&threads, 8); + + Tensor<float, 2, DataLayout> t_left(2, 2); + Tensor<float, 2, DataLayout> t_right(2, 2); + auto expr = t_left.contract(t_right, dims); + using ContractionEval = TensorEvaluator<const decltype(expr), Eigen::ThreadPoolDevice>; + ContractionEval eval(expr, device); + + // Scale k by the packet width so the parallel cost (6.6e4..2.6e5 cycles across FMA and panel-width regimes) sits + // between the old hardcoded threshold (~5.7e4 cycles) and the TensorCostModel threshold (~4.0e5 cycles) under every + // SIMD width. + const Index f32_packet_size = PacketType<float, Eigen::ThreadPoolDevice>::size; + const Index moderate_f32_k = 64 * f32_packet_size; + VERIFY_IS_EQUAL(eval.numThreadsInnerDim(32, 64, moderate_f32_k), 1); + VERIFY(!ContractionEval::shardByInnerDim(32, 64, moderate_f32_k, /*num_threads=*/1, + eval.numThreadsInnerDim(32, 64, moderate_f32_k))); + + // Cover the bfloat16 contraction shape (16 x 128 x k) that motivated the fix. + Tensor<Eigen::bfloat16, 2, DataLayout> bf16_left(2, 2); + Tensor<Eigen::bfloat16, 2, DataLayout> bf16_right(2, 2); + auto bf16_expr = bf16_left.contract(bf16_right, dims); + using Bf16ContractionEval = TensorEvaluator<const decltype(bf16_expr), Eigen::ThreadPoolDevice>; + Bf16ContractionEval bf16_eval(bf16_expr, device); + const Index bf16_packet_size = PacketType<Eigen::bfloat16, Eigen::ThreadPoolDevice>::size; + const Index moderate_bf16_k = 64 * bf16_packet_size; + VERIFY_IS_EQUAL(bf16_eval.numThreadsInnerDim(16, 128, moderate_bf16_k), 1); + VERIFY(!Bf16ContractionEval::shardByInnerDim(16, 128, moderate_bf16_k, /*num_threads=*/1, + bf16_eval.numThreadsInnerDim(16, 128, moderate_bf16_k))); + + // Deep inner dimension with tiny outer dimensions should still shard by k. + const Index deep_f32_k = 30000 * f32_packet_size; + const int nt_k = eval.numThreadsInnerDim(2, 10, deep_f32_k); + VERIFY(nt_k > 1); + VERIFY(ContractionEval::shardByInnerDim(2, 10, deep_f32_k, /*num_threads=*/1, nt_k)); +} + +template <int DataLayout> void test_full_contraction() { int contract_size1 = internal::random<int>(1, 500); int contract_size2 = internal::random<int>(1, 500); @@ -1276,6 +1350,8 @@ CALL_SUBTEST_7(test_sharded_by_inner_dim_contraction<RowMajor>()); CALL_SUBTEST_7(test_sharded_by_inner_dim_contraction_with_output_kernel<ColMajor>()); CALL_SUBTEST_7(test_sharded_by_inner_dim_contraction_with_output_kernel<RowMajor>()); + CALL_SUBTEST_7(test_sharded_by_inner_dim_cost_model<ColMajor>()); + CALL_SUBTEST_7(test_sharded_by_inner_dim_cost_model<RowMajor>()); CALL_SUBTEST_8(test_async_sharded_by_inner_dim_contraction<ColMajor>()); CALL_SUBTEST_8(test_async_sharded_by_inner_dim_contraction<RowMajor>());