SME: Single-vector stores in the full-panel transposer; four depth chunks per fill for 64-bit elements libeigen/eigen!3340
diff --git a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h index 54bd064..3a52e3a 100644 --- a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h
@@ -246,8 +246,86 @@ sve_copy_panel_range<Conjugate>(dst, src, src_stride, Index(0), depth, width); } -// Full 2*svl-wide panel through all four tiles, four rows or depth steps per ZA move, source prefetched into L2 -// two steps ahead; returns the first depth index of the tail (< 2*svl) it leaves to the caller. +// Slices c..c+3 of tiles Lo and Hi as four whole packed lines from d, w scalars apart: Lo's slice is a line's first +// svl scalars, Hi's the rest. Single-vector stores: a two-vector store would first copy each line's halves into a +// consecutive register pair. +template <int Lo, int Hi, typename RealScalar> +static EIGEN_ALWAYS_INLINE void sme_store_lines_vg4(RealScalar* d, int w, int svl, + uint32_t c) __arm_streaming __arm_in("za") { + const svbool_t all = sme_packet_traits<RealScalar>::ptrue(); + const auto lo = sme_read_ver_za_vg4<Lo, RealScalar>(c), hi = sme_read_ver_za_vg4<Hi, RealScalar>(c); + pstoreu(all, d, pget<0>(lo)); + pstoreu(all, d + svl, pget<0>(hi)); + pstoreu(all, d + w, pget<1>(lo)); + pstoreu(all, d + w + svl, pget<1>(hi)); + pstoreu(all, d + 2 * w, pget<2>(lo)); + pstoreu(all, d + 2 * w + svl, pget<2>(hi)); + pstoreu(all, d + 3 * w, pget<3>(lo)); + pstoreu(all, d + 3 * w + svl, pget<3>(hi)); +} + +// Four depth chunks of a full panel per fill, which needs eight tiles. No-op where the element type has four tiles: +// the paired loop's two depth chunks already fill ZA. Returns the first depth index it leaves. +template <typename RealScalar, typename Index> +static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_four(RealScalar*, const RealScalar*, Index, Index k0, + Index) __arm_streaming EIGEN_SME_ZA_AGNOSTIC { + return k0; +} + +#ifdef EIGEN_VECTORIZE_SME_F64F64 +template <int Tile, typename Index> +static EIGEN_ALWAYS_INLINE void sme_write_rows_x4(uint32_t r, svcount_t pn, const double* p, + Index stride) __arm_streaming __arm_inout("za") { + const auto a0 = ploadu_x4(pn, p), a1 = ploadu_x4(pn, p + stride), a2 = ploadu_x4(pn, p + 2 * stride), + a3 = ploadu_x4(pn, p + 3 * stride); + sme_write_hor_za_vg4<Tile>(r, pget<0>(a0), pget<0>(a1), pget<0>(a2), pget<0>(a3)); + sme_write_hor_za_vg4<Tile + 1>(r, pget<1>(a0), pget<1>(a1), pget<1>(a2), pget<1>(a3)); + sme_write_hor_za_vg4<Tile + 2>(r, pget<2>(a0), pget<2>(a1), pget<2>(a2), pget<2>(a3)); + sme_write_hor_za_vg4<Tile + 3>(r, pget<3>(a0), pget<3>(a1), pget<3>(a2), pget<3>(a3)); +} + +// Both row-groups over four depth chunks in the eight ZA.D tiles, row-group 0 in tiles 0..3 and row-group 1 in tiles +// 4..7, so every packed line is stored whole; source prefetched into L2 two steps ahead, two prefetches per row and +// chunk (a whole chunk with 128-byte cache lines, as on Apple M4). +template <typename Index> +static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_four(double* EIGEN_RESTRICT dst, const double* EIGEN_RESTRICT src, + Index src_stride, Index k0, + Index k1) __arm_streaming __arm_inout("za") { + const int svl = sme_packet_traits<double>::size(), w = 2 * svl, d = 4 * svl; + const svcount_t pn = sme_packet_traits<double>::ptrue_c(); + Index k = k0; + for (; k + d <= k1; k += d) { + // Only a chunk inside the range: the prefetch address must stay within the source. + const bool pf = k + 3 * d <= k1; + for (int r = 0; r < svl; r += 4) { + const double* p = src + Index(r) * src_stride + k; + const double* q = p + Index(svl) * src_stride; + if (pf) { + for (int u = 0; u < 4; ++u) { + __builtin_prefetch(p + u * src_stride + 2 * d, 0, 2); + __builtin_prefetch(p + u * src_stride + 2 * d + w, 0, 2); + __builtin_prefetch(q + u * src_stride + 2 * d, 0, 2); + __builtin_prefetch(q + u * src_stride + 2 * d + w, 0, 2); + } + } + sme_write_rows_x4<0>(uint32_t(r), pn, p, src_stride); + sme_write_rows_x4<4>(uint32_t(r), pn, q, src_stride); + } + for (int c = 0; c < svl; c += 4) { + double* d0 = dst + (k + c) * w; + sme_store_lines_vg4<0, 4>(d0, w, svl, uint32_t(c)); + sme_store_lines_vg4<1, 5>(d0 + Index(svl) * w, w, svl, uint32_t(c)); + sme_store_lines_vg4<2, 6>(d0 + Index(2 * svl) * w, w, svl, uint32_t(c)); + sme_store_lines_vg4<3, 7>(d0 + Index(3 * svl) * w, w, svl, uint32_t(c)); + } + } + return k; +} +#endif + +// Full 2*svl-wide panel through tiles 0..3, four rows or depth steps per ZA move, source prefetched into L2 +// two steps ahead, after sme_transpose_pack_four where the element type has eight tiles; returns the first depth +// index of the tail (< 2*svl) it leaves to the caller. template <typename RealScalar, typename Index> static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_pair(RealScalar* EIGEN_RESTRICT dst, const RealScalar* EIGEN_RESTRICT src, Index src_stride, @@ -255,7 +333,7 @@ using Traits = sme_packet_traits<RealScalar>; const int svl = Traits::size(), w = 2 * svl; const svcount_t pn = Traits::ptrue_c(); - Index k = k0; + Index k = sme_transpose_pack_four(dst, src, src_stride, k0, k1); for (; k + w <= k1; k += w) { const bool pf = k + 2 * w <= k1; for (int r = 0; r < svl; r += 4) { @@ -276,20 +354,9 @@ sme_write_hor_za_vg4<3>(uint32_t(r), pget<1>(b0), pget<1>(b1), pget<1>(b2), pget<1>(b3)); } for (int c = 0; c < svl; c += 4) { - const auto t0 = sme_read_ver_za_vg4<0, RealScalar>(uint32_t(c)), - t2 = sme_read_ver_za_vg4<2, RealScalar>(uint32_t(c)); - const auto t1 = sme_read_ver_za_vg4<1, RealScalar>(uint32_t(c)), - t3 = sme_read_ver_za_vg4<3, RealScalar>(uint32_t(c)); RealScalar* d0 = dst + (k + c) * w; - RealScalar* d1 = d0 + Index(svl) * w; - pstoreu_x2(pn, d0, pcreate(pget<0>(t0), pget<0>(t2))); - pstoreu_x2(pn, d0 + w, pcreate(pget<1>(t0), pget<1>(t2))); - pstoreu_x2(pn, d0 + 2 * w, pcreate(pget<2>(t0), pget<2>(t2))); - pstoreu_x2(pn, d0 + 3 * w, pcreate(pget<3>(t0), pget<3>(t2))); - pstoreu_x2(pn, d1, pcreate(pget<0>(t1), pget<0>(t3))); - pstoreu_x2(pn, d1 + w, pcreate(pget<1>(t1), pget<1>(t3))); - pstoreu_x2(pn, d1 + 2 * w, pcreate(pget<2>(t1), pget<2>(t3))); - pstoreu_x2(pn, d1 + 3 * w, pcreate(pget<3>(t1), pget<3>(t3))); + sme_store_lines_vg4<0, 2>(d0, w, svl, uint32_t(c)); + sme_store_lines_vg4<1, 3>(d0 + Index(svl) * w, w, svl, uint32_t(c)); } } return k;
diff --git a/benchmarks/Core/bench_gemm.cpp b/benchmarks/Core/bench_gemm.cpp index 03942c8..7190356 100644 --- a/benchmarks/Core/bench_gemm.cpp +++ b/benchmarks/Core/bench_gemm.cpp
@@ -21,44 +21,87 @@ c.noalias() += a * b; } -static void BM_EigenGemm(benchmark::State& state) { +static void add_gflops_counter(benchmark::State& state) { + const double m = state.range(0), n = state.range(1), p = state.range(2); + state.counters["GFLOPS"] = benchmark::Counter( + kFlopsPerMulAdd * m * n * p, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000); +} + +// Sampled coefficients of c = a * b: |c(i,j) - sum_q a(i,q) b(q,j)| <= 8 p eps sum_q |a(i,q)| |b(q,j)|, p the depth. +template <typename Lhs> +static bool gemm_result_ok(const Lhs& a, const Mat& b, const Mat& c) { + for (Index i : {Index(0), a.rows() / 2, a.rows() - 1}) { + for (Index j : {Index(0), b.cols() / 2, b.cols() - 1}) { + Scalar expected(0); + typename NumTraits<Scalar>::Real scale(0); + for (Index p = 0; p < a.cols(); ++p) { + expected += a(i, p) * b(p, j); + scale += numext::abs(a(i, p)) * numext::abs(b(p, j)); + } + if (!(numext::abs(c(i, j) - expected) <= 8 * a.cols() * NumTraits<Scalar>::epsilon() * scale)) return false; + } + } + return true; +} + +// c += a * b with a stored in LhsOrder; b and c are column-major. +template <int LhsOrder> +static void eigen_gemm(benchmark::State& state) { int m = state.range(0); int n = state.range(1); int p = state.range(2); - Mat a(m, p); + Matrix<Scalar, Dynamic, Dynamic, LhsOrder> a(m, p); a.setRandom(); Mat b(p, n); b.setRandom(); Mat c = Mat::Zero(m, n); + gemm(a, b, c); + if (!gemm_result_ok(a, b, c)) { + state.SkipWithError("GEMM result check failed"); + return; + } for (auto _ : state) { c.setZero(); gemm(a, b, c); benchmark::DoNotOptimize(c.data()); benchmark::ClobberMemory(); } - state.counters["GFLOPS"] = benchmark::Counter( - kFlopsPerMulAdd * m * n * p, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000); + add_gflops_counter(state); } +static void BM_EigenGemm(benchmark::State& state) { eigen_gemm<ColMajor>(state); } +static void BM_EigenGemmRowMajorLhs(benchmark::State& state) { eigen_gemm<RowMajor>(state); } + // clang-format off -BENCHMARK(BM_EigenGemm) - ->Args({8, 8, 8})->Args({16, 16, 16})->Args({32, 32, 32}) - ->Args({64, 64, 64})->Args({96, 96, 96})->Args({128, 128, 128}) - ->Args({160, 160, 160})->Args({192, 192, 192})->Args({224, 224, 224}) - ->Args({256, 256, 256})->Args({288, 288, 288})->Args({320, 320, 320}) - ->Args({384, 384, 384})->Args({448, 448, 448})->Args({512, 512, 512}) - ->Args({768, 768, 768})->Args({1024, 1024, 1024})->Args({1536, 1536, 1536}) - ->Args({2048, 2048, 2048})->Args({4096, 4096, 4096}) - // Non-square sizes - ->Args({64, 64, 1024})->Args({1024, 64, 64})->Args({64, 1024, 64}) - ->Args({256, 256, 1024})->Args({1024, 256, 256}) - // Tall-skinny sizes that straddle the computeProductBlockingSizes cache - // threshold (k*n*sizeof(Lhs) near L1). mc blocking only takes effect when - // m is large; hitting the threshold requires sweeping small n=k with large m. - ->Args({4096, 96, 96})->Args({4096, 128, 128})->Args({4096, 144, 144}) - ->Args({4096, 160, 160})->Args({4096, 176, 176})->Args({8192, 128, 128}); +#define GEMM_SIZES \ + ->Args({8, 8, 8})->Args({16, 16, 16})->Args({32, 32, 32}) \ + ->Args({64, 64, 64})->Args({96, 96, 96})->Args({128, 128, 128}) \ + ->Args({160, 160, 160})->Args({192, 192, 192})->Args({224, 224, 224}) \ + ->Args({256, 256, 256})->Args({288, 288, 288})->Args({320, 320, 320}) \ + ->Args({384, 384, 384})->Args({448, 448, 448})->Args({512, 512, 512}) \ + ->Args({768, 768, 768})->Args({1024, 1024, 1024})->Args({1536, 1536, 1536}) \ + ->Args({2048, 2048, 2048})->Args({4096, 4096, 4096}) \ + /* Non-square sizes */ \ + ->Args({64, 64, 1024})->Args({1024, 64, 64})->Args({64, 1024, 64}) \ + ->Args({256, 256, 1024})->Args({1024, 256, 256}) \ + /* Tall-skinny sizes that straddle the computeProductBlockingSizes cache */ \ + /* threshold (k*n*sizeof(Lhs) near L1). mc blocking only takes effect when */ \ + /* m is large; hitting the threshold requires sweeping small n=k with large m. */ \ + ->Args({4096, 96, 96})->Args({4096, 128, 128})->Args({4096, 144, 144}) \ + ->Args({4096, 160, 160})->Args({4096, 176, 176})->Args({8192, 128, 128}) + +// Depth sweeps through the packers' transposer, which packs b and a row-major a; their leading dimension is k, so +// these also sweep the source stride. At SVL 512, k = 41 (complex<double>) and 81 (double) end in a paired step and a +// tail after the four-chunk fills, and 120 and 136 straddle the float transposer entry depth. +#define DEPTH_SWEEP_SIZES \ + ->ArgsProduct({{64}, {64}, {41, 81, 120, 136, 256, 512, 2048, 4096}}) \ + ->ArgsProduct({{2048}, {64}, {64, 128, 256, 512, 1024}}) \ + ->Args({128, 128, 2048})->Args({256, 256, 2048})->Args({4096, 64, 512})->Args({2048, 128, 512}) // clang-format on +BENCHMARK(BM_EigenGemm) GEMM_SIZES DEPTH_SWEEP_SIZES; +BENCHMARK(BM_EigenGemmRowMajorLhs)->Args({256, 256, 256})->Args({1024, 1024, 1024}) DEPTH_SWEEP_SIZES; + // The reference below calls sgemm_/dgemm_ directly, so it is real-only. #if defined(HAVE_BLAS) && !defined(EIGEN_BENCH_COMPLEX_SCALAR) extern "C" { @@ -94,25 +137,7 @@ benchmark::DoNotOptimize(c.data()); benchmark::ClobberMemory(); } - state.counters["GFLOPS"] = - benchmark::Counter(2.0 * m * n * p, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000); + add_gflops_counter(state); } -// clang-format off -BENCHMARK(BM_BlasGemm) - ->Args({8, 8, 8})->Args({16, 16, 16})->Args({32, 32, 32}) - ->Args({64, 64, 64})->Args({96, 96, 96})->Args({128, 128, 128}) - ->Args({160, 160, 160})->Args({192, 192, 192})->Args({224, 224, 224}) - ->Args({256, 256, 256})->Args({288, 288, 288})->Args({320, 320, 320}) - ->Args({384, 384, 384})->Args({448, 448, 448})->Args({512, 512, 512}) - ->Args({768, 768, 768})->Args({1024, 1024, 1024})->Args({1536, 1536, 1536}) - ->Args({2048, 2048, 2048})->Args({4096, 4096, 4096}) - // Non-square sizes - ->Args({64, 64, 1024})->Args({1024, 64, 64})->Args({64, 1024, 64}) - ->Args({256, 256, 1024})->Args({1024, 256, 256}) - // Tall-skinny sizes that straddle the computeProductBlockingSizes cache - // threshold (k*n*sizeof(Lhs) near L1). mc blocking only takes effect when - // m is large; hitting the threshold requires sweeping small n=k with large m. - ->Args({4096, 96, 96})->Args({4096, 128, 128})->Args({4096, 144, 144}) - ->Args({4096, 160, 160})->Args({4096, 176, 176})->Args({8192, 128, 128}); -// clang-format on +BENCHMARK(BM_BlasGemm) GEMM_SIZES; #endif
diff --git a/test/product_sme.cpp b/test/product_sme.cpp index d48423c..b4b8be7 100644 --- a/test/product_sme.cpp +++ b/test/product_sme.cpp
@@ -785,6 +785,25 @@ verify_symm_pack_rhs<Scalar, ColMajor>(c.N, c.depth, c.cols, c.k2); verify_symm_pack_rhs<Scalar, RowMajor>(c.N, c.depth, c.cols, c.k2); } + + // Full-panel direct regions [s_end, depth) of a ColMajor operand start at s_end = j + w - k2, off the depth chunk + // grid, with real-view lengths 4*svl*a + 2*svl + t that pass through the four-chunk (64-bit), paired and single-tile + // transposer loops in turn. The panel width w is nr and, where smaller, the 2*svl the full-panel path needs. + const int svl = internal::sme_packet_traits<typename NumTraits<Scalar>::Real>::size(); + const int NR = sme_nr<Scalar>(); + const int scale = NumTraits<Scalar>::IsComplex ? 2 : 1; + for (int real_range : {10 * svl + 3, 14 * svl + 1}) { + const int range = (real_range + scale - 1) / scale; + for (int k2 : {5, NR - 3}) { + const int depth = NR - k2 + range, N = numext::maxi(k2 + depth, 2 * NR + 1); + verify_symm_pack_rhs<Scalar, ColMajor>(N, depth, 2 * NR + 1, k2); + verify_symm_pack_rhs<Scalar, RowMajor>(N, depth, 2 * NR + 1, k2); + } + if (svl >= 4 && 2 * svl < NR) { + const int w = 2 * svl, k2 = w - 3, depth = 3 + range; + verify_symm_pack_rhs<Scalar, ColMajor>(k2 + depth, depth, w, k2); + } + } } // --------------------------------------------------------------------------- @@ -1068,6 +1087,31 @@ sweep_pack_direct<Scalar, RowMajor>(n, d); } } + // Depths at and one past whole multiples of four depth chunks, where the full-panel transposer's loops hand over to + // the next and to the single-tile tail; a complex panel transposes a real view of twice the depth. + const int svl = internal::sme_packet_traits<typename NumTraits<Scalar>::Real>::size(); + const int deep = (NumTraits<Scalar>::IsComplex ? 16 : 32) * svl; + std::vector<int> deep_depths = {deep - 1, deep + 1}; + for (int j : {2, 3, 8, 9}) { + deep_depths.push_back(4 * svl * j); + deep_depths.push_back(4 * svl * j + 1); + } + // Deduplicate deep + 1 == 4*svl*8 + 1 for real scalars. + std::sort(deep_depths.begin(), deep_depths.end()); + deep_depths.erase(std::unique(deep_depths.begin(), deep_depths.end()), deep_depths.end()); + // The paired transposer takes only unconjugated 2*svl-wide panels. At SVL 512 that is NR for every scalar and MR for + // all but complex<float>. Below it, eligible real tails reach the paired transposer through tail_pack once deep + // enough, and complex<float> has MR == 2*svl at SVL 256. Complex tails take the NEON tail_transpose_pack, so + // complex<double> at SVL 256 reaches the four-chunk loop only through test_symm_pack's 2*svl-wide direct regions. + std::vector<int> deep_widths = {MR, MR + 1, NR, 2 * NR + 1, 2 * svl}; + std::sort(deep_widths.begin(), deep_widths.end()); + deep_widths.erase(std::unique(deep_widths.begin(), deep_widths.end()), deep_widths.end()); + for (int d : deep_depths) { + for (int n : deep_widths) { + sweep_pack_direct<Scalar, ColMajor>(n, d); + sweep_pack_direct<Scalar, RowMajor>(n, d); + } + } } template <typename Scalar>