SME: Single-vector stores in the full-panel transposer; four depth chunks per fill for 64-bit elements

libeigen/eigen!3340

diff --git a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h
index 54bd064..3a52e3a 100644
--- a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h
+++ b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h
@@ -246,8 +246,86 @@
   sve_copy_panel_range<Conjugate>(dst, src, src_stride, Index(0), depth, width);
 }
 
-// Full 2*svl-wide panel through all four tiles, four rows or depth steps per ZA move, source prefetched into L2
-// two steps ahead; returns the first depth index of the tail (< 2*svl) it leaves to the caller.
+// Slices c..c+3 of tiles Lo and Hi as four whole packed lines from d, w scalars apart: Lo's slice is a line's first
+// svl scalars, Hi's the rest.  Single-vector stores: a two-vector store would first copy each line's halves into a
+// consecutive register pair.
+template <int Lo, int Hi, typename RealScalar>
+static EIGEN_ALWAYS_INLINE void sme_store_lines_vg4(RealScalar* d, int w, int svl,
+                                                    uint32_t c) __arm_streaming __arm_in("za") {
+  const svbool_t all = sme_packet_traits<RealScalar>::ptrue();
+  const auto lo = sme_read_ver_za_vg4<Lo, RealScalar>(c), hi = sme_read_ver_za_vg4<Hi, RealScalar>(c);
+  pstoreu(all, d, pget<0>(lo));
+  pstoreu(all, d + svl, pget<0>(hi));
+  pstoreu(all, d + w, pget<1>(lo));
+  pstoreu(all, d + w + svl, pget<1>(hi));
+  pstoreu(all, d + 2 * w, pget<2>(lo));
+  pstoreu(all, d + 2 * w + svl, pget<2>(hi));
+  pstoreu(all, d + 3 * w, pget<3>(lo));
+  pstoreu(all, d + 3 * w + svl, pget<3>(hi));
+}
+
+// Four depth chunks of a full panel per fill, which needs eight tiles. No-op where the element type has four tiles:
+// the paired loop's two depth chunks already fill ZA. Returns the first depth index it leaves.
+template <typename RealScalar, typename Index>
+static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_four(RealScalar*, const RealScalar*, Index, Index k0,
+                                                         Index) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
+  return k0;
+}
+
+#ifdef EIGEN_VECTORIZE_SME_F64F64
+template <int Tile, typename Index>
+static EIGEN_ALWAYS_INLINE void sme_write_rows_x4(uint32_t r, svcount_t pn, const double* p,
+                                                  Index stride) __arm_streaming __arm_inout("za") {
+  const auto a0 = ploadu_x4(pn, p), a1 = ploadu_x4(pn, p + stride), a2 = ploadu_x4(pn, p + 2 * stride),
+             a3 = ploadu_x4(pn, p + 3 * stride);
+  sme_write_hor_za_vg4<Tile>(r, pget<0>(a0), pget<0>(a1), pget<0>(a2), pget<0>(a3));
+  sme_write_hor_za_vg4<Tile + 1>(r, pget<1>(a0), pget<1>(a1), pget<1>(a2), pget<1>(a3));
+  sme_write_hor_za_vg4<Tile + 2>(r, pget<2>(a0), pget<2>(a1), pget<2>(a2), pget<2>(a3));
+  sme_write_hor_za_vg4<Tile + 3>(r, pget<3>(a0), pget<3>(a1), pget<3>(a2), pget<3>(a3));
+}
+
+// Both row-groups over four depth chunks in the eight ZA.D tiles, row-group 0 in tiles 0..3 and row-group 1 in tiles
+// 4..7, so every packed line is stored whole; source prefetched into L2 two steps ahead, two prefetches per row and
+// chunk (a whole chunk with 128-byte cache lines, as on Apple M4).
+template <typename Index>
+static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_four(double* EIGEN_RESTRICT dst, const double* EIGEN_RESTRICT src,
+                                                         Index src_stride, Index k0,
+                                                         Index k1) __arm_streaming __arm_inout("za") {
+  const int svl = sme_packet_traits<double>::size(), w = 2 * svl, d = 4 * svl;
+  const svcount_t pn = sme_packet_traits<double>::ptrue_c();
+  Index k = k0;
+  for (; k + d <= k1; k += d) {
+    // Only a chunk inside the range: the prefetch address must stay within the source.
+    const bool pf = k + 3 * d <= k1;
+    for (int r = 0; r < svl; r += 4) {
+      const double* p = src + Index(r) * src_stride + k;
+      const double* q = p + Index(svl) * src_stride;
+      if (pf) {
+        for (int u = 0; u < 4; ++u) {
+          __builtin_prefetch(p + u * src_stride + 2 * d, 0, 2);
+          __builtin_prefetch(p + u * src_stride + 2 * d + w, 0, 2);
+          __builtin_prefetch(q + u * src_stride + 2 * d, 0, 2);
+          __builtin_prefetch(q + u * src_stride + 2 * d + w, 0, 2);
+        }
+      }
+      sme_write_rows_x4<0>(uint32_t(r), pn, p, src_stride);
+      sme_write_rows_x4<4>(uint32_t(r), pn, q, src_stride);
+    }
+    for (int c = 0; c < svl; c += 4) {
+      double* d0 = dst + (k + c) * w;
+      sme_store_lines_vg4<0, 4>(d0, w, svl, uint32_t(c));
+      sme_store_lines_vg4<1, 5>(d0 + Index(svl) * w, w, svl, uint32_t(c));
+      sme_store_lines_vg4<2, 6>(d0 + Index(2 * svl) * w, w, svl, uint32_t(c));
+      sme_store_lines_vg4<3, 7>(d0 + Index(3 * svl) * w, w, svl, uint32_t(c));
+    }
+  }
+  return k;
+}
+#endif
+
+// Full 2*svl-wide panel through tiles 0..3, four rows or depth steps per ZA move, source prefetched into L2
+// two steps ahead, after sme_transpose_pack_four where the element type has eight tiles; returns the first depth
+// index of the tail (< 2*svl) it leaves to the caller.
 template <typename RealScalar, typename Index>
 static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_pair(RealScalar* EIGEN_RESTRICT dst,
                                                          const RealScalar* EIGEN_RESTRICT src, Index src_stride,
@@ -255,7 +333,7 @@
   using Traits = sme_packet_traits<RealScalar>;
   const int svl = Traits::size(), w = 2 * svl;
   const svcount_t pn = Traits::ptrue_c();
-  Index k = k0;
+  Index k = sme_transpose_pack_four(dst, src, src_stride, k0, k1);
   for (; k + w <= k1; k += w) {
     const bool pf = k + 2 * w <= k1;
     for (int r = 0; r < svl; r += 4) {
@@ -276,20 +354,9 @@
       sme_write_hor_za_vg4<3>(uint32_t(r), pget<1>(b0), pget<1>(b1), pget<1>(b2), pget<1>(b3));
     }
     for (int c = 0; c < svl; c += 4) {
-      const auto t0 = sme_read_ver_za_vg4<0, RealScalar>(uint32_t(c)),
-                 t2 = sme_read_ver_za_vg4<2, RealScalar>(uint32_t(c));
-      const auto t1 = sme_read_ver_za_vg4<1, RealScalar>(uint32_t(c)),
-                 t3 = sme_read_ver_za_vg4<3, RealScalar>(uint32_t(c));
       RealScalar* d0 = dst + (k + c) * w;
-      RealScalar* d1 = d0 + Index(svl) * w;
-      pstoreu_x2(pn, d0, pcreate(pget<0>(t0), pget<0>(t2)));
-      pstoreu_x2(pn, d0 + w, pcreate(pget<1>(t0), pget<1>(t2)));
-      pstoreu_x2(pn, d0 + 2 * w, pcreate(pget<2>(t0), pget<2>(t2)));
-      pstoreu_x2(pn, d0 + 3 * w, pcreate(pget<3>(t0), pget<3>(t2)));
-      pstoreu_x2(pn, d1, pcreate(pget<0>(t1), pget<0>(t3)));
-      pstoreu_x2(pn, d1 + w, pcreate(pget<1>(t1), pget<1>(t3)));
-      pstoreu_x2(pn, d1 + 2 * w, pcreate(pget<2>(t1), pget<2>(t3)));
-      pstoreu_x2(pn, d1 + 3 * w, pcreate(pget<3>(t1), pget<3>(t3)));
+      sme_store_lines_vg4<0, 2>(d0, w, svl, uint32_t(c));
+      sme_store_lines_vg4<1, 3>(d0 + Index(svl) * w, w, svl, uint32_t(c));
     }
   }
   return k;
diff --git a/benchmarks/Core/bench_gemm.cpp b/benchmarks/Core/bench_gemm.cpp
index 03942c8..7190356 100644
--- a/benchmarks/Core/bench_gemm.cpp
+++ b/benchmarks/Core/bench_gemm.cpp
@@ -21,44 +21,87 @@
   c.noalias() += a * b;
 }
 
-static void BM_EigenGemm(benchmark::State& state) {
+static void add_gflops_counter(benchmark::State& state) {
+  const double m = state.range(0), n = state.range(1), p = state.range(2);
+  state.counters["GFLOPS"] = benchmark::Counter(
+      kFlopsPerMulAdd * m * n * p, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000);
+}
+
+// Sampled coefficients of c = a * b: |c(i,j) - sum_q a(i,q) b(q,j)| <= 8 p eps sum_q |a(i,q)| |b(q,j)|, p the depth.
+template <typename Lhs>
+static bool gemm_result_ok(const Lhs& a, const Mat& b, const Mat& c) {
+  for (Index i : {Index(0), a.rows() / 2, a.rows() - 1}) {
+    for (Index j : {Index(0), b.cols() / 2, b.cols() - 1}) {
+      Scalar expected(0);
+      typename NumTraits<Scalar>::Real scale(0);
+      for (Index p = 0; p < a.cols(); ++p) {
+        expected += a(i, p) * b(p, j);
+        scale += numext::abs(a(i, p)) * numext::abs(b(p, j));
+      }
+      if (!(numext::abs(c(i, j) - expected) <= 8 * a.cols() * NumTraits<Scalar>::epsilon() * scale)) return false;
+    }
+  }
+  return true;
+}
+
+// c += a * b with a stored in LhsOrder; b and c are column-major.
+template <int LhsOrder>
+static void eigen_gemm(benchmark::State& state) {
   int m = state.range(0);
   int n = state.range(1);
   int p = state.range(2);
-  Mat a(m, p);
+  Matrix<Scalar, Dynamic, Dynamic, LhsOrder> a(m, p);
   a.setRandom();
   Mat b(p, n);
   b.setRandom();
   Mat c = Mat::Zero(m, n);
+  gemm(a, b, c);
+  if (!gemm_result_ok(a, b, c)) {
+    state.SkipWithError("GEMM result check failed");
+    return;
+  }
   for (auto _ : state) {
     c.setZero();
     gemm(a, b, c);
     benchmark::DoNotOptimize(c.data());
     benchmark::ClobberMemory();
   }
-  state.counters["GFLOPS"] = benchmark::Counter(
-      kFlopsPerMulAdd * m * n * p, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000);
+  add_gflops_counter(state);
 }
 
+static void BM_EigenGemm(benchmark::State& state) { eigen_gemm<ColMajor>(state); }
+static void BM_EigenGemmRowMajorLhs(benchmark::State& state) { eigen_gemm<RowMajor>(state); }
+
 // clang-format off
-BENCHMARK(BM_EigenGemm)
-    ->Args({8, 8, 8})->Args({16, 16, 16})->Args({32, 32, 32})
-    ->Args({64, 64, 64})->Args({96, 96, 96})->Args({128, 128, 128})
-    ->Args({160, 160, 160})->Args({192, 192, 192})->Args({224, 224, 224})
-    ->Args({256, 256, 256})->Args({288, 288, 288})->Args({320, 320, 320})
-    ->Args({384, 384, 384})->Args({448, 448, 448})->Args({512, 512, 512})
-    ->Args({768, 768, 768})->Args({1024, 1024, 1024})->Args({1536, 1536, 1536})
-    ->Args({2048, 2048, 2048})->Args({4096, 4096, 4096})
-    // Non-square sizes
-    ->Args({64, 64, 1024})->Args({1024, 64, 64})->Args({64, 1024, 64})
-    ->Args({256, 256, 1024})->Args({1024, 256, 256})
-    // Tall-skinny sizes that straddle the computeProductBlockingSizes cache
-    // threshold (k*n*sizeof(Lhs) near L1). mc blocking only takes effect when
-    // m is large; hitting the threshold requires sweeping small n=k with large m.
-    ->Args({4096, 96, 96})->Args({4096, 128, 128})->Args({4096, 144, 144})
-    ->Args({4096, 160, 160})->Args({4096, 176, 176})->Args({8192, 128, 128});
+#define GEMM_SIZES \
+    ->Args({8, 8, 8})->Args({16, 16, 16})->Args({32, 32, 32}) \
+    ->Args({64, 64, 64})->Args({96, 96, 96})->Args({128, 128, 128}) \
+    ->Args({160, 160, 160})->Args({192, 192, 192})->Args({224, 224, 224}) \
+    ->Args({256, 256, 256})->Args({288, 288, 288})->Args({320, 320, 320}) \
+    ->Args({384, 384, 384})->Args({448, 448, 448})->Args({512, 512, 512}) \
+    ->Args({768, 768, 768})->Args({1024, 1024, 1024})->Args({1536, 1536, 1536}) \
+    ->Args({2048, 2048, 2048})->Args({4096, 4096, 4096}) \
+    /* Non-square sizes */ \
+    ->Args({64, 64, 1024})->Args({1024, 64, 64})->Args({64, 1024, 64}) \
+    ->Args({256, 256, 1024})->Args({1024, 256, 256}) \
+    /* Tall-skinny sizes that straddle the computeProductBlockingSizes cache */ \
+    /* threshold (k*n*sizeof(Lhs) near L1). mc blocking only takes effect when */ \
+    /* m is large; hitting the threshold requires sweeping small n=k with large m. */ \
+    ->Args({4096, 96, 96})->Args({4096, 128, 128})->Args({4096, 144, 144}) \
+    ->Args({4096, 160, 160})->Args({4096, 176, 176})->Args({8192, 128, 128})
+
+// Depth sweeps through the packers' transposer, which packs b and a row-major a; their leading dimension is k, so
+// these also sweep the source stride. At SVL 512, k = 41 (complex<double>) and 81 (double) end in a paired step and a
+// tail after the four-chunk fills, and 120 and 136 straddle the float transposer entry depth.
+#define DEPTH_SWEEP_SIZES \
+    ->ArgsProduct({{64}, {64}, {41, 81, 120, 136, 256, 512, 2048, 4096}}) \
+    ->ArgsProduct({{2048}, {64}, {64, 128, 256, 512, 1024}}) \
+    ->Args({128, 128, 2048})->Args({256, 256, 2048})->Args({4096, 64, 512})->Args({2048, 128, 512})
 // clang-format on
 
+BENCHMARK(BM_EigenGemm) GEMM_SIZES DEPTH_SWEEP_SIZES;
+BENCHMARK(BM_EigenGemmRowMajorLhs)->Args({256, 256, 256})->Args({1024, 1024, 1024}) DEPTH_SWEEP_SIZES;
+
 // The reference below calls sgemm_/dgemm_ directly, so it is real-only.
 #if defined(HAVE_BLAS) && !defined(EIGEN_BENCH_COMPLEX_SCALAR)
 extern "C" {
@@ -94,25 +137,7 @@
     benchmark::DoNotOptimize(c.data());
     benchmark::ClobberMemory();
   }
-  state.counters["GFLOPS"] =
-      benchmark::Counter(2.0 * m * n * p, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000);
+  add_gflops_counter(state);
 }
-// clang-format off
-BENCHMARK(BM_BlasGemm)
-    ->Args({8, 8, 8})->Args({16, 16, 16})->Args({32, 32, 32})
-    ->Args({64, 64, 64})->Args({96, 96, 96})->Args({128, 128, 128})
-    ->Args({160, 160, 160})->Args({192, 192, 192})->Args({224, 224, 224})
-    ->Args({256, 256, 256})->Args({288, 288, 288})->Args({320, 320, 320})
-    ->Args({384, 384, 384})->Args({448, 448, 448})->Args({512, 512, 512})
-    ->Args({768, 768, 768})->Args({1024, 1024, 1024})->Args({1536, 1536, 1536})
-    ->Args({2048, 2048, 2048})->Args({4096, 4096, 4096})
-    // Non-square sizes
-    ->Args({64, 64, 1024})->Args({1024, 64, 64})->Args({64, 1024, 64})
-    ->Args({256, 256, 1024})->Args({1024, 256, 256})
-    // Tall-skinny sizes that straddle the computeProductBlockingSizes cache
-    // threshold (k*n*sizeof(Lhs) near L1). mc blocking only takes effect when
-    // m is large; hitting the threshold requires sweeping small n=k with large m.
-    ->Args({4096, 96, 96})->Args({4096, 128, 128})->Args({4096, 144, 144})
-    ->Args({4096, 160, 160})->Args({4096, 176, 176})->Args({8192, 128, 128});
-// clang-format on
+BENCHMARK(BM_BlasGemm) GEMM_SIZES;
 #endif
diff --git a/test/product_sme.cpp b/test/product_sme.cpp
index d48423c..b4b8be7 100644
--- a/test/product_sme.cpp
+++ b/test/product_sme.cpp
@@ -785,6 +785,25 @@
     verify_symm_pack_rhs<Scalar, ColMajor>(c.N, c.depth, c.cols, c.k2);
     verify_symm_pack_rhs<Scalar, RowMajor>(c.N, c.depth, c.cols, c.k2);
   }
+
+  // Full-panel direct regions [s_end, depth) of a ColMajor operand start at s_end = j + w - k2, off the depth chunk
+  // grid, with real-view lengths 4*svl*a + 2*svl + t that pass through the four-chunk (64-bit), paired and single-tile
+  // transposer loops in turn. The panel width w is nr and, where smaller, the 2*svl the full-panel path needs.
+  const int svl = internal::sme_packet_traits<typename NumTraits<Scalar>::Real>::size();
+  const int NR = sme_nr<Scalar>();
+  const int scale = NumTraits<Scalar>::IsComplex ? 2 : 1;
+  for (int real_range : {10 * svl + 3, 14 * svl + 1}) {
+    const int range = (real_range + scale - 1) / scale;
+    for (int k2 : {5, NR - 3}) {
+      const int depth = NR - k2 + range, N = numext::maxi(k2 + depth, 2 * NR + 1);
+      verify_symm_pack_rhs<Scalar, ColMajor>(N, depth, 2 * NR + 1, k2);
+      verify_symm_pack_rhs<Scalar, RowMajor>(N, depth, 2 * NR + 1, k2);
+    }
+    if (svl >= 4 && 2 * svl < NR) {
+      const int w = 2 * svl, k2 = w - 3, depth = 3 + range;
+      verify_symm_pack_rhs<Scalar, ColMajor>(k2 + depth, depth, w, k2);
+    }
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -1068,6 +1087,31 @@
       sweep_pack_direct<Scalar, RowMajor>(n, d);
     }
   }
+  // Depths at and one past whole multiples of four depth chunks, where the full-panel transposer's loops hand over to
+  // the next and to the single-tile tail; a complex panel transposes a real view of twice the depth.
+  const int svl = internal::sme_packet_traits<typename NumTraits<Scalar>::Real>::size();
+  const int deep = (NumTraits<Scalar>::IsComplex ? 16 : 32) * svl;
+  std::vector<int> deep_depths = {deep - 1, deep + 1};
+  for (int j : {2, 3, 8, 9}) {
+    deep_depths.push_back(4 * svl * j);
+    deep_depths.push_back(4 * svl * j + 1);
+  }
+  // Deduplicate deep + 1 == 4*svl*8 + 1 for real scalars.
+  std::sort(deep_depths.begin(), deep_depths.end());
+  deep_depths.erase(std::unique(deep_depths.begin(), deep_depths.end()), deep_depths.end());
+  // The paired transposer takes only unconjugated 2*svl-wide panels. At SVL 512 that is NR for every scalar and MR for
+  // all but complex<float>. Below it, eligible real tails reach the paired transposer through tail_pack once deep
+  // enough, and complex<float> has MR == 2*svl at SVL 256. Complex tails take the NEON tail_transpose_pack, so
+  // complex<double> at SVL 256 reaches the four-chunk loop only through test_symm_pack's 2*svl-wide direct regions.
+  std::vector<int> deep_widths = {MR, MR + 1, NR, 2 * NR + 1, 2 * svl};
+  std::sort(deep_widths.begin(), deep_widths.end());
+  deep_widths.erase(std::unique(deep_widths.begin(), deep_widths.end()), deep_widths.end());
+  for (int d : deep_depths) {
+    for (int n : deep_widths) {
+      sweep_pack_direct<Scalar, ColMajor>(n, d);
+      sweep_pack_direct<Scalar, RowMajor>(n, d);
+    }
+  }
 }
 
 template <typename Scalar>