SME: Pack narrow panels into whole vectors

libeigen/eigen!3372

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h
index d56c6f8..d81759e 100644
--- a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h
+++ b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h
@@ -112,12 +112,37 @@
 // do reach them, from the SYMM above-diagonal transposed pack.
 // ---------------------------------------------------------------------------
 
+// Appends the leading w <= svl lanes of v to the `fill` < svl lanes of cur that `tail` selects, and stores cur at out
+// whenever it fills. Lanes of v left over after a store stay in place, selected by `tail`; the next splice moves them
+// to the front. A packed panel's lines are contiguous, so a panel narrower than whole vectors is stored in whole
+// vectors this way: a predicated store of each partial line costs about 10 ns on Apple M4 in streaming mode. The
+// counts are int64_t, the type whilelt takes, so the loop-carried `fill` is not sign-extended before each whilelt.
+template <typename Scalar>
+EIGEN_ALWAYS_INLINE void sme_append_lanes(typename sme_packet_traits<Scalar>::type& cur, svbool_t& tail, int64_t& fill,
+                                          Scalar*& out, typename sme_packet_traits<Scalar>::type v, int64_t w,
+                                          int64_t svl) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
+  using Traits = sme_packet_traits<Scalar>;
+  const auto joined = psplice(tail, cur, v);
+  if (fill + w < svl) {
+    cur = joined;
+    fill += w;
+    tail = Traits::whilelt(0, fill);
+    return;
+  }
+  pstoreu(Traits::ptrue(), out, joined);
+  out += svl;
+  tail = svnot_b_z(Traits::whilelt(0, w), Traits::whilelt(fill, svl));  // v[svl - fill, w)
+  cur = v;
+  fill += w - svl;
+}
+
 // Copy `width` contiguous source columns per depth step into a depth-major
 // packed panel of width `width`, for the depth sub-range [k0, k1).  Both dst and
 // src are indexed by the absolute depth index k (dst[k*width+off],
 // src[k*src_stride+off]); the caller offsets `src` to the region's column base
-// and `dst` to the panel base.  A panel with svl < width <= 2*svl stores each packed
-// line with one two-vector store; others store svl-wide predicated chunks, each
+// and `dst` to the panel base.  A full 2*svl panel stores each packed line with one
+// two-vector store, a narrower one other than svl appends its lines into whole vectors
+// (sme_append_lanes); others store svl-wide predicated chunks, each
 // streamed over the depth sub-range.  The chunk loop is outermost so each chunk's
 // predicate is computed once instead of per depth step (the runtime chunk count
 // keeps the compiler from hoisting it on its own).  The symm packers reuse this
@@ -146,21 +171,20 @@
     for (; k < k1; ++k) pstoreu_x2(pn, &dst[k * width], ploadu_x2(pn, &src[k * src_stride]));
     return;
   }
-  if (width > svl && width < 2 * svl) {
-    // Tail panel: each packed line in one counter-predicated two-vector store, four steps in flight.
-    const svcount_t pn = Traits::whilelt_c2(0, width);
-    Index k = k0;
-    for (; k + 4 <= k1; k += 4) {
-      const auto v0 = ploadu_x2(pn, &src[k * src_stride]);
-      const auto v1 = ploadu_x2(pn, &src[(k + 1) * src_stride]);
-      const auto v2 = ploadu_x2(pn, &src[(k + 2) * src_stride]);
-      const auto v3 = ploadu_x2(pn, &src[(k + 3) * src_stride]);
-      pstoreu_x2(pn, &dst[k * width], v0);
-      pstoreu_x2(pn, &dst[(k + 1) * width], v1);
-      pstoreu_x2(pn, &dst[(k + 2) * width], v2);
-      pstoreu_x2(pn, &dst[(k + 3) * width], v3);
+  if (width < 2 * svl && width != svl) {
+    // Tail panel: whole-vector stores of the appended lines (see sme_append_lanes).
+    const int w_lo = sme_min(width, svl), w_hi = width - w_lo;
+    const svbool_t pg_lo = Traits::whilelt(0, w_lo), pg_hi = Traits::whilelt(0, w_hi);
+    typename Traits::type cur = pset1<typename Traits::type>(Scalar(0));
+    svbool_t tail = svpfalse_b();
+    int64_t fill = 0;
+    Scalar* out = dst + k0 * width;
+    for (Index k = k0; k < k1; ++k) {
+      const Scalar* s = &src[k * src_stride];
+      sme_append_lanes(cur, tail, fill, out, ploadu(pg_lo, s), w_lo, svl);
+      if (w_hi > 0) sme_append_lanes(cur, tail, fill, out, ploadu(pg_hi, sme_offset(s, svl)), w_hi, svl);
     }
-    for (; k < k1; ++k) pstoreu_x2(pn, &dst[k * width], ploadu_x2(pn, &src[k * src_stride]));
+    if (fill > 0) pstoreu(Traits::whilelt(0, fill), out, psplice(tail, cur, cur));
     return;
   }
   for (int off = 0; off < width; off += svl) {
@@ -232,6 +256,42 @@
   const RealScalar* EIGEN_RESTRICT rsrc = reinterpret_cast<const RealScalar*>(src);
   const int svl = Traits::size();
   const Index step = Index(2 * width);
+  if (width % svl != 0 && width < 2 * svl) {
+    // A depth step is width real parts, then width imaginary parts, of at most two chunks each: appended into
+    // whole vectors (see sme_append_lanes).
+    const int w0 = sme_min(width, svl), w1 = width - w0;
+    const svbool_t pg_w0 = Traits::whilelt(0, w0), pg_w1 = Traits::whilelt(0, w1);
+    const svbool_t pg_lo0 = Traits::whilelt(0, 2 * w0), pg_hi0 = Traits::whilelt(svl, 2 * w0);
+    const svbool_t pg_lo1 = Traits::whilelt(0, 2 * w1), pg_hi1 = Traits::whilelt(svl, 2 * w1);
+    Vec cur = pset1<Vec>(RealScalar(0));
+    svbool_t tail = svpfalse_b();
+    int64_t fill = 0;
+    RealScalar* out = rdst + k0 * step;
+    for (Index k = k0; k < k1; ++k) {
+      const RealScalar* p = rsrc + Index(2) * k * src_stride;
+      const Vec lo0 = ploadu(pg_lo0, p), hi0 = ploadu(pg_hi0, sme_offset(p, svl));
+      Vec im0 = puzp2(lo0, hi0);
+      EIGEN_IF_CONSTEXPR (Conjugate) {
+        im0 = pnegate(pg_w0, im0);
+      }
+      sme_append_lanes(cur, tail, fill, out, puzp1(lo0, hi0), w0, svl);
+      if (w1 > 0) {
+        const RealScalar* p1 = sme_offset(p, 2 * svl);
+        const Vec lo1 = ploadu(pg_lo1, p1), hi1 = ploadu(pg_hi1, sme_offset(p1, svl));
+        Vec im1 = puzp2(lo1, hi1);
+        EIGEN_IF_CONSTEXPR (Conjugate) {
+          im1 = pnegate(pg_w1, im1);
+        }
+        sme_append_lanes(cur, tail, fill, out, puzp1(lo1, hi1), w1, svl);
+        sme_append_lanes(cur, tail, fill, out, im0, w0, svl);
+        sme_append_lanes(cur, tail, fill, out, im1, w1, svl);
+      } else {
+        sme_append_lanes(cur, tail, fill, out, im0, w0, svl);
+      }
+    }
+    if (fill > 0) pstoreu(Traits::whilelt(0, fill), out, psplice(tail, cur, cur));
+    return;
+  }
   for (int off = 0; off < width; off += svl) {
     const int w = width - off;
     const int lanes = 2 * w;
@@ -1440,11 +1500,9 @@
   const Vec vzero = pset1<Vec>(Scalar(0));
   const Vec valpha = pset1<Vec>(alpha);
 
-  // Two C slices are loaded before either is stored: a C line the caller wrote
-  // from non-streaming code just before the kernel does not forward across the
-  // mode switch on Apple M4, and a serial load/store pays that latency per slice.
-  // C = A*B meets the condition on every call, since evalTo zeroes the
-  // destination first. SVE vectors are sizeless, hence the spelled-out pair.
+  // Two C slices are loaded before either is stored: on Apple M4 a C line the SME unit does not hold (as after a
+  // non-streaming access) costs 10-30 ns, which loads issued together overlap. Four measured up to 1.7x slower on
+  // double blocks. SVE vectors are sizeless, hence the spelled-out pair.
   if (C_stride_row == 1) {
     // Column-major C: extract vertical slices (columns of the ZA tile)
     int ci = 0;
@@ -1516,11 +1574,18 @@
   const svbool_t pg = Traits::ptrue();
   const typename Traits::type valpha = pset1<typename Traits::type>(alpha);
   for (int c = 0; c < svl; c += 4) {
+    // GCC 14 can overwrite live ZA tuple components with C loads.
+#if !EIGEN_COMP_GNUC_STRICT
     const auto lo = sme_read_ver_za_vg4<TileLo, Scalar>(uint32_t(c));
     const auto hi = sme_read_ver_za_vg4<TileHi, Scalar>(uint32_t(c));
+#endif
     Scalar* p = C + Index(c) * ldc;
     const auto c0 = ploadu_x2(pn, p), c1 = ploadu_x2(pn, p + ldc), c2 = ploadu_x2(pn, p + 2 * ldc),
                c3 = ploadu_x2(pn, p + 3 * ldc);
+#if EIGEN_COMP_GNUC_STRICT
+    const auto lo = sme_read_ver_za_vg4<TileLo, Scalar>(uint32_t(c));
+    const auto hi = sme_read_ver_za_vg4<TileHi, Scalar>(uint32_t(c));
+#endif
     pstoreu_x2(pn, p,
                pcreate(pmadd(pg, pget<0>(lo), valpha, pget<0>(c0)), pmadd(pg, pget<0>(hi), valpha, pget<1>(c0))));
     pstoreu_x2(pn, p + ldc,
diff --git a/benchmarks/Core/bench_gemm.cpp b/benchmarks/Core/bench_gemm.cpp
index 35ec76c..ffc6c1e 100644
--- a/benchmarks/Core/bench_gemm.cpp
+++ b/benchmarks/Core/bench_gemm.cpp
@@ -93,7 +93,12 @@
     /* Small and thin products on both sides of the SME backend's NEON small-product bound. */ \
     ->Args({17, 17, 17})->Args({20, 20, 20})->Args({25, 25, 25})->Args({28, 28, 28})->Args({31, 31, 31}) \
     ->Args({4, 32, 32})->Args({8, 32, 32})->Args({32, 4, 32})->Args({32, 8, 32})->Args({16, 16, 32}) \
-    ->Args({24, 24, 48})->Args({12, 12, 64})->Args({2, 64, 64})->Args({8, 64, 128})->Args({64, 8, 128})
+    ->Args({24, 24, 48})->Args({12, 12, 64})->Args({2, 64, 64})->Args({8, 64, 128})->Args({64, 8, 128}) \
+    /* Full tiles and narrow panels affected by SME store code generation. */ \
+    ->Args({32, 8, 128})->Args({32, 16, 128})->Args({40, 40, 40})->Args({48, 48, 48}) \
+    /* Ragged C columns at SVL 512, beyond the NEON small-product bound. */ \
+    ->Args({9, 17, 256})->Args({13, 17, 256})->Args({15, 17, 256}) \
+    ->Args({17, 17, 256})->Args({23, 17, 256})->Args({31, 17, 256})
 
 // Depth sweeps through the packers' transposer, which packs b and a row-major a; their leading dimension is k, so
 // these also sweep the source stride. At SVL 512, k = 41 (complex<double>) and 81 (double) end in a paired step and a
@@ -107,6 +112,7 @@
 // floats; 12 doubles for 60 and 252), and n = 60 leaves b's. A column-major b and a row-major a pack through the ZA
 // transposer, a row-major b through the copy; a column-major a is mostly read in place.
 #define TAIL_PANEL_SIZES \
+    ->Args({15, 17, 1024})->Args({17, 15, 1024})->Args({31, 31, 1024}) \
     ->Args({60, 60, 1024})->Args({60, 64, 1024})->Args({64, 60, 1024})->Args({120, 120, 1024})->Args({252, 252, 252})
 // clang-format on
 
diff --git a/test/product_sme.cpp b/test/product_sme.cpp
index d2c91da..ccd597c 100644
--- a/test/product_sme.cpp
+++ b/test/product_sme.cpp
@@ -1092,6 +1092,38 @@
   check_packed_rhs<Scalar, StorageOrder, PanelMode, Conjugate>(Mapper(V.data(), V.outerStride()), V, depth, cols);
 }
 
+template <bool Conjugate, typename Scalar>
+EIGEN_DONT_INLINE __arm_locally_streaming static void sme_copy_range(Scalar* dst, const Scalar* src, Index stride,
+                                                                     Index k0, Index k1, int width) {
+  internal::sve_copy_panel_range<Conjugate>(dst, src, stride, k0, k1, width);
+}
+
+// Every partial-vector width, including copies ending with an unshifted tail and exact vector boundaries.
+template <typename Scalar>
+static void test_copy_panel_range() {
+  const int svl = internal::sme_packet_traits<typename NumTraits<Scalar>::Real>::size();
+  for (int width = 1; width <= 2 * svl; ++width) {
+    for (Index k0 : {0, 1, svl - 1}) {
+      for (Index depth : {0, 1, 2, 3, svl - 1, svl, svl + 1, 2 * svl + 1}) {
+        const Index k1 = k0 + depth;
+        SmeRowMajorMat<Scalar> src = SmeRowMajorMat<Scalar>::Random(k1, width + 3);
+        SmeVector<Scalar> packed = SmeVector<Scalar>::Constant(k1 * width + 2, pack_sentinel<Scalar>());
+        SmeVector<Scalar> ref = packed;
+        sme_copy_range<false>(packed.data() + 1, src.data(), src.outerStride(), k0, k1, width);
+        for (Index k = k0; k < k1; ++k)
+          for (Index c = 0; c < width; ++c) set_packed(ref.data() + 1, Index(width), k, c, Scalar(src(k, c)));
+        VERIFY_IS_EQUAL(packed, ref);
+        packed.setConstant(pack_sentinel<Scalar>());
+        sme_copy_range<true>(packed.data() + 1, src.data(), src.outerStride(), k0, k1, width);
+        for (Index k = k0; k < k1; ++k)
+          for (Index c = 0; c < width; ++c)
+            set_packed(ref.data() + 1, Index(width), k, c, Scalar(numext::conj(src(k, c))));
+        VERIFY_IS_EQUAL(packed, ref);
+      }
+    }
+  }
+}
+
 // Inner-strided blas mapper LHS: element(i, k) laid out with inner stride
 // `incr`.  ColMajor takes the vectorised gather path; RowMajor takes the scalar
 // path (its packets would run along depth, not rows).
@@ -1496,6 +1528,47 @@
   }
 }
 
+// GCC 14 must keep the full-grid store's ZA tuples intact across C loads. Exact integer results and sentinel
+// padding also cover the single-tile stores used for ragged rows and columns.
+template <typename Scalar>
+EIGEN_DONT_INLINE __arm_locally_streaming __arm_new("za") void sme_test_store_grid(Scalar* out, Index ldc,
+                                                                                   const Scalar* a, const Scalar* b,
+                                                                                   int rows, int cols) {
+  using Traits = internal::sme_packet_traits<Scalar>;
+  const int svl = Traits::size();
+  const auto pg = Traits::ptrue();
+  svzero_za();
+  internal::outer_product_2x2<Scalar>(internal::ploadu(pg, a), internal::ploadu(pg, a + svl), internal::ploadu(pg, b),
+                                      internal::ploadu(pg, b + svl));
+  internal::sme_store_2x2_grid(out, Index(1), ldc, Scalar(2), Index(2), svl, rows - svl, Index(1),
+                               internal::sme_min(cols, svl), cols > svl ? cols - svl : 0);
+}
+
+template <typename Scalar>
+static void test_tile_pair_stores() {
+  const int svl = internal::sme_packet_traits<Scalar>::size();
+  const Index ldc = 2 * svl + 7;
+  std::vector<Scalar> a(2 * svl), b(2 * svl);
+  for (int i = 0; i < 2 * svl; ++i) {
+    a[i] = Scalar(i % 7 + 1);
+    b[i] = Scalar(i % 5 + 2);
+  }
+  const int widths[] = {1, 2, 3, 4, 5, svl - 1, svl, svl + 1, svl + 2, 2 * svl - 1, 2 * svl};
+  std::vector<Scalar> out(ldc * (2 * svl + 3)), expected(out.size());
+  for (int rows = svl + 1; rows <= 2 * svl; ++rows) {
+    for (int cols : widths) {
+      if (cols > 2 * svl) continue;
+      std::fill(out.begin(), out.end(), Scalar(-9999));
+      expected = out;
+      for (int j = 0; j < cols; ++j)
+        for (int i = 0; i < rows; ++i) expected[2 + i + (1 + j) * ldc] += Scalar(2) * a[i] * b[j];
+      internal::sme_fpsr_guard guard;
+      sme_test_store_grid(out.data(), ldc, a.data(), b.data(), rows, cols);
+      for (Index i = 0; i < Index(out.size()); ++i) VERIFY_IS_EQUAL(out[i], expected[i]);
+    }
+  }
+}
+
 EIGEN_DECLARE_TEST(product_sme) {
   CALL_SUBTEST_1(test_whilelt_operand_types<float>());
   CALL_SUBTEST_1(test_packet_ops<float>());
@@ -1503,6 +1576,7 @@
   CALL_SUBTEST_1(test_conjugated_products<float>());
   CALL_SUBTEST_1(test_symm_pack<float>());
   CALL_SUBTEST_1(test_pack_direct<float>());
+  CALL_SUBTEST_1(test_copy_panel_range<float>());
   CALL_SUBTEST_1(test_mapper_fallback<float>());
   CALL_SUBTEST_1(test_neon_small_blocks<float>());
   CALL_SUBTEST_1(test_small_products<float>());
@@ -1523,6 +1597,7 @@
   CALL_SUBTEST_2(test_packet_ops<double>());
   CALL_SUBTEST_2(test_symm_pack<double>());
   CALL_SUBTEST_2(test_pack_direct<double>());
+  CALL_SUBTEST_2(test_copy_panel_range<double>());
   CALL_SUBTEST_2(test_mapper_fallback<double>());
   CALL_SUBTEST_2(test_neon_small_blocks<double>());
   CALL_SUBTEST_2(test_small_products<double>());
@@ -1538,6 +1613,7 @@
   CALL_SUBTEST_3(test_conjugated_products<std::complex<float>>());
   CALL_SUBTEST_3(test_symm_pack<std::complex<float>>());
   CALL_SUBTEST_3(test_pack_direct<std::complex<float>>());
+  CALL_SUBTEST_3(test_copy_panel_range<std::complex<float>>());
   CALL_SUBTEST_3(test_mapper_fallback<std::complex<float>>());
   CALL_SUBTEST_3(test_neon_small_blocks<std::complex<float>>());
   CALL_SUBTEST_3(test_small_products<std::complex<float>>());
@@ -1554,6 +1630,7 @@
 #ifdef EIGEN_VECTORIZE_SME_F64F64
   CALL_SUBTEST_4(test_symm_pack<std::complex<double>>());
   CALL_SUBTEST_4(test_pack_direct<std::complex<double>>());
+  CALL_SUBTEST_4(test_copy_panel_range<std::complex<double>>());
   CALL_SUBTEST_4(test_mapper_fallback<std::complex<double>>());
   CALL_SUBTEST_4(test_neon_small_blocks<std::complex<double>>());
   CALL_SUBTEST_4(test_small_products<std::complex<double>>());
@@ -1581,4 +1658,8 @@
   CALL_SUBTEST(test_small_products<std::complex<double>>());
 #endif
 #endif
+  CALL_SUBTEST_9(test_tile_pair_stores<float>());
+#ifdef EIGEN_VECTORIZE_SME_F64F64
+  CALL_SUBTEST_9(test_tile_pair_stores<double>());
+#endif
 }