SME: Pack narrow panels into whole vectors libeigen/eigen!3372 Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h index d56c6f8..d81759e 100644 --- a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h
@@ -112,12 +112,37 @@ // do reach them, from the SYMM above-diagonal transposed pack. // --------------------------------------------------------------------------- +// Appends the leading w <= svl lanes of v to the `fill` < svl lanes of cur that `tail` selects, and stores cur at out +// whenever it fills. Lanes of v left over after a store stay in place, selected by `tail`; the next splice moves them +// to the front. A packed panel's lines are contiguous, so a panel narrower than whole vectors is stored in whole +// vectors this way: a predicated store of each partial line costs about 10 ns on Apple M4 in streaming mode. The +// counts are int64_t, the type whilelt takes, so the loop-carried `fill` is not sign-extended before each whilelt. +template <typename Scalar> +EIGEN_ALWAYS_INLINE void sme_append_lanes(typename sme_packet_traits<Scalar>::type& cur, svbool_t& tail, int64_t& fill, + Scalar*& out, typename sme_packet_traits<Scalar>::type v, int64_t w, + int64_t svl) __arm_streaming EIGEN_SME_ZA_AGNOSTIC { + using Traits = sme_packet_traits<Scalar>; + const auto joined = psplice(tail, cur, v); + if (fill + w < svl) { + cur = joined; + fill += w; + tail = Traits::whilelt(0, fill); + return; + } + pstoreu(Traits::ptrue(), out, joined); + out += svl; + tail = svnot_b_z(Traits::whilelt(0, w), Traits::whilelt(fill, svl)); // v[svl - fill, w) + cur = v; + fill += w - svl; +} + // Copy `width` contiguous source columns per depth step into a depth-major // packed panel of width `width`, for the depth sub-range [k0, k1). Both dst and // src are indexed by the absolute depth index k (dst[k*width+off], // src[k*src_stride+off]); the caller offsets `src` to the region's column base -// and `dst` to the panel base. A panel with svl < width <= 2*svl stores each packed -// line with one two-vector store; others store svl-wide predicated chunks, each +// and `dst` to the panel base. A full 2*svl panel stores each packed line with one +// two-vector store, a narrower one other than svl appends its lines into whole vectors +// (sme_append_lanes); others store svl-wide predicated chunks, each // streamed over the depth sub-range. The chunk loop is outermost so each chunk's // predicate is computed once instead of per depth step (the runtime chunk count // keeps the compiler from hoisting it on its own). The symm packers reuse this @@ -146,21 +171,20 @@ for (; k < k1; ++k) pstoreu_x2(pn, &dst[k * width], ploadu_x2(pn, &src[k * src_stride])); return; } - if (width > svl && width < 2 * svl) { - // Tail panel: each packed line in one counter-predicated two-vector store, four steps in flight. - const svcount_t pn = Traits::whilelt_c2(0, width); - Index k = k0; - for (; k + 4 <= k1; k += 4) { - const auto v0 = ploadu_x2(pn, &src[k * src_stride]); - const auto v1 = ploadu_x2(pn, &src[(k + 1) * src_stride]); - const auto v2 = ploadu_x2(pn, &src[(k + 2) * src_stride]); - const auto v3 = ploadu_x2(pn, &src[(k + 3) * src_stride]); - pstoreu_x2(pn, &dst[k * width], v0); - pstoreu_x2(pn, &dst[(k + 1) * width], v1); - pstoreu_x2(pn, &dst[(k + 2) * width], v2); - pstoreu_x2(pn, &dst[(k + 3) * width], v3); + if (width < 2 * svl && width != svl) { + // Tail panel: whole-vector stores of the appended lines (see sme_append_lanes). + const int w_lo = sme_min(width, svl), w_hi = width - w_lo; + const svbool_t pg_lo = Traits::whilelt(0, w_lo), pg_hi = Traits::whilelt(0, w_hi); + typename Traits::type cur = pset1<typename Traits::type>(Scalar(0)); + svbool_t tail = svpfalse_b(); + int64_t fill = 0; + Scalar* out = dst + k0 * width; + for (Index k = k0; k < k1; ++k) { + const Scalar* s = &src[k * src_stride]; + sme_append_lanes(cur, tail, fill, out, ploadu(pg_lo, s), w_lo, svl); + if (w_hi > 0) sme_append_lanes(cur, tail, fill, out, ploadu(pg_hi, sme_offset(s, svl)), w_hi, svl); } - for (; k < k1; ++k) pstoreu_x2(pn, &dst[k * width], ploadu_x2(pn, &src[k * src_stride])); + if (fill > 0) pstoreu(Traits::whilelt(0, fill), out, psplice(tail, cur, cur)); return; } for (int off = 0; off < width; off += svl) { @@ -232,6 +256,42 @@ const RealScalar* EIGEN_RESTRICT rsrc = reinterpret_cast<const RealScalar*>(src); const int svl = Traits::size(); const Index step = Index(2 * width); + if (width % svl != 0 && width < 2 * svl) { + // A depth step is width real parts, then width imaginary parts, of at most two chunks each: appended into + // whole vectors (see sme_append_lanes). + const int w0 = sme_min(width, svl), w1 = width - w0; + const svbool_t pg_w0 = Traits::whilelt(0, w0), pg_w1 = Traits::whilelt(0, w1); + const svbool_t pg_lo0 = Traits::whilelt(0, 2 * w0), pg_hi0 = Traits::whilelt(svl, 2 * w0); + const svbool_t pg_lo1 = Traits::whilelt(0, 2 * w1), pg_hi1 = Traits::whilelt(svl, 2 * w1); + Vec cur = pset1<Vec>(RealScalar(0)); + svbool_t tail = svpfalse_b(); + int64_t fill = 0; + RealScalar* out = rdst + k0 * step; + for (Index k = k0; k < k1; ++k) { + const RealScalar* p = rsrc + Index(2) * k * src_stride; + const Vec lo0 = ploadu(pg_lo0, p), hi0 = ploadu(pg_hi0, sme_offset(p, svl)); + Vec im0 = puzp2(lo0, hi0); + EIGEN_IF_CONSTEXPR (Conjugate) { + im0 = pnegate(pg_w0, im0); + } + sme_append_lanes(cur, tail, fill, out, puzp1(lo0, hi0), w0, svl); + if (w1 > 0) { + const RealScalar* p1 = sme_offset(p, 2 * svl); + const Vec lo1 = ploadu(pg_lo1, p1), hi1 = ploadu(pg_hi1, sme_offset(p1, svl)); + Vec im1 = puzp2(lo1, hi1); + EIGEN_IF_CONSTEXPR (Conjugate) { + im1 = pnegate(pg_w1, im1); + } + sme_append_lanes(cur, tail, fill, out, puzp1(lo1, hi1), w1, svl); + sme_append_lanes(cur, tail, fill, out, im0, w0, svl); + sme_append_lanes(cur, tail, fill, out, im1, w1, svl); + } else { + sme_append_lanes(cur, tail, fill, out, im0, w0, svl); + } + } + if (fill > 0) pstoreu(Traits::whilelt(0, fill), out, psplice(tail, cur, cur)); + return; + } for (int off = 0; off < width; off += svl) { const int w = width - off; const int lanes = 2 * w; @@ -1440,11 +1500,9 @@ const Vec vzero = pset1<Vec>(Scalar(0)); const Vec valpha = pset1<Vec>(alpha); - // Two C slices are loaded before either is stored: a C line the caller wrote - // from non-streaming code just before the kernel does not forward across the - // mode switch on Apple M4, and a serial load/store pays that latency per slice. - // C = A*B meets the condition on every call, since evalTo zeroes the - // destination first. SVE vectors are sizeless, hence the spelled-out pair. + // Two C slices are loaded before either is stored: on Apple M4 a C line the SME unit does not hold (as after a + // non-streaming access) costs 10-30 ns, which loads issued together overlap. Four measured up to 1.7x slower on + // double blocks. SVE vectors are sizeless, hence the spelled-out pair. if (C_stride_row == 1) { // Column-major C: extract vertical slices (columns of the ZA tile) int ci = 0; @@ -1516,11 +1574,18 @@ const svbool_t pg = Traits::ptrue(); const typename Traits::type valpha = pset1<typename Traits::type>(alpha); for (int c = 0; c < svl; c += 4) { + // GCC 14 can overwrite live ZA tuple components with C loads. +#if !EIGEN_COMP_GNUC_STRICT const auto lo = sme_read_ver_za_vg4<TileLo, Scalar>(uint32_t(c)); const auto hi = sme_read_ver_za_vg4<TileHi, Scalar>(uint32_t(c)); +#endif Scalar* p = C + Index(c) * ldc; const auto c0 = ploadu_x2(pn, p), c1 = ploadu_x2(pn, p + ldc), c2 = ploadu_x2(pn, p + 2 * ldc), c3 = ploadu_x2(pn, p + 3 * ldc); +#if EIGEN_COMP_GNUC_STRICT + const auto lo = sme_read_ver_za_vg4<TileLo, Scalar>(uint32_t(c)); + const auto hi = sme_read_ver_za_vg4<TileHi, Scalar>(uint32_t(c)); +#endif pstoreu_x2(pn, p, pcreate(pmadd(pg, pget<0>(lo), valpha, pget<0>(c0)), pmadd(pg, pget<0>(hi), valpha, pget<1>(c0)))); pstoreu_x2(pn, p + ldc,
diff --git a/benchmarks/Core/bench_gemm.cpp b/benchmarks/Core/bench_gemm.cpp index 35ec76c..ffc6c1e 100644 --- a/benchmarks/Core/bench_gemm.cpp +++ b/benchmarks/Core/bench_gemm.cpp
@@ -93,7 +93,12 @@ /* Small and thin products on both sides of the SME backend's NEON small-product bound. */ \ ->Args({17, 17, 17})->Args({20, 20, 20})->Args({25, 25, 25})->Args({28, 28, 28})->Args({31, 31, 31}) \ ->Args({4, 32, 32})->Args({8, 32, 32})->Args({32, 4, 32})->Args({32, 8, 32})->Args({16, 16, 32}) \ - ->Args({24, 24, 48})->Args({12, 12, 64})->Args({2, 64, 64})->Args({8, 64, 128})->Args({64, 8, 128}) + ->Args({24, 24, 48})->Args({12, 12, 64})->Args({2, 64, 64})->Args({8, 64, 128})->Args({64, 8, 128}) \ + /* Full tiles and narrow panels affected by SME store code generation. */ \ + ->Args({32, 8, 128})->Args({32, 16, 128})->Args({40, 40, 40})->Args({48, 48, 48}) \ + /* Ragged C columns at SVL 512, beyond the NEON small-product bound. */ \ + ->Args({9, 17, 256})->Args({13, 17, 256})->Args({15, 17, 256}) \ + ->Args({17, 17, 256})->Args({23, 17, 256})->Args({31, 17, 256}) // Depth sweeps through the packers' transposer, which packs b and a row-major a; their leading dimension is k, so // these also sweep the source stride. At SVL 512, k = 41 (complex<double>) and 81 (double) end in a paired step and a @@ -107,6 +112,7 @@ // floats; 12 doubles for 60 and 252), and n = 60 leaves b's. A column-major b and a row-major a pack through the ZA // transposer, a row-major b through the copy; a column-major a is mostly read in place. #define TAIL_PANEL_SIZES \ + ->Args({15, 17, 1024})->Args({17, 15, 1024})->Args({31, 31, 1024}) \ ->Args({60, 60, 1024})->Args({60, 64, 1024})->Args({64, 60, 1024})->Args({120, 120, 1024})->Args({252, 252, 252}) // clang-format on
diff --git a/test/product_sme.cpp b/test/product_sme.cpp index d2c91da..ccd597c 100644 --- a/test/product_sme.cpp +++ b/test/product_sme.cpp
@@ -1092,6 +1092,38 @@ check_packed_rhs<Scalar, StorageOrder, PanelMode, Conjugate>(Mapper(V.data(), V.outerStride()), V, depth, cols); } +template <bool Conjugate, typename Scalar> +EIGEN_DONT_INLINE __arm_locally_streaming static void sme_copy_range(Scalar* dst, const Scalar* src, Index stride, + Index k0, Index k1, int width) { + internal::sve_copy_panel_range<Conjugate>(dst, src, stride, k0, k1, width); +} + +// Every partial-vector width, including copies ending with an unshifted tail and exact vector boundaries. +template <typename Scalar> +static void test_copy_panel_range() { + const int svl = internal::sme_packet_traits<typename NumTraits<Scalar>::Real>::size(); + for (int width = 1; width <= 2 * svl; ++width) { + for (Index k0 : {0, 1, svl - 1}) { + for (Index depth : {0, 1, 2, 3, svl - 1, svl, svl + 1, 2 * svl + 1}) { + const Index k1 = k0 + depth; + SmeRowMajorMat<Scalar> src = SmeRowMajorMat<Scalar>::Random(k1, width + 3); + SmeVector<Scalar> packed = SmeVector<Scalar>::Constant(k1 * width + 2, pack_sentinel<Scalar>()); + SmeVector<Scalar> ref = packed; + sme_copy_range<false>(packed.data() + 1, src.data(), src.outerStride(), k0, k1, width); + for (Index k = k0; k < k1; ++k) + for (Index c = 0; c < width; ++c) set_packed(ref.data() + 1, Index(width), k, c, Scalar(src(k, c))); + VERIFY_IS_EQUAL(packed, ref); + packed.setConstant(pack_sentinel<Scalar>()); + sme_copy_range<true>(packed.data() + 1, src.data(), src.outerStride(), k0, k1, width); + for (Index k = k0; k < k1; ++k) + for (Index c = 0; c < width; ++c) + set_packed(ref.data() + 1, Index(width), k, c, Scalar(numext::conj(src(k, c)))); + VERIFY_IS_EQUAL(packed, ref); + } + } + } +} + // Inner-strided blas mapper LHS: element(i, k) laid out with inner stride // `incr`. ColMajor takes the vectorised gather path; RowMajor takes the scalar // path (its packets would run along depth, not rows). @@ -1496,6 +1528,47 @@ } } +// GCC 14 must keep the full-grid store's ZA tuples intact across C loads. Exact integer results and sentinel +// padding also cover the single-tile stores used for ragged rows and columns. +template <typename Scalar> +EIGEN_DONT_INLINE __arm_locally_streaming __arm_new("za") void sme_test_store_grid(Scalar* out, Index ldc, + const Scalar* a, const Scalar* b, + int rows, int cols) { + using Traits = internal::sme_packet_traits<Scalar>; + const int svl = Traits::size(); + const auto pg = Traits::ptrue(); + svzero_za(); + internal::outer_product_2x2<Scalar>(internal::ploadu(pg, a), internal::ploadu(pg, a + svl), internal::ploadu(pg, b), + internal::ploadu(pg, b + svl)); + internal::sme_store_2x2_grid(out, Index(1), ldc, Scalar(2), Index(2), svl, rows - svl, Index(1), + internal::sme_min(cols, svl), cols > svl ? cols - svl : 0); +} + +template <typename Scalar> +static void test_tile_pair_stores() { + const int svl = internal::sme_packet_traits<Scalar>::size(); + const Index ldc = 2 * svl + 7; + std::vector<Scalar> a(2 * svl), b(2 * svl); + for (int i = 0; i < 2 * svl; ++i) { + a[i] = Scalar(i % 7 + 1); + b[i] = Scalar(i % 5 + 2); + } + const int widths[] = {1, 2, 3, 4, 5, svl - 1, svl, svl + 1, svl + 2, 2 * svl - 1, 2 * svl}; + std::vector<Scalar> out(ldc * (2 * svl + 3)), expected(out.size()); + for (int rows = svl + 1; rows <= 2 * svl; ++rows) { + for (int cols : widths) { + if (cols > 2 * svl) continue; + std::fill(out.begin(), out.end(), Scalar(-9999)); + expected = out; + for (int j = 0; j < cols; ++j) + for (int i = 0; i < rows; ++i) expected[2 + i + (1 + j) * ldc] += Scalar(2) * a[i] * b[j]; + internal::sme_fpsr_guard guard; + sme_test_store_grid(out.data(), ldc, a.data(), b.data(), rows, cols); + for (Index i = 0; i < Index(out.size()); ++i) VERIFY_IS_EQUAL(out[i], expected[i]); + } + } +} + EIGEN_DECLARE_TEST(product_sme) { CALL_SUBTEST_1(test_whilelt_operand_types<float>()); CALL_SUBTEST_1(test_packet_ops<float>()); @@ -1503,6 +1576,7 @@ CALL_SUBTEST_1(test_conjugated_products<float>()); CALL_SUBTEST_1(test_symm_pack<float>()); CALL_SUBTEST_1(test_pack_direct<float>()); + CALL_SUBTEST_1(test_copy_panel_range<float>()); CALL_SUBTEST_1(test_mapper_fallback<float>()); CALL_SUBTEST_1(test_neon_small_blocks<float>()); CALL_SUBTEST_1(test_small_products<float>()); @@ -1523,6 +1597,7 @@ CALL_SUBTEST_2(test_packet_ops<double>()); CALL_SUBTEST_2(test_symm_pack<double>()); CALL_SUBTEST_2(test_pack_direct<double>()); + CALL_SUBTEST_2(test_copy_panel_range<double>()); CALL_SUBTEST_2(test_mapper_fallback<double>()); CALL_SUBTEST_2(test_neon_small_blocks<double>()); CALL_SUBTEST_2(test_small_products<double>()); @@ -1538,6 +1613,7 @@ CALL_SUBTEST_3(test_conjugated_products<std::complex<float>>()); CALL_SUBTEST_3(test_symm_pack<std::complex<float>>()); CALL_SUBTEST_3(test_pack_direct<std::complex<float>>()); + CALL_SUBTEST_3(test_copy_panel_range<std::complex<float>>()); CALL_SUBTEST_3(test_mapper_fallback<std::complex<float>>()); CALL_SUBTEST_3(test_neon_small_blocks<std::complex<float>>()); CALL_SUBTEST_3(test_small_products<std::complex<float>>()); @@ -1554,6 +1630,7 @@ #ifdef EIGEN_VECTORIZE_SME_F64F64 CALL_SUBTEST_4(test_symm_pack<std::complex<double>>()); CALL_SUBTEST_4(test_pack_direct<std::complex<double>>()); + CALL_SUBTEST_4(test_copy_panel_range<std::complex<double>>()); CALL_SUBTEST_4(test_mapper_fallback<std::complex<double>>()); CALL_SUBTEST_4(test_neon_small_blocks<std::complex<double>>()); CALL_SUBTEST_4(test_small_products<std::complex<double>>()); @@ -1581,4 +1658,8 @@ CALL_SUBTEST(test_small_products<std::complex<double>>()); #endif #endif + CALL_SUBTEST_9(test_tile_pair_stores<float>()); +#ifdef EIGEN_VECTORIZE_SME_F64F64 + CALL_SUBTEST_9(test_tile_pair_stores<double>()); +#endif }