Core: Keep AArch64 GCC GEBP accumulators in registers

libeigen/eigen!3394

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
index 0a4bec3..b16d8d7 100644
--- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h
+++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
@@ -1258,6 +1258,17 @@
   }
 };
 
+// EIGEN_GEBP_NEON_GCC_WORKAROUNDS: AArch64 NEON with GCC >= 9, where the micro-kernel needs help keeping its
+// accumulators in registers.
+// EIGEN_GEBP_UNROLL_ACCUMULATORS: fully unrolls a loop over the accumulators (at most 3 x 8 = 24), exposing constant
+// indices before GCC's scalar replacement pass at -O2.
+#if EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && EIGEN_GNUC_STRICT_AT_LEAST(9, 0, 0)
+#define EIGEN_GEBP_NEON_GCC_WORKAROUNDS
+#define EIGEN_GEBP_UNROLL_ACCUMULATORS _Pragma("GCC unroll 24")
+#else
+#define EIGEN_GEBP_UNROLL_ACCUMULATORS
+#endif
+
 // One step of the micro-kernel: loads MrPackets LHS packets at step K,
 // then processes NrCols RHS columns via gebp_rhs_cols.
 template <int K, int MrPackets, int NrCols>
@@ -1273,6 +1284,23 @@
     EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]);
 
     gebp_rhs_cols<0, MrPackets, NrCols>::run(traits, blB, Index(NrCols * K), A, rhs_panel, T0, C);
+#if defined(EIGEN_GEBP_NEON_GCC_WORKAROUNDS) && !defined(EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING)
+    using AccPacket = typename GEBPTraits::AccPacket;
+    constexpr bool kRealPacket = std::is_same<AccPacket, Packet4f>::value || std::is_same<AccPacket, Packet2d>::value;
+    EIGEN_IF_CONSTEXPR (MrPackets == 3 && NrCols == 8 && kRealPacket) {
+      // Keep the next depth step's loads from extending live ranges beyond the 24 accumulators.
+      __asm__ volatile("" ::: "memory");
+#if EIGEN_GNUC_STRICT_AT_LEAST(16, 0, 0)
+      // GCC 16 also needs the double accumulator updates to stay on their own depth step.
+      EIGEN_IF_CONSTEXPR (std::is_same<AccPacket, Packet2d>::value) {
+        EIGEN_GEBP_UNROLL_ACCUMULATORS
+        for (int n = 0; n < MrPackets * NrCols; ++n) {
+          EIGEN_OPTIMIZATION_BARRIER(C[n]);
+        }
+      }
+#endif
+    }
+#endif
   }
 };
 // Compiler register allocation workarounds for the GEBP micro-kernel.
@@ -1421,6 +1449,7 @@
   constexpr int CSize = 3 * NrCols > MrPackets * NrCols ? 3 * NrCols : MrPackets * NrCols;
 #endif
   alignas(AccPacketLocal) AccPacketLocal C[CSize];
+  EIGEN_GEBP_UNROLL_ACCUMULATORS
   for (int n = 0; n < MrPackets * NrCols; ++n) traits.initAcc(C[n]);
 
   // Double-accumulation trick for 1pX4 path to break FMA dependency chains
@@ -1469,6 +1498,7 @@
 
   // Merge double accumulators
   EIGEN_IF_CONSTEXPR (use_double_accum) {
+    EIGEN_GEBP_UNROLL_ACCUMULATORS
     for (int n = 0; n < NrCols; ++n) C[n] = padd(C[n], D[n]);
   }
 
@@ -1485,8 +1515,10 @@
 
   // ---- Store results: C[j + p * NrCols] -> res(i + p*ResPacketSz, j2 + j) ----
   alignas(ResPacketLocal) ResPacketLocal alphav = pset1<ResPacketLocal>(alpha);
+  EIGEN_GEBP_UNROLL_ACCUMULATORS
   for (int j = 0; j < NrCols; ++j) {
     LinearMapper_ r = res.getLinearMapper(i, j2 + j);
+    EIGEN_GEBP_UNROLL_ACCUMULATORS
     for (int p = 0; p < MrPackets; ++p) {
       alignas(ResPacketLocal) ResPacketLocal R = r.template loadPacket<ResPacketLocal>(p * ResPacketSz);
       traits.acc(C[j + p * NrCols], alphav, R);
@@ -1494,6 +1526,8 @@
     }
   }
 }
+#undef EIGEN_GEBP_UNROLL_ACCUMULATORS
+#undef EIGEN_GEBP_NEON_GCC_WORKAROUNDS
 
 // Workaround a GCC/AArch64 register-allocation issue (through at least GCC 14.3): the float
 // kernel's mr=12 x nr=8 tile holds 24 of the 32 NEON registers, and GCC's pre-RA scheduler
@@ -1501,7 +1535,8 @@
 // ~halving GEMM throughput (~2.1x fp32 on Cortex-X925; spills 20 -> 2 with it off). Clang
 // allocates the same tile spill-free, so we keep nr=8 and just disable that GCC pass here.
 // This is a known bug in GCC: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100697
-// In-order cores can opt out via EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING.
+// In-order cores can opt out via EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING, which also drops the
+// depth-step barriers in gebp_micro_step.
 #if EIGEN_COMP_GNUC_STRICT && EIGEN_ARCH_ARM64 && !defined(EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING)
 #pragma GCC push_options
 #pragma GCC optimize("no-schedule-insns")
diff --git a/benchmarks/Core/CMakeLists.txt b/benchmarks/Core/CMakeLists.txt
index ae9b2df..d87e0e2 100644
--- a/benchmarks/Core/CMakeLists.txt
+++ b/benchmarks/Core/CMakeLists.txt
@@ -11,6 +11,11 @@
   eigen_add_benchmark(bench_gemm bench_gemm.cpp)
   eigen_add_benchmark(bench_gemm_double bench_gemm.cpp DEFINITIONS SCALAR=double)
 endif()
+# Keep -O2 visible: GCC's fixed-loop unrolling changes GEBP register allocation.
+eigen_add_benchmark(bench_gemm_o2 bench_gemm.cpp)
+eigen_add_benchmark(bench_gemm_double_o2 bench_gemm.cpp DEFINITIONS SCALAR=double)
+target_compile_options(bench_gemm_o2 PRIVATE $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-O2>)
+target_compile_options(bench_gemm_double_o2 PRIVATE $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-O2>)
 eigen_add_benchmark(bench_gemm_complex bench_gemm.cpp
     DEFINITIONS "SCALAR=std::complex<float>" EIGEN_BENCH_COMPLEX_SCALAR)
 eigen_add_benchmark(bench_gemm_complex_double bench_gemm.cpp