Core: Keep AArch64 GCC GEBP accumulators in registers libeigen/eigen!3394 Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h index 0a4bec3..b16d8d7 100644 --- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
@@ -1258,6 +1258,17 @@ } }; +// EIGEN_GEBP_NEON_GCC_WORKAROUNDS: AArch64 NEON with GCC >= 9, where the micro-kernel needs help keeping its +// accumulators in registers. +// EIGEN_GEBP_UNROLL_ACCUMULATORS: fully unrolls a loop over the accumulators (at most 3 x 8 = 24), exposing constant +// indices before GCC's scalar replacement pass at -O2. +#if EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && EIGEN_GNUC_STRICT_AT_LEAST(9, 0, 0) +#define EIGEN_GEBP_NEON_GCC_WORKAROUNDS +#define EIGEN_GEBP_UNROLL_ACCUMULATORS _Pragma("GCC unroll 24") +#else +#define EIGEN_GEBP_UNROLL_ACCUMULATORS +#endif + // One step of the micro-kernel: loads MrPackets LHS packets at step K, // then processes NrCols RHS columns via gebp_rhs_cols. template <int K, int MrPackets, int NrCols> @@ -1273,6 +1284,23 @@ EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]); gebp_rhs_cols<0, MrPackets, NrCols>::run(traits, blB, Index(NrCols * K), A, rhs_panel, T0, C); +#if defined(EIGEN_GEBP_NEON_GCC_WORKAROUNDS) && !defined(EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING) + using AccPacket = typename GEBPTraits::AccPacket; + constexpr bool kRealPacket = std::is_same<AccPacket, Packet4f>::value || std::is_same<AccPacket, Packet2d>::value; + EIGEN_IF_CONSTEXPR (MrPackets == 3 && NrCols == 8 && kRealPacket) { + // Keep the next depth step's loads from extending live ranges beyond the 24 accumulators. + __asm__ volatile("" ::: "memory"); +#if EIGEN_GNUC_STRICT_AT_LEAST(16, 0, 0) + // GCC 16 also needs the double accumulator updates to stay on their own depth step. + EIGEN_IF_CONSTEXPR (std::is_same<AccPacket, Packet2d>::value) { + EIGEN_GEBP_UNROLL_ACCUMULATORS + for (int n = 0; n < MrPackets * NrCols; ++n) { + EIGEN_OPTIMIZATION_BARRIER(C[n]); + } + } +#endif + } +#endif } }; // Compiler register allocation workarounds for the GEBP micro-kernel. @@ -1421,6 +1449,7 @@ constexpr int CSize = 3 * NrCols > MrPackets * NrCols ? 3 * NrCols : MrPackets * NrCols; #endif alignas(AccPacketLocal) AccPacketLocal C[CSize]; + EIGEN_GEBP_UNROLL_ACCUMULATORS for (int n = 0; n < MrPackets * NrCols; ++n) traits.initAcc(C[n]); // Double-accumulation trick for 1pX4 path to break FMA dependency chains @@ -1469,6 +1498,7 @@ // Merge double accumulators EIGEN_IF_CONSTEXPR (use_double_accum) { + EIGEN_GEBP_UNROLL_ACCUMULATORS for (int n = 0; n < NrCols; ++n) C[n] = padd(C[n], D[n]); } @@ -1485,8 +1515,10 @@ // ---- Store results: C[j + p * NrCols] -> res(i + p*ResPacketSz, j2 + j) ---- alignas(ResPacketLocal) ResPacketLocal alphav = pset1<ResPacketLocal>(alpha); + EIGEN_GEBP_UNROLL_ACCUMULATORS for (int j = 0; j < NrCols; ++j) { LinearMapper_ r = res.getLinearMapper(i, j2 + j); + EIGEN_GEBP_UNROLL_ACCUMULATORS for (int p = 0; p < MrPackets; ++p) { alignas(ResPacketLocal) ResPacketLocal R = r.template loadPacket<ResPacketLocal>(p * ResPacketSz); traits.acc(C[j + p * NrCols], alphav, R); @@ -1494,6 +1526,8 @@ } } } +#undef EIGEN_GEBP_UNROLL_ACCUMULATORS +#undef EIGEN_GEBP_NEON_GCC_WORKAROUNDS // Workaround a GCC/AArch64 register-allocation issue (through at least GCC 14.3): the float // kernel's mr=12 x nr=8 tile holds 24 of the 32 NEON registers, and GCC's pre-RA scheduler @@ -1501,7 +1535,8 @@ // ~halving GEMM throughput (~2.1x fp32 on Cortex-X925; spills 20 -> 2 with it off). Clang // allocates the same tile spill-free, so we keep nr=8 and just disable that GCC pass here. // This is a known bug in GCC: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100697 -// In-order cores can opt out via EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING. +// In-order cores can opt out via EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING, which also drops the +// depth-step barriers in gebp_micro_step. #if EIGEN_COMP_GNUC_STRICT && EIGEN_ARCH_ARM64 && !defined(EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING) #pragma GCC push_options #pragma GCC optimize("no-schedule-insns")
diff --git a/benchmarks/Core/CMakeLists.txt b/benchmarks/Core/CMakeLists.txt index ae9b2df..d87e0e2 100644 --- a/benchmarks/Core/CMakeLists.txt +++ b/benchmarks/Core/CMakeLists.txt
@@ -11,6 +11,11 @@ eigen_add_benchmark(bench_gemm bench_gemm.cpp) eigen_add_benchmark(bench_gemm_double bench_gemm.cpp DEFINITIONS SCALAR=double) endif() +# Keep -O2 visible: GCC's fixed-loop unrolling changes GEBP register allocation. +eigen_add_benchmark(bench_gemm_o2 bench_gemm.cpp) +eigen_add_benchmark(bench_gemm_double_o2 bench_gemm.cpp DEFINITIONS SCALAR=double) +target_compile_options(bench_gemm_o2 PRIVATE $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-O2>) +target_compile_options(bench_gemm_double_o2 PRIVATE $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-O2>) eigen_add_benchmark(bench_gemm_complex bench_gemm.cpp DEFINITIONS "SCALAR=std::complex<float>" EIGEN_BENCH_COMPLEX_SCALAR) eigen_add_benchmark(bench_gemm_complex_double bench_gemm.cpp