Core: Iterate one-packet GEBP row panels inside the column loop

libeigen/eigen!3428

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>

Eigen/src/Core/products/GeneralBlockPanelKernel.h

diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
index 160940e..3bebe2a 100644
--- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h
+++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
@@ -1691,7 +1691,8 @@
   }
 
   //---------- Process 1 * LhsProgress rows at once ----------
-  EIGEN_IF_CONSTEXPR (mr >= 1 * Traits::LhsProgress) {
+  // With a wider mr, at most one row panel is left here.
+  EIGEN_IF_CONSTEXPR (mr > 1 * Traits::LhsProgress) {
     for (Index i = peeled_mc2; i < peeled_mc1; i += LhsProgress) {
       EIGEN_IF_CONSTEXPR (nr >= 8) {
         for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
@@ -1705,6 +1706,36 @@
         micro_panel(fix<1>, fix<1>, traits, i, j2);
       }
     }
+  } else EIGEN_IF_CONSTEXPR (mr == 1 * Traits::LhsProgress) {
+    // Every row comes here, blocked like the wider panels above: rows inside columns keep the rhs panel in cache
+    // and walk C down its columns.
+    const Index rhs_block1 = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar);
+    const Index lhs_strip1 = depth * sizeof(LhsScalar) * LhsProgress;
+    const Index lhs_avail1 = (lhs_budget > rhs_block1) ? (lhs_budget - rhs_block1) : 0;
+    const Index mc1_range = peeled_mc1 - peeled_mc2;
+    const Index actual_panel_rows = (lhs_avail1 >= mc1_range * depth * static_cast<Index>(sizeof(LhsScalar)))
+                                        ? mc1_range
+                                        : LhsProgress * std::max<Index>(1, lhs_avail1 / lhs_strip1);
+    for (Index i1 = peeled_mc2; i1 < peeled_mc1; i1 += actual_panel_rows) {
+      const Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc1);
+      EIGEN_IF_CONSTEXPR (nr >= 8) {
+        for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
+          for (Index i = i1; i < actual_panel_end; i += LhsProgress) {
+            micro_panel(fix<1>, fix<8>, traits, i, j2);
+          }
+        }
+      }
+      for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
+        for (Index i = i1; i < actual_panel_end; i += LhsProgress) {
+          micro_panel(fix<1>, fix<4>, traits, i, j2);
+        }
+      }
+      for (Index j2 = packet_cols4; j2 < cols; j2++) {
+        for (Index i = i1; i < actual_panel_end; i += LhsProgress) {
+          micro_panel(fix<1>, fix<1>, traits, i, j2);
+        }
+      }
+    }
   }
 
   //---------- Process LhsProgressHalf rows at once ----------

benchmarks/Core/bench_gemm.cpp

diff --git a/benchmarks/Core/bench_gemm.cpp b/benchmarks/Core/bench_gemm.cpp
index ffc6c1e..cb9329a 100644
--- a/benchmarks/Core/bench_gemm.cpp
+++ b/benchmarks/Core/bench_gemm.cpp
@@ -114,9 +114,14 @@
 #define TAIL_PANEL_SIZES \
     ->Args({15, 17, 1024})->Args({17, 15, 1024})->Args({31, 31, 1024}) \
     ->Args({60, 60, 1024})->Args({60, 64, 1024})->Args({64, 60, 1024})->Args({120, 120, 1024})->Args({252, 252, 252})
+
+// Shallow depth: loading and storing c costs about as much as the k multiply-adds, so these measure the order in which
+// gebp's micro panels walk c.
+#define SHALLOW_DEPTH_SIZES \
+    ->ArgsProduct({{256}, {256}, {4, 8, 16}})->Args({1024, 256, 8})->Args({1024, 1024, 4})
 // clang-format on
 
-BENCHMARK(BM_EigenGemm) GEMM_SIZES DEPTH_SWEEP_SIZES TAIL_PANEL_SIZES;
+BENCHMARK(BM_EigenGemm) GEMM_SIZES DEPTH_SWEEP_SIZES TAIL_PANEL_SIZES SHALLOW_DEPTH_SIZES;
 BENCHMARK(BM_EigenGemmRowMajorLhs)->Args({256, 256, 256})->Args({1024, 1024, 1024}) DEPTH_SWEEP_SIZES TAIL_PANEL_SIZES;
 BENCHMARK(BM_EigenGemmRowMajorRhs)
     ->Args({64, 64, 1024})