Core: Fix the blocked triangular solve for a right-hand side with a runtime inner stride

libeigen/eigen!2985

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>

Eigen/src/Core/products/TriangularSolverMatrix.h

diff --git a/Eigen/src/Core/products/TriangularSolverMatrix.h b/Eigen/src/Core/products/TriangularSolverMatrix.h
index 946fd5d..f720f73 100644
--- a/Eigen/src/Core/products/TriangularSolverMatrix.h
+++ b/Eigen/src/Core/products/TriangularSolverMatrix.h
@@ -282,7 +282,7 @@
 #endif
           trsmKernelL<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride, /*Specialized=*/true>::kernel(
               actualPanelWidth, actual_cols, _tri + i + (i)*triStride, triStride,
-              _other + i * OtherInnerStride + j2 * otherStride, otherIncr, otherStride);
+              _other + i * otherIncr + j2 * otherStride, otherIncr, otherStride);
         }
 
         Index lengthTarget = actual_kc - k1 - actualPanelWidth;
@@ -434,8 +434,8 @@
             trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride,
                         /*Specialized=*/true>::kernel(actualPanelWidth, actual_mc,
                                                       _tri + absolute_j2 + absolute_j2 * triStride, triStride,
-                                                      _other + i2 * OtherInnerStride + absolute_j2 * otherStride,
-                                                      otherIncr, otherStride);
+                                                      _other + i2 * otherIncr + absolute_j2 * otherStride, otherIncr,
+                                                      otherStride);
           }
           // pack the just computed part of lhs to A
           pack_lhs_panel(blockA, lhs.getSubMapper(i2, absolute_j2), actualPanelWidth, actual_mc, actual_kc, j2);

test/product_trsolve.cpp

diff --git a/test/product_trsolve.cpp b/test/product_trsolve.cpp
index cab1f0b..7f95e7f 100644
--- a/test/product_trsolve.cpp
+++ b/test/product_trsolve.cpp
@@ -98,6 +98,21 @@
     VERIFY_TRSM(cmLhs.conjugate().template triangularView<Lower>(), map1);
     buffer.setZero();
     VERIFY_TRSM(cmLhs.template triangularView<Lower>(), map2);
+
+    // A runtime inner stride reaches the blocked kernels as OtherInnerStride == Dynamic, which the
+    // compile-time 2 above does not; their panel offsets once multiplied by that constant.
+    Map<Matrix<Scalar, Size, Cols, colmajor>, 0, Stride<Dynamic, Dynamic> > map3(
+        buffer.data(), cmRhs.rows(), cmRhs.cols(), Stride<Dynamic, Dynamic>(2 * cmRhs.outerStride(), 2));
+    Map<Matrix<Scalar, Size, Cols, rowmajor>, 0, Stride<Dynamic, Dynamic> > map4(
+        buffer.data(), rmRhs.rows(), rmRhs.cols(), Stride<Dynamic, Dynamic>(2 * rmRhs.outerStride(), 2));
+    buffer.setZero();
+    VERIFY_TRSM(cmLhs.template triangularView<Lower>(), map3);
+    buffer.setZero();
+    VERIFY_TRSM(cmLhs.template triangularView<Upper>(), map4);
+    buffer.setZero();
+    VERIFY_TRSM_ONTHERIGHT(cmLhs.template triangularView<Lower>(), map3);
+    buffer.setZero();
+    VERIFY_TRSM_ONTHERIGHT(cmLhs.template triangularView<Upper>(), map4);
   }
 
   if (Size == Dynamic) {
@@ -157,6 +172,24 @@
       VERIFY_IS_APPROX(lhs.triangularView<Upper>().toDenseMatrix() * MatrixX(map), ref);
     }
 
+    // Runtime inner stride (OtherInnerStride == Dynamic), both sides
+    {
+      int cols = 5;
+      MatrixX buffer(2 * n, 2 * cols);
+      Map<MatrixX, 0, Stride<Dynamic, Dynamic> > map(buffer.data(), n, cols, Stride<Dynamic, Dynamic>(2 * n, 2));
+      MatrixX ref(n, cols);
+      buffer.setZero();
+      map.setRandom();
+      ref = map;
+      lhs.triangularView<Lower>().solveInPlace(map);
+      VERIFY_IS_APPROX(lhs.triangularView<Lower>().toDenseMatrix() * MatrixX(map), ref);
+      buffer.setZero();
+      map.setRandom();
+      ref = map;
+      lhs.triangularView<Upper>().template solveInPlace<OnTheRight>(map.transpose());
+      VERIFY_IS_APPROX(MatrixX(map.transpose()) * lhs.triangularView<Upper>().toDenseMatrix(), ref.transpose());
+    }
+
     // InnerStride = 2: UnitLower (tests the UnitDiag path without diagonal scaling)
     {
       int cols = 3;