Core: Unroll cheap unary packet assignments with GCC

libeigen/eigen!3393

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h
index 419f090..9120f95 100644
--- a/Eigen/src/Core/AssignEvaluator.h
+++ b/Eigen/src/Core/AssignEvaluator.h
@@ -535,8 +535,21 @@
 
     head_loop::run(kernel, 0, alignedStart);
 
-    for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize)
-      kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index);
+#if EIGEN_GNUC_STRICT_AT_LEAST(8, 0, 0) && EIGEN_ARCH_i386_OR_x86_64 && !defined(EIGEN_GPUCC)
+    // GCC leaves cheap unary packet operations rolled. Copies, multi-source expressions and expensive functors
+    // showed no consistent benefit, so use the coefficient cost as a conservative unrolling heuristic.
+    using SrcEvaluator = typename Kernel::SrcEvaluatorType;
+    EIGEN_IF_CONSTEXPR (int(SrcEvaluator::CoeffReadCost) > NumTraits<Scalar>::ReadCost &&
+                        int(SrcEvaluator::CoeffReadCost) <= 2 * NumTraits<Scalar>::ReadCost) {
+#pragma GCC unroll 2
+      for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize)
+        kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index);
+    } else
+#endif
+    {
+      for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize)
+        kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index);
+    }
 
     // Derive the tail bound directly; GCC can lose its range when it comes from the packet-loop induction variable.
     const Index alignedEnd = size - (size - alignedStart) % PacketSize;
diff --git a/benchmarks/Core/CMakeLists.txt b/benchmarks/Core/CMakeLists.txt
index 6af7219..173c797 100644
--- a/benchmarks/Core/CMakeLists.txt
+++ b/benchmarks/Core/CMakeLists.txt
@@ -23,6 +23,15 @@
 eigen_add_benchmark(bench_gemm_scaled_complex_double bench_gemm_scaled.cpp DEFINITIONS "SCALAR=std::complex<double>")
 eigen_add_benchmark(bench_gemv bench_gemv.cpp)
 eigen_add_benchmark(bench_vecadd bench_vecadd.cpp)
+foreach(scalar float double)
+  foreach(operation abs:0 square:1 add:2 copy:3 exp:4)
+    string(REPLACE ":" ";" operation ${operation})
+    list(GET operation 0 name)
+    list(GET operation 1 index)
+    eigen_add_benchmark(bench_assignment_${name}_${scalar} bench_assignment.cpp
+        DEFINITIONS EIGEN_BENCH_ASSIGN_SCALAR=${scalar} EIGEN_BENCH_ASSIGN_OP=${index})
+  endforeach()
+endforeach()
 eigen_add_benchmark(bench_trsm bench_trsm.cpp)
 eigen_add_benchmark(bench_trsm_packet bench_trsm_packet.cpp)
 eigen_add_benchmark(bench_reverse bench_reverse.cpp)
diff --git a/benchmarks/Core/bench_assignment.cpp b/benchmarks/Core/bench_assignment.cpp
new file mode 100644
index 0000000..5738b30
--- /dev/null
+++ b/benchmarks/Core/bench_assignment.cpp
@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: The Eigen Authors
+// SPDX-License-Identifier: MPL-2.0
+
+#include <benchmark/benchmark.h>
+#include <Eigen/Core>
+#include <cstdint>
+
+#ifndef EIGEN_BENCH_ASSIGN_SCALAR
+#define EIGEN_BENCH_ASSIGN_SCALAR double
+#endif
+#ifndef EIGEN_BENCH_ASSIGN_OP
+// 0: abs, 1: square, 2: add, 3: copy, 4: exp.
+#define EIGEN_BENCH_ASSIGN_OP 0
+#endif
+
+using namespace Eigen;
+
+static void BM_Assignment(benchmark::State& state) {
+  using Scalar = EIGEN_BENCH_ASSIGN_SCALAR;
+  using Vector = Array<Scalar, Dynamic, 1>;
+  // Control the low address bits independently: matching page offsets can trigger false store/load dependencies.
+  constexpr Index pageSize = 4096;
+  const Index size = state.range(0);
+  Vector storage(3 * size + 3 * pageSize / sizeof(Scalar));
+  Scalar* source = storage.data() + internal::first_aligned<pageSize>(storage.data(), storage.size());
+  const Index separation = (((size + 1) * sizeof(Scalar) + pageSize - 1) / pageSize) * pageSize / sizeof(Scalar);
+  Map<Vector> input(source + state.range(1), size);
+  Map<Vector> output(source + separation + state.range(2), size);
+  Vector other(size);
+  for (Index i = 0; i < size; ++i) {
+    input(i) = Scalar(i % 127 - 63) / Scalar(32);
+    other(i) = Scalar(i % 61 - 30) / Scalar(16);
+  }
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(input.data());
+    benchmark::DoNotOptimize(other.data());
+#if EIGEN_BENCH_ASSIGN_OP == 0
+    output = input.abs();
+#elif EIGEN_BENCH_ASSIGN_OP == 1
+    output = input.square();
+#elif EIGEN_BENCH_ASSIGN_OP == 2
+    output = input + other;
+#elif EIGEN_BENCH_ASSIGN_OP == 3
+    output = input;
+#else
+    output = input.exp();
+#endif
+    benchmark::DoNotOptimize(output.data());
+    benchmark::ClobberMemory();
+  }
+  for (Index i = 0; i < size; ++i) {
+#if EIGEN_BENCH_ASSIGN_OP == 0
+    const Scalar expected = numext::abs(input(i));
+#elif EIGEN_BENCH_ASSIGN_OP == 1
+    const Scalar expected = input(i) * input(i);
+#elif EIGEN_BENCH_ASSIGN_OP == 2
+    const Scalar expected = input(i) + other(i);
+#elif EIGEN_BENCH_ASSIGN_OP == 3
+    const Scalar expected = input(i);
+#else
+    const Scalar expected = numext::exp(input(i));
+#endif
+    if (!(numext::abs(output(i) - expected) <= Scalar(8) * NumTraits<Scalar>::epsilon() * numext::abs(expected))) {
+      state.SkipWithError("assignment differs from the scalar reference");
+      return;
+    }
+  }
+  state.SetBytesProcessed(state.iterations() * size * sizeof(Scalar) * (EIGEN_BENCH_ASSIGN_OP == 2 ? 3 : 2));
+}
+
+BENCHMARK(BM_Assignment)
+    ->ArgsProduct({{31, 32, 33, 127, 128, 129, 255, 256, 257, 1023, 1024, 1025, 4095, 4096, 4097, 16384, 65536,
+                    1048576},
+                   {0, 1},
+                   {0, 16}});
diff --git a/test/unalignedcount.cpp b/test/unalignedcount.cpp
index 1bf94d4..fabeab3 100644
--- a/test/unalignedcount.cpp
+++ b/test/unalignedcount.cpp
@@ -32,23 +32,34 @@
   }
 
 #include "main.h"
+#include <Eigen/Core>
 
-template <typename Scalar>
+template <typename Scalar, int Options = ColMajor>
 void assignment_packet_boundaries() {
-  using VectorType = Matrix<Scalar, Dynamic, 1>;
+  using VectorType = Matrix<Scalar, Options == RowMajor ? 1 : Dynamic, Options == RowMajor ? Dynamic : 1, Options>;
+  using MapType = Map<VectorType>;
+  using Traits = internal::copy_using_evaluator_traits<internal::evaluator<MapType>, internal::evaluator<MapType>,
+                                                       internal::assign_op<Scalar, Scalar>>;
+  STATIC_CHECK((!internal::packet_traits<Scalar>::Vectorizable || Traits::Traversal == LinearVectorizedTraversal));
+  STATIC_CHECK(Traits::Unrolling == NoUnrolling);
   constexpr int packetSize = internal::packet_traits<Scalar>::size;
   for (Index offset = 0; offset < packetSize; ++offset) {
-    for (Index size = 0; size <= 2 * packetSize + 1; ++size) {
-      VectorType source(size), storage(size + 2 * packetSize);
-      for (Index i = 0; i < size; ++i) source(i) = Scalar(i + 1);
+    for (Index size = 0; size <= 9 * packetSize + 1; ++size) {
+      VectorType sourceStorage(size + 2 * packetSize), storage(size + 2 * packetSize);
+      MapType source(sourceStorage.data() + (packetSize - offset) % packetSize, size);
+      for (Index i = 0; i < size; ++i) source(i) = Scalar(i % 7 + 1);
       storage.setConstant(Scalar(-1));
-      Map<VectorType> destination(storage.data() + offset, size);
+      MapType destination(storage.data() + offset, size);
       destination = source;
-      for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(i + 1));
+      for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(i % 7 + 1));
       destination += source;
-      for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(2 * (i + 1)));
+      for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(2 * (i % 7 + 1)));
       destination -= source;
-      for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(i + 1));
+      for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(i % 7 + 1));
+      destination.array() = source.array().square();
+      for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar((i % 7 + 1) * (i % 7 + 1)));
+      destination.array() += destination.array();
+      for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(2 * (i % 7 + 1) * (i % 7 + 1)));
       for (Index i = 0; i < offset; ++i) VERIFY_IS_EQUAL(storage(i), Scalar(-1));
       for (Index i = offset + size; i < storage.size(); ++i) VERIFY_IS_EQUAL(storage(i), Scalar(-1));
     }
@@ -58,6 +69,10 @@
 EIGEN_DECLARE_TEST(unalignedcount) {
   CALL_SUBTEST(assignment_packet_boundaries<float>());
   CALL_SUBTEST(assignment_packet_boundaries<double>());
+  CALL_SUBTEST((assignment_packet_boundaries<float, RowMajor>()));
+  CALL_SUBTEST(assignment_packet_boundaries<int>());
+  CALL_SUBTEST(assignment_packet_boundaries<std::complex<float>>());
+  CALL_SUBTEST(assignment_packet_boundaries<std::complex<double>>());
 #if defined(EIGEN_VECTORIZE_AVX512)
   VectorXf a(48), b(48);
   a.fill(0);