Core: Unroll cheap unary packet assignments with GCC libeigen/eigen!3393 Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h index 419f090..9120f95 100644 --- a/Eigen/src/Core/AssignEvaluator.h +++ b/Eigen/src/Core/AssignEvaluator.h
@@ -535,8 +535,21 @@ head_loop::run(kernel, 0, alignedStart); - for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize) - kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index); +#if EIGEN_GNUC_STRICT_AT_LEAST(8, 0, 0) && EIGEN_ARCH_i386_OR_x86_64 && !defined(EIGEN_GPUCC) + // GCC leaves cheap unary packet operations rolled. Copies, multi-source expressions and expensive functors + // showed no consistent benefit, so use the coefficient cost as a conservative unrolling heuristic. + using SrcEvaluator = typename Kernel::SrcEvaluatorType; + EIGEN_IF_CONSTEXPR (int(SrcEvaluator::CoeffReadCost) > NumTraits<Scalar>::ReadCost && + int(SrcEvaluator::CoeffReadCost) <= 2 * NumTraits<Scalar>::ReadCost) { +#pragma GCC unroll 2 + for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize) + kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index); + } else +#endif + { + for (Index index = alignedStart; index <= size - PacketSize; index += PacketSize) + kernel.template assignPacket<Alignment, SrcAlignment, PacketType>(index); + } // Derive the tail bound directly; GCC can lose its range when it comes from the packet-loop induction variable. const Index alignedEnd = size - (size - alignedStart) % PacketSize;
diff --git a/benchmarks/Core/CMakeLists.txt b/benchmarks/Core/CMakeLists.txt index 6af7219..173c797 100644 --- a/benchmarks/Core/CMakeLists.txt +++ b/benchmarks/Core/CMakeLists.txt
@@ -23,6 +23,15 @@ eigen_add_benchmark(bench_gemm_scaled_complex_double bench_gemm_scaled.cpp DEFINITIONS "SCALAR=std::complex<double>") eigen_add_benchmark(bench_gemv bench_gemv.cpp) eigen_add_benchmark(bench_vecadd bench_vecadd.cpp) +foreach(scalar float double) + foreach(operation abs:0 square:1 add:2 copy:3 exp:4) + string(REPLACE ":" ";" operation ${operation}) + list(GET operation 0 name) + list(GET operation 1 index) + eigen_add_benchmark(bench_assignment_${name}_${scalar} bench_assignment.cpp + DEFINITIONS EIGEN_BENCH_ASSIGN_SCALAR=${scalar} EIGEN_BENCH_ASSIGN_OP=${index}) + endforeach() +endforeach() eigen_add_benchmark(bench_trsm bench_trsm.cpp) eigen_add_benchmark(bench_trsm_packet bench_trsm_packet.cpp) eigen_add_benchmark(bench_reverse bench_reverse.cpp)
diff --git a/benchmarks/Core/bench_assignment.cpp b/benchmarks/Core/bench_assignment.cpp new file mode 100644 index 0000000..5738b30 --- /dev/null +++ b/benchmarks/Core/bench_assignment.cpp
@@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include <benchmark/benchmark.h> +#include <Eigen/Core> +#include <cstdint> + +#ifndef EIGEN_BENCH_ASSIGN_SCALAR +#define EIGEN_BENCH_ASSIGN_SCALAR double +#endif +#ifndef EIGEN_BENCH_ASSIGN_OP +// 0: abs, 1: square, 2: add, 3: copy, 4: exp. +#define EIGEN_BENCH_ASSIGN_OP 0 +#endif + +using namespace Eigen; + +static void BM_Assignment(benchmark::State& state) { + using Scalar = EIGEN_BENCH_ASSIGN_SCALAR; + using Vector = Array<Scalar, Dynamic, 1>; + // Control the low address bits independently: matching page offsets can trigger false store/load dependencies. + constexpr Index pageSize = 4096; + const Index size = state.range(0); + Vector storage(3 * size + 3 * pageSize / sizeof(Scalar)); + Scalar* source = storage.data() + internal::first_aligned<pageSize>(storage.data(), storage.size()); + const Index separation = (((size + 1) * sizeof(Scalar) + pageSize - 1) / pageSize) * pageSize / sizeof(Scalar); + Map<Vector> input(source + state.range(1), size); + Map<Vector> output(source + separation + state.range(2), size); + Vector other(size); + for (Index i = 0; i < size; ++i) { + input(i) = Scalar(i % 127 - 63) / Scalar(32); + other(i) = Scalar(i % 61 - 30) / Scalar(16); + } + for (auto _ : state) { + benchmark::DoNotOptimize(input.data()); + benchmark::DoNotOptimize(other.data()); +#if EIGEN_BENCH_ASSIGN_OP == 0 + output = input.abs(); +#elif EIGEN_BENCH_ASSIGN_OP == 1 + output = input.square(); +#elif EIGEN_BENCH_ASSIGN_OP == 2 + output = input + other; +#elif EIGEN_BENCH_ASSIGN_OP == 3 + output = input; +#else + output = input.exp(); +#endif + benchmark::DoNotOptimize(output.data()); + benchmark::ClobberMemory(); + } + for (Index i = 0; i < size; ++i) { +#if EIGEN_BENCH_ASSIGN_OP == 0 + const Scalar expected = numext::abs(input(i)); +#elif EIGEN_BENCH_ASSIGN_OP == 1 + const Scalar expected = input(i) * input(i); +#elif EIGEN_BENCH_ASSIGN_OP == 2 + const Scalar expected = input(i) + other(i); +#elif EIGEN_BENCH_ASSIGN_OP == 3 + const Scalar expected = input(i); +#else + const Scalar expected = numext::exp(input(i)); +#endif + if (!(numext::abs(output(i) - expected) <= Scalar(8) * NumTraits<Scalar>::epsilon() * numext::abs(expected))) { + state.SkipWithError("assignment differs from the scalar reference"); + return; + } + } + state.SetBytesProcessed(state.iterations() * size * sizeof(Scalar) * (EIGEN_BENCH_ASSIGN_OP == 2 ? 3 : 2)); +} + +BENCHMARK(BM_Assignment) + ->ArgsProduct({{31, 32, 33, 127, 128, 129, 255, 256, 257, 1023, 1024, 1025, 4095, 4096, 4097, 16384, 65536, + 1048576}, + {0, 1}, + {0, 16}});
diff --git a/test/unalignedcount.cpp b/test/unalignedcount.cpp index 1bf94d4..fabeab3 100644 --- a/test/unalignedcount.cpp +++ b/test/unalignedcount.cpp
@@ -32,23 +32,34 @@ } #include "main.h" +#include <Eigen/Core> -template <typename Scalar> +template <typename Scalar, int Options = ColMajor> void assignment_packet_boundaries() { - using VectorType = Matrix<Scalar, Dynamic, 1>; + using VectorType = Matrix<Scalar, Options == RowMajor ? 1 : Dynamic, Options == RowMajor ? Dynamic : 1, Options>; + using MapType = Map<VectorType>; + using Traits = internal::copy_using_evaluator_traits<internal::evaluator<MapType>, internal::evaluator<MapType>, + internal::assign_op<Scalar, Scalar>>; + STATIC_CHECK((!internal::packet_traits<Scalar>::Vectorizable || Traits::Traversal == LinearVectorizedTraversal)); + STATIC_CHECK(Traits::Unrolling == NoUnrolling); constexpr int packetSize = internal::packet_traits<Scalar>::size; for (Index offset = 0; offset < packetSize; ++offset) { - for (Index size = 0; size <= 2 * packetSize + 1; ++size) { - VectorType source(size), storage(size + 2 * packetSize); - for (Index i = 0; i < size; ++i) source(i) = Scalar(i + 1); + for (Index size = 0; size <= 9 * packetSize + 1; ++size) { + VectorType sourceStorage(size + 2 * packetSize), storage(size + 2 * packetSize); + MapType source(sourceStorage.data() + (packetSize - offset) % packetSize, size); + for (Index i = 0; i < size; ++i) source(i) = Scalar(i % 7 + 1); storage.setConstant(Scalar(-1)); - Map<VectorType> destination(storage.data() + offset, size); + MapType destination(storage.data() + offset, size); destination = source; - for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(i + 1)); + for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(i % 7 + 1)); destination += source; - for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(2 * (i + 1))); + for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(2 * (i % 7 + 1))); destination -= source; - for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(i + 1)); + for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(i % 7 + 1)); + destination.array() = source.array().square(); + for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar((i % 7 + 1) * (i % 7 + 1))); + destination.array() += destination.array(); + for (Index i = 0; i < size; ++i) VERIFY_IS_EQUAL(destination(i), Scalar(2 * (i % 7 + 1) * (i % 7 + 1))); for (Index i = 0; i < offset; ++i) VERIFY_IS_EQUAL(storage(i), Scalar(-1)); for (Index i = offset + size; i < storage.size(); ++i) VERIFY_IS_EQUAL(storage(i), Scalar(-1)); } @@ -58,6 +69,10 @@ EIGEN_DECLARE_TEST(unalignedcount) { CALL_SUBTEST(assignment_packet_boundaries<float>()); CALL_SUBTEST(assignment_packet_boundaries<double>()); + CALL_SUBTEST((assignment_packet_boundaries<float, RowMajor>())); + CALL_SUBTEST(assignment_packet_boundaries<int>()); + CALL_SUBTEST(assignment_packet_boundaries<std::complex<float>>()); + CALL_SUBTEST(assignment_packet_boundaries<std::complex<double>>()); #if defined(EIGEN_VECTORIZE_AVX512) VectorXf a(48), b(48); a.fill(0);