Core bug fixes: SSE, AVX and AVX-512 backends libeigen/eigen!3348 Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/arch/AVX/MathFunctions.h b/Eigen/src/Core/arch/AVX/MathFunctions.h index 63a529d..f09e144 100644 --- a/Eigen/src/Core/arch/AVX/MathFunctions.h +++ b/Eigen/src/Core/arch/AVX/MathFunctions.h
@@ -88,13 +88,13 @@ #if EIGEN_FAST_MATH template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8f prsqrt<Packet8f>(const Packet8f& a) { - // _mm256_rsqrt_ps returns -inf for negative denormals. - // _mm512_rsqrt**_ps returns -NaN for negative denormals. We may want - // consistency here. - // const Packet8f rsqrt = pselect(pcmp_lt(a, pzero(a)), - // pset1<Packet8f>(-NumTraits<float>::quiet_NaN()), - // _mm256_rsqrt_ps(a)); - return generic_rsqrt_newton_step<Packet8f, /*Steps=*/1>::run(a, _mm256_rsqrt_ps(a)); + // vrsqrtps reads a subnormal a as zero, giving e = +-inf. a * e, the Newton step's first product, is therefore +inf + // for subnormal a, and finite or NaN for every other a; those lanes take 1 / sqrt(a) instead. Under DAZ the multiply + // reads a subnormal as zero too, and the estimate's +-inf is the DAZ result. + const Packet8f e = _mm256_rsqrt_ps(a); + const Packet8f r = generic_rsqrt_newton_step<Packet8f, /*Steps=*/1>::run(a, e); + const Packet8f redo = pcmp_eq(pmul(a, e), pset1<Packet8f>(NumTraits<float>::infinity())); + return predux_any(redo) ? pselect(redo, pdiv(pset1<Packet8f>(1.0f), psqrt(a)), r) : r; } template <>
diff --git a/Eigen/src/Core/arch/AVX/PacketMath.h b/Eigen/src/Core/arch/AVX/PacketMath.h index ba7b402..a5399b7 100644 --- a/Eigen/src/Core/arch/AVX/PacketMath.h +++ b/Eigen/src/Core/arch/AVX/PacketMath.h
@@ -1932,7 +1932,7 @@ EIGEN_STRONG_INLINE Packet4d pldexp<Packet4d>(const Packet4d& a, const Packet4d& exponent) { // Clamp exponent to [-2099, 2099] const Packet4d max_exponent = pset1<Packet4d>(2099.0); - const Packet4i e = _mm256_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); + const Packet4i e = _mm256_cvttpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); // The single-rounding split of pldexp_generic. const Packet4i bias = pset1<Packet4i>(1023);
diff --git a/Eigen/src/Core/arch/AVX512/PacketMath.h b/Eigen/src/Core/arch/AVX512/PacketMath.h index c0a244a..7d31c46 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMath.h +++ b/Eigen/src/Core/arch/AVX512/PacketMath.h
@@ -1221,12 +1221,40 @@ EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_pd(to, mask, from); } +// The gathers and scatters below address lane i through the 32-bit offset i * stride. Where that overflows an int, +// the lanes are addressed one at a time. +template <typename Packet> +EIGEN_STRONG_INLINE bool avx512_lane_offsets_fit(Index stride) { + const Index max_stride = Index(NumTraits<int>::highest()) / Index(unpacket_traits<Packet>::size - 1); + return stride <= max_stride && stride >= -max_stride; +} + +template <typename Scalar, typename Packet> +EIGEN_STRONG_INLINE Packet avx512_pgather_lanes(const Packet& src, const Scalar* from, Index stride, + typename unpacket_traits<Packet>::mask_t umask) { + EIGEN_ALIGN64 Scalar values[unpacket_traits<Packet>::size]; + pstore(values, src); + for (int i = 0; i < unpacket_traits<Packet>::size; ++i) + if ((umask >> i) & 1) values[i] = from[i * stride]; + return pload<Packet>(values); +} + +template <typename Scalar, typename Packet> +EIGEN_STRONG_INLINE void avx512_pscatter_lanes(Scalar* to, const Packet& from, Index stride, + typename unpacket_traits<Packet>::mask_t umask) { + EIGEN_ALIGN64 Scalar values[unpacket_traits<Packet>::size]; + pstore(values, from); + for (int i = 0; i < unpacket_traits<Packet>::size; ++i) + if ((umask >> i) & 1) to[i * stride] = values[i]; +} + template <typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline Packet pgather(const Packet& src, const Scalar* from, Index stride, typename unpacket_traits<Packet>::mask_t umask); template <> EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const Packet16f& src, const float* from, Index stride, uint16_t umask) { + if (!avx512_lane_offsets_fit<Packet16f>(stride)) return avx512_pgather_lanes(src, from, stride, umask); Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride)); Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); @@ -1237,6 +1265,7 @@ template <> EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const Packet8d& src, const double* from, Index stride, uint8_t umask) { + if (!avx512_lane_offsets_fit<Packet8d>(stride)) return avx512_pgather_lanes(src, from, stride, umask); Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride)); Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); @@ -1254,6 +1283,7 @@ const Packet16i indices = _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 17, 19, 21, 23, 25, 27, 29, 31); return _mm512_permutex2var_ps(low, indices, high); } + if (!avx512_lane_offsets_fit<Packet16f>(stride)) return pgather_partial<float, Packet16f>(from, stride, 16); Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride)); Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); @@ -1268,6 +1298,7 @@ const __m512i indices = _mm512_setr_epi64(0, 2, 4, 6, 9, 11, 13, 15); return _mm512_permutex2var_pd(low, indices, high); } + if (!avx512_lane_offsets_fit<Packet8d>(stride)) return pgather_partial<double, Packet8d>(from, stride, 8); Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride)); Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); @@ -1276,6 +1307,7 @@ } template <> EIGEN_DEVICE_FUNC inline Packet8l pgather<int64_t, Packet8l>(const int64_t* from, Index stride) { + if (!avx512_lane_offsets_fit<Packet8l>(stride)) return pgather_partial<int64_t, Packet8l>(from, stride, 8); Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride)); Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); @@ -1284,6 +1316,7 @@ } template <> EIGEN_DEVICE_FUNC inline Packet16i pgather<int, Packet16i>(const int* from, Index stride) { + if (!avx512_lane_offsets_fit<Packet16i>(stride)) return pgather_partial<int, Packet16i>(from, stride, 16); Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride)); Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); @@ -1296,6 +1329,7 @@ template <> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride, uint16_t umask) { + if (!avx512_lane_offsets_fit<Packet16f>(stride)) return avx512_pscatter_lanes(to, from, stride, umask); Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride)); Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); @@ -1305,6 +1339,7 @@ template <> EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride, uint8_t umask) { + if (!avx512_lane_offsets_fit<Packet8d>(stride)) return avx512_pscatter_lanes(to, from, stride, umask); Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride)); Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); @@ -1313,6 +1348,7 @@ } template <> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride) { + if (!avx512_lane_offsets_fit<Packet16f>(stride)) return pscatter_partial<float, Packet16f>(to, from, stride, 16); Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride)); Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); @@ -1320,6 +1356,7 @@ } template <> EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride) { + if (!avx512_lane_offsets_fit<Packet8d>(stride)) return pscatter_partial<double, Packet8d>(to, from, stride, 8); Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride)); Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); @@ -1327,6 +1364,7 @@ } template <> EIGEN_DEVICE_FUNC inline void pscatter<int64_t, Packet8l>(int64_t* to, const Packet8l& from, Index stride) { + if (!avx512_lane_offsets_fit<Packet8l>(stride)) return pscatter_partial<int64_t, Packet8l>(to, from, stride, 8); Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride)); Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); @@ -1334,6 +1372,7 @@ } template <> EIGEN_DEVICE_FUNC inline void pscatter<int, Packet16i>(int* to, const Packet16i& from, Index stride) { + if (!avx512_lane_offsets_fit<Packet16i>(stride)) return pscatter_partial<int, Packet16i>(to, from, stride, 16); Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride)); Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
diff --git a/Eigen/src/Core/arch/SSE/MathFunctions.h b/Eigen/src/Core/arch/SSE/MathFunctions.h index a8fbd09..24a70f1 100644 --- a/Eigen/src/Core/arch/SSE/MathFunctions.h +++ b/Eigen/src/Core/arch/SSE/MathFunctions.h
@@ -44,10 +44,18 @@ } #if EIGEN_FAST_MATH -// Even on Skylake, using Newton iteration is a win for reciprocal square root. +// Even on Skylake, using Newton iteration is a win for reciprocal square root. rsqrtps reads a subnormal x as zero, +// giving +-inf, so the subnormal lanes, 1 <= bits(|x|) <= 0x007fffff, take 1 / sqrt(x) instead. Adding 0x7f800000 +// moves exactly that range above 0x7f800000 and wraps every larger value negative. With SSE this test of x alone is +// faster than the test of x * estimate in prsqrt<Packet8f>, which wins with AVX. Under DAZ the fallback reads a +// subnormal as zero too, and returns the estimate's +-inf. template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f prsqrt<Packet4f>(const Packet4f& x) { - return generic_rsqrt_newton_step<Packet4f, /*Steps=*/1>::run(x, _mm_rsqrt_ps(x)); + const Packet4f r = generic_rsqrt_newton_step<Packet4f, /*Steps=*/1>::run(x, _mm_rsqrt_ps(x)); + const Packet4i bias = pset1<Packet4i>(0x7f800000); + const Packet4i abs_bits = pand(Packet4i(_mm_castps_si128(x)), pset1<Packet4i>(0x7fffffff)); + const Packet4f redo = _mm_castsi128_ps(pcmp_lt(bias, padd(abs_bits, bias))); + return predux_any(redo) ? pselect(redo, pdiv(pset1<Packet4f>(1.0f), psqrt(x)), r) : r; } #ifdef EIGEN_VECTORIZE_FMA
diff --git a/Eigen/src/Core/arch/SSE/PacketMath.h b/Eigen/src/Core/arch/SSE/PacketMath.h index af0898c..47d4ac4 100644 --- a/Eigen/src/Core/arch/SSE/PacketMath.h +++ b/Eigen/src/Core/arch/SSE/PacketMath.h
@@ -1726,8 +1726,8 @@ // 32 first selects it. const Packet2d max_exponent = pset1<Packet2d>(2099.0); const Packet2d last_max = pset1<Packet2d>(1022.0); - const Packet4i e = _mm_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); - const Packet4i b = _mm_cvtpd_epi32(pmin(pmax(exponent, pnegate(last_max)), last_max)); + const Packet4i e = _mm_cvttpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); + const Packet4i b = _mm_cvttpd_epi32(pmin(pmax(exponent, pnegate(last_max)), last_max)); const Packet4i t = pandnot(psub(e, b), pset1<Packet4i>(1)); // even const Packet4i biased = padd(Packet4i(_mm_unpacklo_epi32(t, psub(e, t))), Packet4i(_mm_set_epi32(1023, 2046, 1023, 2046)));
diff --git a/benchmarks/Core/bench_cwise_math.cpp b/benchmarks/Core/bench_cwise_math.cpp index 1a684de..2428af4 100644 --- a/benchmarks/Core/bench_cwise_math.cpp +++ b/benchmarks/Core/bench_cwise_math.cpp
@@ -7,6 +7,7 @@ #include <benchmark/benchmark.h> #include <cstdint> +#include <random> #include <Eigen/Core> #include <contrib/Eigen/SpecialFunctions> @@ -181,6 +182,32 @@ state.SetBytesProcessed(state.iterations() * n * Index(sizeof(Scalar)) * 2); } +// The SSE and AVX float rsqrt estimates flush subnormal inputs, so packets holding one take a 1 / sqrt fallback, while +// zeros stay on the fast path. mode: 0 = inputs in [0.01, 100), 1 = one zero per 8 coefficients, 2 = one subnormal +// per 64 coefficients, 3 = subnormal inputs only. Zeros and subnormals sit at pseudo-random positions, so a branch on +// them is unpredictable. +template <typename Scalar> +static void BM_RsqrtSpecialInputs(benchmark::State& state) { + const Index n = state.range(0); + const int mode = int(state.range(1)); + const Scalar subnormal = (std::numeric_limits<Scalar>::min)() / Scalar(3); + std::mt19937 gen(42); + Array<Scalar, Dynamic, 1> a(n), b(n); + for (Index i = 0; i < n; ++i) { + const std::uint32_t r = gen(); + a(i) = Scalar(0.01 + double(i % 1000) / 10.0); + if (mode == 1 && r % 8 == 0) a(i) = Scalar(0); + if ((mode == 2 && r % 64 == 0) || mode == 3) a(i) = subnormal; + } + for (auto _ : state) { + benchmark::DoNotOptimize(a.data()); + b = a.rsqrt(); + benchmark::DoNotOptimize(b.data()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(state.iterations() * n * sizeof(Scalar) * 2); +} + // mode: 0 = inputs in [0.5, 2), 1 = subnormal inputs, 2 = one subnormal per 64 coefficients. The subnormal is // min / 3, whose reciprocal is finite. The SSE and AVX rcp estimates flush it, so packets holding one take the // division fallback of preciprocal; the AVX-512 rcp14 estimate does not. @@ -333,6 +360,8 @@ ->ArgNames({"size", "mode"})->Name("LogSpecialInputs_float"); BENCHMARK(BM_Log1pSpecialInputs<float>)->ArgsProduct({{4096, 65536}, {0, 1, 2}}) ->ArgNames({"size", "mode"})->Name("Log1pSpecialInputs_float"); +BENCHMARK(BM_RsqrtSpecialInputs<float>)->ArgsProduct({{4096, 65536}, {0, 1, 2, 3}}) + ->ArgNames({"size", "mode"})->Name("RsqrtSpecialInputs_float"); BENCHMARK(BM_Inverse<float>)->ArgsProduct({{1024, 4096, 16384, 65536, 262144, 1048576}, {0, 1, 2}}) ->ArgNames({"size", "mode"})->Name("Inverse_float"); BENCHMARK_TEMPLATE(BM_Sign, float)->ArgsProduct({{1024, 16384, 262144}, {0, 1, 2}})->ArgNames({"size", "mode"})->Name("Sign_float");
diff --git a/test/packetmath.cpp b/test/packetmath.cpp index b7dc1e1..366fe6a 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp
@@ -1291,6 +1291,11 @@ data2[i + PacketSize] = Scalar(internal::random<double>(-4, 4)); } CHECK_CWISE2_IF(PacketTraits::HasExp, REF_LDEXP, internal::pldexp); + // pldexp takes (int)exponent: +-0.75, +-1.75, +-2.75, +-3.75 truncate to a different power than they round to. + for (int i = 0; i < PacketSize; ++i) { + data1[i + PacketSize] = Scalar((i % 2 == 0 ? 1 : -1) * ((i / 2) % 4 + 0.75)); + } + CHECK_CWISE2_IF(PacketTraits::HasExp, REF_LDEXP, internal::pldexp); if (PacketTraits::HasExp) { data1[0] = Scalar(-1); // underflow to zero @@ -1841,8 +1846,9 @@ std::enable_if_t<!NumTraits<Scalar>::IsComplex, void> packetmath_ieee_special_values() { typedef internal::packet_traits<Scalar> PacketTraits; run_ieee_cases<PacketTraits::HasSqrt, Scalar, Packet>(sqrt_fun()); - // TODO(rmlarsen): See if we can fix rsqrt for denorms without wreaking performance. - run_ieee_cases<PacketTraits::HasRsqrt, Scalar, Packet, true>(rsqrt_fun()); + // The x86 float rsqrt recomputes the subnormal inputs its estimate flushes; other estimates may still flush them. + run_ieee_cases<PacketTraits::HasRsqrt, Scalar, Packet, + !(EIGEN_ARCH_i386_OR_x86_64 && std::is_same<Scalar, float>::value)>(rsqrt_fun()); run_ieee_cases<PacketTraits::HasCbrt, Scalar, Packet>(cbrt_fun()); run_ieee_cases<PacketTraits::HasExp, Scalar, Packet>(exp_fun()); run_ieee_cases<PacketTraits::HasExp, Scalar, Packet>(exp2_fun()); @@ -1860,8 +1866,8 @@ run_ieee_cases<PacketTraits::HasATanh, Scalar, Packet>(atanh_fun()); } -// log, log2 and log1p handle special inputs only in packets that contain one, so place each special input alone -// among ordinary lanes, in every lane. +// log, log2, log1p and the x86 float rsqrt handle special inputs only in packets that contain one, so place each +// special input alone among ordinary lanes, in every lane. template <typename Scalar, typename Packet> std::enable_if_t<NumTraits<Scalar>::IsComplex || NumTraits<Scalar>::IsInteger, void> packetmath_special_value_lanes() {} @@ -1869,6 +1875,9 @@ std::enable_if_t<!NumTraits<Scalar>::IsComplex && !NumTraits<Scalar>::IsInteger, void> packetmath_special_value_lanes() { using PacketTraits = internal::packet_traits<Scalar>; + // As in packetmath_ieee_special_values: other rsqrt estimates may still flush subnormal inputs. + constexpr bool kCheckRsqrt = + PacketTraits::HasRsqrt && EIGEN_ARCH_i386_OR_x86_64 && std::is_same<Scalar, float>::value; const int PacketSize = internal::unpacket_traits<Packet>::size; const Scalar norm_min = (std::numeric_limits<Scalar>::min)(); const Scalar inf = NumTraits<Scalar>::infinity(); @@ -1890,6 +1899,7 @@ CHECK_CWISE1_IF(PacketTraits::HasLog, std::log, internal::plog); CHECK_CWISE1_IF(PacketTraits::HasLog, log2, internal::plog2); CHECK_CWISE1_IF(PacketTraits::HasLog1p, std::log1p, internal::plog1p); + CHECK_CWISE1_IF(kCheckRsqrt, numext::rsqrt, internal::prsqrt); } } }
diff --git a/test/packetmath_gather.cpp b/test/packetmath_gather.cpp index f88e6c4..4a68fa2 100644 --- a/test/packetmath_gather.cpp +++ b/test/packetmath_gather.cpp
@@ -4,9 +4,100 @@ #include "packetmath_test_shared.h" #include <memory> +// SVE's 32-bit gathers form the lane offsets the same way and still overflow; see pgather<float, PacketXf>. +#if defined(__unix__) && (defined(__LP64__) || defined(_LP64)) && !defined(EIGEN_VECTORIZE_SVE) +#include <sys/mman.h> +#include <unistd.h> +#include <cstdint> +#define EIGEN_TEST_LARGE_STRIDE_GATHER +#endif + namespace Eigen { namespace test { +#ifdef EIGEN_TEST_LARGE_STRIDE_GATHER +// Lane i at from[i * stride] for a stride near the limit of 32-bit lane offsets (lanes - 1) * stride. Only the pages +// holding the lanes set in `mapped` are mapped, so the reserved range costs address space but no memory. +template <typename Scalar> +struct large_stride_lanes { + Index stride; + std::size_t bytes = 0; + void* reserved = MAP_FAILED; + Scalar* from = nullptr; + + large_stride_lanes(int lanes, Index stride_, std::uint32_t mapped = ~std::uint32_t(0)) : stride(stride_) { + const std::uintptr_t page = std::uintptr_t(sysconf(_SC_PAGESIZE)); + const Index span = (lanes - 1) * numext::abs(stride); + bytes = std::size_t(span + 1) * sizeof(Scalar) + page; + reserved = mmap(nullptr, bytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (reserved == MAP_FAILED) { + std::cerr << "Skipping the gather test at stride " << stride << ": cannot reserve " << bytes << " bytes.\n"; + return; + } + from = static_cast<Scalar*>(reserved) + (stride < 0 ? span : 0); + for (Index i = 0; i < lanes; ++i) { + if (((mapped >> i) & 1) == 0) continue; + void* lane_page = reinterpret_cast<void*>(reinterpret_cast<std::uintptr_t>(from + i * stride) & ~(page - 1)); + VERIFY_IS_EQUAL(mprotect(lane_page, page, PROT_READ | PROT_WRITE), 0); + from[i * stride] = Scalar(i + 1); + } + } + ~large_stride_lanes() { + if (reserved != MAP_FAILED) VERIFY_IS_EQUAL(munmap(reserved, bytes), 0); + } +}; + +// The largest stride whose lane offsets fit in an int, the next one, and its negation. +template <typename Packet> +std::vector<Index> large_strides() { + const Index fits = Index(NumTraits<int>::highest()) / (internal::unpacket_traits<Packet>::size - 1); + return {fits, fits + 1, -(fits + 1)}; +} + +template <typename Scalar, typename Packet> +void pgather_pscatter_large_stride() { + constexpr int PacketSize = internal::unpacket_traits<Packet>::size; + if (PacketSize < 2) return; + for (Index s : large_strides<Packet>()) { + // Opaque, as in production: clang 14-20 merge _mm_set_pd(p[0x7fffffff], p[0]) into one load at a constant stride. + volatile Index runtime_stride = s; + const Index stride = runtime_stride; + large_stride_lanes<Scalar> lanes(PacketSize, stride); + if (lanes.from == nullptr) continue; + Scalar actual[PacketSize], expected[PacketSize]; + for (Index i = 0; i < PacketSize; ++i) expected[i] = Scalar(i + 1); + internal::pstoreu(actual, internal::pgather<Scalar, Packet>(lanes.from, stride)); + VERIFY(areEqualBits(expected, actual, PacketSize, false)); + for (Index i = 0; i < PacketSize; ++i) expected[i] = Scalar(2 * i + 3); + internal::pscatter<Scalar, Packet>(lanes.from, internal::ploadu<Packet>(expected), stride); + for (Index i = 0; i < PacketSize; ++i) actual[i] = lanes.from[i * stride]; + VERIFY(areEqualBits(expected, actual, PacketSize, false)); + } +} + +#ifdef EIGEN_VECTORIZE_AVX512 +// The masked forms, which the AVX-512 GEMM kernel uses on a strided result. The masked-off lanes stay unmapped, so +// touching one faults. +template <typename Scalar, typename Packet> +void masked_pgather_pscatter_large_stride() { + constexpr int PacketSize = internal::unpacket_traits<Packet>::size; + const auto mask = static_cast<typename internal::unpacket_traits<Packet>::mask_t>(0x5555); + for (Index s : large_strides<Packet>()) { + volatile Index runtime_stride = s; // see pgather_pscatter_large_stride + const Index stride = runtime_stride; + large_stride_lanes<Scalar> lanes(PacketSize, stride, mask); + if (lanes.from == nullptr) continue; + Scalar actual[PacketSize], expected[PacketSize]; + for (Index i = 0; i < PacketSize; ++i) expected[i] = (i % 2 == 0) ? Scalar(i + 1) : Scalar(-1); + internal::pstoreu(actual, internal::pgather(internal::pset1<Packet>(Scalar(-1)), lanes.from, stride, mask)); + VERIFY(areEqualBits(expected, actual, PacketSize, false)); + internal::pscatter(lanes.from, internal::pset1<Packet>(Scalar(0)), stride, mask); + for (Index i = 0; i < PacketSize; i += 2) VERIFY_IS_EQUAL(lanes.from[i * stride], Scalar(0)); + } +} +#endif +#endif + template <typename Scalar, typename Packet> struct runall<Scalar, Packet, false, false> { static void run() { @@ -36,6 +127,9 @@ } } } +#ifdef EIGEN_TEST_LARGE_STRIDE_GATHER + pgather_pscatter_large_stride<Scalar, Packet>(); +#endif } }; @@ -45,4 +139,12 @@ EIGEN_DECLARE_TEST(packetmath_gather) { CALL_SUBTEST_1(test::runner<float>::run()); CALL_SUBTEST_2(test::runner<double>::run()); +#ifdef EIGEN_TEST_LARGE_STRIDE_GATHER + CALL_SUBTEST_1((test::pgather_pscatter_large_stride<int32_t, internal::packet_traits<int32_t>::type>())); + CALL_SUBTEST_2((test::pgather_pscatter_large_stride<int64_t, internal::packet_traits<int64_t>::type>())); +#ifdef EIGEN_VECTORIZE_AVX512 + CALL_SUBTEST_1((test::masked_pgather_pscatter_large_stride<float, internal::Packet16f>())); + CALL_SUBTEST_2((test::masked_pgather_pscatter_large_stride<double, internal::Packet8d>())); +#endif +#endif }