Core bug fixes: SSE, AVX and AVX-512 backends

libeigen/eigen!3348

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/src/Core/arch/AVX/MathFunctions.h b/Eigen/src/Core/arch/AVX/MathFunctions.h
index 63a529d..f09e144 100644
--- a/Eigen/src/Core/arch/AVX/MathFunctions.h
+++ b/Eigen/src/Core/arch/AVX/MathFunctions.h
@@ -88,13 +88,13 @@
 #if EIGEN_FAST_MATH
 template <>
 EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8f prsqrt<Packet8f>(const Packet8f& a) {
-  // _mm256_rsqrt_ps returns -inf for negative denormals.
-  // _mm512_rsqrt**_ps returns -NaN for negative denormals.  We may want
-  // consistency here.
-  // const Packet8f rsqrt = pselect(pcmp_lt(a, pzero(a)),
-  //                                pset1<Packet8f>(-NumTraits<float>::quiet_NaN()),
-  //                                _mm256_rsqrt_ps(a));
-  return generic_rsqrt_newton_step<Packet8f, /*Steps=*/1>::run(a, _mm256_rsqrt_ps(a));
+  // vrsqrtps reads a subnormal a as zero, giving e = +-inf. a * e, the Newton step's first product, is therefore +inf
+  // for subnormal a, and finite or NaN for every other a; those lanes take 1 / sqrt(a) instead. Under DAZ the multiply
+  // reads a subnormal as zero too, and the estimate's +-inf is the DAZ result.
+  const Packet8f e = _mm256_rsqrt_ps(a);
+  const Packet8f r = generic_rsqrt_newton_step<Packet8f, /*Steps=*/1>::run(a, e);
+  const Packet8f redo = pcmp_eq(pmul(a, e), pset1<Packet8f>(NumTraits<float>::infinity()));
+  return predux_any(redo) ? pselect(redo, pdiv(pset1<Packet8f>(1.0f), psqrt(a)), r) : r;
 }
 
 template <>
diff --git a/Eigen/src/Core/arch/AVX/PacketMath.h b/Eigen/src/Core/arch/AVX/PacketMath.h
index ba7b402..a5399b7 100644
--- a/Eigen/src/Core/arch/AVX/PacketMath.h
+++ b/Eigen/src/Core/arch/AVX/PacketMath.h
@@ -1932,7 +1932,7 @@
 EIGEN_STRONG_INLINE Packet4d pldexp<Packet4d>(const Packet4d& a, const Packet4d& exponent) {
   // Clamp exponent to [-2099, 2099]
   const Packet4d max_exponent = pset1<Packet4d>(2099.0);
-  const Packet4i e = _mm256_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
+  const Packet4i e = _mm256_cvttpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
 
   // The single-rounding split of pldexp_generic.
   const Packet4i bias = pset1<Packet4i>(1023);
diff --git a/Eigen/src/Core/arch/AVX512/PacketMath.h b/Eigen/src/Core/arch/AVX512/PacketMath.h
index c0a244a..7d31c46 100644
--- a/Eigen/src/Core/arch/AVX512/PacketMath.h
+++ b/Eigen/src/Core/arch/AVX512/PacketMath.h
@@ -1221,12 +1221,40 @@
   EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_pd(to, mask, from);
 }
 
+// The gathers and scatters below address lane i through the 32-bit offset i * stride. Where that overflows an int,
+// the lanes are addressed one at a time.
+template <typename Packet>
+EIGEN_STRONG_INLINE bool avx512_lane_offsets_fit(Index stride) {
+  const Index max_stride = Index(NumTraits<int>::highest()) / Index(unpacket_traits<Packet>::size - 1);
+  return stride <= max_stride && stride >= -max_stride;
+}
+
+template <typename Scalar, typename Packet>
+EIGEN_STRONG_INLINE Packet avx512_pgather_lanes(const Packet& src, const Scalar* from, Index stride,
+                                                typename unpacket_traits<Packet>::mask_t umask) {
+  EIGEN_ALIGN64 Scalar values[unpacket_traits<Packet>::size];
+  pstore(values, src);
+  for (int i = 0; i < unpacket_traits<Packet>::size; ++i)
+    if ((umask >> i) & 1) values[i] = from[i * stride];
+  return pload<Packet>(values);
+}
+
+template <typename Scalar, typename Packet>
+EIGEN_STRONG_INLINE void avx512_pscatter_lanes(Scalar* to, const Packet& from, Index stride,
+                                               typename unpacket_traits<Packet>::mask_t umask) {
+  EIGEN_ALIGN64 Scalar values[unpacket_traits<Packet>::size];
+  pstore(values, from);
+  for (int i = 0; i < unpacket_traits<Packet>::size; ++i)
+    if ((umask >> i) & 1) to[i * stride] = values[i];
+}
+
 template <typename Scalar, typename Packet>
 EIGEN_DEVICE_FUNC inline Packet pgather(const Packet& src, const Scalar* from, Index stride,
                                         typename unpacket_traits<Packet>::mask_t umask);
 template <>
 EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const Packet16f& src, const float* from, Index stride,
                                                              uint16_t umask) {
+  if (!avx512_lane_offsets_fit<Packet16f>(stride)) return avx512_pgather_lanes(src, from, stride, umask);
   Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
   Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
   Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
@@ -1237,6 +1265,7 @@
 template <>
 EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const Packet8d& src, const double* from, Index stride,
                                                             uint8_t umask) {
+  if (!avx512_lane_offsets_fit<Packet8d>(stride)) return avx512_pgather_lanes(src, from, stride, umask);
   Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
   Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
   Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
@@ -1254,6 +1283,7 @@
     const Packet16i indices = _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 17, 19, 21, 23, 25, 27, 29, 31);
     return _mm512_permutex2var_ps(low, indices, high);
   }
+  if (!avx512_lane_offsets_fit<Packet16f>(stride)) return pgather_partial<float, Packet16f>(from, stride, 16);
   Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
   Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
   Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
@@ -1268,6 +1298,7 @@
     const __m512i indices = _mm512_setr_epi64(0, 2, 4, 6, 9, 11, 13, 15);
     return _mm512_permutex2var_pd(low, indices, high);
   }
+  if (!avx512_lane_offsets_fit<Packet8d>(stride)) return pgather_partial<double, Packet8d>(from, stride, 8);
   Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
   Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
   Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
@@ -1276,6 +1307,7 @@
 }
 template <>
 EIGEN_DEVICE_FUNC inline Packet8l pgather<int64_t, Packet8l>(const int64_t* from, Index stride) {
+  if (!avx512_lane_offsets_fit<Packet8l>(stride)) return pgather_partial<int64_t, Packet8l>(from, stride, 8);
   Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
   Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
   Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
@@ -1284,6 +1316,7 @@
 }
 template <>
 EIGEN_DEVICE_FUNC inline Packet16i pgather<int, Packet16i>(const int* from, Index stride) {
+  if (!avx512_lane_offsets_fit<Packet16i>(stride)) return pgather_partial<int, Packet16i>(from, stride, 16);
   Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
   Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
   Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
@@ -1296,6 +1329,7 @@
 template <>
 EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride,
                                                          uint16_t umask) {
+  if (!avx512_lane_offsets_fit<Packet16f>(stride)) return avx512_pscatter_lanes(to, from, stride, umask);
   Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
   Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
   Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
@@ -1305,6 +1339,7 @@
 template <>
 EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride,
                                                          uint8_t umask) {
+  if (!avx512_lane_offsets_fit<Packet8d>(stride)) return avx512_pscatter_lanes(to, from, stride, umask);
   Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
   Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
   Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
@@ -1313,6 +1348,7 @@
 }
 template <>
 EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride) {
+  if (!avx512_lane_offsets_fit<Packet16f>(stride)) return pscatter_partial<float, Packet16f>(to, from, stride, 16);
   Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
   Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
   Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
@@ -1320,6 +1356,7 @@
 }
 template <>
 EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride) {
+  if (!avx512_lane_offsets_fit<Packet8d>(stride)) return pscatter_partial<double, Packet8d>(to, from, stride, 8);
   Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
   Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
   Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
@@ -1327,6 +1364,7 @@
 }
 template <>
 EIGEN_DEVICE_FUNC inline void pscatter<int64_t, Packet8l>(int64_t* to, const Packet8l& from, Index stride) {
+  if (!avx512_lane_offsets_fit<Packet8l>(stride)) return pscatter_partial<int64_t, Packet8l>(to, from, stride, 8);
   Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
   Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
   Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
@@ -1334,6 +1372,7 @@
 }
 template <>
 EIGEN_DEVICE_FUNC inline void pscatter<int, Packet16i>(int* to, const Packet16i& from, Index stride) {
+  if (!avx512_lane_offsets_fit<Packet16i>(stride)) return pscatter_partial<int, Packet16i>(to, from, stride, 16);
   Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
   Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
   Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
diff --git a/Eigen/src/Core/arch/SSE/MathFunctions.h b/Eigen/src/Core/arch/SSE/MathFunctions.h
index a8fbd09..24a70f1 100644
--- a/Eigen/src/Core/arch/SSE/MathFunctions.h
+++ b/Eigen/src/Core/arch/SSE/MathFunctions.h
@@ -44,10 +44,18 @@
 }
 
 #if EIGEN_FAST_MATH
-// Even on Skylake, using Newton iteration is a win for reciprocal square root.
+// Even on Skylake, using Newton iteration is a win for reciprocal square root. rsqrtps reads a subnormal x as zero,
+// giving +-inf, so the subnormal lanes, 1 <= bits(|x|) <= 0x007fffff, take 1 / sqrt(x) instead. Adding 0x7f800000
+// moves exactly that range above 0x7f800000 and wraps every larger value negative. With SSE this test of x alone is
+// faster than the test of x * estimate in prsqrt<Packet8f>, which wins with AVX. Under DAZ the fallback reads a
+// subnormal as zero too, and returns the estimate's +-inf.
 template <>
 EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f prsqrt<Packet4f>(const Packet4f& x) {
-  return generic_rsqrt_newton_step<Packet4f, /*Steps=*/1>::run(x, _mm_rsqrt_ps(x));
+  const Packet4f r = generic_rsqrt_newton_step<Packet4f, /*Steps=*/1>::run(x, _mm_rsqrt_ps(x));
+  const Packet4i bias = pset1<Packet4i>(0x7f800000);
+  const Packet4i abs_bits = pand(Packet4i(_mm_castps_si128(x)), pset1<Packet4i>(0x7fffffff));
+  const Packet4f redo = _mm_castsi128_ps(pcmp_lt(bias, padd(abs_bits, bias)));
+  return predux_any(redo) ? pselect(redo, pdiv(pset1<Packet4f>(1.0f), psqrt(x)), r) : r;
 }
 
 #ifdef EIGEN_VECTORIZE_FMA
diff --git a/Eigen/src/Core/arch/SSE/PacketMath.h b/Eigen/src/Core/arch/SSE/PacketMath.h
index af0898c..47d4ac4 100644
--- a/Eigen/src/Core/arch/SSE/PacketMath.h
+++ b/Eigen/src/Core/arch/SSE/PacketMath.h
@@ -1726,8 +1726,8 @@
   // 32 first selects it.
   const Packet2d max_exponent = pset1<Packet2d>(2099.0);
   const Packet2d last_max = pset1<Packet2d>(1022.0);
-  const Packet4i e = _mm_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
-  const Packet4i b = _mm_cvtpd_epi32(pmin(pmax(exponent, pnegate(last_max)), last_max));
+  const Packet4i e = _mm_cvttpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
+  const Packet4i b = _mm_cvttpd_epi32(pmin(pmax(exponent, pnegate(last_max)), last_max));
   const Packet4i t = pandnot(psub(e, b), pset1<Packet4i>(1));  // even
   const Packet4i biased =
       padd(Packet4i(_mm_unpacklo_epi32(t, psub(e, t))), Packet4i(_mm_set_epi32(1023, 2046, 1023, 2046)));
diff --git a/benchmarks/Core/bench_cwise_math.cpp b/benchmarks/Core/bench_cwise_math.cpp
index 1a684de..2428af4 100644
--- a/benchmarks/Core/bench_cwise_math.cpp
+++ b/benchmarks/Core/bench_cwise_math.cpp
@@ -7,6 +7,7 @@
 
 #include <benchmark/benchmark.h>
 #include <cstdint>
+#include <random>
 #include <Eigen/Core>
 #include <contrib/Eigen/SpecialFunctions>
 
@@ -181,6 +182,32 @@
   state.SetBytesProcessed(state.iterations() * n * Index(sizeof(Scalar)) * 2);
 }
 
+// The SSE and AVX float rsqrt estimates flush subnormal inputs, so packets holding one take a 1 / sqrt fallback, while
+// zeros stay on the fast path. mode: 0 = inputs in [0.01, 100), 1 = one zero per 8 coefficients, 2 = one subnormal
+// per 64 coefficients, 3 = subnormal inputs only. Zeros and subnormals sit at pseudo-random positions, so a branch on
+// them is unpredictable.
+template <typename Scalar>
+static void BM_RsqrtSpecialInputs(benchmark::State& state) {
+  const Index n = state.range(0);
+  const int mode = int(state.range(1));
+  const Scalar subnormal = (std::numeric_limits<Scalar>::min)() / Scalar(3);
+  std::mt19937 gen(42);
+  Array<Scalar, Dynamic, 1> a(n), b(n);
+  for (Index i = 0; i < n; ++i) {
+    const std::uint32_t r = gen();
+    a(i) = Scalar(0.01 + double(i % 1000) / 10.0);
+    if (mode == 1 && r % 8 == 0) a(i) = Scalar(0);
+    if ((mode == 2 && r % 64 == 0) || mode == 3) a(i) = subnormal;
+  }
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(a.data());
+    b = a.rsqrt();
+    benchmark::DoNotOptimize(b.data());
+    benchmark::ClobberMemory();
+  }
+  state.SetBytesProcessed(state.iterations() * n * sizeof(Scalar) * 2);
+}
+
 // mode: 0 = inputs in [0.5, 2), 1 = subnormal inputs, 2 = one subnormal per 64 coefficients. The subnormal is
 // min / 3, whose reciprocal is finite. The SSE and AVX rcp estimates flush it, so packets holding one take the
 // division fallback of preciprocal; the AVX-512 rcp14 estimate does not.
@@ -333,6 +360,8 @@
     ->ArgNames({"size", "mode"})->Name("LogSpecialInputs_float");
 BENCHMARK(BM_Log1pSpecialInputs<float>)->ArgsProduct({{4096, 65536}, {0, 1, 2}})
     ->ArgNames({"size", "mode"})->Name("Log1pSpecialInputs_float");
+BENCHMARK(BM_RsqrtSpecialInputs<float>)->ArgsProduct({{4096, 65536}, {0, 1, 2, 3}})
+    ->ArgNames({"size", "mode"})->Name("RsqrtSpecialInputs_float");
 BENCHMARK(BM_Inverse<float>)->ArgsProduct({{1024, 4096, 16384, 65536, 262144, 1048576}, {0, 1, 2}})
     ->ArgNames({"size", "mode"})->Name("Inverse_float");
 BENCHMARK_TEMPLATE(BM_Sign, float)->ArgsProduct({{1024, 16384, 262144}, {0, 1, 2}})->ArgNames({"size", "mode"})->Name("Sign_float");
diff --git a/test/packetmath.cpp b/test/packetmath.cpp
index b7dc1e1..366fe6a 100644
--- a/test/packetmath.cpp
+++ b/test/packetmath.cpp
@@ -1291,6 +1291,11 @@
     data2[i + PacketSize] = Scalar(internal::random<double>(-4, 4));
   }
   CHECK_CWISE2_IF(PacketTraits::HasExp, REF_LDEXP, internal::pldexp);
+  // pldexp takes (int)exponent: +-0.75, +-1.75, +-2.75, +-3.75 truncate to a different power than they round to.
+  for (int i = 0; i < PacketSize; ++i) {
+    data1[i + PacketSize] = Scalar((i % 2 == 0 ? 1 : -1) * ((i / 2) % 4 + 0.75));
+  }
+  CHECK_CWISE2_IF(PacketTraits::HasExp, REF_LDEXP, internal::pldexp);
   if (PacketTraits::HasExp) {
     data1[0] = Scalar(-1);
     // underflow to zero
@@ -1841,8 +1846,9 @@
 std::enable_if_t<!NumTraits<Scalar>::IsComplex, void> packetmath_ieee_special_values() {
   typedef internal::packet_traits<Scalar> PacketTraits;
   run_ieee_cases<PacketTraits::HasSqrt, Scalar, Packet>(sqrt_fun());
-  // TODO(rmlarsen): See if we can fix rsqrt for denorms without wreaking performance.
-  run_ieee_cases<PacketTraits::HasRsqrt, Scalar, Packet, true>(rsqrt_fun());
+  // The x86 float rsqrt recomputes the subnormal inputs its estimate flushes; other estimates may still flush them.
+  run_ieee_cases<PacketTraits::HasRsqrt, Scalar, Packet,
+                 !(EIGEN_ARCH_i386_OR_x86_64 && std::is_same<Scalar, float>::value)>(rsqrt_fun());
   run_ieee_cases<PacketTraits::HasCbrt, Scalar, Packet>(cbrt_fun());
   run_ieee_cases<PacketTraits::HasExp, Scalar, Packet>(exp_fun());
   run_ieee_cases<PacketTraits::HasExp, Scalar, Packet>(exp2_fun());
@@ -1860,8 +1866,8 @@
   run_ieee_cases<PacketTraits::HasATanh, Scalar, Packet>(atanh_fun());
 }
 
-// log, log2 and log1p handle special inputs only in packets that contain one, so place each special input alone
-// among ordinary lanes, in every lane.
+// log, log2, log1p and the x86 float rsqrt handle special inputs only in packets that contain one, so place each
+// special input alone among ordinary lanes, in every lane.
 template <typename Scalar, typename Packet>
 std::enable_if_t<NumTraits<Scalar>::IsComplex || NumTraits<Scalar>::IsInteger, void> packetmath_special_value_lanes() {}
 
@@ -1869,6 +1875,9 @@
 std::enable_if_t<!NumTraits<Scalar>::IsComplex && !NumTraits<Scalar>::IsInteger, void>
 packetmath_special_value_lanes() {
   using PacketTraits = internal::packet_traits<Scalar>;
+  // As in packetmath_ieee_special_values: other rsqrt estimates may still flush subnormal inputs.
+  constexpr bool kCheckRsqrt =
+      PacketTraits::HasRsqrt && EIGEN_ARCH_i386_OR_x86_64 && std::is_same<Scalar, float>::value;
   const int PacketSize = internal::unpacket_traits<Packet>::size;
   const Scalar norm_min = (std::numeric_limits<Scalar>::min)();
   const Scalar inf = NumTraits<Scalar>::infinity();
@@ -1890,6 +1899,7 @@
       CHECK_CWISE1_IF(PacketTraits::HasLog, std::log, internal::plog);
       CHECK_CWISE1_IF(PacketTraits::HasLog, log2, internal::plog2);
       CHECK_CWISE1_IF(PacketTraits::HasLog1p, std::log1p, internal::plog1p);
+      CHECK_CWISE1_IF(kCheckRsqrt, numext::rsqrt, internal::prsqrt);
     }
   }
 }
diff --git a/test/packetmath_gather.cpp b/test/packetmath_gather.cpp
index f88e6c4..4a68fa2 100644
--- a/test/packetmath_gather.cpp
+++ b/test/packetmath_gather.cpp
@@ -4,9 +4,100 @@
 #include "packetmath_test_shared.h"
 #include <memory>
 
+// SVE's 32-bit gathers form the lane offsets the same way and still overflow; see pgather<float, PacketXf>.
+#if defined(__unix__) && (defined(__LP64__) || defined(_LP64)) && !defined(EIGEN_VECTORIZE_SVE)
+#include <sys/mman.h>
+#include <unistd.h>
+#include <cstdint>
+#define EIGEN_TEST_LARGE_STRIDE_GATHER
+#endif
+
 namespace Eigen {
 namespace test {
 
+#ifdef EIGEN_TEST_LARGE_STRIDE_GATHER
+// Lane i at from[i * stride] for a stride near the limit of 32-bit lane offsets (lanes - 1) * stride. Only the pages
+// holding the lanes set in `mapped` are mapped, so the reserved range costs address space but no memory.
+template <typename Scalar>
+struct large_stride_lanes {
+  Index stride;
+  std::size_t bytes = 0;
+  void* reserved = MAP_FAILED;
+  Scalar* from = nullptr;
+
+  large_stride_lanes(int lanes, Index stride_, std::uint32_t mapped = ~std::uint32_t(0)) : stride(stride_) {
+    const std::uintptr_t page = std::uintptr_t(sysconf(_SC_PAGESIZE));
+    const Index span = (lanes - 1) * numext::abs(stride);
+    bytes = std::size_t(span + 1) * sizeof(Scalar) + page;
+    reserved = mmap(nullptr, bytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+    if (reserved == MAP_FAILED) {
+      std::cerr << "Skipping the gather test at stride " << stride << ": cannot reserve " << bytes << " bytes.\n";
+      return;
+    }
+    from = static_cast<Scalar*>(reserved) + (stride < 0 ? span : 0);
+    for (Index i = 0; i < lanes; ++i) {
+      if (((mapped >> i) & 1) == 0) continue;
+      void* lane_page = reinterpret_cast<void*>(reinterpret_cast<std::uintptr_t>(from + i * stride) & ~(page - 1));
+      VERIFY_IS_EQUAL(mprotect(lane_page, page, PROT_READ | PROT_WRITE), 0);
+      from[i * stride] = Scalar(i + 1);
+    }
+  }
+  ~large_stride_lanes() {
+    if (reserved != MAP_FAILED) VERIFY_IS_EQUAL(munmap(reserved, bytes), 0);
+  }
+};
+
+// The largest stride whose lane offsets fit in an int, the next one, and its negation.
+template <typename Packet>
+std::vector<Index> large_strides() {
+  const Index fits = Index(NumTraits<int>::highest()) / (internal::unpacket_traits<Packet>::size - 1);
+  return {fits, fits + 1, -(fits + 1)};
+}
+
+template <typename Scalar, typename Packet>
+void pgather_pscatter_large_stride() {
+  constexpr int PacketSize = internal::unpacket_traits<Packet>::size;
+  if (PacketSize < 2) return;
+  for (Index s : large_strides<Packet>()) {
+    // Opaque, as in production: clang 14-20 merge _mm_set_pd(p[0x7fffffff], p[0]) into one load at a constant stride.
+    volatile Index runtime_stride = s;
+    const Index stride = runtime_stride;
+    large_stride_lanes<Scalar> lanes(PacketSize, stride);
+    if (lanes.from == nullptr) continue;
+    Scalar actual[PacketSize], expected[PacketSize];
+    for (Index i = 0; i < PacketSize; ++i) expected[i] = Scalar(i + 1);
+    internal::pstoreu(actual, internal::pgather<Scalar, Packet>(lanes.from, stride));
+    VERIFY(areEqualBits(expected, actual, PacketSize, false));
+    for (Index i = 0; i < PacketSize; ++i) expected[i] = Scalar(2 * i + 3);
+    internal::pscatter<Scalar, Packet>(lanes.from, internal::ploadu<Packet>(expected), stride);
+    for (Index i = 0; i < PacketSize; ++i) actual[i] = lanes.from[i * stride];
+    VERIFY(areEqualBits(expected, actual, PacketSize, false));
+  }
+}
+
+#ifdef EIGEN_VECTORIZE_AVX512
+// The masked forms, which the AVX-512 GEMM kernel uses on a strided result. The masked-off lanes stay unmapped, so
+// touching one faults.
+template <typename Scalar, typename Packet>
+void masked_pgather_pscatter_large_stride() {
+  constexpr int PacketSize = internal::unpacket_traits<Packet>::size;
+  const auto mask = static_cast<typename internal::unpacket_traits<Packet>::mask_t>(0x5555);
+  for (Index s : large_strides<Packet>()) {
+    volatile Index runtime_stride = s;  // see pgather_pscatter_large_stride
+    const Index stride = runtime_stride;
+    large_stride_lanes<Scalar> lanes(PacketSize, stride, mask);
+    if (lanes.from == nullptr) continue;
+    Scalar actual[PacketSize], expected[PacketSize];
+    for (Index i = 0; i < PacketSize; ++i) expected[i] = (i % 2 == 0) ? Scalar(i + 1) : Scalar(-1);
+    internal::pstoreu(actual, internal::pgather(internal::pset1<Packet>(Scalar(-1)), lanes.from, stride, mask));
+    VERIFY(areEqualBits(expected, actual, PacketSize, false));
+    internal::pscatter(lanes.from, internal::pset1<Packet>(Scalar(0)), stride, mask);
+    for (Index i = 0; i < PacketSize; i += 2) VERIFY_IS_EQUAL(lanes.from[i * stride], Scalar(0));
+  }
+}
+#endif
+#endif
+
 template <typename Scalar, typename Packet>
 struct runall<Scalar, Packet, false, false> {
   static void run() {
@@ -36,6 +127,9 @@
         }
       }
     }
+#ifdef EIGEN_TEST_LARGE_STRIDE_GATHER
+    pgather_pscatter_large_stride<Scalar, Packet>();
+#endif
   }
 };
 
@@ -45,4 +139,12 @@
 EIGEN_DECLARE_TEST(packetmath_gather) {
   CALL_SUBTEST_1(test::runner<float>::run());
   CALL_SUBTEST_2(test::runner<double>::run());
+#ifdef EIGEN_TEST_LARGE_STRIDE_GATHER
+  CALL_SUBTEST_1((test::pgather_pscatter_large_stride<int32_t, internal::packet_traits<int32_t>::type>()));
+  CALL_SUBTEST_2((test::pgather_pscatter_large_stride<int64_t, internal::packet_traits<int64_t>::type>()));
+#ifdef EIGEN_VECTORIZE_AVX512
+  CALL_SUBTEST_1((test::masked_pgather_pscatter_large_stride<float, internal::Packet16f>()));
+  CALL_SUBTEST_2((test::masked_pgather_pscatter_large_stride<double, internal::Packet8d>()));
+#endif
+#endif
 }