blob: b81b89fabc5c798f105ce3ad8b0084d0af2f7010 [file] [edit]
// SPDX-FileCopyrightText: The Eigen Authors
// SPDX-License-Identifier: MPL-2.0
#include "main.h"
#include "twoprod_helpers.h"
// Checks twoprod, or fast_twoprod if `fast` is set, on {x_hi, x_lo, y_hi, y_lo, ref_hi, ref_lo, ref_tail}.
template <typename Scalar, typename Packet>
void check_doubleword_product(const Scalar (&data)[7], bool fast = false) {
constexpr Index size = internal::unpacket_traits<Packet>::size;
EIGEN_ALIGN_MAX Scalar hi[size], lo[size];
const Scalar epsilon = NumTraits<Scalar>::epsilon();
const Scalar bound = Scalar(1.25) * epsilon * epsilon * numext::abs(data[4]); // 5*u^2, u = eps/2.
for (int exponent : {-20, 0, 20}) {
// Keep the operands runtime so constant folding cannot replace the arithmetic under test.
volatile Scalar x_scale = numext::ldexp(Scalar(1), exponent), y_scale = numext::ldexp(Scalar(1), -exponent);
for (int sign : {-1, 1}) {
const Scalar x_hi = Scalar(sign) * data[0] * x_scale;
const Scalar x_lo = Scalar(sign) * data[1] * x_scale;
const Scalar y_hi = data[2] * y_scale;
const Scalar y_lo = data[3] * y_scale;
VERIFY_IS_EQUAL(x_hi + x_lo, x_hi);
VERIFY_IS_EQUAL(y_hi + y_lo, y_hi);
Packet p_hi, p_lo;
const Packet px_hi = internal::pset1<Packet>(x_hi), px_lo = internal::pset1<Packet>(x_lo);
const Packet py_hi = internal::pset1<Packet>(y_hi), py_lo = internal::pset1<Packet>(y_lo);
if (fast) {
internal::fast_twoprod(px_hi, px_lo, py_hi, py_lo, p_hi, p_lo);
} else {
internal::twoprod(px_hi, px_lo, py_hi, py_lo, p_hi, p_lo);
}
internal::pstore(hi, p_hi);
internal::pstore(lo, p_lo);
for (Index i = 0; i < size; ++i) {
VERIFY_IS_EQUAL(hi[i], Scalar(sign) * data[4]);
const Scalar error = numext::abs((lo[i] - Scalar(sign) * data[5]) - Scalar(sign) * data[6]);
VERIFY(error < bound);
}
}
}
}
// Where scalar and packet pmadd are both fused, fast_twoprod takes the same path for both, so int_pow's scalar and
// vectorized paths round alike.
template <typename Scalar, typename Packet>
void check_fast_twoprod_scalar_matches_packet(const Scalar (&data)[7]) {
#ifdef EIGEN_VECTORIZE_FMA
if (!internal::is_fused_madd<Packet>::value) return;
constexpr Index size = internal::unpacket_traits<Packet>::size;
EIGEN_ALIGN_MAX Scalar hi[size], lo[size];
// Keep the operands runtime so constant folding cannot replace the arithmetic under test.
volatile Scalar operands[4] = {data[0], data[1], data[2], data[3]};
const Scalar x_hi = operands[0], x_lo = operands[1], y_hi = operands[2], y_lo = operands[3];
Scalar s_hi, s_lo;
internal::fast_twoprod<Scalar>(x_hi, x_lo, y_hi, y_lo, s_hi, s_lo);
const Packet px_hi = internal::pset1<Packet>(x_hi), px_lo = internal::pset1<Packet>(x_lo);
const Packet py_hi = internal::pset1<Packet>(y_hi), py_lo = internal::pset1<Packet>(y_lo);
Packet p_hi, p_lo;
internal::fast_twoprod(px_hi, px_lo, py_hi, py_lo, p_hi, p_lo);
internal::pstore(hi, p_hi);
internal::pstore(lo, p_lo);
for (Index i = 0; i < size; ++i) {
VERIFY_IS_EQUAL(hi[i], s_hi);
VERIFY_IS_EQUAL(lo[i], s_lo);
}
#else
EIGEN_UNUSED_VARIABLE(data);
#endif
}
EIGEN_DECLARE_TEST(twoprod) {
using FloatPacket = internal::packet_traits<float>::type;
using DoublePacket = internal::packet_traits<double>::type;
check_twoprod_packet<float>();
check_twoprod_packet<double>();
check_twoprod_packet<FloatPacket>();
check_twoprod_packet<DoublePacket>();
// {x_hi, x_lo, y_hi, y_lo, ref_hi, ref_lo, ref_tail}. The reference product is
// decomposed with 600-bit MPFR, rounding each word to the tested scalar type.
// These normalized inputs exceed the former 2*u^2 bound in both precisions.
const float float_data[] = {1.1675887107849121f, -3.9186936362511915e-08f, 1.7997856140136719f,
5.2592689314678864e-08f, 2.1014094352722168f, -7.963821957446271e-08f,
1.2973732602550997e-15f};
const double double_data[] = {1.4946183601517316, 9.493375847471864e-17, 1.4603476327002127,
-8.3856771222625945e-17, 2.182662384037855, 1.9088477361435543e-16,
-3.7615759087187478e-33};
check_doubleword_product<float, float>(float_data);
check_doubleword_product<double, double>(double_data);
check_doubleword_product<float, FloatPacket>(float_data);
check_doubleword_product<double, DoublePacket>(double_data);
// Worst cases found by search for fast_twoprod: a chain of fused pmadds, with error up to 5*u^2, and (where pmadd
// is not fused) Algorithm 10 of Joldes, Muller, & Popescu, which stays below 5*u^2 on these operands. An unfused
// pmadd chain reaches 6.9*u^2 on the "unfused" operands. References are decomposed as above.
const float fast_float_data[][7] = {
// fused general.
{-0.250000864f, -1.48879558e-08f, -0.508609235f, -2.97569827e-08f, 0.127152771f, -7.35021777e-09f,
-4.21284906e-17f},
// fused square.
{-1.00657368f, -5.95824865e-08f, -1.00657368f, -5.95824865e-08f, 1.01319075f, -5.90029394e-08f, -2.8103879e-18f},
// unfused general.
{8.21313381f, 4.76301125e-07f, 1.00737178f, 5.95087641e-08f, 8.27368069f, -4.64115629e-07f, -1.17886943e-15f},
// unfused square.
{-2.0135932f, 1.19185245e-07f, -2.0135932f, 1.19185245e-07f, 4.05455685f, 2.35876868e-07f, -1.45683893e-17f},
// Algorithm 10 general.
{-1.00040495f, -5.96022929e-08f, 0.0625946224f, 3.72474185e-09f, -0.0626199767f, -1.07125442e-09f,
5.91333584e-19f},
// Algorithm 10 square.
{8.00068283f, 4.76670664e-07f, 8.00068283f, 4.76670664e-07f, 64.0109329f, 4.64245204e-07f, -1.60054737e-16f},
};
const double fast_double_data[][7] = {
// fused general.
{0.25002041616236204, 2.7754358076265322e-17, 1.0410038358637521, 1.1094124097677933e-16, 0.26027221226927061,
-2.7065762180931695e-17, -5.4781762140699477e-36},
// fused square.
{-4.0061437100749089, -4.4397637134168864e-16, -4.0061437100749089, -4.4397637134168864e-16, 16.049187425772761,
-1.7728980890933963e-15, -1.3739735310886058e-34},
// unfused general.
{-8.8249679712992339, -8.8761466869110876e-16, 0.062562782408520978, 6.915930645409851e-18, -0.55211455095056094,
4.9974548464202032e-17, 8.7122901865361381e-35},
// unfused square.
{2.0092063436530401, 2.2171312984987409e-16, 2.0092063436530401, 2.2171312984987409e-16, 4.0369101313756195,
-4.417540122138019e-16, -2.2355717681196923e-34},
// Algorithm 10 general.
{-16.239637864125285, 1.7758716492754939e-15, 16.01732882694828, -1.7740284508235665e-15, -260.11561970025468,
2.0571644619504938e-14, 8.9404545263427861e-33},
// Algorithm 10 square.
{16.000001245592074, 1.7762960189520815e-15, 16.000001245592074, 1.7762960189520815e-15, 256.00003985894796,
1.6725363562404628e-14, -2.1609212338112685e-34},
};
for (const auto& data : fast_float_data) {
check_doubleword_product<float, float>(data, true);
check_doubleword_product<float, FloatPacket>(data, true);
check_fast_twoprod_scalar_matches_packet<float, FloatPacket>(data);
}
for (const auto& data : fast_double_data) {
check_doubleword_product<double, double>(data, true);
check_doubleword_product<double, DoublePacket>(data, true);
check_fast_twoprod_scalar_matches_packet<double, DoublePacket>(data);
}
}