blob: 73f25c2e681b5816dcabaa658c216fc9b824e87d [file] [edit]
// SPDX-FileCopyrightText: The Eigen Authors
// SPDX-License-Identifier: MPL-2.0
#include "main.h"
#include "fp_control.h"
#define EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW
#include "AnnoyingScalar.h"
template <typename T>
void check_power_of_two_scaling_factor() {
static_assert(internal::supports_power_of_two_scaling<T>::value, "power-of-two scaling must be enabled");
using Factors = internal::safe_scaling_factors<T>;
const auto check_factor = [](const T& value) {
const auto factors = internal::safe_scaling<T>::compute_floor_factors(value);
VERIFY(factors.scale >= (std::numeric_limits<T>::min)());
VERIFY(factors.invScale >= (std::numeric_limits<T>::min)());
VERIFY_IS_EQUAL(factors.scale * factors.invScale, T(1));
return factors;
};
const auto check_round_trip = [&](const T& value) {
const auto factors = check_factor(value);
Matrix<T, 1, 1> input;
input(0) = value;
Matrix<T, 1, 1> scaled;
const auto scaleToFactors = internal::safe_scaling<T>::scale_to(scaled, input, value);
VERIFY_IS_EQUAL(scaleToFactors.scale, factors.scale);
VERIFY_IS_EQUAL(scaleToFactors.invScale, factors.invScale);
Matrix<T, 1, 1> scaledInPlace;
scaledInPlace(0) = value;
internal::safe_scaling<T>::scale_in_place(scaledInPlace, value, factors);
VERIFY_IS_EQUAL(scaledInPlace, scaled);
scaledInPlace(0) = value;
const auto inPlaceFactors = internal::safe_scaling<T>::scale_in_place(scaledInPlace, value);
VERIFY_IS_EQUAL(inPlaceFactors.scale, factors.scale);
VERIFY_IS_EQUAL(inPlaceFactors.invScale, factors.invScale);
VERIFY_IS_EQUAL(scaledInPlace, scaled);
T restored;
internal::safe_scaling<T>::unscale_to(restored, scaled(0), factors);
VERIFY_IS_EQUAL(restored, value);
Matrix<T, 1, 1> restoredMatrix;
internal::safe_scaling<T>::unscale_to(restoredMatrix, scaled, value, factors);
VERIFY_IS_EQUAL(restoredMatrix, input);
internal::safe_scaling<T>::unscale_in_place(scaled, factors);
VERIFY_IS_EQUAL(scaled(0), value);
};
const auto denormFactors = check_factor(std::numeric_limits<T>::denorm_min());
VERIFY_IS_EQUAL(denormFactors.scale, (std::numeric_limits<T>::min)());
Factors normalReciprocalFactors;
VERIFY(internal::safe_scaling<T>::try_compute_ceiling_factors_with_normal_reciprocal(
std::numeric_limits<T>::denorm_min(), (std::numeric_limits<T>::min)(), normalReciprocalFactors));
VERIFY_IS_EQUAL(normalReciprocalFactors.scale, denormFactors.scale);
VERIFY_IS_EQUAL(normalReciprocalFactors.invScale, denormFactors.invScale);
// bfloat16 arithmetic widens to float, whose subnormal inputs may be flushed before scaling can recover them.
if (!std::is_same<T, bfloat16>::value || !ScopedFlushToZero::hardwareFlushesSubnormalInputs()) {
using InputScalar = std::conditional_t<std::is_floating_point<T>::value, T, float>;
volatile InputScalar denormInput = static_cast<InputScalar>(std::numeric_limits<T>::denorm_min());
const T denormMin = static_cast<T>(denormInput);
Matrix<T, 1, 1> denormInputMatrix;
denormInputMatrix(0) = denormMin;
Matrix<T, 1, 1> scaledDenorm;
const auto factors = internal::safe_scaling<T>::scale_to(scaledDenorm, denormInputMatrix, denormMin);
Matrix<T, 1, 1> scaledDenormInPlace;
scaledDenormInPlace(0) = denormMin;
internal::safe_scaling<T>::scale_in_place(scaledDenormInPlace, denormMin, factors);
VERIFY_IS_EQUAL(scaledDenormInPlace, scaledDenorm);
VERIFY(scaledDenorm(0) > T(0));
}
check_round_trip((std::numeric_limits<T>::min)());
check_round_trip(T(0.75));
check_round_trip(T(1.5));
check_round_trip(T(3));
check_round_trip((std::numeric_limits<T>::max)());
const auto floorFactors = internal::safe_scaling<T>::compute_floor_factors(T(1.5));
const auto ceilingFactors = internal::safe_scaling<T>::compute_ceiling_factors_with_normal_reciprocal(T(1.5));
VERIFY_IS_EQUAL(floorFactors.scale, T(1));
VERIFY_IS_EQUAL(ceilingFactors.scale, T(2));
const T highest = (std::numeric_limits<T>::max)();
const auto highestCeilingFactors = internal::safe_scaling<T>::compute_ceiling_factors_with_normal_reciprocal(highest);
VERIFY_IS_EQUAL(highestCeilingFactors.scale, check_factor(highest).scale);
VERIFY_IS_EQUAL(highestCeilingFactors.scale * highestCeilingFactors.invScale, T(1));
const T scaledHighest = highest * highestCeilingFactors.invScale;
VERIFY(scaledHighest >= T(1));
if (std::numeric_limits<T>::max_exponent + std::numeric_limits<T>::min_exponent <= 3) {
VERIFY(scaledHighest < T(4));
}
}
template <typename T>
void check_safe_scaling_special_values() {
using Scaling = internal::safe_scaling<T>;
using Factors = internal::safe_scaling_factors<T>;
const Factors identity;
VERIFY_IS_EQUAL(identity.scale, T(1));
VERIFY_IS_EQUAL(identity.invScale, T(1));
Matrix<T, 1, 1> zero = Matrix<T, 1, 1>::Zero();
Matrix<T, 1, 1> scaledZero;
const auto zeroFactors = Scaling::scale_to(scaledZero, zero, T(0));
VERIFY_IS_EQUAL(zeroFactors.scale, T(1));
VERIFY_IS_EQUAL(zeroFactors.invScale, T(1));
VERIFY_IS_EQUAL(scaledZero(0), T(0));
for (const T special : {std::numeric_limits<T>::infinity(), std::numeric_limits<T>::quiet_NaN()}) {
Matrix<T, 2, 1> input;
input << special, T(2);
Matrix<T, 2, 1> scaled;
const auto factors = Scaling::scale_to(scaled, input, special);
VERIFY_IS_EQUAL(factors.scale, T(1));
VERIFY_IS_EQUAL(factors.invScale, T(1));
VERIFY_IS_EQUAL(scaled(1), T(2));
if ((numext::isnan)(special)) {
VERIFY((numext::isnan)(scaled(0)));
} else {
VERIFY((numext::isinf)(scaled(0)));
}
}
}
void check_safe_scaling_special_value_frontends() {
using Scaling = internal::safe_scaling<float>;
using Factors = internal::safe_scaling_factors<float>;
for (const float special : {std::numeric_limits<float>::infinity(), std::numeric_limits<float>::quiet_NaN()}) {
Vector2f input(special, 2.0f);
Vector2f scaledTo;
const auto scaleToFactors = Scaling::scale_to(scaledTo, input, special);
VERIFY_IS_EQUAL(scaleToFactors.scale, 1.0f);
VERIFY_IS_EQUAL(scaleToFactors.invScale, 1.0f);
Vector2f scaledExpression;
const Factors expressionFactors =
Scaling::with_scaled(input, special, [&](const auto& expression) { scaledExpression = expression; });
VERIFY_IS_EQUAL(expressionFactors.scale, 1.0f);
VERIFY_IS_EQUAL(expressionFactors.invScale, 1.0f);
VERIFY_IS_EQUAL(scaledTo(1), 2.0f);
VERIFY_IS_EQUAL(scaledExpression(1), 2.0f);
if ((numext::isnan)(special)) {
VERIFY((numext::isnan)(scaledTo(0)));
VERIFY((numext::isnan)(scaledExpression(0)));
} else {
VERIFY((numext::isinf)(scaledTo(0)));
VERIFY((numext::isinf)(scaledExpression(0)));
}
}
}
void check_arithmetic_safe_scaling_fallback() {
using Scaling = internal::safe_scaling<double, false>;
using Factors = internal::safe_scaling_factors<double>;
Matrix<double, 2, 1> input;
input << 3.0, 6.0;
Matrix<double, 2, 1> scaled;
const auto factors = Scaling::scale_to(scaled, input, 3.0);
VERIFY_IS_EQUAL(factors.scale, 3.0);
VERIFY_IS_EQUAL(factors.invScale, 1.0);
VERIFY_IS_EQUAL(scaled(0), 1.0);
VERIFY_IS_EQUAL(scaled(1), 2.0);
Matrix<double, 2, 1> restored;
Scaling::unscale_to(restored, scaled, 3.0, factors);
VERIFY_IS_EQUAL(restored, input);
Scaling::unscale_in_place(scaled, factors);
VERIFY_IS_EQUAL(scaled, input);
const auto check_extreme_unscale = [](const double value) {
const Factors extremeFactors = Scaling::compute_floor_factors(value);
double restored;
Scaling::unscale_to(restored, 1.0, extremeFactors);
VERIFY_IS_EQUAL(restored, value);
restored = 1.0;
Scaling::unscale_in_place(restored, extremeFactors);
VERIFY_IS_EQUAL(restored, value);
};
check_extreme_unscale((std::numeric_limits<double>::max)());
volatile double denormMinInput = std::numeric_limits<double>::denorm_min();
const double denormMin = denormMinInput;
if (denormMin > 0.0) check_extreme_unscale(denormMin);
Factors normalReciprocalFactors;
const double normalMin = (std::numeric_limits<double>::min)();
VERIFY(Scaling::try_compute_ceiling_factors_with_normal_reciprocal(3.0, normalMin, normalReciprocalFactors));
VERIFY_IS_EQUAL(normalReciprocalFactors.scale, 3.0);
VERIFY_IS_EQUAL(normalReciprocalFactors.invScale, 1.0 / 3.0);
VERIFY(!Scaling::try_compute_ceiling_factors_with_normal_reciprocal(std::numeric_limits<double>::denorm_min(),
normalMin, normalReciprocalFactors));
VERIFY(!Scaling::try_compute_ceiling_factors_with_normal_reciprocal((std::numeric_limits<double>::max)(), normalMin,
normalReciprocalFactors));
}
void check_custom_scalar_scaling_exceptions() {
using Scalar = AnnoyingScalar;
using Scaling = internal::safe_scaling<Scalar>;
static_assert(!internal::supports_power_of_two_scaling<Scalar>::value, "exercise arithmetic scaling");
volatile float denormInput = std::numeric_limits<float>::denorm_min();
const float denorm = denormInput;
const float twiceDenorm = denorm + denorm;
if (!(denorm > 0.0f && twiceDenorm > 0.0f)) return; // No custom-scalar subnormal recovery is promised.
Matrix<Scalar, 2, 1> input;
input << Scalar(denorm), Scalar(twiceDenorm);
Matrix<Scalar, 2, 1> scaled;
internal::safe_scaling_factors<Scalar> factors;
const int overflow =
fp_exceptions_raised_by(FE_OVERFLOW, [&] { factors = Scaling::scale_to(scaled, input, Scalar(twiceDenorm)); });
if (overflow >= 0) VERIFY_IS_EQUAL(overflow, 0);
VERIFY_IS_EQUAL(scaled(0), Scalar(0.5));
VERIFY_IS_EQUAL(scaled(1), Scalar(1));
Matrix<Scalar, 2, 1> restored;
Scaling::unscale_to(restored, scaled, Scalar(twiceDenorm), factors);
VERIFY_IS_EQUAL(restored, input);
Scaling::unscale_in_place(scaled, factors);
VERIFY_IS_EQUAL(scaled, input);
}
template <typename Scalar>
void check_scale_binary_by_power_of_two() {
using Binary = internal::binary_floating_point_traits<Scalar>;
using Bits = typename Binary::Bits;
constexpr int kMinFactorExponent = 1 - std::numeric_limits<Scalar>::min_exponent - Binary::kFractionBits;
constexpr int kMaxFactorExponent = 1 - std::numeric_limits<Scalar>::min_exponent;
const Bits magnitudes[] = {Bits(0),
Bits(1),
Bits(3),
Binary::kFractionMask,
Binary::kExponentUnit,
Binary::kExponentUnit + (Binary::kExponentUnit >> 1)};
ScopedFlushToZero flushToZero;
// Cover the entire factor range selected by tiny-input recovery. Form the reference from integer significands,
// not subnormal floating-point inputs, so it remains valid under FTZ/DAZ.
for (int exponent = kMinFactorExponent; exponent <= kMaxFactorExponent; ++exponent) {
const Scalar factor = numext::ldexp(Scalar(1), exponent);
for (Bits magnitude : magnitudes) {
const Scalar expected = numext::ldexp(Scalar(magnitude), std::numeric_limits<Scalar>::min_exponent -
std::numeric_limits<Scalar>::digits + exponent);
for (Bits sign : {Bits(0), Bits(Binary::kSignBit)}) {
const Scalar value = numext::bit_cast<Scalar>(sign | magnitude);
const Scalar actual = internal::scale_binary_by_power_of_two(value, factor);
VERIFY_IS_EQUAL(Binary::bits(actual), sign | Binary::bits(expected));
}
}
}
}
// Rounds a subnormal result exactly where hardware arithmetic may flush it: the exact product m * 2^-k denorm_min is
// normal in double, and its nearest integer, ties to even, is the expected magnitude's bit pattern (a carry into the
// exponent field encodes the smallest normal).
template <typename Scalar>
void check_scale_binary_by_power_of_two_rounding() {
using Binary = internal::binary_floating_point_traits<Scalar>;
using Bits = typename Binary::Bits;
// Leading-bit significands in [kExponentUnit, 2 * kExponentUnit): ties and their neighbours at small shifts, and an
// all-ones tail that carries into the smallest normal.
const Bits significands[] = {Binary::kExponentUnit,
Binary::kExponentUnit | Bits(1),
Binary::kExponentUnit | Bits(2),
Binary::kExponentUnit | Bits(3),
Binary::kExponentUnit | Bits(5),
Binary::kExponentUnit + (Binary::kExponentUnit >> 1),
(Binary::kExponentUnit << 1) - Bits(1)};
// A biased input exponent e scales m by 2^(e - 1); the factor 2^-(e - 1 + k) cancels it and shifts by k.
const int exponentFields[] = {1, 2, Binary::kExponentBias - Binary::kFractionBits - 3};
ScopedFlushToZero flushToZero;
for (int field : exponentFields) {
for (int k = 0; k <= Binary::kFractionBits + 3; ++k) {
const Scalar factor = numext::ldexp(Scalar(1), -(field - 1 + k));
for (Bits m : significands) {
const Bits expected = Bits(std::nearbyint(std::ldexp(double(m), -k)));
for (Bits sign : {Bits(0), Bits(Binary::kSignBit)}) {
const Scalar value =
numext::bit_cast<Scalar>(sign | (Bits(field) << Binary::kFractionBits) | (m & Binary::kFractionMask));
VERIFY_IS_EQUAL(Binary::bits(internal::scale_binary_by_power_of_two(value, factor)), sign | expected);
}
}
}
}
// Subnormal inputs normalize before they round.
for (Bits magnitude : {Bits(1), Bits(3), Binary::kExponentUnit >> 1, Binary::kFractionMask}) {
for (int k = 1; k <= Binary::kFractionBits + 2; ++k) {
const Bits expected = Bits(std::nearbyint(std::ldexp(double(magnitude), -k)));
for (Bits sign : {Bits(0), Bits(Binary::kSignBit)}) {
const Scalar actual = internal::scale_binary_by_power_of_two(numext::bit_cast<Scalar>(sign | magnitude),
numext::ldexp(Scalar(1), -k));
VERIFY_IS_EQUAL(Binary::bits(actual), sign | expected);
}
}
}
}
// Under DAZ a comparison reads a subnormal as zero, so compare representations.
template <typename RealScalar>
bool same_bits(const RealScalar& a, const RealScalar& b) {
using Binary = internal::binary_floating_point_traits<RealScalar>;
return Binary::bits(a) == Binary::bits(b);
}
template <typename RealScalar>
bool same_bits(const std::complex<RealScalar>& a, const std::complex<RealScalar>& b) {
return same_bits(a.real(), b.real()) && same_bits(a.imag(), b.imag());
}
template <typename Scalar>
struct scaling_test_value {
using RealScalar = typename NumTraits<Scalar>::Real;
static Scalar run(RealScalar real, RealScalar) { return real; }
};
template <typename RealScalar>
struct scaling_test_value<std::complex<RealScalar>> {
static std::complex<RealScalar> run(RealScalar real, RealScalar imag) { return std::complex<RealScalar>(real, imag); }
};
template <typename Scalar>
void check_arithmetic_scaling_expression() {
using RealScalar = typename NumTraits<Scalar>::Real;
using Scaling = internal::safe_scaling<RealScalar, false>;
using Vector2 = Matrix<Scalar, 2, 1>;
using Value = scaling_test_value<Scalar>;
// Lazy scaling and its adjoint must retain the division policy even below the binary-recovery threshold.
for (const RealScalar maxCoeff :
{RealScalar(3), numext::ldexp(RealScalar(1.5), std::numeric_limits<RealScalar>::min_exponent + 7)}) {
Vector2 input;
input << Value::run(maxCoeff, RealScalar(0)), Value::run(maxCoeff / RealScalar(2), -maxCoeff / RealScalar(4));
Vector2 expected;
expected << Value::run(RealScalar(1), RealScalar(0)), Value::run(RealScalar(0.5), RealScalar(-0.25));
Vector2 materialized;
const auto factors = Scaling::scale_to(materialized, input, maxCoeff);
Vector2 lazy;
Matrix<Scalar, 1, 2> adjoint;
const auto expressionFactors = Scaling::with_scaled(input, maxCoeff, [&](const auto& expression) {
lazy = expression;
adjoint = expression.adjoint();
});
VERIFY_IS_EQUAL(materialized, expected);
VERIFY_IS_EQUAL(lazy, expected);
VERIFY_IS_EQUAL(adjoint, expected.adjoint());
VERIFY_IS_EQUAL(expressionFactors.scale, factors.scale);
VERIFY_IS_EQUAL(expressionFactors.invScale, factors.invScale);
Vector2 inPlace = input;
const auto inPlaceFactors = Scaling::scale_in_place(inPlace, maxCoeff);
VERIFY_IS_EQUAL(inPlace, expected);
VERIFY_IS_EQUAL(inPlaceFactors.scale, factors.scale);
VERIFY_IS_EQUAL(inPlaceFactors.invScale, factors.invScale);
}
}
template <typename Scalar>
void check_subnormal_preserving_scaling() {
using RealScalar = typename NumTraits<Scalar>::Real;
using Binary = internal::binary_floating_point_traits<RealScalar>;
using Bits = typename Binary::Bits;
const RealScalar maxCoeff = numext::ldexp((std::numeric_limits<RealScalar>::min)(), 8);
// Arithmetic or narrowing conversions could flush these inputs before they reach the scaling helper.
const RealScalar subnormalMaxCoeff = numext::bit_cast<RealScalar>(Bits(64));
const auto factors = internal::safe_scaling<RealScalar>::compute_floor_factors(maxCoeff);
Matrix<Scalar, 2, 1> input;
input(0) = scaling_test_value<Scalar>::run(maxCoeff, -maxCoeff);
input(1) = scaling_test_value<Scalar>::run(numext::bit_cast<RealScalar>(Binary::kExponentUnit >> 1),
numext::bit_cast<RealScalar>(Binary::kExponentUnit >> 2));
Matrix<Scalar, 1, 1> subnormalInput;
subnormalInput(0) = scaling_test_value<Scalar>::run(subnormalMaxCoeff, RealScalar(0));
ScopedFlushToZero flushToZero;
Matrix<Scalar, 2, 1> scaled;
internal::safe_scaling<RealScalar>::scale_to(scaled, input, maxCoeff, factors);
Matrix<Scalar, 2, 1> scaledExpression;
const auto expressionFactors = internal::safe_scaling<RealScalar>::with_scaled(
input, maxCoeff, [&](const auto& expression) { scaledExpression = expression; });
VERIFY_IS_EQUAL(scaled(0), scaling_test_value<Scalar>::run(RealScalar(1), RealScalar(-1)));
VERIFY_IS_EQUAL(scaled(1),
scaling_test_value<Scalar>::run(RealScalar(1) / RealScalar(512), RealScalar(1) / RealScalar(1024)));
VERIFY_IS_EQUAL(expressionFactors.scale, factors.scale);
VERIFY_IS_EQUAL(expressionFactors.invScale, factors.invScale);
VERIFY_IS_EQUAL(scaledExpression, scaled);
Matrix<Scalar, 2, 1> scaledInPlace = input;
const auto inPlaceFactors = internal::safe_scaling<RealScalar>::scale_in_place(scaledInPlace, maxCoeff);
VERIFY_IS_EQUAL(inPlaceFactors.scale, factors.scale);
VERIFY_IS_EQUAL(inPlaceFactors.invScale, factors.invScale);
VERIFY_IS_EQUAL(scaledInPlace, scaled);
Matrix<Scalar, 1, 1> scaledSubnormal;
const auto subnormalFactors =
internal::safe_scaling<RealScalar>::scale_to(scaledSubnormal, subnormalInput, subnormalMaxCoeff);
VERIFY(subnormalFactors.scale != RealScalar(1) || subnormalFactors.invScale != RealScalar(1));
VERIFY(numext::abs(scaledSubnormal(0)) > RealScalar(0));
// Unscaling restores the subnormal components exactly, still under FTZ.
Matrix<Scalar, 2, 1> restored;
internal::safe_scaling<RealScalar>::unscale_to(restored, scaled, maxCoeff, factors);
for (Index i = 0; i < input.size(); ++i) VERIFY(same_bits(restored(i), input(i)));
internal::safe_scaling<RealScalar>::unscale_in_place(scaled, maxCoeff, factors);
internal::safe_scaling<RealScalar>::unscale_in_place(scaledSubnormal, subnormalMaxCoeff, subnormalFactors);
for (Index i = 0; i < input.size(); ++i) VERIFY(same_bits(scaled(i), input(i)));
VERIFY(same_bits(scaledSubnormal(0), subnormalInput(0)));
// A NaN, infinite or zero maxCoeff selects identity factors, and unscaling leaves the coefficients alone.
for (const RealScalar special :
{std::numeric_limits<RealScalar>::quiet_NaN(), std::numeric_limits<RealScalar>::infinity(), RealScalar(0)}) {
Matrix<Scalar, 2, 1> unchanged;
const auto specialFactors = internal::safe_scaling<RealScalar>::scale_to(unchanged, input, special);
internal::safe_scaling<RealScalar>::unscale_in_place(unchanged, special, specialFactors);
for (Index i = 0; i < input.size(); ++i) VERIFY(same_bits(unchanged(i), input(i)));
const auto inPlaceSpecialFactors = internal::safe_scaling<RealScalar>::scale_in_place(unchanged, special);
VERIFY_IS_EQUAL(inPlaceSpecialFactors.scale, RealScalar(1));
VERIFY_IS_EQUAL(inPlaceSpecialFactors.invScale, RealScalar(1));
for (Index i = 0; i < input.size(); ++i) VERIFY(same_bits(unchanged(i), input(i)));
}
}
// Unscaling rounds through integer significands exactly when 0 < maxCoeff < min / eps. With floor factors, eps / 2
// unscales to min / 2 at the threshold and to min / 4 one ulp below it; FTZ flushes only the former.
template <typename Scalar>
void check_unscale_recovery_threshold() {
using Binary = internal::binary_floating_point_traits<Scalar>;
using Scaling = internal::safe_scaling<Scalar>;
const Scalar threshold = (std::numeric_limits<Scalar>::min)() / NumTraits<Scalar>::epsilon();
const Scalar belowThreshold = numext::nextafter(threshold, Scalar(0));
// Read through volatile so the products are formed at run time, under FTZ.
volatile Scalar halfEpsilon = NumTraits<Scalar>::epsilon() / Scalar(2);
Matrix<Scalar, 1, 1> atThreshold, below;
atThreshold(0) = halfEpsilon;
below(0) = halfEpsilon;
ScopedFlushToZero flushToZero;
Scaling::unscale_in_place(below, belowThreshold, Scaling::compute_floor_factors(belowThreshold));
VERIFY(same_bits(below(0), numext::bit_cast<Scalar>(Binary::kExponentUnit >> 2)));
Scaling::unscale_in_place(atThreshold, threshold, Scaling::compute_floor_factors(threshold));
if (flushToZero.isSupported()) VERIFY(same_bits(atThreshold(0), Scalar(0)));
}
template <typename Scalar>
void check_identity_scaling() {
using RealScalar = typename NumTraits<Scalar>::Real;
using Binary = internal::binary_floating_point_traits<RealScalar>;
using Scaling = internal::safe_scaling<RealScalar>;
Matrix<Scalar, 2, 1> input;
input(0) = scaling_test_value<Scalar>::run(RealScalar(1), RealScalar(-1));
input(1) = scaling_test_value<Scalar>::run(numext::bit_cast<RealScalar>(Binary::kExponentUnit >> 1),
numext::bit_cast<RealScalar>(+Binary::kSignBit));
const ScopedFlushToZero flush;
for (RealScalar maxCoeff : {RealScalar(0), RealScalar(1), RealScalar(1.5), NumTraits<RealScalar>::infinity(),
NumTraits<RealScalar>::quiet_NaN()}) {
// Keep the factor selection at run time, where a compiler can merge a copy with multiplication by one.
volatile RealScalar runtimeMaxCoeff = maxCoeff;
Matrix<Scalar, 2, 1> scaled, restored;
const auto factors = Scaling::scale_to(scaled, input, RealScalar(runtimeMaxCoeff));
VERIFY_IS_EQUAL(factors.scale, RealScalar(1));
VERIFY_IS_EQUAL(factors.invScale, RealScalar(1));
for (Index i = 0; i < input.size(); ++i) VERIFY(same_bits(scaled(i), input(i)));
Scaling::unscale_to(restored, input, factors);
for (Index i = 0; i < input.size(); ++i) VERIFY(same_bits(restored(i), input(i)));
Scaling::with_scaled(input, RealScalar(runtimeMaxCoeff), [&](const auto& expression) { scaled = expression; });
for (Index i = 0; i < input.size(); ++i) VERIFY(same_bits(scaled(i), input(i)));
Scaling::scale_in_place(scaled, RealScalar(runtimeMaxCoeff));
for (Index i = 0; i < input.size(); ++i) VERIFY(same_bits(scaled(i), input(i)));
Scaling::unscale_in_place(scaled, factors);
for (Index i = 0; i < input.size(); ++i) VERIFY(same_bits(scaled(i), input(i)));
}
}
// A reduction that flushes subnormal inputs returns zero; the rescan reads the largest real or imaginary magnitude from
// the representation instead, and leaves a nonzero maximum alone.
template <typename Scalar>
void check_recover_flushed_max_coeff() {
using RealScalar = typename NumTraits<Scalar>::Real;
using Binary = internal::binary_floating_point_traits<RealScalar>;
using Bits = typename Binary::Bits;
using Scaling = internal::safe_scaling<RealScalar>;
using Value = scaling_test_value<Scalar>;
const RealScalar small = numext::bit_cast<RealScalar>(Bits(3));
const RealScalar large = numext::bit_cast<RealScalar>(Bits(Binary::kFractionMask));
Matrix<Scalar, 2, 2> m;
m << Value::run(small, RealScalar(0)), Value::run(-small, -large), Value::run(RealScalar(0), small),
Value::run(-small, RealScalar(0));
const RealScalar expected = NumTraits<Scalar>::IsComplex ? large : small;
VERIFY(same_bits(Scaling::recover_flushed_max_coeff(m, RealScalar(0)), expected));
VERIFY(same_bits(Scaling::recover_flushed_max_coeff(m, RealScalar(1)), RealScalar(1)));
VERIFY(same_bits(Scaling::recover_flushed_max_coeff(Matrix<Scalar, 2, 2>::Zero(), RealScalar(0)), RealScalar(0)));
}
// A strided view reduces coefficient by coefficient, and under Arm FZ a scalar max(a, b) is a compare and a select
// that keeps the first subnormal operand rather than producing a zero: the rescan must not trust a subnormal maximum.
// Every flush-to-zero mode, with the largest magnitude last.
template <typename Scalar>
void check_recover_flushed_scalar_max_coeff() {
using Binary = internal::binary_floating_point_traits<Scalar>;
using Bits = typename Binary::Bits;
using Scaling = internal::safe_scaling<Scalar>;
using VectorType = Matrix<Scalar, Dynamic, 1>;
VectorType storage = VectorType::Zero(6);
const Bits significands[3] = {Bits(5), Bits(8), Bits(23)};
for (Index i = 0; i < 3; ++i) storage(2 * i) = numext::bit_cast<Scalar>(significands[i]);
const Map<const VectorType, 0, InnerStride<2>> strided(storage.data(), 3);
forEachFlushToZeroMode([&](FlushToZeroMode) {
const Scalar maxCoeff = strided.cwiseAbs().maxCoeff();
VERIFY(same_bits(Scaling::recover_flushed_max_coeff(strided, maxCoeff), numext::bit_cast<Scalar>(Bits(23))));
VERIFY_IS_EQUAL(internal::binary_frexp_exponent(Scaling::recover_flushed_max_coeff(strided, maxCoeff)),
5 + std::numeric_limits<Scalar>::min_exponent - std::numeric_limits<Scalar>::digits);
});
}
// Each flush-to-zero mode the host selects does what its name says, read from the results of two probes through the
// representation, since a comparison under DAZ reads any subnormal as zero; the modes restore the ambient state.
template <typename Scalar>
void check_flush_to_zero_modes() {
using Binary = internal::binary_floating_point_traits<Scalar>;
const auto resultsFlushed = []() { return Binary::bits(underflowProbe<Scalar>()) == 0; };
const auto inputsFlushed = []() { return Binary::bits(subnormalInputProbe<Scalar>()) == 0; };
const bool resultsBefore = resultsFlushed(), inputsBefore = inputsFlushed();
int modes = 0;
forEachFlushToZeroMode([&](FlushToZeroMode mode) {
++modes;
VERIFY_IS_EQUAL(resultsFlushed(), flushToZeroModeIncludes(mode, FlushToZeroMode::Results));
VERIFY_IS_EQUAL(inputsFlushed(), flushToZeroModeIncludes(mode, FlushToZeroMode::Inputs));
VERIFY_IS_EQUAL(ScopedFlushToZero::hardwareFlushesSubnormalInputs(),
flushToZeroModeIncludes(mode, FlushToZeroMode::Inputs));
});
VERIFY(modes >= 1);
VERIFY_IS_EQUAL(resultsFlushed(), resultsBefore);
VERIFY_IS_EQUAL(inputsFlushed(), inputsBefore);
// The default request flushes at least results, as it always did.
{
const ScopedFlushToZero flushToZero;
if (flushToZero.isSupported()) {
VERIFY(flushToZeroModeIncludes(flushToZero.mode(), FlushToZeroMode::Results));
VERIFY(resultsFlushed());
}
}
VERIFY_IS_EQUAL(resultsFlushed(), resultsBefore);
}
// value * 2^exponent through integer significands agrees bit for bit with the correctly rounded ldexp for every input
// class and every exponent that changes the result, and keeps doing so in every flush-to-zero mode, where the
// references are taken beforehand. Beyond the exponent range it saturates to the signed zero or infinity ldexp returns.
template <typename Scalar>
void check_scale_binary_by_exponent() {
using Binary = internal::binary_floating_point_traits<Scalar>;
using Bits = typename Binary::Bits;
using Scaling = internal::safe_scaling<Scalar>;
// 2^recovery is the min / eps threshold check_unscale_recovery_threshold() exercises.
VERIFY_IS_EQUAL(numext::ldexp(Scalar(1), Scaling::subnormal_recovery_exponent()),
(std::numeric_limits<Scalar>::min)() / NumTraits<Scalar>::epsilon());
// Subnormals down to denorm_min, the normal boundary, full fractions, and the largest finite value.
const Bits magnitudes[] = {Bits(1),
Bits(3),
Binary::kExponentUnit >> 1,
Binary::kFractionMask,
Binary::kExponentUnit,
Binary::kExponentUnit | Bits(1),
Binary::kExponentUnit + (Binary::kExponentUnit >> 1),
(Binary::kExponentUnit << 1) - Bits(1),
Binary::bits(Scalar(0.75)),
Binary::bits(Scalar(1)),
Binary::bits(Scalar(1.5)),
Binary::bits((std::numeric_limits<Scalar>::max)())};
// Every exponent from full underflow to full overflow, then values that only the clamp can handle.
const int span = std::numeric_limits<Scalar>::max_exponent - std::numeric_limits<Scalar>::min_exponent +
std::numeric_limits<Scalar>::digits + 3;
std::vector<int> exponents;
for (int e = -span; e <= span; ++e) exponents.push_back(e);
for (int e : {1 << 24, 1 << 30, (std::numeric_limits<int>::max)()}) {
exponents.push_back(e);
exponents.push_back(-e);
}
exponents.push_back((std::numeric_limits<int>::min)());
struct Case {
Bits value;
int exponent;
Bits expected;
Bits expectedUnit; // ldexp(1, exponent)
int frexpExponent; // of value
Bits frexpSignificand;
};
std::vector<Case> cases;
for (Bits magnitude : magnitudes) {
for (int e : exponents) {
for (Bits sign : {Bits(0), Bits(Binary::kSignBit)}) {
const Scalar value = numext::bit_cast<Scalar>(sign | magnitude);
int frexpExponent = 0;
EIGEN_USING_STD(frexp);
const Scalar frexpSignificand = frexp(value, &frexpExponent);
cases.push_back(Case{sign | magnitude, e, Binary::bits(numext::ldexp(value, e)),
Binary::bits(numext::ldexp(Scalar(1), e)), frexpExponent, Binary::bits(frexpSignificand)});
}
}
}
for (const Scalar special : {Scalar(0), -Scalar(0), std::numeric_limits<Scalar>::infinity(),
-std::numeric_limits<Scalar>::infinity(), std::numeric_limits<Scalar>::quiet_NaN()}) {
for (int e : {-1000, -1, 0, 1, 1000}) {
cases.push_back(Case{Binary::bits(special), e, Binary::bits(special), Binary::bits(numext::ldexp(Scalar(1), e)),
0, Binary::bits(special)});
}
}
const auto check = [&]() {
for (const Case& c : cases) {
const Scalar actual = internal::scale_binary_by_exponent(numext::bit_cast<Scalar>(c.value), c.exponent);
VERIFY_IS_EQUAL(Binary::bits(actual), c.expected);
const std::complex<Scalar> complexActual = internal::scale_binary_by_exponent(
std::complex<Scalar>(numext::bit_cast<Scalar>(c.value), Scalar(1)), c.exponent);
VERIFY_IS_EQUAL(Binary::bits(complexActual.real()), c.expected);
VERIFY_IS_EQUAL(Binary::bits(complexActual.imag()), c.expectedUnit);
// The representation-based classifiers agree with the IEEE ones on every value.
const Scalar value = numext::bit_cast<Scalar>(c.value);
VERIFY_IS_EQUAL(Binary::bits(internal::ldexp_preserving_subnormals(value, c.exponent)), c.expected);
const Bits magnitude = c.value & ~Binary::kSignBit;
VERIFY_IS_EQUAL(numext::is_exactly_zero_no_flush(value), magnitude == 0);
VERIFY_IS_EQUAL(internal::is_subnormal_magnitude(value), magnitude != 0 && magnitude < Binary::kExponentUnit);
if (magnitude == c.value &&
(numext::isfinite)(value)) { // non-negative: the maximum against every larger power of two
VERIFY_IS_EQUAL(Binary::bits(internal::max_preserving_subnormals(value, numext::bit_cast<Scalar>(Bits(1)))),
magnitude == 0 ? Bits(1) : magnitude);
}
// frexp from the representation agrees with frexp on every finite value; zeros give exponent 0.
if ((numext::isfinite)(value)) {
int frexpExponent = 0;
const Scalar significand = internal::binary_frexp(value, frexpExponent);
VERIFY_IS_EQUAL(frexpExponent, c.frexpExponent);
VERIFY_IS_EQUAL(Binary::bits(significand), c.frexpSignificand);
VERIFY_IS_EQUAL(internal::binary_frexp_exponent(value), c.frexpExponent);
}
}
// A normal power-of-two factor is the exponent form of the same scaling.
for (int e = std::numeric_limits<Scalar>::min_exponent - 1; e < std::numeric_limits<Scalar>::max_exponent; ++e) {
const Scalar factor = numext::bit_cast<Scalar>(Bits(e + Binary::kExponentBias) << Binary::kFractionBits);
for (Bits magnitude : magnitudes) {
const Scalar value = numext::bit_cast<Scalar>(magnitude);
VERIFY_IS_EQUAL(Binary::bits(internal::scale_binary_by_power_of_two(value, factor)),
Binary::bits(internal::scale_binary_by_exponent(value, e)));
}
}
// The functor is the coefficient-wise scalar form, complex components included.
Matrix<std::complex<Scalar>, 3, 1> input;
input << std::complex<Scalar>(numext::bit_cast<Scalar>(Bits(3)), numext::bit_cast<Scalar>(Binary::kFractionMask)),
std::complex<Scalar>(Scalar(1.5), -Scalar(0.75)),
std::complex<Scalar>((std::numeric_limits<Scalar>::max)(), numext::bit_cast<Scalar>(Bits(1)));
for (int e : {-(std::numeric_limits<Scalar>::digits + 1), -3, 5, std::numeric_limits<Scalar>::max_exponent}) {
const Matrix<std::complex<Scalar>, 3, 1> scaled = input.unaryExpr(internal::scale_by_exponent_op<Scalar>(e));
for (Index i = 0; i < input.size(); ++i) {
VERIFY(same_bits(scaled(i), internal::scale_binary_by_exponent(input(i), e)));
}
}
};
forEachFlushToZeroMode([&](FlushToZeroMode) { check(); });
}
// The scalars without a representation path take the C library.
void check_preserving_subnormals_fallbacks() {
VERIFY_IS_EQUAL(internal::ldexp_preserving_subnormals(1.5L, 3), 12.0L);
VERIFY_IS_EQUAL(internal::frexp_exponent_preserving_subnormals(12.0L), 4);
int exponent = 0;
VERIFY_IS_EQUAL(internal::frexp_preserving_subnormals(-12.0L, exponent), -0.75L);
VERIFY_IS_EQUAL(exponent, 4);
VERIFY_IS_EQUAL(internal::frexp_exponent_preserving_subnormals(0.0L), 0);
VERIFY(numext::is_exactly_zero_no_flush(0.0L));
VERIFY(!internal::is_subnormal_magnitude(1.0L));
VERIFY(internal::is_subnormal_magnitude(std::numeric_limits<long double>::denorm_min()));
VERIFY_IS_EQUAL(internal::max_preserving_subnormals(1.5L, 2.5L), 2.5L);
}
// prod_k x_k with every factor and every partial product renormalized by the C library's frexp: the form
// scaled_product must agree with bit for bit, since scaling by a power of two is exact (see
// scaling_commutes_with_multiplication() for the exception).
template <typename RealScalar>
struct EagerProduct {
void multiply(const RealScalar& factor) {
if (factor == RealScalar(0) || !(std::isfinite)(factor) || !(std::isfinite)(mantissa)) {
mantissa *= factor;
return;
}
int factorExponent = 0, productExponent = 0;
mantissa = std::frexp(mantissa * std::frexp(factor, &factorExponent), &productExponent);
exponent += factorExponent + productExponent;
}
RealScalar value() const {
const long limit = 4L * std::numeric_limits<RealScalar>::max_exponent;
return std::ldexp(mantissa, int((std::min)((std::max)(exponent, -limit), limit)));
}
long double logAbs() const {
return std::log(std::abs(static_cast<long double>(mantissa))) + static_cast<long double>(exponent) * std::log(2.0L);
}
RealScalar mantissa = RealScalar(1);
long exponent = 0;
};
// The same value, a zero with its sign. long double has no bit pattern to compare.
template <typename RealScalar>
bool identical(const RealScalar& a, const RealScalar& b) {
return ((std::isnan)(a) && (std::isnan)(b)) || (a == b && (std::signbit)(a) == (std::signbit)(b));
}
// A significand wider than double's on double's exponent range can only be the pair of doubles of IBM double-double
// (long double on ppc64le), whose low half is a double of its own: it underflows to a multiple of 2^-1074 for a result
// a few bits above numeric_limits<>::min() (2^-969), so one multiplication there rounds differently from the same one
// scaled by a power of two, by about 2^-105 relative. Elsewhere scaling by a power of two commutes with every
// operation.
template <typename RealScalar>
constexpr bool scaling_commutes_with_multiplication() {
return !(std::numeric_limits<RealScalar>::digits > std::numeric_limits<double>::digits &&
std::numeric_limits<RealScalar>::max_exponent == std::numeric_limits<double>::max_exponent);
}
// identical(), up to the rounding of n multiplications where scaling does not commute with them. A zero, an infinity
// or a NaN is not within reach of that rounding.
template <typename RealScalar>
bool same_product(const RealScalar& a, const RealScalar& b, Index n) {
if (scaling_commutes_with_multiplication<RealScalar>() || !(std::isfinite)(a) || !(std::isfinite)(b) ||
a == RealScalar(0) || b == RealScalar(0))
return identical(a, b);
return (std::signbit)(a) == (std::signbit)(b) &&
numext::abs(a - b) <= RealScalar(4 * n) * NumTraits<RealScalar>::epsilon() * numext::abs(b);
}
template <typename RealScalar>
void check_scaled_product_real() {
using Product = internal::scaled_product<RealScalar>;
using Vector = Matrix<RealScalar, Dynamic, 1>;
const int maxExponent = std::numeric_limits<RealScalar>::max_exponent;
const RealScalar big = std::ldexp(RealScalar(1), maxExponent / 2 + 2);
const RealScalar small = std::ldexp(RealScalar(1), -(maxExponent / 2 + 2));
const RealScalar inf = std::numeric_limits<RealScalar>::infinity();
const RealScalar nan = std::numeric_limits<RealScalar>::quiet_NaN();
const auto product = [](std::initializer_list<RealScalar> factors) {
Product result;
for (const RealScalar& factor : factors) result.multiply(factor);
return result;
};
// While the plain product stays in the normal range, value() is that product.
for (int trial = 0; trial < 100; ++trial) {
const Vector factors = Vector::NullaryExpr(
40, []() { return std::ldexp(internal::random<RealScalar>(-2, 2), internal::random(-3, 3)); });
RealScalar plain(1);
Product sequential;
for (Index i = 0; i < factors.size(); ++i) {
plain *= factors[i];
sequential.multiply(factors[i]);
}
VERIFY_IS_EQUAL(sequential.value(), plain);
// scaled_prod() multiplies whole blocks on independent chains, so it associates differently: each of the n
// multiplications rounds once either way.
VERIFY(numext::abs(internal::scaled_prod(factors).value() - plain) <=
RealScalar(2 * factors.size()) * NumTraits<RealScalar>::epsilon() * numext::abs(plain));
}
// Factors across the whole exponent range, subnormals included: the same bits as the eager form, whether the result
// is normal, subnormal, zero or infinite.
int normalResults = 0, saturatedResults = 0;
for (int trial = 0; trial < 2000; ++trial) {
const Index size = internal::random<Index>(1, 40);
// Narrow spreads keep the result in range through many excursions; wide ones saturate it.
const int spread = internal::random(1, 2 * maxExponent);
Product scaled;
EagerProduct<RealScalar> eager;
for (Index i = 0; i < size; ++i) {
const RealScalar factor = std::ldexp(internal::random<RealScalar>(-2, 2),
(i % 2 == 0 ? 1 : -1) * (maxExponent - internal::random(0, spread)));
scaled.multiply(factor);
eager.multiply(factor);
}
const RealScalar expected = eager.value();
VERIFY(same_product(scaled.value(), expected, size));
VERIFY(same_product(scaled.abs(), RealScalar(std::abs(expected)), size));
// log(mantissa) and exponent log(2) each round once, and the mantissa is the eager one: the log is right
// whether or not the product itself is representable.
const long double expectedLog = eager.logAbs();
const long double tolerance = 4 * NumTraits<RealScalar>::epsilon() * (std::max)(1.0L, std::abs(expectedLog));
if ((std::isfinite)(expectedLog))
VERIFY(std::abs(static_cast<long double>(scaled.logAbs()) - expectedLog) <= tolerance);
else // a zero factor
VERIFY_IS_EQUAL(static_cast<long double>(scaled.logAbs()), expectedLog);
if ((std::isnormal)(expected))
++normalResults;
else
++saturatedResults;
}
VERIFY(normalResults > 100 && saturatedResults > 100);
// The same factors through scaled_prod(), which multiplies a block of 8 on independent chains when every factor is
// within 2^(+-max_exponent / 10) and falls back to one factor at a time otherwise. The log has the same bound
// whether or not the product is representable; each of the n multiplications rounds once.
for (int trial = 0; trial < 2000; ++trial) {
const Index size = internal::random<Index>(1, 40);
const int spread = internal::random(1, 2 * maxExponent);
Vector factors(size);
EagerProduct<RealScalar> eager;
for (Index i = 0; i < size; ++i) {
factors[i] = std::ldexp(internal::random<RealScalar>(-2, 2),
(i % 2 == 0 ? 1 : -1) * (maxExponent - internal::random(0, spread)));
eager.multiply(factors[i]);
}
const long double expectedLog = eager.logAbs();
if (!(std::isfinite)(expectedLog)) continue;
const long double tolerance =
(4 + 2 * size) * NumTraits<RealScalar>::epsilon() * (std::max)(1.0L, std::abs(expectedLog));
VERIFY(std::abs(static_cast<long double>(internal::scaled_prod(factors).logAbs()) - expectedLog) <= tolerance);
}
// At the block bound, just inside it and just outside it, with partial products that overflow and that do not.
for (int offset = -1; offset <= 1; ++offset) {
const int blockExponent = maxExponent / 10 + offset;
const RealScalar atBound = std::ldexp(RealScalar(1), blockExponent);
Vector alternating(32), sorted(32);
for (Index i = 0; i < 32; ++i) {
alternating[i] = i % 2 == 0 ? RealScalar(3) * atBound : RealScalar(1) / atBound;
sorted[i] = i < 16 ? -atBound : RealScalar(1) / atBound;
}
VERIFY_IS_EQUAL(internal::scaled_prod(alternating).value(), RealScalar(43046721));
VERIFY_IS_EQUAL(internal::scaled_prod(sorted).value(), RealScalar(1));
VERIFY_IS_EQUAL(internal::scaled_prod(Vector::Constant(32, -atBound)).value(), inf);
VERIFY_IS_EQUAL(internal::scaled_prod(Vector::Constant(32, RealScalar(1) / atBound)).value(), RealScalar(0));
sorted[31] = RealScalar(0);
VERIFY_IS_EQUAL(internal::scaled_prod(sorted).value(), RealScalar(0));
sorted[30] = nan;
VERIFY((numext::isnan)(internal::scaled_prod(sorted).value()));
}
// Partial products that leave the range on the way to an exact result.
VERIFY_IS_EQUAL(product({big, big, big, small, small, small, RealScalar(3), RealScalar(-5)}).value(),
RealScalar(-15));
VERIFY_IS_EQUAL(product({small, small, small, big, big, big, RealScalar(3), RealScalar(-5)}).value(),
RealScalar(-15));
// A zero factor decides, wherever the partial product was.
VERIFY_IS_EQUAL(product({big, big, big, RealScalar(0)}).value(), RealScalar(0));
VERIFY_IS_EQUAL(product({RealScalar(0), big, big, big}).value(), RealScalar(0));
VERIFY_IS_EQUAL(product({big, big, big, RealScalar(0)}).logAbs(), -inf);
// Only the result saturates, with its sign.
VERIFY_IS_EQUAL(product({big, -big, big}).value(), -inf);
VERIFY_IS_EQUAL(product({small, -small, small}).value(), RealScalar(0));
VERIFY((std::signbit)(product({small, -small, small}).value()));
// log(mantissa), exponent log(2) and their sum each round once.
VERIFY(numext::abs(product({big, -big, big}).logAbs() - RealScalar(3) * std::log(big)) <=
RealScalar(4) * NumTraits<RealScalar>::epsilon() * RealScalar(3) * std::log(big));
// Infinities and NaNs propagate as in the plain product.
VERIFY_IS_EQUAL(product({big, big, -inf, small}).value(), -inf);
VERIFY((numext::isnan)(product({big, big, inf, RealScalar(0)}).value()));
VERIFY((numext::isnan)(product({RealScalar(0), inf}).value()));
VERIFY((numext::isnan)(product({big, nan, RealScalar(0)}).value()));
VERIFY((numext::isnan)(product({big, nan, small}).logAbs()));
// logAbs() of a product next to one is the log of that product, not a cancelling sum.
const RealScalar nearOne = RealScalar(1) + 4 * NumTraits<RealScalar>::epsilon();
VERIFY_IS_EQUAL(product({big, nearOne, small}).logAbs(), std::log(nearOne));
VERIFY_IS_EQUAL(Product().logAbs(), RealScalar(0));
// Products of products, including a product with itself, and quotients.
Product left = product({big, RealScalar(3)});
left.multiply(product({big, big, RealScalar(5)}));
VERIFY_IS_EQUAL(left.value(), inf);
left.divide(big);
left.divide(big);
VERIFY_IS_EQUAL(left.value(), RealScalar(15) * big);
Product square = product({big, RealScalar(-3)});
square.multiply(square);
square.divide(big);
VERIFY_IS_EQUAL(square.value(), RealScalar(9) * big);
Product quotient = product({small});
quotient.divide(big);
quotient.divide(big);
quotient.multiply(big);
quotient.multiply(big);
VERIFY_IS_EQUAL(quotient.value(), small);
VERIFY_IS_EQUAL(internal::scaled_prod_squared(Vector::Constant(3, RealScalar(3) * big)).value(), inf);
VERIFY_IS_EQUAL(internal::scaled_prod_squared(Vector::Constant(3, RealScalar(3))).value(), RealScalar(729));
VERIFY_IS_EQUAL(internal::scaled_prod(Vector(0)).value(), RealScalar(1));
// One log for the whole expression: within the rounding of that log and of the product.
Vector graded(6);
graded << big, RealScalar(3) * big, big, small, RealScalar(-5) * small, small;
VERIFY(numext::abs(internal::log_abs_prod(graded) - std::log(RealScalar(15))) <=
RealScalar(4) * NumTraits<RealScalar>::epsilon() * std::log(RealScalar(15)));
}
template <typename RealScalar>
void check_scaled_product_complex() {
using Complex = std::complex<RealScalar>;
using Product = internal::scaled_product<Complex>;
using Vector = Matrix<Complex, Dynamic, 1>;
const int maxExponent = std::numeric_limits<RealScalar>::max_exponent;
const RealScalar big = std::ldexp(RealScalar(1), maxExponent / 2 + 2);
const RealScalar small = std::ldexp(RealScalar(1), -(maxExponent / 2 + 2));
const auto product = [](std::initializer_list<Complex> factors) {
Product result;
for (const Complex& factor : factors) result.multiply(factor);
return result;
};
for (int trial = 0; trial < 100; ++trial) {
const Vector factors = Vector::Random(30);
const Complex scaled = internal::scaled_prod(factors).value();
Complex plain(1);
for (Index k = 0; k < factors.size(); ++k) plain *= factors[k];
// The same multiplications, but a compiler may contract either complex product to FMAs: each then rounds
// differently, by at most 2 sqrt(2) epsilon.
VERIFY(std::abs(scaled - plain) <=
RealScalar(8 * factors.size()) * NumTraits<RealScalar>::epsilon() * std::abs(plain));
}
// x_k 2^(e_k) with sum_k e_k = 0 has the product of the x_k, which stays in range; each multiplication rounds once.
for (int trial = 0; trial < 500; ++trial) {
const Index pairs = internal::random<Index>(1, 15);
Product scaled;
Complex expected(1);
for (Index i = 0; i < pairs; ++i) {
const int exponent = internal::random(-maxExponent + 4, maxExponent - 4);
const Complex first = internal::random<Complex>(), second = internal::random<Complex>();
scaled.multiply(Complex(std::ldexp(first.real(), exponent), std::ldexp(first.imag(), exponent)));
scaled.multiply(Complex(std::ldexp(second.real(), -exponent), std::ldexp(second.imag(), -exponent)));
expected *= first;
expected *= second;
}
const RealScalar tolerance = RealScalar(4 * 2 * pairs) * NumTraits<RealScalar>::epsilon() * std::abs(expected);
VERIFY(std::abs(scaled.value() - expected) <= tolerance);
VERIFY(numext::abs(scaled.abs() - std::abs(expected)) <= tolerance);
}
const Complex i(0, 1);
VERIFY_IS_EQUAL(product({big * i, big * i, big * i, small, small, small * i, Complex(3, 4)}).value(), Complex(3, 4));
VERIFY_IS_EQUAL(product({big * i, big, big, Complex(0)}).value(), Complex(0));
// |mantissa| overflows here, |product| does not.
const RealScalar high = NumTraits<RealScalar>::highest() / RealScalar(2);
VERIFY_IS_EQUAL(product({Complex(small, 0), Complex(high, high), Complex(high, high), Complex(small, 0)}).abs(),
RealScalar(2) * (high * small) * (high * small));
VERIFY(numext::abs(product({Complex(0, big), Complex(big, 0), Complex(0, big)}).logAbs() -
RealScalar(3) * std::log(big)) <=
RealScalar(4) * NumTraits<RealScalar>::epsilon() * RealScalar(3) * std::log(big));
Product quotient = product({Complex(small, small)});
quotient.divide(Complex(0, big));
quotient.divide(Complex(0, big));
quotient.multiply(Complex(big, 0));
quotient.multiply(Complex(big, 0));
VERIFY_IS_EQUAL(quotient.value(), Complex(-small, -small));
}
// A subnormal factor, a subnormal partial product and a subnormal result all survive flush-to-zero, where the plain
// product reads or writes a zero.
template <typename RealScalar>
void check_scaled_product_flush_to_zero() {
const int minExponent = std::numeric_limits<RealScalar>::min_exponent;
const RealScalar subnormal = std::ldexp(RealScalar(3), minExponent - 8);
const RealScalar up = std::ldexp(RealScalar(1), 40);
const RealScalar down = std::ldexp(RealScalar(1), minExponent + 20);
const RealScalar expectedNormal = std::ldexp(RealScalar(3), minExponent + 72);
const RealScalar expectedSubnormal = std::ldexp(RealScalar(1), minExponent - 4);
forEachFlushToZeroMode([&](FlushToZeroMode) {
internal::scaled_product<RealScalar> fromSubnormal;
fromSubnormal.multiply(subnormal);
fromSubnormal.multiply(up);
fromSubnormal.multiply(up);
VERIFY(same_bits(fromSubnormal.value(), expectedNormal));
internal::scaled_product<RealScalar> toSubnormal;
toSubnormal.multiply(down);
toSubnormal.multiply(std::ldexp(RealScalar(1), -24));
VERIFY(same_bits(toSubnormal.value(), expectedSubnormal));
toSubnormal.multiply(up);
VERIFY(same_bits(toSubnormal.value(), std::ldexp(RealScalar(1), minExponent + 36)));
});
}
// Scalars without an exact frexp and ldexp multiply directly.
void check_scaled_product_selection() {
STATIC_CHECK(internal::has_scaled_product<float>::value);
STATIC_CHECK(internal::has_scaled_product<double>::value);
STATIC_CHECK(internal::has_scaled_product<long double>::value);
STATIC_CHECK(internal::has_scaled_product<std::complex<float>>::value);
STATIC_CHECK(internal::has_scaled_product<std::complex<double>>::value);
STATIC_CHECK(!internal::has_scaled_product<int>::value);
STATIC_CHECK(!internal::has_scaled_product<std::complex<int>>::value);
STATIC_CHECK(!internal::has_scaled_product<half>::value);
STATIC_CHECK(!internal::has_scaled_product<bfloat16>::value);
STATIC_CHECK(!internal::has_scaled_product<AnnoyingScalar>::value);
const Vector4i integers(2, -3, 5, 7);
VERIFY_IS_EQUAL(internal::scaled_prod(integers).value(), -210);
VERIFY_IS_EQUAL(internal::scaled_prod_squared(integers).value(), 44100);
internal::scaled_product<int> quotient = internal::scaled_prod(integers);
quotient.divide(-21);
VERIFY_IS_EQUAL(quotient.value(), 10);
VERIFY_IS_EQUAL(quotient.abs(), 10);
Matrix<half, 3, 1> halves;
halves << half(2.0f), half(-4.0f), half(0.5f);
VERIFY_IS_EQUAL(internal::scaled_prod(halves).value(), half(-4.0f));
VERIFY_IS_APPROX(float(internal::log_abs_prod(halves)), std::log(4.0f));
}
EIGEN_DECLARE_TEST(safe_scaling) {
CALL_SUBTEST(check_flush_to_zero_modes<float>());
CALL_SUBTEST(check_flush_to_zero_modes<double>());
CALL_SUBTEST(check_recover_flushed_scalar_max_coeff<float>());
CALL_SUBTEST(check_recover_flushed_scalar_max_coeff<double>());
CALL_SUBTEST(check_scale_binary_by_exponent<float>());
CALL_SUBTEST(check_scale_binary_by_exponent<double>());
CALL_SUBTEST(check_preserving_subnormals_fallbacks());
CALL_SUBTEST(check_identity_scaling<float>());
CALL_SUBTEST(check_identity_scaling<double>());
CALL_SUBTEST(check_identity_scaling<std::complex<float>>());
CALL_SUBTEST(check_identity_scaling<std::complex<double>>());
CALL_SUBTEST(check_power_of_two_scaling_factor<half>());
CALL_SUBTEST(check_power_of_two_scaling_factor<bfloat16>());
CALL_SUBTEST(check_power_of_two_scaling_factor<float>());
CALL_SUBTEST(check_power_of_two_scaling_factor<double>());
CALL_SUBTEST(check_power_of_two_scaling_factor<long double>());
CALL_SUBTEST(check_safe_scaling_special_values<half>());
CALL_SUBTEST(check_safe_scaling_special_values<bfloat16>());
CALL_SUBTEST(check_safe_scaling_special_values<float>());
CALL_SUBTEST(check_safe_scaling_special_values<double>());
CALL_SUBTEST(check_safe_scaling_special_value_frontends());
CALL_SUBTEST(check_arithmetic_safe_scaling_fallback());
CALL_SUBTEST(check_custom_scalar_scaling_exceptions());
CALL_SUBTEST(check_arithmetic_scaling_expression<float>());
CALL_SUBTEST(check_arithmetic_scaling_expression<double>());
CALL_SUBTEST(check_arithmetic_scaling_expression<std::complex<float>>());
CALL_SUBTEST(check_arithmetic_scaling_expression<std::complex<double>>());
CALL_SUBTEST(check_scale_binary_by_power_of_two<float>());
CALL_SUBTEST(check_scale_binary_by_power_of_two<double>());
CALL_SUBTEST(check_scale_binary_by_power_of_two_rounding<float>());
CALL_SUBTEST(check_scale_binary_by_power_of_two_rounding<double>());
CALL_SUBTEST(check_subnormal_preserving_scaling<float>());
CALL_SUBTEST(check_subnormal_preserving_scaling<std::complex<float>>());
CALL_SUBTEST(check_subnormal_preserving_scaling<double>());
CALL_SUBTEST(check_subnormal_preserving_scaling<std::complex<double>>());
CALL_SUBTEST(check_unscale_recovery_threshold<float>());
CALL_SUBTEST(check_unscale_recovery_threshold<double>());
CALL_SUBTEST(check_recover_flushed_max_coeff<float>());
CALL_SUBTEST(check_recover_flushed_max_coeff<std::complex<float>>());
CALL_SUBTEST(check_recover_flushed_max_coeff<double>());
CALL_SUBTEST(check_recover_flushed_max_coeff<std::complex<double>>());
for (int i = 0; i < g_repeat; ++i) {
CALL_SUBTEST(check_scaled_product_real<float>());
CALL_SUBTEST(check_scaled_product_real<double>());
CALL_SUBTEST(check_scaled_product_real<long double>());
CALL_SUBTEST(check_scaled_product_complex<float>());
CALL_SUBTEST(check_scaled_product_complex<double>());
}
CALL_SUBTEST(check_scaled_product_flush_to_zero<float>());
CALL_SUBTEST(check_scaled_product_flush_to_zero<double>());
CALL_SUBTEST(check_scaled_product_selection());
}