blob: e46bea4af7de704845158aa64e7069e09dd8a2f0 [file] [edit]
// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2015 Gael Guennebaud <gael.guennebaud@inria.fr>
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
// SPDX-License-Identifier: MPL-2.0
#include "main.h"
#include "fp_control.h"
#include <Eigen/QR>
#include <iomanip>
#include <vector>
void check(bool b, bool ref) {
std::cout << b;
if (b == ref)
std::cout << " OK ";
else
std::cout << " BAD ";
}
template <typename T>
void check_inf_nan(bool dryrun) {
Matrix<T, Dynamic, 1> m(10);
m.setRandom();
m(3) = std::numeric_limits<T>::quiet_NaN();
if (dryrun) {
std::cout << "std::isfinite(" << m(3) << ") = ";
check((std::isfinite)(m(3)), false);
std::cout << " ; numext::isfinite = ";
check((numext::isfinite)(m(3)), false);
std::cout << "\n";
std::cout << "std::isinf(" << m(3) << ") = ";
check((std::isinf)(m(3)), false);
std::cout << " ; numext::isinf = ";
check((numext::isinf)(m(3)), false);
std::cout << "\n";
std::cout << "std::isnan(" << m(3) << ") = ";
check((std::isnan)(m(3)), true);
std::cout << " ; numext::isnan = ";
check((numext::isnan)(m(3)), true);
std::cout << "\n";
std::cout << "allFinite: ";
check(m.allFinite(), 0);
std::cout << "\n";
std::cout << "hasNaN: ";
check(m.hasNaN(), 1);
std::cout << "\n";
std::cout << "\n";
} else {
if ((std::isfinite)(m(3))) {
g_test_level = 1;
VERIFY(!(numext::isfinite)(m(3)));
g_test_level = 0;
}
if ((std::isinf)(m(3))) {
g_test_level = 1;
VERIFY(!(numext::isinf)(m(3)));
g_test_level = 0;
}
if (!(std::isnan)(m(3))) {
g_test_level = 1;
VERIFY((numext::isnan)(m(3)));
g_test_level = 0;
}
if ((std::isfinite)(m(3))) {
g_test_level = 1;
VERIFY(!m.allFinite());
g_test_level = 0;
}
if (!(std::isnan)(m(3))) {
g_test_level = 1;
VERIFY(m.hasNaN());
g_test_level = 0;
}
}
T hidden_zero = (std::numeric_limits<T>::min)() * (std::numeric_limits<T>::min)();
m(4) /= hidden_zero;
if (dryrun) {
std::cout << "std::isfinite(" << m(4) << ") = ";
check((std::isfinite)(m(4)), false);
std::cout << " ; numext::isfinite = ";
check((numext::isfinite)(m(4)), false);
std::cout << "\n";
std::cout << "std::isinf(" << m(4) << ") = ";
check((std::isinf)(m(4)), true);
std::cout << " ; numext::isinf = ";
check((numext::isinf)(m(4)), true);
std::cout << "\n";
std::cout << "std::isnan(" << m(4) << ") = ";
check((std::isnan)(m(4)), false);
std::cout << " ; numext::isnan = ";
check((numext::isnan)(m(4)), false);
std::cout << "\n";
std::cout << "allFinite: ";
check(m.allFinite(), 0);
std::cout << "\n";
std::cout << "hasNaN: ";
check(m.hasNaN(), 1);
std::cout << "\n";
std::cout << "\n";
} else {
if ((std::isfinite)(m(3))) {
g_test_level = 1;
VERIFY(!(numext::isfinite)(m(4)));
g_test_level = 0;
}
if (!(std::isinf)(m(3))) {
g_test_level = 1;
VERIFY((numext::isinf)(m(4)));
g_test_level = 0;
}
if ((std::isnan)(m(3))) {
g_test_level = 1;
VERIFY(!(numext::isnan)(m(4)));
g_test_level = 0;
}
if ((std::isfinite)(m(3))) {
g_test_level = 1;
VERIFY(!m.allFinite());
g_test_level = 0;
}
if (!(std::isnan)(m(3))) {
g_test_level = 1;
VERIFY(m.hasNaN());
g_test_level = 0;
}
}
m(3) = 0;
if (dryrun) {
std::cout << "std::isfinite(" << m(3) << ") = ";
check((std::isfinite)(m(3)), true);
std::cout << " ; numext::isfinite = ";
check((numext::isfinite)(m(3)), true);
std::cout << "\n";
std::cout << "std::isinf(" << m(3) << ") = ";
check((std::isinf)(m(3)), false);
std::cout << " ; numext::isinf = ";
check((numext::isinf)(m(3)), false);
std::cout << "\n";
std::cout << "std::isnan(" << m(3) << ") = ";
check((std::isnan)(m(3)), false);
std::cout << " ; numext::isnan = ";
check((numext::isnan)(m(3)), false);
std::cout << "\n";
std::cout << "allFinite: ";
check(m.allFinite(), 0);
std::cout << "\n";
std::cout << "hasNaN: ";
check(m.hasNaN(), 0);
std::cout << "\n";
std::cout << "\n\n";
} else {
if (!(std::isfinite)(m(3))) {
g_test_level = 1;
VERIFY((numext::isfinite)(m(3)));
g_test_level = 0;
}
if ((std::isinf)(m(3))) {
g_test_level = 1;
VERIFY(!(numext::isinf)(m(3)));
g_test_level = 0;
}
if ((std::isnan)(m(3))) {
g_test_level = 1;
VERIFY(!(numext::isnan)(m(3)));
g_test_level = 0;
}
if ((std::isfinite)(m(3))) {
g_test_level = 1;
VERIFY(!m.allFinite());
g_test_level = 0;
}
if ((std::isnan)(m(3))) {
g_test_level = 1;
VERIFY(!m.hasNaN());
g_test_level = 0;
}
}
}
template <typename RealScalar>
void check_complex_rowmajor_adjoint_product() {
using Scalar = std::complex<RealScalar>;
using RowMatrix = Matrix<Scalar, Dynamic, Dynamic, RowMajor>;
using ColMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>;
RowMatrix mat(2, 2);
mat << Scalar(1, 2), Scalar(3, -4), Scalar(-5, 6), Scalar(7, 8);
RowMatrix expected(2, 2);
expected << Scalar(66, 0), Scalar(8, -92), Scalar(8, 92), Scalar(138, 0);
const RowMatrix row_major_result = mat.adjoint() * mat;
const ColMatrix col_major_result = mat.adjoint() * mat;
VERIFY_IS_APPROX(mat.adjoint() * mat, expected);
VERIFY_IS_APPROX(row_major_result, expected);
VERIFY_IS_APPROX(col_major_result, expected);
}
template <typename RealScalar>
void check_complex_packet_arithmetic() {
using Scalar = std::complex<RealScalar>;
using Vector2 = Matrix<Scalar, 2, 1>;
Vector2 values;
values << Scalar(RealScalar(0.53645928880954319), RealScalar(-0.60489662966980218)),
Scalar(RealScalar(0.25774142970757641), RealScalar(0.10793998506041591));
Scalar divisor(RealScalar(1.6611441458336193), RealScalar(-0.21123424512127231));
Scalar factor(RealScalar(1.2499121678643004), RealScalar(0.36146968008699221));
Vector2 quotient = values / divisor;
Vector2 expected_quotient;
expected_quotient << values.coeff(0) / divisor, values.coeff(1) / divisor;
VERIFY_IS_APPROX(quotient, expected_quotient);
Vector2 inverse = values.array().inverse();
Vector2 expected_inverse;
expected_inverse << Scalar(RealScalar(1)) / values.coeff(0), Scalar(RealScalar(1)) / values.coeff(1);
VERIFY_IS_APPROX(inverse, expected_inverse);
Vector2 product = values * factor;
Vector2 expected_product;
expected_product << values.coeff(0) * factor, values.coeff(1) * factor;
VERIFY_IS_APPROX(product, expected_product);
Vector2 conjugate_product = values.conjugate().cwiseProduct(Vector2::Constant(factor));
Vector2 expected_conjugate_product;
expected_conjugate_product << numext::conj(values.coeff(0)) * factor, numext::conj(values.coeff(1)) * factor;
VERIFY_IS_APPROX(conjugate_product, expected_conjugate_product);
// numext::divide guards against intermediate overflow (c^2 + d^2) near max representable bounds.
// The divisor stays below max_val / 4: above that its reciprocal is subnormal, and GCC's
// -ffast-math vector division (rcpps plus a Newton step) flushes it to zero.
const RealScalar max_val = (std::numeric_limits<RealScalar>::max)();
const RealScalar big_val = max_val / RealScalar(16);
const Scalar big_num(RealScalar(0.9) * big_val, RealScalar(0));
const Scalar big_denom(big_val, RealScalar(0));
const Scalar safe_div_result = numext::divide(big_num, big_denom);
VERIFY((numext::isfinite)(safe_div_result.real()));
VERIFY_IS_APPROX(safe_div_result, Scalar(RealScalar(0.9), RealScalar(0)));
VERIFY_IS_APPROX(numext::divide(RealScalar(0.9) * max_val, max_val), RealScalar(0.9));
// Verify vectorized expression division (vec / denom) on numbers that would cause
// intermediate overflow (c^2 + d^2 > max) in naive complex division.
const RealScalar intermediate_scale = numext::sqrt(max_val) * RealScalar(2);
const Scalar intermediate_num(RealScalar(0.9) * intermediate_scale, RealScalar(0));
const Scalar intermediate_denom(intermediate_scale, RealScalar(0));
Matrix<Scalar, Dynamic, 1> vec(8);
vec.fill(intermediate_num);
Matrix<Scalar, Dynamic, 1> vec_div = vec / intermediate_denom;
for (Index i = 0; i < vec.size(); ++i) {
VERIFY((numext::isfinite)(vec_div(i).real()));
VERIFY_IS_APPROX(vec_div(i), Scalar(RealScalar(0.9), RealScalar(0)));
}
}
template <typename RealScalar>
void check_complex_packet_math_functions() {
using Scalar = std::complex<RealScalar>;
using Vector4 = Matrix<Scalar, 4, 1>;
Vector4 values;
values << Scalar(RealScalar(0.53645928880954319), RealScalar(-0.60489662966980218)),
Scalar(RealScalar(0.25774142970757641), RealScalar(0.10793998506041591)),
Scalar(RealScalar(-0.83239073966000054), RealScalar(0.026801457199547407)),
Scalar(RealScalar(1.6611441458336193), RealScalar(-0.21123424512127231));
Vector4 sqrt_result = values.array().sqrt();
Vector4 log_result = values.array().log();
Vector4 exp_result = values.array().exp();
Vector4 expected_sqrt, expected_log, expected_exp;
for (Index i = 0; i < values.size(); ++i) {
expected_sqrt[i] = std::sqrt(values[i]);
expected_log[i] = std::log(values[i]);
expected_exp[i] = std::exp(values[i]);
}
VERIFY_IS_APPROX(sqrt_result, expected_sqrt);
VERIFY_IS_APPROX(log_result, expected_log);
VERIFY_IS_APPROX(exp_result, expected_exp);
}
template <typename RealScalar>
void check_complex_householder_qr() {
using Scalar = std::complex<RealScalar>;
using Matrix32 = Matrix<Scalar, 3, 2>;
using Matrix22 = Matrix<Scalar, 2, 2>;
Matrix32 mat;
mat << Scalar(RealScalar(0.59688070592806186), RealScalar(-0.21123424512127231)),
Scalar(RealScalar(0.83239073966000054), RealScalar(0.026801457199547407)),
Scalar(RealScalar(0.53645928880954319), RealScalar(-0.60489662966980218)),
Scalar(RealScalar(0.21393694076781777), RealScalar(0.43459380694554106)),
Scalar(RealScalar(0.25774142970757641), RealScalar(0.10793998506041591)),
Scalar(RealScalar(0.60835279200704262), RealScalar(-0.51422689790671194));
HouseholderQR<Matrix32> qr(mat);
Matrix32 q = qr.householderQ() * Matrix32::Identity();
Matrix22 r = qr.matrixQR().template topRows<2>().template triangularView<Upper>();
VERIFY_IS_APPROX(mat, q * r);
}
// The complex self-adjoint 1-norm takes sqrt(re^2 + im^2) in packets and falls back to the scalar
// abs when a component is too large to square. The decision must not depend on an infinity or
// NaN surviving fast-math: an approximate packet sqrt can turn the overflow into a NaN that a
// maximum then discards. Large entries both on the diagonal and inside a packet of a column.
template <typename RealScalar>
void check_complex_selfadjoint_l1norm() {
using Scalar = std::complex<RealScalar>;
RealScalar big = numext::sqrt(NumTraits<RealScalar>::highest()) * RealScalar(1e3);
Matrix<Scalar, 8, 8> m = Matrix<Scalar, 8, 8>::Identity() * big;
VERIFY_IS_APPROX(m.template selfadjointView<Lower>().l1Norm(), big);
VERIFY_IS_APPROX(m.template selfadjointView<Upper>().l1Norm(), big);
m.setIdentity();
m(2, 0) = m(0, 2) = Scalar(big, big);
RealScalar expected = numext::abs(Scalar(big, big)) + RealScalar(1);
VERIFY_IS_APPROX(m.template selfadjointView<Lower>().l1Norm(), expected);
VERIFY_IS_APPROX(m.template selfadjointView<Upper>().l1Norm(), expected);
}
template <typename RealScalar>
void check_complex_fastmath() {
check_complex_rowmajor_adjoint_product<RealScalar>();
check_complex_packet_arithmetic<RealScalar>();
check_complex_packet_math_functions<RealScalar>();
check_complex_householder_qr<RealScalar>();
check_complex_selfadjoint_l1norm<RealScalar>();
}
// The packet implementations of these functions manipulate signs of non-zero values through a
// -0.0 bitmask. Under fast-math flags compilers consider -0.0 and +0.0 interchangeable and may
// substitute one such constant for the other (GCC's value numbering does this on RISC-V), which
// silently zeroes the mask unless it is constructed from integer bits (see psignmask and
// https://gitlab.com/libeigen/eigen/-/merge_requests/2698). Sign handling of *non-zero* inputs
// and outputs is not relaxed by fast-math, so these checks must hold.
template <typename Scalar>
void check_sign_dependent_functions() {
using ArrayType = Array<Scalar, Dynamic, 1>;
const Index n = 64;
ArrayType ya(n), xa(n);
for (Index i = 0; i < n; ++i) {
Scalar s = Scalar(1) + Scalar(i) / Scalar(n);
// All four quadrants.
ya[i] = (i & 1) ? s : -s;
xa[i] = (i & 2) ? Scalar(2) * s : Scalar(-2) * s;
}
// Cover the |x| == |y| special path of patan2 in all four quadrants.
ya[0] = Scalar(1), xa[0] = Scalar(1);
ya[1] = Scalar(1), xa[1] = Scalar(-1);
ya[2] = Scalar(-1), xa[2] = Scalar(1);
ya[3] = Scalar(-1), xa[3] = Scalar(-1);
const ArrayType atan2_result = ya.atan2(xa);
for (Index i = 0; i < n; ++i) {
VERIFY_IS_APPROX(atan2_result[i], std::atan2(ya[i], xa[i]));
}
// Mixed-sign inputs, no exact zeros, |w| up to ~3 to hit the small- and large-|x| branches.
const ArrayType w = ArrayType::LinSpaced(n, Scalar(-3), Scalar(3)) + Scalar(0.017);
const ArrayType w_unit = w / Scalar(3.2); // in (-1, 1) for atanh
const ArrayType atan_result = w.atan();
const ArrayType sinh_result = w.sinh();
const ArrayType tanh_result = w.tanh();
const ArrayType asinh_result = w.asinh();
const ArrayType atanh_result = w_unit.atanh();
const ArrayType cbrt_result = w.cbrt();
for (Index i = 0; i < n; ++i) {
VERIFY_IS_APPROX(atan_result[i], std::atan(w[i]));
VERIFY_IS_APPROX(sinh_result[i], std::sinh(w[i]));
VERIFY_IS_APPROX(tanh_result[i], std::tanh(w[i]));
VERIFY_IS_APPROX(asinh_result[i], std::asinh(w[i]));
VERIFY_IS_APPROX(atanh_result[i], std::atanh(w_unit[i]));
VERIFY_IS_APPROX(cbrt_result[i], std::cbrt(w[i]));
}
}
// Packet exp, expm1, log1p, asinh and acosh under -ffast-math, against exact references. A wrong value, not isfinite or
// isnan (which fast-math folds), shows an overflow or a cancellation. The inputs and the C library references go
// through volatiles: fast-math folds calls on constants and rewrites acosh and the like, even inside std::acosh.
using LongDoubleFunction = long double (*)(long double);
// Whether v is finite, from its bits: fast-math folds isfinite and compares as if a NaN or an infinity never occurred.
template <typename Scalar>
bool has_finite_bits(Scalar v) {
using Bits = typename numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type;
constexpr int kMantissaBits = NumTraits<Scalar>::digits() - 1;
constexpr int kExponentBits = int(8 * sizeof(Scalar)) - 1 - kMantissaBits;
const Bits exponent_mask = static_cast<Bits>(((Bits(1) << kExponentBits) - 1) << kMantissaBits);
return (numext::bit_cast<Bits>(v) & exponent_mask) != exponent_mask;
}
template <typename Scalar, typename Func>
bool check_fastmath_accuracy(const char* name, const std::vector<Scalar>& values, Func func,
LongDoubleFunction reference) {
LongDoubleFunction volatile ref = reference;
using ArrayType = Array<Scalar, Dynamic, 1>;
// Long enough for several packets of any size and a remainder.
const Index n = 3 * 64 + 5;
// The kernels are within 2 ulp, and the limit leaves room for the reciprocal approximation that fast-math divides
// by.
const long double tolerance = 4 * static_cast<long double>(NumTraits<Scalar>::epsilon());
ArrayType x(n);
for (Index i = 0; i < n; ++i) {
const volatile Scalar* v = &values[static_cast<size_t>(i) % values.size()];
x[i] = *v;
}
const ArrayType y = func(x);
bool passed = true;
for (Index i = 0; i < n; ++i) {
const long double expected = ref(static_cast<long double>(x[i]));
const long double error = numext::abs(static_cast<long double>(y[i]) - expected);
// An infinite reference would make the comparison below true for every finite y.
if (!has_finite_bits(static_cast<Scalar>(expected)) || !has_finite_bits(y[i]) ||
!(error <= tolerance * numext::abs(expected))) {
passed = false;
if (static_cast<size_t>(i) < values.size()) {
std::cerr << std::setprecision(std::numeric_limits<Scalar>::max_digits10) << name << "(" << x[i]
<< ") = " << y[i] << ", expected " << static_cast<Scalar>(expected) << "\n";
}
}
}
return passed;
}
template <typename Scalar>
void check_exp_family_accuracy() {
using ArrayType = Array<Scalar, Dynamic, 1>;
const bool is_float = std::is_same<Scalar, float>::value;
// Small and moderate arguments, where expm1, log1p and asinh rely on the low bits of fl(1 + x).
const Scalar tiny = is_float ? Scalar(1e-7) : Scalar(1e-10);
const std::vector<Scalar> small_values = {tiny, -tiny, Scalar(1e-3), Scalar(-1e-3),
Scalar(0.1), Scalar(0.7), Scalar(3)};
// The largest finite results of exp and expm1 need a scale 2^m with m at the top of the exponent range.
const Scalar exp_edge = is_float ? Scalar(88.38) : Scalar(709.1);
const std::vector<Scalar> large_values = {
exp_edge, is_float ? Scalar(88.5) : Scalar(709.5), is_float ? Scalar(88.72) : Scalar(709.78), Scalar(10),
Scalar(-10), is_float ? Scalar(-80) : Scalar(-700)};
// Arguments up to the largest finite value, where asinh and acosh switch to log(2 x) without squaring x.
const std::vector<Scalar> huge_values = {is_float ? Scalar(1e10) : Scalar(1e150),
is_float ? Scalar(1e30) : Scalar(1e300),
is_float ? Scalar(1e38) : Scalar(1e307), NumTraits<Scalar>::highest()};
auto exp_fn = [](const ArrayType& x) -> ArrayType { return x.exp(); };
LongDoubleFunction exp_ref = ::expl;
auto expm1_fn = [](const ArrayType& x) -> ArrayType { return x.expm1(); };
LongDoubleFunction expm1_ref = ::expm1l;
auto log1p_fn = [](const ArrayType& x) -> ArrayType { return x.log1p(); };
LongDoubleFunction log1p_ref = ::log1pl;
auto asinh_fn = [](const ArrayType& x) -> ArrayType { return x.asinh(); };
LongDoubleFunction asinh_ref = ::asinhl;
auto acosh_fn = [](const ArrayType& x) -> ArrayType { return x.acosh(); };
LongDoubleFunction acosh_ref = ::acoshl;
auto sinh_fn = [](const ArrayType& x) -> ArrayType { return x.sinh(); };
LongDoubleFunction sinh_ref = ::sinhl;
auto cosh_fn = [](const ArrayType& x) -> ArrayType { return x.cosh(); };
LongDoubleFunction cosh_ref = ::coshl;
// Every call runs, so that one failure report lists all the functions that are wrong.
bool passed = true;
passed &= check_fastmath_accuracy("exp", small_values, exp_fn, exp_ref);
passed &= check_fastmath_accuracy("exp", large_values, exp_fn, exp_ref);
passed &= check_fastmath_accuracy("expm1", small_values, expm1_fn, expm1_ref);
passed &= check_fastmath_accuracy("expm1", large_values, expm1_fn, expm1_ref);
passed &= check_fastmath_accuracy("log1p", small_values, log1p_fn, log1p_ref);
passed &= check_fastmath_accuracy("log1p", huge_values, log1p_fn, log1p_ref);
passed &= check_fastmath_accuracy("asinh", small_values, asinh_fn, asinh_ref);
passed &= check_fastmath_accuracy("asinh", huge_values, asinh_fn, asinh_ref);
std::vector<Scalar> negative_huge_values;
for (Scalar v : huge_values) negative_huge_values.push_back(-v);
passed &= check_fastmath_accuracy("asinh", negative_huge_values, asinh_fn, asinh_ref);
// acosh is defined for x >= 1: x = 1 + tiny, 1 + 1e-3, ... and the huge values.
std::vector<Scalar> acosh_values = huge_values;
for (Scalar v : small_values) {
if (v > Scalar(0)) acosh_values.push_back(Scalar(1) + v);
}
passed &= check_fastmath_accuracy("acosh", acosh_values, acosh_fn, acosh_ref);
// sinh and cosh scale exp(r) by 2^(n - 1) from the same reduction as expm1, and are finite up to ln(2 highest).
std::vector<Scalar> hyperbolic_values = large_values;
// sinhl and coshl overflow at 710.3 where long double has the range of double (MSVC, Apple arm64).
if (std::numeric_limits<long double>::max_exponent > std::numeric_limits<Scalar>::max_exponent) {
hyperbolic_values.push_back(is_float ? Scalar(89.2) : Scalar(710.3));
hyperbolic_values.push_back(is_float ? Scalar(-89.2) : Scalar(-710.3));
}
hyperbolic_values.push_back(Scalar(-1.5));
passed &= check_fastmath_accuracy("sinh", hyperbolic_values, sinh_fn, sinh_ref);
passed &= check_fastmath_accuracy("cosh", hyperbolic_values, cosh_fn, cosh_ref);
VERIFY(passed);
}
// For x = 1000, the packet logistic computes exp(x) = inf, which it must recognize and map to 1, though fast-math
// deletes a comparison with an infinity literal.
long double logistic_reference(long double x) { return 1 / (1 + ::expl(-x)); }
template <typename Scalar>
void check_logistic_overflow() {
using ArrayType = Array<Scalar, Dynamic, 1>;
const std::vector<Scalar> values = {Scalar(1000), Scalar(-5), Scalar(100), Scalar(-50), Scalar(0.5)};
VERIFY(check_fastmath_accuracy(
"logistic", values, [](const ArrayType& x) -> ArrayType { return x.logistic(); }, logistic_reference));
}
// Finite arguments of sqrt and rsqrt, whose Newton steps compare the argument with an infinity.
long double rsqrt_reference(long double x) { return 1 / ::sqrtl(x); }
// bfloat16 divides in float, where -ffast-math may compute 1 / (1 + e) by an approximation that flushes it to 0 once
// 1 + e exceeds 2^126: logistic(88) has e = exp(88) = 1.6e38 and is 1.
void check_logistic_bfloat16() {
const float values[] = {88, 89, 100, 30, -30, 0, 50, -50};
const int n = 37;
Array<bfloat16, Dynamic, 1> x(n);
for (int i = 0; i < n; ++i) {
const volatile float* v = &values[i % 8];
x[i] = bfloat16(*v);
}
const Array<bfloat16, Dynamic, 1> y = x.logistic();
LongDoubleFunction volatile reference = logistic_reference;
for (int i = 0; i < n; ++i) {
const long double expected = reference(static_cast<long double>(static_cast<float>(x[i])));
const long double tolerance = 2 * static_cast<long double>(NumTraits<bfloat16>::epsilon());
VERIFY(has_finite_bits(static_cast<float>(y[i])));
VERIFY(numext::abs(static_cast<long double>(static_cast<float>(y[i])) - expected) <= tolerance * expected);
}
}
template <typename Scalar>
void check_sqrt_rsqrt_finite() {
using ArrayType = Array<Scalar, Dynamic, 1>;
const std::vector<Scalar> values = {Scalar(4), Scalar(9), Scalar(2), Scalar(1e-3), Scalar(7.5), Scalar(1e20)};
bool passed = true;
passed &= check_fastmath_accuracy(
"sqrt", values, [](const ArrayType& x) -> ArrayType { return x.sqrt(); }, ::sqrtl);
passed &= check_fastmath_accuracy(
"rsqrt", values, [](const ArrayType& x) -> ArrayType { return x.rsqrt(); }, rsqrt_reference);
VERIFY(passed);
}
// exp(x + iy) with exp(x) above the largest finite value and cos(y) or sin(y) small enough that the component is
// finite: the packet code splits exp(x) = 2^n exp(r) for those, and finds them by the overflow of exp(x), which
// fast-math must not hide.
template <typename Scalar>
void check_complex_exp_overflow() {
using Complex = std::complex<Scalar>;
const bool is_float = std::is_same<Scalar, float>::value;
// Real parts at x_overflow of pexp_complex, the bound of the split, which lies 4 epsilon below ln(highest), and next
// to it; ln(highest), where exp(x) is the largest finite value; and larger ones. Up to x_overflow exp(x) is computed
// by the packet exp.
using Bits = typename numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type;
const Scalar x_overflow = Scalar(NumTraits<Scalar>::max_exponent()) * Scalar(EIGEN_LN2) *
(Scalar(1) - Scalar(4) * NumTraits<Scalar>::epsilon());
const Bits x_overflow_bits = numext::bit_cast<Bits>(x_overflow);
const std::vector<Scalar> x_values = {
is_float ? Scalar(89) : Scalar(710),
is_float ? Scalar(92) : Scalar(720),
is_float ? Scalar(95) : Scalar(735),
numext::bit_cast<Scalar>(Bits(x_overflow_bits - 1)),
x_overflow,
numext::bit_cast<Scalar>(Bits(x_overflow_bits + 1)),
static_cast<Scalar>(::logl(static_cast<long double>(NumTraits<Scalar>::highest())))};
const Scalar y_values[] = {Scalar(1.5707963267948966), Scalar(3.141592653589793), Scalar(-1.5707963267948966),
Scalar(0.7)};
const Index n = 4 * static_cast<Index>(x_values.size());
Array<Complex, Dynamic, 1> z(n);
for (Index i = 0; i < n; ++i) {
const volatile Scalar* x = &x_values[static_cast<size_t>(i / 4)];
const volatile Scalar* y = &y_values[i % 4];
const Scalar re = *x, im = *y; // std::complex takes const Scalar&, which cannot bind to a volatile lvalue.
z[i] = Complex(re, im);
}
const Array<Complex, Dynamic, 1> w = z.exp();
// The reference is exp(x) cos(y) = 2^n m with m = exp(r) cos(y), x = n ln2 + r, |r| <= ln2 / 2, in double: exp(x)
// itself overflows, and long double is no wider than double with MSVC and on Apple arm64. The constants are those of
// fdlibm's exp: ln2_hi has 32 trailing zero bits, so n ln2_hi is exact, and the first difference goes through a
// volatile because fast-math fuses n ln2_hi + n ln2_lo into the rounded n (ln2_hi + ln2_lo).
const volatile double ln2_hi = 6.93147180369123816490e-01, ln2_lo = 1.90821492927058770002e-10;
const double highest = static_cast<double>(NumTraits<Scalar>::highest());
const double tolerance = 8 * static_cast<double>(NumTraits<Scalar>::epsilon());
for (Index i = 0; i < n; ++i) {
const double x = static_cast<double>(z[i].real()), y = static_cast<double>(z[i].imag());
const int exponent = static_cast<int>(std::floor(x / EIGEN_LN2 + 0.5));
const volatile double reduced_hi = x - exponent * ln2_hi;
const double r = reduced_hi - exponent * ln2_lo;
const double m = std::exp(r);
const double mantissa[2] = {m * std::cos(y), m * std::sin(y)};
const Scalar actual[2] = {w[i].real(), w[i].imag()};
for (int k = 0; k < 2; ++k) {
// A component above the largest finite value is infinite, which fast-math cannot be asked about. Compared before
// scaling, so that no infinity occurs.
if (numext::abs(mantissa[k]) <= std::ldexp(highest, -exponent)) {
const double expected = std::ldexp(mantissa[k], exponent);
VERIFY(has_finite_bits(actual[k]));
VERIFY(numext::abs(static_cast<double>(actual[k]) - expected) <= tolerance * numext::abs(expected));
}
}
}
}
template <typename Scalar>
EIGEN_DONT_INLINE Scalar fastmath_scalar_sign(const Scalar& z) {
return numext::sign(z);
}
// The scalar numext::sign of z with max(|Re z|, |Im z|) above highest / 2 tests the components, not an inf |z|, and
// divides by m / 4 behind a barrier, because -ffast-math turns z / m into z * (1 / m) and 1 / m is flushed there. The
// operands come from a volatile scale, so that nothing is folded at compile time, and subnormals are flushed.
// Array::sign() is not checked: its packet path is wrong near overflow under -ffast-math. Only m >= 0.6 max is covered;
// the scalar result under -ffast-math can still be 0 or NaN for sqrt(max) < |z| <= max / 2 and for |z| < sqrt(min).
template <typename RealScalar>
void check_complex_sign_overflow() {
using Scalar = std::complex<RealScalar>;
// Subnormals are flushed, as in a program linked with -ffast-math, which a test linked without it would not do.
ScopedFlushToZero flush_to_zero(FlushToZeroMode::Both);
volatile RealScalar unit = RealScalar(1);
const RealScalar highest = (std::numeric_limits<RealScalar>::max)() * unit;
const RealScalar h = RealScalar(0.75) * highest;
// The quotient, the square root and the final quotient round at most 0.5 ulp each (about 1.5 eps together), and the
// constant sqrt_half rounds 0.5 ulp more; 8 eps leaves margin. Not a measured maximum.
const RealScalar tol = RealScalar(8) * NumTraits<RealScalar>::epsilon();
const RealScalar sqrt_half = RealScalar(0.70710678118654752440);
// A NaN result must fail: under -ffast-math a comparison with a NaN is not reliable, but the bits of a nonnegative
// float order like its value, and those of a NaN lie above every finite one.
using Bits = typename numext::get_integer_by_size<sizeof(RealScalar)>::unsigned_type;
const auto close = [&](const Scalar& x, const Scalar& y) {
const RealScalar bound = tol * numext::abs(y);
return numext::bit_cast<Bits>(numext::abs(numext::real(x) - numext::real(y))) <= numext::bit_cast<Bits>(bound) &&
numext::bit_cast<Bits>(numext::abs(numext::imag(x) - numext::imag(y))) <= numext::bit_cast<Bits>(bound);
};
VERIFY(close(fastmath_scalar_sign(Scalar(highest, highest)), Scalar(sqrt_half, sqrt_half)));
VERIFY(close(fastmath_scalar_sign(Scalar(-highest, highest)), Scalar(-sqrt_half, sqrt_half)));
VERIFY(close(fastmath_scalar_sign(Scalar(h, -h)), Scalar(sqrt_half, -sqrt_half)));
VERIFY(close(fastmath_scalar_sign(Scalar(RealScalar(0.6) * highest, unit)), Scalar(RealScalar(1), RealScalar(0))));
}
EIGEN_DECLARE_TEST(fastmath) {
CALL_SUBTEST_14(check_complex_sign_overflow<float>());
CALL_SUBTEST_15(check_complex_sign_overflow<double>());
std::cout << "*** float *** \n\n";
check_inf_nan<float>(true);
std::cout << "*** double ***\n\n";
check_inf_nan<double>(true);
std::cout << "*** long double *** \n\n";
check_inf_nan<long double>(true);
check_inf_nan<float>(false);
check_inf_nan<double>(false);
check_inf_nan<long double>(false);
CALL_SUBTEST_1(check_complex_fastmath<float>());
CALL_SUBTEST_2(check_complex_fastmath<double>());
CALL_SUBTEST_3(check_sign_dependent_functions<float>());
CALL_SUBTEST_4(check_sign_dependent_functions<double>());
CALL_SUBTEST_5(check_exp_family_accuracy<float>());
CALL_SUBTEST_6(check_exp_family_accuracy<double>());
CALL_SUBTEST_7(check_sqrt_rsqrt_finite<float>());
CALL_SUBTEST_8(check_sqrt_rsqrt_finite<double>());
CALL_SUBTEST_9(check_logistic_overflow<float>());
CALL_SUBTEST_10(check_logistic_overflow<double>());
CALL_SUBTEST_11(check_logistic_bfloat16());
CALL_SUBTEST_12(check_complex_exp_overflow<float>());
CALL_SUBTEST_13(check_complex_exp_overflow<double>());
}