blob: 0a8a298bc9e414a39e1b3260bfc66dd9fbf503b2 [file] [edit]
// SPDX-FileCopyrightText: The Eigen Authors
// SPDX-License-Identifier: MPL-2.0
#include "main.h"
#include "guard_page_buffer.h"
#if defined(__GNUC__) && __GNUC__ >= 6
#pragma GCC diagnostic ignored "-Wignored-attributes"
#endif
template <typename Scalar, typename Packet = typename internal::packet_traits<Scalar>::type,
int Size = internal::unpacket_traits<Packet>::size>
void loaddup_boundary() {
constexpr int Count = (Size + 1) / 2;
const test::GuardPageBuffer buffer(Count * sizeof(Scalar));
Scalar* const data = buffer.place_at_end<Scalar>(Count);
for (Index i = 0; i < Count; ++i) data[i] = Scalar(i + 1);
Scalar actual[Size];
internal::pstoreu(actual, internal::ploaddup<Packet>(data));
for (Index i = 0; i < Size; ++i) {
const Index source = i / 2;
VERIFY_IS_EQUAL(actual[i], Scalar(source + 1));
}
}
template <typename Scalar>
void loaddup_packet_boundaries() {
using Packet = typename internal::packet_traits<Scalar>::type;
using Half = typename internal::unpacket_traits<Packet>::half;
using Quarter = typename internal::unpacket_traits<Half>::half;
loaddup_boundary<Scalar, Packet>();
EIGEN_IF_CONSTEXPR ((!internal::is_same<Packet, Half>::value)) {
loaddup_boundary<Scalar, Half>();
}
EIGEN_IF_CONSTEXPR ((!internal::is_same<Half, Quarter>::value)) {
loaddup_boundary<Scalar, Quarter>();
}
}
template <typename Real, int Rows, int Cols, int Order>
void mixed_product_loaddup_boundary() {
using Complex = std::complex<Real>;
using MatrixType = Matrix<Real, Rows, Cols, Order>;
const test::GuardPageBuffer buffer(Rows * Cols * sizeof(Real));
Map<MatrixType, Unaligned> lhs(buffer.place_at_end<Real>(Rows * Cols));
for (Index j = 0; j < Cols; ++j)
for (Index i = 0; i < Rows; ++i) lhs(i, j) = Real(i + j + 1);
Matrix<Complex, Cols, 1> rhs;
for (Index j = 0; j < Cols; ++j) rhs(j) = Complex(Real(j + 1), Real(1 - j));
const Matrix<Complex, Rows, 1> actual = lhs * rhs;
const Matrix<Complex, Rows, 1> lazy = lhs.lazyProduct(rhs);
for (Index i = 0; i < Rows; ++i) {
Complex expected(0);
for (Index j = 0; j < Cols; ++j) expected += lhs(i, j) * rhs(j);
VERIFY_IS_APPROX(actual(i), expected);
VERIFY_IS_APPROX(lazy(i), expected);
}
}
template <typename Real>
void mixed_product_loaddup_boundaries() {
mixed_product_loaddup_boundary<Real, 2, 2, ColMajor>();
mixed_product_loaddup_boundary<Real, 2, 2, RowMajor>();
mixed_product_loaddup_boundary<Real, 8, 17, ColMajor>();
mixed_product_loaddup_boundary<Real, 8, 17, RowMajor>();
}
EIGEN_DECLARE_TEST(packetmath_loaddup) {
CALL_SUBTEST_1(loaddup_packet_boundaries<float>());
CALL_SUBTEST_2(loaddup_packet_boundaries<double>());
CALL_SUBTEST_3(loaddup_packet_boundaries<int>());
CALL_SUBTEST_1(mixed_product_loaddup_boundaries<float>());
CALL_SUBTEST_2(mixed_product_loaddup_boundaries<double>());
#if defined(EIGEN_VECTORIZE_LSX) || defined(EIGEN_VECTORIZE_ALTIVEC)
CALL_SUBTEST_3(loaddup_boundary<numext::int8_t>());
CALL_SUBTEST_3(loaddup_boundary<numext::uint8_t>());
CALL_SUBTEST_3(loaddup_boundary<numext::int16_t>());
CALL_SUBTEST_3(loaddup_boundary<numext::uint16_t>());
#if defined(EIGEN_VECTORIZE_LSX)
CALL_SUBTEST_3(loaddup_boundary<numext::uint32_t>());
#endif
#if defined(EIGEN_VECTORIZE_ALTIVEC)
CALL_SUBTEST_3(loaddup_boundary<bfloat16>());
#endif
#endif
#if defined(EIGEN_VECTORIZE_RVV10)
// Explicit LMULs cover half packets even when the default packet uses a larger register group.
#define EIGEN_TEST_RVV_DUP(Scalar, Packet) \
loaddup_boundary<Scalar, internal::Packet, internal::unpacket_traits<internal::Packet>::size>()
CALL_SUBTEST_1((EIGEN_TEST_RVV_DUP(float, Packet1Xf)));
CALL_SUBTEST_1((EIGEN_TEST_RVV_DUP(float, Packet2Xf)));
CALL_SUBTEST_1((EIGEN_TEST_RVV_DUP(float, Packet4Xf)));
CALL_SUBTEST_2((EIGEN_TEST_RVV_DUP(double, Packet1Xd)));
CALL_SUBTEST_2((EIGEN_TEST_RVV_DUP(double, Packet2Xd)));
CALL_SUBTEST_2((EIGEN_TEST_RVV_DUP(double, Packet4Xd)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(numext::int32_t, Packet1Xi)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(numext::int32_t, Packet2Xi)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(numext::int32_t, Packet4Xi)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(numext::int64_t, Packet1Xl)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(numext::int64_t, Packet2Xl)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(numext::int64_t, Packet4Xl)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(numext::int16_t, Packet1Xs)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(numext::int16_t, Packet2Xs)));
#if defined(EIGEN_VECTORIZE_RVV10FP16)
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(half, Packet1Xh)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(half, Packet2Xh)));
#endif
#if defined(EIGEN_VECTORIZE_RVV10BF16)
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(bfloat16, Packet1Xbf)));
CALL_SUBTEST_3((EIGEN_TEST_RVV_DUP(bfloat16, Packet2Xbf)));
#endif
#undef EIGEN_TEST_RVV_DUP
#endif
}