blob: 8a6a7676a2f2f8190f4ba406c73cfd401edf242f [file] [edit]
// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
// SPDX-FileCopyrightText: The Eigen Authors
// SPDX-License-Identifier: MPL-2.0
// SVE packet types are fixed-length vectors of exactly EIGEN_ARM64_SVE_VL bits. Running such a
// binary where the hardware vector length differs makes every packet type a lie and lets the
// rest of the suite pass vacuously. qemu is the common way in: above 512 bits it keeps the
// default 512-bit task vector length, so `-cpu max,sve2048=on` alone runs at 512 unless
// `sve-default-vector-length=<bytes>` is also given.
// The backend raises Eigen's own default to fit its blocked kernel panels, but an explicit limit
// is the caller's policy and must survive. 1 is below the requirement at every vector length, so
// the assertion is never vacuous, and nothing in this file allocates a fixed-size Eigen object.
#define EIGEN_STACK_ALLOCATION_LIMIT 1
#include "main.h"
static_assert(EIGEN_STACK_ALLOCATION_LIMIT == 1, "SVE must preserve an explicit stack allocation limit");
// Registered only under EIGEN_TEST_SVE (test/CMakeLists.txt). Without an SVE -march this would
// compile against NEON and trivially pass.
#if !defined(EIGEN_VECTORIZE_SVE)
#error \
"sve_vector_length requires the SVE backend. Build with an SVE -march (e.g. " \
"-march=armv8.2-a+sve) and -DEIGEN_TEST_SVE=ON, which supplies -DEIGEN_ARM64_USE_SVE and " \
"-msve-vector-bits."
#endif
// Must stay inline asm: under -msve-vector-bits the compiler folds svcntb() to the compile-time
// constant, making the comparison below a tautology that holds on a mismatched machine.
static EIGEN_DONT_INLINE int runtime_vector_length_bits() {
numext::uint64_t vl_bytes;
asm("rdvl %0, #1" : "=r"(vl_bytes));
return int(vl_bytes) * CHAR_BIT;
}
void check_runtime_vector_length() { VERIFY_IS_EQUAL(runtime_vector_length_bits(), int(EIGEN_ARM64_SVE_VL)); }
// Cross-checked against RDVL rather than svcntw()/svcntd(), which fold as above.
void check_packet_sizes() {
const int vl = runtime_vector_length_bits();
VERIFY_IS_EQUAL(int(internal::packet_traits<numext::int32_t>::size), vl / (8 * int(sizeof(numext::int32_t))));
VERIFY_IS_EQUAL(int(internal::packet_traits<float>::size), vl / (8 * int(sizeof(float))));
VERIFY_IS_EQUAL(int(internal::packet_traits<double>::size), vl / (8 * int(sizeof(double))));
}
// GCC can lose lanes in full-register constructors of at least eight elements when the even or odd half has
// leading constants followed by nonconstant elements. SLP can produce these from ordinary scalar stores.
EIGEN_DONT_INLINE void scale_zero_prefixed_values(float* dst, float a, float b, float c, float d, float e, float f,
float g, float h, float i, float j, float scale) {
dst[0] = dst[1] = dst[2] = dst[3] = dst[4] = dst[5] = 0 * scale;
dst[6] = a * scale;
dst[7] = b * scale;
dst[8] = c * scale;
dst[9] = d * scale;
dst[10] = e * scale;
dst[11] = f * scale;
dst[12] = g * scale;
dst[13] = h * scale;
dst[14] = i * scale;
dst[15] = j * scale;
}
template <typename Scalar, int Prefix>
EIGEN_DONT_INLINE void scale_constant_prefixed_values(Scalar* dst, Scalar a, Scalar b, Scalar c, Scalar d, Scalar e,
Scalar scale) {
dst[0] = Scalar(Prefix) * scale;
dst[1] = Scalar(Prefix == 0 ? 0 : Prefix + 1) * scale;
dst[2] = Scalar(Prefix == 0 ? 0 : Prefix + 2) * scale;
dst[3] = a * scale;
dst[4] = b * scale;
dst[5] = c * scale;
dst[6] = d * scale;
dst[7] = e * scale;
}
void check_scaled_zero_prefix() {
float values[10], result[16];
for (float& value : values) value = internal::random<float>(1, 2);
const float scale = internal::random<float>(1, 2);
scale_zero_prefixed_values(result, values[0], values[1], values[2], values[3], values[4], values[5], values[6],
values[7], values[8], values[9], scale);
for (int i = 0; i < 16; ++i) VERIFY_IS_EQUAL(result[i], i < 6 ? 0.0f : values[i - 6] * scale);
}
template <typename Scalar, int Prefix>
void check_scaled_constant_prefix() {
Scalar values[5], result[8];
for (Scalar& value : values) value = internal::random<Scalar>(1, 2);
const Scalar scale = internal::random<Scalar>(1, 2);
scale_constant_prefixed_values<Scalar, Prefix>(result, values[0], values[1], values[2], values[3], values[4], scale);
for (int i = 0; i < 8; ++i) {
const Scalar expected = i < 3 ? Scalar(Prefix == 0 ? 0 : Prefix + i) : values[i - 3];
VERIFY_IS_EQUAL(result[i], expected * scale);
}
}
// The opaque barriers keep the store/reload from being constant-folded away; on a narrower
// machine the tail of the buffer then keeps its sentinel.
template <typename Scalar, typename Packet>
void check_full_width_store() {
constexpr int PacketSize = internal::unpacket_traits<Packet>::size;
EIGEN_ALIGN_MAX Scalar buffer[PacketSize];
Scalar value = Scalar(1);
Scalar* p = buffer;
asm volatile("" : "+r"(p) : : "memory");
for (int i = 0; i < PacketSize; ++i) p[i] = Scalar(-12345);
asm volatile("" : "+r"(p), "+r"(value) : : "memory");
internal::pstore(p, internal::pset1<Packet>(value));
asm volatile("" : "+r"(p) : : "memory");
for (int i = 0; i < PacketSize; ++i) {
VERIFY_IS_EQUAL(p[i], Scalar(1));
}
}
EIGEN_DECLARE_TEST(sve_vector_length) {
CALL_SUBTEST(check_runtime_vector_length());
CALL_SUBTEST(check_packet_sizes());
CALL_SUBTEST(check_scaled_zero_prefix());
CALL_SUBTEST((check_scaled_constant_prefix<float, 0>()));
CALL_SUBTEST((check_scaled_constant_prefix<float, 1>()));
CALL_SUBTEST((check_scaled_constant_prefix<double, 0>()));
CALL_SUBTEST((check_scaled_constant_prefix<double, 1>()));
CALL_SUBTEST((check_full_width_store<numext::int32_t, internal::PacketXi>()));
CALL_SUBTEST((check_full_width_store<float, internal::PacketXf>()));
CALL_SUBTEST((check_full_width_store<double, internal::PacketXd>()));
}