Core: Read the cache topology from sysfs on Linux libeigen/eigen!2910 Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
diff --git a/Eigen/Core b/Eigen/Core index cca76e4..e6a4bb5 100644 --- a/Eigen/Core +++ b/Eigen/Core
@@ -126,6 +126,10 @@ // Required for querying cache sizes on Linux and macOS. #if EIGEN_OS_LINUX +#if !defined(EIGEN_NO_CPU_CACHE_SYSFS) +#include <cstdio> +#include <sched.h> +#endif #include <unistd.h> #elif EIGEN_OS_MAC #include <sys/types.h>
diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h index e24d5ca..96efe53 100644 --- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
@@ -81,9 +81,9 @@ /** \internal */ struct CacheSizes { - CacheSizes() : m_l1(-1), m_l2(-1), m_l3(-1) { + CacheSizes() : m_l1(-1), m_l2(-1), m_l3(-1), m_l3_per_cpu(0) { std::ptrdiff_t l1CacheSize, l2CacheSize, l3CacheSize; - queryCacheSizes(l1CacheSize, l2CacheSize, l3CacheSize); + queryCacheSizes(l1CacheSize, l2CacheSize, l3CacheSize, m_l3_per_cpu); m_l1 = manage_caching_sizes_helper(l1CacheSize, defaultL1CacheSize); m_l2 = manage_caching_sizes_helper(l2CacheSize, defaultL2CacheSize); m_l3 = manage_caching_sizes_helper(l3CacheSize, defaultL3CacheSize); @@ -92,10 +92,14 @@ std::ptrdiff_t m_l1; std::ptrdiff_t m_l2; std::ptrdiff_t m_l3; + // Bytes of L3 backing one CPU, or 0 when unknown. Cleared by setCpuCacheSizes so that an + // explicit override is never overruled by the detected geometry. + std::ptrdiff_t m_l3_per_cpu; }; /** \internal */ -inline void manage_caching_sizes(Action action, std::ptrdiff_t* l1, std::ptrdiff_t* l2, std::ptrdiff_t* l3) { +inline void manage_caching_sizes(Action action, std::ptrdiff_t* l1, std::ptrdiff_t* l2, std::ptrdiff_t* l3, + std::ptrdiff_t* l3_per_cpu = nullptr) { static CacheSizes m_cacheSizes; if (action == SetAction) { @@ -104,10 +108,12 @@ m_cacheSizes.m_l1 = *l1; m_cacheSizes.m_l2 = *l2; m_cacheSizes.m_l3 = *l3; + m_cacheSizes.m_l3_per_cpu = l3_per_cpu != nullptr ? *l3_per_cpu : 0; } else if (action == GetAction) { eigen_internal_assert(l1 != 0 && l2 != 0); *l1 = m_cacheSizes.m_l1; *l2 = m_cacheSizes.m_l2; + if (l3_per_cpu != nullptr) *l3_per_cpu = m_cacheSizes.m_l3_per_cpu; *l3 = m_cacheSizes.m_l3; } else { eigen_internal_assert(false); @@ -212,8 +218,8 @@ // kc x nc blocks B' on the rhs. B' has to fit into L2/L3 cache. Moreover, A' is processed // per mr x kc horizontal small panels where mr is the blocking size along the m dimension // at the register level. This small horizontal panel has to stay within L1 cache. - std::ptrdiff_t l1, l2, l3; - manage_caching_sizes(GetAction, &l1, &l2, &l3); + std::ptrdiff_t l1, l2, l3, l3_per_cpu; + manage_caching_sizes(GetAction, &l1, &l2, &l3, &l3_per_cpu); #ifdef EIGEN_VECTORIZE_AVX512 const std::ptrdiff_t phys_l1 = l1; // We need to find a rationale for that, but without this adjustment, @@ -279,6 +285,8 @@ l1 = 9 * 1024; l2 = 32 * 1024; l3 = 512 * 1024; + // The detected share would otherwise swamp these synthetic sizes and defeat the whole point. + l3_per_cpu = 0; #endif // Early return for small problems because the computation below are time consuming for small problems. @@ -349,6 +357,15 @@ const Index actual_l2 = static_cast<Index>(l2 * 3 / 2); #endif + // Budget for the packed rhs panel. The 1.5x above stands in for an L3 whose geometry was + // unknown, and was calibrated against a 1MB placeholder L2, so it underestimates the reachable + // working set on a core whose real L2 is much smaller. Prefer this CPU's measured share of L3 + // where the platform reports it -- a share rather than the whole cache, since sizing one CPU's + // panel to all of a server's L3 would evict every other CPU's working set. This deliberately + // does not feed actual_lm below: that governs the blockA allocation, whose L1/L2 tuning is + // separate. + const Index rhs_panel_budget = numext::maxi<Index>(actual_l2, static_cast<Index>(l3_per_cpu)); + // Here, nc is chosen such that a block of kc x nc of the rhs fit within half of L2. // The second half is implicitly reserved to access the result and lhs coefficients. // When k<max_kc, then nc can grow without bound. In practice, it seems to be fruitful @@ -364,10 +381,10 @@ } else { // L2 blocking: use actual kc (k) rather than max_kc so that nc is not // unnecessarily squeezed when k < max_kc (e.g. on CPUs with large L1). - max_nc = (3 * actual_l2) / (2 * 2 * k * sizeof(RhsScalar)); + max_nc = (3 * rhs_panel_budget) / (2 * 2 * k * sizeof(RhsScalar)); } // WARNING Below, we assume that Traits::nr is a power of two. - Index nc = numext::mini<Index>(actual_l2 / (2 * k * sizeof(RhsScalar)), max_nc) & (~(Traits::nr - 1)); + Index nc = numext::mini<Index>(rhs_panel_budget / (2 * k * sizeof(RhsScalar)), max_nc) & (~(Traits::nr - 1)); if (n > nc) { // We are really blocking over the columns: // -> reduce blocking size to make sure the last block is as large as possible
diff --git a/Eigen/src/Core/util/Memory.h b/Eigen/src/Core/util/Memory.h index c2ff37d..020397a 100644 --- a/Eigen/src/Core/util/Memory.h +++ b/Eigen/src/Core/util/Memory.h
@@ -966,6 +966,11 @@ //---------- Cache sizes ---------- +#if EIGEN_OS_LINUX && !defined(EIGEN_NO_CPU_CACHE_SYSFS) +// Linux publishes the cache topology under sysfs on every architecture. +#define EIGEN_CPU_CACHE_SYSFS 1 +#endif + #if !defined(EIGEN_NO_CPUID) #if EIGEN_COMP_GNUC && EIGEN_ARCH_i386_OR_x86_64 #if defined(__PIC__) && EIGEN_ARCH_i386 @@ -1264,9 +1269,194 @@ } #endif +#ifdef EIGEN_CPU_CACHE_SYSFS + +/** \internal Data-cache geometry as Linux describes it; a zero field means sysfs did not say. */ +struct CpuCacheTopology { + std::ptrdiff_t l1 = 0; + std::ptrdiff_t l2 = 0; + std::ptrdiff_t l3 = 0; + std::ptrdiff_t l3_per_cpu = 0; +}; + +/** \internal Reads the first line of the file at \a relative under the CPU topology directory \a root into + * \a value. The directory is /sys/devices/system/cpu in production; tests substitute a fixture tree. */ +template <int Size> +inline bool readSysfsLine(const char* root, const char* relative, char (&value)[Size]) { + char path[512]; + const int length = std::snprintf(path, sizeof(path), "%s/%s", root, relative); + if (length <= 0 || length >= static_cast<int>(sizeof(path))) return false; + std::FILE* file = std::fopen(path, "r"); + if (file == nullptr) return false; + const bool ok = std::fgets(value, Size, file) != nullptr; + std::fclose(file); + return ok; +} + +/** \internal Reads the first line of \a cpu's cache attribute \a name into \a value. */ +template <int Size> +inline bool readCpuCacheAttribute(const char* root, int cpu, int index, const char* name, char (&value)[Size]) { + char relative[64]; + const int length = std::snprintf(relative, sizeof(relative), "cpu%d/cache/index%d/%s", cpu, index, name); + return length > 0 && length < static_cast<int>(sizeof(relative)) && readSysfsLine(root, relative, value); +} + +/** \internal Parses a sysfs cache size such as "64K". \returns 0 if \a text is not a positive size. */ +inline std::ptrdiff_t parseCpuCacheSize(const char* text) { + char* suffix = nullptr; + const std::ptrdiff_t value = std::strtol(text, &suffix, 10); + if (value <= 0) return 0; + // The kernel writes kibibytes ("%uK"), but only scale on an explicit unit: reading a bare byte + // count as kibibytes would overstate a cache 1024x, which is much worse than the reverse. + const std::ptrdiff_t multiplier = (*suffix == 'K' || *suffix == 'k') ? 1024 + : (*suffix == 'M' || *suffix == 'm') ? 1024 * 1024 + : 1; + return value * multiplier; +} + +/** \internal Whether \a text is where a sysfs line legitimately ends. */ +inline bool isCpuListTerminator(const char* text) { return *text == '\0' || *text == '\n' || *text == '\r'; } + /** \internal - * Queries and returns the cache sizes in Bytes of the L1, L2, and L3 data caches respectively */ -inline void queryCacheSizes(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) { + * Calls \a visit(first, last) for each range of a sysfs cpu list such as "0-3" or "0-3,8-11". \returns false + * if \a text is malformed, so that an unparsable list reads as "unknown" rather than as whatever the ranges + * before the parse went wrong added up to. */ +template <typename Visitor> +inline bool parseCpuList(const char* text, Visitor&& visit) { + if (isCpuListTerminator(text)) return false; + // Far above the 8192 CPUs current kernels can number, so a larger id means the line is not a cpu list; + // bounding it also keeps a caller that walks the ranges from visiting ids that cannot exist. + const long max_cpu_id = 1 << 16; + const char* cursor = text; + for (;;) { + char* end = nullptr; + const long first = std::strtol(cursor, &end, 10); + if (end == cursor || first < 0) return false; + long last = first; + if (*end == '-') { + cursor = end + 1; + last = std::strtol(cursor, &end, 10); + if (end == cursor || last < first) return false; + } + if (last >= max_cpu_id) return false; + visit(static_cast<int>(first), static_cast<int>(last)); + // A comma promises another range, so a list ending on one is malformed: going round the loop + // lands on the terminator and fails the strtol above. + if (*end == ',') { + cursor = end + 1; + continue; + } + // Anything other than a separator or the end of the line means the format is not what this + // parser assumes. + return isCpuListTerminator(end); + } +} + +/** \internal + * Counts the CPUs in a sysfs cpu list. \returns 0 if \a text is malformed: reading such a line as a small + * count would inflate l3_per_cpu, so the whole list is treated as unknown instead. */ +inline int parseCpuListCount(const char* text) { + int count = 0; + return parseCpuList(text, [&count](int first, int last) { count += last - first + 1; }) ? count : 0; +} + +/** \internal + * \returns \a cpu's data-cache geometry as published under \a root. + * + * l3_per_cpu is the L3 instance size divided by the CPUs sharing it, and is deliberately derived + * within this single pass: the l3 reported elsewhere is a package total on a multi-die part -- 128MB + * on a 32-core Threadripper whose cores each reach one 16MB slice -- so pairing it with one + * instance's sharer count would overstate the share eightfold. */ +inline CpuCacheTopology readCpuCacheTopologySysfs(const char* root, int cpu) { + CpuCacheTopology topology; + // One directory per cache, numbered contiguously from zero. The bound only guards a malformed + // sysfs; it is far above what any current CPU reports. + for (int index = 0; index < 16; ++index) { + // A shared_cpu_list can be long on a large machine, and truncating it would undercount the + // sharers and hand out too large a share. + char value[512]; + + if (!readCpuCacheAttribute(root, cpu, index, "level", value)) break; + const long level = std::strtol(value, nullptr, 10); + std::ptrdiff_t* target = + level == 1 ? &topology.l1 : (level == 2 ? &topology.l2 : (level == 3 ? &topology.l3 : nullptr)); + if (target == nullptr || *target > 0) continue; + + // "Data", "Instruction", or "Unified". An instruction cache never holds the operands a product + // blocks for, so it must not be mistaken for the L1 data cache. + if (!readCpuCacheAttribute(root, cpu, index, "type", value) || value[0] == 'I') continue; + + if (!readCpuCacheAttribute(root, cpu, index, "size", value)) continue; + const std::ptrdiff_t size = parseCpuCacheSize(value); + if (size <= 0) continue; + *target = size; + + if (level == 3 && readCpuCacheAttribute(root, cpu, index, "shared_cpu_list", value) && + std::strchr(value, '\n') != nullptr) { + const int sharing = parseCpuListCount(value); + if (sharing > 0) topology.l3_per_cpu = size / sharing; + } + } + return topology; +} + +/** \internal The smaller of two reported sizes, where 0 stands for "not reported". */ +inline std::ptrdiff_t smallerReportedCacheSize(std::ptrdiff_t a, std::ptrdiff_t b) { + return a == 0 ? b : (b == 0 ? a : (std::min)(a, b)); +} + +/** \internal + * \returns the geometry a thread confined to the CPUs \a is_allowed(cpu) admits can rely on: per level, the + * smallest cache any of those CPUs reports, so that blocking sized from it fits wherever the thread lands. + * The candidates are the CPUs listed online under \a root; one that publishes no cache directory contributes + * nothing, and a level it does not report is left to the CPUs that do. */ +template <typename IsAllowed> +inline CpuCacheTopology queryCpuCacheTopologySysfs(const char* root, IsAllowed&& is_allowed) { + CpuCacheTopology topology; + char online[512]; + if (!readSysfsLine(root, "online", online) || std::strchr(online, '\n') == nullptr) return topology; + // Validate the whole line before acting on any range of it, so that a malformed list reads as "unknown". + if (!parseCpuList(online, [](int, int) {})) return topology; + parseCpuList(online, [&](int first, int last) { + for (int cpu = first; cpu <= last; ++cpu) { + if (!is_allowed(cpu)) continue; + const CpuCacheTopology candidate = readCpuCacheTopologySysfs(root, cpu); + topology.l1 = smallerReportedCacheSize(topology.l1, candidate.l1); + topology.l2 = smallerReportedCacheSize(topology.l2, candidate.l2); + topology.l3 = smallerReportedCacheSize(topology.l3, candidate.l3); + topology.l3_per_cpu = smallerReportedCacheSize(topology.l3_per_cpu, candidate.l3_per_cpu); + } + }); + return topology; +} + +/** \internal + * \returns the geometry the calling thread can rely on: that of the CPUs its affinity mask admits, narrowed as + * queryCpuCacheTopologySysfs(root, is_allowed) describes. Sampling one fixed CPU would not do: a cpuset or + * taskset can exclude CPU 0 while its directory stays readable, and on a heterogeneous part the excluded CPUs + * can be the ones with the large caches. Where the mask is unavailable every online CPU is a candidate, which + * is the conservative answer. */ +inline CpuCacheTopology queryCpuCacheTopologySysfs(const char* root = "/sys/devices/system/cpu") { +#ifdef CPU_SETSIZE + // The affinity API is a GNU extension that glibc, musl and bionic expose under _GNU_SOURCE, which g++ and + // clang++ predefine for C++; CPU_SETSIZE is defined exactly when it is exposed. The query fails on a machine + // with more CPU ids than cpu_set_t holds. + cpu_set_t allowed; + if (sched_getaffinity(0, sizeof(allowed), &allowed) == 0) { + return queryCpuCacheTopologySysfs(root, + [&allowed](int cpu) { return cpu < CPU_SETSIZE && CPU_ISSET(cpu, &allowed); }); + } +#endif + return queryCpuCacheTopologySysfs(root, [](int) { return true; }); +} + +#endif // EIGEN_CPU_CACHE_SYSFS + +/** \internal + * Queries and returns the cache sizes in Bytes of the L1, L2, and L3 data caches respectively, and in + * \a l3_per_cpu one CPU's share of the L3 where the platform publishes the sharing, 0 otherwise. */ +inline void queryCacheSizes(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3, std::ptrdiff_t& l3_per_cpu) { + l3_per_cpu = 0; #ifdef EIGEN_CPUID int abcd[4]; const int GenuineIntel[] = {0x756e6547, 0x49656e69, 0x6c65746e}; @@ -1326,13 +1516,31 @@ if (sysctlbyname("hw.l3cachesize", &val, &val_size, nullptr, 0) == 0 && val > 0) l3 = val; } #elif EIGEN_OS_UNIX && defined(_SC_LEVEL1_DCACHE_SIZE) - // On Linux and other POSIX systems, use sysconf to query cache sizes. + // A glibc extension: POSIX specifies no cache queries, and musl defines none of these names. l1 = sysconf(_SC_LEVEL1_DCACHE_SIZE); l2 = sysconf(_SC_LEVEL2_CACHE_SIZE); l3 = sysconf(_SC_LEVEL3_CACHE_SIZE); #else l1 = l2 = l3 = -1; #endif +#ifdef EIGEN_CPU_CACHE_SYSFS + // glibc answers the _SC_LEVEL*_CACHE_SIZE queries from CPUID and so only implements them on x86; every + // other architecture gets 0, and musl has no such queries at all. Whatever the platform left unknown comes + // from the topology Linux publishes on every architecture, as does the L3 share, so that all four numbers + // describe the same CPUs. + const CpuCacheTopology topology = queryCpuCacheTopologySysfs(); + if (l1 <= 0) l1 = topology.l1; + if (l2 <= 0) l2 = topology.l2; + if (l3 <= 0) l3 = topology.l3; + l3_per_cpu = topology.l3_per_cpu; +#endif +} + +/** \internal + * Queries and returns the cache sizes in Bytes of the L1, L2, and L3 data caches respectively */ +inline void queryCacheSizes(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) { + std::ptrdiff_t l3_per_cpu; + queryCacheSizes(l1, l2, l3, l3_per_cpu); } /** \internal
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a780a60..bbe9c57 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt
@@ -279,6 +279,7 @@ ei_add_test(nomalloc) ei_add_test(stack_allocation_limit) ei_add_test(first_aligned) +ei_add_test(cache_sizes) ei_add_test(type_alias) ei_add_test(nullary) ei_add_test(mixingtypes)
diff --git a/test/cache_sizes.cpp b/test/cache_sizes.cpp new file mode 100644 index 0000000..1880325 --- /dev/null +++ b/test/cache_sizes.cpp
@@ -0,0 +1,225 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include "main.h" + +#ifdef EIGEN_CPU_CACHE_SYSFS +#include <sched.h> +#include <sys/stat.h> +#include <cstdlib> +#include <string> +#include <vector> +#endif + +void cache_sizes_plausible() { + // Whatever the platform reports has to be a plausible data-cache size: a stray unit suffix + // would otherwise be read as a handful of bytes and silently shrink every blocking size. + std::ptrdiff_t l1 = -1, l2 = -1, l3 = -1, l3_per_cpu = -1; + internal::queryCacheSizes(l1, l2, l3, l3_per_cpu); + for (std::ptrdiff_t size : {l1, l2, l3, l3_per_cpu}) + if (size > 0) VERIFY(size >= 1024); + // A share is one CPU's slice of one L3 instance, so it can never exceed the reported L3. + VERIFY(l3_per_cpu >= 0); + if (l3_per_cpu > 0) VERIFY(l3 > 0 && l3_per_cpu <= l3); +} + +#ifdef EIGEN_CPU_CACHE_SYSFS + +using internal::CpuCacheTopology; + +bool same_topology(const CpuCacheTopology& a, const CpuCacheTopology& b) { + return a.l1 == b.l1 && a.l2 == b.l2 && a.l3 == b.l3 && a.l3_per_cpu == b.l3_per_cpu; +} + +CpuCacheTopology topology_of(std::ptrdiff_t l1, std::ptrdiff_t l2, std::ptrdiff_t l3, std::ptrdiff_t l3_per_cpu) { + CpuCacheTopology topology; + topology.l1 = l1; + topology.l2 = l2; + topology.l3 = l3; + topology.l3_per_cpu = l3_per_cpu; + return topology; +} + +void cache_sizes_parsers() { + VERIFY(internal::parseCpuCacheSize("64K\n") == 64 * 1024); + VERIFY(internal::parseCpuCacheSize("2048K\n") == 2048 * 1024); + VERIFY(internal::parseCpuCacheSize("32M\n") == 32 * 1024 * 1024); + // A size Eigen cannot make sense of has to read as "unknown", never as a few bytes. + VERIFY(internal::parseCpuCacheSize("bogus") == 0); + VERIFY(internal::parseCpuCacheSize("0K") == 0); + // A bare count is bytes, not kibibytes. + VERIFY(internal::parseCpuCacheSize("512\n") == 512); + + VERIFY(internal::parseCpuListCount("0\n") == 1); + VERIFY(internal::parseCpuListCount("0-3\n") == 4); + VERIFY(internal::parseCpuListCount("0-3,8-11\n") == 8); + VERIFY(internal::parseCpuListCount("0,2,4\n") == 3); + VERIFY(internal::parseCpuListCount("") == 0); + VERIFY(internal::parseCpuListCount("3-0") == 0); + VERIFY(internal::parseCpuListCount("0-3\r\n") == 4); + // Trailing text means the format is not the one assumed, so the count is not trustworthy: + // a small count here would inflate l3_per_cpu rather than leave the share unknown. + VERIFY(internal::parseCpuListCount("0-3junk") == 0); + VERIFY(internal::parseCpuListCount("0-3 8-11") == 0); + VERIFY(internal::parseCpuListCount("0-3,") == 0); + VERIFY(internal::parseCpuListCount("0-3,junk") == 0); + VERIFY(internal::parseCpuListCount("junk") == 0); + // strtol would accept a sign, but no CPU has a negative id, and none has one beyond what a kernel can number. + VERIFY(internal::parseCpuListCount("-1") == 0); + VERIFY(internal::parseCpuListCount("0-70000") == 0); + + // The ranges are handed over one at a time, and a list that turns out malformed is reported as such even + // though its leading ranges were already visited. + int ranges = 0, cpus = 0; + const auto tally = [&](int first, int last) { + ++ranges; + cpus += last - first + 1; + }; + VERIFY(internal::parseCpuList("0-3,8,10-11\n", tally)); + VERIFY(ranges == 3 && cpus == 7); + VERIFY(!internal::parseCpuList("0-3,8,10-11junk\n", tally)); +} + +// A sysfs tree laid out exactly as Linux publishes it, describing whatever machine a test needs. +struct SysfsFixture { + std::string root; + std::vector<std::string> created; + + SysfsFixture() { + const char* base = std::getenv("TMPDIR"); + const std::string pattern = std::string(base != nullptr ? base : "/tmp") + "/eigen-cache-sizes-XXXXXX"; + std::vector<char> buffer(pattern.begin(), pattern.end()); + buffer.push_back('\0'); + if (mkdtemp(buffer.data()) != nullptr) { + root = buffer.data(); + created.push_back(root); + } + } + ~SysfsFixture() { + for (auto it = created.rbegin(); it != created.rend(); ++it) std::remove(it->c_str()); + } + + void directory(const std::string& path) { + if (mkdir(path.c_str(), 0700) == 0) created.push_back(path); + } + void file(const std::string& relative, const char* text) { + const std::string path = root + "/" + relative; + std::FILE* stream = std::fopen(path.c_str(), "w"); + VERIFY(stream != nullptr); + std::fputs(text, stream); + std::fclose(stream); + created.push_back(path); + } + void cpu(int id) { + directory(root + "/cpu" + std::to_string(id)); + directory(root + "/cpu" + std::to_string(id) + "/cache"); + } + void cache(int id, int index, const char* level, const char* type, const char* size, const char* shared) { + const std::string dir = "cpu" + std::to_string(id) + "/cache/index" + std::to_string(index); + directory(root + "/" + dir); + file(dir + "/level", level); + file(dir + "/type", type); + file(dir + "/size", size); + file(dir + "/shared_cpu_list", shared); + } +}; + +void cache_sizes_fixture_topology() { + SysfsFixture fixture; + if (fixture.root.empty()) return; + const char* const root = fixture.root.c_str(); + + // Two big cores (0 and 3) with a 16MB L3 between them, and two little cores (1 and 2) with a 2MB + // cluster L3. CPU 3 is online but publishes no cache directory at all, and CPU 2 has no L2, so its + // L3 sits at the index the others use for L2. + fixture.cpu(0); + fixture.cache(0, 0, "1\n", "Data\n", "64K\n", "0\n"); + fixture.cache(0, 1, "1\n", "Instruction\n", "128K\n", "0\n"); + fixture.cache(0, 2, "2\n", "Unified\n", "1024K\n", "0\n"); + fixture.cache(0, 3, "3\n", "Unified\n", "16384K\n", "0,3\n"); + fixture.cpu(1); + fixture.cache(1, 0, "1\n", "Data\n", "32K\n", "1\n"); + fixture.cache(1, 1, "1\n", "Instruction\n", "32K\n", "1\n"); + fixture.cache(1, 2, "2\n", "Unified\n", "256K\n", "1\n"); + fixture.cache(1, 3, "3\n", "Unified\n", "2048K\n", "1-2\n"); + fixture.cpu(2); + fixture.cache(2, 0, "1\n", "Data\n", "32K\n", "2\n"); + fixture.cache(2, 1, "1\n", "Instruction\n", "32K\n", "2\n"); + fixture.cache(2, 2, "3\n", "Unified\n", "2048K\n", "1-2\n"); + fixture.directory(fixture.root + "/cpu3"); + fixture.file("online", "0-3\n"); + + const CpuCacheTopology big = topology_of(64 * 1024, 1024 * 1024, 16384 * 1024, 8192 * 1024); + const CpuCacheTopology little = topology_of(32 * 1024, 256 * 1024, 2048 * 1024, 1024 * 1024); + const CpuCacheTopology unknown = topology_of(0, 0, 0, 0); + + // Each CPU on its own. The larger instruction cache must not pass for the L1 data cache. + VERIFY(same_topology(internal::readCpuCacheTopologySysfs(root, 0), big)); + VERIFY(same_topology(internal::readCpuCacheTopologySysfs(root, 1), little)); + VERIFY( + same_topology(internal::readCpuCacheTopologySysfs(root, 2), topology_of(32 * 1024, 0, 2048 * 1024, 1024 * 1024))); + VERIFY(same_topology(internal::readCpuCacheTopologySysfs(root, 3), unknown)); + + // A thread that may land on any CPU has to block for the smallest caches among them. + const auto every_cpu = [](int) { return true; }; + VERIFY(same_topology(internal::queryCpuCacheTopologySysfs(root, every_cpu), little)); + // One confined to the big cores sees only their geometry, and a mask that excludes CPU 0 must not carry + // CPU 0's geometry along. + VERIFY(same_topology(internal::queryCpuCacheTopologySysfs(root, [](int cpu) { return cpu == 0; }), big)); + VERIFY(same_topology(internal::queryCpuCacheTopologySysfs(root, [](int cpu) { return cpu == 0 || cpu == 3; }), big)); + VERIFY(same_topology(internal::queryCpuCacheTopologySysfs(root, [](int cpu) { return cpu != 0; }), little)); + VERIFY( + same_topology(internal::queryCpuCacheTopologySysfs(root, [](int cpu) { return cpu == 1 || cpu == 3; }), little)); + // A CPU that reports nothing, or no CPU at all, leaves the geometry unknown rather than defaulted. + VERIFY(same_topology(internal::queryCpuCacheTopologySysfs(root, [](int cpu) { return cpu == 3; }), unknown)); + VERIFY(same_topology(internal::queryCpuCacheTopologySysfs(root, [](int) { return false; }), unknown)); + + // An online list that cannot be trusted, or that is missing, means no CPU is known. + fixture.file("online", "0-3junk\n"); + VERIFY(same_topology(internal::queryCpuCacheTopologySysfs(root, every_cpu), unknown)); + fixture.file("online", "0-3"); + VERIFY(same_topology(internal::queryCpuCacheTopologySysfs(root, every_cpu), unknown)); + std::remove((fixture.root + "/online").c_str()); + VERIFY(same_topology(internal::queryCpuCacheTopologySysfs(root, every_cpu), unknown)); +} + +void cache_sizes_affinity() { + // On a kernel that publishes the topology, Eigen has to pick it up instead of falling back to + // its compiled-in defaults. glibc's sysconf answers only on x86, so before the sysfs fallback + // this failed on every other Linux architecture. + std::ptrdiff_t l1, l2, l3, l3_per_cpu; + internal::queryCacheSizes(l1, l2, l3, l3_per_cpu); + const CpuCacheTopology mine = internal::queryCpuCacheTopologySysfs(); + if (mine.l1 > 0) VERIFY(l1 > 0); + VERIFY(l3_per_cpu == mine.l3_per_cpu); + + // Confine the thread to every CPU it may use but CPU 0: the detected geometry has to follow the mask. + cpu_set_t original; + if (sched_getaffinity(0, sizeof(original), &original) != 0) return; + if (CPU_COUNT(&original) < 2 || !CPU_ISSET(0, &original)) return; + cpu_set_t restricted = original; + CPU_CLR(0, &restricted); + VERIFY(sched_setaffinity(0, sizeof(restricted), &restricted) == 0); + const CpuCacheTopology seen = internal::queryCpuCacheTopologySysfs(); + const CpuCacheTopology expected = internal::queryCpuCacheTopologySysfs( + "/sys/devices/system/cpu", [&restricted](int cpu) { return cpu < CPU_SETSIZE && CPU_ISSET(cpu, &restricted); }); + VERIFY(sched_setaffinity(0, sizeof(original), &original) == 0); + VERIFY(same_topology(seen, expected)); +} + +#endif // EIGEN_CPU_CACHE_SYSFS + +EIGEN_DECLARE_TEST(cache_sizes) { + CALL_SUBTEST(cache_sizes_plausible()); +#ifdef EIGEN_CPU_CACHE_SYSFS + CALL_SUBTEST(cache_sizes_parsers()); + CALL_SUBTEST(cache_sizes_fixture_topology()); + CALL_SUBTEST(cache_sizes_affinity()); +#endif +}
diff --git a/test/product_large.cpp b/test/product_large.cpp index 6f19aa0..0e17db6 100644 --- a/test/product_large.cpp +++ b/test/product_large.cpp
@@ -81,6 +81,16 @@ VERIFY(m2 > 0); VERIFY(n2 > 0); + // An explicit override has to govern blocking on its own. The detected per-CPU L3 share also + // feeds the rhs-panel budget, so leaving it set would silently overrule callers that force + // small cache sizes to exercise multi-pass blocking (unsupported/test/tensor_contraction.cpp + // does exactly that). + setCpuCacheSizes(896, 1920, 2944); + std::ptrdiff_t forced_l1, forced_l2, forced_l3, forced_l3_per_cpu = -1; + internal::manage_caching_sizes(GetAction, &forced_l1, &forced_l2, &forced_l3, &forced_l3_per_cpu); + VERIFY(forced_l1 == 896 && forced_l2 == 1920 && forced_l3 == 2944); + VERIFY(forced_l3_per_cpu == 0); + setCpuCacheSizes(old_l1, old_l2, old_l3); }