| #!/usr/bin/env python3 | |
| # SPDX-FileCopyrightText: The Eigen Authors | |
| # SPDX-License-Identifier: MPL-2.0 | |
| # | |
| # Content-addressed pass cache for CI test jobs (driven by | |
| # test.linux.script.sh, run from inside the build directory). | |
| # | |
| # `plan` reads a `ctest --show-only=json-v1` dump (stdin via | |
| # `--tests-json -`) and keys every enumerated test by an environment | |
| # fingerprint plus the content of its definition: command elements that are | |
| # files (the test binary, and the emulator for cross-compiled jobs) are | |
| # replaced by a digest of their bytes, as are the build-directory shared | |
| # libraries they load; remaining elements and the CTest properties are taken | |
| # literally with build-directory paths normalized away. | |
| # Tests whose key the manifest already records as a first-attempt pass | |
| # become the anchored exclusion regex in --skip-regex-out; every keyable | |
| # test goes to --keys-out for the later `record` step. | |
| # | |
| # The environment fingerprint covers everything that can change a test's | |
| # outcome while its binary stays identical: the image and its C library, the | |
| # CONFIG_PATHS subtrees of the checked-in CI configuration, the dpkg version | |
| # state of every lib* package (a superset of anything a test binary can | |
| # dynamically load from outside the build directory, cross sysroots included), | |
| # and KEYED_ENV_PREFIXES variables read directly for values set outside the | |
| # tree. Job-to-job differences are already isolated by the per-job-name GitLab | |
| # cache key; the fingerprint guards against one job's environment drifting over | |
| # time. | |
| # A tool that is genuinely absent (e.g. no dpkg on the hand-assembled | |
| # riscv64 image) contributes a fixed sentinel so images with and without it | |
| # never share keys; unexpected errors propagate, `plan` fails, and the | |
| # driving script then runs the full selection. | |
| # | |
| # `record` folds the run's first-attempt results back into the manifest, | |
| # using the dashboard run's Test.xml as the authoritative status source: | |
| # only tests with Status="passed" are recorded ("failed" and "notrun" cover | |
| # failures, timeouts, missing executables, and SKIP_RETURN_CODE skips). | |
| # New passes are appended, entries that proved useful again (skipped or | |
| # re-passed) move to the end, and the file is trimmed to --max-entries, so | |
| # it ages out roughly least-recently-useful first. | |
| # | |
| # A test is keyable only if its command ends in an executable inside the | |
| # build directory. Anything else (e.g. a test that invokes the compiler on | |
| # tree sources) has an outcome that depends on more than the artifact's | |
| # content and must always run. | |
| import argparse | |
| import functools | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import struct | |
| import subprocess | |
| import sys | |
| import xml.etree.ElementTree as ElementTree | |
| # Paths under the config root whose content can change a test's outcome while | |
| # its binary stays identical. Deliberately not the whole ci/ tree: the | |
| # *.gitlab-ci.yml files are orchestration, and everything in them that reaches | |
| # a test outcome reaches this key by value already -- job variables through | |
| # KEYED_ENV_PREFIXES, the image through CI_JOB_IMAGE, compiler flags and the | |
| # cross emulator through the digests of the files in the test's command, ctest | |
| # timeouts through the properties hash. Hashing the file that sets them only | |
| # means every CI-maintenance merge request throws away every job's manifest. | |
| # | |
| # ci/docker/ stays: those images are referenced by a moving :latest tag, so | |
| # CI_JOB_IMAGE cannot see a rebuild, and the dpkg-query below covers only lib* | |
| # packages, not a non-lib* one such as qemu-user. | |
| # | |
| # This is a whole-directory walk rather than a list of the scripts on the test | |
| # path, because a list is complete only until someone adds a script and forgets | |
| # to name it here -- and that failure mode is a silently reused stale pass. | |
| # | |
| # The exchange for the narrowing: a job's tags: are now invisible to the key. | |
| # Moving a job to a runner pool whose CPU differs (an AVX-512 job off the | |
| # avx512 pool, say) will not invalidate its manifest, so pair such a move with | |
| # a cache clear. Nothing today distinguishes two hosts within one tag pool | |
| # either, so this widens an accepted hole rather than opening a new one. | |
| CONFIG_PATHS = ("ci/scripts", "ci/docker") | |
| # Environment variables (matched by prefix) that can change a test's outcome | |
| # without changing its binary. Since the CI YAML is no longer hashed, this | |
| # tuple is load-bearing: a new job variable that can change a test's outcome | |
| # must be added here, because setting it in the YAML alone no longer keys it. | |
| KEYED_ENV_PREFIXES = ( | |
| "EIGEN_REPEAT", | |
| "EIGEN_SEED", | |
| "EIGEN_CI_CTEST_ARGS", | |
| "QEMU_", | |
| "ASAN_", | |
| "UBSAN_", | |
| "LSAN_", | |
| "TSAN_", | |
| "MSAN_", | |
| "LD_LIBRARY_PATH", | |
| "LD_PRELOAD", | |
| # Host-derived values that legitimately vary across runners of one job | |
| # (EIGEN_CI_CTEST_PARALLEL, NPROC) are deliberately absent. | |
| ) | |
| @functools.lru_cache(maxsize=None) | |
| def file_digest(path): | |
| with open(path, "rb") as f: | |
| return hashlib.file_digest(f, "sha256").hexdigest() | |
| PT_LOAD, PT_DYNAMIC = 1, 2 | |
| DT_NULL, DT_NEEDED, DT_STRTAB = 0, 1, 5 | |
| @functools.lru_cache(maxsize=None) | |
| def needed_libraries(path): | |
| """The DT_NEEDED names of an ELF file, found through its program headers | |
| as the dynamic loader finds them, so any ELF class, byte order, or | |
| target architecture works; () for a file that is not ELF or is | |
| statically linked. A malformed ELF file raises.""" | |
| with open(path, "rb") as f: | |
| ident = f.read(16) | |
| if ident[:4] != b"\x7fELF": | |
| return () | |
| is64 = {1: False, 2: True}[ident[4]] | |
| order = {1: "<", 2: ">"}[ident[5]] | |
| def read(fmt, offset): | |
| fmt = order + fmt | |
| f.seek(offset) | |
| return struct.unpack(fmt, f.read(struct.calcsize(fmt))) | |
| def read_string(offset): | |
| chunks = [] | |
| f.seek(offset) | |
| while True: | |
| chunk = f.read(256) | |
| if not chunk: | |
| raise ValueError("%s: unterminated dynamic string" % path) | |
| end = chunk.find(b"\0") | |
| if end >= 0: | |
| return os.fsdecode(b"".join(chunks) + chunk[:end]) | |
| chunks.append(chunk) | |
| if is64: | |
| (phoff,) = read("Q", 32) | |
| phentsize, phnum = read("HH", 54) | |
| else: | |
| (phoff,) = read("I", 28) | |
| phentsize, phnum = read("HH", 42) | |
| loads = [] | |
| dynamic = None | |
| for i in range(phnum): | |
| if is64: | |
| p_type, _, p_offset, p_vaddr, _, p_filesz = read("IIQQQQ", phoff + i * phentsize) | |
| else: | |
| p_type, p_offset, p_vaddr, _, p_filesz = read("IIIII", phoff + i * phentsize) | |
| if p_type == PT_LOAD: | |
| loads.append((p_vaddr, p_filesz, p_offset)) | |
| elif p_type == PT_DYNAMIC: | |
| dynamic = (p_offset, p_filesz) | |
| if dynamic is None: | |
| return () | |
| dynamic_offset, dynamic_size = dynamic | |
| entry = "qQ" if is64 else "iI" | |
| needed = [] | |
| strtab = None | |
| for offset in range(dynamic_offset, dynamic_offset + dynamic_size, struct.calcsize(entry)): | |
| tag, value = read(entry, offset) | |
| if tag == DT_NULL: | |
| break | |
| if tag == DT_NEEDED: | |
| needed.append(value) | |
| elif tag == DT_STRTAB: | |
| strtab = value | |
| if not needed: | |
| return () | |
| # DT_STRTAB holds a virtual address; the PT_LOAD segment that maps it | |
| # gives its file offset. | |
| for vaddr, filesz, offset in loads: | |
| if strtab is not None and vaddr <= strtab < vaddr + filesz: | |
| return tuple(read_string(offset + strtab - vaddr + name) for name in needed) | |
| raise ValueError("%s: DT_STRTAB is not in a PT_LOAD segment" % path) | |
| @functools.lru_cache(maxsize=None) | |
| def build_tree_files(builddir): | |
| """Every file name under the build directory, mapped to its paths.""" | |
| files = {} | |
| for dirpath, _, filenames in os.walk(builddir): | |
| for name in filenames: | |
| files.setdefault(name, []).append(os.path.join(dirpath, name)) | |
| return files | |
| def build_tree_libraries(paths, builddir): | |
| """The build-directory shared libraries that the files in `paths` load, | |
| directly or through one another, as sorted real paths. A DT_NEEDED name | |
| matches every build-directory file of that name, wherever RUNPATH points, | |
| so the result is a superset of what the loader picks. Libraries outside | |
| the build directory are in the fingerprint already.""" | |
| files = build_tree_files(builddir) | |
| found = set() | |
| pending = list(paths) | |
| while pending: | |
| for name in needed_libraries(pending.pop()): | |
| for candidate in files.get(os.path.basename(name), ()): | |
| library = os.path.realpath(candidate) | |
| if library not in found and os.path.isfile(library): | |
| found.add(library) | |
| pending.append(library) | |
| return sorted(found) | |
| def fingerprint(config_root): | |
| parts = ["image:" + os.environ.get("CI_JOB_IMAGE", "")] | |
| try: | |
| parts.append("libc:" + (os.confstr("CS_GNU_LIBC_VERSION") or "")) | |
| except (ValueError, OSError): | |
| parts.append("libc:unavailable") | |
| config = hashlib.sha256() | |
| for top in CONFIG_PATHS: | |
| top_path = os.path.join(config_root, top) | |
| if os.path.isfile(top_path): | |
| config.update(top.encode() + b"\0" + file_digest(top_path).encode() + b"\0") | |
| continue | |
| for dirpath, dirnames, filenames in os.walk(top_path): | |
| dirnames.sort() | |
| for filename in sorted(filenames): | |
| path = os.path.join(dirpath, filename) | |
| rel = os.path.relpath(path, config_root) | |
| config.update(rel.encode() + b"\0" + file_digest(path).encode() + b"\0") | |
| parts.append("config:" + config.hexdigest()) | |
| try: | |
| packages = subprocess.run(["dpkg-query", "-W", "lib*"], check=True, capture_output=True, text=True).stdout | |
| parts.append("libs:" + hashlib.sha256(packages.encode()).hexdigest()) | |
| except (FileNotFoundError, subprocess.CalledProcessError): | |
| parts.append("libs:unavailable") | |
| env = ("%s=%s" % (k, v) for k, v in os.environ.items() if k.startswith(KEYED_ENV_PREFIXES)) | |
| parts.append("env:" + ",".join(sorted(env))) | |
| return "|".join(parts) | |
| def normalize(text, builddir): | |
| """Strips the build-directory prefix so the key does not depend on where | |
| the checkout happens to live.""" | |
| return text.replace(builddir, ".") | |
| def test_key(test, fingerprint_bytes, builddir): | |
| """Digest of the fingerprint and the test's definition, or None if the | |
| command does not end in an executable under the build directory.""" | |
| command = test.get("command") | |
| if not command: | |
| return None | |
| last = os.path.realpath(command[-1]) | |
| if not (last.startswith(builddir + os.sep) and os.path.isfile(last) and os.access(last, os.X_OK)): | |
| return None | |
| h = hashlib.sha256() | |
| h.update(fingerprint_bytes) | |
| built = [] | |
| for element in command: | |
| path = os.path.realpath(element) | |
| if os.path.isfile(path): | |
| h.update(b"file:" + file_digest(path).encode() + b"\0") | |
| if path.startswith(builddir + os.sep): | |
| built.append(path) | |
| else: | |
| h.update(b"arg:" + normalize(element, builddir).encode() + b"\0") | |
| for library in build_tree_libraries(built, builddir): | |
| h.update(b"lib:" + normalize(library, builddir).encode() + b"\0" + file_digest(library).encode() + b"\0") | |
| properties = sorted(test.get("properties", []), key=lambda p: str(p.get("name"))) | |
| h.update(b"props:" + normalize(json.dumps(properties, sort_keys=True), builddir).encode()) | |
| return h.hexdigest() | |
| def read_manifest(path): | |
| """The manifest as an insertion-ordered {key: testname} dict.""" | |
| entries = {} | |
| if os.path.exists(path): | |
| with open(path) as f: | |
| for line in f: | |
| fields = line.split() | |
| if fields: | |
| entries[fields[0]] = fields[1] if len(fields) > 1 else "" | |
| return entries | |
| def write_manifest(path, entries, max_entries): | |
| tmp = path + ".tmp" | |
| with open(tmp, "w") as f: | |
| f.writelines("%s %s\n" % item for item in list(entries.items())[-max_entries:]) | |
| os.replace(tmp, path) | |
| def plan(args): | |
| with (sys.stdin if args.tests_json == "-" else open(args.tests_json)) as f: | |
| tests = json.load(f)["tests"] | |
| manifest = read_manifest(args.manifest) | |
| builddir = os.path.realpath(os.getcwd()) | |
| fingerprint_bytes = fingerprint(args.config_root).encode() + b"\0" | |
| keys = {} | |
| skip = [] | |
| for test in tests: | |
| key = test_key(test, fingerprint_bytes, builddir) | |
| if key is None: | |
| continue | |
| keys[test["name"]] = key | |
| if key in manifest: | |
| skip.append(test["name"]) | |
| with open(args.keys_out, "w") as f: | |
| f.writelines("%s %s\n" % (key, name) for name, key in keys.items()) | |
| with open(args.skip_regex_out, "w") as f: | |
| if skip: | |
| f.write("^(%s)$" % "|".join(re.escape(name) for name in skip)) | |
| print( | |
| "test cache plan: %d tests selected, %d cached passes to skip, %d to run (%d not keyable)" | |
| % (len(tests), len(skip), len(tests) - len(skip), len(tests) - len(keys)) | |
| ) | |
| def record(args): | |
| keys = {} | |
| with open(args.keys) as f: | |
| for line in f: | |
| fields = line.split() | |
| if len(fields) == 2: | |
| keys[fields[1]] = fields[0] | |
| # The dashboard <Test> elements carry a Status attribute ("passed", | |
| # "failed", or "notrun"); the bare <TestList> entries do not and are | |
| # ignored. Streamed with iterparse because --no-compress-output embeds | |
| # every test's stdout in the file. A missing or unparsable Test.xml | |
| # means ctest died without completing the test phase: record nothing | |
| # rather than trust it. | |
| passed = set() | |
| try: | |
| with open(os.path.join(args.testing_dir, "TAG")) as f: | |
| tag = f.readline().strip() | |
| for _, element in ElementTree.iterparse(os.path.join(args.testing_dir, tag, "Test.xml")): | |
| if element.tag != "Test": | |
| continue | |
| if element.get("Status") == "passed": | |
| name = element.find("Name") | |
| if name is not None: | |
| passed.add(name.text) | |
| element.clear() | |
| except (OSError, ElementTree.ParseError) as error: | |
| print("test cache record: cannot read test results (%s); recording nothing" % error) | |
| return | |
| manifest = read_manifest(args.manifest) | |
| new = refreshed = 0 | |
| for name, key in keys.items(): | |
| if name in passed: | |
| if manifest.pop(key, None) is None: | |
| new += 1 | |
| else: | |
| refreshed += 1 | |
| manifest[key] = name | |
| elif manifest.pop(key, None) is not None: | |
| # Not run because this entry made plan skip it; move the entry to | |
| # the back so trimming ages out unused entries first. | |
| manifest[key] = name | |
| refreshed += 1 | |
| write_manifest(args.manifest, manifest, args.max_entries) | |
| print( | |
| "test cache record: %d new first-attempt passes, %d entries refreshed, %d entries total" | |
| % (new, refreshed, len(manifest)) | |
| ) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| sub = parser.add_subparsers(dest="cmd", required=True) | |
| p = sub.add_parser("plan") | |
| p.add_argument("--tests-json", required=True, help="json-v1 dump, or - for stdin") | |
| p.add_argument("--manifest", required=True) | |
| p.add_argument("--config-root", required=True) | |
| p.add_argument("--skip-regex-out", required=True) | |
| p.add_argument("--keys-out", required=True) | |
| p.set_defaults(func=plan) | |
| p = sub.add_parser("record") | |
| p.add_argument("--manifest", required=True) | |
| p.add_argument("--keys", required=True) | |
| p.add_argument("--testing-dir", required=True) | |
| p.add_argument("--max-entries", type=int, default=20000) | |
| p.set_defaults(func=record) | |
| args = parser.parse_args() | |
| args.func(args) | |
| if __name__ == "__main__": | |
| main() |