ci/scripts/test_cache.py - mirror - Git at Google

 #!/usr/bin/env python3
 # SPDX-FileCopyrightText: The Eigen Authors
 # SPDX-License-Identifier: MPL-2.0
 #
 # Content-addressed pass cache for CI test jobs (driven by
 # test.linux.script.sh, run from inside the build directory).
 #
 # `plan` reads a `ctest --show-only=json-v1` dump (stdin via
 # `--tests-json -`) and keys every enumerated test by an environment
 # fingerprint plus the content of its definition: command elements that are
 # files (the test binary, and the emulator for cross-compiled jobs) are
 # replaced by a digest of their bytes, remaining elements and the CTest
 # properties are taken literally with build-directory paths normalized away.
 # Tests whose key the manifest already records as a first-attempt pass
 # become the anchored exclusion regex in --skip-regex-out; every keyable
 # test goes to --keys-out for the later `record` step.
 #
 # The environment fingerprint covers everything that can change a test's
 # outcome while its binary stays identical: the image and its C library, the
 # CONFIG_PATHS subtrees of the checked-in CI configuration, the dpkg version
 # state of every lib* package (a superset of anything a test binary can
 # dynamically load, cross sysroots included), and KEYED_ENV_PREFIXES variables
 # read directly for values set outside the tree.  Job-to-job differences are
 # already isolated by the per-job-name GitLab cache key; the fingerprint guards
 # against one job's environment drifting over time.
 # A tool that is genuinely absent (e.g. no dpkg on the hand-assembled
 # riscv64 image) contributes a fixed sentinel so images with and without it
 # never share keys; unexpected errors propagate, `plan` fails, and the
 # driving script then runs the full selection.
 #
 # `record` folds the run's first-attempt results back into the manifest,
 # using the dashboard run's Test.xml as the authoritative status source:
 # only tests with Status="passed" are recorded ("failed" and "notrun" cover
 # failures, timeouts, missing executables, and SKIP_RETURN_CODE skips).
 # New passes are appended, entries that proved useful again (skipped or
 # re-passed) move to the end, and the file is trimmed to --max-entries, so
 # it ages out roughly least-recently-useful first.
 #
 # A test is keyable only if its command ends in an executable inside the
 # build directory.  Anything else (e.g. a test that invokes the compiler on
 # tree sources) has an outcome that depends on more than the artifact's
 # content and must always run.

 import argparse
 import functools
 import hashlib
 import json
 import os
 import re
 import subprocess
 import sys
 import xml.etree.ElementTree as ElementTree

 # Paths under the config root whose content can change a test's outcome while
 # its binary stays identical.  Deliberately not the whole ci/ tree: the
 # *.gitlab-ci.yml files are orchestration, and everything in them that reaches
 # a test outcome reaches this key by value already -- job variables through
 # KEYED_ENV_PREFIXES, the image through CI_JOB_IMAGE, compiler flags and the
 # cross emulator through the digests of the files in the test's command, ctest
 # timeouts through the properties hash.  Hashing the file that sets them only
 # means every CI-maintenance merge request throws away every job's manifest.
 #
 # ci/docker/ stays: those images are referenced by a moving :latest tag, so
 # CI_JOB_IMAGE cannot see a rebuild, and the dpkg-query below covers only lib*
 # packages, not a non-lib* one such as qemu-user.
 #
 # This is a whole-directory walk rather than a list of the scripts on the test
 # path, because a list is complete only until someone adds a script and forgets
 # to name it here -- and that failure mode is a silently reused stale pass.
 #
 # The exchange for the narrowing: a job's tags: are now invisible to the key.
 # Moving a job to a runner pool whose CPU differs (an AVX-512 job off the
 # avx512 pool, say) will not invalidate its manifest, so pair such a move with
 # a cache clear.  Nothing today distinguishes two hosts within one tag pool
 # either, so this widens an accepted hole rather than opening a new one.
 CONFIG_PATHS = ("ci/scripts", "ci/docker")

 # Environment variables (matched by prefix) that can change a test's outcome
 # without changing its binary.  Since the CI YAML is no longer hashed, this
 # tuple is load-bearing: a new job variable that can change a test's outcome
 # must be added here, because setting it in the YAML alone no longer keys it.
 KEYED_ENV_PREFIXES = (
     "EIGEN_REPEAT",
     "EIGEN_SEED",
     "EIGEN_CI_CTEST_ARGS",
     "QEMU_",
     "ASAN_",
     "UBSAN_",
     "LSAN_",
     "TSAN_",
     "MSAN_",
     "LD_LIBRARY_PATH",
     "LD_PRELOAD",
     # Host-derived values that legitimately vary across runners of one job
     # (EIGEN_CI_CTEST_PARALLEL, NPROC) are deliberately absent.
 )


 @functools.lru_cache(maxsize=None)
 def file_digest(path):
     with open(path, "rb") as f:
         return hashlib.file_digest(f, "sha256").hexdigest()


 def fingerprint(config_root):
     parts = ["image:" + os.environ.get("CI_JOB_IMAGE", "")]
     try:
         parts.append("libc:" + (os.confstr("CS_GNU_LIBC_VERSION") or ""))
     except (ValueError, OSError):
         parts.append("libc:unavailable")
     config = hashlib.sha256()
     for top in CONFIG_PATHS:
         top_path = os.path.join(config_root, top)
         if os.path.isfile(top_path):
             config.update(top.encode() + b"\0" + file_digest(top_path).encode() + b"\0")
             continue
         for dirpath, dirnames, filenames in os.walk(top_path):
             dirnames.sort()
             for filename in sorted(filenames):
                 path = os.path.join(dirpath, filename)
                 rel = os.path.relpath(path, config_root)
                 config.update(rel.encode() + b"\0" + file_digest(path).encode() + b"\0")
     parts.append("config:" + config.hexdigest())
     try:
         packages = subprocess.run(["dpkg-query", "-W", "lib*"], check=True, capture_output=True, text=True).stdout
         parts.append("libs:" + hashlib.sha256(packages.encode()).hexdigest())
     except (FileNotFoundError, subprocess.CalledProcessError):
         parts.append("libs:unavailable")
     env = ("%s=%s" % (k, v) for k, v in os.environ.items() if k.startswith(KEYED_ENV_PREFIXES))
     parts.append("env:" + ",".join(sorted(env)))
     return "|".join(parts)


 def normalize(text, builddir):
     """Strips the build-directory prefix so the key does not depend on where
     the checkout happens to live."""
     return text.replace(builddir, ".")


 def test_key(test, fingerprint_bytes, builddir):
     """Digest of the fingerprint and the test's definition, or None if the
     command does not end in an executable under the build directory."""
     command = test.get("command")
     if not command:
         return None
     last = os.path.realpath(command[-1])
     if not (last.startswith(builddir + os.sep) and os.path.isfile(last) and os.access(last, os.X_OK)):
         return None
     h = hashlib.sha256()
     h.update(fingerprint_bytes)
     for element in command:
         path = os.path.realpath(element)
         if os.path.isfile(path):
             h.update(b"file:" + file_digest(path).encode() + b"\0")
         else:
             h.update(b"arg:" + normalize(element, builddir).encode() + b"\0")
     properties = sorted(test.get("properties", []), key=lambda p: str(p.get("name")))
     h.update(b"props:" + normalize(json.dumps(properties, sort_keys=True), builddir).encode())
     return h.hexdigest()


 def read_manifest(path):
     """The manifest as an insertion-ordered {key: testname} dict."""
     entries = {}
     if os.path.exists(path):
         with open(path) as f:
             for line in f:
                 fields = line.split()
                 if fields:
                     entries[fields[0]] = fields[1] if len(fields) > 1 else ""
     return entries


 def write_manifest(path, entries, max_entries):
     tmp = path + ".tmp"
     with open(tmp, "w") as f:
         f.writelines("%s %s\n" % item for item in list(entries.items())[-max_entries:])
     os.replace(tmp, path)


 def plan(args):
     with (sys.stdin if args.tests_json == "-" else open(args.tests_json)) as f:
         tests = json.load(f)["tests"]
     manifest = read_manifest(args.manifest)
     builddir = os.path.realpath(os.getcwd())
     fingerprint_bytes = fingerprint(args.config_root).encode() + b"\0"
     keys = {}
     skip = []
     for test in tests:
         key = test_key(test, fingerprint_bytes, builddir)
         if key is None:
             continue
         keys[test["name"]] = key
         if key in manifest:
             skip.append(test["name"])
     with open(args.keys_out, "w") as f:
         f.writelines("%s %s\n" % (key, name) for name, key in keys.items())
     with open(args.skip_regex_out, "w") as f:
         if skip:
             f.write("^(%s)$" % "|".join(re.escape(name) for name in skip))
     print(
         "test cache plan: %d tests selected, %d cached passes to skip, %d to run (%d not keyable)"
         % (len(tests), len(skip), len(tests) - len(skip), len(tests) - len(keys))
     )


 def record(args):
     keys = {}
     with open(args.keys) as f:
         for line in f:
             fields = line.split()
             if len(fields) == 2:
                 keys[fields[1]] = fields[0]
     # The dashboard <Test> elements carry a Status attribute ("passed",
     # "failed", or "notrun"); the bare <TestList> entries do not and are
     # ignored.  Streamed with iterparse because --no-compress-output embeds
     # every test's stdout in the file.  A missing or unparsable Test.xml
     # means ctest died without completing the test phase: record nothing
     # rather than trust it.
     passed = set()
     try:
         with open(os.path.join(args.testing_dir, "TAG")) as f:
             tag = f.readline().strip()
         for _, element in ElementTree.iterparse(os.path.join(args.testing_dir, tag, "Test.xml")):
             if element.tag != "Test":
                 continue
             if element.get("Status") == "passed":
                 name = element.find("Name")
                 if name is not None:
                     passed.add(name.text)
             element.clear()
     except (OSError, ElementTree.ParseError) as error:
         print("test cache record: cannot read test results (%s); recording nothing" % error)
         return
     manifest = read_manifest(args.manifest)
     new = refreshed = 0
     for name, key in keys.items():
         if name in passed:
             if manifest.pop(key, None) is None:
                 new += 1
             else:
                 refreshed += 1
             manifest[key] = name
         elif manifest.pop(key, None) is not None:
             # Not run because this entry made plan skip it; move the entry to
             # the back so trimming ages out unused entries first.
             manifest[key] = name
             refreshed += 1
     write_manifest(args.manifest, manifest, args.max_entries)
     print(
         "test cache record: %d new first-attempt passes, %d entries refreshed, %d entries total"
         % (new, refreshed, len(manifest))
     )


 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     sub = parser.add_subparsers(dest="cmd", required=True)

     p = sub.add_parser("plan")
     p.add_argument("--tests-json", required=True, help="json-v1 dump, or - for stdin")
     p.add_argument("--manifest", required=True)
     p.add_argument("--config-root", required=True)
     p.add_argument("--skip-regex-out", required=True)
     p.add_argument("--keys-out", required=True)
     p.set_defaults(func=plan)

     p = sub.add_parser("record")
     p.add_argument("--manifest", required=True)
     p.add_argument("--keys", required=True)
     p.add_argument("--testing-dir", required=True)
     p.add_argument("--max-entries", type=int, default=20000)
     p.set_defaults(func=record)

     args = parser.parse_args()
     args.func(args)


 if __name__ == "__main__":
     main()
	#!/usr/bin/env python3
	# SPDX-FileCopyrightText: The Eigen Authors
	# SPDX-License-Identifier: MPL-2.0
	#
	# Content-addressed pass cache for CI test jobs (driven by
	# test.linux.script.sh, run from inside the build directory).
	#
	# `plan` reads a `ctest --show-only=json-v1` dump (stdin via
	# `--tests-json -`) and keys every enumerated test by an environment
	# fingerprint plus the content of its definition: command elements that are
	# files (the test binary, and the emulator for cross-compiled jobs) are
	# replaced by a digest of their bytes, remaining elements and the CTest
	# properties are taken literally with build-directory paths normalized away.
	# Tests whose key the manifest already records as a first-attempt pass
	# become the anchored exclusion regex in --skip-regex-out; every keyable
	# test goes to --keys-out for the later `record` step.
	#
	# The environment fingerprint covers everything that can change a test's
	# outcome while its binary stays identical: the image and its C library, the
	# CONFIG_PATHS subtrees of the checked-in CI configuration, the dpkg version
	# state of every lib* package (a superset of anything a test binary can
	# dynamically load, cross sysroots included), and KEYED_ENV_PREFIXES variables
	# read directly for values set outside the tree. Job-to-job differences are
	# already isolated by the per-job-name GitLab cache key; the fingerprint guards
	# against one job's environment drifting over time.
	# A tool that is genuinely absent (e.g. no dpkg on the hand-assembled
	# riscv64 image) contributes a fixed sentinel so images with and without it
	# never share keys; unexpected errors propagate, `plan` fails, and the
	# driving script then runs the full selection.
	#
	# `record` folds the run's first-attempt results back into the manifest,
	# using the dashboard run's Test.xml as the authoritative status source:
	# only tests with Status="passed" are recorded ("failed" and "notrun" cover
	# failures, timeouts, missing executables, and SKIP_RETURN_CODE skips).
	# New passes are appended, entries that proved useful again (skipped or
	# re-passed) move to the end, and the file is trimmed to --max-entries, so
	# it ages out roughly least-recently-useful first.
	#
	# A test is keyable only if its command ends in an executable inside the
	# build directory. Anything else (e.g. a test that invokes the compiler on
	# tree sources) has an outcome that depends on more than the artifact's
	# content and must always run.

	import argparse
	import functools
	import hashlib
	import json
	import os
	import re
	import subprocess
	import sys
	import xml.etree.ElementTree as ElementTree

	# Paths under the config root whose content can change a test's outcome while
	# its binary stays identical. Deliberately not the whole ci/ tree: the
	# *.gitlab-ci.yml files are orchestration, and everything in them that reaches
	# a test outcome reaches this key by value already -- job variables through
	# KEYED_ENV_PREFIXES, the image through CI_JOB_IMAGE, compiler flags and the
	# cross emulator through the digests of the files in the test's command, ctest
	# timeouts through the properties hash. Hashing the file that sets them only
	# means every CI-maintenance merge request throws away every job's manifest.
	#
	# ci/docker/ stays: those images are referenced by a moving :latest tag, so
	# CI_JOB_IMAGE cannot see a rebuild, and the dpkg-query below covers only lib*
	# packages, not a non-lib* one such as qemu-user.
	#
	# This is a whole-directory walk rather than a list of the scripts on the test
	# path, because a list is complete only until someone adds a script and forgets
	# to name it here -- and that failure mode is a silently reused stale pass.
	#
	# The exchange for the narrowing: a job's tags: are now invisible to the key.
	# Moving a job to a runner pool whose CPU differs (an AVX-512 job off the
	# avx512 pool, say) will not invalidate its manifest, so pair such a move with
	# a cache clear. Nothing today distinguishes two hosts within one tag pool
	# either, so this widens an accepted hole rather than opening a new one.
	CONFIG_PATHS = ("ci/scripts", "ci/docker")

	# Environment variables (matched by prefix) that can change a test's outcome
	# without changing its binary. Since the CI YAML is no longer hashed, this
	# tuple is load-bearing: a new job variable that can change a test's outcome
	# must be added here, because setting it in the YAML alone no longer keys it.
	KEYED_ENV_PREFIXES = (
	"EIGEN_REPEAT",
	"EIGEN_SEED",
	"EIGEN_CI_CTEST_ARGS",
	"QEMU_",
	"ASAN_",
	"UBSAN_",
	"LSAN_",
	"TSAN_",
	"MSAN_",
	"LD_LIBRARY_PATH",
	"LD_PRELOAD",
	# Host-derived values that legitimately vary across runners of one job
	# (EIGEN_CI_CTEST_PARALLEL, NPROC) are deliberately absent.
	)


	@functools.lru_cache(maxsize=None)
	def file_digest(path):
	with open(path, "rb") as f:
	return hashlib.file_digest(f, "sha256").hexdigest()


	def fingerprint(config_root):
	parts = ["image:" + os.environ.get("CI_JOB_IMAGE", "")]
	try:
	parts.append("libc:" + (os.confstr("CS_GNU_LIBC_VERSION") or ""))
	except (ValueError, OSError):
	parts.append("libc:unavailable")
	config = hashlib.sha256()
	for top in CONFIG_PATHS:
	top_path = os.path.join(config_root, top)
	if os.path.isfile(top_path):
	config.update(top.encode() + b"\0" + file_digest(top_path).encode() + b"\0")
	continue
	for dirpath, dirnames, filenames in os.walk(top_path):
	dirnames.sort()
	for filename in sorted(filenames):
	path = os.path.join(dirpath, filename)
	rel = os.path.relpath(path, config_root)
	config.update(rel.encode() + b"\0" + file_digest(path).encode() + b"\0")
	parts.append("config:" + config.hexdigest())
	try:
	packages = subprocess.run(["dpkg-query", "-W", "lib*"], check=True, capture_output=True, text=True).stdout
	parts.append("libs:" + hashlib.sha256(packages.encode()).hexdigest())
	except (FileNotFoundError, subprocess.CalledProcessError):
	parts.append("libs:unavailable")
	env = ("%s=%s" % (k, v) for k, v in os.environ.items() if k.startswith(KEYED_ENV_PREFIXES))
	parts.append("env:" + ",".join(sorted(env)))
	return "\|".join(parts)


	def normalize(text, builddir):
	"""Strips the build-directory prefix so the key does not depend on where
	the checkout happens to live."""
	return text.replace(builddir, ".")


	def test_key(test, fingerprint_bytes, builddir):
	"""Digest of the fingerprint and the test's definition, or None if the
	command does not end in an executable under the build directory."""
	command = test.get("command")
	if not command:
	return None
	last = os.path.realpath(command[-1])
	if not (last.startswith(builddir + os.sep) and os.path.isfile(last) and os.access(last, os.X_OK)):
	return None
	h = hashlib.sha256()
	h.update(fingerprint_bytes)
	for element in command:
	path = os.path.realpath(element)
	if os.path.isfile(path):
	h.update(b"file:" + file_digest(path).encode() + b"\0")
	else:
	h.update(b"arg:" + normalize(element, builddir).encode() + b"\0")
	properties = sorted(test.get("properties", []), key=lambda p: str(p.get("name")))
	h.update(b"props:" + normalize(json.dumps(properties, sort_keys=True), builddir).encode())
	return h.hexdigest()


	def read_manifest(path):
	"""The manifest as an insertion-ordered {key: testname} dict."""
	entries = {}
	if os.path.exists(path):
	with open(path) as f:
	for line in f:
	fields = line.split()
	if fields:
	entries[fields[0]] = fields[1] if len(fields) > 1 else ""
	return entries


	def write_manifest(path, entries, max_entries):
	tmp = path + ".tmp"
	with open(tmp, "w") as f:
	f.writelines("%s %s\n" % item for item in list(entries.items())[-max_entries:])
	os.replace(tmp, path)


	def plan(args):
	with (sys.stdin if args.tests_json == "-" else open(args.tests_json)) as f:
	tests = json.load(f)["tests"]
	manifest = read_manifest(args.manifest)
	builddir = os.path.realpath(os.getcwd())
	fingerprint_bytes = fingerprint(args.config_root).encode() + b"\0"
	keys = {}
	skip = []
	for test in tests:
	key = test_key(test, fingerprint_bytes, builddir)
	if key is None:
	continue
	keys[test["name"]] = key
	if key in manifest:
	skip.append(test["name"])
	with open(args.keys_out, "w") as f:
	f.writelines("%s %s\n" % (key, name) for name, key in keys.items())
	with open(args.skip_regex_out, "w") as f:
	if skip:
	f.write("^(%s)$" % "\|".join(re.escape(name) for name in skip))
	print(
	"test cache plan: %d tests selected, %d cached passes to skip, %d to run (%d not keyable)"
	% (len(tests), len(skip), len(tests) - len(skip), len(tests) - len(keys))
	)


	def record(args):
	keys = {}
	with open(args.keys) as f:
	for line in f:
	fields = line.split()
	if len(fields) == 2:
	keys[fields[1]] = fields[0]
	# The dashboard <Test> elements carry a Status attribute ("passed",
	# "failed", or "notrun"); the bare <TestList> entries do not and are
	# ignored. Streamed with iterparse because --no-compress-output embeds
	# every test's stdout in the file. A missing or unparsable Test.xml
	# means ctest died without completing the test phase: record nothing
	# rather than trust it.
	passed = set()
	try:
	with open(os.path.join(args.testing_dir, "TAG")) as f:
	tag = f.readline().strip()
	for _, element in ElementTree.iterparse(os.path.join(args.testing_dir, tag, "Test.xml")):
	if element.tag != "Test":
	continue
	if element.get("Status") == "passed":
	name = element.find("Name")
	if name is not None:
	passed.add(name.text)
	element.clear()
	except (OSError, ElementTree.ParseError) as error:
	print("test cache record: cannot read test results (%s); recording nothing" % error)
	return
	manifest = read_manifest(args.manifest)
	new = refreshed = 0
	for name, key in keys.items():
	if name in passed:
	if manifest.pop(key, None) is None:
	new += 1
	else:
	refreshed += 1
	manifest[key] = name
	elif manifest.pop(key, None) is not None:
	# Not run because this entry made plan skip it; move the entry to
	# the back so trimming ages out unused entries first.
	manifest[key] = name
	refreshed += 1
	write_manifest(args.manifest, manifest, args.max_entries)
	print(
	"test cache record: %d new first-attempt passes, %d entries refreshed, %d entries total"
	% (new, refreshed, len(manifest))
	)


	def main():
	parser = argparse.ArgumentParser(description=__doc__)
	sub = parser.add_subparsers(dest="cmd", required=True)

	p = sub.add_parser("plan")
	p.add_argument("--tests-json", required=True, help="json-v1 dump, or - for stdin")
	p.add_argument("--manifest", required=True)
	p.add_argument("--config-root", required=True)
	p.add_argument("--skip-regex-out", required=True)
	p.add_argument("--keys-out", required=True)
	p.set_defaults(func=plan)

	p = sub.add_parser("record")
	p.add_argument("--manifest", required=True)
	p.add_argument("--keys", required=True)
	p.add_argument("--testing-dir", required=True)
	p.add_argument("--max-entries", type=int, default=20000)
	p.set_defaults(func=record)

	args = parser.parse_args()
	args.func(args)


	if __name__ == "__main__":
	main()