CI: Address MR review feedback on sccache overrides and multi-arch caching
Address review comments from @rmlarsen1 on MR !3171:
- Add sccache directory and size override fallback chain (SCCACHE_DIR, EIGEN_CI_SCCACHE_DIR) to respect runner-level overrides while capping default project-local cache at 4G.
- Print notice on download_file failure when curl, wget, and python3 are all unavailable.
- Guard SCCACHE_GCS_CREDENTIALS_URL export in build scripts when credential server port is empty.
- Key compiler cache binary directories by architecture (.sccache-bin/${arch}, .ccache-bin/${arch}) so multi-arch runners share linux-ccache-bin without thrashing.
diff --git a/ci/scripts/build.linux.script.sh b/ci/scripts/build.linux.script.sh
index 04fffae..b0572db 100755
--- a/ci/scripts/build.linux.script.sh
+++ b/ci/scripts/build.linux.script.sh
@@ -33,8 +33,8 @@
. "${rootdir}/ci/scripts/install_compiler_cache.sh"
if [[ "${EIGEN_CI_SCCACHE:-on}" != "off" && -n "${sccache_bin}" ]]; then
- export SCCACHE_DIR="${CI_PROJECT_DIR}/.sccache"
- export SCCACHE_CACHE_SIZE="${CCACHE_MAXSIZE:-4G}"
+ export SCCACHE_DIR="${SCCACHE_DIR:-${EIGEN_CI_SCCACHE_DIR:-${CI_PROJECT_DIR}/.sccache}}"
+ export SCCACHE_CACHE_SIZE="${SCCACHE_CACHE_SIZE:-${EIGEN_CI_SCCACHE_CACHE_SIZE:-4G}}"
export SCCACHE_BASEDIRS="${rootdir}"
export SCCACHE_SERVER_PORT="$((4226 + ${CI_CONCURRENT_ID:-0}))"
@@ -78,6 +78,7 @@
set -x
# Wait up to 2 seconds for local credential server to be ready
+ cred_port=""
for _ in {1..40}; do
if [[ -s "${cred_port_file}" ]]; then
cred_port=$(cat "${cred_port_file}")
@@ -85,7 +86,16 @@
fi
sleep 0.05
done
- export SCCACHE_GCS_CREDENTIALS_URL="http://127.0.0.1:${cred_port}/token/${GCS_URL_SECRET}"
+ if [[ -n "${cred_port:-}" ]]; then
+ export SCCACHE_GCS_CREDENTIALS_URL="http://127.0.0.1:${cred_port}/token/${GCS_URL_SECRET}"
+ else
+ echo "Notice: Local credential server failed to bind or respond; skipping GCS remote cache." >&2
+ unset SCCACHE_GCS_BUCKET
+ unset SCCACHE_GCS_RW_MODE
+ unset SCCACHE_MULTILEVEL_CHAIN
+ [[ -n "${sccache_cred_server_pid}" ]] && kill "${sccache_cred_server_pid}" 2>/dev/null || true
+ sccache_cred_server_pid=""
+ fi
rm -f "${cred_port_file}"
unset GCS_URL_SECRET
fi
diff --git a/ci/scripts/build.windows.script.ps1 b/ci/scripts/build.windows.script.ps1
index bac52b6..9fb4a05 100644
--- a/ci/scripts/build.windows.script.ps1
+++ b/ci/scripts/build.windows.script.ps1
@@ -85,8 +85,8 @@
# 1. Try starting sccache server if available
if ($sccache_exe) {
- $env:SCCACHE_DIR = Join-Path $env:CI_PROJECT_DIR ".sccache"
- $env:SCCACHE_CACHE_SIZE = if ($env:EIGEN_CI_CCACHE_MAXSIZE) { $env:EIGEN_CI_CCACHE_MAXSIZE } else { "4G" }
+ $env:SCCACHE_DIR = if ($env:SCCACHE_DIR) { $env:SCCACHE_DIR } elseif ($env:EIGEN_CI_SCCACHE_DIR) { $env:EIGEN_CI_SCCACHE_DIR } else { Join-Path $env:CI_PROJECT_DIR ".sccache" }
+ $env:SCCACHE_CACHE_SIZE = if ($env:SCCACHE_CACHE_SIZE) { $env:SCCACHE_CACHE_SIZE } elseif ($env:EIGEN_CI_SCCACHE_CACHE_SIZE) { $env:EIGEN_CI_SCCACHE_CACHE_SIZE } else { "4G" }
# Rewrite paths relative to rootdir for cross-runner / cross-directory cache hits.
$env:SCCACHE_BASEDIRS = $rootdir
# Isolate daemon port per runner slot to avoid port collisions and process cross-kill.
@@ -151,7 +151,19 @@
}
Start-Sleep -Milliseconds 50
}
- $env:SCCACHE_GCS_CREDENTIALS_URL = "http://127.0.0.1:$cred_port/token/$url_secret"
+ if ($cred_port) {
+ $env:SCCACHE_GCS_CREDENTIALS_URL = "http://127.0.0.1:$cred_port/token/$url_secret"
+ } else {
+ Write-Warning "Local credential server failed to bind or respond; skipping GCS remote cache."
+ Remove-Item Env:SCCACHE_GCS_BUCKET -ErrorAction SilentlyContinue
+ Remove-Item Env:SCCACHE_GCS_RW_MODE -ErrorAction SilentlyContinue
+ Remove-Item Env:SCCACHE_MULTILEVEL_CHAIN -ErrorAction SilentlyContinue
+ if ($cred_server_job) {
+ Stop-Job $cred_server_job -ErrorAction SilentlyContinue
+ Remove-Job $cred_server_job -Force -ErrorAction SilentlyContinue
+ $cred_server_job = $null
+ }
+ }
}
& $sccache_exe --start-server | Out-Null
diff --git a/ci/scripts/install_compiler_cache.sh b/ci/scripts/install_compiler_cache.sh
index cc16828..d3cc403 100755
--- a/ci/scripts/install_compiler_cache.sh
+++ b/ci/scripts/install_compiler_cache.sh
@@ -21,6 +21,7 @@
if command -v python3 >/dev/null 2>&1; then
python3 -c "import sys, urllib.request; urllib.request.urlretrieve(sys.argv[1], sys.argv[2])" "${url}" "${dest}" && return 0
fi
+ echo "Notice: Neither curl, wget, nor python3 is available to download $(basename "${dest}")." >&2
return 1
}
@@ -29,7 +30,7 @@
if [[ -z "${sccache_bin}" ]]; then
sccache_ver="0.18.0"
- sccache_bindir="${root_dir}/.sccache-bin"
+ sccache_bindir="${root_dir}/.sccache-bin/${arch}"
sccache_tar_sha256=""
sccache_bin_sha256=""
@@ -78,7 +79,7 @@
if [[ -z "${ccache_bin}" ]]; then
ccache_ver="4.13.6"
- ccache_bindir="${root_dir}/.ccache-bin"
+ ccache_bindir="${root_dir}/.ccache-bin/${arch}"
ccache_tar_sha256=""
ccache_bin_sha256=""