From 7a07251acfa3ecf27aab252d3c1c2e98e9eb1d7a Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:05:59 +0200 Subject: [PATCH] CI disk: short artifact retention, runner cleanup, sccache on the Mac runners Releases now ship as often as changes land, so nothing they leave behind may grow without bound. Workflow uploads keep 3 days (nightly parity logs 7); the releases carry their own assets. tools/ci/runner-gc.sh, run every 6 hours by launchd on the Mac mini from a copy of main, skips a runner in a job, removes a checkout's cargo build folders past 12 GB and harness work folders under .local/ untouched for 3 days, and keeps the 2 newest shared preparations while no runner works. The runners build through the machine's sccache, so what it removes comes back from the cache. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 ++ .github/workflows/launcher.yml | 3 +- .github/workflows/nightly.yml | 2 +- .github/workflows/release.yml | 3 +- docs/release-flow.md | 13 +++++++++ tools/ci/runner-env.sh | 5 ++++ tools/ci/runner-gc.sh | 53 ++++++++++++++++++++++++++++++++++ 7 files changed, 78 insertions(+), 3 deletions(-) create mode 100755 tools/ci/runner-gc.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd9cde557..57b7842b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,6 +80,7 @@ jobs: name: parity-logs path: .local/parity-runner/logs/ if-no-files-found: ignore + retention-days: 3 # Horizon never downloads the game. These jobs need a self-hosted Linux runner # (label horizon-game) holding the maintainer's own copy: the official launcher's # game directory in HORIZON_MINECRAFT_DIR and the pinned server jar in @@ -541,3 +542,4 @@ jobs: dist/client-graphics-smoke.json dist/client-graphics-macos-title.json dist/client-graphics-macos-world.json + retention-days: 3 diff --git a/.github/workflows/launcher.yml b/.github/workflows/launcher.yml index 60d0ba5c1..f0ac59a2b 100644 --- a/.github/workflows/launcher.yml +++ b/.github/workflows/launcher.yml @@ -209,7 +209,7 @@ jobs: with: name: ${{ matrix.asset }} path: launcher/out/* - retention-days: 7 + retention-days: 3 release: needs: [decide, build] @@ -250,3 +250,4 @@ jobs: with: name: launcher-release-notes path: notes/ + retention-days: 3 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 59560e262..689cdae59 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -76,4 +76,4 @@ jobs: name: nightly-parity-logs path: .local/parity-runner/logs/ if-no-files-found: ignore - retention-days: 14 + retention-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0ffa29d26..3162b6f1b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -181,7 +181,7 @@ jobs: name: ${{ matrix.asset }} path: dist/${{ matrix.asset }} if-no-files-found: error - retention-days: 7 + retention-days: 3 publish: needs: [decide, build] @@ -222,3 +222,4 @@ jobs: with: name: release-notes path: notes/ + retention-days: 3 diff --git a/docs/release-flow.md b/docs/release-flow.md index 4a8951296..f494bf538 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -229,3 +229,16 @@ time, on the Mac mini's runner, after preparing the resources again if the prepa The bugs session picks the issues up. The audit's logs are the run's `nightly-parity-logs` artifact; harnesses listed `linux_only` are skipped there (see above). By hand: `gh workflow run "nightly audit"` (add `-f dry_run=true` to leave issues alone). + +## Disk + +Releases ship often, so nothing they leave behind may grow without bound. + +- **Artifacts:** every upload keeps 3 days (nightly parity logs 7); a release's assets live on the release itself. +- **Mac runners:** `tools/ci/runner-gc.sh` runs every 6 hours from launchd on the Mac mini, as a copy taken from main + (`~/.actions-runners/runner-gc.sh`, plist `dev.horizon.runner-gc`), never from a pull request's checkout. It skips a + runner in a job, removes a checkout's cargo build folders past 12 GB (`MAX_TARGET_GB`) and harness work folders + under `.local/` untouched for 3 days (`STALE_DAYS`), and keeps the 2 newest shared preparations (`KEEP_PREPARED`) + while no runner works. Runner builds go through the machine's sccache (`tools/ci/runner-env.sh`), so what it removes + comes back from the cache. Update the copy after changing the script: + `ssh macmini 'curl -fsSL https://raw.githubusercontent.com/ludovic111/horizon/main/tools/ci/runner-gc.sh -o ~/.actions-runners/runner-gc.sh'`. diff --git a/tools/ci/runner-env.sh b/tools/ci/runner-env.sh index b4c5094bd..49158ac91 100755 --- a/tools/ci/runner-env.sh +++ b/tools/ci/runner-env.sh @@ -12,5 +12,10 @@ if [ "$(uname)" = Darwin ]; then java_home=$(/usr/libexec/java_home -v "$java_major") { echo "$bin"; echo "$java_home/bin"; echo /opt/homebrew/bin; } >> "$GITHUB_PATH" echo "JAVA_HOME=$java_home" >> "$GITHUB_ENV" + # Builds go through the machine's sccache (shared with ~/fleet/rbuild), so the build folders runner-gc.sh + # removes come back from the cache instead of compiling cold. + if [ -x /opt/homebrew/bin/sccache ]; then + { echo "RUSTC_WRAPPER=/opt/homebrew/bin/sccache"; echo "CARGO_INCREMENTAL=0"; } >> "$GITHUB_ENV" + fi fi echo "HORIZON_NO_AUDIO=1" >> "$GITHUB_ENV" diff --git a/tools/ci/runner-gc.sh b/tools/ci/runner-gc.sh new file mode 100755 index 000000000..0f4f1e4c2 --- /dev/null +++ b/tools/ci/runner-gc.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Keeps the Mac mini's Horizon runners from filling its disk. launchd runs a copy taken from main +# (~/.actions-runners/runner-gc.sh, never a pull request's checkout) every 6 hours; see docs/release-flow.md. +# - A runner in the middle of a job is left alone. +# - A checkout whose cargo build folders (target/ and the harness probes' targets under .local/, the +# directories holding a CACHEDIR.TAG) pass MAX_TARGET_GB loses them: sccache (tools/ci/runner-env.sh) +# makes the next build cheap. Symlinks into the shared vanilla data are never followed. +# - Harness work folders under .local/ (vanilla exports, probe outputs: real directories, not the links into +# the shared vanilla data) that nothing touched for STALE_DAYS go; a harness that runs again rebuilds its own. +# .local/parity-runner (run state, oracle cache) stays. +# - The shared prepared resources keep their KEEP_PREPARED newest preparations, only while no runner works. +set -euo pipefail +root=${RUNNERS:-$HOME/.actions-runners} +max_kb=$(( ${MAX_TARGET_GB:-12} * 1024 * 1024 )) +keep=${KEEP_PREPARED:-2} +stale_min=$(( ${STALE_DAYS:-3} * 24 * 60 )) +busy() { pgrep -f "$1/bin/Runner.Worker" > /dev/null; } + +any_busy=0 +for runner in "$root"/horizon "$root"/horizon-[0-9]*; do + [ -d "$runner/_work" ] || continue + if busy "$runner"; then echo "$(basename "$runner"): in a job, skipped"; any_busy=1; continue; fi + rm -rf "${runner:?}/_work/_temp/"* 2>/dev/null || true + checkout="$runner/_work/horizon/horizon" + [ -d "$checkout" ] || continue + stale=0 + for dir in "$checkout"/.local/*/; do + dir=${dir%/} + [ -L "$dir" ] || [ "$(basename "$dir")" = parity-runner ] && continue + if [ -z "$(find "$dir" -mmin -"$stale_min" -print -quit 2>/dev/null)" ]; then rm -rf "$dir"; stale=$((stale + 1)); fi + done + [ "$stale" = 0 ] || echo "$(basename "$runner"): removed $stale harness folders untouched for ${STALE_DAYS:-3} days" + targets=() + [ -f "$checkout/target/CACHEDIR.TAG" ] && targets+=("$checkout/target") + while IFS= read -r tag; do targets+=("$(dirname "$tag")"); done < <( + find "$checkout/.local" -mindepth 2 -maxdepth 2 -name CACHEDIR.TAG -type f 2>/dev/null) + [ ${#targets[@]} -gt 0 ] || continue + size=$(du -sk "${targets[@]}" | awk '{s += $1} END {print s}') + if [ "$size" -gt "$max_kb" ]; then + rm -rf "${targets[@]}" + echo "$(basename "$runner"): removed ${#targets[@]} build folders ($(( size / 1024 / 1024 )) GB)" + fi +done + +prepared="$root/horizon/_work/horizon/shared-local/prepared" +if [ "$any_busy" = 0 ] && [ -d "$prepared" ] && mkdir "$prepared/.lock" 2>/dev/null; then + trap 'rmdir "$prepared/.lock" 2>/dev/null || true' EXIT + ls -1t "$prepared" | tail -n +"$(( keep + 1 ))" | while read -r old; do + rm -rf "${prepared:?}/$old" + echo "prepared resources $old removed" + done +fi +df -h "$root" | tail -1