diff --git a/docs/release-flow.md b/docs/release-flow.md index ed7be2418..e6a0a3856 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -432,6 +432,9 @@ Releases ship often, so nothing they leave behind may grow without bound. (`~/.actions-runners/runner-gc.sh`, plist `dev.horizon.runner-gc`), never from a pull request's checkout. It skips a runner in a job, removes a checkout's cargo build folders past 12 GB (`MAX_TARGET_GB`) and harness work folders under `.local/` untouched for 3 days (`STALE_DAYS`), and keeps the 2 newest shared preparations (`KEEP_PREPARED`) - while no runner works. Runner builds go through the machine's sccache (`tools/ci/runner-env.sh`), so what it removes + while no runner works. It also stops harness JVMs (vanilla servers, oracles) left orphaned for 3 hours + (`ORPHAN_HOURS`): parent gone, command line under a runner's checkout or a worktree's `.local/`. `tools/parity.py` + stops its own harnesses on SIGTERM and SIGHUP (a cancelled job), so these come only from a SIGKILL or a harness run + by hand. Runner builds go through the machine's sccache (`tools/ci/runner-env.sh`), so what it removes comes back from the cache. Update the copy after changing the script: `ssh macmini 'curl -fsSL https://raw.githubusercontent.com/ludovic111/horizon/main/tools/ci/runner-gc.sh -o ~/.actions-runners/runner-gc.sh'`. diff --git a/tools/ci/runner-gc.sh b/tools/ci/runner-gc.sh index 0f4f1e4c2..7ccf69ff0 100755 --- a/tools/ci/runner-gc.sh +++ b/tools/ci/runner-gc.sh @@ -9,6 +9,10 @@ # the shared vanilla data) that nothing touched for STALE_DAYS go; a harness that runs again rebuilds its own. # .local/parity-runner (run state, oracle cache) stays. # - The shared prepared resources keep their KEEP_PREPARED newest preparations, only while no runner works. +# - Java processes harnesses left behind (vanilla servers, oracles) go once ORPHAN_HOURS old: those whose parent +# is gone (adopted by launchd, init or systemd --user) and whose command line names a runner's checkout or a +# worktree's .local/. A job cancelled while it ran left such servers for days (2026-10-26); tools/parity.py now +# stops its harnesses on SIGTERM, and this catches what it can't (SIGKILL, a harness run by hand). set -euo pipefail root=${RUNNERS:-$HOME/.actions-runners} max_kb=$(( ${MAX_TARGET_GB:-12} * 1024 * 1024 )) @@ -42,6 +46,25 @@ for runner in "$root"/horizon "$root"/horizon-[0-9]*; do fi done +# [[dd-]hh:]mm:ss, as ps prints a process's age, in minutes. +minutes() { + local t=$1 d=0 h=0 m a b c + if [[ $t == *-* ]]; then d=${t%%-*}; t=${t#*-}; fi + IFS=: read -r a b c <<< "$t" + if [ -n "$c" ]; then h=$a m=$b; else m=$a; fi + echo $(( 10#$d * 1440 + 10#$h * 60 + 10#$m )) +} +orphan_min=$(( ${ORPHAN_HOURS:-3} * 60 )) +ps -A -o pid=,ppid=,etime=,command= | while read -r pid ppid etime command; do + case "$command" in *java*) ;; *) continue ;; esac + case "$command" in *"$root"/*/_work/*|*/.claude/worktrees/*/.local/*|*/code/horizon/.local/*) ;; *) continue ;; esac + [ "$(minutes "$etime")" -ge "$orphan_min" ] || continue + if [ "$ppid" != 1 ]; then + case "$(ps -o command= -p "$ppid" 2>/dev/null)" in *"systemd --user"*) ;; *) continue ;; esac + fi + if kill "$pid" 2>/dev/null; then echo "orphaned java $pid ($etime old) stopped: ${command:0:160}"; fi +done + prepared="$root/horizon/_work/horizon/shared-local/prepared" if [ "$any_busy" = 0 ] && [ -d "$prepared" ] && mkdir "$prepared/.lock" 2>/dev/null; then trap 'rmdir "$prepared/.lock" 2>/dev/null || true' EXIT diff --git a/tools/parity.py b/tools/parity.py index 58e3b4d13..cd8d1e2f6 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -48,6 +48,7 @@ import os from pathlib import Path, PurePosixPath import re import shutil +import signal import socket import subprocess import sys @@ -1134,6 +1135,14 @@ def run(selected, state, hasher, environment_digest, executors, reuse_oracles, r return results, time.monotonic() - started_all +def exit_on_termination(): + """SIGTERM and SIGHUP (a cancelled CI job, a dropped ssh) end this process through its finally blocks, which stop + the running harnesses. Each runs in its own session, so without that they, and the vanilla servers they started, + outlive it (orphans of days were found on the Mac mini's runners, 2026-10-26).""" + for number in (signal.SIGTERM, signal.SIGHUP): + signal.signal(number, lambda received, _frame: sys.exit(128 + received)) + + def worker_main(shared_local): """`parity.py --worker SHARED_LOCAL`: run harnesses sent as JSON lines on stdin, answer on stdout.""" import queue as queues @@ -1169,13 +1178,20 @@ def worker_main(shared_local): built, errors = prebuild(first["prebuild"], default_build_jobs()) if first["prebuild"] else (True, "") executor = LocalExecutor(MACHINE_JOBS, int(memory_available() * 0.75), first["timeout"]) send({"hello": {"jobs": MACHINE_JOBS, "memory": executor.memory_budget, "built": built, "errors": errors[-4000:]}}) + try: + return serve(executor, inbox, queues.Empty, send, state, hasher) + finally: + executor.stop() + + +def serve(executor, inbox, empty, send, state, hasher): + """The worker's loop: start what the coordinating runner sends, report what finished, until its stdin ends.""" while True: try: message = inbox.get(timeout=0.2) - except queues.Empty: + except empty: message = False if message is None: - executor.stop() save_state(state) return 0 if message: @@ -1479,6 +1495,7 @@ def finished_run(failed): def main(): if len(sys.argv) > 1 and sys.argv[1] == "--trace": return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:]) + exit_on_termination() if len(sys.argv) > 1 and sys.argv[1] == "--worker": return worker_main(sys.argv[2]) parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) @@ -1608,6 +1625,7 @@ def main(): jobs = 1 # harnesses that can only run here still need a slot remotes = [RemoteExecutor(host, config["hosts"][host], artifacts, config.get("local_only", {}), args.timeout) for host in hosts] + local = None try: if jobs: print("Building probes...", flush=True) @@ -1620,12 +1638,12 @@ def main(): where = "".join(f", plus {host}" for host in hosts) print(f"Running {len(selected)} harnesses, {jobs} at a time here{where}...", flush=True) reuse_oracles = not (args.fresh or args.all) - executors = [*remotes, LocalExecutor(jobs, int(memory_available() * 0.75), args.timeout, - REMOTE_LOCAL_PEAK if remotes and args.jobs is None else None)] - results, elapsed = run(selected, state, hasher, environment_key(hasher), executors, reuse_oracles, rounds, - known_flaky(config)) + local = LocalExecutor(jobs, int(memory_available() * 0.75), args.timeout, + REMOTE_LOCAL_PEAK if remotes and args.jobs is None else None) + results, elapsed = run(selected, state, hasher, environment_key(hasher), [*remotes, local], reuse_oracles, + rounds, known_flaky(config)) finally: - for executor in remotes: + for executor in [*remotes, *([local] if local else [])]: executor.stop() save_state(state) failed = sorted(name for name, record in results.items() if record["status"] != "pass")