Merge pull request #2486 from ludovic111/fix/parity-reap-orphans

🤖 Agent: parity.py reaps the harness processes a failed or timed-out run leaves behind; parity passed on playhorizon.
This commit is contained in:
Ludovic Marie
2026-10-27 04:16:36 +01:00
committed by GitHub
3 changed files with 52 additions and 8 deletions
+4 -1
View File
@@ -432,6 +432,9 @@ Releases ship often, so nothing they leave behind may grow without bound.
(`~/.actions-runners/runner-gc.sh`, plist `dev.horizon.runner-gc`), never from a pull request's checkout. It skips a
runner in a job, removes a checkout's cargo build folders past 12 GB (`MAX_TARGET_GB`) and harness work folders
under `.local/` untouched for 3 days (`STALE_DAYS`), and keeps the 2 newest shared preparations (`KEEP_PREPARED`)
while no runner works. Runner builds go through the machine's sccache (`tools/ci/runner-env.sh`), so what it removes
while no runner works. It also stops harness JVMs (vanilla servers, oracles) left orphaned for 3 hours
(`ORPHAN_HOURS`): parent gone, command line under a runner's checkout or a worktree's `.local/`. `tools/parity.py`
stops its own harnesses on SIGTERM and SIGHUP (a cancelled job), so these come only from a SIGKILL or a harness run
by hand. Runner builds go through the machine's sccache (`tools/ci/runner-env.sh`), so what it removes
comes back from the cache. Update the copy after changing the script:
`ssh macmini 'curl -fsSL https://raw.githubusercontent.com/ludovic111/horizon/main/tools/ci/runner-gc.sh -o ~/.actions-runners/runner-gc.sh'`.
+23
View File
@@ -9,6 +9,10 @@
# the shared vanilla data) that nothing touched for STALE_DAYS go; a harness that runs again rebuilds its own.
# .local/parity-runner (run state, oracle cache) stays.
# - The shared prepared resources keep their KEEP_PREPARED newest preparations, only while no runner works.
# - Java processes harnesses left behind (vanilla servers, oracles) go once ORPHAN_HOURS old: those whose parent
# is gone (adopted by launchd, init or systemd --user) and whose command line names a runner's checkout or a
# worktree's .local/. A job cancelled while it ran left such servers for days (2026-10-26); tools/parity.py now
# stops its harnesses on SIGTERM, and this catches what it can't (SIGKILL, a harness run by hand).
set -euo pipefail
root=${RUNNERS:-$HOME/.actions-runners}
max_kb=$(( ${MAX_TARGET_GB:-12} * 1024 * 1024 ))
@@ -42,6 +46,25 @@ for runner in "$root"/horizon "$root"/horizon-[0-9]*; do
fi
done
# [[dd-]hh:]mm:ss, as ps prints a process's age, in minutes.
minutes() {
local t=$1 d=0 h=0 m a b c
if [[ $t == *-* ]]; then d=${t%%-*}; t=${t#*-}; fi
IFS=: read -r a b c <<< "$t"
if [ -n "$c" ]; then h=$a m=$b; else m=$a; fi
echo $(( 10#$d * 1440 + 10#$h * 60 + 10#$m ))
}
orphan_min=$(( ${ORPHAN_HOURS:-3} * 60 ))
ps -A -o pid=,ppid=,etime=,command= | while read -r pid ppid etime command; do
case "$command" in *java*) ;; *) continue ;; esac
case "$command" in *"$root"/*/_work/*|*/.claude/worktrees/*/.local/*|*/code/horizon/.local/*) ;; *) continue ;; esac
[ "$(minutes "$etime")" -ge "$orphan_min" ] || continue
if [ "$ppid" != 1 ]; then
case "$(ps -o command= -p "$ppid" 2>/dev/null)" in *"systemd --user"*) ;; *) continue ;; esac
fi
if kill "$pid" 2>/dev/null; then echo "orphaned java $pid ($etime old) stopped: ${command:0:160}"; fi
done
prepared="$root/horizon/_work/horizon/shared-local/prepared"
if [ "$any_busy" = 0 ] && [ -d "$prepared" ] && mkdir "$prepared/.lock" 2>/dev/null; then
trap 'rmdir "$prepared/.lock" 2>/dev/null || true' EXIT
+25 -7
View File
@@ -48,6 +48,7 @@ import os
from pathlib import Path, PurePosixPath
import re
import shutil
import signal
import socket
import subprocess
import sys
@@ -1134,6 +1135,14 @@ def run(selected, state, hasher, environment_digest, executors, reuse_oracles, r
return results, time.monotonic() - started_all
def exit_on_termination():
"""SIGTERM and SIGHUP (a cancelled CI job, a dropped ssh) end this process through its finally blocks, which stop
the running harnesses. Each runs in its own session, so without that they, and the vanilla servers they started,
outlive it (orphans of days were found on the Mac mini's runners, 2026-10-26)."""
for number in (signal.SIGTERM, signal.SIGHUP):
signal.signal(number, lambda received, _frame: sys.exit(128 + received))
def worker_main(shared_local):
"""`parity.py --worker SHARED_LOCAL`: run harnesses sent as JSON lines on stdin, answer on stdout."""
import queue as queues
@@ -1169,13 +1178,20 @@ def worker_main(shared_local):
built, errors = prebuild(first["prebuild"], default_build_jobs()) if first["prebuild"] else (True, "")
executor = LocalExecutor(MACHINE_JOBS, int(memory_available() * 0.75), first["timeout"])
send({"hello": {"jobs": MACHINE_JOBS, "memory": executor.memory_budget, "built": built, "errors": errors[-4000:]}})
try:
return serve(executor, inbox, queues.Empty, send, state, hasher)
finally:
executor.stop()
def serve(executor, inbox, empty, send, state, hasher):
"""The worker's loop: start what the coordinating runner sends, report what finished, until its stdin ends."""
while True:
try:
message = inbox.get(timeout=0.2)
except queues.Empty:
except empty:
message = False
if message is None:
executor.stop()
save_state(state)
return 0
if message:
@@ -1479,6 +1495,7 @@ def finished_run(failed):
def main():
if len(sys.argv) > 1 and sys.argv[1] == "--trace":
return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:])
exit_on_termination()
if len(sys.argv) > 1 and sys.argv[1] == "--worker":
return worker_main(sys.argv[2])
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -1608,6 +1625,7 @@ def main():
jobs = 1 # harnesses that can only run here still need a slot
remotes = [RemoteExecutor(host, config["hosts"][host], artifacts, config.get("local_only", {}), args.timeout)
for host in hosts]
local = None
try:
if jobs:
print("Building probes...", flush=True)
@@ -1620,12 +1638,12 @@ def main():
where = "".join(f", plus {host}" for host in hosts)
print(f"Running {len(selected)} harnesses, {jobs} at a time here{where}...", flush=True)
reuse_oracles = not (args.fresh or args.all)
executors = [*remotes, LocalExecutor(jobs, int(memory_available() * 0.75), args.timeout,
REMOTE_LOCAL_PEAK if remotes and args.jobs is None else None)]
results, elapsed = run(selected, state, hasher, environment_key(hasher), executors, reuse_oracles, rounds,
known_flaky(config))
local = LocalExecutor(jobs, int(memory_available() * 0.75), args.timeout,
REMOTE_LOCAL_PEAK if remotes and args.jobs is None else None)
results, elapsed = run(selected, state, hasher, environment_key(hasher), [*remotes, local], reuse_oracles,
rounds, known_flaky(config))
finally:
for executor in remotes:
for executor in [*remotes, *([local] if local else [])]:
executor.stop()
save_state(state)
failed = sorted(name for name, record in results.items() if record["status"] != "pass")