Merge pull request #2486 from ludovic111/fix/parity-reap-orphans
🤖 Agent: parity.py reaps the harness processes a failed or timed-out run leaves behind; parity passed on playhorizon.
This commit is contained in:
@@ -432,6 +432,9 @@ Releases ship often, so nothing they leave behind may grow without bound.
|
||||
(`~/.actions-runners/runner-gc.sh`, plist `dev.horizon.runner-gc`), never from a pull request's checkout. It skips a
|
||||
runner in a job, removes a checkout's cargo build folders past 12 GB (`MAX_TARGET_GB`) and harness work folders
|
||||
under `.local/` untouched for 3 days (`STALE_DAYS`), and keeps the 2 newest shared preparations (`KEEP_PREPARED`)
|
||||
while no runner works. Runner builds go through the machine's sccache (`tools/ci/runner-env.sh`), so what it removes
|
||||
while no runner works. It also stops harness JVMs (vanilla servers, oracles) left orphaned for 3 hours
|
||||
(`ORPHAN_HOURS`): parent gone, command line under a runner's checkout or a worktree's `.local/`. `tools/parity.py`
|
||||
stops its own harnesses on SIGTERM and SIGHUP (a cancelled job), so these come only from a SIGKILL or a harness run
|
||||
by hand. Runner builds go through the machine's sccache (`tools/ci/runner-env.sh`), so what it removes
|
||||
comes back from the cache. Update the copy after changing the script:
|
||||
`ssh macmini 'curl -fsSL https://raw.githubusercontent.com/ludovic111/horizon/main/tools/ci/runner-gc.sh -o ~/.actions-runners/runner-gc.sh'`.
|
||||
|
||||
@@ -9,6 +9,10 @@
|
||||
# the shared vanilla data) that nothing touched for STALE_DAYS go; a harness that runs again rebuilds its own.
|
||||
# .local/parity-runner (run state, oracle cache) stays.
|
||||
# - The shared prepared resources keep their KEEP_PREPARED newest preparations, only while no runner works.
|
||||
# - Java processes harnesses left behind (vanilla servers, oracles) go once ORPHAN_HOURS old: those whose parent
|
||||
# is gone (adopted by launchd, init or systemd --user) and whose command line names a runner's checkout or a
|
||||
# worktree's .local/. A job cancelled while it ran left such servers for days (2026-10-26); tools/parity.py now
|
||||
# stops its harnesses on SIGTERM, and this catches what it can't (SIGKILL, a harness run by hand).
|
||||
set -euo pipefail
|
||||
root=${RUNNERS:-$HOME/.actions-runners}
|
||||
max_kb=$(( ${MAX_TARGET_GB:-12} * 1024 * 1024 ))
|
||||
@@ -42,6 +46,25 @@ for runner in "$root"/horizon "$root"/horizon-[0-9]*; do
|
||||
fi
|
||||
done
|
||||
|
||||
# [[dd-]hh:]mm:ss, as ps prints a process's age, in minutes.
|
||||
minutes() {
|
||||
local t=$1 d=0 h=0 m a b c
|
||||
if [[ $t == *-* ]]; then d=${t%%-*}; t=${t#*-}; fi
|
||||
IFS=: read -r a b c <<< "$t"
|
||||
if [ -n "$c" ]; then h=$a m=$b; else m=$a; fi
|
||||
echo $(( 10#$d * 1440 + 10#$h * 60 + 10#$m ))
|
||||
}
|
||||
orphan_min=$(( ${ORPHAN_HOURS:-3} * 60 ))
|
||||
ps -A -o pid=,ppid=,etime=,command= | while read -r pid ppid etime command; do
|
||||
case "$command" in *java*) ;; *) continue ;; esac
|
||||
case "$command" in *"$root"/*/_work/*|*/.claude/worktrees/*/.local/*|*/code/horizon/.local/*) ;; *) continue ;; esac
|
||||
[ "$(minutes "$etime")" -ge "$orphan_min" ] || continue
|
||||
if [ "$ppid" != 1 ]; then
|
||||
case "$(ps -o command= -p "$ppid" 2>/dev/null)" in *"systemd --user"*) ;; *) continue ;; esac
|
||||
fi
|
||||
if kill "$pid" 2>/dev/null; then echo "orphaned java $pid ($etime old) stopped: ${command:0:160}"; fi
|
||||
done
|
||||
|
||||
prepared="$root/horizon/_work/horizon/shared-local/prepared"
|
||||
if [ "$any_busy" = 0 ] && [ -d "$prepared" ] && mkdir "$prepared/.lock" 2>/dev/null; then
|
||||
trap 'rmdir "$prepared/.lock" 2>/dev/null || true' EXIT
|
||||
|
||||
+25
-7
@@ -48,6 +48,7 @@ import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -1134,6 +1135,14 @@ def run(selected, state, hasher, environment_digest, executors, reuse_oracles, r
|
||||
return results, time.monotonic() - started_all
|
||||
|
||||
|
||||
def exit_on_termination():
|
||||
"""SIGTERM and SIGHUP (a cancelled CI job, a dropped ssh) end this process through its finally blocks, which stop
|
||||
the running harnesses. Each runs in its own session, so without that they, and the vanilla servers they started,
|
||||
outlive it (orphans of days were found on the Mac mini's runners, 2026-10-26)."""
|
||||
for number in (signal.SIGTERM, signal.SIGHUP):
|
||||
signal.signal(number, lambda received, _frame: sys.exit(128 + received))
|
||||
|
||||
|
||||
def worker_main(shared_local):
|
||||
"""`parity.py --worker SHARED_LOCAL`: run harnesses sent as JSON lines on stdin, answer on stdout."""
|
||||
import queue as queues
|
||||
@@ -1169,13 +1178,20 @@ def worker_main(shared_local):
|
||||
built, errors = prebuild(first["prebuild"], default_build_jobs()) if first["prebuild"] else (True, "")
|
||||
executor = LocalExecutor(MACHINE_JOBS, int(memory_available() * 0.75), first["timeout"])
|
||||
send({"hello": {"jobs": MACHINE_JOBS, "memory": executor.memory_budget, "built": built, "errors": errors[-4000:]}})
|
||||
try:
|
||||
return serve(executor, inbox, queues.Empty, send, state, hasher)
|
||||
finally:
|
||||
executor.stop()
|
||||
|
||||
|
||||
def serve(executor, inbox, empty, send, state, hasher):
|
||||
"""The worker's loop: start what the coordinating runner sends, report what finished, until its stdin ends."""
|
||||
while True:
|
||||
try:
|
||||
message = inbox.get(timeout=0.2)
|
||||
except queues.Empty:
|
||||
except empty:
|
||||
message = False
|
||||
if message is None:
|
||||
executor.stop()
|
||||
save_state(state)
|
||||
return 0
|
||||
if message:
|
||||
@@ -1479,6 +1495,7 @@ def finished_run(failed):
|
||||
def main():
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "--trace":
|
||||
return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:])
|
||||
exit_on_termination()
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "--worker":
|
||||
return worker_main(sys.argv[2])
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
@@ -1608,6 +1625,7 @@ def main():
|
||||
jobs = 1 # harnesses that can only run here still need a slot
|
||||
remotes = [RemoteExecutor(host, config["hosts"][host], artifacts, config.get("local_only", {}), args.timeout)
|
||||
for host in hosts]
|
||||
local = None
|
||||
try:
|
||||
if jobs:
|
||||
print("Building probes...", flush=True)
|
||||
@@ -1620,12 +1638,12 @@ def main():
|
||||
where = "".join(f", plus {host}" for host in hosts)
|
||||
print(f"Running {len(selected)} harnesses, {jobs} at a time here{where}...", flush=True)
|
||||
reuse_oracles = not (args.fresh or args.all)
|
||||
executors = [*remotes, LocalExecutor(jobs, int(memory_available() * 0.75), args.timeout,
|
||||
REMOTE_LOCAL_PEAK if remotes and args.jobs is None else None)]
|
||||
results, elapsed = run(selected, state, hasher, environment_key(hasher), executors, reuse_oracles, rounds,
|
||||
known_flaky(config))
|
||||
local = LocalExecutor(jobs, int(memory_available() * 0.75), args.timeout,
|
||||
REMOTE_LOCAL_PEAK if remotes and args.jobs is None else None)
|
||||
results, elapsed = run(selected, state, hasher, environment_key(hasher), [*remotes, local], reuse_oracles,
|
||||
rounds, known_flaky(config))
|
||||
finally:
|
||||
for executor in remotes:
|
||||
for executor in [*remotes, *([local] if local else [])]:
|
||||
executor.stop()
|
||||
save_state(state)
|
||||
failed = sorted(name for name, record in results.items() if record["status"] != "pass")
|
||||
|
||||
Reference in New Issue
Block a user