Files
horizon-airdef/tools/ci/runner_pause.py
T
Ludovic Marie d81aa97e84 CI: the desktop runner works only while nobody is at the screen; desktop as a parity host
runner_pause.py pauses on any seat0 session (locked too), games, GPU use or a pause file, and is
the only thing that starts the runner; a job under way is throttled, then the runner stops. The
desktop host in parity_hosts.json refuses work while paused and its workers stop on a pause. The
nightly audit stays on the Mac mini. Docs and a unit test of the watcher.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 23:12:09 +01:00

195 lines
8.2 KiB
Python
Executable File

#!/usr/bin/env python3
"""Pauses the Horizon CI runner (tools/ci/linux-runner.sh) while the maintainer is at this machine or plays on it.
The runner works only while nobody is at the screen (rule of 2026-10-25, after ludodesktop was restarted under a
CI load while he used it). It pauses when any of these holds:
- a graphical session is open on seat0, locked or not: what `~/fleet/power status` calls "IN USE";
- fleet-load says a game runs (~/fleet/load: Steam, gamescope, Wine or Proton processes);
- a process of his matches GAME below (the same, plus Minecraft and a Horizon game he started himself);
- a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY %, default 50, in two
samples in a row; the browser, Discord and the compositor stay under 20 %);
- ~/.actions-runners/horizon/pause exists (pause by hand: `touch` it; `rm` it to let the watcher decide).
Paused, the runner takes no new job: while it is idle its service is stopped, so GitHub sends the jobs to the
Mac mini, and the parity workers other machines started here stop (their harnesses go back to those machines). A
job already running is throttled, not stopped: horizon-ci.slice drops to CPU weight 1, I/O weight 1 and a quota of
THROTTLE_CPUS threads, and the service stops once the job ends. Freezing it (SIGSTOP or the cgroup freezer)
would stop the runner's heartbeat, so GitHub would drop the job as lost, and the harnesses' wall-clock timeouts
would fail runs that were only waiting. The runner comes back RESUME_AFTER seconds after the last of these signs.
Every change is written to ~/.actions-runners/horizon/pause.log.
"""
import json
import os
import re
import subprocess
import time
from pathlib import Path
DIR = Path.home() / ".actions-runners" / "horizon"
SERVICE, SLICE = "horizon-runner.service", "horizon-ci.slice"
MANUAL, STATE, LOG = DIR / "pause", DIR / "paused", DIR / "pause.log"
GAME = re.compile(r"(SteamLaunch|gamescope|wine64-preloader|wineserver|wine-preloader|\.exe(\s|$)|proton\b"
r"|net\.minecraft\.client\.main\.Main|minecraft-launcher|prismlauncher)", re.I)
GPU_BUSY = int(os.environ.get("GPU_BUSY", 50))
INTERVAL = int(os.environ.get("INTERVAL", 15))
RESUME_AFTER = int(os.environ.get("RESUME_AFTER", 300))
THROTTLE_CPUS = int(os.environ.get("THROTTLE_CPUS", 2))
def log(text):
with LOG.open("a") as handle:
handle.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S')} {text}\n")
def systemctl(*arguments):
return subprocess.run(["systemctl", "--user", *arguments], capture_output=True, text=True)
def ci_process(pid):
"""A process of the runner's slice (its jobs, the parity workers other machines start here)."""
try:
return SLICE in Path(f"/proc/{pid}/cgroup").read_text()
except OSError:
return True # gone
def at_the_screen():
"""A graphical login on seat0, as ~/fleet/power status counts it (a locked screen counts too)."""
sessions = subprocess.run(["loginctl", "list-sessions", "--no-legend"], capture_output=True, text=True).stdout
for line in sessions.splitlines():
fields = line.split()
if "seat0" in fields:
return f"session {fields[0]} of {fields[2] if len(fields) > 2 else '?'} at the screen (seat0)"
return None
def fleet_load_gaming():
status = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "fleet-load" / "status.json"
try:
if time.time() - status.stat().st_mtime > 120:
return None
return "fleet-load: playing a game" if json.loads(status.read_text())["human"]["gaming"] else None
except (OSError, ValueError, KeyError, TypeError):
return None
def game_process():
uid = os.getuid()
for entry in Path("/proc").iterdir():
if not entry.name.isdigit():
continue
try:
if entry.stat().st_uid != uid:
continue
command = (entry / "cmdline").read_bytes().replace(b"\0", b" ").decode(errors="replace")
except OSError:
continue
# A Horizon game counts when he started it (its parent is his launcher), not when a harness or agent did.
if GAME.search(command) or (" --offscreen" not in command and "/horizon-launcher/" in command
and "horizon-client" in command.split(" ")[0]):
if not ci_process(int(entry.name)):
return f"process {entry.name}: {command[:120]}"
return None
def gpu_users():
"""Processes outside the CI slice using the GPU's shader cores, as {pid: SM %}."""
try:
# Two samples: the first one nvidia-smi prints has no utilisation yet.
output = subprocess.run(["nvidia-smi", "pmon", "-c", "2", "-s", "u"], capture_output=True, text=True,
timeout=20).stdout
except (OSError, subprocess.TimeoutExpired):
return {}
users = {}
for line in output.splitlines():
fields = line.split()
if line.startswith("#") or len(fields) < 4 or not fields[1].isdigit() or not fields[3].isdigit():
continue
if not ci_process(int(fields[1])):
users[int(fields[1])] = max(users.get(int(fields[1]), 0), int(fields[3]))
return users
def runner_busy():
"""The runner is in a job: a Runner.Worker runs in its service."""
group = systemctl("show", "-p", "ControlGroup", "--value", SERVICE).stdout.strip()
try:
pids = Path(f"/sys/fs/cgroup{group}/cgroup.procs").read_text().split() if group else []
except OSError:
return False
for pid in pids:
try:
if Path(f"/proc/{pid}/comm").read_text().strip() == "Runner.Worker":
return True
except OSError:
pass
return False
def throttle(on):
if on:
systemctl("set-property", "--runtime", SLICE, "CPUWeight=1", "IOWeight=1", f"CPUQuota={THROTTLE_CPUS * 100}%")
else:
systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", "CPUQuota=")
def pause_reason(hot):
"""Why the runner may not work now, or None. hot: the GPU users seen busy at the last sample (updated)."""
if MANUAL.exists():
return f"paused by hand ({MANUAL})"
reason = at_the_screen() or fleet_load_gaming() or game_process()
if reason:
return reason
busy = {pid for pid, sm in gpu_users().items() if sm >= GPU_BUSY}
twice = sorted(busy & set(hot))
hot.clear()
hot.update(dict.fromkeys(busy))
if twice:
try:
name = Path(f"/proc/{twice[0]}/comm").read_text().strip()
except OSError:
name = "?"
return f"GPU busy: process {twice[0]} ({name})"
return None
def main():
hot = {}
# A boot (or a restart of this watcher) counts as a sign: the runner starts RESUME_AFTER seconds later at the
# earliest, which leaves time to log in after switching the machine on.
last_sign = time.monotonic()
paused = None
while True:
reason = pause_reason(hot)
now = time.monotonic()
if reason:
last_sign = now
active = systemctl("is-active", "--quiet", SERVICE).returncode == 0
if reason or now - last_sign < RESUME_AFTER:
if paused is not True:
paused = True
STATE.write_text((reason or "starting") + "\n")
throttle(True)
# Parity workers other machines started here (tools/parity_hosts.json): they stop, and those
# machines run the harnesses themselves.
systemctl("stop", "horizon-parity-worker-*.scope")
log(f"pause: {reason or 'waiting ' + str(RESUME_AFTER) + ' s after starting'}"
+ (f"; a job is running: throttled to {THROTTLE_CPUS} threads, stops after it" if active else ""))
if active and not runner_busy():
systemctl("stop", SERVICE)
log("runner stopped")
else:
if paused is not False:
paused = False
STATE.unlink(missing_ok=True)
throttle(False)
log(f"resume: nobody at the screen and no game for {RESUME_AFTER} s")
if not active:
systemctl("start", SERVICE)
log("runner started")
time.sleep(INTERVAL)
if __name__ == "__main__":
main()