From 8463325648f90045f83808dd201c07bfad2a264f Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:28:03 +0100 Subject: [PATCH 1/6] CI: Linux runners for the parity job (desktop, ThinkPad), paused while a game runs runner-env.sh and vanilla-data.sh find Python 3.13, the pinned JDK and sccache on NixOS; linux-runner.sh installs a runner as a systemd user service with its own screen; runner-pause.py stops it from taking jobs while a game runs. Co-Authored-By: Claude Opus 5.5 --- tools/ci/linux-runner.sh | 191 +++++++++++++++++++++++++++++++++++++++ tools/ci/runner-env.sh | 32 ++++++- tools/ci/runner-pause.py | 167 ++++++++++++++++++++++++++++++++++ tools/ci/vanilla-data.sh | 8 +- 4 files changed, 392 insertions(+), 6 deletions(-) create mode 100755 tools/ci/linux-runner.sh create mode 100755 tools/ci/runner-pause.py diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh new file mode 100755 index 000000000..0cb677c87 --- /dev/null +++ b/tools/ci/linux-runner.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# A Horizon CI runner (labels self-hosted, Linux, X64, horizon-game and the host's name) on a Linux machine of the +# maintainer's (NixOS: ludodesktop, ludothinkpad), as a systemd user service of the account that holds his +# Minecraft files, like the Mac mini's runners (docs/release-flow.md). Run it on that machine, as that user: +# +# bash tools/ci/linux-runner.sh install # register (gh login), install, start; again to update the scripts +# bash tools/ci/linux-runner.sh status # the services, the runner's state, the pause watcher's last lines +# bash tools/ci/linux-runner.sh uninstall # stop and unregister +# +# Everything lives in ~/.actions-runners/horizon (never in ~/code): the runner, its own rustup and cargo homes, +# the work folder, and copies of this script and runner-pause.py taken at install time (a pull request's checkout +# never runs as the service). The service runs nice, in horizon-ci.slice (low CPU and I/O weight, memory capped), +# on its own virtual screen: vscreen (a headless sway) where it is installed, else Xvfb from nixpkgs with Mesa's +# Vulkan drivers. runner-pause.py pauses it while a game runs (horizon-runner-pause.service). +# HORIZON_SHARED_SOURCE: the imported vanilla data (default ~/horizon-local/shared-local, else +# ~/code/horizon/.local); RUNNER_JOBS: cargo jobs and harnesses at once (default 2/3 of the threads). +set -euo pipefail +repo=ludovic111/horizon +dir=$HOME/.actions-runners/horizon +units=${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user +version=2.338.0 +sha=af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32 +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Nix packages the jobs need, kept alive by GC roots in $dir/nix. +nix_out() { nix-build '' -A "$1" -o "$dir/nix/$2" > /dev/null && readlink -f "$dir/nix/$2"; } + +# The runner is a .NET program: on NixOS its native libraries (libstdc++, ICU, OpenSSL) come from nix-ld. +export LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} +# NixOS has no /bin/bash: the runner's scripts (again after each of its self-updates) get a portable one. +fix_shebangs() { sed -i '1s|^#!/bin/bash|#!/usr/bin/env bash|' "$dir"/*.sh "$dir"/bin/*.sh; } + +display_env() { + # The jobs' screen. Never the maintainer's: the user manager hands every service his DISPLAY and WAYLAND_DISPLAY. + if command -v vscreen > /dev/null; then + VSCREEN=ci-horizon vscreen env + else + echo "unset WAYLAND_DISPLAY; export DISPLAY=:$(cat "$dir/display")" + fi +} + +install() { + local name=${RUNNER_NAME:-$(hostname)-horizon} jobs=${RUNNER_JOBS:-$(( $(nproc) * 2 / 3 ))} + local source=${HORIZON_SHARED_SOURCE:-} + if [ -z "$source" ]; then + for candidate in "$HOME/horizon-local/shared-local" "$HOME/code/horizon/.local"; do + [ -d "$candidate/minecraft" ] && { source=$candidate; break; } + done + fi + [ -n "$source" ] && [ -d "$source/minecraft" ] || { echo "no imported Minecraft (HORIZON_SHARED_SOURCE)"; exit 1; } + mkdir -p "$dir/nix" "$units" + cp "$here/linux-runner.sh" "$here/runner-pause.py" "$dir/" + + if [ ! -f "$dir/.runner" ]; then + local cache=$HOME/.cache/gha-runner/actions-runner-linux-x64-$version.tar.gz + if ! echo "$sha $cache" | sha256sum -c --quiet - > /dev/null 2>&1; then + mkdir -p "$(dirname "$cache")" + curl -fsSL -o "$cache" "https://github.com/actions/runner/releases/download/v$version/actions-runner-linux-x64-$version.tar.gz" + echo "$sha $cache" | sha256sum -c --quiet - + fi + tar xzf "$cache" -C "$dir" + fix_shebangs + gh api -X POST "repos/$repo/actions/runners/registration-token" --jq .token | + (cd "$dir" && bash ./config.sh --unattended --replace --url "https://github.com/$repo" --token "$(cat)" \ + --name "$name" --labels "${RUNNER_LABELS:-horizon-game,$(hostname)}" --work _work > /dev/null) + fi + + # Libraries for building (pkg-config) and running (dlopen) the game, the JDK pinned by research-lock.json, sccache. + local alsa udev java sccache + alsa=$(nix_out alsa-lib.dev alsa-dev) + udev=$(nix_out systemd.dev systemd-dev) + java=$(nix_out "jdk$(sed -n 's/.*"java_major": *\([0-9]*\).*/\1/p' "$here/../../research-lock.json" | head -1)_headless" jdk)/lib/openjdk + sccache=$(nix_out sccache sccache) + local libs=/run/opengl-driver/lib:/run/current-system/sw/share/nix-ld/lib + if ! command -v vscreen > /dev/null; then + local mesa x + mesa=$(nix_out mesa mesa) + x=$(nix_out xvfb-run xvfb-run) + for lib in libx11 libxcursor libxrandr libxi libxext libxinerama libxrender libxxf86vm libxcb libxkbcommon \ + vulkan-loader libGL; do + libs=$libs:$(nix_out "$lib" "$lib")/lib + done + libs=$mesa/lib:$libs + echo "VK_DRIVER_FILES=$mesa/share/vulkan/icd.d/intel_icd.x86_64.json:$mesa/share/vulkan/icd.d/lvp_icd.x86_64.json" \ + > "$dir/.env.display" + echo "$x/bin" > "$dir/.path.display" + fi + cat > "$dir/.env" <> "$dir/.env"; else echo "VSCREEN=ci-horizon" >> "$dir/.env"; fi + echo "$dir/cargo/bin:$(cat "$dir/.path.display" 2>/dev/null || true)${dir}/nix/sccache/bin:$java/bin:$HOME/.local/bin:/run/wrappers/bin:/etc/profiles/per-user/$USER/bin:/run/current-system/sw/bin" \ + | sed 's/::/:/g' > "$dir/.path" + + cat > "$units/horizon-ci.slice" < "$units/horizon-runner.service" < "$units/horizon-runner-pause.service" </dev/null)" != yes ]; then + sudo -n loginctl enable-linger "$USER" || echo "Run once: sudo loginctl enable-linger $USER" + fi + systemctl --user daemon-reload + systemctl --user enable horizon-runner.service horizon-runner-pause.service > /dev/null + systemctl --user restart horizon-runner-pause.service + [ -e "$dir/paused" ] || systemctl --user start horizon-runner.service + status +} + +run() { + cd "$dir" + fix_shebangs + if command -v vscreen > /dev/null; then + eval "$(display_env)" + else + # Xvfb on the first free display from :90, for this service's lifetime. + local n=90 + while [ -e "/tmp/.X$n-lock" ]; do n=$((n + 1)); done + echo "$n" > "$dir/display" + Xvfb ":$n" -screen 0 1280x720x24 -nolisten tcp & + eval "$(display_env)" + fi + exec bash ./bin/runsvc.sh +} + +status() { + systemctl --user --no-pager list-units 'horizon-runner*' 'vscreen-ci-horizon*' + [ -e "$dir/paused" ] && echo "paused: $(cat "$dir/paused")" + tail -5 "$dir/pause.log" 2>/dev/null || true +} + +uninstall() { + systemctl --user disable --now horizon-runner-pause.service horizon-runner.service 2>/dev/null || true + gh api -X POST "repos/$repo/actions/runners/remove-token" --jq .token | + (cd "$dir" && bash ./config.sh remove --token "$(cat)") + rm -f "$units/horizon-runner.service" "$units/horizon-runner-pause.service" "$units/horizon-ci.slice" + systemctl --user daemon-reload +} + +case ${1:-} in + install|run|status|uninstall) "$1" ;; + *) sed -n '2,10p' "$0"; exit 2 ;; +esac diff --git a/tools/ci/runner-env.sh b/tools/ci/runner-env.sh index 49158ac91..39828b7d7 100755 --- a/tools/ci/runner-env.sh +++ b/tools/ci/runner-env.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# The tools the vanilla harnesses need, on the self-hosted Mac mini runner (label horizon-game): its service -# starts with a bare PATH. Python 3.13 and the JDK pinned by research-lock.json come from the machine, never -# from a download. Writes them to $GITHUB_PATH and $GITHUB_ENV for the next steps. +# The tools the vanilla harnesses need, on a self-hosted runner (label horizon-game): the Mac mini's, whose service +# starts with a bare PATH, and the Linux ones (tools/ci/linux-runner.sh, NixOS). Python 3.13 and the JDK pinned by +# research-lock.json come from the machine, never from a download. Writes them to $GITHUB_PATH and $GITHUB_ENV for +# the next steps. set -euo pipefail java_major=$(sed -n 's/.*"java_major": *\([0-9]*\).*/\1/p' research-lock.json | head -1) bin="$RUNNER_TEMP/horizon-bin" @@ -17,5 +18,30 @@ if [ "$(uname)" = Darwin ]; then if [ -x /opt/homebrew/bin/sccache ]; then { echo "RUSTC_WRAPPER=/opt/homebrew/bin/sccache"; echo "CARGO_INCREMENTAL=0"; } >> "$GITHUB_ENV" fi +else + python=$(command -v python3.13 || true) + [ -n "$python" ] || { echo "::error::no python3.13 on this runner"; exit 1; } + ln -sfn "$python" "$bin/python3" + ln -sfn "$python" "$bin/python" + # The JDK: the runner's JAVA_HOME (linux-runner.sh pins one), else the java on PATH, else nixpkgs' jdk. + major_of() { "$1/bin/java" -version 2>&1 | sed -n 's/.*version "\([0-9]*\).*/\1/p' | head -1; } + java_home=${JAVA_HOME:-} + if [ -z "$java_home" ] || [ "$(major_of "$java_home")" != "$java_major" ]; then + java_home="" + if command -v java > /dev/null; then + candidate=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")") + [ "$(major_of "$candidate")" = "$java_major" ] && java_home=$candidate + fi + fi + if [ -z "$java_home" ] && command -v nix-build > /dev/null; then + out=$(nix-build '' --no-out-link -A "jdk${java_major}_headless" 2>/dev/null || true) + [ -n "$out" ] && java_home=$out/lib/openjdk + fi + [ -n "$java_home" ] || { echo "::error::no JDK $java_major on this runner"; exit 1; } + { echo "$bin"; echo "$java_home/bin"; } >> "$GITHUB_PATH" + echo "JAVA_HOME=$java_home" >> "$GITHUB_ENV" + if sccache=$(command -v sccache); then + { echo "RUSTC_WRAPPER=$sccache"; echo "CARGO_INCREMENTAL=0"; } >> "$GITHUB_ENV" + fi fi echo "HORIZON_NO_AUDIO=1" >> "$GITHUB_ENV" diff --git a/tools/ci/runner-pause.py b/tools/ci/runner-pause.py new file mode 100755 index 000000000..4a88c6ae4 --- /dev/null +++ b/tools/ci/runner-pause.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Pauses the Horizon CI runner (tools/ci/linux-runner.sh) while the maintainer plays a game on this machine. + +A game is running when any of these holds: + - fleet-load says so (~/fleet/load: Steam, gamescope, Wine or Proton processes); + - a process of his matches GAME below (the same, plus Minecraft and a Horizon game he started himself); + - a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY % twice in a row); + - ~/.actions-runners/horizon/pause exists (pause by hand: `touch` it; `rm` it to let the watcher decide). + +Paused, the runner takes no new job: while it is idle its service is stopped, so GitHub sends the jobs to the +Mac mini. A job already running is throttled, not stopped: horizon-ci.slice (the job, and the parity workers other machines +send here) drops to CPU weight 1, I/O weight 1 +and a quota of THROTTLE_CPUS threads, and the service stops once the job ends. Freezing it (SIGSTOP or the cgroup +freezer) would stop the runner's heartbeat, so GitHub would drop the job as lost, and the harnesses' wall-clock +timeouts would fail runs that were only waiting. The runner comes back RESUME_AFTER seconds after the last sign +of a game. Every change is written to ~/.actions-runners/horizon/pause.log. +""" +import json +import os +import re +import subprocess +import time +from pathlib import Path + +DIR = Path.home() / ".actions-runners" / "horizon" +SERVICE, SLICE = "horizon-runner.service", "horizon-ci.slice" +MANUAL, STATE, LOG = DIR / "pause", DIR / "paused", DIR / "pause.log" +GAME = re.compile(r"(SteamLaunch|gamescope|wine64-preloader|wineserver|wine-preloader|\.exe(\s|$)|proton\b" + r"|net\.minecraft\.client\.main\.Main|minecraft-launcher|prismlauncher)", re.I) +GPU_BUSY = int(os.environ.get("GPU_BUSY", 30)) +INTERVAL = int(os.environ.get("INTERVAL", 15)) +RESUME_AFTER = int(os.environ.get("RESUME_AFTER", 300)) +THROTTLE_CPUS = int(os.environ.get("THROTTLE_CPUS", 2)) + + +def log(text): + with LOG.open("a") as handle: + handle.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S')} {text}\n") + + +def systemctl(*arguments): + return subprocess.run(["systemctl", "--user", *arguments], capture_output=True, text=True) + + +def ci_process(pid): + """A process of the runner's slice (its jobs, the parity workers other machines start here).""" + try: + return SLICE in Path(f"/proc/{pid}/cgroup").read_text() + except OSError: + return True # gone + + +def fleet_load_gaming(): + status = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "fleet-load" / "status.json" + try: + if time.time() - status.stat().st_mtime > 120: + return None + return "fleet-load: playing a game" if json.loads(status.read_text())["human"]["gaming"] else None + except (OSError, ValueError, KeyError, TypeError): + return None + + +def game_process(): + uid = os.getuid() + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + if entry.stat().st_uid != uid: + continue + command = (entry / "cmdline").read_bytes().replace(b"\0", b" ").decode(errors="replace") + except OSError: + continue + # A Horizon game counts when he started it (its parent is his launcher), not when a harness or agent did. + if GAME.search(command) or (" --offscreen" not in command and "/horizon-launcher/" in command + and "horizon-client" in command.split(" ")[0]): + if not ci_process(int(entry.name)): + return f"process {entry.name}: {command[:120]}" + return None + + +def gpu_users(): + """Processes outside the CI slice using the GPU's shader cores, as {pid: SM %}.""" + try: + output = subprocess.run(["nvidia-smi", "pmon", "-c", "1", "-s", "u"], capture_output=True, text=True, + timeout=20).stdout + except (OSError, subprocess.TimeoutExpired): + return {} + users = {} + for line in output.splitlines(): + fields = line.split() + if line.startswith("#") or len(fields) < 4 or not fields[1].isdigit() or not fields[3].isdigit(): + continue + if not ci_process(int(fields[1])): + users[int(fields[1])] = int(fields[3]) + return users + + +def runner_busy(): + """The runner is in a job: a Runner.Worker runs in its service.""" + group = systemctl("show", "-p", "ControlGroup", "--value", SERVICE).stdout.strip() + try: + pids = Path(f"/sys/fs/cgroup{group}/cgroup.procs").read_text().split() if group else [] + except OSError: + return False + for pid in pids: + try: + if Path(f"/proc/{pid}/comm").read_text().strip() == "Runner.Worker": + return True + except OSError: + pass + return False + + +def throttle(on): + if on: + systemctl("set-property", "--runtime", SLICE, "CPUWeight=1", "IOWeight=1", f"CPUQuota={THROTTLE_CPUS * 100}%") + else: + systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", "CPUQuota=") + + +def main(): + hot = {} # GPU users seen busy at the last sample + last_game = 0.0 + paused = STATE.exists() + while True: + reason = None + if MANUAL.exists(): + reason = "paused by hand (" + str(MANUAL) + ")" + reason = reason or fleet_load_gaming() or game_process() + if not reason: + busy = {pid for pid, sm in gpu_users().items() if sm >= GPU_BUSY} + twice = busy & set(hot) + hot = dict.fromkeys(busy) + if twice: + pid = min(twice) + try: + name = Path(f"/proc/{pid}/comm").read_text().strip() + except OSError: + name = "?" + reason = f"GPU busy: process {pid} ({name})" + now = time.monotonic() + if reason: + last_game = now + if reason and not paused: + paused = True + STATE.write_text(reason + "\n") + throttle(True) # also what other machines' parity workers run here meanwhile + if runner_busy(): + log(f"pause: {reason}; a job is running: throttled to {THROTTLE_CPUS} threads, stops after it") + else: + systemctl("stop", SERVICE) + log(f"pause: {reason}; runner stopped") + elif paused and reason is None and now - last_game >= RESUME_AFTER: + paused = False + STATE.unlink(missing_ok=True) + throttle(False) + systemctl("start", SERVICE) + log(f"resume: no game for {RESUME_AFTER} s; runner started") + elif paused and systemctl("is-active", "--quiet", SERVICE).returncode == 0 and not runner_busy(): + systemctl("stop", SERVICE) + log("the job ended: runner stopped until the game ends") + time.sleep(INTERVAL) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/vanilla-data.sh b/tools/ci/vanilla-data.sh index 5276c54e9..a7bd4a407 100755 --- a/tools/ci/vanilla-data.sh +++ b/tools/ci/vanilla-data.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # The vanilla data for this checkout on the self-hosted runner. Horizon never downloads the game: the runner # holds the maintainer's own copy, imported and decompiled once (HORIZON_SHARED_SOURCE, default -# ~/code/horizon-parity/shared-local). It is cloned once into the runner's work folder ($RUNNER_WORKSPACE/ -# shared-local; APFS clones, so no second copy of the bytes), and .local/ links to it as in a parity worktree. +# ~/code/horizon-parity/shared-local; tools/ci/linux-runner.sh sets it on Linux). It is cloned once into the runner's +# work folder ($RUNNER_WORKSPACE/shared-local; APFS clones, or reflinks where the file system has them, so no second +# copy of the bytes), and .local/ links to it as in a parity worktree. # .local/horizon-client and the exporter outputs beside it, what horizon-prepare makes, are prepared by this # checkout's own preparer once per preparer version (its sources, the exporters and research-lock.json), so a # change to the preparer is checked with what it prepares; the four newest preparations are kept. @@ -31,7 +32,8 @@ for entry in "$shared"/*; do done # HORIZON_PREPARE_KEY: the same key, computed where the Git history is (tools/pr_ready.py on another machine). -key=${HORIZON_PREPARE_KEY:-$(git ls-files -s crates/horizon-prepare mods/horizon/src/test/java research-lock.json | shasum -a 256 | cut -c1-16)} +sha256() { if command -v shasum > /dev/null; then shasum -a 256; else sha256sum; fi; } +key=${HORIZON_PREPARE_KEY:-$(git ls-files -s crates/horizon-prepare mods/horizon/src/test/java research-lock.json | sha256 | cut -c1-16)} prepared="$shared/prepared/$key" # Two runners share the vanilla data (~/.actions-runners/horizon-2/_work/horizon/shared-local links to the first # one's): one preparation at a time. A lock older than 30 minutes belongs to a job that died. From d81aa97e842a753e231ae34b181bc1bd2654c589 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:12:08 +0100 Subject: [PATCH 2/6] CI: the desktop runner works only while nobody is at the screen; desktop as a parity host runner_pause.py pauses on any seat0 session (locked too), games, GPU use or a pause file, and is the only thing that starts the runner; a job under way is throttled, then the runner stops. The desktop host in parity_hosts.json refuses work while paused and its workers stop on a pause. The nightly audit stays on the Mac mini. Docs and a unit test of the watcher. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 3 +- docs/parity-workflow.md | 2 +- docs/release-flow.md | 28 +++- tests/test_runner_pause.py | 82 ++++++++++++ tools/ci/linux-runner.sh | 52 +++++--- tools/ci/{runner-pause.py => runner_pause.py} | 125 +++++++++++------- tools/parity_hosts.json | 9 ++ 7 files changed, 230 insertions(+), 71 deletions(-) create mode 100644 tests/test_runner_pause.py rename tools/ci/{runner-pause.py => runner_pause.py} (51%) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 51f9e89cd..60bbbf057 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -29,7 +29,8 @@ env: jobs: audit: - runs-on: [self-hosted, horizon-game] + # The Mac mini's runners only: the audit compares with the previous one, so it stays on one platform. + runs-on: [self-hosted, horizon-game, macOS] timeout-minutes: 150 steps: - name: 03:00 in Paris diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index f9f222071..80426fa59 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -38,7 +38,7 @@ The game logs into `HORIZON_HOME` and falls back to the player's own game folder ### Other machines -Harnesses can also run on the Mac mini and the ThinkPad (`tools/parity_hosts.json` describes them; its `default` list is empty, so pass `--hosts macmini,thinkpad` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 3 and only those under 800 MB. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. +Harnesses can also run on the Mac mini, the ThinkPad and the desktop (`tools/parity_hosts.json` describes them; `desktop` uses its CI runner's environment and vanilla data, and refuses work, or stops it, while someone is at it: docs/release-flow.md, "The desktop's runner"; its `default` list is empty, so pass `--hosts macmini,thinkpad` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 3 and only those under 800 MB. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. Harnesses a host can't run are listed in the file: `local_only` (`save_roundtrip` runs the game on a virtual screen) and each host's `exclude`. The Mac excludes nothing: vanilla's `Math.sin`, `pow`, `log` and the like don't round the same on an Apple silicon Mac as on Linux (HotSpot uses fdlibm there, Intel's stubs on x86-64), and Rust's float functions round like neither (they call the platform libm), so code that mirrors a vanilla `Math` call uses `horizon_core::jmath`, which computes what HotSpot computes on the host architecture. The ThinkPad helps too, gently: Linux, 2 harnesses at `nice 19`, since it hosts the Claude account pool; it finds ALSA through `~/code/horizon-parity/alsa-dev` like the desktop. diff --git a/docs/release-flow.md b/docs/release-flow.md index 4fb0209d9..e72bebc73 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -13,7 +13,7 @@ reopened; labels never start it): | Job | Runs on | What | Time | | --- | --- | --- | --- | | `verify` | GitHub `ubuntu-latest` | Public-tree check, Python tests, rustfmt, clippy with warnings denied, Rust tests. Dependencies and the workspace build come from a cache saved by main's runs | ~4 min cold, less warm | -| `parity / parity` | The Mac mini's runner (`ludovics-mac-mini`, label `horizon-game`) | The vanilla harnesses the change can affect (`tools/parity.py --changed `) | seconds when nothing is affected; minutes per affected harness | +| `parity / parity` | A runner with the label `horizon-game`: the Mac mini's two, and the desktop's when no game runs there (below) | The vanilla harnesses the change can affect (`tools/parity.py --changed `) | seconds when nothing is affected; minutes per affected harness | Pushes to main run `verify` only; the nightly audit covers main's harnesses. @@ -28,10 +28,13 @@ The job is `.github/workflows/parity.yml`, a reusable workflow: `checks` calls i repository, `run-parity.yml` for a fork's pull request a maintainer labels `run-parity`. Horizon never downloads the game, so harnesses that compare with vanilla run only where the maintainer's own -copy is: the Mac mini. Its runner runs as the maintainer's account and keeps its checkout between runs -(`clean: false`), so the probes build incrementally into `.local/parity-target` and earlier passes stand. +copy is: the Mac mini and his Linux desktop. Their runners run as the maintainer's account and keep their checkout +between runs (`clean: false`), so the probes build incrementally into `.local/parity-target` and earlier passes +stand. -1. `tools/ci/runner-env.sh` puts the machine's Python 3.13 and the JDK `research-lock.json` pins on the PATH. +1. `tools/ci/runner-env.sh` puts the machine's Python 3.13 and the JDK `research-lock.json` pins on the PATH + (Homebrew and `java_home` on the Mac; on NixOS what `linux-runner.sh` pinned, else nixpkgs), and sccache when + the machine has it. 2. `tools/ci/vanilla-data.sh` links `.local/` to the runner's copy of the vanilla data, cloned once from `~/code/horizon-parity/shared-local` into `~/.actions-runners/horizon/_work/horizon/shared-local`, and prepares `.local/horizon-client` with the pull request's own `horizon-prepare` once per preparer version @@ -64,6 +67,23 @@ release's macOS build back. Each keeps its own checkout, `target/` and Rust inst `_work/horizon/shared-local` links to the first's, so they share the vanilla data, the preparations (one at a time, under a lock) and the parity passes and slots. +**The desktop's runner.** `ludodesktop-horizon` (labels `self-hosted`, `Linux`, `X64`, `ludodesktop`, and +`horizon-game` once a full trial has passed there) runs on the maintainer's Linux desktop, only while nobody is +at it. `tools/ci/linux-runner.sh install`, run there as him, registers it and installs two systemd user services: +`horizon-runner` (the runner, in `~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O +weight 20 and memory capped at two thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the +machine froze under ~7 cold cargo builds on 2026-10-14; its own Rust install; the JDK, sccache and the build +libraries from nixpkgs; its own `vscreen` so no game it starts reaches his screen) and `horizon-runner-pause` +(`tools/ci/runner_pause.py`), the only thing that starts the runner. The watcher stops it while a graphical +session is open on the machine, locked or not (`~/fleet/power status` says "IN USE"), while a game runs (Steam, +gamescope, Wine, Proton or Minecraft processes, fleet-load's own detection, or a process outside the CI slice +using the GPU at 50 % or more), or while `~/.actions-runners/horizon/pause` exists, and starts it again 5 minutes +after the last such sign. A job already under way is throttled to 2 threads at the lowest weight and the runner +stops after it: freezing the job would make GitHub drop it as lost and the harnesses time out. Old pull requests' +`runner-env.sh` know only the Mac: the runner's `.env` and `.path` give their jobs Python, the JDK and the +vanilla data (`HORIZON_SHARED_SOURCE`) all the same. The nightly audit stays on the Mac mini's runners, so its +comparison with the previous night is always made on one platform. + ## Merging Pull requests merge with a merge commit. The orchestrator sessions review a pull request, then turn on diff --git a/tests/test_runner_pause.py b/tests/test_runner_pause.py new file mode 100644 index 000000000..c6df08b82 --- /dev/null +++ b/tests/test_runner_pause.py @@ -0,0 +1,82 @@ +"""tools/ci/runner_pause.py: the Linux CI runner works only while nobody is at the screen or playing.""" +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from tools.ci import runner_pause + + +class Machine: + """A fake desktop: a clock, who is there, and the runner service. The watcher samples every 15 s.""" + + def __init__(self, signs): + self.signs = signs # (from, until, reason): a sign of someone at the screen or playing + self.clock = 0.0 + self.active = False + self.job_until = None # the runner is in a job until then + self.throttles = [] # (clock, on) + self.calls = [] + + def reason(self, hot): + return next((reason for start, end, reason in self.signs if start <= self.clock < end), None) + + def systemctl(self, *arguments): + self.calls.append((self.clock, arguments)) + if arguments[1:] == (runner_pause.SERVICE,): + self.active = {"start": True, "stop": False}.get(arguments[0], self.active) + return mock.Mock(returncode=0 if arguments[:2] != ("is-active", "--quiet") or self.active else 3) + + def busy(self): + return self.job_until is not None and self.clock < self.job_until + + def sleep(self, seconds): + self.clock += seconds + if self.clock > 2000: + raise StopIteration + + def run(self): + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.multiple(runner_pause, MANUAL=Path(directory, "pause"), STATE=Path(directory, "paused"), + LOG=Path(directory, "pause.log"), INTERVAL=15, RESUME_AFTER=300, + pause_reason=self.reason, systemctl=self.systemctl, runner_busy=self.busy, + throttle=lambda on: self.throttles.append((self.clock, on))), \ + mock.patch.object(runner_pause.time, "monotonic", lambda: self.clock), \ + mock.patch.object(runner_pause.time, "sleep", self.sleep): + try: + runner_pause.main() + except StopIteration: # the clock ran out + pass + + def first(self, verb, after=0): + return next(clock for clock, arguments in self.calls + if arguments == (verb, runner_pause.SERVICE) and clock >= after) + + +class RunnerPauseTests(unittest.TestCase): + def test_starts_only_after_nobody_was_there_for_resume_after(self): + machine = Machine([(0, 600, "session 3 of ludovic at the screen (seat0)")]) + machine.run() + self.assertEqual(machine.first("start"), 585 + 300) # the last sample that saw him + + def test_a_game_stops_an_idle_runner_at_once(self): + machine = Machine([(1000, 1300, "process 42: wine64-preloader game.exe")]) + machine.run() + self.assertEqual(machine.first("start"), 300) + self.assertEqual(machine.first("stop", after=300), 1005) + self.assertEqual(machine.first("start", after=1005), 1290 + 300) + + def test_a_job_under_way_is_throttled_then_stopped_when_it_ends(self): + machine = Machine([(1000, 1300, "GPU busy: process 7 (game)")]) + machine.job_until = 1200 + machine.run() + self.assertIn((1005, True), machine.throttles) # the game started: the job slows down at once, + self.assertIn((1005, ("stop", "horizon-parity-worker-*.scope")), machine.calls) # other machines' work goes + stop = machine.first("stop", after=300) + self.assertGreaterEqual(stop, 1200) # and the runner stops once it ends, not before + self.assertLess(stop, 1215) + self.assertEqual(machine.throttles[-1], (1290 + 300, False)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index 0cb677c87..adc2b0130 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -8,12 +8,14 @@ # bash tools/ci/linux-runner.sh uninstall # stop and unregister # # Everything lives in ~/.actions-runners/horizon (never in ~/code): the runner, its own rustup and cargo homes, -# the work folder, and copies of this script and runner-pause.py taken at install time (a pull request's checkout -# never runs as the service). The service runs nice, in horizon-ci.slice (low CPU and I/O weight, memory capped), +# the work folder, and copies of this script and runner_pause.py taken at install time (a pull request's checkout +# never runs as the service). horizon-runner-pause.service (runner_pause.py) starts the runner only while nobody is +# at the screen or playing, and stops it otherwise. The runner runs nice, in horizon-ci.slice (low CPU and I/O weight, memory capped), # on its own virtual screen: vscreen (a headless sway) where it is installed, else Xvfb from nixpkgs with Mesa's -# Vulkan drivers. runner-pause.py pauses it while a game runs (horizon-runner-pause.service). +# Vulkan drivers. # HORIZON_SHARED_SOURCE: the imported vanilla data (default ~/horizon-local/shared-local, else -# ~/code/horizon/.local); RUNNER_JOBS: cargo jobs and harnesses at once (default 2/3 of the threads). +# ~/code/horizon/.local); RUNNER_JOBS: cargo jobs and harnesses at once (default a third of the threads, at most 8: +# ludodesktop froze under ~7 cold cargo builds on 2026-10-14, and it is the maintainer's own machine). set -euo pipefail repo=ludovic111/horizon dir=$HOME/.actions-runners/horizon @@ -30,6 +32,18 @@ export LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib${LD_LIBRARY_PATH: # NixOS has no /bin/bash: the runner's scripts (again after each of its self-updates) get a portable one. fix_shebangs() { sed -i '1s|^#!/bin/bash|#!/usr/bin/env bash|' "$dir"/*.sh "$dir"/bin/*.sh; } +# Other machines' parity runs can send harnesses here (tools/parity_hosts.json "desktop": parity.py --worker over +# ssh, in horizon-ci.slice). Their vanilla data is the runner checkout's: parity-host/shared-local links to its +# .local links, so it follows the preparation of the runner's last job. +parity_host_links() { + local checkout=$dir/_work/horizon/horizon/.local shared=$dir/parity-host/shared-local entry + [ -d "$checkout" ] || return 0 + mkdir -p "$shared" + for entry in "$checkout"/*; do + [ -L "$entry" ] && ln -sfn "$entry" "$shared/$(basename "$entry")" + done +} + display_env() { # The jobs' screen. Never the maintainer's: the user manager hands every service his DISPLAY and WAYLAND_DISPLAY. if command -v vscreen > /dev/null; then @@ -40,7 +54,9 @@ display_env() { } install() { - local name=${RUNNER_NAME:-$(hostname)-horizon} jobs=${RUNNER_JOBS:-$(( $(nproc) * 2 / 3 ))} + local name=${RUNNER_NAME:-$(hostname)-horizon} threads jobs + threads=$(nproc) + jobs=${RUNNER_JOBS:-$(( threads / 3 > 8 ? 8 : threads / 3 > 1 ? threads / 3 : 1 ))} local source=${HORIZON_SHARED_SOURCE:-} if [ -z "$source" ]; then for candidate in "$HOME/horizon-local/shared-local" "$HOME/code/horizon/.local"; do @@ -49,7 +65,7 @@ install() { fi [ -n "$source" ] && [ -d "$source/minecraft" ] || { echo "no imported Minecraft (HORIZON_SHARED_SOURCE)"; exit 1; } mkdir -p "$dir/nix" "$units" - cp "$here/linux-runner.sh" "$here/runner-pause.py" "$dir/" + cp "$here/linux-runner.sh" "$here/runner_pause.py" "$dir/" if [ ! -f "$dir/.runner" ]; then local cache=$HOME/.cache/gha-runner/actions-runner-linux-x64-$version.tar.gz @@ -66,11 +82,11 @@ install() { fi # Libraries for building (pkg-config) and running (dlopen) the game, the JDK pinned by research-lock.json, sccache. - local alsa udev java sccache + local alsa udev java alsa=$(nix_out alsa-lib.dev alsa-dev) udev=$(nix_out systemd.dev systemd-dev) java=$(nix_out "jdk$(sed -n 's/.*"java_major": *\([0-9]*\).*/\1/p' "$here/../../research-lock.json" | head -1)_headless" jdk)/lib/openjdk - sccache=$(nix_out sccache sccache) + nix_out sccache sccache > /dev/null # on the PATH below: runner-env.sh hands it to cargo local libs=/run/opengl-driver/lib:/run/current-system/sw/share/nix-ld/lib if ! command -v vscreen > /dev/null; then local mesa x @@ -91,6 +107,7 @@ RUSTUP_HOME=$dir/rustup CARGO_HOME=$dir/cargo CARGO_BUILD_JOBS=$jobs PARITY_MACHINE_JOBS=$jobs +PARITY_LOCAL_JOBS=$jobs JAVA_HOME=$java HORIZON_SHARED_SOURCE=$source HORIZON_NO_AUDIO=1 @@ -118,27 +135,25 @@ MemoryMax=$(( $(awk '/MemTotal/ {print int($2 / 1048576)}' /proc/meminfo) * 2 / EOF cat > "$units/horizon-runner.service" < "$units/horizon-runner-pause.service" < /dev/null + # Only the watcher starts the runner (never the boot itself): it decides whether anyone is at the screen. + systemctl --user enable horizon-runner-pause.service > /dev/null + # The runner reads .env when it starts: stop it, unless it is in a job (that would cancel the job); the watcher + # starts it again when it may run. + if pgrep -f "$dir/bin/Runner.Worker" > /dev/null; then echo "In a job: stop horizon-runner.service after it" + else systemctl --user stop horizon-runner.service; rm -f "$dir/paused"; fi systemctl --user restart horizon-runner-pause.service - [ -e "$dir/paused" ] || systemctl --user start horizon-runner.service status } run() { cd "$dir" fix_shebangs + parity_host_links if command -v vscreen > /dev/null; then eval "$(display_env)" else diff --git a/tools/ci/runner-pause.py b/tools/ci/runner_pause.py similarity index 51% rename from tools/ci/runner-pause.py rename to tools/ci/runner_pause.py index 4a88c6ae4..57f304f1f 100755 --- a/tools/ci/runner-pause.py +++ b/tools/ci/runner_pause.py @@ -1,19 +1,22 @@ #!/usr/bin/env python3 -"""Pauses the Horizon CI runner (tools/ci/linux-runner.sh) while the maintainer plays a game on this machine. +"""Pauses the Horizon CI runner (tools/ci/linux-runner.sh) while the maintainer is at this machine or plays on it. -A game is running when any of these holds: - - fleet-load says so (~/fleet/load: Steam, gamescope, Wine or Proton processes); +The runner works only while nobody is at the screen (rule of 2026-10-25, after ludodesktop was restarted under a +CI load while he used it). It pauses when any of these holds: + - a graphical session is open on seat0, locked or not: what `~/fleet/power status` calls "IN USE"; + - fleet-load says a game runs (~/fleet/load: Steam, gamescope, Wine or Proton processes); - a process of his matches GAME below (the same, plus Minecraft and a Horizon game he started himself); - - a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY % twice in a row); + - a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY %, default 50, in two + samples in a row; the browser, Discord and the compositor stay under 20 %); - ~/.actions-runners/horizon/pause exists (pause by hand: `touch` it; `rm` it to let the watcher decide). Paused, the runner takes no new job: while it is idle its service is stopped, so GitHub sends the jobs to the -Mac mini. A job already running is throttled, not stopped: horizon-ci.slice (the job, and the parity workers other machines -send here) drops to CPU weight 1, I/O weight 1 -and a quota of THROTTLE_CPUS threads, and the service stops once the job ends. Freezing it (SIGSTOP or the cgroup -freezer) would stop the runner's heartbeat, so GitHub would drop the job as lost, and the harnesses' wall-clock -timeouts would fail runs that were only waiting. The runner comes back RESUME_AFTER seconds after the last sign -of a game. Every change is written to ~/.actions-runners/horizon/pause.log. +Mac mini, and the parity workers other machines started here stop (their harnesses go back to those machines). A +job already running is throttled, not stopped: horizon-ci.slice drops to CPU weight 1, I/O weight 1 and a quota of +THROTTLE_CPUS threads, and the service stops once the job ends. Freezing it (SIGSTOP or the cgroup freezer) +would stop the runner's heartbeat, so GitHub would drop the job as lost, and the harnesses' wall-clock timeouts +would fail runs that were only waiting. The runner comes back RESUME_AFTER seconds after the last of these signs. +Every change is written to ~/.actions-runners/horizon/pause.log. """ import json import os @@ -27,7 +30,7 @@ SERVICE, SLICE = "horizon-runner.service", "horizon-ci.slice" MANUAL, STATE, LOG = DIR / "pause", DIR / "paused", DIR / "pause.log" GAME = re.compile(r"(SteamLaunch|gamescope|wine64-preloader|wineserver|wine-preloader|\.exe(\s|$)|proton\b" r"|net\.minecraft\.client\.main\.Main|minecraft-launcher|prismlauncher)", re.I) -GPU_BUSY = int(os.environ.get("GPU_BUSY", 30)) +GPU_BUSY = int(os.environ.get("GPU_BUSY", 50)) INTERVAL = int(os.environ.get("INTERVAL", 15)) RESUME_AFTER = int(os.environ.get("RESUME_AFTER", 300)) THROTTLE_CPUS = int(os.environ.get("THROTTLE_CPUS", 2)) @@ -50,6 +53,16 @@ def ci_process(pid): return True # gone +def at_the_screen(): + """A graphical login on seat0, as ~/fleet/power status counts it (a locked screen counts too).""" + sessions = subprocess.run(["loginctl", "list-sessions", "--no-legend"], capture_output=True, text=True).stdout + for line in sessions.splitlines(): + fields = line.split() + if "seat0" in fields: + return f"session {fields[0]} of {fields[2] if len(fields) > 2 else '?'} at the screen (seat0)" + return None + + def fleet_load_gaming(): status = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "fleet-load" / "status.json" try: @@ -82,7 +95,8 @@ def game_process(): def gpu_users(): """Processes outside the CI slice using the GPU's shader cores, as {pid: SM %}.""" try: - output = subprocess.run(["nvidia-smi", "pmon", "-c", "1", "-s", "u"], capture_output=True, text=True, + # Two samples: the first one nvidia-smi prints has no utilisation yet. + output = subprocess.run(["nvidia-smi", "pmon", "-c", "2", "-s", "u"], capture_output=True, text=True, timeout=20).stdout except (OSError, subprocess.TimeoutExpired): return {} @@ -92,7 +106,7 @@ def gpu_users(): if line.startswith("#") or len(fields) < 4 or not fields[1].isdigit() or not fields[3].isdigit(): continue if not ci_process(int(fields[1])): - users[int(fields[1])] = int(fields[3]) + users[int(fields[1])] = max(users.get(int(fields[1]), 0), int(fields[3])) return users @@ -119,47 +133,60 @@ def throttle(on): systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", "CPUQuota=") +def pause_reason(hot): + """Why the runner may not work now, or None. hot: the GPU users seen busy at the last sample (updated).""" + if MANUAL.exists(): + return f"paused by hand ({MANUAL})" + reason = at_the_screen() or fleet_load_gaming() or game_process() + if reason: + return reason + busy = {pid for pid, sm in gpu_users().items() if sm >= GPU_BUSY} + twice = sorted(busy & set(hot)) + hot.clear() + hot.update(dict.fromkeys(busy)) + if twice: + try: + name = Path(f"/proc/{twice[0]}/comm").read_text().strip() + except OSError: + name = "?" + return f"GPU busy: process {twice[0]} ({name})" + return None + + def main(): - hot = {} # GPU users seen busy at the last sample - last_game = 0.0 - paused = STATE.exists() + hot = {} + # A boot (or a restart of this watcher) counts as a sign: the runner starts RESUME_AFTER seconds later at the + # earliest, which leaves time to log in after switching the machine on. + last_sign = time.monotonic() + paused = None while True: - reason = None - if MANUAL.exists(): - reason = "paused by hand (" + str(MANUAL) + ")" - reason = reason or fleet_load_gaming() or game_process() - if not reason: - busy = {pid for pid, sm in gpu_users().items() if sm >= GPU_BUSY} - twice = busy & set(hot) - hot = dict.fromkeys(busy) - if twice: - pid = min(twice) - try: - name = Path(f"/proc/{pid}/comm").read_text().strip() - except OSError: - name = "?" - reason = f"GPU busy: process {pid} ({name})" + reason = pause_reason(hot) now = time.monotonic() if reason: - last_game = now - if reason and not paused: - paused = True - STATE.write_text(reason + "\n") - throttle(True) # also what other machines' parity workers run here meanwhile - if runner_busy(): - log(f"pause: {reason}; a job is running: throttled to {THROTTLE_CPUS} threads, stops after it") - else: + last_sign = now + active = systemctl("is-active", "--quiet", SERVICE).returncode == 0 + if reason or now - last_sign < RESUME_AFTER: + if paused is not True: + paused = True + STATE.write_text((reason or "starting") + "\n") + throttle(True) + # Parity workers other machines started here (tools/parity_hosts.json): they stop, and those + # machines run the harnesses themselves. + systemctl("stop", "horizon-parity-worker-*.scope") + log(f"pause: {reason or 'waiting ' + str(RESUME_AFTER) + ' s after starting'}" + + (f"; a job is running: throttled to {THROTTLE_CPUS} threads, stops after it" if active else "")) + if active and not runner_busy(): systemctl("stop", SERVICE) - log(f"pause: {reason}; runner stopped") - elif paused and reason is None and now - last_game >= RESUME_AFTER: - paused = False - STATE.unlink(missing_ok=True) - throttle(False) - systemctl("start", SERVICE) - log(f"resume: no game for {RESUME_AFTER} s; runner started") - elif paused and systemctl("is-active", "--quiet", SERVICE).returncode == 0 and not runner_busy(): - systemctl("stop", SERVICE) - log("the job ended: runner stopped until the game ends") + log("runner stopped") + else: + if paused is not False: + paused = False + STATE.unlink(missing_ok=True) + throttle(False) + log(f"resume: nobody at the screen and no game for {RESUME_AFTER} s") + if not active: + systemctl("start", SERVICE) + log("runner started") time.sleep(INTERVAL) diff --git a/tools/parity_hosts.json b/tools/parity_hosts.json index 0aa013a5b..5d561175d 100644 --- a/tools/parity_hosts.json +++ b/tools/parity_hosts.json @@ -11,6 +11,15 @@ "jobs": 14, "exclude": {} }, + "desktop": { + "ssh": "desktop", + "root": ".actions-runners/horizon/parity-host", + "shell": "systemd-run --user --scope --quiet --slice=horizon-ci.slice --unit=horizon-parity-worker-$$ bash -c", + "setup": "test ! -e ~/.actions-runners/horizon/paused && set -a && . ~/.actions-runners/horizon/.env && set +a && export PATH=$(cat ~/.actions-runners/horizon/.path) && eval \"$(vscreen env)\"", + "jobs": 8, + "nice": 19, + "note": "ludodesktop, the maintainer's own machine: only while nobody is at the screen (tools/ci/runner_pause.py: a paused desktop refuses workers and stops the running ones, whose harnesses then run here); 8 jobs at most, in the CI runner's environment and slice" + }, "thinkpad": { "ssh": "thinkpad", "root": "code/horizon-parity", From 0baf6cbd62e5dc053732354b5ebd356aae3ffd83 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:27:22 +0100 Subject: [PATCH 3/6] CI: the desktop runner works while its screen is locked or untouched for 15 min MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ludovic's rule (2026-10-26): runner_pause.py pauses at once on keyboard, mouse or gamepad input (read from /dev/input, only whether bytes came), on an unlocked session touched in the last 15 min, at 90 °C, on a game or the GPU. Fixes the service start: the link loop's last test failed the script under set -e. Co-Authored-By: Claude Opus 5.5 --- docs/parity-workflow.md | 2 +- docs/release-flow.md | 32 +++++++------- tests/test_runner_pause.py | 13 +++++- tools/ci/linux-runner.sh | 12 +++--- tools/ci/runner_pause.py | 88 ++++++++++++++++++++++++++++++++------ tools/parity_hosts.json | 2 +- 6 files changed, 113 insertions(+), 36 deletions(-) diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index 80426fa59..7eecdd8bb 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -38,7 +38,7 @@ The game logs into `HORIZON_HOME` and falls back to the player's own game folder ### Other machines -Harnesses can also run on the Mac mini, the ThinkPad and the desktop (`tools/parity_hosts.json` describes them; `desktop` uses its CI runner's environment and vanilla data, and refuses work, or stops it, while someone is at it: docs/release-flow.md, "The desktop's runner"; its `default` list is empty, so pass `--hosts macmini,thinkpad` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 3 and only those under 800 MB. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. +Harnesses can also run on the Mac mini, the ThinkPad and the desktop (`tools/parity_hosts.json` describes them; `desktop` uses its CI runner's environment and vanilla data, and refuses work, or stops it, while someone uses it: docs/release-flow.md, "The desktop's runner"; its `default` list is empty, so pass `--hosts macmini,thinkpad` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 8. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. Harnesses a host can't run are listed in the file: `local_only` (`save_roundtrip` runs the game on a virtual screen) and each host's `exclude`. The Mac excludes nothing: vanilla's `Math.sin`, `pow`, `log` and the like don't round the same on an Apple silicon Mac as on Linux (HotSpot uses fdlibm there, Intel's stubs on x86-64), and Rust's float functions round like neither (they call the platform libm), so code that mirrors a vanilla `Math` call uses `horizon_core::jmath`, which computes what HotSpot computes on the host architecture. The ThinkPad helps too, gently: Linux, 2 harnesses at `nice 19`, since it hosts the Claude account pool; it finds ALSA through `~/code/horizon-parity/alsa-dev` like the desktop. diff --git a/docs/release-flow.md b/docs/release-flow.md index e72bebc73..d9d1801aa 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -68,21 +68,23 @@ release's macOS build back. Each keeps its own checkout, `target/` and Rust inst time, under a lock) and the parity passes and slots. **The desktop's runner.** `ludodesktop-horizon` (labels `self-hosted`, `Linux`, `X64`, `ludodesktop`, and -`horizon-game` once a full trial has passed there) runs on the maintainer's Linux desktop, only while nobody is -at it. `tools/ci/linux-runner.sh install`, run there as him, registers it and installs two systemd user services: -`horizon-runner` (the runner, in `~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O -weight 20 and memory capped at two thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the -machine froze under ~7 cold cargo builds on 2026-10-14; its own Rust install; the JDK, sccache and the build -libraries from nixpkgs; its own `vscreen` so no game it starts reaches his screen) and `horizon-runner-pause` -(`tools/ci/runner_pause.py`), the only thing that starts the runner. The watcher stops it while a graphical -session is open on the machine, locked or not (`~/fleet/power status` says "IN USE"), while a game runs (Steam, -gamescope, Wine, Proton or Minecraft processes, fleet-load's own detection, or a process outside the CI slice -using the GPU at 50 % or more), or while `~/.actions-runners/horizon/pause` exists, and starts it again 5 minutes -after the last such sign. A job already under way is throttled to 2 threads at the lowest weight and the runner -stops after it: freezing the job would make GitHub drop it as lost and the harnesses time out. Old pull requests' -`runner-env.sh` know only the Mac: the runner's `.env` and `.path` give their jobs Python, the JDK and the -vanilla data (`HORIZON_SHARED_SOURCE`) all the same. The nightly audit stays on the Mac mini's runners, so its -comparison with the previous night is always made on one platform. +`horizon-game` once a full trial has passed there) runs on the maintainer's Linux desktop while nobody uses it: its +screen locked, or untouched for 15 minutes (his rule, 2026-10-26). `tools/ci/linux-runner.sh install`, run there as +him, registers it and installs two systemd user services: `horizon-runner` (the runner, in +`~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O weight 20 and memory capped at two +thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the machine froze under ~7 cold cargo +builds on 2026-10-14; its own Rust install; the JDK, sccache and the build libraries from nixpkgs; its own +`vscreen` so no game it starts reaches his screen) and `horizon-runner-pause` (`tools/ci/runner_pause.py`), the +only thing that starts the runner. The watcher stops it at once on keyboard, mouse or gamepad input (read from +`/dev/input`: only whether bytes came, never what), while an unlocked session has had input in the last 15 minutes, +while the CPU package is at 90 °C or more, while a game runs (Steam, gamescope, Wine, Proton or Minecraft +processes, fleet-load's own detection, or a process outside the CI slice using the GPU at 50 % or more), or while +`~/.actions-runners/horizon/pause` exists, and starts it again 5 minutes after the last such sign. A job already +under way is throttled to 2 threads at the lowest weight and the runner stops after it: freezing the job would make +GitHub drop it as lost and the harnesses time out. Old pull requests' `runner-env.sh` know only the Mac: the +runner's `.env` and `.path` give their jobs Python, the JDK and the vanilla data (`HORIZON_SHARED_SOURCE`) all the +same. The nightly audit stays on the Mac mini's runners, so its comparison with the previous night is always made +on one platform. ## Merging diff --git a/tests/test_runner_pause.py b/tests/test_runner_pause.py index c6df08b82..38c4affd8 100644 --- a/tests/test_runner_pause.py +++ b/tests/test_runner_pause.py @@ -18,7 +18,7 @@ class Machine: self.throttles = [] # (clock, on) self.calls = [] - def reason(self, hot): + def reason(self, hot, keys): return next((reason for start, end, reason in self.signs if start <= self.clock < end), None) def systemctl(self, *arguments): @@ -77,6 +77,17 @@ class RunnerPauseTests(unittest.TestCase): self.assertLess(stop, 1215) self.assertEqual(machine.throttles[-1], (1290 + 300, False)) + def test_the_desktop_is_free_when_locked_or_untouched_for_15_minutes(self): + cases = [ # (session, seconds since the last input, someone there) + (("3", False), 30, True), (("3", True), 30, True), # any input pauses at once, even on the lock screen + (("3", False), 120, True), (("3", False), 1000, False), # unlocked: free after 15 min untouched + (("3", True), 120, False), (None, 120, False), # locked, or nobody logged in + ] + for session, idle, there in cases: + with self.subTest(session=session, idle=idle), \ + mock.patch.object(runner_pause, "seat_session", lambda: session): + self.assertEqual(runner_pause.at_the_screen(idle) is not None, there) + if __name__ == "__main__": unittest.main() diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index adc2b0130..1bd587e79 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -9,10 +9,10 @@ # # Everything lives in ~/.actions-runners/horizon (never in ~/code): the runner, its own rustup and cargo homes, # the work folder, and copies of this script and runner_pause.py taken at install time (a pull request's checkout -# never runs as the service). horizon-runner-pause.service (runner_pause.py) starts the runner only while nobody is -# at the screen or playing, and stops it otherwise. The runner runs nice, in horizon-ci.slice (low CPU and I/O weight, memory capped), -# on its own virtual screen: vscreen (a headless sway) where it is installed, else Xvfb from nixpkgs with Mesa's -# Vulkan drivers. +# never runs as the service). horizon-runner-pause.service (runner_pause.py) starts the runner only while nobody +# uses the machine (screen locked or untouched for 15 min, no game), and stops it otherwise. The runner runs nice, +# in horizon-ci.slice (low CPU and I/O weight, memory capped), on its own virtual screen: vscreen (a headless sway) +# where it is installed, else Xvfb from nixpkgs with Mesa's Vulkan drivers. # HORIZON_SHARED_SOURCE: the imported vanilla data (default ~/horizon-local/shared-local, else # ~/code/horizon/.local); RUNNER_JOBS: cargo jobs and harnesses at once (default a third of the threads, at most 8: # ludodesktop froze under ~7 cold cargo builds on 2026-10-14, and it is the maintainer's own machine). @@ -40,7 +40,7 @@ parity_host_links() { [ -d "$checkout" ] || return 0 mkdir -p "$shared" for entry in "$checkout"/*; do - [ -L "$entry" ] && ln -sfn "$entry" "$shared/$(basename "$entry")" + if [ -L "$entry" ]; then ln -sfn "$entry" "$shared/$(basename "$entry")"; fi done } @@ -151,7 +151,7 @@ RestartSec=30 EOF cat > "$units/horizon-runner-pause.service" <= GPU_BUSY %, default 50, in two @@ -34,6 +36,9 @@ GPU_BUSY = int(os.environ.get("GPU_BUSY", 50)) INTERVAL = int(os.environ.get("INTERVAL", 15)) RESUME_AFTER = int(os.environ.get("RESUME_AFTER", 300)) THROTTLE_CPUS = int(os.environ.get("THROTTLE_CPUS", 2)) +INPUT_PAUSE = int(os.environ.get("INPUT_PAUSE", 60)) +IDLE_AFTER = int(os.environ.get("IDLE_AFTER", 900)) +TEMP_LIMIT = int(os.environ.get("TEMP_LIMIT", 90)) def log(text): @@ -53,13 +58,71 @@ def ci_process(pid): return True # gone -def at_the_screen(): - """A graphical login on seat0, as ~/fleet/power status counts it (a locked screen counts too).""" +class Input: + """Keyboard, mouse and gamepad activity, from /dev/input (the input group): only whether bytes came, never what.""" + + def __init__(self): + self.devices = {} # path -> file descriptor + self.last = time.monotonic() # a start counts as input: nothing is known yet + + def poll(self): + try: + text = Path("/proc/bus/input/devices").read_text() + except OSError: + text = "" + wanted = {f"/dev/input/{handler}" for block in text.split("\n\n") for line in block.splitlines() + if line.startswith("H: Handlers=") and re.search(r"\b(kbd|mouse\d*|js\d*)\b", line) + for handler in line.split("=", 1)[1].split() if handler.startswith("event")} + for path in set(self.devices) - wanted: + os.close(self.devices.pop(path)) + for path in wanted - set(self.devices): + try: + self.devices[path] = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + except OSError: + pass + for path, descriptor in list(self.devices.items()): + try: + while os.read(descriptor, 4096): + self.last = time.monotonic() + except BlockingIOError: + pass + except OSError: # unplugged + os.close(self.devices.pop(path)) + return time.monotonic() - self.last + + +def seat_session(): + """The graphical session on seat0 and whether its screen is locked, or None.""" sessions = subprocess.run(["loginctl", "list-sessions", "--no-legend"], capture_output=True, text=True).stdout for line in sessions.splitlines(): fields = line.split() if "seat0" in fields: - return f"session {fields[0]} of {fields[2] if len(fields) > 2 else '?'} at the screen (seat0)" + locked = subprocess.run(["loginctl", "show-session", fields[0], "-p", "LockedHint", "--value"], + capture_output=True, text=True).stdout.strip() == "yes" + return fields[0], locked + return None + + +def at_the_screen(input_idle): + """Someone is at the machine: input in the last INPUT_PAUSE seconds, or an unlocked session with input in the last + IDLE_AFTER seconds. A locked screen, or one nobody touched for IDLE_AFTER, leaves the machine to CI.""" + if input_idle < INPUT_PAUSE: + return f"input {input_idle:.0f} s ago" + session = seat_session() + if session and not session[1] and input_idle < IDLE_AFTER: + return f"session {session[0]} unlocked, input {input_idle:.0f} s ago" + return None + + +def too_hot(): + """The CPU package above TEMP_LIMIT °C (coretemp or k10temp).""" + for monitor in Path("/sys/class/hwmon").glob("hwmon*"): + try: + if (monitor / "name").read_text().strip() in ("coretemp", "k10temp"): + celsius = int((monitor / "temp1_input").read_text()) / 1000 + return f"CPU at {celsius:.0f} °C" if celsius >= TEMP_LIMIT else None + except (OSError, ValueError): + pass return None @@ -133,11 +196,12 @@ def throttle(on): systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", "CPUQuota=") -def pause_reason(hot): - """Why the runner may not work now, or None. hot: the GPU users seen busy at the last sample (updated).""" +def pause_reason(hot, keys): + """Why the runner may not work now, or None. hot: the GPU users seen busy at the last sample (updated); + keys: the Input watcher.""" if MANUAL.exists(): return f"paused by hand ({MANUAL})" - reason = at_the_screen() or fleet_load_gaming() or game_process() + reason = at_the_screen(keys.poll()) or too_hot() or fleet_load_gaming() or game_process() if reason: return reason busy = {pid for pid, sm in gpu_users().items() if sm >= GPU_BUSY} @@ -154,13 +218,13 @@ def pause_reason(hot): def main(): - hot = {} + hot, keys = {}, Input() # A boot (or a restart of this watcher) counts as a sign: the runner starts RESUME_AFTER seconds later at the # earliest, which leaves time to log in after switching the machine on. last_sign = time.monotonic() paused = None while True: - reason = pause_reason(hot) + reason = pause_reason(hot, keys) now = time.monotonic() if reason: last_sign = now diff --git a/tools/parity_hosts.json b/tools/parity_hosts.json index 5d561175d..f2ba65c68 100644 --- a/tools/parity_hosts.json +++ b/tools/parity_hosts.json @@ -18,7 +18,7 @@ "setup": "test ! -e ~/.actions-runners/horizon/paused && set -a && . ~/.actions-runners/horizon/.env && set +a && export PATH=$(cat ~/.actions-runners/horizon/.path) && eval \"$(vscreen env)\"", "jobs": 8, "nice": 19, - "note": "ludodesktop, the maintainer's own machine: only while nobody is at the screen (tools/ci/runner_pause.py: a paused desktop refuses workers and stops the running ones, whose harnesses then run here); 8 jobs at most, in the CI runner's environment and slice" + "note": "ludodesktop, the maintainer's own machine: only while nobody uses it (tools/ci/runner_pause.py: screen locked or untouched for 15 min, no game; a paused desktop refuses workers and stops the running ones, whose harnesses then run here); 8 jobs at most, in the CI runner's environment and slice" }, "thinkpad": { "ssh": "thinkpad", From 9db6a0ee48a6862df8e27fe0e5cb587a61c67e0f Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:23:27 +0100 Subject: [PATCH 4/6] CI: parity-main stays on the Mac; heat check on a median, boot as the first sign; nix roots fixed nix-build names a dev output's GC root -dev, so the runner's PKG_CONFIG_PATH pointed at links that didn't exist and every client build failed on libudev; it now uses the store path nix-build prints. The package temperature spikes for a second under load, so the heat check takes the median of 5 readings. Restarting the watcher no longer throttles a running job. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parity-main.yml | 4 +++- docs/release-flow.md | 5 +++-- tools/ci/linux-runner.sh | 5 +++-- tools/ci/runner_pause.py | 26 ++++++++++++++++---------- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/parity-main.yml b/.github/workflows/parity-main.yml index 1ddfb24bb..1debcc694 100644 --- a/.github/workflows/parity-main.yml +++ b/.github/workflows/parity-main.yml @@ -23,7 +23,9 @@ env: jobs: parity-main: - runs-on: [self-hosted, horizon-game] + # The Mac mini's runners only: a desktop job slowed down because the maintainer came back could time out, and a + # failure here bisects and comments on merges. + runs-on: [self-hosted, horizon-game, macOS] timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures steps: - uses: actions/checkout@v4 diff --git a/docs/release-flow.md b/docs/release-flow.md index 28d1feef9..dc96498a0 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -90,8 +90,9 @@ processes, fleet-load's own detection, or a process outside the CI slice using t under way is throttled to 2 threads at the lowest weight and the runner stops after it: freezing the job would make GitHub drop it as lost and the harnesses time out. Old pull requests' `runner-env.sh` know only the Mac: the runner's `.env` and `.path` give their jobs Python, the JDK and the vanilla data (`HORIZON_SHARED_SOURCE`) all the -same. The nightly audit stays on the Mac mini's runners, so its comparison with the previous night is always made -on one platform. +same. The nightly audit and parity-main stay on the Mac mini's runners: the audit compares with the previous night, +so it stays on one platform, and a parity-main job slowed down on the desktop because he came back could time out, +and its failures bisect and comment on merges. ## Merging diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index 1bd587e79..c7235f5c1 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -24,8 +24,9 @@ version=2.338.0 sha=af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32 here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# Nix packages the jobs need, kept alive by GC roots in $dir/nix. -nix_out() { nix-build '' -A "$1" -o "$dir/nix/$2" > /dev/null && readlink -f "$dir/nix/$2"; } +# Nix packages the jobs need, kept alive by GC roots in $dir/nix. nix-build prints the store path; the root's name +# gets the output's suffix (alsa-dev-dev for alsa-lib.dev), so it is not read back from the link. +nix_out() { nix-build '' -A "$1" -o "$dir/nix/$2" | tail -1; } # The runner is a .NET program: on NixOS its native libraries (libstdc++, ICU, OpenSSL) come from nix-ld. export LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} diff --git a/tools/ci/runner_pause.py b/tools/ci/runner_pause.py index 35476a0b2..8fa0a4c5b 100755 --- a/tools/ci/runner_pause.py +++ b/tools/ci/runner_pause.py @@ -5,7 +5,7 @@ The runner works while nobody uses the machine: its screen locked, or untouched It pauses when any of these holds: - keyboard, mouse or gamepad input in the last INPUT_PAUSE seconds (60), or an unlocked session on seat0 with input in the last IDLE_AFTER seconds; - - the CPU package at TEMP_LIMIT °C (90) or more; + - the CPU package at TEMP_LIMIT °C (90) or more (the median of 5 readings); - fleet-load says a game runs (~/fleet/load: Steam, gamescope, Wine or Proton processes); - a process of his matches GAME below (the same, plus Minecraft and a Horizon game he started himself); - a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY %, default 50, in two @@ -63,7 +63,7 @@ class Input: def __init__(self): self.devices = {} # path -> file descriptor - self.last = time.monotonic() # a start counts as input: nothing is known yet + self.last = 0.0 # the boot (the monotonic clock's zero on Linux) counts as input def poll(self): try: @@ -115,14 +115,20 @@ def at_the_screen(input_idle): def too_hot(): - """The CPU package above TEMP_LIMIT °C (coretemp or k10temp).""" + """The CPU package at TEMP_LIMIT °C or more (coretemp or k10temp), as the median of 5 readings over 2 s: under + load it jumps for a second at a time (to 96 °C on ludodesktop at a median of 68 °C), which is no reason to stop.""" for monitor in Path("/sys/class/hwmon").glob("hwmon*"): try: - if (monitor / "name").read_text().strip() in ("coretemp", "k10temp"): - celsius = int((monitor / "temp1_input").read_text()) / 1000 - return f"CPU at {celsius:.0f} °C" if celsius >= TEMP_LIMIT else None + if (monitor / "name").read_text().strip() not in ("coretemp", "k10temp"): + continue + readings = [] + for _ in range(5): + readings.append(int((monitor / "temp1_input").read_text()) / 1000) + time.sleep(0.5) except (OSError, ValueError): - pass + continue + celsius = sorted(readings)[2] + return f"CPU at {celsius:.0f} °C" if celsius >= TEMP_LIMIT else None return None @@ -219,9 +225,9 @@ def pause_reason(hot, keys): def main(): hot, keys = {}, Input() - # A boot (or a restart of this watcher) counts as a sign: the runner starts RESUME_AFTER seconds later at the - # earliest, which leaves time to log in after switching the machine on. - last_sign = time.monotonic() + # The boot counts as a sign: the runner starts RESUME_AFTER seconds after it at the earliest, which leaves time + # to log in after switching the machine on. Restarting this watcher later changes nothing for a running job. + last_sign = 0.0 paused = None while True: reason = pause_reason(hot, keys) From 49bbdf8954bdc8958bc6f8a2e7917bb8035f710c Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:38:23 +0100 Subject: [PATCH 5/6] CI: the desktop's CI slice gets half the threads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On all 24, a cargo build held ludodesktop's CPU at 92 °C and the watcher kept pausing; at half, harnesses run at a median of 80 °C. Co-Authored-By: Claude Opus 5.5 --- tools/ci/linux-runner.sh | 1 + tools/ci/runner_pause.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index c7235f5c1..699742bf1 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -131,6 +131,7 @@ Description=Horizon CI: the runner's jobs and parity workers sent from other mac [Slice] CPUWeight=20 IOWeight=20 +CPUQuota=$(( threads * 50 ))% MemoryHigh=$(( $(awk '/MemTotal/ {print int($2 / 1048576)}' /proc/meminfo) / 2 ))G MemoryMax=$(( $(awk '/MemTotal/ {print int($2 / 1048576)}' /proc/meminfo) * 2 / 3 ))G EOF diff --git a/tools/ci/runner_pause.py b/tools/ci/runner_pause.py index 8fa0a4c5b..845627243 100755 --- a/tools/ci/runner_pause.py +++ b/tools/ci/runner_pause.py @@ -36,6 +36,8 @@ GPU_BUSY = int(os.environ.get("GPU_BUSY", 50)) INTERVAL = int(os.environ.get("INTERVAL", 15)) RESUME_AFTER = int(os.environ.get("RESUME_AFTER", 300)) THROTTLE_CPUS = int(os.environ.get("THROTTLE_CPUS", 2)) +# Working, the slice gets half the threads: on all of them ludodesktop's CPU sits at 92 °C through a cargo build. +NORMAL_CPUS = int(os.environ.get("NORMAL_CPUS", max(1, (os.cpu_count() or 2) // 2))) INPUT_PAUSE = int(os.environ.get("INPUT_PAUSE", 60)) IDLE_AFTER = int(os.environ.get("IDLE_AFTER", 900)) TEMP_LIMIT = int(os.environ.get("TEMP_LIMIT", 90)) @@ -199,7 +201,7 @@ def throttle(on): if on: systemctl("set-property", "--runtime", SLICE, "CPUWeight=1", "IOWeight=1", f"CPUQuota={THROTTLE_CPUS * 100}%") else: - systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", "CPUQuota=") + systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", f"CPUQuota={NORMAL_CPUS * 100}%") def pause_reason(hot, keys): From d782e455a41f21e79675347c4314c9e8e2bdb535 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:55:57 +0100 Subject: [PATCH 6/6] docs: the desktop's CI slice gets half the threads Co-Authored-By: Claude Opus 5.5 --- docs/release-flow.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release-flow.md b/docs/release-flow.md index dc96498a0..7bb64d21e 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -78,8 +78,8 @@ time, under a lock) and the parity passes and slots. `horizon-game` once a full trial has passed there) runs on the maintainer's Linux desktop while nobody uses it: its screen locked, or untouched for 15 minutes (his rule, 2026-10-26). `tools/ci/linux-runner.sh install`, run there as him, registers it and installs two systemd user services: `horizon-runner` (the runner, in -`~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O weight 20 and memory capped at two -thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the machine froze under ~7 cold cargo +`~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O weight 20, half the threads (on all of +them the CPU package reached 92–96 °C) and memory capped at two thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the machine froze under ~7 cold cargo builds on 2026-10-14; its own Rust install; the JDK, sccache and the build libraries from nixpkgs; its own `vscreen` so no game it starts reaches his screen) and `horizon-runner-pause` (`tools/ci/runner_pause.py`), the only thing that starts the runner. The watcher stops it at once on keyboard, mouse or gamepad input (read from