diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index 6cb235c3c..73ac5c47c 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -40,13 +40,13 @@ The game logs into `HORIZON_HOME` and falls back to the player's own game folder ### Other machines -Harnesses can also run on the Mac mini, the desktop and playhorizon (`tools/parity_hosts.json` describes them; `desktop` and `playhorizon` use their CI runner's environment and vanilla data, and the desktop refuses work, or stops it, while someone uses it: docs/release-flow.md, "The desktop's runner"; its `default` list is empty, so pass `--hosts macmini,playhorizon` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 8, playhorizon 6. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. +Harnesses can also run on the Mac mini, the desktop and playhorizon (`tools/parity_hosts.json` describes them; `desktop` and `playhorizon` use their CI runner's environment and vanilla data, and the desktop refuses work, or stops it, while someone uses it: docs/release-flow.md, "The desktop's runner"; its `default` list is empty, so pass `--hosts macmini,playhorizon` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 8, playhorizon 3 (on CPUs 0-3: its CI runner has 4-7). Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. Harnesses a host can't run are listed in the file: `local_only` (`save_roundtrip` runs the game on a virtual screen) and each host's `exclude`. The Mac excludes nothing: vanilla's `Math.sin`, `pow`, `log` and the like don't round the same on an Apple silicon Mac as on Linux (HotSpot uses fdlibm there, Intel's stubs on x86-64), and Rust's float functions round like neither (they call the platform libm), so code that mirrors a vanilla `Math` call uses `horizon_core::jmath`, which computes what HotSpot computes on the host architecture. The ThinkPad no longer runs harnesses (2026-10-26): playhorizon is the Linux host that is always on. ### Any command on playhorizon: `tools/remote.sh` -For Linux work that doesn't need a whole parity run (a crate's tests, a few harnesses, a server build), `tools/remote.sh ` runs the command on playhorizon at the worktree's commit and streams its output back; its exit status is the command's. Commit and push first: the host fetches the commit from GitHub, and nothing else leaves this machine (no files, no credentials, no environment). Each worktree keeps one checkout there, `~/remote-work/` with its own `target/`, so the next run only rebuilds what changed, and one run per name goes at a time. The command runs niced in the host's CI slice, pinned to CPUs 4-7 (`--cpus`; 0-3 stay free for benchmarks), with scratch game and launcher homes and the vanilla data linked as CI links it; a command naming `parity`, `lan_e2e` or `horizon-prepare` also gets `.local/horizon-client`, prepared as CI prepares it. Whatever the command leaves running (a game, a daemon) is stopped when it ends, and stopping `remote.sh` (or losing the connection) stops the command on the host. Logs stay in `~/remote-work/.logs/`; `--list` shows the checkouts, `--gc [DAYS]` removes old ones. +For Linux work that doesn't need a whole parity run (a crate's tests, a few harnesses, a server build), `tools/remote.sh ` runs the command on playhorizon at the worktree's commit and streams its output back; its exit status is the command's. Commit and push first: the host fetches the commit from GitHub, and nothing else leaves this machine (no files, no credentials, no environment). Each worktree keeps one checkout there, `~/remote-work/` with its own `target/`, so the next run only rebuilds what changed, and one run per name goes at a time. The command runs niced in the host's CI slice, pinned to CPUs 0-3 (`--cpus`): the CI runner has 4-7, so these runs never slow a pull request's parity job. A benchmark asks for its CPUs with `--cpus 0-3 --exclusive` (and says so in its pull request): it starts only when no other run uses them, and later runs wait for it to end. The command gets scratch game and launcher homes and the vanilla data linked as CI links it; a command naming `parity`, `lan_e2e` or `horizon-prepare` also gets `.local/horizon-client`, prepared as CI prepares it. Whatever the command leaves running (a game, a daemon) is stopped when it ends, and stopping `remote.sh` (or losing the connection) stops the command on the host. Logs stay in `~/remote-work/.logs/`; `--list` shows the checkouts, `--gc [DAYS]` removes old ones. ```sh tools/remote.sh .claude/worktrees/my-fix cargo test -p horizon-world --lib # 33 s, warm diff --git a/docs/release-flow.md b/docs/release-flow.md index e6a0a3856..be0ccbee8 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -108,12 +108,12 @@ and its failures bisect and comment on merges. AMD EPYC threads, 30 GB, in Detroit; reachable only through the maintainer's tailnet). Nobody sits at it, so the watcher never pauses it except by hand. It gets half the machine: `RUNNER_MEMORY=24`, `CARGO_BUILD_JOBS` and the harness slots at 4, and a drop-in (`~/.config/systemd/user/horizon-runner.service.d/cpus.conf`) that starts it under -`taskset -c 4-7`, so CPUs 0-3 stay free for benchmarks pinned there (the user slice can't set `AllowedCPUs`: systemd -doesn't delegate the cpuset controller to it). +`taskset -c 4-7` (the user slice can't set `AllowedCPUs`: systemd doesn't delegate the cpuset controller to it). +Agents' `tools/remote.sh` runs and benchmarks share CPUs 0-3, so they never slow a pull request's job. Without Nix, `linux-runner.sh` takes the build libraries, the JDK, sccache, Xvfb and Mesa's software Vulkan (lavapipe; there is no GPU) from apt, and Python 3.13 from uv. The partner can reach the machine, so gh is not logged in there: the registration token comes from another machine (`RUNNER_TOKEN`), and no job with secrets may -run there (parity has none). It is also a parity host (`playhorizon` in `tools/parity_hosts.json`, 6 jobs). Since +run there (parity has none). It is also a parity host (`playhorizon` in `tools/parity_hosts.json`, 3 jobs on CPUs 0-3). Since 2026-10-26 it has the `horizon-game` label: pull requests' parity jobs run there or on the Mac mini, and `full-parity` ones only there. diff --git a/tools/parity_hosts.json b/tools/parity_hosts.json index 60e6205e8..29815a9f6 100644 --- a/tools/parity_hosts.json +++ b/tools/parity_hosts.json @@ -23,17 +23,17 @@ "playhorizon": { "ssh": "ludovic@playhorizon", "root": ".actions-runners/horizon/parity-host", - "shell": "systemd-run --user --scope --quiet --slice=horizon-ci.slice --unit=horizon-parity-worker-$$ bash -c", + "shell": "systemd-run --user --scope --quiet --slice=horizon-ci.slice --unit=horizon-parity-worker-$$ taskset -c 0-3 bash -c", "setup": "test ! -e ~/.actions-runners/horizon/paused && set -a && . ~/.actions-runners/horizon/.env && set +a && export PATH=$(cat ~/.actions-runners/horizon/.path) && unset WAYLAND_DISPLAY && export DISPLAY=:$(cat ~/.actions-runners/horizon/display)", "python": "python3.13", - "jobs": 6, + "jobs": 3, "nice": 10, "exclude": { "mob_ai": "the vanilla oracle runs out of its 3 GB heap here (OutOfMemoryError in IO workers; passes on the desktop and the Mac) - 2026-10-26 trial", "lan_integration": "two clients rendered on the CPU (lavapipe): different timing subtests fail on each run (elytra, consume-release, barter, critical-hits) - 2026-10-26 trial", "world_reopen": "failed twice here (no Save and Quit button after reopening; the minecart-count flake), passes on the desktop - 2026-10-26 trial" }, - "note": "a partner's server (Ubuntu, 8 EPYC threads, 30 GB, Detroit; tailnet only), used by nothing else: 6 jobs, in the CI runner's environment, slice and Xvfb screen (tools/ci/linux-runner.sh); the always-on Linux host since the ThinkPad stopped running harnesses (2026-10-26)" + "note": "a partner's server (Ubuntu, 8 EPYC threads, 30 GB, Detroit; tailnet only), its CI runner has CPUs 4-7, so workers get 0-3 (as tools/remote.sh runs do) and 3 jobs, in the CI runner's environment, slice and Xvfb screen (tools/ci/linux-runner.sh); the always-on Linux host since the ThinkPad stopped running harnesses (2026-10-26)" } }, "local_only": { diff --git a/tools/remote.sh b/tools/remote.sh index 0955666cb..fa913e7cc 100755 --- a/tools/remote.sh +++ b/tools/remote.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash # Run a command on a Linux build host (default playhorizon) at a worktree's commit, and get its output back. # -# tools/remote.sh [--host H] [--name N] [--cpus LIST] [--jobs J] +# tools/remote.sh [--host H] [--name N] [--cpus LIST] [--exclusive] [--jobs J] # tools/remote.sh .claude/worktrees/my-fix cargo test -p horizon-world --lib # tools/remote.sh . python3 tools/parity.py food light +# tools/remote.sh --cpus 0-3 --exclusive . python3 tools/perf_bench.py # a benchmark: CPUs 0-3 to itself # tools/remote.sh --list # checkouts on the host, their size and last use # tools/remote.sh --gc [DAYS] # remove checkouts unused for DAYS (default 2) # @@ -18,19 +19,24 @@ # What the command leaves running is stopped when it ends; stopping this script stops the command on the host. # The exit status is the command's. # +# CPUs: playhorizon's CI runner has CPUs 4-7 (a systemd drop-in pins it, docs/release-flow.md), so runs here get +# 0-3 by default and never slow a pull request's parity job. A run holds its CPU set's lock shared; --exclusive (a +# benchmark: name it with --cpus and say so in its pull request) holds it alone, so it starts only when no other run +# uses those CPUs, and the others wait for it to end. --cpus 4-7 or all compete with CI: only when it is idle. +# # The host is lent by a partner who keeps hypervisor access: never send secrets, tokens or credentials there, # and never treat a result from it as a CI result. set -euo pipefail -# CPUs 0-3 stay free for benchmarks (perf-world-entry measures on playhorizon pinned to them); --cpus all for every one. -host=playhorizon name="" jobs="" cpus=4-7 -usage() { sed -n '2,8s/^# \{0,1\}//p' "$0" >&2; exit 2; } +host=playhorizon name="" jobs="" cpus=0-3 exclusive=0 +usage() { sed -n '2,9s/^# \{0,1\}//p' "$0" >&2; exit 2; } while [ $# -gt 0 ]; do case $1 in --host) host=$2; shift 2;; --name) name=$2; shift 2;; --jobs) jobs=$2; shift 2;; --cpus) cpus=$2; shift 2;; + --exclusive) exclusive=1; shift;; --list) exec ssh "ludovic@$host" 'cd ~/remote-work 2>/dev/null && for d in */; do d=${d%/}; printf "%-32s %8s last used %s\n" "$d" "$(du -sh "$d" 2>/dev/null | cut -f1)" "$(stat -c %y "$d.logs" 2>/dev/null | cut -c1-16)"; done';; --gc) days=${2:-2}; exec ssh "ludovic@$host" "cd ~/remote-work 2>/dev/null || exit 0 @@ -58,7 +64,8 @@ command=$(printf '%q ' "$@") # The host side reads its script and these four values on stdin (nothing is re-split by a remote shell); its # own environment comes from the runner's files there. { -printf 'name=%q sha=%q jobs=%q cpus=%q command=%q\n' "$name" "$sha" "$jobs" "$cpus" "$command" +[[ $cpus =~ ^[0-9,-]+$|^all$ ]] || { echo "remote.sh: --cpus takes a list such as 0-3, or all" >&2; exit 2; } +printf 'name=%q sha=%q jobs=%q cpus=%q exclusive=%q command=%q\n' "$name" "$sha" "$jobs" "$cpus" "$exclusive" "$command" cat <<'REMOTE' set -euo pipefail if [ "$cpus" = all ]; then pin=(); jobs=${jobs:-$(nproc)}; else pin=(taskset -c "$cpus"); jobs=${jobs:-$(taskset -c "$cpus" nproc)}; fi @@ -71,19 +78,27 @@ unset WAYLAND_DISPLAY mkdir -p ~/remote-work && cd ~/remote-work exec 9> "$name.lock" flock -n 9 || { echo "remote.sh: $name is busy with another run" >&2; exit 75; } +# The CPU set's lock: shared by ordinary runs, held alone by an --exclusive one (a benchmark). +exec 8> ".cpus-$cpus.lock" +if [ "$exclusive" = 1 ]; then + flock -x -n 8 || { echo "remote.sh: other runs use CPUs $cpus; try again when they are done (--list)" >&2; exit 75; } +elif ! flock -s -n 8; then + echo "remote.sh: an exclusive run (a benchmark) has CPUs $cpus; waiting for it to end" >&2 + flock -s 8 +fi if [ ! -d "$name/.git" ]; then - git clone -q --filter=blob:none --no-checkout https://github.com/ludovic111/horizon "$name" 9>&- + git clone -q --filter=blob:none --no-checkout https://github.com/ludovic111/horizon "$name" 8>&- 9>&- fi cd "$name" -# The lock's descriptor stays out of everything started here (9>&-): a daemon one of them starts (sccache's server, +# The lock's descriptor stays out of everything started here (8>&- 9>&-): a daemon one of them starts (sccache's server, # from the preparer's build or the command's) would hold the lock after the run. -git fetch -q --no-tags origin "$sha" 9>&- +git fetch -q --no-tags origin "$sha" 8>&- 9>&- git checkout -q --force --detach "$sha" git clean -qfd -e /target -e /.local # The host's vanilla data, linked as CI's runner links it; for harnesses also .local/horizon-client, prepared by # this commit's own preparer (once per preparer version, kept beside the data and shared by every name). if [[ $command =~ parity|lan_e2e|horizon-prepare ]]; then - RUNNER_WORKSPACE=$(dirname "$HORIZON_SHARED_SOURCE") RUNNER_TEMP=$(mktemp -d) bash tools/ci/vanilla-data.sh >&2 9>&- + RUNNER_WORKSPACE=$(dirname "$HORIZON_SHARED_SOURCE") RUNNER_TEMP=$(mktemp -d) bash tools/ci/vanilla-data.sh >&2 8>&- 9>&- else mkdir -p .local for d in minecraft tools datagen; do @@ -103,11 +118,11 @@ unit=remote-$name.scope systemctl --user stop "$unit" 2>/dev/null || true session=$PPID ( while kill -0 "$session" 2>/dev/null; do sleep 2; done; systemctl --user stop "$unit" 2>/dev/null ) \ - < /dev/null > /dev/null 2>&1 9>&- & + < /dev/null > /dev/null 2>&1 8>&- 9>&- & watcher=$! status=0 systemd-run --user --scope --quiet --slice=horizon-ci.slice --unit="$unit" \ - "${pin[@]}" nice -n 10 bash -c "$command" 2>&1 < /dev/null 9>&- | tee "$log" || status=${PIPESTATUS[0]} + "${pin[@]}" nice -n 10 bash -c "$command" 2>&1 < /dev/null 8>&- 9>&- | tee "$log" || status=${PIPESTATUS[0]} kill "$watcher" 2>/dev/null || true systemctl --user stop "$unit" 2>/dev/null || true exit "$status" diff --git a/tools/test_server_player_sculk_parity.py b/tools/test_server_player_sculk_parity.py index 33489ad5a..75c89ea44 100644 --- a/tools/test_server_player_sculk_parity.py +++ b/tools/test_server_player_sculk_parity.py @@ -8,6 +8,7 @@ hosts: the guest always uses the server-player path. import json import math import subprocess +import threading import lab import lan_e2e as lan @@ -43,17 +44,29 @@ def check_walk(host, guest, out, role): until(host, lambda: phase(host), lambda s: 'sculk_sensor_phase=inactive' in s) player.wait(10) assert 'sculk_sensor_phase=inactive' in phase(host), 'setup activated the sensor' - player.raw('key', 'w', 'down') + # The sensor stays active 30 ticks. Each client samples it every tick from inside one ctl script, so a + # runner that answers control requests slowly (frames drawn on the CPU) can't miss that window between + # separate requests; the other client samples alongside the walk. + probe = f'getblock {sensor[0]} {sensor[1]} {sensor[2]}' + other = guest if player is host else host + seen = {} + + def watch(viewer, lines): + seen[viewer.name] = [r['block'] for r in viewer.script(lines, every=True) if 'block' in r] + watcher = threading.Thread(target=watch, args=(other, ['wait 1', probe] * 100)) + watcher.start() try: - player.wait(15) + watch(player, ['key w down', *['wait 1', probe] * 15, 'key w up', *['wait 1', probe] * 85]) finally: player.raw('key', 'w', 'up') + watcher.join() moved = {'client': math.dist(lan.pos_of(player), start), 'server': math.dist(lan.server_player(host, player)['position'], start)} assert all(d > 1 for d in moved.values()), moved - for viewer in (host, guest): - until(viewer, lambda: phase(viewer), lambda s: 'sculk_sensor_phase=active' in s, ticks=85) - return {'moved': moved, 'phase': phase(host)} + active = {name: next((i for i, s in enumerate(states) if 'sculk_sensor_phase=active' in s), None) + for name, states in seen.items()} + assert len(active) == 2 and all(i is not None for i in active.values()), f'sensor never active: {seen}' + return {'moved': moved, 'first_active_sample': active} finally: player.raw('key', 'w', 'up') host('command', command=f'fill {x - 2} {y} {z - 2} {x + 5} {y + 3} {z + 5} minecraft:air') diff --git a/tools/test_world_reopen_parity.py b/tools/test_world_reopen_parity.py index 4ef9a5e87..9487fd1b7 100644 --- a/tools/test_world_reopen_parity.py +++ b/tools/test_world_reopen_parity.py @@ -68,14 +68,23 @@ def main(): return json.loads(result.stdout) if result.stdout.strip().startswith('{') else result.stdout def carts(): - found = ctl('status')['carts'] - return sorted((c['kind'], *(round(v, 1) for v in c['position'])) for c in found) + """The loaded minecarts once their list stops changing: on a loaded runner the mineshaft's chunks and their + entities keep arriving after the world counts as loaded, so one reading can be short of a cart.""" + last = None + for _ in range(30): + found = sorted((c['kind'], *(round(v, 1) for v in c['position'])) for c in ctl('status')['carts']) + if found == last: + return found + last = found + ctl('wait', '40') + return last ctl('launch', '--resources', str(RESOURCES), '--world', 'reopen', '--seed', '7', '--creative', '--render-distance', '4', *extra) failures = 0 try: ctl('wait-ready', '300') + ctl('wait-for', 'loaded', '--timeout', '300') # as after each reopen below ctl('command', 'summon minecraft:minecart ~2 ~ ~2') ctl('wait', '20') expected = carts()