diff --git a/.github/workflows/parity-main.yml b/.github/workflows/parity-main.yml index 1ddfb24bb..1debcc694 100644 --- a/.github/workflows/parity-main.yml +++ b/.github/workflows/parity-main.yml @@ -23,7 +23,9 @@ env: jobs: parity-main: - runs-on: [self-hosted, horizon-game] + # The Mac mini's runners only: a desktop job slowed down because the maintainer came back could time out, and a + # failure here bisects and comments on merges. + runs-on: [self-hosted, horizon-game, macOS] timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures steps: - uses: actions/checkout@v4 diff --git a/docs/release-flow.md b/docs/release-flow.md index 28d1feef9..dc96498a0 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -90,8 +90,9 @@ processes, fleet-load's own detection, or a process outside the CI slice using t under way is throttled to 2 threads at the lowest weight and the runner stops after it: freezing the job would make GitHub drop it as lost and the harnesses time out. Old pull requests' `runner-env.sh` know only the Mac: the runner's `.env` and `.path` give their jobs Python, the JDK and the vanilla data (`HORIZON_SHARED_SOURCE`) all the -same. The nightly audit stays on the Mac mini's runners, so its comparison with the previous night is always made -on one platform. +same. The nightly audit and parity-main stay on the Mac mini's runners: the audit compares with the previous night, +so it stays on one platform, and a parity-main job slowed down on the desktop because he came back could time out, +and its failures bisect and comment on merges. ## Merging diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index 1bd587e79..c7235f5c1 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -24,8 +24,9 @@ version=2.338.0 sha=af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32 here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# Nix packages the jobs need, kept alive by GC roots in $dir/nix. -nix_out() { nix-build '' -A "$1" -o "$dir/nix/$2" > /dev/null && readlink -f "$dir/nix/$2"; } +# Nix packages the jobs need, kept alive by GC roots in $dir/nix. nix-build prints the store path; the root's name +# gets the output's suffix (alsa-dev-dev for alsa-lib.dev), so it is not read back from the link. +nix_out() { nix-build '' -A "$1" -o "$dir/nix/$2" | tail -1; } # The runner is a .NET program: on NixOS its native libraries (libstdc++, ICU, OpenSSL) come from nix-ld. export LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} diff --git a/tools/ci/runner_pause.py b/tools/ci/runner_pause.py index 35476a0b2..8fa0a4c5b 100755 --- a/tools/ci/runner_pause.py +++ b/tools/ci/runner_pause.py @@ -5,7 +5,7 @@ The runner works while nobody uses the machine: its screen locked, or untouched It pauses when any of these holds: - keyboard, mouse or gamepad input in the last INPUT_PAUSE seconds (60), or an unlocked session on seat0 with input in the last IDLE_AFTER seconds; - - the CPU package at TEMP_LIMIT °C (90) or more; + - the CPU package at TEMP_LIMIT °C (90) or more (the median of 5 readings); - fleet-load says a game runs (~/fleet/load: Steam, gamescope, Wine or Proton processes); - a process of his matches GAME below (the same, plus Minecraft and a Horizon game he started himself); - a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY %, default 50, in two @@ -63,7 +63,7 @@ class Input: def __init__(self): self.devices = {} # path -> file descriptor - self.last = time.monotonic() # a start counts as input: nothing is known yet + self.last = 0.0 # the boot (the monotonic clock's zero on Linux) counts as input def poll(self): try: @@ -115,14 +115,20 @@ def at_the_screen(input_idle): def too_hot(): - """The CPU package above TEMP_LIMIT °C (coretemp or k10temp).""" + """The CPU package at TEMP_LIMIT °C or more (coretemp or k10temp), as the median of 5 readings over 2 s: under + load it jumps for a second at a time (to 96 °C on ludodesktop at a median of 68 °C), which is no reason to stop.""" for monitor in Path("/sys/class/hwmon").glob("hwmon*"): try: - if (monitor / "name").read_text().strip() in ("coretemp", "k10temp"): - celsius = int((monitor / "temp1_input").read_text()) / 1000 - return f"CPU at {celsius:.0f} °C" if celsius >= TEMP_LIMIT else None + if (monitor / "name").read_text().strip() not in ("coretemp", "k10temp"): + continue + readings = [] + for _ in range(5): + readings.append(int((monitor / "temp1_input").read_text()) / 1000) + time.sleep(0.5) except (OSError, ValueError): - pass + continue + celsius = sorted(readings)[2] + return f"CPU at {celsius:.0f} °C" if celsius >= TEMP_LIMIT else None return None @@ -219,9 +225,9 @@ def pause_reason(hot, keys): def main(): hot, keys = {}, Input() - # A boot (or a restart of this watcher) counts as a sign: the runner starts RESUME_AFTER seconds later at the - # earliest, which leaves time to log in after switching the machine on. - last_sign = time.monotonic() + # The boot counts as a sign: the runner starts RESUME_AFTER seconds after it at the earliest, which leaves time + # to log in after switching the machine on. Restarting this watcher later changes nothing for a running job. + last_sign = 0.0 paused = None while True: reason = pause_reason(hot, keys)