diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml
new file mode 100644
index 000000000..af951fe12
--- /dev/null
+++ b/.github/actionlint.yaml
@@ -0,0 +1,3 @@
+self-hosted-runner:
+ # The Mac mini's runner (ludovics-mac-mini), which holds the maintainer's own Minecraft files.
+ labels: [horizon-game, ludovics-mac-mini]
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7b1a63710..dd9cde557 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,14 +1,25 @@
name: checks
+# verify: every pull request and every push to main, on GitHub's runners (the repository is public).
+# parity: the vanilla harnesses a pull request's changes can affect, on the Mac mini's runner, which holds the
+# maintainer's own Minecraft files (Horizon never downloads the game). A fork's pull request runs it only when
+# a maintainer adds the run-parity label, once per label. See docs/release-flow.md.
on:
push:
branches: [main]
pull_request:
+ types: [opened, synchronize, reopened, labeled]
workflow_dispatch:
permissions:
contents: read
+concurrency:
+ # A new push to a pull request replaces its running checks; main's runs all finish (releases follow them).
+ group: checks-${{ github.event.pull_request.number || github.sha }}-${{ github.event.action == 'labeled' && 'label' || 'run' }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
verify:
+ if: github.event.action != 'labeled'
runs-on: ubuntu-latest
+ timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
@@ -26,12 +37,49 @@ jobs:
run: |
sudo apt-get update && sudo apt-get install -y --no-install-recommends libasound2-dev libudev-dev
rustup toolchain install stable --profile minimal --component rustfmt --component clippy
+ # Dependencies and the workspace's own build, saved from main's runs (pull requests read main's cache).
+ - uses: Swatinem/rust-cache@v2
+ with:
+ shared-key: verify
+ save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Check formatting and lint
run: |
cargo +stable fmt --all -- --check
cargo +stable clippy --workspace --all-targets -- -D warnings
- name: Test Rust core and voxel kernel correctness
run: cargo +stable test --workspace
+ parity:
+ if: >-
+ github.event_name == 'pull_request' && (
+ (github.event.pull_request.head.repo.full_name == github.repository && github.event.action != 'labeled') ||
+ (github.event.action == 'labeled' && github.event.label.name == 'run-parity'))
+ runs-on: [self-hosted, horizon-game]
+ timeout-minutes: 45
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ # The commit the event names: a fork's later pushes need another run-parity label.
+ ref: ${{ github.event.pull_request.head.sha }}
+ fetch-depth: 0
+ persist-credentials: false
+ # Keeps .local/ (parity state, probe builds, oracle results) and target/ between runs.
+ clean: false
+ - uses: dtolnay/rust-toolchain@stable
+ - name: Tools from the runner
+ run: bash tools/ci/runner-env.sh
+ - name: Vanilla data, and resources prepared by this preparer
+ run: bash tools/ci/vanilla-data.sh
+ - name: The harnesses this change can affect
+ env:
+ PARITY_HOSTS: ''
+ run: python3 tools/parity.py --changed ${{ github.event.pull_request.base.sha }} --timeout 1500
+ - name: Logs of the failed harnesses
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: parity-logs
+ path: .local/parity-runner/logs/
+ if-no-files-found: ignore
# Horizon never downloads the game. These jobs need a self-hosted Linux runner
# (label horizon-game) holding the maintainer's own copy: the official launcher's
# game directory in HORIZON_MINECRAFT_DIR and the pinned server jar in
diff --git a/docs/release-flow.md b/docs/release-flow.md
new file mode 100644
index 000000000..abccac599
--- /dev/null
+++ b/docs/release-flow.md
@@ -0,0 +1,61 @@
+# Release flow
+
+How a change gets from a pull request to players: the checks a pull request passes, how it merges, and the
+automatic releases that follow. Releases are automatic and frequent so the latest Horizon is always playable;
+there is no separate development channel.
+
+## Pull request checks
+
+The `checks` workflow (`.github/workflows/ci.yml`) runs two jobs on every pull request:
+
+| Job | Runs on | What | Time |
+| --- | --- | --- | --- |
+| `verify` | GitHub `ubuntu-latest` | Public-tree check, Python tests, rustfmt, clippy with warnings denied, Rust tests. Dependencies and the workspace build come from a cache saved by main's runs | ~4 min cold, less warm |
+| `parity` | The Mac mini's runner (`ludovics-mac-mini`, label `horizon-game`) | The vanilla harnesses the change can affect (`tools/parity.py --changed `) | seconds when nothing is affected; minutes per affected harness |
+
+Pushes to main run `verify` only; the nightly audit covers main's harnesses.
+
+### The parity job
+
+Horizon never downloads the game, so harnesses that compare with vanilla run only where the maintainer's own
+copy is: the Mac mini. Its runner runs as the maintainer's account and keeps its checkout between runs
+(`clean: false`), so the probes build incrementally into `.local/parity-target` and earlier passes stand.
+
+1. `tools/ci/runner-env.sh` puts the machine's Python 3.13 and the JDK `research-lock.json` pins on the PATH.
+2. `tools/ci/vanilla-data.sh` links `.local/` to the runner's copy of the vanilla data, cloned once from
+ `~/code/horizon-parity/shared-local` into `~/.actions-runners/horizon/_work/horizon/shared-local`, and
+ prepares `.local/horizon-client` with the pull request's own `horizon-prepare` once per preparer version
+ (its sources, the exporters in `mods/horizon/src/test/java` and `research-lock.json`).
+3. `tools/parity.py --changed ` picks the harnesses whose traced dependencies (their script, Python
+ modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any
+ harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to
+ `Cargo.lock` or `research-lock.json` affects every harness. Each harness has 25 minutes, the job 45.
+
+Failed harnesses' logs are uploaded as the `parity-logs` artifact.
+
+**Platform-specific harnesses.** A harness listed under `linux_only` in `tools/parity_hosts.json` gives
+different results on macOS for a reason that is not a Horizon bug (the reason is printed). `parity.py` skips it on
+other platforms unless it is named on the command line.
+
+**Pull requests from forks** never run code on the Mac mini by themselves. A maintainer who has read the change
+adds the `run-parity` label: that runs the job once, on the commit the pull request had then. To run it again
+after new pushes, remove the label and add it again.
+
+**One runner.** The Mac mini has one runner for this repository: parity jobs, the macOS release builds and the
+nightly audit queue behind each other.
+
+## Merging
+
+Pull requests merge with a merge commit. The orchestrator sessions review a pull request, then turn on
+auto-merge (`gh pr merge --auto --merge`): GitHub merges it once the required checks pass. Branch
+protection on main requires `verify`. Branches don't have to be up to date with main: two sessions merge in
+parallel, and main's own `checks` run catches the rare semantic conflict.
+
+`parity` becomes required once it has run green on pull requests for a while (a week without a false failure):
+
+```sh
+gh api -X POST repos/ludovic111/horizon/branches/main/protection/required_status_checks/contexts -f 'contexts[]=parity'
+```
+
+A skipped `parity` (a fork without the label, a run started by a label) counts as passed, so requiring it never
+blocks a fork's pull request; the maintainer decides whether to run it.
diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py
index ad69e8c67..78716787a 100644
--- a/tests/test_parity_runner.py
+++ b/tests/test_parity_runner.py
@@ -44,6 +44,25 @@ class ParityRunnerTests(unittest.TestCase):
source.write_text("two!")
self.assertTrue(parity.stale("x", state, hasher, "env"))
+ def test_a_change_affects_the_harnesses_that_depend_on_it(self):
+ with tempfile.TemporaryDirectory() as temp, mock.patch.object(parity, "ROOT", Path(temp)), \
+ mock.patch.object(parity, "SHARED", Path(temp, "shared")):
+ found = {name: Path(temp, "tools", f"test_{name}_parity.py") for name in ("food", "light", "new")}
+ records = {"food": {"dependencies": ["crates/a.rs", "tools/lab.py"]},
+ "light": {"dependencies": ["crates/lib.rs#keep:light"]}}
+ self.assertEqual(parity.affected_harnesses(found, records, {"crates/a.rs"}), {"food", "new"})
+ self.assertEqual(parity.affected_harnesses(found, records, {"crates/lib.rs"}), {"light", "new"})
+ self.assertEqual(parity.affected_harnesses(found, records, {"tools/test_light_parity.py"}), {"light", "new"})
+ self.assertEqual(parity.affected_harnesses(found, records, {"Cargo.lock"}), {"food", "light", "new"})
+ self.assertEqual(parity.affected_harnesses(found, records, {"README.md"}), {"new"})
+
+ def test_platform_specific_harnesses_are_skipped_elsewhere(self):
+ config = {"linux_only": {"options": "libm"}}
+ with mock.patch.object(parity.sys, "platform", "linux"):
+ self.assertEqual(parity.platform_skips(config), {})
+ with mock.patch.object(parity.sys, "platform", "darwin"):
+ self.assertEqual(parity.platform_skips(config), {"options": "libm"})
+
def test_a_pass_from_another_worktree_counts_only_for_the_same_content(self):
with tempfile.TemporaryDirectory() as temp:
elsewhere, here, shared = Path(temp, "a"), Path(temp, "b"), Path(temp, "shared")
diff --git a/tools/ci/runner-env.sh b/tools/ci/runner-env.sh
new file mode 100755
index 000000000..b4c5094bd
--- /dev/null
+++ b/tools/ci/runner-env.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# The tools the vanilla harnesses need, on the self-hosted Mac mini runner (label horizon-game): its service
+# starts with a bare PATH. Python 3.13 and the JDK pinned by research-lock.json come from the machine, never
+# from a download. Writes them to $GITHUB_PATH and $GITHUB_ENV for the next steps.
+set -euo pipefail
+java_major=$(sed -n 's/.*"java_major": *\([0-9]*\).*/\1/p' research-lock.json | head -1)
+bin="$RUNNER_TEMP/horizon-bin"
+mkdir -p "$bin"
+if [ "$(uname)" = Darwin ]; then
+ ln -sfn /opt/homebrew/bin/python3.13 "$bin/python3"
+ ln -sfn /opt/homebrew/bin/python3.13 "$bin/python"
+ java_home=$(/usr/libexec/java_home -v "$java_major")
+ { echo "$bin"; echo "$java_home/bin"; echo /opt/homebrew/bin; } >> "$GITHUB_PATH"
+ echo "JAVA_HOME=$java_home" >> "$GITHUB_ENV"
+fi
+echo "HORIZON_NO_AUDIO=1" >> "$GITHUB_ENV"
diff --git a/tools/ci/vanilla-data.sh b/tools/ci/vanilla-data.sh
new file mode 100755
index 000000000..c5e76b388
--- /dev/null
+++ b/tools/ci/vanilla-data.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# The vanilla data for this checkout on the self-hosted runner. Horizon never downloads the game: the runner
+# holds the maintainer's own copy, imported and decompiled once (HORIZON_SHARED_SOURCE, default
+# ~/code/horizon-parity/shared-local). It is cloned once into the runner's work folder ($RUNNER_WORKSPACE/
+# shared-local; APFS clones, so no second copy of the bytes), and .local/ links to it as in a parity worktree.
+# .local/horizon-client, the resources horizon-prepare makes, is prepared by this checkout's own preparer
+# once per preparer version (its sources, the exporters and research-lock.json), so a change to the preparer
+# is checked with what it prepares; the four newest preparations are kept.
+set -euo pipefail
+shared="${RUNNER_WORKSPACE:?}/shared-local"
+source="${HORIZON_SHARED_SOURCE:-$HOME/code/horizon-parity/shared-local}"
+version=$(sed -n 's/.*"version": *"\([^"]*\)".*/\1/p' research-lock.json | head -1)
+if [ ! -f "$shared/minecraft/$version/prepared.json" ]; then
+ [ -f "$source/minecraft/$version/prepared.json" ] || { echo "::error::no imported Minecraft $version in $source"; exit 1; }
+ mkdir -p "$shared"
+ if [ "$(uname)" = Darwin ]; then
+ for entry in "$source"/*; do
+ [ "$(basename "$entry")" = horizon-client ] || cp -cR "$entry" "$shared/"
+ done
+ else
+ for entry in "$source"/*; do
+ [ "$(basename "$entry")" = horizon-client ] || cp -R --reflink=auto "$entry" "$shared/"
+ done
+ fi
+fi
+mkdir -p .local "$shared/prepared"
+for entry in "$shared"/*; do
+ name=$(basename "$entry")
+ case "$name" in prepared|horizon-client|*.log) continue ;; esac
+ if [ -L ".local/$name" ] || [ ! -e ".local/$name" ]; then ln -sfn "$entry" ".local/$name"; fi
+done
+
+key=$(git ls-files -s crates/horizon-prepare mods/horizon/src/test/java research-lock.json | shasum -a 256 | cut -c1-16)
+prepared="$shared/prepared/$key"
+ln -sfn "$prepared" .local/horizon-client
+if [ ! -f "$prepared/.done" ]; then
+ echo "Preparing resources with this checkout's horizon-prepare ($key)"
+ rm -rf "$prepared" && mkdir -p "$prepared"
+ started=$SECONDS
+ cargo build --release --locked -p horizon-prepare
+ if ! target/release/horizon-prepare --root "$PWD" > "$RUNNER_TEMP/prepare.log" 2>&1; then
+ tail -40 "$RUNNER_TEMP/prepare.log"
+ rm -rf "$prepared"
+ exit 1
+ fi
+ touch "$prepared/.done"
+ echo "Prepared in $((SECONDS - started)) s"
+fi
+touch "$prepared"
+# The four newest preparations stay.
+ls -1dt "$shared/prepared"/*/ 2>/dev/null | tail -n +5 | while read -r old; do rm -rf "$old"; done
diff --git a/tools/parity.py b/tools/parity.py
index c7063f987..85d079c14 100644
--- a/tools/parity.py
+++ b/tools/parity.py
@@ -6,6 +6,7 @@
python3 tools/parity.py --fresh ... # don't reuse vanilla oracle results
python3 tools/parity.py food light # these harnesses (names without test_/_parity.py)
python3 tools/parity.py --failed # the ones that failed last time
+ python3 tools/parity.py --changed origin/main # those a branch's changes can affect (pull requests)
python3 tools/parity.py --list # what would run, without running it
python3 tools/parity.py --status # last result of every harness
@@ -1059,18 +1060,29 @@ def trace_main(output, script, arguments):
runpy.run_path(script, run_name="__main__")
-def changed_harnesses(found, records, ref):
- """Harnesses new or changed since the merge base of HEAD and ref: their script, or a dependency of theirs that is
- an oracle (Java under mods/horizon/src/test) or a probe (a crate's example or binary), changed or was added."""
+# Files every harness's result depends on (part of environment_key): a change to one affects them all.
+SHARED_INPUTS = {"research-lock.json", "Cargo.lock"}
+
+
+def changed_files(ref):
+ """Files changed between the merge base of HEAD and ref and HEAD (committed or not), or None without a merge base."""
def git(*arguments):
result = subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, text=True)
return result.stdout.split("\n") if result.returncode == 0 else None
base = git("merge-base", "HEAD", ref)
if not base or not base[0]:
+ return None
+ return set(filter(None, git("diff", "--name-only", base[0]) or [])) | \
+ set(filter(None, git("ls-files", "--others", "--exclude-standard") or []))
+
+
+def changed_harnesses(found, records, ref):
+ """Harnesses new or changed since the merge base of HEAD and ref: their script, or a dependency of theirs that is
+ an oracle (Java under mods/horizon/src/test) or a probe (a crate's example or binary), changed or was added."""
+ changed = changed_files(ref)
+ if changed is None:
print(f"note: no merge base with {ref}; the flake check repeats nothing", flush=True)
return set()
- changed = set(filter(None, git("diff", "--name-only", base[0]) or [])) | \
- set(filter(None, git("ls-files", "--others", "--exclude-standard") or []))
found_changed = set()
for name, script in found.items():
specific = [path for path in records.get(name, {}).get("dependencies", [])
@@ -1080,6 +1092,30 @@ def changed_harnesses(found, records, ref):
return found_changed
+def affected_harnesses(found, records, changed):
+ """Harnesses the changed files can affect: their script or a recorded dependency (here, or in the shared
+ dependency lists) changed, a file every harness depends on changed, or nothing is recorded about them yet."""
+ if changed & SHARED_INPUTS:
+ return set(found)
+ affected = set()
+ for name, script in found.items():
+ dependencies = records.get(name, {}).get("dependencies")
+ template = SHARED / "dependencies" / f"{name}.json"
+ if dependencies is None and template.exists():
+ dependencies = json.loads(template.read_text()).get("dependencies")
+ if dependencies is None or str(Path(script).relative_to(ROOT)) in changed or \
+ any(path.split(KEEP)[0] in changed for path in dependencies):
+ affected.add(name)
+ return affected
+
+
+def platform_skips(config):
+ """Harnesses tools/parity_hosts.json marks as giving different results on this platform: {name: reason}."""
+ if sys.platform == "linux":
+ return {}
+ return dict(config.get("linux_only", {}))
+
+
def finished_run(failed):
"""Record that this run completed, for tools/gate.py: a crashed runner never writes it."""
STATE_DIR.mkdir(parents=True, exist_ok=True)
@@ -1110,6 +1146,8 @@ def main():
parser.add_argument("--flake-check", metavar="REF", help="harnesses new or changed since the merge base with REF "
"must pass --flake-rounds times in a row (tools/gate.py)")
parser.add_argument("--flake-rounds", type=int, default=3, help="runs required by --flake-check (default 3)")
+ parser.add_argument("--changed", metavar="REF", help="only harnesses the changes since the merge base with REF can "
+ "affect (and that have not passed with their current inputs): the pull-request check")
args = parser.parse_args()
found = harnesses()
@@ -1137,8 +1175,20 @@ def main():
chosen = list(found)
elif args.failed:
chosen = [name for name in found if records.get(name, {}).get("status") == "fail"]
+ elif args.changed:
+ changed = changed_files(args.changed)
+ if changed is None:
+ parser.error(f"no merge base with {args.changed}")
+ affected = affected_harnesses(found, records, changed)
+ chosen = [name for name in found if name in affected and stale(name, state, hasher, environment)]
+ print(f"{len(changed)} files changed since the merge base with {args.changed}: {len(affected)} harnesses "
+ f"depend on them, {len(chosen)} without a pass for their current inputs.", flush=True)
else:
chosen = [name for name in found if stale(name, state, hasher, environment)]
+ skips = platform_skips(host_config())
+ for name in [name for name in chosen if base_name(name) in skips and not args.names]:
+ print(f"skipped on {sys.platform}: {name}: {skips[base_name(name)]}", flush=True)
+ chosen.remove(name)
rounds = {name: args.repeat for name in chosen} if args.repeat > 1 else {}
if args.flake_check:
changed = changed_harnesses(found, records, args.flake_check)