From 116a7f7dcafc4e36d14b75600f402446ec8056ac Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:48:43 +0100 Subject: [PATCH] horizon-prepare: OUTPUT_VERSION, the number of what it prepares, checked by CI Launchers copy a player's prepared resources to a new release while its preparer declares the same OUTPUT_VERSION (the launcher side is its own pull request). The number is bumped by hand; CI keeps it honest: the parity job already prepares with every change to the preparer, and tools/prepare_output.py check now fails when that preparation's digest is not the one crates/horizon-prepare/output-versions.txt gives the current number, or when the file changed other than by appending a version. The digest canonicalises what differs between two runs of the same preparer (JSON key order, creative tab tag order, lambda class addresses, logs, the sounds): 10 preparations by 5 preparer versions on the three Mac runners give one digest, and a Linux preparation by an older preparer gives the Mac runners' digest for it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parity.yml | 4 + crates/horizon-prepare/output-versions.txt | 3 + crates/horizon-prepare/src/lib.rs | 32 ++++ docs/release-flow.md | 4 + tests/test_prepare_output.py | 64 ++++++++ tools/prepare_output.py | 176 +++++++++++++++++++++ 6 files changed, 283 insertions(+) create mode 100644 crates/horizon-prepare/output-versions.txt create mode 100644 tests/test_prepare_output.py create mode 100755 tools/prepare_output.py diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index a18fc5019..0588c12a3 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -37,6 +37,10 @@ jobs: run: bash tools/ci/runner-env.sh - name: Vanilla data, and resources prepared by this preparer run: bash tools/ci/vanilla-data.sh + - name: The prepared output is the one the preparer's OUTPUT_VERSION says + env: + BASE_SHA: ${{ inputs.base }} + run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA" - name: The version constants are the pinned jar's run: python3 tools/gen_version_consts.py --check - name: The harnesses this change can affect diff --git a/crates/horizon-prepare/output-versions.txt b/crates/horizon-prepare/output-versions.txt new file mode 100644 index 000000000..71a531fab --- /dev/null +++ b/crates/horizon-prepare/output-versions.txt @@ -0,0 +1,3 @@ +# OUTPUT_VERSION (src/lib.rs) and the digest of what it prepares (python3 tools/prepare_output.py digest), +# one line per version, never edited: a change to the prepared files appends the next version. +1 0a608bf4115a8ed2e2c4f5f2b2609dc10c54e63479af8e56fabaa10af1060389 diff --git a/crates/horizon-prepare/src/lib.rs b/crates/horizon-prepare/src/lib.rs index c7e443c7d..5ea1121df 100644 --- a/crates/horizon-prepare/src/lib.rs +++ b/crates/horizon-prepare/src/lib.rs @@ -31,6 +31,15 @@ pub type Log<'a> = &'a dyn Fn(&str); /// The manifest's `schema`: the prepared files' layout, which the launcher compares. pub const SCHEMA: i64 = 7; +/// What this preparer makes. The launcher copies a prepared version's resources to a newly installed +/// version whose preparer has the same number (and the same pinned jars), so a release that changes +/// this crate without changing its output asks nobody to prepare again. Bump it, and append the new +/// digest to output-versions.txt, when the prepared files change: CI prepares with every change to +/// the preparer and fails when the output is not the one output-versions.txt gives this number +/// (`python3 tools/prepare_output.py check`). The launcher reads this line from the version +/// folder's sources: keep it on one line, as it is. +pub const OUTPUT_VERSION: u32 = 1; + /// Digest of this crate's sources (build.rs): the launcher prepares a version in-process when /// that version ships these same sources, and runs the version's own binary otherwise. pub const SOURCES_SHA256: &str = env!("HORIZON_PREPARE_SOURCES_SHA256"); @@ -1209,6 +1218,29 @@ mod tests { fs::remove_dir_all(dir).unwrap(); } + #[test] + fn output_versions_end_at_the_output_version() { + let text = include_str!("../output-versions.txt"); + let versions: Vec<(u32, &str)> = text + .lines() + .filter(|l| !l.trim().is_empty() && !l.starts_with('#')) + .map(|l| { + let (n, digest) = l.split_once(' ').unwrap(); + (n.parse().unwrap(), digest.trim()) + }) + .collect(); + let numbers: Vec = versions.iter().map(|v| v.0).collect(); + assert_eq!(numbers, (1..=OUTPUT_VERSION).collect::>()); + for (i, (_, digest)) in versions.iter().enumerate() { + assert_eq!(digest.len(), 64); + assert!(!versions[..i].iter().any(|(_, d)| d == digest)); + } + // The launcher finds the number with this line in a version folder's sources. + let lib = include_str!("lib.rs"); + let line = format!("pub const OUTPUT_VERSION: u32 = {OUTPUT_VERSION};"); + assert!(lib.lines().any(|l| l == line)); + } + #[test] fn the_source_digest_is_the_one_build_rs_computed() { assert_eq!( diff --git a/docs/release-flow.md b/docs/release-flow.md index 4fb0209d9..1413cde01 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -39,6 +39,10 @@ copy is: the Mac mini. Its runner runs as the maintainer's account and keeps its depend only on the Minecraft version, are kept once in `shared-local/sounds//` (copied from a folder `horizon-prepare sounds` made from the maintainer's own game; the Mac mini has no official launcher of its own) and cloned into each preparation (not linked: vanilla's packs refuse symbolic links), for the audio harnesses. + `tools/prepare_output.py check` then compares what was prepared with `crates/horizon-prepare/output-versions.txt`: + launchers keep a player's prepared resources across releases whose preparer declares the same `OUTPUT_VERSION` + (`crates/horizon-prepare/src/lib.rs`), so a change to the prepared files fails here until it bumps the number + and appends the new digest (the error gives both lines). 3. `tools/parity.py --changed ` picks the harnesses whose traced dependencies (their script, Python modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to diff --git a/tests/test_prepare_output.py b/tests/test_prepare_output.py new file mode 100644 index 000000000..00c489fbb --- /dev/null +++ b/tests/test_prepare_output.py @@ -0,0 +1,64 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from tools import prepare_output + + +class PrepareOutputTests(unittest.TestCase): + def preparation(self, root: Path, files: dict[str, bytes]) -> Path: + for rel, data in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return root + + def test_what_varies_between_runs_does_not_count(self): + manifest = {'schema': 7, 'data_version': 5023, 'render_files_sha256': {'a.json': 'x'}} + a = { + 'horizon-resources.json': json.dumps(manifest).encode(), + 'render-definitions.json': b'{"a": 1, "b": [1, 2]}', + 'creative-inventory.json': b'{"tags": [["b", "a"]]}', + 'block-behaviours.json': b'{"c": "Foo$$Lambda/0x0000012a"}', + 'export.log': b'run 1', + 'assets/minecraft/sounds/a.ogg': b'one', + } + manifest['render_files_sha256'] = {'a.json': 'y'} + b = { + 'horizon-resources.json': json.dumps(manifest).encode(), + 'render-definitions.json': b'{"b": [1, 2], "a": 1}', + 'creative-inventory.json': b'{"tags": [["a", "b"]]}', + 'block-behaviours.json': b'{"c": "Foo$$Lambda/0x00000ff0"}', + 'export.log': b'run 2', + 'assets/minecraft/sounds/a.ogg': b'two', + } + with tempfile.TemporaryDirectory() as tmp: + first = prepare_output.digest(self.preparation(Path(tmp) / 'a', a)) + second = prepare_output.digest(self.preparation(Path(tmp) / 'b', b)) + self.assertEqual(first, second) + # The data does, and so do the manifest's schema and the files there are. + (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [2, 1], "a": 1}') + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [1, 2], "a": 1}') + (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps({**manifest, 'schema': 8})) + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps(manifest)) + (Path(tmp) / 'b/new.nbt').write_bytes(b'') + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + + def test_the_versions_file_ends_at_the_constant(self): + versions = prepare_output.parse_versions('# comment\n1 aa\n2 bb\n') + self.assertEqual(versions, [(1, 'aa'), (2, 'bb')]) + self.assertEqual(prepare_output.problems(versions, 2), []) + self.assertTrue(prepare_output.problems(versions, 1)) + self.assertTrue(prepare_output.problems([(1, 'aa'), (3, 'bb')], 3)) + self.assertTrue(prepare_output.problems([(1, 'aa'), (2, 'aa')], 2)) + root = Path(prepare_output.__file__).resolve().parent.parent + current = prepare_output.parse_versions((root / prepare_output.VERSIONS).read_text()) + constant = prepare_output.CONSTANT.search((root / prepare_output.LIB).read_text()) + self.assertEqual(prepare_output.problems(current, int(constant.group(1))), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/prepare_output.py b/tools/prepare_output.py new file mode 100755 index 000000000..81ed156cb --- /dev/null +++ b/tools/prepare_output.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""The prepared output's version: what keeps `OUTPUT_VERSION` (crates/horizon-prepare/src/lib.rs) honest. + +The launcher copies a prepared version's resources (`.local/horizon-client/`) to a newly installed +one when both pin the same Minecraft jars and their preparers declare the same `OUTPUT_VERSION`, so a release +that changes the preparer's code without changing what it makes asks nobody to prepare again. The number is +bumped by hand; this tool catches a change to the output without a bump. + +crates/horizon-prepare/output-versions.txt lists every `OUTPUT_VERSION` with the digest of what it prepares. +It only grows: the last line is the current version, and a change to the output appends a line. + + python3 tools/prepare_output.py digest [RESOURCES] the digest of a preparation + python3 tools/prepare_output.py check [--base SHA] [--cache] [RESOURCES] + fails when RESOURCES (default: this checkout's .local/horizon-client/, as CI prepares it) is + not what the last line says, or when the file changed other than by appending since SHA. --cache keeps + the digest beside RESOURCES (CI: a preparation never changes once made; it takes ~30 s to compute) + +The digest covers every file of the preparation, canonicalised where the preparer is not deterministic between +runs (verified across the CI runners): JSON is compared as data (key order, the order of each `tags` list of +creative-inventory.json, and the addresses in `$$Lambda/0x...` class names don't count); logs, compiled classes, +the sounds (copied from the player's launcher by another step) and the manifest's file hashes (raw bytes) don't +count. The manifest's own scalar fields (schema, data version, jar hashes, file count) do. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +VERSIONS = 'crates/horizon-prepare/output-versions.txt' +LIB = 'crates/horizon-prepare/src/lib.rs' +MANIFEST = 'horizon-resources.json' +LAMBDA = re.compile(r'\$\$Lambda/0x[0-9a-fA-F]+') +CONSTANT = re.compile(r'^pub const OUTPUT_VERSION: u32 = (\d+);', re.M) + + +def canonical_json(value, key=''): + if isinstance(value, dict): + return {k: canonical_json(v, k) for k, v in value.items()} + if isinstance(value, list): + items = [canonical_json(v, key) for v in value] + if key == 'tags' and all(isinstance(x, list) and all(isinstance(t, str) for t in x) for x in items): + items = [sorted(x) for x in items] + return items + if isinstance(value, str): + return LAMBDA.sub('$$Lambda', value) + return value + + +def canonical(rel: str, data: bytes) -> bytes: + if rel == MANIFEST: + manifest = json.loads(data) + scalars = {k: v for k, v in manifest.items() if not isinstance(v, (dict, list))} + return json.dumps(scalars, sort_keys=True).encode() + if rel.endswith('.json'): + try: + value = json.loads(data) + except ValueError: + return data + return json.dumps(canonical_json(value), sort_keys=True).encode() + return data + + +def skipped(rel: str) -> bool: + parts = rel.split('/') + return rel.endswith(('.log', '.class')) or parts[:3] == ['assets', 'minecraft', 'sounds'] \ + or rel == 'assets/minecraft/sounds.json' + + +def digest(resources: Path) -> tuple[str, int]: + """The digest of a preparation, and how many files it covers.""" + if not (resources / MANIFEST).is_file(): + raise SystemExit(f'{resources} is not a preparation: no {MANIFEST}') + files = [] + for dirpath, dirs, names in os.walk(resources): + dirs.sort() + for name in names: + path = Path(dirpath) / name + rel = path.relative_to(resources).as_posix() + if path.is_symlink() or skipped(rel): + continue + files.append(rel) + total = hashlib.sha256() + for rel in sorted(files): + data = hashlib.sha256(canonical(rel, (resources / rel).read_bytes())).hexdigest() + total.update(f'{rel}\0{data}\n'.encode()) + return total.hexdigest(), len(files) + + +def parse_versions(text: str) -> list[tuple[int, str]]: + out = [] + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + number, value = line.split() + out.append((int(number), value)) + return out + + +def problems(versions: list[tuple[int, str]], constant: int | None) -> list[str]: + found = [] + if not versions: + return [f'{VERSIONS} lists no version'] + if [n for n, _ in versions] != list(range(1, len(versions) + 1)): + found.append(f'{VERSIONS}: the versions must be 1, 2, 3... in order') + if len({d for _, d in versions}) != len(versions): + found.append(f'{VERSIONS}: two versions with the same digest') + if constant != versions[-1][0]: + found.append(f'{LIB} says OUTPUT_VERSION = {constant}, {VERSIONS} ends at {versions[-1][0]}') + return found + + +def default_resources() -> Path: + lock = json.loads((ROOT / 'research-lock.json').read_text()) + return ROOT / '.local/horizon-client' / lock['minecraft']['version'] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + parser.add_argument('command', choices=['digest', 'check']) + parser.add_argument('resources', nargs='?', type=Path) + parser.add_argument('--base', help='the commit the change is based on (check: the file only grew since)') + parser.add_argument('--cache', action='store_true', help='keep the digest beside the preparation') + args = parser.parse_args() + resources = (args.resources or default_resources()).resolve() + tool = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()[:16] + cache = resources.parent / f'.output-digest-{resources.name}-{tool}' + if args.cache and cache.is_file(): + value, count = cache.read_text().split() + else: + value, count = digest(resources) + if args.cache: + cache.write_text(f'{value} {count}\n') + if args.command == 'digest': + print(value) + return 0 + + versions = parse_versions((ROOT / VERSIONS).read_text()) + match = CONSTANT.search((ROOT / LIB).read_text()) + found = problems(versions, int(match.group(1)) if match else None) + if args.base: + shown = subprocess.run(['git', 'show', f'{args.base}:{VERSIONS}'], cwd=ROOT, capture_output=True, text=True) + if shown.returncode == 0: + before = parse_versions(shown.stdout) + if versions[:len(before)] != before: + found.append(f'{VERSIONS} changed lines it already had (since {args.base[:12]}): ' + 'a new output gets a new version, appended') + current = versions[-1] if versions else (0, '') + if value != current[1]: + known = [n for n, d in versions if d == value] + if known: + found.append(f'the preparation is OUTPUT_VERSION {known[0]}\'s output, but OUTPUT_VERSION is {current[0]}') + else: + found.append( + f'the prepared output changed ({count} files, digest {value}) but OUTPUT_VERSION is still ' + f'{current[0]}. Players would keep resources prepared by the old preparer. Bump it:\n' + f' set `pub const OUTPUT_VERSION: u32 = {current[0] + 1};` in {LIB}\n' + f' append `{current[0] + 1} {value}` to {VERSIONS}\n' + ' (if the change is only a nondeterminism this tool should ignore, teach canonical() instead)') + for problem in found: + print(f'::error::{problem}' if os.environ.get('GITHUB_ACTIONS') else f'error: {problem}', file=sys.stderr) + if not found: + print(f'prepared output: OUTPUT_VERSION {current[0]} ({count} files, {value[:16]})') + return 1 if found else 0 + + +if __name__ == '__main__': + sys.exit(main())