horizon-prepare: OUTPUT_VERSION, the number of what it prepares, checked by CI

Launchers copy a player's prepared resources to a new release while its preparer declares the
same OUTPUT_VERSION (the launcher side is its own pull request). The number is bumped by hand;
CI keeps it honest: the parity job already prepares with every change to the preparer, and
tools/prepare_output.py check now fails when that preparation's digest is not the one
crates/horizon-prepare/output-versions.txt gives the current number, or when the file changed
other than by appending a version. The digest canonicalises what differs between two runs of
the same preparer (JSON key order, creative tab tag order, lambda class addresses, logs, the
sounds): 10 preparations by 5 preparer versions on the three Mac runners give one digest, and a Linux preparation by an older preparer gives the Mac runners' digest for it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/parity.yml

@@ -37,6 +37,10 @@ jobs:
         run: bash tools/ci/runner-env.sh
       - name: Vanilla data, and resources prepared by this preparer
         run: bash tools/ci/vanilla-data.sh
+      - name: The prepared output is the one the preparer's OUTPUT_VERSION says
+        env:
+          BASE_SHA: ${{ inputs.base }}
+        run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA"
       - name: The version constants are the pinned jar's
         run: python3 tools/gen_version_consts.py --check
       - name: The harnesses this change can affect

crates/horizon-prepare/output-versions.txt

@@ -0,0 +1,3 @@
+# OUTPUT_VERSION (src/lib.rs) and the digest of what it prepares (python3 tools/prepare_output.py digest),
+# one line per version, never edited: a change to the prepared files appends the next version.
+1 0a608bf4115a8ed2e2c4f5f2b2609dc10c54e63479af8e56fabaa10af1060389

crates/horizon-prepare/src/lib.rs

@@ -31,6 +31,15 @@ pub type Log<'a> = &'a dyn Fn(&str);
 /// The manifest's `schema`: the prepared files' layout, which the launcher compares.
 pub const SCHEMA: i64 = 7;
 
+/// What this preparer makes. The launcher copies a prepared version's resources to a newly installed
+/// version whose preparer has the same number (and the same pinned jars), so a release that changes
+/// this crate without changing its output asks nobody to prepare again. Bump it, and append the new
+/// digest to output-versions.txt, when the prepared files change: CI prepares with every change to
+/// the preparer and fails when the output is not the one output-versions.txt gives this number
+/// (`python3 tools/prepare_output.py check`). The launcher reads this line from the version
+/// folder's sources: keep it on one line, as it is.
+pub const OUTPUT_VERSION: u32 = 1;
+
 /// Digest of this crate's sources (build.rs): the launcher prepares a version in-process when
 /// that version ships these same sources, and runs the version's own binary otherwise.
 pub const SOURCES_SHA256: &str = env!("HORIZON_PREPARE_SOURCES_SHA256");
@@ -1209,6 +1218,29 @@ mod tests {
         fs::remove_dir_all(dir).unwrap();
     }
 
+    #[test]
+    fn output_versions_end_at_the_output_version() {
+        let text = include_str!("../output-versions.txt");
+        let versions: Vec<(u32, &str)> = text
+            .lines()
+            .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
+            .map(|l| {
+                let (n, digest) = l.split_once(' ').unwrap();
+                (n.parse().unwrap(), digest.trim())
+            })
+            .collect();
+        let numbers: Vec<u32> = versions.iter().map(|v| v.0).collect();
+        assert_eq!(numbers, (1..=OUTPUT_VERSION).collect::<Vec<_>>());
+        for (i, (_, digest)) in versions.iter().enumerate() {
+            assert_eq!(digest.len(), 64);
+            assert!(!versions[..i].iter().any(|(_, d)| d == digest));
+        }
+        // The launcher finds the number with this line in a version folder's sources.
+        let lib = include_str!("lib.rs");
+        let line = format!("pub const OUTPUT_VERSION: u32 = {OUTPUT_VERSION};");
+        assert!(lib.lines().any(|l| l == line));
+    }
+
     #[test]
     fn the_source_digest_is_the_one_build_rs_computed() {
         assert_eq!(

docs/release-flow.md

@@ -39,6 +39,10 @@ copy is: the Mac mini. Its runner runs as the maintainer's account and keeps its
    depend only on the Minecraft version, are kept once in `shared-local/sounds/<version>/` (copied from a folder
    `horizon-prepare sounds` made from the maintainer's own game; the Mac mini has no official launcher of its own)
    and cloned into each preparation (not linked: vanilla's packs refuse symbolic links), for the audio harnesses.
+   `tools/prepare_output.py check` then compares what was prepared with `crates/horizon-prepare/output-versions.txt`:
+   launchers keep a player's prepared resources across releases whose preparer declares the same `OUTPUT_VERSION`
+   (`crates/horizon-prepare/src/lib.rs`), so a change to the prepared files fails here until it bumps the number
+   and appends the new digest (the error gives both lines).
 3. `tools/parity.py --changed <base sha>` picks the harnesses whose traced dependencies (their script, Python
    modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any
    harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to

tests/test_prepare_output.py

@@ -0,0 +1,64 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from tools import prepare_output
+
+
+class PrepareOutputTests(unittest.TestCase):
+    def preparation(self, root: Path, files: dict[str, bytes]) -> Path:
+        for rel, data in files.items():
+            path = root / rel
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(data)
+        return root
+
+    def test_what_varies_between_runs_does_not_count(self):
+        manifest = {'schema': 7, 'data_version': 5023, 'render_files_sha256': {'a.json': 'x'}}
+        a = {
+            'horizon-resources.json': json.dumps(manifest).encode(),
+            'render-definitions.json': b'{"a": 1, "b": [1, 2]}',
+            'creative-inventory.json': b'{"tags": [["b", "a"]]}',
+            'block-behaviours.json': b'{"c": "Foo$$Lambda/0x0000012a"}',
+            'export.log': b'run 1',
+            'assets/minecraft/sounds/a.ogg': b'one',
+        }
+        manifest['render_files_sha256'] = {'a.json': 'y'}
+        b = {
+            'horizon-resources.json': json.dumps(manifest).encode(),
+            'render-definitions.json': b'{"b": [1, 2], "a": 1}',
+            'creative-inventory.json': b'{"tags": [["a", "b"]]}',
+            'block-behaviours.json': b'{"c": "Foo$$Lambda/0x00000ff0"}',
+            'export.log': b'run 2',
+            'assets/minecraft/sounds/a.ogg': b'two',
+        }
+        with tempfile.TemporaryDirectory() as tmp:
+            first = prepare_output.digest(self.preparation(Path(tmp) / 'a', a))
+            second = prepare_output.digest(self.preparation(Path(tmp) / 'b', b))
+            self.assertEqual(first, second)
+            # The data does, and so do the manifest's schema and the files there are.
+            (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [2, 1], "a": 1}')
+            self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b'))
+            (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [1, 2], "a": 1}')
+            (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps({**manifest, 'schema': 8}))
+            self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b'))
+            (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps(manifest))
+            (Path(tmp) / 'b/new.nbt').write_bytes(b'')
+            self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b'))
+
+    def test_the_versions_file_ends_at_the_constant(self):
+        versions = prepare_output.parse_versions('# comment\n1 aa\n2 bb\n')
+        self.assertEqual(versions, [(1, 'aa'), (2, 'bb')])
+        self.assertEqual(prepare_output.problems(versions, 2), [])
+        self.assertTrue(prepare_output.problems(versions, 1))
+        self.assertTrue(prepare_output.problems([(1, 'aa'), (3, 'bb')], 3))
+        self.assertTrue(prepare_output.problems([(1, 'aa'), (2, 'aa')], 2))
+        root = Path(prepare_output.__file__).resolve().parent.parent
+        current = prepare_output.parse_versions((root / prepare_output.VERSIONS).read_text())
+        constant = prepare_output.CONSTANT.search((root / prepare_output.LIB).read_text())
+        self.assertEqual(prepare_output.problems(current, int(constant.group(1))), [])
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/prepare_output.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""The prepared output's version: what keeps `OUTPUT_VERSION` (crates/horizon-prepare/src/lib.rs) honest.
+
+The launcher copies a prepared version's resources (`.local/horizon-client/<version>`) to a newly installed
+one when both pin the same Minecraft jars and their preparers declare the same `OUTPUT_VERSION`, so a release
+that changes the preparer's code without changing what it makes asks nobody to prepare again. The number is
+bumped by hand; this tool catches a change to the output without a bump.
+
+crates/horizon-prepare/output-versions.txt lists every `OUTPUT_VERSION` with the digest of what it prepares.
+It only grows: the last line is the current version, and a change to the output appends a line.
+
+    python3 tools/prepare_output.py digest [RESOURCES]          the digest of a preparation
+    python3 tools/prepare_output.py check [--base SHA] [--cache] [RESOURCES]
+        fails when RESOURCES (default: this checkout's .local/horizon-client/<version>, as CI prepares it) is
+        not what the last line says, or when the file changed other than by appending since SHA. --cache keeps
+        the digest beside RESOURCES (CI: a preparation never changes once made; it takes ~30 s to compute)
+
+The digest covers every file of the preparation, canonicalised where the preparer is not deterministic between
+runs (verified across the CI runners): JSON is compared as data (key order, the order of each `tags` list of
+creative-inventory.json, and the addresses in `$$Lambda/0x...` class names don't count); logs, compiled classes,
+the sounds (copied from the player's launcher by another step) and the manifest's file hashes (raw bytes) don't
+count. The manifest's own scalar fields (schema, data version, jar hashes, file count) do.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+VERSIONS = 'crates/horizon-prepare/output-versions.txt'
+LIB = 'crates/horizon-prepare/src/lib.rs'
+MANIFEST = 'horizon-resources.json'
+LAMBDA = re.compile(r'\$\$Lambda/0x[0-9a-fA-F]+')
+CONSTANT = re.compile(r'^pub const OUTPUT_VERSION: u32 = (\d+);', re.M)
+
+
+def canonical_json(value, key=''):
+    if isinstance(value, dict):
+        return {k: canonical_json(v, k) for k, v in value.items()}
+    if isinstance(value, list):
+        items = [canonical_json(v, key) for v in value]
+        if key == 'tags' and all(isinstance(x, list) and all(isinstance(t, str) for t in x) for x in items):
+            items = [sorted(x) for x in items]
+        return items
+    if isinstance(value, str):
+        return LAMBDA.sub('$$Lambda', value)
+    return value
+
+
+def canonical(rel: str, data: bytes) -> bytes:
+    if rel == MANIFEST:
+        manifest = json.loads(data)
+        scalars = {k: v for k, v in manifest.items() if not isinstance(v, (dict, list))}
+        return json.dumps(scalars, sort_keys=True).encode()
+    if rel.endswith('.json'):
+        try:
+            value = json.loads(data)
+        except ValueError:
+            return data
+        return json.dumps(canonical_json(value), sort_keys=True).encode()
+    return data
+
+
+def skipped(rel: str) -> bool:
+    parts = rel.split('/')
+    return rel.endswith(('.log', '.class')) or parts[:3] == ['assets', 'minecraft', 'sounds'] \
+        or rel == 'assets/minecraft/sounds.json'
+
+
+def digest(resources: Path) -> tuple[str, int]:
+    """The digest of a preparation, and how many files it covers."""
+    if not (resources / MANIFEST).is_file():
+        raise SystemExit(f'{resources} is not a preparation: no {MANIFEST}')
+    files = []
+    for dirpath, dirs, names in os.walk(resources):
+        dirs.sort()
+        for name in names:
+            path = Path(dirpath) / name
+            rel = path.relative_to(resources).as_posix()
+            if path.is_symlink() or skipped(rel):
+                continue
+            files.append(rel)
+    total = hashlib.sha256()
+    for rel in sorted(files):
+        data = hashlib.sha256(canonical(rel, (resources / rel).read_bytes())).hexdigest()
+        total.update(f'{rel}\0{data}\n'.encode())
+    return total.hexdigest(), len(files)
+
+
+def parse_versions(text: str) -> list[tuple[int, str]]:
+    out = []
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        number, value = line.split()
+        out.append((int(number), value))
+    return out
+
+
+def problems(versions: list[tuple[int, str]], constant: int | None) -> list[str]:
+    found = []
+    if not versions:
+        return [f'{VERSIONS} lists no version']
+    if [n for n, _ in versions] != list(range(1, len(versions) + 1)):
+        found.append(f'{VERSIONS}: the versions must be 1, 2, 3... in order')
+    if len({d for _, d in versions}) != len(versions):
+        found.append(f'{VERSIONS}: two versions with the same digest')
+    if constant != versions[-1][0]:
+        found.append(f'{LIB} says OUTPUT_VERSION = {constant}, {VERSIONS} ends at {versions[-1][0]}')
+    return found
+
+
+def default_resources() -> Path:
+    lock = json.loads((ROOT / 'research-lock.json').read_text())
+    return ROOT / '.local/horizon-client' / lock['minecraft']['version']
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
+    parser.add_argument('command', choices=['digest', 'check'])
+    parser.add_argument('resources', nargs='?', type=Path)
+    parser.add_argument('--base', help='the commit the change is based on (check: the file only grew since)')
+    parser.add_argument('--cache', action='store_true', help='keep the digest beside the preparation')
+    args = parser.parse_args()
+    resources = (args.resources or default_resources()).resolve()
+    tool = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()[:16]
+    cache = resources.parent / f'.output-digest-{resources.name}-{tool}'
+    if args.cache and cache.is_file():
+        value, count = cache.read_text().split()
+    else:
+        value, count = digest(resources)
+        if args.cache:
+            cache.write_text(f'{value} {count}\n')
+    if args.command == 'digest':
+        print(value)
+        return 0
+
+    versions = parse_versions((ROOT / VERSIONS).read_text())
+    match = CONSTANT.search((ROOT / LIB).read_text())
+    found = problems(versions, int(match.group(1)) if match else None)
+    if args.base:
+        shown = subprocess.run(['git', 'show', f'{args.base}:{VERSIONS}'], cwd=ROOT, capture_output=True, text=True)
+        if shown.returncode == 0:
+            before = parse_versions(shown.stdout)
+            if versions[:len(before)] != before:
+                found.append(f'{VERSIONS} changed lines it already had (since {args.base[:12]}): '
+                             'a new output gets a new version, appended')
+    current = versions[-1] if versions else (0, '')
+    if value != current[1]:
+        known = [n for n, d in versions if d == value]
+        if known:
+            found.append(f'the preparation is OUTPUT_VERSION {known[0]}\'s output, but OUTPUT_VERSION is {current[0]}')
+        else:
+            found.append(
+                f'the prepared output changed ({count} files, digest {value}) but OUTPUT_VERSION is still '
+                f'{current[0]}. Players would keep resources prepared by the old preparer. Bump it:\n'
+                f'  set `pub const OUTPUT_VERSION: u32 = {current[0] + 1};` in {LIB}\n'
+                f'  append `{current[0] + 1} {value}` to {VERSIONS}\n'
+                '  (if the change is only a nondeterminism this tool should ignore, teach canonical() instead)')
+    for problem in found:
+        print(f'::error::{problem}' if os.environ.get('GITHUB_ACTIONS') else f'error: {problem}', file=sys.stderr)
+    if not found:
+        print(f'prepared output: OUTPUT_VERSION {current[0]} ({count} files, {value[:16]})')
+    return 1 if found else 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())