From 938e6b1191d07da24615e49487ea544ef88f81f6 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:57:15 +0200 Subject: [PATCH 1/3] version_port.py regen: a candidate worktree pinned to the new version, every table regenerated Co-Authored-By: Claude Opus 5.5 --- tools/port_regen.py | 285 ++++++++++++++++++++++++++++++++++++++++++ tools/port_report.py | 36 ++++-- tools/version_port.py | 43 ++++++- 3 files changed, 350 insertions(+), 14 deletions(-) create mode 100644 tools/port_regen.py diff --git a/tools/port_regen.py b/tools/port_regen.py new file mode 100644 index 000000000..025b69ed8 --- /dev/null +++ b/tools/port_regen.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""Regenerate what Horizon derives from vanilla, for a new version, in a candidate worktree +(docs/version-ports.md, "Regenerate"). + +Used through `version_port.py regen NEW`: + +1. the candidate: a worktree of `port/NEW` (from --base, default origin/main), its research-lock.json pinned to + NEW (version, version JSON, jars, Java); +2. its .local: NEW's jars, libraries and manifest prepared by tools/lab.py prepare from the version store (a + launcher-shaped copy of the download), NEW's data generator reports, the shared tools; +3. the two vanilla classes the generators parse (EntityTypes, BlockStateData) decompiled alone, with the full + jar and its libraries for types, into the version store; +4. every generator run there, each recorded as unchanged, regenerated (with its diff stat) or failed (with the + end of its output); the regenerated tree is committed on port/NEW, never pushed. + +Nothing here writes into the shared .local (~/code/horizon-parity/shared-local): the candidate links only what +it reads. +""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import zipfile + +try: + from . import lab +except ImportError: + import lab + +# Classes whose decompiled source a generator parses: (side, class). +SOURCES = [("client", "net/minecraft/world/entity/EntityTypes"), + ("server", "net/minecraft/util/datafix/fixes/BlockStateData")] +PARITY = "mods/horizon/src/test/java/dev/horizonlab/parity" + + +# --- the candidate ----------------------------------------------------------------------------------------- + +def git(*args, cwd=lab.ROOT, check=True): + return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=check).stdout + + +def candidate(version, base, path): + """The port/VERSION worktree at path, made from base unless it exists.""" + if (path / ".git").exists(): + return path + path.parent.mkdir(parents=True, exist_ok=True) + branch = f"port/{version}" + exists = git("rev-parse", "--verify", "--quiet", f"refs/heads/{branch}", check=False).strip() + git("worktree", "add", *([str(path), branch] if exists else ["-b", branch, str(path), base])) + return path + + +def pinned_lock(lock, manifest, manifest_sha1): + """research-lock.json pinned to the version this manifest describes. The same version keeps its block (and + the republished version JSONs it accepts); another version gets a new one.""" + game = lock["minecraft"] + same = (game["version"] == manifest["id"] + and manifest_sha1 in [game["manifest_sha1"], *game.get("accepted_manifest_sha1", [])] + and game["client_sha1"] == manifest["downloads"]["client"]["sha1"] + and game["server_sha1"] == manifest["downloads"]["server"]["sha1"]) + if same: + return lock + return {**lock, "minecraft": { + "version": manifest["id"], "manifest_sha1": manifest_sha1, + "client_sha1": manifest["downloads"]["client"]["sha1"], + "server_sha1": manifest["downloads"]["server"]["sha1"], + "java_major": manifest["javaVersion"]["majorVersion"], + }} + + +def pin(root, version_json): + lock_path = root / "research-lock.json" + lock = json.loads(lock_path.read_text()) + pinned = pinned_lock(lock, json.loads(version_json.read_text()), lab.digest(version_json, "sha1")) + if pinned is not lock: + lab.write_json(lock_path, pinned) + + +def link(target, link_path): + link_path.parent.mkdir(parents=True, exist_ok=True) + if link_path.is_symlink() or link_path.exists(): + if link_path.is_symlink() and Path(os.readlink(link_path)) == target: + return + raise SystemExit(f"{link_path} exists and is not a link to {target}; remove it") + link_path.symlink_to(target) + + +def launcher_tree(store, manifest): + """A launcher-shaped copy of a downloaded version (versions/ID/ID.json and .jar, libraries/), so that + tools/lab.py prepare imports it exactly as it imports a player's install.""" + tree, version = store / "launcher", manifest["id"] + versions = tree / "versions" / version + versions.mkdir(parents=True, exist_ok=True) + shutil.copyfile(store / "version.json", versions / f"{version}.json") + jar = versions / f"{version}.jar" + if not jar.exists(): + os.link(store / "jars" / "client.jar", jar) + for artifact in lab.client_library_artifacts(manifest): + lab.download(artifact["url"], tree / "libraries" / artifact["path"], artifact["sha1"]) + return tree + + +def prepare_local(root, version, found, store): + """The candidate's .local: tools from the shared one, the version's prepared jars and its reports.""" + shared = Path(os.environ.get("HORIZON_SHARED_LOCAL", Path.home() / "code/horizon-parity/shared-local")) + local = root / ".local" + if local.is_symlink(): + raise SystemExit(f"{local} links to a shared directory; a candidate needs its own") + link(shared / "tools", local / "tools") + link(found["reports"], local / "datagen" / "out" / "reports") + jars = found["client"].parent + if jars.parent.name == version and (jars.parent / "prepared.json").exists() and store not in jars.parents: + link(jars.parent, local / "minecraft" / version) # The pinned version, prepared by lab.py already. + return + game = store / "game" + game.mkdir(exist_ok=True) + link(game, local / "minecraft" / version) + if not (game / "prepared.json").exists(): + tree = launcher_tree(store, json.loads(found["version_json"].read_text())) + run([python(), "tools/lab.py", "prepare", "--minecraft-dir", tree, "--server-jar", found["server"]], root) + + +def python(): + return shutil.which("python3") or "python3" + + +def run(command, cwd, log=None, env=None): + """Run a command; on failure raise with the end of its output.""" + result = subprocess.run([str(c) for c in command], cwd=cwd, capture_output=True, text=True, env=env) + if log: + log.write_text(result.stdout + result.stderr) + if result.returncode: + raise RuntimeError(f"{' '.join(map(str, command))[:300]} exited {result.returncode}:\n" + + (result.stdout + result.stderr)[-2500:]) + return result.stdout + + +# --- decompiling the classes the generators parse ------------------------------------------------------------ + +def decompile_classes(root, version, store): + """Decompile only SOURCES' classes (with their inner classes) into store/sources/SIDE, with the full jar and + the side's libraries on the decompiler's path so types resolve as in a full run.""" + lock = json.loads((root / "research-lock.json").read_text()) + vineflower = root / ".local" / "tools" / f"vineflower-{lock['vineflower']['version']}.jar" + prepared = json.loads((root / ".local" / "minecraft" / version / "prepared.json").read_text()) + for side, name in SOURCES: + output = store / "sources" / side + if (output / f"{name}.java").exists(): + continue + jar = root / prepared["jars"][side] + with tempfile.TemporaryDirectory(prefix=".decompile-", dir=store) as scratch: + scratch = Path(scratch) + small = scratch / "classes.jar" + with zipfile.ZipFile(jar) as full, zipfile.ZipFile(small, "w") as out: + for entry in full.namelist(): + if entry == f"{name}.class" or (entry.startswith(f"{name}$") and entry.endswith(".class")): + out.writestr(entry, full.read(entry)) + command = ["java", "-Xmx2G", "-jar", vineflower, "--folder", "--skip-extra-files=1", "--log-level=warn", + f"--add-external={jar}", *(f"--add-external={root / p}" for p in prepared["libraries"][side]), + small, scratch / "out"] + run(command, root) + target = output / f"{name}.java" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.move(scratch / "out" / f"{name}.java", target) + + +# --- the generators ------------------------------------------------------------------------------------------ + +def compile_oracle(root, version, classpath, sources, build): + lock = json.loads((root / "research-lock.json").read_text()) + classes = build / "classes" + classes.mkdir(parents=True, exist_ok=True) + run(["javac", "--release", lock["minecraft"]["java_major"], "-proc:none", "-sourcepath", + root / "mods/horizon/src/test/java", "-cp", os.pathsep.join(map(str, classpath)), "-d", classes, + *sources], root) + return classes + + +def classpaths(root, version): + prepared = json.loads((root / ".local" / "minecraft" / version / "prepared.json").read_text()) + return {side: [root / prepared["jars"][side], *(root / p for p in prepared["libraries"][side])] + for side in ("client", "server")} + + +def gen_suggestions(root, version, store, build): + client = classpaths(root, version)["client"] + classes = compile_oracle(root, version, client, [root / PARITY / "VanillaCommandSuggestionsExporter.java"], + build / "suggestions") + tsv = build / "suggestions.tsv" + run(["java", "-Xmx1G", "-cp", os.pathsep.join(map(str, [classes, *client])), + "dev.horizonlab.parity.VanillaCommandSuggestionsExporter", tsv], build) + return [python(), "tools/generate_command_suggestions.py", tsv] + + +def gen_vanilla_order(root, version, store, build): + """vanilla's tree (the command-parse oracle's export) against Horizon's (command_parse_fixture tree).""" + server = classpaths(root, version)["server"] + classes = compile_oracle(root, version, server, [root / PARITY / "VanillaCommandParseOracle.java"], + build / "parse") + run(["java", "-Xmx1G", "-cp", os.pathsep.join(map(str, [classes, *server])), + "dev.horizonlab.parity.VanillaCommandParseOracle", "tree", build], build) + env = dict(os.environ, CARGO_TARGET_DIR=str(root / ".local" / "port-target")) + jobs = os.environ.get("FLEET_LOAD_JOBS") + run(["cargo", "build", "-p", "horizon-client", "--example", "command_parse_fixture", + *(["-j", jobs] if jobs else [])], root, env=env) + fixture = root / ".local/port-target/debug/examples/command_parse_fixture" + run([fixture, "tree", build / "rust_tree.json"], root) + return [python(), "tools/generate_command_order.py", build / "tree.json", build / "rust_tree.json"] + + +# (name, outputs, how to get the generator's command: a list, or a function that prepares its input first). +GENERATORS = [ + ("version_consts", ["crates/horizon-core/src/version.rs"], + lambda root, version, store, build: [python(), "tools/gen_version_consts.py"]), + ("packet_ids", ["crates/horizon-protocol/src/ids.rs"], + lambda root, version, store, build: [python(), "tools/generate_packet_ids.py"]), + ("entity_tracking", ["crates/horizon-client/src/lan/tracking.rs"], + lambda root, version, store, build: [python(), "tools/generate_entity_tracking.py", "--sources", + store / "sources" / "client"]), + ("block_state_data", ["crates/horizon-world/src/datafix/block_state_data.txt"], + lambda root, version, store, build: [python(), "tools/gen_block_state_data.py", + store / "sources/server" / f"{SOURCES[1][1]}.java"]), + ("lan_command_order", ["crates/horizon-client/src/lan/command_order.rs"], + lambda root, version, store, build: [python(), "tools/generate_lan_command_order.py"]), + ("command_suggestions", ["crates/horizon-client/src/lan/command_suggestions.rs"], gen_suggestions), + ("vanilla_command_order", ["crates/horizon-client/src/command/vanilla_order.rs"], gen_vanilla_order), +] + + +def status(root, outputs): + """unchanged, or regenerated with {path: [added, removed]} against the candidate's last commit.""" + stat = {} + for line in git("diff", "--numstat", "HEAD", "--", *outputs, cwd=root).splitlines(): + added, removed, path = line.split("\t") + stat[path] = [int(added), int(removed)] + return ("regenerated", stat) if stat else ("unchanged", {}) + + +def run_generators(root, version, store, names=None): + """Every generator in the candidate: {name: {status, outputs, stat | error}}.""" + build = root / ".local" / "port-regen" + build.mkdir(parents=True, exist_ok=True) + results = {} + for name, outputs, command in GENERATORS: + if names and name not in names: + continue + print(f" {name}", flush=True) + try: + run(command(root, version, store, build), root, log=build / f"{name}.log") + rust = [root / o for o in outputs if o.endswith(".rs")] + if rust: + run(["rustfmt", "--edition", "2024", *rust], root) + state, stat = status(root, outputs) + results[name] = {"status": state, "outputs": outputs, "stat": stat} + except (RuntimeError, OSError, subprocess.CalledProcessError, KeyError, ValueError) as error: + git("checkout", "HEAD", "--", *outputs, cwd=root, check=False) + results[name] = {"status": "failed", "outputs": outputs, "error": str(error)[-2500:]} + print(f" {name}: {results[name]['status']}", flush=True) + return results + + +def commit(root, version, results): + """Commit the pin and what regenerated cleanly on port/VERSION (local only).""" + paths = ["research-lock.json"] + [o for r in results.values() if r["status"] == "regenerated" for o in r["outputs"]] + git("add", "--", *paths, cwd=root) + if git("diff", "--cached", "--name-only", cwd=root).strip(): + git("commit", "-q", "-m", f"Port to Minecraft {version}: pin it and regenerate (version_port.py regen)", + cwd=root) + return git("rev-parse", "HEAD", cwd=root).strip() + + +def regen(version, found, store, base, path, names=None): + """The whole stage: {old, new, candidate, commit, generators}.""" + old = json.loads(git("show", f"{base}:research-lock.json"))["minecraft"]["version"] + root = candidate(version, base, path) + pin(root, found["version_json"]) + print(f"Candidate {root} (port/{version}, was {old})", flush=True) + prepare_local(root, version, found, store) + decompile_classes(root, version, store) + results = run_generators(root, version, store, names) + return {"old": old, "new": version, "candidate": str(root), "commit": commit(root, version, results), + "generators": results} diff --git a/tools/port_report.py b/tools/port_report.py index 94ad0b91d..eafcaa8bd 100644 --- a/tools/port_report.py +++ b/tools/port_report.py @@ -104,7 +104,7 @@ def item(rank, kind, title, **details): return {"rank": rank, "kind": kind, "title": title, **details} -def blocking_items(diff, mapping, pinned_facts, root): +def blocking_items(diff, mapping, pinned_facts, root, regen=None): items = [] if diff["facts"]: changes = ", ".join(f"{k} {a} -> {b}" for k, (a, b) in diff["facts"].items()) @@ -147,9 +147,15 @@ def blocking_items(diff, mapping, pinned_facts, root): items.append(item(1, "report-format", f"blocks.json format: {', '.join(diff['blocks']['format'])}", files=files_containing("blocks.json", root), details={"fields": diff["blocks"]["format"]})) - for output, command, stale in GENERATORS: - if stale(diff): - items.append(item(1, "regenerate", f"Regenerate {output}", files=[output], command=command)) + if regen is None: + for output, command, stale in GENERATORS: + if stale(diff): + items.append(item(1, "regenerate", f"Regenerate {output}", files=[output], command=command)) + else: + for name, entry in regen["generators"].items(): + if entry["status"] == "failed": + items.append(item(1, "regen-failed", f"Generator {name} failed on {regen['new']}", + files=entry["outputs"], error=entry["error"])) return items @@ -237,10 +243,11 @@ def content_items(diff, mapping): return items -def build(diff, mapping, old, new, pinned_facts, results=None, root=ROOT): +def build(diff, mapping, old, new, pinned_facts, results=None, root=ROOT, regen=None): """The report: summary, ranked items and what came out clean. pinned_facts: the facts of the version the - code states today (to find where it states them); results: {harness: "pass" | "fail"} against NEW.""" - items = blocking_items(diff, mapping, pinned_facts, root) + code states today (to find where it states them); results: {harness: "pass" | "fail"} against NEW; regen: + version_port.py regen's result (without it, the tables a diff makes stale are listed to regenerate).""" + items = blocking_items(diff, mapping, pinned_facts, root, regen) items += code_items(diff, mapping, results) + content_items(diff, mapping) items.sort(key=lambda i: (i["rank"], -i.get("score", 0), i["title"])) for number, entry in enumerate(items, 1): @@ -256,8 +263,12 @@ def build(diff, mapping, old, new, pinned_facts, results=None, root=ROOT): "classes": {"added": len(classes["added"]), "removed": len(classes["removed"]), "moved": len(classes["moved"]), "changed": len(classes["changed"]), "horizon": len(classes["horizon"])}, - "generators": [i["files"][0] for i in items if i["kind"] == "regenerate"], + "generators": ({name: e["status"] for name, e in regen["generators"].items()} if regen + else {i["files"][0]: "stale" for i in items if i["kind"] == "regenerate"}), }, + "regen": {"candidate": regen["candidate"], "commit": regen["commit"], + "stat": {p: s for e in regen["generators"].values() for p, s in e.get("stat", {}).items()}} + if regen else None, "items": items, "clean": clean, } @@ -278,6 +289,13 @@ def markdown(report, limit=12): lines.append(f"- work items by rank: " + ", ".join(f"{r}: {n}" for r, n in summary["ranks"].items())) if report["clean"]: lines.append(f"- unchanged: {', '.join(report['clean'])}") + if report.get("regen"): + generators = summary["generators"] + lines.append(f"- generators in `{report['regen']['candidate']}` at {report['regen']['commit'][:10]}: " + + ", ".join(f"{s} {sum(1 for v in generators.values() if v == s)}" + for s in ("regenerated", "unchanged", "failed"))) + for path, (added, removed) in report["regen"]["stat"].items(): + lines.append(f" - regenerated `{path}` (+{added} -{removed})") names = {1: "Blocks the switch", 2: "Proved by a harness: rerun it", 3: "Ported, no harness", 4: "New content", 5: "Not ported"} for rank in sorted({i["rank"] for i in report["items"]}): @@ -286,6 +304,8 @@ def markdown(report, limit=12): lines.append(f"### #{entry['id']} {entry['title']}") if entry.get("command"): lines.append(f"- run: `{entry['command']}`") + if entry.get("error"): + lines.append("- error: `" + entry["error"].strip().splitlines()[-1][:300] + "`") if entry.get("failed"): lines.append(f"- failing harnesses: {', '.join(entry['failed'])}") if entry.get("harnesses"): diff --git a/tools/version_port.py b/tools/version_port.py index c46efa202..f99727fd0 100644 --- a/tools/version_port.py +++ b/tools/version_port.py @@ -8,12 +8,15 @@ version_port.py diff OLD NEW facts, packets, registries, blocks, items, commands, data packs, assets, lang and classes, with the Horizon code each changed class concerns: diff-OLD-NEW/diff.json + version_port.py regen NEW [--base REF] a port/NEW worktree pinned to NEW with every generated table + regenerated (tools/port_regen.py): diff-OLD-NEW/regen.json version_port.py report OLD NEW [--results S] the ranked port report: diff-OLD-NEW/report.md and .json - version_port.py port NEW [--old OLD] prepare both, diff, report + version_port.py port NEW [--old OLD] prepare both, diff, regen, report version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep The downloads are for development only: they go to the version store (~/code/horizon-parity/versions, -HORIZON_VERSIONS overrides it), never into the repository, and nothing from them is shipped. The pinned +HORIZON_VERSIONS overrides it), never into the repository, and nothing from them is shipped. Candidate +worktrees go to ~/code/horizon-parity/ports/NEW (HORIZON_PORTS overrides it). The pinned version is not downloaded: its files are read where tools/lab.py prepared them (.local/). detect exits 0 when nothing is new, 10 for a new release and 11 for a new snapshot (with --snapshots). @@ -33,15 +36,17 @@ import urllib.request import zipfile try: - from . import class_index, lab, port_report, vanilla_map + from . import class_index, lab, port_regen, port_report, vanilla_map except ImportError: import class_index import lab + import port_regen import port_report import vanilla_map MANIFEST_URL = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json" STORE = Path(os.environ.get("HORIZON_VERSIONS", Path.home() / "code" / "horizon-parity" / "versions")) +PORTS = Path(os.environ.get("HORIZON_PORTS", STORE.parent / "ports")) STATES = ["handshake", "status", "login", "configuration", "play"] # How many changed JSON paths one changed file keeps in diff.json. PATH_LIMIT = 50 @@ -545,6 +550,20 @@ def diff(args): return 0 +# --- regen ------------------------------------------------------------------------------------------------- + +def regen(args): + version = safe_version(args.new) + result = port_regen.regen(version, paths(version), version_dir(version), args.base, + Path(args.worktree) if args.worktree else PORTS / version, args.only) + lab.write_json(diff_dir(result["old"], version) / "regen.json", result) + for name, entry in result["generators"].items(): + stat = ", ".join(f"{p} +{a} -{r}" for p, (a, r) in entry.get("stat", {}).items()) + print(f"{name}: {entry['status']}" + (f" ({stat})" if stat else "")) + print(f"\n{result['candidate']} at {result['commit'][:10]}; {diff_dir(result['old'], version) / 'regen.json'}") + return 1 if any(e["status"] == "failed" for e in result["generators"].values()) else 0 + + # --- report ------------------------------------------------------------------------------------------------ def harness_results(state_path): @@ -563,7 +582,10 @@ def report(args): result = json.loads((directory / "diff.json").read_text()) mapping = json.loads((directory / "vanilla_map.json").read_text()) results = harness_results(args.results) if args.results else None - built = port_report.build(result, mapping, args.old, args.new, load(pinned(), "facts.json"), results) + regenerated = directory / "regen.json" + regenerated = json.loads(regenerated.read_text()) if regenerated.exists() else None + built = port_report.build(result, mapping, args.old, args.new, load(pinned(), "facts.json"), results, + regen=regenerated) lab.write_json(directory / "report.json", built) (directory / "report.md").write_text(port_report.markdown(built)) print("\n".join(port_report.markdown(built).splitlines()[:args.lines])) @@ -572,11 +594,13 @@ def report(args): def port(args): - """prepare both, diff, report.""" + """prepare both, diff, regen, report.""" old = args.old or pinned() for version in (old, args.new): prepare(argparse.Namespace(version=version, download=False, jobs=args.jobs)) diff(argparse.Namespace(old=old, new=args.new)) + if not args.no_regen: + regen(argparse.Namespace(new=args.new, base=args.base, worktree=None, only=None)) return report(argparse.Namespace(old=old, new=args.new, results=args.results, lines=args.lines)) @@ -623,6 +647,11 @@ def main(): command = commands.add_parser("diff", help="diff two prepared versions") command.add_argument("old") command.add_argument("new") + command = commands.add_parser("regen", help="a candidate worktree pinned to NEW, every table regenerated") + command.add_argument("new") + command.add_argument("--base", default="origin/main", help="what port/NEW starts from (default origin/main)") + command.add_argument("--worktree", help="where (default: $HORIZON_PORTS/NEW)") + command.add_argument("--only", nargs="*", help="these generators only (names in tools/port_regen.py)") command = commands.add_parser("report", help="the ranked port report of a diff") command.add_argument("old") command.add_argument("new") @@ -633,13 +662,15 @@ def main(): command.add_argument("--old", help="default: the pinned version") command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW") command.add_argument("--lines", type=int, default=60) + command.add_argument("--base", default="origin/main", help="what the candidate starts from") + command.add_argument("--no-regen", action="store_true", help="diff and report only (snapshots)") command.add_argument("-j", "--jobs", type=int, default=int(os.environ.get("FLEET_LOAD_JOBS", 0)) or None) command = commands.add_parser("gc", help="delete versions and scratch the store doesn't need") command.add_argument("--keep", nargs="*", default=[]) command.add_argument("--dry-run", action="store_true") args = parser.parse_args() STORE.mkdir(parents=True, exist_ok=True) - return {"detect": detect, "prepare": prepare, "diff": diff, "report": report, "port": port, + return {"detect": detect, "prepare": prepare, "diff": diff, "regen": regen, "report": report, "port": port, "gc": gc}[args.command](args) From 3b5eabaf5f96501cf9b1e6dd926cbc52ca140d42 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:03:43 +0200 Subject: [PATCH 2/3] regen: command_parse_fixture exports Horizon's registration order; javac errors summarized Co-Authored-By: Claude Opus 5.5 --- .../examples/command_parse_fixture.rs | 2 +- crates/horizon-client/src/command/vanilla.rs | 9 +++- tests/test_port_regen.py | 52 +++++++++++++++++++ tests/test_port_report.py | 25 ++++++++- tools/generate_command_order.py | 2 +- tools/port_regen.py | 34 ++++++++++-- tools/port_report.py | 10 ++-- 7 files changed, 122 insertions(+), 12 deletions(-) create mode 100644 tests/test_port_regen.py diff --git a/crates/horizon-client/examples/command_parse_fixture.rs b/crates/horizon-client/examples/command_parse_fixture.rs index 39d738b7f..1cc1f95d5 100644 --- a/crates/horizon-client/examples/command_parse_fixture.rs +++ b/crates/horizon-client/examples/command_parse_fixture.rs @@ -135,7 +135,7 @@ fn tree(d: &Dispatcher, node: usize) -> Value { fn main() -> horizon_client::Result<()> { let args: Vec = std::env::args().collect(); if args.get(1).map(String::as_str) == Some("tree") { - let d = vanilla::release_dispatcher(); + let d = &vanilla::registered(false); let mut v = tree(d, 0); v["type"] = json!("root"); fs::write(&args[2], v.to_string())?; diff --git a/crates/horizon-client/src/command/vanilla.rs b/crates/horizon-client/src/command/vanilla.rs index daf4ecf86..fd23b82cd 100644 --- a/crates/horizon-client/src/command/vanilla.rs +++ b/crates/horizon-client/src/command/vanilla.rs @@ -36,6 +36,14 @@ pub fn build() -> Dispatcher { /// The tree Commands registers; `dev` adds the commands vanilla only /// registers in development builds (raid, warden_spawn_tracker). pub fn build_with(dev: bool) -> Dispatcher { + let mut d = registered(dev); + apply_vanilla_order(&mut d); + d +} +/// The tree in the order Horizon's code registers it, before +/// [`vanilla_order`](super::vanilla_order) puts children in vanilla's order +/// (tools/generate_command_order.py compares the two). +pub fn registered(dev: bool) -> Dispatcher { let mut d = Dispatcher::new(); let root = 0; @@ -403,7 +411,6 @@ pub fn build_with(dev: bool) -> Dispatcher { super::tree_misc::register(&mut d, root); super::tree_world::register(&mut d, root); super::tree_extra::register(&mut d, root); - apply_vanilla_order(&mut d); d } diff --git a/tests/test_port_regen.py b/tests/test_port_regen.py new file mode 100644 index 000000000..c52499417 --- /dev/null +++ b/tests/test_port_regen.py @@ -0,0 +1,52 @@ +import unittest + +from tools import port_regen as regen + +LOCK = {"minecraft": {"version": "26.3", "manifest_sha1": "m3", "accepted_manifest_sha1": ["m3old"], + "accepted_manifest_note": "republished", "client_sha1": "c3", "server_sha1": "s3", + "java_major": 25}, + "vineflower": {"version": "1.12.0"}} + + +def manifest(version, client, server, java=25): + return {"id": version, "downloads": {"client": {"sha1": client}, "server": {"sha1": server}}, + "javaVersion": {"majorVersion": java}} + + +class PinTests(unittest.TestCase): + def test_another_version_gets_a_new_block(self): + pinned = regen.pinned_lock(LOCK, manifest("26.4", "c4", "s4", 26), "m4") + self.assertEqual(pinned["minecraft"], {"version": "26.4", "manifest_sha1": "m4", "client_sha1": "c4", + "server_sha1": "s4", "java_major": 26}) + self.assertEqual(pinned["vineflower"], LOCK["vineflower"]) + + def test_the_same_version_keeps_its_block(self): + for sha1 in ["m3", "m3old"]: + self.assertIs(regen.pinned_lock(LOCK, manifest("26.3", "c3", "s3"), sha1), LOCK) + + def test_the_same_version_with_other_jars_is_repinned(self): + self.assertEqual(regen.pinned_lock(LOCK, manifest("26.3", "c3b", "s3"), "m3b")["minecraft"]["client_sha1"], + "c3b") + + +class CompileErrorTests(unittest.TestCase): + OUTPUT = """/x/mods/horizon/src/test/java/dev/horizonlab/parity/ParityData.java:45: error: cannot find symbol + var context = TagLoader.buildUpdatedLookups(layers.getAccessForLoading(RegistryLayer.WORLD), staticTags); + ^ + symbol: variable WORLD + location: class RegistryLayer +/x/VanillaCommandParseOracle.java:12: error: incompatible types: int cannot be converted to String +2 errors +""" + + def test_names_the_symbols(self): + self.assertEqual(regen.compile_errors(self.OUTPUT), [ + "ParityData.java:45: cannot find symbol: variable WORLD (class RegistryLayer)", + "VanillaCommandParseOracle.java:12: incompatible types: int cannot be converted to String"]) + + def test_nothing_from_other_failures(self): + self.assertEqual(regen.compile_errors("expected the root's order to differ"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_port_report.py b/tests/test_port_report.py index f1a8e1f35..9db732862 100644 --- a/tests/test_port_report.py +++ b/tests/test_port_report.py @@ -62,11 +62,32 @@ def diff(): class PortReportTests(unittest.TestCase): - def build(self, results=None): + def build(self, results=None, regen=None): with mock.patch.object(report, "version_sites", return_value=["crates/p/src/lib.rs:20"]), \ mock.patch.object(report, "files_naming", return_value=["crates/p/src/play.rs"]), \ mock.patch.object(report, "files_containing", return_value=["crates/w/src/blocks.rs"]): - return report.build(diff(), mapping(), "26.2", "26.3", {"id": "26.3"}, results) + return report.build(diff(), mapping(), "26.2", "26.3", {"id": "26.3"}, results, regen=regen) + + def test_regen_results_replace_the_stale_tables(self): + regen = {"new": "26.3", "candidate": "/ports/26.3", "commit": "0123456789abcdef", "generators": { + "packet_ids": {"status": "regenerated", "outputs": ["crates/horizon-protocol/src/ids.rs"], + "stat": {"crates/horizon-protocol/src/ids.rs": [142, 134]}}, + "entity_tracking": {"status": "unchanged", "outputs": ["crates/horizon-client/src/lan/tracking.rs"], + "stat": {}}, + "lan_command_order": {"status": "failed", "outputs": ["crates/horizon-client/src/lan/command_order.rs"], + "error": "javac exited 1", + "compile_errors": ["ParityData.java:45: cannot find symbol: variable WORLD"]}}} + built = self.build(regen=regen) + self.assertFalse([i for i in built["items"] if i["kind"] == "regenerate"]) + [failed] = [i for i in built["items"] if i["kind"] == "regen-failed"] + self.assertEqual(failed["rank"], 1) + self.assertIn("no longer compiles", failed["title"]) + self.assertEqual(built["summary"]["generators"], + {"packet_ids": "regenerated", "entity_tracking": "unchanged", "lan_command_order": "failed"}) + text = report.markdown(built) + self.assertIn("regenerated 1, unchanged 1, failed 1", text) + self.assertIn("`crates/horizon-protocol/src/ids.rs` (+142 -134)", text) + self.assertIn("- javac: `ParityData.java:45: cannot find symbol: variable WORLD`", text) def test_ranks_and_kinds(self): built = self.build() diff --git a/tools/generate_command_order.py b/tools/generate_command_order.py index 6eabc909b..ab78b3ada 100644 --- a/tools/generate_command_order.py +++ b/tools/generate_command_order.py @@ -5,7 +5,7 @@ registers them in a different order. Brigadier keeps children in registration order, and /help (smart usage) lists them that way. python3 tools/test_command_parse_parity.py # exports .local/command-parse-parity/tree.json - command_parse_fixture tree rust_tree.json # Horizon's tree with child order + command_parse_fixture tree rust_tree.json # Horizon's tree in its registration order generate_command_order.py VANILLA_TREE RUST_TREE """ import argparse diff --git a/tools/port_regen.py b/tools/port_regen.py index 025b69ed8..be2d31843 100644 --- a/tools/port_regen.py +++ b/tools/port_regen.py @@ -20,6 +20,7 @@ it reads. import json import os from pathlib import Path +import re import shutil import subprocess import tempfile @@ -127,17 +128,41 @@ def python(): return shutil.which("python3") or "python3" +class Failed(RuntimeError): + """A command that exited non-zero, with all it printed.""" + + def __init__(self, command, code, output): + super().__init__(f"{' '.join(map(str, command))[:300]} exited {code}:\n{output[-2500:]}") + self.output = output + + def run(command, cwd, log=None, env=None): - """Run a command; on failure raise with the end of its output.""" + """Run a command; on failure raise Failed.""" result = subprocess.run([str(c) for c in command], cwd=cwd, capture_output=True, text=True, env=env) if log: log.write_text(result.stdout + result.stderr) if result.returncode: - raise RuntimeError(f"{' '.join(map(str, command))[:300]} exited {result.returncode}:\n" - + (result.stdout + result.stderr)[-2500:]) + raise Failed(command, result.returncode, result.stdout + result.stderr) return result.stdout +def compile_errors(output, limit=12): + """javac's errors, one line each: "ParityData.java:45: cannot find symbol: variable WORLD (class + RegistryLayer)" names the vanilla API an oracle used that the new version changed.""" + lines = output.splitlines() + found = [] + for index, line in enumerate(lines): + match = re.match(r".*?([\w$]+\.java):(\d+): error: (.*)", line) + if not match: + continue + entry = f"{match[1]}:{match[2]}: {match[3]}" + after = [text.strip() for text in lines[index + 1:index + 6]] + symbol = next((text.split(":", 1)[1].strip() for text in after if text.startswith("symbol:")), None) + where = next((text.split(":", 1)[1].strip() for text in after if text.startswith("location:")), None) + found.append(entry + (f": {symbol}" if symbol else "") + (f" ({where})" if where else "")) + return found[:limit] + + # --- decompiling the classes the generators parse ------------------------------------------------------------ def decompile_classes(root, version, store): @@ -257,7 +282,8 @@ def run_generators(root, version, store, names=None): results[name] = {"status": state, "outputs": outputs, "stat": stat} except (RuntimeError, OSError, subprocess.CalledProcessError, KeyError, ValueError) as error: git("checkout", "HEAD", "--", *outputs, cwd=root, check=False) - results[name] = {"status": "failed", "outputs": outputs, "error": str(error)[-2500:]} + results[name] = {"status": "failed", "outputs": outputs, "error": str(error)[-2500:], + "compile_errors": compile_errors(getattr(error, "output", str(error)))} print(f" {name}: {results[name]['status']}", flush=True) return results diff --git a/tools/port_report.py b/tools/port_report.py index eafcaa8bd..0e5766ed9 100644 --- a/tools/port_report.py +++ b/tools/port_report.py @@ -154,8 +154,10 @@ def blocking_items(diff, mapping, pinned_facts, root, regen=None): else: for name, entry in regen["generators"].items(): if entry["status"] == "failed": - items.append(item(1, "regen-failed", f"Generator {name} failed on {regen['new']}", - files=entry["outputs"], error=entry["error"])) + errors = entry.get("compile_errors", []) + what = "its vanilla exporter no longer compiles" if errors else "failed" + items.append(item(1, "regen-failed", f"Generator {name}: {what} on {regen['new']}", + files=entry["outputs"], error=entry["error"], compile_errors=errors)) return items @@ -304,7 +306,9 @@ def markdown(report, limit=12): lines.append(f"### #{entry['id']} {entry['title']}") if entry.get("command"): lines.append(f"- run: `{entry['command']}`") - if entry.get("error"): + if entry.get("compile_errors"): + lines += [f"- javac: `{e}`" for e in entry["compile_errors"][:limit]] + elif entry.get("error"): lines.append("- error: `" + entry["error"].strip().splitlines()[-1][:300] + "`") if entry.get("failed"): lines.append(f"- failing harnesses: {', '.join(entry['failed'])}") From b3bc3c076831513b69ee362f278218849a42d0b8 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:07:13 +0200 Subject: [PATCH 3/3] docs/version-ports.md: regen, its replay, status Co-Authored-By: Claude Opus 5.5 --- docs/version-ports.md | 67 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/docs/version-ports.md b/docs/version-ports.md index cdc2b37de..96c6dab6e 100644 --- a/docs/version-ports.md +++ b/docs/version-ports.md @@ -169,20 +169,35 @@ neither, is reported as such. ### Regenerate -Each generated output has a generator entry in `version_port.py` (`regen`): its command, its input in the version -store, the files it writes. On a candidate worktree (`port/NEW` from `origin/main`, with `.local/` linking to the -new version), `regen` runs them all and records, per generator: unchanged, regenerated (with the `--stat`), or -failed (the generator's own error: a source pattern no longer matches, a report changed shape). Two additions make -this complete: +`version_port.py regen NEW` (`tools/port_regen.py`) makes the candidate and regenerates every table in it: -- `version_consts`: a generator for the version facts table above, from `facts.json`. Before it, a small refactor - makes every Rust site use the `horizon-core` / `horizon-protocol` constants instead of its own copy, so the pin - bump touches one generated file. -- `horizon-prepare` against the new version: its Java exporters compile and run against the new jar, or fail to - compile, which names exactly the vanilla API they used that changed. +1. **The candidate**: a worktree of the branch `port/NEW` (from `--base`, default `origin/main`) in + `~/code/horizon-parity/ports/NEW` (`HORIZON_PORTS`), with `research-lock.json` pinned to NEW: version, the + version JSON's sha1, the jars' sha1s, Java. Every tool reads the version from the lock (`lab.VERSION`, + `lab.GAME`, `lab.RESOURCES`), so nothing else in the tree names the old one. +2. **Its `.local/`**, a real directory: `minecraft/NEW` is prepared by `tools/lab.py prepare` itself, from a + launcher-shaped copy of the download in the version store (`versions/NEW/launcher`: the version JSON, the client + jar and its libraries, verified by sha1) into `versions/NEW/game`, so the candidate's jars, libraries and + manifest are exactly what a player's install gives. `datagen/out/reports` links to NEW's reports, `tools` to the + shared tools. Nothing is written into `shared-local/`. +3. **Decompiled sources**: only the classes a generator parses (`EntityTypes`, `BlockStateData`), each decompiled + alone with the full jar and its libraries on Vineflower's path, into `versions/NEW/sources/` (seconds instead of + a 10-minute full run). The source is not the full run's text (inner classes of the jar's other classes are + imported, chained calls are not wrapped), but what the generators read from it is: from 26.3's they write the + committed tables byte for byte. +4. **The generators**, every row of the table above, each recorded in `diff-OLD-NEW/regen.json` as unchanged, + regenerated (with its `--numstat`) or failed (with the end of its output and, when an exporter or oracle no + longer compiles, javac's errors one per line: they name the vanilla API that changed). Rust outputs are + rustfmt'ed. The pin and what regenerated are committed on `port/NEW`, locally; nothing is pushed. -Generators that read decompiled sources (`EntityTypes`, `BlockStateData`) get those two files from the changed-class -decompilation, so no full Vineflower run is needed for them either. +The report then lists what was regenerated in its summary and makes each failed generator a rank-1 item, instead of +guessing which tables the diff made stale. `--only NAME...` reruns some generators; `port NEW` runs `regen` after +`diff` (`--no-regen` for snapshots). + +Three generators run vanilla through Horizon's Java oracles (`VanillaCommandSuggestionsExporter`, +`VanillaCommandParseOracle`) and one builds `command_parse_fixture` (Horizon's command tree in its own registration +order, before `vanilla_order.rs` reorders it), so `regen` needs Java and Cargo. `horizon-prepare` (the resources the +harnesses read) is the next stage's first step. ### Run every harness against the new version @@ -276,6 +291,27 @@ name. The rest are classes whose changed methods Horizon doesn't name. Many of t `--results`) is what turns a rank-2 lead into a confirmed failure; the replay cannot do that, because the oracles are written against 26.3. +### `regen` + +`regen 26.2` from a branch pinned to 26.3, then `regen 26.3 --base port/26.2`, both from versions downloaded from +Mojang (`prepare --download`): + +| Generator | 26.3 → 26.2 | 26.2 → 26.3 | +| --- | --- | --- | +| `version_consts` | regenerated (+8 -8: protocol 776, data version 4903, packs 88.0 / 107.1, build time) | regenerated, back to the committed file | +| `packet_ids` | regenerated (+134 -142) | regenerated (+142 -134), back to the committed file | +| `entity_tracking` | regenerated (+10 -13) | regenerated (+13 -10), back to the committed file | +| `block_state_data` | regenerated (+1 -1: the version in its header; `BlockStateData` is unchanged) | regenerated, back to the committed file | +| `lan_command_order`, `vanilla_command_order` | failed: `VanillaCommandParseOracle.java:81: no suitable constructor found for CommandSourceStack(...)` and 2 more | unchanged | +| `command_suggestions` | failed: `ParityData.java:45: cannot find symbol: variable WORLD (class RegistryLayer)`, `WORLD_REGISTRIES` | unchanged | + +After the round trip, `port/26.3` equals the branch it started from except `research-lock.json`'s five lines of +republished 26.3 version JSONs it accepts (which a port cannot know). So every table Horizon commits is what the +generators make from vanilla's data, and going 26.3 → 26.2 fails exactly where Horizon's oracles use 26.3-only API, +with javac naming it. The report for 26.2 → 26.3 then has 5 rank-1 items instead of 10: the 5 "regenerate" guesses +are replaced by the generators' results in its summary. Both regens took under three minutes, most of it building +`command_parse_fixture`. + ## Implementation status ```sh @@ -283,8 +319,9 @@ python3 tools/version_port.py detect --snapshots # exit 10: new release, 11: python3 tools/version_port.py prepare 26.3 # the pinned version: indexes .local, downloads nothing python3 tools/version_port.py prepare 26.4 # downloads, verifies, runs the data generator (about 10 s) python3 tools/version_port.py diff 26.3 26.4 # summary; the full diff in versions/diff-26.3-26.4/diff.json +python3 tools/version_port.py regen 26.4 # ports/26.4 on port/26.4: pinned, every table regenerated python3 tools/version_port.py report 26.3 26.4 [--results .local/parity-runner/state.json] # report.md, report.json -python3 tools/version_port.py port 26.4 # prepare both, diff, report +python3 tools/version_port.py port 26.4 # prepare both, diff, regen, report python3 tools/version_port.py gc # keep the pinned version and the newest two others python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food.FoodData # its Rust files and harnesses ``` @@ -296,4 +333,6 @@ python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food | Class index (`tools/class_index.py`), class diff with moves, `vanilla_map.json` (`tools/vanilla_map.py`) | done | | Port report (`tools/port_report.py`, `report`), `port` | done | | `version_consts`: one generated file for the version facts (`tools/gen_version_consts.py`) | done | -| Changed-class decompilation, `regen`, candidate harness run, `publish`, launchd agent | later | +| Tools read the version from the lock (`lab.VERSION`, `lab.GAME`, `lab.RESOURCES`, `lab.version_info()`) | done | +| `regen` (`tools/port_regen.py`): candidate worktree, its `.local/`, the generators' classes decompiled alone, every generator, `regen.json` in the report | done | +| Candidate harness run, `publish`, launchd agent | later |