diff --git a/docs/version-ports.md b/docs/version-ports.md index 65793ee53..d625fcda8 100644 --- a/docs/version-ports.md +++ b/docs/version-ports.md @@ -260,10 +260,18 @@ The replay of 26.2 → 26.3 below measures how much that is for one release. ## Implementation status +```sh +python3 tools/version_port.py detect --snapshots # exit 10: new release, 11: new snapshot +python3 tools/version_port.py prepare 26.3 # the pinned version: indexes .local, downloads nothing +python3 tools/version_port.py prepare 26.4 # downloads, verifies, runs the data generator (about 10 s) +python3 tools/version_port.py diff 26.3 26.4 # summary; the full diff in versions/diff-26.3-26.4/diff.json +python3 tools/version_port.py gc # keep the pinned version and the newest two others +``` + | Slice | Status | | --- | --- | | This design | this page | -| `detect`, `prepare`, `diff` (facts, packets, registries, blocks, items, commands, data, assets) | next PR | +| `detect`, `prepare`, `diff` (facts, packets, registries, blocks, items, commands, data, assets), `gc` | done | | Class index, class diff, `vanilla_map.json` | next PR | | Port report (`report`), `port` | next PR | | Changed-class decompilation, `regen`, `version_consts` refactor, candidate harness run, `publish`, launchd agent | later | diff --git a/tests/test_version_port.py b/tests/test_version_port.py new file mode 100644 index 000000000..dd6c2f73b --- /dev/null +++ b/tests/test_version_port.py @@ -0,0 +1,144 @@ +import json +from pathlib import Path +import tempfile +import unittest +import zipfile + +from tools import version_port as port + + +def jar(path, files): + with zipfile.ZipFile(path, "w") as archive: + for name, value in files.items(): + archive.writestr(name, value if isinstance(value, (str, bytes)) else json.dumps(value)) + return path + + +class DetectTests(unittest.TestCase): + MANIFEST = {"latest": {"release": "26.4", "snapshot": "26.5-snapshot-1"}} + + def test_new_release_and_snapshot(self): + self.assertEqual(port.news(self.MANIFEST, "26.3", {}), [("release", "26.4")]) + self.assertEqual(port.news(self.MANIFEST, "26.3", {}, snapshots=True), + [("release", "26.4"), ("snapshot", "26.5-snapshot-1")]) + + def test_pinned_or_seen_is_not_news(self): + self.assertEqual(port.news(self.MANIFEST, "26.4", {}), []) + self.assertEqual(port.news(self.MANIFEST, "26.3", {"release": "26.4", "snapshot": "26.5-snapshot-1"}, + snapshots=True), []) + + def test_a_release_is_not_also_reported_as_the_latest_snapshot(self): + manifest = {"latest": {"release": "26.4", "snapshot": "26.4"}} + self.assertEqual(port.news(manifest, "26.3", {}, snapshots=True), [("release", "26.4")]) + + def test_unsafe_versions_are_refused(self): + for version in ["..", "26.4/../x", ""]: + with self.assertRaises(ValueError): + port.safe_version(version) + + +class ReportDiffTests(unittest.TestCase): + def test_json_diff_paths(self): + self.assertEqual(port.json_diff({"a": 1, "b": [1, 2]}, {"a": 2, "b": [1, 3], "c": True}), + [("/a", 1, 2), ("/b[1]", 2, 3), ("/c", None, True)]) + self.assertEqual(port.json_diff([1], [1, 2]), [("/", [1], [1, 2])]) + self.assertEqual(len(port.json_diff({str(i): i for i in range(9)}, {}, limit=3)), 3) + + def test_packets_added_removed_renumbered(self): + old = {"play": {"serverbound": {"minecraft:swing": {"protocol_id": 0}, "minecraft:use": {"protocol_id": 1}}}} + new = {"play": {"serverbound": {"minecraft:punch": {"protocol_id": 0}, "minecraft:use": {"protocol_id": 2}, + "minecraft:x": {"protocol_id": 1}}}} + self.assertEqual(port.diff_packets(old, new), {"play/serverbound": { + "added": {"minecraft:punch": 0, "minecraft:x": 1}, "removed": {"minecraft:swing": 0}, + "renumbered": {"minecraft:use": [1, 2]}, "count": [2, 3]}}) + self.assertEqual(port.diff_packets(old, old), {}) + + def test_registries(self): + def registry(*names, default=None): + return {"default": default, "entries": {n: {"protocol_id": i} for i, n in enumerate(names)}} + old = {"minecraft:item": registry("air", "stone"), "minecraft:gone": registry("a")} + new = {"minecraft:item": registry("air", "dirt", "stone"), "minecraft:new": registry("b")} + self.assertEqual(port.diff_registries(old, new), { + "added": {"minecraft:new": ["b"]}, "removed": ["minecraft:gone"], + "changed": {"minecraft:item": {"added": ["dirt"], "ids_shifted": 1}}}) + + def test_blocks_state_ids_and_format(self): + def block(first, *values, default=0, **extra): + states = [{"id": first + i, "properties": {"p": v}} for i, v in enumerate(values)] + states[default]["default"] = True + return {"properties": {"p": list(values)}, "states": states, **extra} + old = {"a": block(0, "x", "y"), "b": block(2, "x")} + new = {"a": block(0, "x", "y", "z", default=1, definition={}), + "b": block(3, "x", definition={}), "c": block(4, "x", definition={})} + result = port.diff_blocks(old, new) + self.assertEqual(result["added"], ["c"]) + self.assertEqual(result["format"], ["definition"]) + self.assertEqual(set(result["changed"]), {"a"}) + self.assertEqual(result["changed"]["a"]["states"], [2, 3]) + self.assertEqual(result["changed"]["a"]["default"], [{"p": "x"}, {"p": "y"}]) + self.assertEqual((result["state_ids_shifted"], result["first_shifted"]), (1, "b")) + + def test_items_by_component(self): + old = {"stone": {"swing": {}, "name": "Stone"}, "dirt": {"swing": {}}} + new = {"stone": {"attack": {}, "name": "Rock"}, "dirt": {"attack": {}}, "mud": {}} + result = port.diff_items(old, new) + self.assertEqual(result["added"], ["mud"]) + self.assertEqual(result["components"], {"attack": {"added": 2, "removed": 0, "changed": 0}, + "swing": {"added": 0, "removed": 2, "changed": 0}, + "name": {"added": 0, "removed": 0, "changed": 1}}) + + def test_command_paths(self): + def literal(**children): + return {"type": "literal", "children": children} if children else {"type": "literal", "executable": True} + old = {"type": "root", "children": {"data": literal(get=literal(block=literal(targetPos=literal())))}} + new = {"type": "root", "children": {"data": literal(get=literal(block=literal(pos=literal()))), + "compute": literal()}} + result = port.diff_commands(old, new) + self.assertEqual(result["added"], ["compute", "data get block pos"]) + self.assertEqual(result["removed"], ["data get block targetPos"]) + + +class JarDiffTests(unittest.TestCase): + def test_contents_lang_and_facts(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + common = {"data/minecraft/tags/block/logs.json": {"values": ["oak"]}} + old = jar(root / "old.jar", {**common, + "data/minecraft/recipe/a.json": {"result": "a", "count": 1}, + "data/minecraft/recipe/gone.json": {}, + "assets/minecraft/lang/en_us.json": {"k": "K", "same": "S"}}) + new = jar(root / "new.jar", {**common, + "data/minecraft/recipe/a.json": {"result": "a", "count": 2}, + "data/minecraft/worldgen/biome/new.json": {}, + "assets/minecraft/lang/en_us.json": {"k": "K2", "same": "S", "n": "N"}, + "version.json": {"id": "26.4", "world_version": 5100, + "protocol_version": 778, "java_version": 25, + "pack_version": {"resource_major": 98, "resource_minor": 0, + "data_major": 122, "data_minor": 0}}}) + data = port.diff_contents(old, new, "data/minecraft/") + self.assertEqual(data["counts"], {"added": 1, "removed": 1, "changed": 1}) + self.assertEqual(data["kinds"]["recipe"], {"added": [], "removed": ["data/minecraft/recipe/gone.json"], + "changed": ["data/minecraft/recipe/a.json"]}) + self.assertEqual(data["kinds"]["worldgen/biome"]["added"], ["data/minecraft/worldgen/biome/new.json"]) + self.assertEqual(data["json"], {"data/minecraft/recipe/a.json": [("/count", 1, 2)]}) + self.assertEqual(port.diff_lang(old, new), {"added": ["n"], "removed": [], "changed": ["k"]}) + facts = port.jar_facts(new) + self.assertEqual((facts["world_version"], facts["protocol_version"], facts["data_pack"]), + (5100, 778, "122.0")) + self.assertEqual(port.diff_facts({"id": "a", "protocol_version": 777}, {"id": "b", "protocol_version": 778}), + {"protocol_version": [777, 778]}) + + +class GcTests(unittest.TestCase): + def test_keeps_pinned_newest_two_and_their_diffs(self): + entries = [("26.1", 1, True), ("26.2", 2, True), ("26.3", 3, True), ("26.4-snapshot-1", 4, True), + ("26.4-snapshot-2", 5, True), ("diff-26.2-26.3", 6, False), + ("diff-26.3-26.4-snapshot-2", 7, False), (".datagen-x", 8, False), ("detect-state.json", 9, False)] + self.assertEqual(port.to_remove(entries, "26.3", set()), + [".datagen-x", "26.1", "26.2", "diff-26.2-26.3"]) + self.assertEqual(port.to_remove(entries, "26.3", {"26.1"}), + [".datagen-x", "26.2", "diff-26.2-26.3"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/version_port.py b/tools/version_port.py new file mode 100644 index 000000000..4330b7503 --- /dev/null +++ b/tools/version_port.py @@ -0,0 +1,550 @@ +#!/usr/bin/env python3 +"""Port a new Minecraft release to Horizon: detect it, prepare it, diff it against the pinned one +(docs/version-ports.md). + + version_port.py detect [--snapshots] what Mojang released that Horizon doesn't pin + version_port.py prepare VERSION download, verify, run the data generator, read its facts + version_port.py diff OLD NEW facts, packets, registries, blocks, items, commands, + data packs, assets and lang: diff-OLD-NEW/diff.json + version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep + +The downloads are for development only: they go to the version store (~/code/horizon-parity/versions, +HORIZON_VERSIONS overrides it), never into the repository, and nothing from them is shipped. The pinned +version is not downloaded: its files are read where tools/lab.py prepared them (.local/). + +detect exits 0 when nothing is new, 10 for a new release and 11 for a new snapshot (with --snapshots). +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import zipfile + +try: + from . import lab +except ImportError: + import lab + +MANIFEST_URL = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json" +STORE = Path(os.environ.get("HORIZON_VERSIONS", Path.home() / "code" / "horizon-parity" / "versions")) +STATES = ["handshake", "status", "login", "configuration", "play"] +# How many changed JSON paths one changed file keeps in diff.json. +PATH_LIMIT = 50 + + +def safe_version(version): + if not re.fullmatch(r"[A-Za-z0-9_.-]+", version) or version in {".", ".."}: + raise ValueError(f"Unsafe version identifier: {version!r}") + return version + + +def fetch_json(url, sha1=None): + request = urllib.request.Request(url, headers={"User-Agent": "horizon/0.1"}) + with urllib.request.urlopen(request, timeout=60) as response: + data = response.read() + if sha1 is not None and hashlib.sha1(data).hexdigest() != sha1: + raise ValueError(f"{url}: sha1 mismatch") + return data + + +def pinned(): + return json.loads(lab.LOCK.read_text(encoding="utf-8"))["minecraft"]["version"] + + +# --- detect ------------------------------------------------------------------------------------------------ + +def news(manifest, pin, seen, snapshots=False): + """[(kind, version)] Mojang lists that are neither pinned nor seen before, newest kind first.""" + found = [] + for kind in ["release"] + (["snapshot"] if snapshots else []): + latest = manifest["latest"][kind] + if latest != pin and latest != seen.get(kind): + # The latest snapshot is the latest release right after a release: report it once. + if kind == "snapshot" and latest == manifest["latest"]["release"]: + continue + found.append((kind, latest)) + return found + + +def detect(args): + manifest = json.loads(fetch_json(MANIFEST_URL)) + state_path = STORE / "detect-state.json" + seen = json.loads(state_path.read_text()) if state_path.exists() else {} + found = news(manifest, pinned(), seen, args.snapshots) + if not found: + print(f"Nothing new: latest release {manifest['latest']['release']}, Horizon pins {pinned()}") + return 0 + for kind, version in found: + print(f"New {kind}: {version} (Horizon pins {pinned()})") + seen[kind] = version + if not args.dry_run: + lab.write_json(state_path, seen) + return 10 if found[0][0] == "release" else 11 + + +# --- prepare ----------------------------------------------------------------------------------------------- + +def version_dir(version): + return STORE / safe_version(version) + + +def paths(version): + """The prepared files of a version: {client, server, server_game, reports, version_json}.""" + record = version_dir(version) / "prepared.json" + if not record.exists(): + raise SystemExit(f"{version} is not prepared; run version_port.py prepare {version}") + return {key: Path(value) for key, value in json.loads(record.read_text())["paths"].items()} + + +def jar_facts(client_jar): + """The facts the game states about itself in the client jar's version.json.""" + with zipfile.ZipFile(client_jar) as archive: + info = json.loads(archive.read("version.json")) + pack = info.get("pack_version", {}) + return { + "id": info["id"], + "world_version": info["world_version"], + "protocol_version": info["protocol_version"], + "resource_pack": f"{pack.get('resource_major')}.{pack.get('resource_minor')}", + "data_pack": f"{pack.get('data_major')}.{pack.get('data_minor')}", + "java_version": info["java_version"], + "stable": info.get("stable"), + } + + +def unpack_game(server, destination, version): + """The server download is a bundler: take its game jar only (the libraries stay inside it).""" + with zipfile.ZipFile(server) as archive: + if "META-INF/versions.list" not in archive.namelist(): + shutil.copyfile(server, destination) + return destination + entries = list(lab.bundle_entries(archive, "META-INF/versions.list")) + if len(entries) != 1 or entries[0][1] != version: + raise ValueError(f"Expected one bundled server for {version}") + checksum, _, relative = entries[0] + lab.extract_verified(archive, f"META-INF/versions/{relative}", destination, checksum) + return destination + + +def java_command(major): + java = Path(os.environ["JAVA_HOME"]) / "bin" / "java" if os.environ.get("JAVA_HOME") else shutil.which("java") + if not java: + raise SystemExit("Java not found: set JAVA_HOME or put java on the PATH") + result = subprocess.run([str(java), "-version"], capture_output=True, text=True) + found = re.search(r'version "(\d+)', result.stderr + result.stdout) + if not found or int(found[1]) < major: + raise SystemExit(f"This version needs Java {major}; {java} is {found[1] if found else 'unknown'}. " + f"Install JDK {major} (a fleet toolchain change) and rerun.") + return str(java) + + +def run_datagen(server, reports, major): + """vanilla's data generator, --reports only. The bundler unpacks its libraries into a scratch folder.""" + scratch = Path(tempfile.mkdtemp(prefix=".datagen-", dir=STORE)) + out = scratch / "out" + try: + log = scratch / "datagen.log" + with log.open("w") as stream: + result = subprocess.run([java_command(major), "-DbundlerMainClass=net.minecraft.data.Main", "-jar", + str(server), "--reports", "--output", str(out)], + cwd=scratch, stdout=stream, stderr=subprocess.STDOUT) + if result.returncode: + raise SystemExit(f"The data generator failed:\n{log.read_text()[-3000:]}") + if reports.exists(): + shutil.rmtree(reports) + shutil.move(str(out / "reports"), reports) + finally: + shutil.rmtree(scratch, ignore_errors=True) + + +def prepare_pinned(version, directory): + """The pinned version's files are already in .local: record where, don't copy them.""" + game = lab.LOCAL / "minecraft" / version + found = { + "client": game / "jars" / "client.jar", + "server": game / "jars" / "server.jar", + "server_game": game / "jars" / "server-game.jar", + "reports": lab.LOCAL / "datagen" / "out" / "reports", + "version_json": game / "manifest.json", + } + missing = [str(p) for p in found.values() if not p.exists()] + if missing: + raise SystemExit("The pinned version is not prepared in .local (tools/lab.py prepare, horizon-prepare): " + + ", ".join(missing)) + return {key: path.resolve() for key, path in found.items()} + + +def prepare_download(version, directory): + manifest = json.loads(fetch_json(MANIFEST_URL)) + entry = next((v for v in manifest["versions"] if v["id"] == version), None) + if entry is None: + raise SystemExit(f"Mojang's manifest has no version {version}") + version_json = directory / "version.json" + if not (version_json.exists() and lab.digest(version_json, "sha1") == entry["sha1"]): + version_json.write_bytes(fetch_json(entry["url"], entry["sha1"])) + meta = json.loads(version_json.read_text()) + if "client_mappings" in meta["downloads"]: + raise SystemExit(f"{version} is obfuscated; the pipeline handles unobfuscated releases (26.1 and later)") + jars = directory / "jars" + found = {"version_json": version_json} + for side in ["client", "server"]: + download = meta["downloads"][side] + print(f"Fetching the {version} {side} jar ({download['size'] // 1_000_000} MB)", flush=True) + found[side] = lab.download(download["url"], jars / f"{side}.jar", download["sha1"]) + found["server_game"] = unpack_game(found["server"], jars / "server-game.jar", version) + found["reports"] = directory / "datagen" / "reports" + if not found["reports"].exists(): + print(f"Running the {version} data generator", flush=True) + run_datagen(found["server"], found["reports"], meta["javaVersion"]["majorVersion"]) + return found + + +def prepare(args): + version = safe_version(args.version) + directory = version_dir(version) + directory.mkdir(parents=True, exist_ok=True) + found = (prepare_pinned if version == pinned() and not args.download else prepare_download)(version, directory) + facts = jar_facts(found["client"]) + if facts["id"] != version: + raise ValueError(f"The client jar says it is {facts['id']}, not {version}") + lab.write_json(directory / "facts.json", facts) + lab.write_json(directory / "prepared.json", { + "version": version, "pinned": version == pinned() and not args.download, + "paths": {key: str(path) for key, path in found.items()}, + }) + print(json.dumps(facts, indent=2)) + return 0 + + +# --- diff -------------------------------------------------------------------------------------------------- + +def clip(value, limit=200): + text = json.dumps(value, sort_keys=True) + return value if len(text) <= limit else text[:limit] + "..." + + +def json_diff(old, new, path="", limit=PATH_LIMIT): + """[(path, old, new)] where two JSON values differ; lists of different lengths count as one change.""" + changes = [] + + def walk(a, b, at): + if len(changes) >= limit or a == b: + return + if isinstance(a, dict) and isinstance(b, dict): + for key in sorted(a.keys() | b.keys()): + walk(a.get(key), b.get(key), f"{at}/{key}") + elif isinstance(a, list) and isinstance(b, list) and len(a) == len(b): + for index, (x, y) in enumerate(zip(a, b)): + walk(x, y, f"{at}[{index}]") + else: + changes.append((at or "/", clip(a), clip(b))) + + walk(old, new, path) + return changes + + +def keyed_diff(old, new): + """Added, removed and common keys of two mappings.""" + return sorted(new.keys() - old.keys()), sorted(old.keys() - new.keys()), sorted(old.keys() & new.keys()) + + +def diff_facts(old, new): + return {key: [old.get(key), new.get(key)] for key in sorted(old.keys() | new.keys()) + if key != "id" and old.get(key) != new.get(key)} + + +def diff_packets(old, new): + """Per state and direction: packets added, removed and renumbered (the ids are on the wire).""" + result = {} + for state in sorted(old.keys() | new.keys(), key=lambda s: (STATES.index(s) if s in STATES else 99, s)): + for direction in sorted(old.get(state, {}).keys() | new.get(state, {}).keys()): + a = {name: e["protocol_id"] for name, e in old.get(state, {}).get(direction, {}).items()} + b = {name: e["protocol_id"] for name, e in new.get(state, {}).get(direction, {}).items()} + added, removed, common = keyed_diff(a, b) + renumbered = {name: [a[name], b[name]] for name in common if a[name] != b[name]} + if added or removed or renumbered: + result[f"{state}/{direction}"] = { + "added": {name: b[name] for name in added}, "removed": {name: a[name] for name in removed}, + "renumbered": renumbered, "count": [len(a), len(b)]} + return result + + +def diff_registries(old, new): + """Registries added and removed; per registry its entries added and removed, and how many common + entries changed protocol id (the network sends those ids).""" + added, removed, common = keyed_diff(old, new) + result = {"added": {name: sorted(new[name].get("entries", {})) for name in added}, "removed": removed, + "changed": {}} + for name in common: + a, b = old[name].get("entries", {}), new[name].get("entries", {}) + entries_added, entries_removed, shared = keyed_diff(a, b) + shifted = [e for e in shared if a[e].get("protocol_id") != b[e].get("protocol_id")] + change = {} + if entries_added: + change["added"] = entries_added + if entries_removed: + change["removed"] = entries_removed + if shifted: + change["ids_shifted"] = len(shifted) + if old[name].get("default") != new[name].get("default"): + change["default"] = [old[name].get("default"), new[name].get("default")] + if change: + result["changed"][name] = change + return result + + +def diff_blocks(old, new): + """Blocks added and removed; per block property changes, state counts and default state; and the + global block-state ids (the network and the chunk palette use them).""" + added, removed, common = keyed_diff(old, new) + # A field one report has for no block at all is a change of the report's format, not of every block. + fields = [{key for block in side.values() for key in block} for side in (old, new)] + format_change = sorted(fields[0] ^ fields[1]) + + def default(block): + return next((s.get("properties", {}) for s in block["states"] if s.get("default")), {}) + + def first_id(block): + return min(s["id"] for s in block["states"]) + + changed = {} + for name in common: + a, b = old[name], new[name] + change = {} + if a.get("properties", {}) != b.get("properties", {}): + change["properties"] = json_diff(a.get("properties", {}), b.get("properties", {})) + if len(a["states"]) != len(b["states"]): + change["states"] = [len(a["states"]), len(b["states"])] + if default(a) != default(b): + change["default"] = [default(a), default(b)] + if a.get("definition") != b.get("definition") and "definition" not in format_change: + change["definition"] = json_diff(a.get("definition"), b.get("definition")) + if change: + changed[name] = change + shifted = [name for name in common if first_id(old[name]) != first_id(new[name])] + first = min(shifted, key=lambda n: first_id(new[n]), default=None) + return {"added": added, "removed": removed, "changed": changed, "format": format_change, + "state_count": [sum(len(b["states"]) for b in old.values()), sum(len(b["states"]) for b in new.values())], + "state_ids_shifted": len(shifted), "first_shifted": first} + + +def diff_items(old, new): + """Items added and removed; per item the default components added, removed and changed.""" + added, removed, common = keyed_diff(old, new) + changed = {} + for name in common: + a, b = old[name], new[name] + components_added, components_removed, shared = keyed_diff(a, b) + values = {c: json_diff(a[c], b[c], limit=10) for c in shared if a[c] != b[c]} + if components_added or components_removed or values: + changed[name] = {"added": components_added, "removed": components_removed, "changed": values} + components = {} + for change in changed.values(): + for label in ("added", "removed", "changed"): + for component in change[label]: + components.setdefault(component, {"added": 0, "removed": 0, "changed": 0})[label] += 1 + return {"added": added, "removed": removed, "changed": changed, "components": components} + + +def command_nodes(tree): + """Every node of a commands report by its path ("execute as targets"), with what defines it.""" + nodes = {} + + def walk(node, path): + for name, child in node.get("children", {}).items(): + at = f"{path} {name}".strip() + nodes[at] = {k: child[k] for k in ("type", "parser", "properties", "executable", "redirect", "permissions") + if k in child} + walk(child, at) + + walk(tree, "") + return nodes + + +def diff_commands(old, new): + a, b = command_nodes(old), command_nodes(new) + added, removed, common = keyed_diff(a, b) + return {"added": added, "removed": removed, + "changed": {path: json_diff(a[path], b[path], limit=10) for path in common if a[path] != b[path]}} + + +def jar_entries(jar, prefix): + """{path: crc} of a jar's files under prefix.""" + with zipfile.ZipFile(jar) as archive: + return {i.filename: i.CRC for i in archive.infolist() if i.filename.startswith(prefix) and not i.is_dir()} + + +def content_kind(path): + """data/minecraft/worldgen/biome/x.json -> worldgen/biome; assets/minecraft/models/block/x.json -> models/block.""" + parts = path.split("/")[2:-1] + if parts and parts[0] in {"worldgen", "tags"} and len(parts) > 1: + return "/".join(parts[:2]) + return parts[0] if parts else "(root)" + + +def diff_contents(old_jar, new_jar, prefix): + """Files under prefix added, removed and changed (by CRC), by kind; changed JSON files with their paths.""" + a, b = jar_entries(old_jar, prefix), jar_entries(new_jar, prefix) + added, removed, common = keyed_diff(a, b) + changed = [p for p in common if a[p] != b[p]] + kinds = {} + for label, names in [("added", added), ("removed", removed), ("changed", changed)]: + for name in names: + kinds.setdefault(content_kind(name), {"added": [], "removed": [], "changed": []})[label].append(name) + details = {} + with zipfile.ZipFile(old_jar) as left, zipfile.ZipFile(new_jar) as right: + for name in changed: + if name.endswith(".json") and not name.endswith("/lang/en_us.json"): + try: + details[name] = json_diff(json.loads(left.read(name)), json.loads(right.read(name))) + except ValueError: + pass + return {"kinds": dict(sorted(kinds.items())), "json": details, + "counts": {"added": len(added), "removed": len(removed), "changed": len(changed)}} + + +def diff_lang(old_jar, new_jar, name="assets/minecraft/lang/en_us.json"): + with zipfile.ZipFile(old_jar) as left, zipfile.ZipFile(new_jar) as right: + a, b = json.loads(left.read(name)), json.loads(right.read(name)) + added, removed, common = keyed_diff(a, b) + return {"added": added, "removed": removed, "changed": [k for k in common if a[k] != b[k]]} + + +def read_reports(reports): + items = {} + for path in sorted((reports / "minecraft" / "components" / "item").glob("*.json")): + items[f"minecraft:{path.stem}"] = json.loads(path.read_text())["components"] + return { + "packets": json.loads((reports / "packets.json").read_text()), + "registries": json.loads((reports / "registries.json").read_text()), + "blocks": json.loads((reports / "blocks.json").read_text()), + "commands": json.loads((reports / "commands.json").read_text()), + "items": items, + } + + +def diff_versions(old_paths, new_paths, old_facts, new_facts): + a, b = read_reports(old_paths["reports"]), read_reports(new_paths["reports"]) + return { + "facts": diff_facts(old_facts, new_facts), + "packets": diff_packets(a["packets"], b["packets"]), + "registries": diff_registries(a["registries"], b["registries"]), + "blocks": diff_blocks(a["blocks"], b["blocks"]), + "items": diff_items(a["items"], b["items"]), + "commands": diff_commands(a["commands"], b["commands"]), + "data": diff_contents(old_paths["client"], new_paths["client"], "data/minecraft/"), + "assets": diff_contents(old_paths["client"], new_paths["client"], "assets/minecraft/"), + "lang": diff_lang(old_paths["client"], new_paths["client"]), + } + + +def summary(old, new, result): + """A few lines a person reads first.""" + lines = [f"Minecraft {old} -> {new}"] + for key, (a, b) in result["facts"].items(): + lines.append(f" {key}: {a} -> {b}") + for flow, change in result["packets"].items(): + lines.append(f" packets {flow}: {change['count'][0]} -> {change['count'][1]}, +{len(change['added'])} " + f"-{len(change['removed'])}, {len(change['renumbered'])} renumbered") + registries = result["registries"] + lines.append(f" registries: +{len(registries['added'])} -{len(registries['removed'])}, " + f"{len(registries['changed'])} changed") + for name, change in registries["changed"].items(): + lines.append(f" {name}: +{len(change.get('added', []))} -{len(change.get('removed', []))}" + + (f", {change['ids_shifted']} ids shifted" if change.get("ids_shifted") else "")) + blocks = result["blocks"] + lines.append(f" blocks: +{len(blocks['added'])} -{len(blocks['removed'])}, {len(blocks['changed'])} changed, " + f"states {blocks['state_count'][0]} -> {blocks['state_count'][1]}, " + f"{blocks['state_ids_shifted']} blocks' state ids shifted") + if blocks["format"]: + lines.append(f" report format: fields {', '.join(blocks['format'])} appear or disappear") + items = result["items"] + lines.append(f" items: +{len(items['added'])} -{len(items['removed'])}, {len(items['changed'])} with " + f"changed default components") + ranked = sorted(items["components"].items(), key=lambda kv: -sum(kv[1].values())) + for component, counts in ranked[:10]: + lines.append(f" {component}: " + ", ".join(f"{k} on {n}" for k, n in counts.items() if n)) + commands = result["commands"] + lines.append(f" command nodes: +{len(commands['added'])} -{len(commands['removed'])}, " + f"{len(commands['changed'])} changed") + for key in ["data", "assets"]: + counts = result[key]["counts"] + lines.append(f" {key} files: +{counts['added']} -{counts['removed']} ~{counts['changed']}") + lang = result["lang"] + lines.append(f" en_us keys: +{len(lang['added'])} -{len(lang['removed'])} ~{len(lang['changed'])}") + return "\n".join(lines) + + +def diff_dir(old, new): + return STORE / f"diff-{safe_version(old)}-{safe_version(new)}" + + +def diff(args): + facts = {v: json.loads((version_dir(v) / "facts.json").read_text()) for v in (args.old, args.new)} + result = diff_versions(paths(args.old), paths(args.new), facts[args.old], facts[args.new]) + directory = diff_dir(args.old, args.new) + lab.write_json(directory / "diff.json", result) + print(summary(args.old, args.new, result)) + print(f"\n{directory / 'diff.json'}") + return 0 + + +# --- gc ---------------------------------------------------------------------------------------------------- + +def to_remove(entries, pin, keep): + """Version store entries to delete: every version but the pinned one, --keep and the newest two others; + diffs whose versions are gone; scratch folders. entries: [(name, mtime, is_version)].""" + versions = sorted(((m, n) for n, m, is_version in entries if is_version and n not in keep | {pin}), reverse=True) + kept = keep | {pin} | {n for _, n in versions[:2]} + remove = [] + for name, _, is_version in entries: + if name.startswith("."): + remove.append(name) + elif is_version and name not in kept: + remove.append(name) + elif name.startswith("diff-") and not any(name == f"diff-{a}-{b}" for a in kept for b in kept): + remove.append(name) + return sorted(remove) + + +def gc(args): + if not STORE.exists(): + return 0 + entries = [(p.name, p.stat().st_mtime, (p / "prepared.json").exists()) for p in STORE.iterdir() if p.is_dir()] + for name in to_remove(entries, pinned(), set(args.keep)): + print(f"Removing {STORE / name}") + if not args.dry_run: + shutil.rmtree(STORE / name) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("detect", help="what Mojang released that Horizon doesn't pin") + command.add_argument("--snapshots", action="store_true") + command.add_argument("--dry-run", action="store_true", help="don't remember what was found") + command = commands.add_parser("prepare", help="download, verify and run the data generator") + command.add_argument("version") + command.add_argument("--download", action="store_true", help="download even the pinned version") + command = commands.add_parser("diff", help="diff two prepared versions") + command.add_argument("old") + command.add_argument("new") + command = commands.add_parser("gc", help="delete versions and scratch the store doesn't need") + command.add_argument("--keep", nargs="*", default=[]) + command.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + STORE.mkdir(parents=True, exist_ok=True) + return {"detect": detect, "prepare": prepare, "diff": diff, "gc": gc}[args.command](args) + + +if __name__ == "__main__": + sys.exit(main())