version_port.py: detect, prepare and diff Minecraft releases
detect reads Mojang's manifest; prepare downloads and verifies a version into the version store (outside the repository), runs its data generator and reads the jar's own version facts; diff compares facts, packets, registries, blocks, item components, commands, data packs, assets and lang. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
@@ -260,10 +260,18 @@ The replay of 26.2 → 26.3 below measures how much that is for one release.
|
||||
|
||||
## Implementation status
|
||||
|
||||
```sh
|
||||
python3 tools/version_port.py detect --snapshots # exit 10: new release, 11: new snapshot
|
||||
python3 tools/version_port.py prepare 26.3 # the pinned version: indexes .local, downloads nothing
|
||||
python3 tools/version_port.py prepare 26.4 # downloads, verifies, runs the data generator (about 10 s)
|
||||
python3 tools/version_port.py diff 26.3 26.4 # summary; the full diff in versions/diff-26.3-26.4/diff.json
|
||||
python3 tools/version_port.py gc # keep the pinned version and the newest two others
|
||||
```
|
||||
|
||||
| Slice | Status |
|
||||
| --- | --- |
|
||||
| This design | this page |
|
||||
| `detect`, `prepare`, `diff` (facts, packets, registries, blocks, items, commands, data, assets) | next PR |
|
||||
| `detect`, `prepare`, `diff` (facts, packets, registries, blocks, items, commands, data, assets), `gc` | done |
|
||||
| Class index, class diff, `vanilla_map.json` | next PR |
|
||||
| Port report (`report`), `port` | next PR |
|
||||
| Changed-class decompilation, `regen`, `version_consts` refactor, candidate harness run, `publish`, launchd agent | later |
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
|
||||
from tools import version_port as port
|
||||
|
||||
|
||||
def jar(path, files):
|
||||
with zipfile.ZipFile(path, "w") as archive:
|
||||
for name, value in files.items():
|
||||
archive.writestr(name, value if isinstance(value, (str, bytes)) else json.dumps(value))
|
||||
return path
|
||||
|
||||
|
||||
class DetectTests(unittest.TestCase):
|
||||
MANIFEST = {"latest": {"release": "26.4", "snapshot": "26.5-snapshot-1"}}
|
||||
|
||||
def test_new_release_and_snapshot(self):
|
||||
self.assertEqual(port.news(self.MANIFEST, "26.3", {}), [("release", "26.4")])
|
||||
self.assertEqual(port.news(self.MANIFEST, "26.3", {}, snapshots=True),
|
||||
[("release", "26.4"), ("snapshot", "26.5-snapshot-1")])
|
||||
|
||||
def test_pinned_or_seen_is_not_news(self):
|
||||
self.assertEqual(port.news(self.MANIFEST, "26.4", {}), [])
|
||||
self.assertEqual(port.news(self.MANIFEST, "26.3", {"release": "26.4", "snapshot": "26.5-snapshot-1"},
|
||||
snapshots=True), [])
|
||||
|
||||
def test_a_release_is_not_also_reported_as_the_latest_snapshot(self):
|
||||
manifest = {"latest": {"release": "26.4", "snapshot": "26.4"}}
|
||||
self.assertEqual(port.news(manifest, "26.3", {}, snapshots=True), [("release", "26.4")])
|
||||
|
||||
def test_unsafe_versions_are_refused(self):
|
||||
for version in ["..", "26.4/../x", ""]:
|
||||
with self.assertRaises(ValueError):
|
||||
port.safe_version(version)
|
||||
|
||||
|
||||
class ReportDiffTests(unittest.TestCase):
|
||||
def test_json_diff_paths(self):
|
||||
self.assertEqual(port.json_diff({"a": 1, "b": [1, 2]}, {"a": 2, "b": [1, 3], "c": True}),
|
||||
[("/a", 1, 2), ("/b[1]", 2, 3), ("/c", None, True)])
|
||||
self.assertEqual(port.json_diff([1], [1, 2]), [("/", [1], [1, 2])])
|
||||
self.assertEqual(len(port.json_diff({str(i): i for i in range(9)}, {}, limit=3)), 3)
|
||||
|
||||
def test_packets_added_removed_renumbered(self):
|
||||
old = {"play": {"serverbound": {"minecraft:swing": {"protocol_id": 0}, "minecraft:use": {"protocol_id": 1}}}}
|
||||
new = {"play": {"serverbound": {"minecraft:punch": {"protocol_id": 0}, "minecraft:use": {"protocol_id": 2},
|
||||
"minecraft:x": {"protocol_id": 1}}}}
|
||||
self.assertEqual(port.diff_packets(old, new), {"play/serverbound": {
|
||||
"added": {"minecraft:punch": 0, "minecraft:x": 1}, "removed": {"minecraft:swing": 0},
|
||||
"renumbered": {"minecraft:use": [1, 2]}, "count": [2, 3]}})
|
||||
self.assertEqual(port.diff_packets(old, old), {})
|
||||
|
||||
def test_registries(self):
|
||||
def registry(*names, default=None):
|
||||
return {"default": default, "entries": {n: {"protocol_id": i} for i, n in enumerate(names)}}
|
||||
old = {"minecraft:item": registry("air", "stone"), "minecraft:gone": registry("a")}
|
||||
new = {"minecraft:item": registry("air", "dirt", "stone"), "minecraft:new": registry("b")}
|
||||
self.assertEqual(port.diff_registries(old, new), {
|
||||
"added": {"minecraft:new": ["b"]}, "removed": ["minecraft:gone"],
|
||||
"changed": {"minecraft:item": {"added": ["dirt"], "ids_shifted": 1}}})
|
||||
|
||||
def test_blocks_state_ids_and_format(self):
|
||||
def block(first, *values, default=0, **extra):
|
||||
states = [{"id": first + i, "properties": {"p": v}} for i, v in enumerate(values)]
|
||||
states[default]["default"] = True
|
||||
return {"properties": {"p": list(values)}, "states": states, **extra}
|
||||
old = {"a": block(0, "x", "y"), "b": block(2, "x")}
|
||||
new = {"a": block(0, "x", "y", "z", default=1, definition={}),
|
||||
"b": block(3, "x", definition={}), "c": block(4, "x", definition={})}
|
||||
result = port.diff_blocks(old, new)
|
||||
self.assertEqual(result["added"], ["c"])
|
||||
self.assertEqual(result["format"], ["definition"])
|
||||
self.assertEqual(set(result["changed"]), {"a"})
|
||||
self.assertEqual(result["changed"]["a"]["states"], [2, 3])
|
||||
self.assertEqual(result["changed"]["a"]["default"], [{"p": "x"}, {"p": "y"}])
|
||||
self.assertEqual((result["state_ids_shifted"], result["first_shifted"]), (1, "b"))
|
||||
|
||||
def test_items_by_component(self):
|
||||
old = {"stone": {"swing": {}, "name": "Stone"}, "dirt": {"swing": {}}}
|
||||
new = {"stone": {"attack": {}, "name": "Rock"}, "dirt": {"attack": {}}, "mud": {}}
|
||||
result = port.diff_items(old, new)
|
||||
self.assertEqual(result["added"], ["mud"])
|
||||
self.assertEqual(result["components"], {"attack": {"added": 2, "removed": 0, "changed": 0},
|
||||
"swing": {"added": 0, "removed": 2, "changed": 0},
|
||||
"name": {"added": 0, "removed": 0, "changed": 1}})
|
||||
|
||||
def test_command_paths(self):
|
||||
def literal(**children):
|
||||
return {"type": "literal", "children": children} if children else {"type": "literal", "executable": True}
|
||||
old = {"type": "root", "children": {"data": literal(get=literal(block=literal(targetPos=literal())))}}
|
||||
new = {"type": "root", "children": {"data": literal(get=literal(block=literal(pos=literal()))),
|
||||
"compute": literal()}}
|
||||
result = port.diff_commands(old, new)
|
||||
self.assertEqual(result["added"], ["compute", "data get block pos"])
|
||||
self.assertEqual(result["removed"], ["data get block targetPos"])
|
||||
|
||||
|
||||
class JarDiffTests(unittest.TestCase):
|
||||
def test_contents_lang_and_facts(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
common = {"data/minecraft/tags/block/logs.json": {"values": ["oak"]}}
|
||||
old = jar(root / "old.jar", {**common,
|
||||
"data/minecraft/recipe/a.json": {"result": "a", "count": 1},
|
||||
"data/minecraft/recipe/gone.json": {},
|
||||
"assets/minecraft/lang/en_us.json": {"k": "K", "same": "S"}})
|
||||
new = jar(root / "new.jar", {**common,
|
||||
"data/minecraft/recipe/a.json": {"result": "a", "count": 2},
|
||||
"data/minecraft/worldgen/biome/new.json": {},
|
||||
"assets/minecraft/lang/en_us.json": {"k": "K2", "same": "S", "n": "N"},
|
||||
"version.json": {"id": "26.4", "world_version": 5100,
|
||||
"protocol_version": 778, "java_version": 25,
|
||||
"pack_version": {"resource_major": 98, "resource_minor": 0,
|
||||
"data_major": 122, "data_minor": 0}}})
|
||||
data = port.diff_contents(old, new, "data/minecraft/")
|
||||
self.assertEqual(data["counts"], {"added": 1, "removed": 1, "changed": 1})
|
||||
self.assertEqual(data["kinds"]["recipe"], {"added": [], "removed": ["data/minecraft/recipe/gone.json"],
|
||||
"changed": ["data/minecraft/recipe/a.json"]})
|
||||
self.assertEqual(data["kinds"]["worldgen/biome"]["added"], ["data/minecraft/worldgen/biome/new.json"])
|
||||
self.assertEqual(data["json"], {"data/minecraft/recipe/a.json": [("/count", 1, 2)]})
|
||||
self.assertEqual(port.diff_lang(old, new), {"added": ["n"], "removed": [], "changed": ["k"]})
|
||||
facts = port.jar_facts(new)
|
||||
self.assertEqual((facts["world_version"], facts["protocol_version"], facts["data_pack"]),
|
||||
(5100, 778, "122.0"))
|
||||
self.assertEqual(port.diff_facts({"id": "a", "protocol_version": 777}, {"id": "b", "protocol_version": 778}),
|
||||
{"protocol_version": [777, 778]})
|
||||
|
||||
|
||||
class GcTests(unittest.TestCase):
|
||||
def test_keeps_pinned_newest_two_and_their_diffs(self):
|
||||
entries = [("26.1", 1, True), ("26.2", 2, True), ("26.3", 3, True), ("26.4-snapshot-1", 4, True),
|
||||
("26.4-snapshot-2", 5, True), ("diff-26.2-26.3", 6, False),
|
||||
("diff-26.3-26.4-snapshot-2", 7, False), (".datagen-x", 8, False), ("detect-state.json", 9, False)]
|
||||
self.assertEqual(port.to_remove(entries, "26.3", set()),
|
||||
[".datagen-x", "26.1", "26.2", "diff-26.2-26.3"])
|
||||
self.assertEqual(port.to_remove(entries, "26.3", {"26.1"}),
|
||||
[".datagen-x", "26.2", "diff-26.2-26.3"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,550 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Port a new Minecraft release to Horizon: detect it, prepare it, diff it against the pinned one
|
||||
(docs/version-ports.md).
|
||||
|
||||
version_port.py detect [--snapshots] what Mojang released that Horizon doesn't pin
|
||||
version_port.py prepare VERSION download, verify, run the data generator, read its facts
|
||||
version_port.py diff OLD NEW facts, packets, registries, blocks, items, commands,
|
||||
data packs, assets and lang: diff-OLD-NEW/diff.json
|
||||
version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep
|
||||
|
||||
The downloads are for development only: they go to the version store (~/code/horizon-parity/versions,
|
||||
HORIZON_VERSIONS overrides it), never into the repository, and nothing from them is shipped. The pinned
|
||||
version is not downloaded: its files are read where tools/lab.py prepared them (.local/).
|
||||
|
||||
detect exits 0 when nothing is new, 10 for a new release and 11 for a new snapshot (with --snapshots).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
import zipfile
|
||||
|
||||
try:
|
||||
from . import lab
|
||||
except ImportError:
|
||||
import lab
|
||||
|
||||
MANIFEST_URL = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"
|
||||
STORE = Path(os.environ.get("HORIZON_VERSIONS", Path.home() / "code" / "horizon-parity" / "versions"))
|
||||
STATES = ["handshake", "status", "login", "configuration", "play"]
|
||||
# How many changed JSON paths one changed file keeps in diff.json.
|
||||
PATH_LIMIT = 50
|
||||
|
||||
|
||||
def safe_version(version):
|
||||
if not re.fullmatch(r"[A-Za-z0-9_.-]+", version) or version in {".", ".."}:
|
||||
raise ValueError(f"Unsafe version identifier: {version!r}")
|
||||
return version
|
||||
|
||||
|
||||
def fetch_json(url, sha1=None):
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "horizon/0.1"})
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
data = response.read()
|
||||
if sha1 is not None and hashlib.sha1(data).hexdigest() != sha1:
|
||||
raise ValueError(f"{url}: sha1 mismatch")
|
||||
return data
|
||||
|
||||
|
||||
def pinned():
|
||||
return json.loads(lab.LOCK.read_text(encoding="utf-8"))["minecraft"]["version"]
|
||||
|
||||
|
||||
# --- detect ------------------------------------------------------------------------------------------------
|
||||
|
||||
def news(manifest, pin, seen, snapshots=False):
|
||||
"""[(kind, version)] Mojang lists that are neither pinned nor seen before, newest kind first."""
|
||||
found = []
|
||||
for kind in ["release"] + (["snapshot"] if snapshots else []):
|
||||
latest = manifest["latest"][kind]
|
||||
if latest != pin and latest != seen.get(kind):
|
||||
# The latest snapshot is the latest release right after a release: report it once.
|
||||
if kind == "snapshot" and latest == manifest["latest"]["release"]:
|
||||
continue
|
||||
found.append((kind, latest))
|
||||
return found
|
||||
|
||||
|
||||
def detect(args):
|
||||
manifest = json.loads(fetch_json(MANIFEST_URL))
|
||||
state_path = STORE / "detect-state.json"
|
||||
seen = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||
found = news(manifest, pinned(), seen, args.snapshots)
|
||||
if not found:
|
||||
print(f"Nothing new: latest release {manifest['latest']['release']}, Horizon pins {pinned()}")
|
||||
return 0
|
||||
for kind, version in found:
|
||||
print(f"New {kind}: {version} (Horizon pins {pinned()})")
|
||||
seen[kind] = version
|
||||
if not args.dry_run:
|
||||
lab.write_json(state_path, seen)
|
||||
return 10 if found[0][0] == "release" else 11
|
||||
|
||||
|
||||
# --- prepare -----------------------------------------------------------------------------------------------
|
||||
|
||||
def version_dir(version):
|
||||
return STORE / safe_version(version)
|
||||
|
||||
|
||||
def paths(version):
|
||||
"""The prepared files of a version: {client, server, server_game, reports, version_json}."""
|
||||
record = version_dir(version) / "prepared.json"
|
||||
if not record.exists():
|
||||
raise SystemExit(f"{version} is not prepared; run version_port.py prepare {version}")
|
||||
return {key: Path(value) for key, value in json.loads(record.read_text())["paths"].items()}
|
||||
|
||||
|
||||
def jar_facts(client_jar):
|
||||
"""The facts the game states about itself in the client jar's version.json."""
|
||||
with zipfile.ZipFile(client_jar) as archive:
|
||||
info = json.loads(archive.read("version.json"))
|
||||
pack = info.get("pack_version", {})
|
||||
return {
|
||||
"id": info["id"],
|
||||
"world_version": info["world_version"],
|
||||
"protocol_version": info["protocol_version"],
|
||||
"resource_pack": f"{pack.get('resource_major')}.{pack.get('resource_minor')}",
|
||||
"data_pack": f"{pack.get('data_major')}.{pack.get('data_minor')}",
|
||||
"java_version": info["java_version"],
|
||||
"stable": info.get("stable"),
|
||||
}
|
||||
|
||||
|
||||
def unpack_game(server, destination, version):
|
||||
"""The server download is a bundler: take its game jar only (the libraries stay inside it)."""
|
||||
with zipfile.ZipFile(server) as archive:
|
||||
if "META-INF/versions.list" not in archive.namelist():
|
||||
shutil.copyfile(server, destination)
|
||||
return destination
|
||||
entries = list(lab.bundle_entries(archive, "META-INF/versions.list"))
|
||||
if len(entries) != 1 or entries[0][1] != version:
|
||||
raise ValueError(f"Expected one bundled server for {version}")
|
||||
checksum, _, relative = entries[0]
|
||||
lab.extract_verified(archive, f"META-INF/versions/{relative}", destination, checksum)
|
||||
return destination
|
||||
|
||||
|
||||
def java_command(major):
|
||||
java = Path(os.environ["JAVA_HOME"]) / "bin" / "java" if os.environ.get("JAVA_HOME") else shutil.which("java")
|
||||
if not java:
|
||||
raise SystemExit("Java not found: set JAVA_HOME or put java on the PATH")
|
||||
result = subprocess.run([str(java), "-version"], capture_output=True, text=True)
|
||||
found = re.search(r'version "(\d+)', result.stderr + result.stdout)
|
||||
if not found or int(found[1]) < major:
|
||||
raise SystemExit(f"This version needs Java {major}; {java} is {found[1] if found else 'unknown'}. "
|
||||
f"Install JDK {major} (a fleet toolchain change) and rerun.")
|
||||
return str(java)
|
||||
|
||||
|
||||
def run_datagen(server, reports, major):
|
||||
"""vanilla's data generator, --reports only. The bundler unpacks its libraries into a scratch folder."""
|
||||
scratch = Path(tempfile.mkdtemp(prefix=".datagen-", dir=STORE))
|
||||
out = scratch / "out"
|
||||
try:
|
||||
log = scratch / "datagen.log"
|
||||
with log.open("w") as stream:
|
||||
result = subprocess.run([java_command(major), "-DbundlerMainClass=net.minecraft.data.Main", "-jar",
|
||||
str(server), "--reports", "--output", str(out)],
|
||||
cwd=scratch, stdout=stream, stderr=subprocess.STDOUT)
|
||||
if result.returncode:
|
||||
raise SystemExit(f"The data generator failed:\n{log.read_text()[-3000:]}")
|
||||
if reports.exists():
|
||||
shutil.rmtree(reports)
|
||||
shutil.move(str(out / "reports"), reports)
|
||||
finally:
|
||||
shutil.rmtree(scratch, ignore_errors=True)
|
||||
|
||||
|
||||
def prepare_pinned(version, directory):
|
||||
"""The pinned version's files are already in .local: record where, don't copy them."""
|
||||
game = lab.LOCAL / "minecraft" / version
|
||||
found = {
|
||||
"client": game / "jars" / "client.jar",
|
||||
"server": game / "jars" / "server.jar",
|
||||
"server_game": game / "jars" / "server-game.jar",
|
||||
"reports": lab.LOCAL / "datagen" / "out" / "reports",
|
||||
"version_json": game / "manifest.json",
|
||||
}
|
||||
missing = [str(p) for p in found.values() if not p.exists()]
|
||||
if missing:
|
||||
raise SystemExit("The pinned version is not prepared in .local (tools/lab.py prepare, horizon-prepare): "
|
||||
+ ", ".join(missing))
|
||||
return {key: path.resolve() for key, path in found.items()}
|
||||
|
||||
|
||||
def prepare_download(version, directory):
|
||||
manifest = json.loads(fetch_json(MANIFEST_URL))
|
||||
entry = next((v for v in manifest["versions"] if v["id"] == version), None)
|
||||
if entry is None:
|
||||
raise SystemExit(f"Mojang's manifest has no version {version}")
|
||||
version_json = directory / "version.json"
|
||||
if not (version_json.exists() and lab.digest(version_json, "sha1") == entry["sha1"]):
|
||||
version_json.write_bytes(fetch_json(entry["url"], entry["sha1"]))
|
||||
meta = json.loads(version_json.read_text())
|
||||
if "client_mappings" in meta["downloads"]:
|
||||
raise SystemExit(f"{version} is obfuscated; the pipeline handles unobfuscated releases (26.1 and later)")
|
||||
jars = directory / "jars"
|
||||
found = {"version_json": version_json}
|
||||
for side in ["client", "server"]:
|
||||
download = meta["downloads"][side]
|
||||
print(f"Fetching the {version} {side} jar ({download['size'] // 1_000_000} MB)", flush=True)
|
||||
found[side] = lab.download(download["url"], jars / f"{side}.jar", download["sha1"])
|
||||
found["server_game"] = unpack_game(found["server"], jars / "server-game.jar", version)
|
||||
found["reports"] = directory / "datagen" / "reports"
|
||||
if not found["reports"].exists():
|
||||
print(f"Running the {version} data generator", flush=True)
|
||||
run_datagen(found["server"], found["reports"], meta["javaVersion"]["majorVersion"])
|
||||
return found
|
||||
|
||||
|
||||
def prepare(args):
|
||||
version = safe_version(args.version)
|
||||
directory = version_dir(version)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
found = (prepare_pinned if version == pinned() and not args.download else prepare_download)(version, directory)
|
||||
facts = jar_facts(found["client"])
|
||||
if facts["id"] != version:
|
||||
raise ValueError(f"The client jar says it is {facts['id']}, not {version}")
|
||||
lab.write_json(directory / "facts.json", facts)
|
||||
lab.write_json(directory / "prepared.json", {
|
||||
"version": version, "pinned": version == pinned() and not args.download,
|
||||
"paths": {key: str(path) for key, path in found.items()},
|
||||
})
|
||||
print(json.dumps(facts, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
# --- diff --------------------------------------------------------------------------------------------------
|
||||
|
||||
def clip(value, limit=200):
|
||||
text = json.dumps(value, sort_keys=True)
|
||||
return value if len(text) <= limit else text[:limit] + "..."
|
||||
|
||||
|
||||
def json_diff(old, new, path="", limit=PATH_LIMIT):
|
||||
"""[(path, old, new)] where two JSON values differ; lists of different lengths count as one change."""
|
||||
changes = []
|
||||
|
||||
def walk(a, b, at):
|
||||
if len(changes) >= limit or a == b:
|
||||
return
|
||||
if isinstance(a, dict) and isinstance(b, dict):
|
||||
for key in sorted(a.keys() | b.keys()):
|
||||
walk(a.get(key), b.get(key), f"{at}/{key}")
|
||||
elif isinstance(a, list) and isinstance(b, list) and len(a) == len(b):
|
||||
for index, (x, y) in enumerate(zip(a, b)):
|
||||
walk(x, y, f"{at}[{index}]")
|
||||
else:
|
||||
changes.append((at or "/", clip(a), clip(b)))
|
||||
|
||||
walk(old, new, path)
|
||||
return changes
|
||||
|
||||
|
||||
def keyed_diff(old, new):
|
||||
"""Added, removed and common keys of two mappings."""
|
||||
return sorted(new.keys() - old.keys()), sorted(old.keys() - new.keys()), sorted(old.keys() & new.keys())
|
||||
|
||||
|
||||
def diff_facts(old, new):
|
||||
return {key: [old.get(key), new.get(key)] for key in sorted(old.keys() | new.keys())
|
||||
if key != "id" and old.get(key) != new.get(key)}
|
||||
|
||||
|
||||
def diff_packets(old, new):
|
||||
"""Per state and direction: packets added, removed and renumbered (the ids are on the wire)."""
|
||||
result = {}
|
||||
for state in sorted(old.keys() | new.keys(), key=lambda s: (STATES.index(s) if s in STATES else 99, s)):
|
||||
for direction in sorted(old.get(state, {}).keys() | new.get(state, {}).keys()):
|
||||
a = {name: e["protocol_id"] for name, e in old.get(state, {}).get(direction, {}).items()}
|
||||
b = {name: e["protocol_id"] for name, e in new.get(state, {}).get(direction, {}).items()}
|
||||
added, removed, common = keyed_diff(a, b)
|
||||
renumbered = {name: [a[name], b[name]] for name in common if a[name] != b[name]}
|
||||
if added or removed or renumbered:
|
||||
result[f"{state}/{direction}"] = {
|
||||
"added": {name: b[name] for name in added}, "removed": {name: a[name] for name in removed},
|
||||
"renumbered": renumbered, "count": [len(a), len(b)]}
|
||||
return result
|
||||
|
||||
|
||||
def diff_registries(old, new):
|
||||
"""Registries added and removed; per registry its entries added and removed, and how many common
|
||||
entries changed protocol id (the network sends those ids)."""
|
||||
added, removed, common = keyed_diff(old, new)
|
||||
result = {"added": {name: sorted(new[name].get("entries", {})) for name in added}, "removed": removed,
|
||||
"changed": {}}
|
||||
for name in common:
|
||||
a, b = old[name].get("entries", {}), new[name].get("entries", {})
|
||||
entries_added, entries_removed, shared = keyed_diff(a, b)
|
||||
shifted = [e for e in shared if a[e].get("protocol_id") != b[e].get("protocol_id")]
|
||||
change = {}
|
||||
if entries_added:
|
||||
change["added"] = entries_added
|
||||
if entries_removed:
|
||||
change["removed"] = entries_removed
|
||||
if shifted:
|
||||
change["ids_shifted"] = len(shifted)
|
||||
if old[name].get("default") != new[name].get("default"):
|
||||
change["default"] = [old[name].get("default"), new[name].get("default")]
|
||||
if change:
|
||||
result["changed"][name] = change
|
||||
return result
|
||||
|
||||
|
||||
def diff_blocks(old, new):
|
||||
"""Blocks added and removed; per block property changes, state counts and default state; and the
|
||||
global block-state ids (the network and the chunk palette use them)."""
|
||||
added, removed, common = keyed_diff(old, new)
|
||||
# A field one report has for no block at all is a change of the report's format, not of every block.
|
||||
fields = [{key for block in side.values() for key in block} for side in (old, new)]
|
||||
format_change = sorted(fields[0] ^ fields[1])
|
||||
|
||||
def default(block):
|
||||
return next((s.get("properties", {}) for s in block["states"] if s.get("default")), {})
|
||||
|
||||
def first_id(block):
|
||||
return min(s["id"] for s in block["states"])
|
||||
|
||||
changed = {}
|
||||
for name in common:
|
||||
a, b = old[name], new[name]
|
||||
change = {}
|
||||
if a.get("properties", {}) != b.get("properties", {}):
|
||||
change["properties"] = json_diff(a.get("properties", {}), b.get("properties", {}))
|
||||
if len(a["states"]) != len(b["states"]):
|
||||
change["states"] = [len(a["states"]), len(b["states"])]
|
||||
if default(a) != default(b):
|
||||
change["default"] = [default(a), default(b)]
|
||||
if a.get("definition") != b.get("definition") and "definition" not in format_change:
|
||||
change["definition"] = json_diff(a.get("definition"), b.get("definition"))
|
||||
if change:
|
||||
changed[name] = change
|
||||
shifted = [name for name in common if first_id(old[name]) != first_id(new[name])]
|
||||
first = min(shifted, key=lambda n: first_id(new[n]), default=None)
|
||||
return {"added": added, "removed": removed, "changed": changed, "format": format_change,
|
||||
"state_count": [sum(len(b["states"]) for b in old.values()), sum(len(b["states"]) for b in new.values())],
|
||||
"state_ids_shifted": len(shifted), "first_shifted": first}
|
||||
|
||||
|
||||
def diff_items(old, new):
|
||||
"""Items added and removed; per item the default components added, removed and changed."""
|
||||
added, removed, common = keyed_diff(old, new)
|
||||
changed = {}
|
||||
for name in common:
|
||||
a, b = old[name], new[name]
|
||||
components_added, components_removed, shared = keyed_diff(a, b)
|
||||
values = {c: json_diff(a[c], b[c], limit=10) for c in shared if a[c] != b[c]}
|
||||
if components_added or components_removed or values:
|
||||
changed[name] = {"added": components_added, "removed": components_removed, "changed": values}
|
||||
components = {}
|
||||
for change in changed.values():
|
||||
for label in ("added", "removed", "changed"):
|
||||
for component in change[label]:
|
||||
components.setdefault(component, {"added": 0, "removed": 0, "changed": 0})[label] += 1
|
||||
return {"added": added, "removed": removed, "changed": changed, "components": components}
|
||||
|
||||
|
||||
def command_nodes(tree):
|
||||
"""Every node of a commands report by its path ("execute as targets"), with what defines it."""
|
||||
nodes = {}
|
||||
|
||||
def walk(node, path):
|
||||
for name, child in node.get("children", {}).items():
|
||||
at = f"{path} {name}".strip()
|
||||
nodes[at] = {k: child[k] for k in ("type", "parser", "properties", "executable", "redirect", "permissions")
|
||||
if k in child}
|
||||
walk(child, at)
|
||||
|
||||
walk(tree, "")
|
||||
return nodes
|
||||
|
||||
|
||||
def diff_commands(old, new):
|
||||
a, b = command_nodes(old), command_nodes(new)
|
||||
added, removed, common = keyed_diff(a, b)
|
||||
return {"added": added, "removed": removed,
|
||||
"changed": {path: json_diff(a[path], b[path], limit=10) for path in common if a[path] != b[path]}}
|
||||
|
||||
|
||||
def jar_entries(jar, prefix):
|
||||
"""{path: crc} of a jar's files under prefix."""
|
||||
with zipfile.ZipFile(jar) as archive:
|
||||
return {i.filename: i.CRC for i in archive.infolist() if i.filename.startswith(prefix) and not i.is_dir()}
|
||||
|
||||
|
||||
def content_kind(path):
|
||||
"""data/minecraft/worldgen/biome/x.json -> worldgen/biome; assets/minecraft/models/block/x.json -> models/block."""
|
||||
parts = path.split("/")[2:-1]
|
||||
if parts and parts[0] in {"worldgen", "tags"} and len(parts) > 1:
|
||||
return "/".join(parts[:2])
|
||||
return parts[0] if parts else "(root)"
|
||||
|
||||
|
||||
def diff_contents(old_jar, new_jar, prefix):
|
||||
"""Files under prefix added, removed and changed (by CRC), by kind; changed JSON files with their paths."""
|
||||
a, b = jar_entries(old_jar, prefix), jar_entries(new_jar, prefix)
|
||||
added, removed, common = keyed_diff(a, b)
|
||||
changed = [p for p in common if a[p] != b[p]]
|
||||
kinds = {}
|
||||
for label, names in [("added", added), ("removed", removed), ("changed", changed)]:
|
||||
for name in names:
|
||||
kinds.setdefault(content_kind(name), {"added": [], "removed": [], "changed": []})[label].append(name)
|
||||
details = {}
|
||||
with zipfile.ZipFile(old_jar) as left, zipfile.ZipFile(new_jar) as right:
|
||||
for name in changed:
|
||||
if name.endswith(".json") and not name.endswith("/lang/en_us.json"):
|
||||
try:
|
||||
details[name] = json_diff(json.loads(left.read(name)), json.loads(right.read(name)))
|
||||
except ValueError:
|
||||
pass
|
||||
return {"kinds": dict(sorted(kinds.items())), "json": details,
|
||||
"counts": {"added": len(added), "removed": len(removed), "changed": len(changed)}}
|
||||
|
||||
|
||||
def diff_lang(old_jar, new_jar, name="assets/minecraft/lang/en_us.json"):
|
||||
with zipfile.ZipFile(old_jar) as left, zipfile.ZipFile(new_jar) as right:
|
||||
a, b = json.loads(left.read(name)), json.loads(right.read(name))
|
||||
added, removed, common = keyed_diff(a, b)
|
||||
return {"added": added, "removed": removed, "changed": [k for k in common if a[k] != b[k]]}
|
||||
|
||||
|
||||
def read_reports(reports):
|
||||
items = {}
|
||||
for path in sorted((reports / "minecraft" / "components" / "item").glob("*.json")):
|
||||
items[f"minecraft:{path.stem}"] = json.loads(path.read_text())["components"]
|
||||
return {
|
||||
"packets": json.loads((reports / "packets.json").read_text()),
|
||||
"registries": json.loads((reports / "registries.json").read_text()),
|
||||
"blocks": json.loads((reports / "blocks.json").read_text()),
|
||||
"commands": json.loads((reports / "commands.json").read_text()),
|
||||
"items": items,
|
||||
}
|
||||
|
||||
|
||||
def diff_versions(old_paths, new_paths, old_facts, new_facts):
|
||||
a, b = read_reports(old_paths["reports"]), read_reports(new_paths["reports"])
|
||||
return {
|
||||
"facts": diff_facts(old_facts, new_facts),
|
||||
"packets": diff_packets(a["packets"], b["packets"]),
|
||||
"registries": diff_registries(a["registries"], b["registries"]),
|
||||
"blocks": diff_blocks(a["blocks"], b["blocks"]),
|
||||
"items": diff_items(a["items"], b["items"]),
|
||||
"commands": diff_commands(a["commands"], b["commands"]),
|
||||
"data": diff_contents(old_paths["client"], new_paths["client"], "data/minecraft/"),
|
||||
"assets": diff_contents(old_paths["client"], new_paths["client"], "assets/minecraft/"),
|
||||
"lang": diff_lang(old_paths["client"], new_paths["client"]),
|
||||
}
|
||||
|
||||
|
||||
def summary(old, new, result):
|
||||
"""A few lines a person reads first."""
|
||||
lines = [f"Minecraft {old} -> {new}"]
|
||||
for key, (a, b) in result["facts"].items():
|
||||
lines.append(f" {key}: {a} -> {b}")
|
||||
for flow, change in result["packets"].items():
|
||||
lines.append(f" packets {flow}: {change['count'][0]} -> {change['count'][1]}, +{len(change['added'])} "
|
||||
f"-{len(change['removed'])}, {len(change['renumbered'])} renumbered")
|
||||
registries = result["registries"]
|
||||
lines.append(f" registries: +{len(registries['added'])} -{len(registries['removed'])}, "
|
||||
f"{len(registries['changed'])} changed")
|
||||
for name, change in registries["changed"].items():
|
||||
lines.append(f" {name}: +{len(change.get('added', []))} -{len(change.get('removed', []))}"
|
||||
+ (f", {change['ids_shifted']} ids shifted" if change.get("ids_shifted") else ""))
|
||||
blocks = result["blocks"]
|
||||
lines.append(f" blocks: +{len(blocks['added'])} -{len(blocks['removed'])}, {len(blocks['changed'])} changed, "
|
||||
f"states {blocks['state_count'][0]} -> {blocks['state_count'][1]}, "
|
||||
f"{blocks['state_ids_shifted']} blocks' state ids shifted")
|
||||
if blocks["format"]:
|
||||
lines.append(f" report format: fields {', '.join(blocks['format'])} appear or disappear")
|
||||
items = result["items"]
|
||||
lines.append(f" items: +{len(items['added'])} -{len(items['removed'])}, {len(items['changed'])} with "
|
||||
f"changed default components")
|
||||
ranked = sorted(items["components"].items(), key=lambda kv: -sum(kv[1].values()))
|
||||
for component, counts in ranked[:10]:
|
||||
lines.append(f" {component}: " + ", ".join(f"{k} on {n}" for k, n in counts.items() if n))
|
||||
commands = result["commands"]
|
||||
lines.append(f" command nodes: +{len(commands['added'])} -{len(commands['removed'])}, "
|
||||
f"{len(commands['changed'])} changed")
|
||||
for key in ["data", "assets"]:
|
||||
counts = result[key]["counts"]
|
||||
lines.append(f" {key} files: +{counts['added']} -{counts['removed']} ~{counts['changed']}")
|
||||
lang = result["lang"]
|
||||
lines.append(f" en_us keys: +{len(lang['added'])} -{len(lang['removed'])} ~{len(lang['changed'])}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def diff_dir(old, new):
|
||||
return STORE / f"diff-{safe_version(old)}-{safe_version(new)}"
|
||||
|
||||
|
||||
def diff(args):
|
||||
facts = {v: json.loads((version_dir(v) / "facts.json").read_text()) for v in (args.old, args.new)}
|
||||
result = diff_versions(paths(args.old), paths(args.new), facts[args.old], facts[args.new])
|
||||
directory = diff_dir(args.old, args.new)
|
||||
lab.write_json(directory / "diff.json", result)
|
||||
print(summary(args.old, args.new, result))
|
||||
print(f"\n{directory / 'diff.json'}")
|
||||
return 0
|
||||
|
||||
|
||||
# --- gc ----------------------------------------------------------------------------------------------------
|
||||
|
||||
def to_remove(entries, pin, keep):
|
||||
"""Version store entries to delete: every version but the pinned one, --keep and the newest two others;
|
||||
diffs whose versions are gone; scratch folders. entries: [(name, mtime, is_version)]."""
|
||||
versions = sorted(((m, n) for n, m, is_version in entries if is_version and n not in keep | {pin}), reverse=True)
|
||||
kept = keep | {pin} | {n for _, n in versions[:2]}
|
||||
remove = []
|
||||
for name, _, is_version in entries:
|
||||
if name.startswith("."):
|
||||
remove.append(name)
|
||||
elif is_version and name not in kept:
|
||||
remove.append(name)
|
||||
elif name.startswith("diff-") and not any(name == f"diff-{a}-{b}" for a in kept for b in kept):
|
||||
remove.append(name)
|
||||
return sorted(remove)
|
||||
|
||||
|
||||
def gc(args):
|
||||
if not STORE.exists():
|
||||
return 0
|
||||
entries = [(p.name, p.stat().st_mtime, (p / "prepared.json").exists()) for p in STORE.iterdir() if p.is_dir()]
|
||||
for name in to_remove(entries, pinned(), set(args.keep)):
|
||||
print(f"Removing {STORE / name}")
|
||||
if not args.dry_run:
|
||||
shutil.rmtree(STORE / name)
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
command = commands.add_parser("detect", help="what Mojang released that Horizon doesn't pin")
|
||||
command.add_argument("--snapshots", action="store_true")
|
||||
command.add_argument("--dry-run", action="store_true", help="don't remember what was found")
|
||||
command = commands.add_parser("prepare", help="download, verify and run the data generator")
|
||||
command.add_argument("version")
|
||||
command.add_argument("--download", action="store_true", help="download even the pinned version")
|
||||
command = commands.add_parser("diff", help="diff two prepared versions")
|
||||
command.add_argument("old")
|
||||
command.add_argument("new")
|
||||
command = commands.add_parser("gc", help="delete versions and scratch the store doesn't need")
|
||||
command.add_argument("--keep", nargs="*", default=[])
|
||||
command.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
STORE.mkdir(parents=True, exist_ok=True)
|
||||
return {"detect": detect, "prepare": prepare, "diff": diff, "gc": gc}[args.command](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user