From 7c0d3ceea68880c545e040999aa6ab2bc2018012 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:51:39 +0200 Subject: [PATCH] Port report: ranked work items from a version diff tools/port_report.py turns a diff into work items: rank 1 blocks the switch (facts, packets, DataFixers, data component types, report formats, tables to regenerate), rank 2 and 3 one item per Horizon file that ports changed vanilla code (with or without a harness), rank 4 new content, rank 5 summaries. version_port.py report writes report.md and report.json; port prepares, diffs and reports in one go. The class map now resolves names of both versions. The doc records the 26.2 -> 26.3 replay against Horizon's own code. Co-Authored-By: Claude Opus 5.5 --- docs/version-ports.md | 27 +++- tests/test_port_report.py | 145 +++++++++++++++++ tools/port_report.py | 322 ++++++++++++++++++++++++++++++++++++++ tools/version_port.py | 80 ++++++++-- 4 files changed, 560 insertions(+), 14 deletions(-) create mode 100644 tests/test_port_report.py create mode 100644 tools/port_report.py diff --git a/docs/version-ports.md b/docs/version-ports.md index 719f99a8c..39765044e 100644 --- a/docs/version-ports.md +++ b/docs/version-ports.md @@ -256,7 +256,28 @@ Manual, by agents (and Ludovic's review): - **New JDK**: a fleet toolchain install, once per Java bump. - **Judgment on the switch**: merging the pin bump and releasing stay Ludovic's call. -The replay of 26.2 → 26.3 below measures how much that is for one release. +## Replay: 26.2 → 26.3 + +Horizon was written against 26.3 from its first commit, so its code is the ground truth of what porting 26.2 to 26.3 +takes. On the Mac mini, `prepare 26.2` took 9 s (136 MB), `diff 26.2 26.3` 6 s and `report` under a second. + +| What 26.3 needed (in Horizon's code) | What the report found | +| --- | --- | +| Protocol 777, DataVersion 5023, pack formats | rank 1: protocol 776 → 777, world version 4903 → 5023, resource pack 88.0 → 97.1, data pack 107.1 → 121.0, with every site stating them (`PROTOCOL_VERSION`, both `DATA_VERSION`s, the `"26.3"` names) | +| `ids.rs`: `ADD_TRANSIENT_BLOCK`, `POST_EFFECTS` (configuration and play), `SWING_ANIMATION`, serverbound `PUNCH` instead of `SWING`; handlers in `packets/play.rs`, `lan/hub.rs` | rank 1: exactly those packets, 131 renumbered, those files, and "regenerate `ids.rs`". Regenerating from 26.3's report reproduces the committed file byte for byte (after rustfmt); from 26.2's it differs by 274 lines | +| DataFixers 4996-5016 in `datafix/fixes_c.rs` | rank 1: schemas V4996, V4996_1, V4997, V5000 and 16 new fixes (`MergeTerrainChunkStatusFix`, `ExplorerMapItemFix`, `SwingAnimationComponentSplitFix`...), with `datafix/` | +| `swing_animation` split into `attack_animation` and `interact_animation` (`first_person.rs`, `game/hand.rs`, `item_model.rs`) | rank 1: data component types +13 -2, those files among the ones naming them; per component, 1,537 items affected | +| `lan/tracking.rs` and the command tables | rank 1: regenerate `tracking.rs`, `command_suggestions.rs`, `command_order.rs`, `vanilla_order.rs` | +| 133 methods that exist only in 26.3 and that Horizon's comments name (`Minecraft.startAttack`, `MultiPlayerGameMode.piercingAttack`, `SkyRenderer.buildEndSky`, `GameRenderer.render3dHud`...) | all 133: 132 in rank 2-3 items with the method named, on the file that ports it; 1 in the save-format item | +| 49 classes that exist only in 26.3 and that Horizon names (`ComputeCommand`, `PostEffectCommand`, `LivingEntity.SwingState`, `BlockTransformer`, `StrawBedBlock`...) | all 49, in rank 4 "New in " items | + +The report has 499 items: 10 of rank 1, 311 of rank 2, 96 of rank 3, 80 of rank 4, and 2 rank-5 summaries. Rank 1 is +exact. Ranks 2 and 3 are leads, not all work: 173 of their 407 items have a changed method that Horizon's comments +name. The rest are classes whose changed methods Horizon doesn't name. Many of those are mechanical (26.3 renamed +`GuiGraphics` to `GuiGraphicsExtractor` and reworked the render pipeline), and 105 are "widely named" buckets +(classes such as `Entity` that many files mention in passing). Running every harness against the new version (with +`--results`) is what turns a rank-2 lead into a confirmed failure; the replay cannot do that, because the oracles are +written against 26.3. ## Implementation status @@ -265,6 +286,8 @@ python3 tools/version_port.py detect --snapshots # exit 10: new release, 11: python3 tools/version_port.py prepare 26.3 # the pinned version: indexes .local, downloads nothing python3 tools/version_port.py prepare 26.4 # downloads, verifies, runs the data generator (about 10 s) python3 tools/version_port.py diff 26.3 26.4 # summary; the full diff in versions/diff-26.3-26.4/diff.json +python3 tools/version_port.py report 26.3 26.4 [--results .local/parity-runner/state.json] # report.md, report.json +python3 tools/version_port.py port 26.4 # prepare both, diff, report python3 tools/version_port.py gc # keep the pinned version and the newest two others python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food.FoodData # its Rust files and harnesses ``` @@ -274,5 +297,5 @@ python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food | This design | this page | | `detect`, `prepare`, `diff` (facts, packets, registries, blocks, items, commands, data, assets), `gc` | done | | Class index (`tools/class_index.py`), class diff with moves, `vanilla_map.json` (`tools/vanilla_map.py`) | done | -| Port report (`report`), `port` | next PR | +| Port report (`tools/port_report.py`, `report`), `port` | done | | Changed-class decompilation, `regen`, `version_consts` refactor, candidate harness run, `publish`, launchd agent | later | diff --git a/tests/test_port_report.py b/tests/test_port_report.py new file mode 100644 index 000000000..75e9a3a40 --- /dev/null +++ b/tests/test_port_report.py @@ -0,0 +1,145 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from tools import port_report as report +from tools import version_port as port + + +def mapping(): + return { + "classes": { + "net/minecraft/world/food/FoodData": { + "rust": {"crates/w/src/food.rs": ["tick"], "crates/c/src/hud.rs": []}, "harnesses": ["food"]}, + "net/minecraft/world/entity/Entity": { + "rust": {f"crates/c/src/f{i}.rs": [] for i in range(5)}, "harnesses": ["food", "move"]}, + "net/minecraft/client/Sky": {"rust": {"crates/c/src/sky.rs": []}, "harnesses": []}, + "net/minecraft/world/entity/NewMob": {"rust": {"crates/c/src/new_mob.rs": []}, "harnesses": []}, + "net/minecraft/util/datafix/DataFixers": {"rust": {"crates/w/src/datafix/mod.rs": []}, + "harnesses": ["datafix"]}, + }, + "harnesses": { + "food": {"rust": ["crates/w/src/food.rs"], "classes": ["net/minecraft/world/food/FoodData"]}, + "move": {"rust": [], "classes": ["net/minecraft/world/entity/Entity"]}, + "datafix": {"rust": [], "classes": []}, + }, + } + + +def diff(): + def methods(**kinds): + return {"methods": {k: kinds.get(k, []) for k in ("added", "removed", "changed")}} + changed = { + "net/minecraft/world/food/FoodData": methods(changed=["tick()V", "eat(I)V"]), + "net/minecraft/world/entity/Entity": methods(changed=["move()V"]), + "net/minecraft/client/Sky": methods(added=["stars()V"]), + "net/minecraft/util/datafix/DataFixers": methods(changed=["addFixers()V"]), + "net/minecraft/server/Unported": methods(changed=["run()V"]), + } + m = mapping() + return { + "facts": {"protocol_version": [776, 777], "world_version": [4903, 5023]}, + "packets": {"play/serverbound": {"added": {"minecraft:punch": 46}, "removed": {"minecraft:swing": 63}, + "renumbered": {}, "count": [69, 69]}}, + "registries": {"added": {}, "removed": [], "changed": { + "minecraft:entity_type": {"added": ["minecraft:new_mob"]}, + "minecraft:data_component_type": {"added": ["minecraft:attack_animation"], + "removed": ["minecraft:swing_animation"]}}}, + "blocks": {"added": [], "removed": [], "changed": {}, "format": ["definition"]}, + "items": {"added": [], "removed": [], "changed": {}, "components": {}}, + "commands": {"added": [], "removed": [], "changed": {}}, + "classes": { + "added": ["net/minecraft/world/entity/NewMob", "net/minecraft/util/datafix/schemas/V4996", + "net/minecraft/util/datafix/fixes/SwingSplitFix", "net/minecraft/network/protocol/game/Punch", + "net/minecraft/elsewhere/Thing"], + "removed": ["net/minecraft/network/protocol/game/Swing"], "moved": {}, + "changed": changed, + "horizon": {name: m["classes"][name] for name in changed if name in m["classes"]}, + }, + } + + +class PortReportTests(unittest.TestCase): + def build(self, results=None): + with mock.patch.object(report, "version_sites", return_value=["crates/p/src/lib.rs:20"]), \ + mock.patch.object(report, "files_naming", return_value=["crates/p/src/play.rs"]), \ + mock.patch.object(report, "files_containing", return_value=["crates/w/src/blocks.rs"]): + return report.build(diff(), mapping(), "26.2", "26.3", {"id": "26.3"}, results) + + def test_ranks_and_kinds(self): + built = self.build() + kinds = [(i["rank"], i["kind"]) for i in built["items"]] + self.assertEqual(sorted(k for r, k in kinds if r == 1), + ["components", "facts", "protocol", "regenerate", "regenerate", "report-format", + "save-format"]) + self.assertEqual([i["id"] for i in built["items"]], list(range(1, len(kinds) + 1))) + self.assertEqual(kinds, sorted(kinds, key=lambda k: k[0])) + regenerated = {i["files"][0] for i in built["items"] if i["kind"] == "regenerate"} + self.assertEqual(regenerated, {"crates/horizon-protocol/src/ids.rs", "crates/horizon-client/src/lan/tracking.rs"}) + + def test_save_format_lists_new_schemas_and_fixes(self): + save = next(i for i in self.build()["items"] if i["kind"] == "save-format") + self.assertEqual(save["details"]["schemas"], ["net/minecraft/util/datafix/schemas/V4996"]) + self.assertEqual(save["details"]["fixes"], ["net/minecraft/util/datafix/fixes/SwingSplitFix"]) + self.assertEqual(save["files"], ["crates/w/src/datafix/mod.rs"]) + + def test_code_items_by_file(self): + items = {i["module"]: i for i in self.build()["items"] if "module" in i} + food = items["crates/w/src/food.rs"] + self.assertEqual((food["rank"], food["kind"], food["harnesses"]), (2, "rerun", ["food"])) + self.assertEqual(food["classes"]["net/minecraft/world/food/FoodData"]["named"], ["tick"]) + sky = items["crates/c/src/sky.rs"] + self.assertEqual((sky["rank"], sky["kind"]), (3, "unproved")) + # Named by many files and none names the changed method: grouped by package, harnesses by coverage. + widely = items["widely named: world/entity"] + self.assertEqual(widely["harnesses"], ["food", "move"]) + unported = next(i for i in self.build()["items"] if i["kind"] == "unported") + self.assertEqual(unported["packages"], {"net/minecraft/server": 1}) + + def test_failed_harness_ranks_first_in_rank_two(self): + items = self.build({"food": "fail", "move": "pass"})["items"] + first = next(i for i in items if i["rank"] == 2) + self.assertEqual((first["kind"], first["failed"]), ("harness-failed", ["food"])) + + def test_new_content(self): + items = self.build()["items"] + self.assertIn("minecraft:new_mob", next(i for i in items if i["kind"] == "content")["entries"]) + new = next(i for i in items if i["kind"] == "new-classes") + self.assertEqual((new["classes"], new["ported"]), + (["net/minecraft/world/entity/NewMob"], ["net/minecraft/world/entity/NewMob"])) + self.assertEqual(next(i for i in items if i["kind"] == "new-unported")["packages"], + {"net/minecraft/elsewhere": 1}) + + def test_markdown(self): + text = report.markdown(self.build()) + self.assertTrue(text.startswith("# Port report: Minecraft 26.2 -> 26.3")) + self.assertIn("## 1. Blocks the switch", text) + self.assertIn("play/serverbound: added minecraft:punch; removed minecraft:swing", text) + self.assertIn("FoodData (2 methods; tick)", text) + + def test_helpers(self): + self.assertEqual(report.snake("LivingEntity"), "living_entity") + entry = {"rust": {"a/x.rs": ["tick"], "a/living_entity.rs": [], "a/y.rs": ["tick", "eat"]}} + self.assertEqual(report.primary_file(entry, {"tick", "eat"}, "n/LivingEntity"), "a/y.rs") + self.assertEqual(report.primary_file(entry, {"other"}, "n/LivingEntity"), "a/living_entity.rs") + self.assertEqual(report.primary_file(entry, set(), "n/Other"), "a/living_entity.rs") + self.assertIsNone(report.primary_file({"rust": {f"{i}.rs": [] for i in range(4)}}, set(), "n/Other")) + + +class VersionPortReportTests(unittest.TestCase): + def test_harness_results_merge_shards(self): + with tempfile.TemporaryDirectory() as directory: + state = Path(directory) / "state.json" + state.write_text(json.dumps({"harnesses": {"food": {"status": "pass"}, "datafix#1": {"status": "pass"}, + "datafix#2": {"status": "fail"}}})) + self.assertEqual(port.harness_results(state), {"food": "pass", "datafix": "fail"}) + + def test_class_names_drop_moved_old_names(self): + self.assertEqual(port.class_names({"a/X": 0, "a/Y": 0}, {"b/X": 0, "a/Y": 0}, {"a/X": "b/X"}), + ["a/Y", "b/X"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/port_report.py b/tools/port_report.py new file mode 100644 index 000000000..afbb65861 --- /dev/null +++ b/tools/port_report.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""The port report: a version diff (tools/version_port.py diff) turned into ranked, area-sized work items +(docs/version-ports.md, "The port report"). + +Ranks: + 1 blocks the switch: version facts, packets, the save format (DataFixers), data component types, report + formats, generated tables to regenerate + 2 a harness failed against the new version (with --results), else changed vanilla code a harness proves: + rerun it + 3 changed vanilla code Horizon ports and no harness proves + 4 new content: registry entries, and new classes in the vanilla packages Horizon ports + 5 changed and new vanilla code Horizon doesn't port (two summary items) + +Used through `version_port.py report OLD NEW`; the functions here take plain data so tests can build it. +""" + +from collections import Counter, defaultdict +from pathlib import Path +import re +import subprocess + +try: + from . import class_index, vanilla_map +except ImportError: + import class_index + import vanilla_map + +ROOT = Path(__file__).resolve().parents[1] +PROTOCOL = "net/minecraft/network/protocol/" +DATAFIX = "net/minecraft/util/datafix/" +# Generated tables, what makes them stale, and how to regenerate them (docs/version-ports.md, "Regenerate"). +GENERATORS = [ + ("crates/horizon-protocol/src/ids.rs", "python3 tools/generate_packet_ids.py", + lambda d: bool(d["packets"])), + ("crates/horizon-client/src/lan/tracking.rs", "python3 tools/generate_entity_tracking.py", + lambda d: "minecraft:entity_type" in d["registries"]["changed"] + or "net/minecraft/world/entity/EntityType" in d["classes"]["changed"]), + ("crates/horizon-client/src/lan/command_suggestions.rs", "python3 tools/generate_command_suggestions.py", + lambda d: bool(d["commands"]["added"] or d["commands"]["removed"] or d["commands"]["changed"])), + ("crates/horizon-client/src/lan/command_order.rs", "python3 tools/generate_lan_command_order.py", + lambda d: bool(d["commands"]["added"] or d["commands"]["removed"])), + ("crates/horizon-client/src/command/vanilla_order.rs", "python3 tools/generate_command_order.py", + lambda d: bool(d["commands"]["added"] or d["commands"]["removed"])), + ("crates/horizon-world/src/datafix/block_state_data.txt", "python3 tools/gen_block_state_data.py", + lambda d: "net/minecraft/util/datafix/fixes/BlockStateData" in d["classes"]["changed"]), +] +# Registries whose new entries are new things to port. +CONTENT = ["minecraft:block", "minecraft:item", "minecraft:entity_type", "minecraft:block_entity_type", + "minecraft:particle_type", "minecraft:menu", "minecraft:worldgen/feature", "minecraft:sound_event"] + + +def version_sites(facts, root=ROOT): + """file:line of every place the code states these version facts (not tests).""" + pattern = ["-e", str(facts["protocol_version"]), "-e", str(facts["world_version"])] + found = [] + for command in (["git", "grep", "-n", "-w"] + pattern, ["git", "grep", "-n", "-F", f'"{facts["id"]}"']): + result = subprocess.run(command + ["--", "crates/*.rs", "launcher/*.rs", "tools/*.py"], + cwd=root, capture_output=True, text=True) + found += [line.split(":", 2)[0] + ":" + line.split(":", 2)[1] for line in result.stdout.splitlines()] + return sorted({site for site in found if not re.search(r"(^|/)tests?(/|\.rs)|/test_|_tests?\.rs", site)}) + + +def files_containing(text, root=ROOT): + """Rust and Python files that contain this text (a report's file name).""" + command = ["git", "grep", "-l", "-F", text, "--", "crates/*.rs", "launcher/*.rs", "tools/*.py"] + return subprocess.run(command, cwd=root, capture_output=True, text=True).stdout.split() + + +def files_naming(words, root=ROOT): + """Rust files that contain any of these words (component or packet names).""" + if not words: + return [] + command = ["git", "grep", "-l", "-w"] + [arg for word in words for arg in ("-e", word)] + ["--", "crates/*.rs"] + return subprocess.run(command, cwd=root, capture_output=True, text=True).stdout.split() + + +def snake(name): + """LivingEntity -> living_entity.""" + return re.sub(r"(? {b}" for k, (a, b) in diff["facts"].items()) + items.append(item(1, "facts", f"Version facts: {changes}", files=version_sites(pinned_facts, root), + details=diff["facts"])) + if diff["packets"]: + classes = diff["classes"] + packets = {"added": [c for c in classes["added"] if c.startswith(PROTOCOL)], + "removed": [c for c in classes["removed"] if c.startswith(PROTOCOL)], + "changed": [c for c in classes["changed"] if c.startswith(PROTOCOL)]} + names = sorted({n.split(":")[1] for flow in diff["packets"].values() + for n in list(flow["added"]) + list(flow["removed"])}) + constants = [n.upper().replace("/", "_") for n in names] + counts = ", ".join(f"{flow} +{len(c['added'])} -{len(c['removed'])} ~{len(c['renumbered'])}" + for flow, c in diff["packets"].items()) + items.append(item(1, "protocol", f"Protocol: packets {counts}", + files=sorted(set(files_naming(constants, root)) | {"crates/horizon-protocol/src/ids.rs"}), + details={"packets": diff["packets"], "classes": packets})) + if "world_version" in diff["facts"]: + classes = diff["classes"] + added = [c for c in classes["added"] if c.startswith(DATAFIX)] + changed = [c for c in classes["changed"] if c.startswith(DATAFIX)] + schemas = sorted(c for c in added if "/schemas/V" in c and "$" not in c) + fixes = sorted(c for c in added if "/fixes/" in c and "$" not in c) + horizon = vanilla_map.lookup(mapping, "net/minecraft/util/datafix/DataFixers") + named = {f for c in schemas + fixes for f in mapping["classes"].get(c, {}).get("rust", {})} + a, b = diff["facts"]["world_version"] + items.append(item(1, "save-format", f"Save format: DataFixers {a} -> {b}: {len(schemas)} schemas, " + f"{len(fixes)} fixes added, {len(changed)} datafix classes changed", + files=sorted(set(horizon["rust"]) | named), harnesses=horizon["harnesses"], + details={"schemas": schemas, "fixes": fixes, "changed": changed})) + registry = diff["registries"]["changed"].get("minecraft:data_component_type", {}) + if registry.get("added") or registry.get("removed"): + names = [n.split(":")[1] for n in registry.get("added", []) + registry.get("removed", [])] + items.append(item(1, "components", f"Data component types: +{len(registry.get('added', []))} " + f"-{len(registry.get('removed', []))} (sent on the wire and saved)", + files=files_naming(names, root), + details={"types": registry, "items": diff["items"]["components"]})) + if diff["blocks"].get("format"): + items.append(item(1, "report-format", f"blocks.json format: {', '.join(diff['blocks']['format'])}", + files=files_containing("blocks.json", root), + details={"fields": diff["blocks"]["format"]})) + for output, command, stale in GENERATORS: + if stale(diff): + items.append(item(1, "regenerate", f"Regenerate {output}", files=[output], command=command)) + return items + + +def item_harnesses(path, classes, mapping, limit=8): + """The harnesses for an item: those whose probes reach its file, else those covering the most of its + classes, the most specific first.""" + harnesses = mapping["harnesses"] + reaching = sorted(h for h, data in harnesses.items() if path in data["rust"]) + if reaching: + return reaching + covering = Counter(h for c in classes for h in vanilla_map.lookup(mapping, c)["harnesses"]) + return sorted(sorted(covering, key=lambda h: (-covering[h], len(harnesses[h]["classes"]), h))[:limit]) + + +def code_items(diff, mapping, results=None): + """Ranks 2, 3 and 5 from the changed and removed vanilla classes, one item per Horizon file that ports + them (classes many files name in passing go by vanilla package).""" + classes = diff["classes"] + groups = defaultdict(lambda: {"classes": {}, "files": set(), "precise": 0, "methods": 0}) + unported = Counter() + for name in list(classes["changed"]) + classes["removed"]: + if name.startswith((PROTOCOL, DATAFIX)): + continue # The protocol and save-format items have them. + entry = classes["horizon"].get(name) + if not entry: + unported[name.rsplit("/", 1)[0]] += 1 + continue + removed = name not in classes["changed"] + change = classes["changed"].get(name, {}) + members = method_names(change) + path = primary_file(entry, members, name) if entry["rust"] else None + key = path or "widely named: " + name.rsplit("/", 1)[0].removeprefix("net/minecraft/") + group = groups[key] + precise = sorted({m for named in entry["rust"].values() for m in named} & members) + methods = 1 if removed else class_index.changed_methods(change) + group["classes"][name] = {"methods": methods, "named": precise, "removed": removed} + group["files"].update([path] if path else entry["rust"]) + group["precise"] += len(precise) + group["methods"] += methods + failing = {h for h, status in (results or {}).items() if status != "pass"} + items = [] + for key, group in groups.items(): + harnesses = item_harnesses(key, group["classes"], mapping) + failed = sorted(set(harnesses) & failing) + rank, kind = (2, "harness-failed") if failed else (2, "rerun") if harnesses else (3, "unproved") + top = sorted(group["classes"], key=lambda c: (-group["classes"][c]["methods"], c)) + title = (f"{key}: {len(group['classes'])} vanilla classes changed " + f"({group['methods']} methods, {group['precise']} named in Horizon)") + items.append(item(rank, kind, title, module=key, files=sorted(group["files"]), harnesses=harnesses, + failed=failed, classes={c: group["classes"][c] for c in top}, + score=group["methods"] + 10 * group["precise"] + (1000 if failed else 0))) + if unported: + items.append(item(5, "unported", f"{sum(unported.values())} changed or removed vanilla classes Horizon " + f"doesn't port, in {len(unported)} packages", packages=dict(unported.most_common()))) + return items + + +def content_items(diff, mapping): + """Rank 4: new registry entries, and new classes by vanilla package where Horizon ports other classes of + that package (with those Horizon already names, after a port); new classes elsewhere go in one summary.""" + items = [] + for registry in CONTENT: + entries = diff["registries"]["changed"].get(registry, {}).get("added", []) + if entries: + items.append(item(4, "content", f"New {registry.split(':')[1]}: {len(entries)} entries", entries=entries)) + ported_packages = Counter(c.rsplit("/", 1)[0] for c, e in mapping["classes"].items() if e["rust"]) + by_package = defaultdict(list) + for name in diff["classes"]["added"]: + if not re.search(r"\$\d", name) and not name.startswith((PROTOCOL, DATAFIX)): + by_package[name.rsplit("/", 1)[0]].append(name) + elsewhere = Counter() + for package, names in sorted(by_package.items()): + if package not in ported_packages: + elsewhere[package] += len(names) + continue + ported = sorted(c for c in names if mapping["classes"].get(c, {}).get("rust")) + nearby = Counter(f for c, e in mapping["classes"].items() if c.rsplit("/", 1)[0] == package for f in e["rust"]) + files = sorted({f for c in ported for f in mapping["classes"][c]["rust"]}) or [f for f, _ in nearby.most_common(5)] + items.append(item(4, "new-classes", f"New in {package.removeprefix('net/minecraft/')}: {len(names)} classes" + f" ({len(ported)} already named in Horizon)", classes=sorted(names), ported=ported, + files=files, score=len(names))) + if elsewhere: + items.append(item(5, "new-unported", f"{sum(elsewhere.values())} new vanilla classes in {len(elsewhere)} " + f"packages Horizon doesn't port", packages=dict(elsewhere.most_common()))) + return items + + +def build(diff, mapping, old, new, pinned_facts, results=None, root=ROOT): + """The report: summary, ranked items and what came out clean. pinned_facts: the facts of the version the + code states today (to find where it states them); results: {harness: "pass" | "fail"} against NEW.""" + items = blocking_items(diff, mapping, pinned_facts, root) + items += code_items(diff, mapping, results) + content_items(diff, mapping) + items.sort(key=lambda i: (i["rank"], -i.get("score", 0), i["title"])) + for number, entry in enumerate(items, 1): + entry["id"] = number + clean = [kind for kind in ("packets", "commands") if not diff[kind]] + clean += [kind for kind in ("blocks", "items") if not (diff[kind]["added"] or diff[kind]["removed"] + or diff[kind]["changed"])] + classes = diff["classes"] + return { + "old": old, "new": new, + "summary": { + "facts": diff["facts"], "ranks": dict(sorted(Counter(i["rank"] for i in items).items())), + "classes": {"added": len(classes["added"]), "removed": len(classes["removed"]), + "moved": len(classes["moved"]), "changed": len(classes["changed"]), + "horizon": len(classes["horizon"])}, + "generators": [i["files"][0] for i in items if i["kind"] == "regenerate"], + }, + "items": items, "clean": clean, + } + + +def short(name): + return name.rsplit("/", 1)[-1].replace("$", ".") + + +def markdown(report, limit=12): + """report.md: what a person or an agent reads first.""" + lines = [f"# Port report: Minecraft {report['old']} -> {report['new']}", ""] + summary = report["summary"] + for key, (a, b) in summary["facts"].items(): + lines.append(f"- {key}: {a} -> {b}") + c = summary["classes"] + lines.append(f"- classes: +{c['added']} -{c['removed']}, {c['moved']} moved, {c['changed']} changed; " + f"{c['horizon']} changed or removed classes concern Horizon code or harnesses") + lines.append(f"- work items by rank: " + ", ".join(f"{r}: {n}" for r, n in summary["ranks"].items())) + if report["clean"]: + lines.append(f"- unchanged: {', '.join(report['clean'])}") + names = {1: "Blocks the switch", 2: "Proved by a harness: rerun it", 3: "Ported, no harness", + 4: "New content", 5: "Not ported"} + for rank in sorted({i["rank"] for i in report["items"]}): + lines += ["", f"## {rank}. {names[rank]}", ""] + for entry in [i for i in report["items"] if i["rank"] == rank]: + lines.append(f"### #{entry['id']} {entry['title']}") + if entry.get("command"): + lines.append(f"- run: `{entry['command']}`") + if entry.get("failed"): + lines.append(f"- failing harnesses: {', '.join(entry['failed'])}") + if entry.get("harnesses"): + lines.append(f"- harnesses: {', '.join(entry['harnesses'][:limit])}" + + (f" (+{len(entry['harnesses']) - limit})" if len(entry["harnesses"]) > limit else "")) + if entry.get("files"): + lines.append(f"- Horizon: {', '.join(f'`{f}`' for f in entry['files'][:limit])}" + + (f" (+{len(entry['files']) - limit})" if len(entry["files"]) > limit else "")) + if entry.get("classes") and isinstance(entry["classes"], dict): + shown = list(entry["classes"].items())[:limit] + lines.append("- vanilla: " + ", ".join( + f"{short(n)} ({'removed' if v['removed'] else str(v['methods']) + ' methods'}" + + (f"; {', '.join(v['named'])}" if v["named"] else "") + ")" for n, v in shown) + + (f" (+{len(entry['classes']) - limit})" if len(entry["classes"]) > limit else "")) + elif entry.get("classes"): + lines.append(f"- new classes: {', '.join(short(n) for n in entry['classes'][:limit])}" + + (f" (+{len(entry['classes']) - limit})" if len(entry["classes"]) > limit else "")) + if entry.get("entries"): + lines.append(f"- entries: {', '.join(n.split(':')[-1] for n in entry['entries'][:limit])}" + + (f" (+{len(entry['entries']) - limit})" if len(entry["entries"]) > limit else "")) + if entry.get("ported"): + lines.append(f"- already in Horizon: {', '.join(short(n) for n in entry['ported'][:limit])}") + details = entry.get("details", {}) + for key in ("schemas", "fixes"): + if details.get(key): + lines.append(f"- {key}: {', '.join(short(n) for n in details[key])}") + if entry["kind"] == "protocol": + for flow, change in details["packets"].items(): + lines.append(f"- {flow}: added {', '.join(change['added']) or 'none'}; " + f"removed {', '.join(change['removed']) or 'none'}") + if entry["kind"] in ("unported", "new-unported"): + lines.append("- " + ", ".join(f"{p.removeprefix('net/minecraft/')} {n}" + for p, n in list(entry["packages"].items())[:limit])) + lines.append("") + return "\n".join(lines).rstrip() + "\n" diff --git a/tools/version_port.py b/tools/version_port.py index c718be0c4..c46efa202 100644 --- a/tools/version_port.py +++ b/tools/version_port.py @@ -8,6 +8,8 @@ version_port.py diff OLD NEW facts, packets, registries, blocks, items, commands, data packs, assets, lang and classes, with the Horizon code each changed class concerns: diff-OLD-NEW/diff.json + version_port.py report OLD NEW [--results S] the ranked port report: diff-OLD-NEW/report.md and .json + version_port.py port NEW [--old OLD] prepare both, diff, report version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep The downloads are for development only: they go to the version store (~/code/horizon-parity/versions, @@ -31,10 +33,11 @@ import urllib.request import zipfile try: - from . import class_index, lab, vanilla_map + from . import class_index, lab, port_report, vanilla_map except ImportError: import class_index import lab + import port_report import vanilla_map MANIFEST_URL = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json" @@ -438,13 +441,17 @@ def read_reports(reports): } -def diff_classes(old, new, mapping): - """The class diff, each changed or removed class with the Horizon files and harnesses that concern it - (looked up under its old name: Horizon's code names the pinned version's classes).""" - result = class_index.diff(old, new) +def class_names(old, new, moved): + """Every class name of either version, but a moved class only under its new name, so that its simple + name stays unambiguous.""" + return sorted(set(new) | (set(old) - set(moved))) + + +def diff_classes(result, mapping): + """The class diff, each changed or removed class with the Horizon files and harnesses that concern it.""" horizon = {} - for name, change in result["changed"].items(): - entry = vanilla_map.lookup(mapping, change.get("moved_from", name)) + for name in result["changed"]: + entry = vanilla_map.lookup(mapping, name) if entry["rust"] or entry["harnesses"]: horizon[name] = entry for name in result["removed"]: @@ -455,9 +462,9 @@ def diff_classes(old, new, mapping): return result -def diff_versions(old_paths, new_paths, old_facts, new_facts, old_classes=None, new_classes=None, mapping=None): +def diff_versions(old_paths, new_paths, old_facts, new_facts, classes=None, mapping=None): a, b = read_reports(old_paths["reports"]), read_reports(new_paths["reports"]) - classes = diff_classes(old_classes, new_classes, mapping) if old_classes is not None else None + classes = diff_classes(classes, mapping) if classes is not None else None return { "facts": diff_facts(old_facts, new_facts), "packets": diff_packets(a["packets"], b["packets"]), @@ -526,16 +533,53 @@ def load(version, name): def diff(args): directory = diff_dir(args.old, args.new) old_classes, new_classes = load(args.old, "classes.json"), load(args.new, "classes.json") - mapping = vanilla_map.build(old_classes, dependencies=vanilla_map.recorded_dependencies(vanilla_map.default_shared())) + classes = class_index.diff(old_classes, new_classes) + mapping = vanilla_map.build(class_names(old_classes, new_classes, classes["moved"]), + dependencies=vanilla_map.recorded_dependencies(vanilla_map.default_shared())) lab.write_json(directory / "vanilla_map.json", mapping) result = diff_versions(paths(args.old), paths(args.new), load(args.old, "facts.json"), load(args.new, "facts.json"), - old_classes, new_classes, mapping) + classes, mapping) lab.write_json(directory / "diff.json", result) print(summary(args.old, args.new, result)) print(f"\n{directory / 'diff.json'}") return 0 +# --- report ------------------------------------------------------------------------------------------------ + +def harness_results(state_path): + """{harness: "pass" | "fail"} from tools/parity.py's state; a harness fails if one of its shards does.""" + found = {} + for name, record in json.loads(Path(state_path).read_text()).get("harnesses", {}).items(): + harness = name.split("#")[0] + found[harness] = "fail" if record.get("status") != "pass" or found.get(harness) == "fail" else "pass" + return found + + +def report(args): + directory = diff_dir(args.old, args.new) + if not (directory / "diff.json").exists(): + raise SystemExit(f"No diff yet; run version_port.py diff {args.old} {args.new}") + result = json.loads((directory / "diff.json").read_text()) + mapping = json.loads((directory / "vanilla_map.json").read_text()) + results = harness_results(args.results) if args.results else None + built = port_report.build(result, mapping, args.old, args.new, load(pinned(), "facts.json"), results) + lab.write_json(directory / "report.json", built) + (directory / "report.md").write_text(port_report.markdown(built)) + print("\n".join(port_report.markdown(built).splitlines()[:args.lines])) + print(f"\n{directory / 'report.md'} ({len(built['items'])} items)") + return 0 + + +def port(args): + """prepare both, diff, report.""" + old = args.old or pinned() + for version in (old, args.new): + prepare(argparse.Namespace(version=version, download=False, jobs=args.jobs)) + diff(argparse.Namespace(old=old, new=args.new)) + return report(argparse.Namespace(old=old, new=args.new, results=args.results, lines=args.lines)) + + # --- gc ---------------------------------------------------------------------------------------------------- def to_remove(entries, pin, keep): @@ -579,12 +623,24 @@ def main(): command = commands.add_parser("diff", help="diff two prepared versions") command.add_argument("old") command.add_argument("new") + command = commands.add_parser("report", help="the ranked port report of a diff") + command.add_argument("old") + command.add_argument("new") + command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW") + command.add_argument("--lines", type=int, default=60, help="how much of report.md to print") + command = commands.add_parser("port", help="prepare, diff and report in one go") + command.add_argument("new") + command.add_argument("--old", help="default: the pinned version") + command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW") + command.add_argument("--lines", type=int, default=60) + command.add_argument("-j", "--jobs", type=int, default=int(os.environ.get("FLEET_LOAD_JOBS", 0)) or None) command = commands.add_parser("gc", help="delete versions and scratch the store doesn't need") command.add_argument("--keep", nargs="*", default=[]) command.add_argument("--dry-run", action="store_true") args = parser.parse_args() STORE.mkdir(parents=True, exist_ok=True) - return {"detect": detect, "prepare": prepare, "diff": diff, "gc": gc}[args.command](args) + return {"detect": detect, "prepare": prepare, "diff": diff, "report": report, "port": port, + "gc": gc}[args.command](args) if __name__ == "__main__":