Merge pull request #325 from ludovic111/feat/version-port-report

Port report: ranked work items from a Minecraft version diff
This commit is contained in:
Ludovic Marie
2026-10-15 14:00:06 +02:00
committed by GitHub
4 changed files with 560 additions and 14 deletions
+25 -2
View File
@@ -256,7 +256,28 @@ Manual, by agents (and Ludovic's review):
- **New JDK**: a fleet toolchain install, once per Java bump.
- **Judgment on the switch**: merging the pin bump and releasing stay Ludovic's call.
The replay of 26.2 → 26.3 below measures how much that is for one release.
## Replay: 26.2 → 26.3
Horizon was written against 26.3 from its first commit, so its code is the ground truth of what porting 26.2 to 26.3
takes. On the Mac mini, `prepare 26.2` took 9 s (136 MB), `diff 26.2 26.3` 6 s and `report` under a second.
| What 26.3 needed (in Horizon's code) | What the report found |
| --- | --- |
| Protocol 777, DataVersion 5023, pack formats | rank 1: protocol 776 → 777, world version 4903 → 5023, resource pack 88.0 → 97.1, data pack 107.1 → 121.0, with every site stating them (`PROTOCOL_VERSION`, both `DATA_VERSION`s, the `"26.3"` names) |
| `ids.rs`: `ADD_TRANSIENT_BLOCK`, `POST_EFFECTS` (configuration and play), `SWING_ANIMATION`, serverbound `PUNCH` instead of `SWING`; handlers in `packets/play.rs`, `lan/hub.rs` | rank 1: exactly those packets, 131 renumbered, those files, and "regenerate `ids.rs`". Regenerating from 26.3's report reproduces the committed file byte for byte (after rustfmt); from 26.2's it differs by 274 lines |
| DataFixers 4996-5016 in `datafix/fixes_c.rs` | rank 1: schemas V4996, V4996_1, V4997, V5000 and 16 new fixes (`MergeTerrainChunkStatusFix`, `ExplorerMapItemFix`, `SwingAnimationComponentSplitFix`...), with `datafix/` |
| `swing_animation` split into `attack_animation` and `interact_animation` (`first_person.rs`, `game/hand.rs`, `item_model.rs`) | rank 1: data component types +13 -2, those files among the ones naming them; per component, 1,537 items affected |
| `lan/tracking.rs` and the command tables | rank 1: regenerate `tracking.rs`, `command_suggestions.rs`, `command_order.rs`, `vanilla_order.rs` |
| 133 methods that exist only in 26.3 and that Horizon's comments name (`Minecraft.startAttack`, `MultiPlayerGameMode.piercingAttack`, `SkyRenderer.buildEndSky`, `GameRenderer.render3dHud`...) | all 133: 132 in rank 2-3 items with the method named, on the file that ports it; 1 in the save-format item |
| 49 classes that exist only in 26.3 and that Horizon names (`ComputeCommand`, `PostEffectCommand`, `LivingEntity.SwingState`, `BlockTransformer`, `StrawBedBlock`...) | all 49, in rank 4 "New in <package>" items |
The report has 499 items: 10 of rank 1, 311 of rank 2, 96 of rank 3, 80 of rank 4, and 2 rank-5 summaries. Rank 1 is
exact. Ranks 2 and 3 are leads, not all work: 173 of their 407 items have a changed method that Horizon's comments
name. The rest are classes whose changed methods Horizon doesn't name. Many of those are mechanical (26.3 renamed
`GuiGraphics` to `GuiGraphicsExtractor` and reworked the render pipeline), and 105 are "widely named" buckets
(classes such as `Entity` that many files mention in passing). Running every harness against the new version (with
`--results`) is what turns a rank-2 lead into a confirmed failure; the replay cannot do that, because the oracles are
written against 26.3.
## Implementation status
@@ -265,6 +286,8 @@ python3 tools/version_port.py detect --snapshots # exit 10: new release, 11:
python3 tools/version_port.py prepare 26.3 # the pinned version: indexes .local, downloads nothing
python3 tools/version_port.py prepare 26.4 # downloads, verifies, runs the data generator (about 10 s)
python3 tools/version_port.py diff 26.3 26.4 # summary; the full diff in versions/diff-26.3-26.4/diff.json
python3 tools/version_port.py report 26.3 26.4 [--results .local/parity-runner/state.json] # report.md, report.json
python3 tools/version_port.py port 26.4 # prepare both, diff, report
python3 tools/version_port.py gc # keep the pinned version and the newest two others
python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food.FoodData # its Rust files and harnesses
```
@@ -274,5 +297,5 @@ python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food
| This design | this page |
| `detect`, `prepare`, `diff` (facts, packets, registries, blocks, items, commands, data, assets), `gc` | done |
| Class index (`tools/class_index.py`), class diff with moves, `vanilla_map.json` (`tools/vanilla_map.py`) | done |
| Port report (`report`), `port` | next PR |
| Port report (`tools/port_report.py`, `report`), `port` | done |
| Changed-class decompilation, `regen`, `version_consts` refactor, candidate harness run, `publish`, launchd agent | later |
+145
View File
@@ -0,0 +1,145 @@
import json
from pathlib import Path
import tempfile
import unittest
from unittest import mock
from tools import port_report as report
from tools import version_port as port
def mapping():
return {
"classes": {
"net/minecraft/world/food/FoodData": {
"rust": {"crates/w/src/food.rs": ["tick"], "crates/c/src/hud.rs": []}, "harnesses": ["food"]},
"net/minecraft/world/entity/Entity": {
"rust": {f"crates/c/src/f{i}.rs": [] for i in range(5)}, "harnesses": ["food", "move"]},
"net/minecraft/client/Sky": {"rust": {"crates/c/src/sky.rs": []}, "harnesses": []},
"net/minecraft/world/entity/NewMob": {"rust": {"crates/c/src/new_mob.rs": []}, "harnesses": []},
"net/minecraft/util/datafix/DataFixers": {"rust": {"crates/w/src/datafix/mod.rs": []},
"harnesses": ["datafix"]},
},
"harnesses": {
"food": {"rust": ["crates/w/src/food.rs"], "classes": ["net/minecraft/world/food/FoodData"]},
"move": {"rust": [], "classes": ["net/minecraft/world/entity/Entity"]},
"datafix": {"rust": [], "classes": []},
},
}
def diff():
def methods(**kinds):
return {"methods": {k: kinds.get(k, []) for k in ("added", "removed", "changed")}}
changed = {
"net/minecraft/world/food/FoodData": methods(changed=["tick()V", "eat(I)V"]),
"net/minecraft/world/entity/Entity": methods(changed=["move()V"]),
"net/minecraft/client/Sky": methods(added=["stars()V"]),
"net/minecraft/util/datafix/DataFixers": methods(changed=["addFixers()V"]),
"net/minecraft/server/Unported": methods(changed=["run()V"]),
}
m = mapping()
return {
"facts": {"protocol_version": [776, 777], "world_version": [4903, 5023]},
"packets": {"play/serverbound": {"added": {"minecraft:punch": 46}, "removed": {"minecraft:swing": 63},
"renumbered": {}, "count": [69, 69]}},
"registries": {"added": {}, "removed": [], "changed": {
"minecraft:entity_type": {"added": ["minecraft:new_mob"]},
"minecraft:data_component_type": {"added": ["minecraft:attack_animation"],
"removed": ["minecraft:swing_animation"]}}},
"blocks": {"added": [], "removed": [], "changed": {}, "format": ["definition"]},
"items": {"added": [], "removed": [], "changed": {}, "components": {}},
"commands": {"added": [], "removed": [], "changed": {}},
"classes": {
"added": ["net/minecraft/world/entity/NewMob", "net/minecraft/util/datafix/schemas/V4996",
"net/minecraft/util/datafix/fixes/SwingSplitFix", "net/minecraft/network/protocol/game/Punch",
"net/minecraft/elsewhere/Thing"],
"removed": ["net/minecraft/network/protocol/game/Swing"], "moved": {},
"changed": changed,
"horizon": {name: m["classes"][name] for name in changed if name in m["classes"]},
},
}
class PortReportTests(unittest.TestCase):
def build(self, results=None):
with mock.patch.object(report, "version_sites", return_value=["crates/p/src/lib.rs:20"]), \
mock.patch.object(report, "files_naming", return_value=["crates/p/src/play.rs"]), \
mock.patch.object(report, "files_containing", return_value=["crates/w/src/blocks.rs"]):
return report.build(diff(), mapping(), "26.2", "26.3", {"id": "26.3"}, results)
def test_ranks_and_kinds(self):
built = self.build()
kinds = [(i["rank"], i["kind"]) for i in built["items"]]
self.assertEqual(sorted(k for r, k in kinds if r == 1),
["components", "facts", "protocol", "regenerate", "regenerate", "report-format",
"save-format"])
self.assertEqual([i["id"] for i in built["items"]], list(range(1, len(kinds) + 1)))
self.assertEqual(kinds, sorted(kinds, key=lambda k: k[0]))
regenerated = {i["files"][0] for i in built["items"] if i["kind"] == "regenerate"}
self.assertEqual(regenerated, {"crates/horizon-protocol/src/ids.rs", "crates/horizon-client/src/lan/tracking.rs"})
def test_save_format_lists_new_schemas_and_fixes(self):
save = next(i for i in self.build()["items"] if i["kind"] == "save-format")
self.assertEqual(save["details"]["schemas"], ["net/minecraft/util/datafix/schemas/V4996"])
self.assertEqual(save["details"]["fixes"], ["net/minecraft/util/datafix/fixes/SwingSplitFix"])
self.assertEqual(save["files"], ["crates/w/src/datafix/mod.rs"])
def test_code_items_by_file(self):
items = {i["module"]: i for i in self.build()["items"] if "module" in i}
food = items["crates/w/src/food.rs"]
self.assertEqual((food["rank"], food["kind"], food["harnesses"]), (2, "rerun", ["food"]))
self.assertEqual(food["classes"]["net/minecraft/world/food/FoodData"]["named"], ["tick"])
sky = items["crates/c/src/sky.rs"]
self.assertEqual((sky["rank"], sky["kind"]), (3, "unproved"))
# Named by many files and none names the changed method: grouped by package, harnesses by coverage.
widely = items["widely named: world/entity"]
self.assertEqual(widely["harnesses"], ["food", "move"])
unported = next(i for i in self.build()["items"] if i["kind"] == "unported")
self.assertEqual(unported["packages"], {"net/minecraft/server": 1})
def test_failed_harness_ranks_first_in_rank_two(self):
items = self.build({"food": "fail", "move": "pass"})["items"]
first = next(i for i in items if i["rank"] == 2)
self.assertEqual((first["kind"], first["failed"]), ("harness-failed", ["food"]))
def test_new_content(self):
items = self.build()["items"]
self.assertIn("minecraft:new_mob", next(i for i in items if i["kind"] == "content")["entries"])
new = next(i for i in items if i["kind"] == "new-classes")
self.assertEqual((new["classes"], new["ported"]),
(["net/minecraft/world/entity/NewMob"], ["net/minecraft/world/entity/NewMob"]))
self.assertEqual(next(i for i in items if i["kind"] == "new-unported")["packages"],
{"net/minecraft/elsewhere": 1})
def test_markdown(self):
text = report.markdown(self.build())
self.assertTrue(text.startswith("# Port report: Minecraft 26.2 -> 26.3"))
self.assertIn("## 1. Blocks the switch", text)
self.assertIn("play/serverbound: added minecraft:punch; removed minecraft:swing", text)
self.assertIn("FoodData (2 methods; tick)", text)
def test_helpers(self):
self.assertEqual(report.snake("LivingEntity"), "living_entity")
entry = {"rust": {"a/x.rs": ["tick"], "a/living_entity.rs": [], "a/y.rs": ["tick", "eat"]}}
self.assertEqual(report.primary_file(entry, {"tick", "eat"}, "n/LivingEntity"), "a/y.rs")
self.assertEqual(report.primary_file(entry, {"other"}, "n/LivingEntity"), "a/living_entity.rs")
self.assertEqual(report.primary_file(entry, set(), "n/Other"), "a/living_entity.rs")
self.assertIsNone(report.primary_file({"rust": {f"{i}.rs": [] for i in range(4)}}, set(), "n/Other"))
class VersionPortReportTests(unittest.TestCase):
def test_harness_results_merge_shards(self):
with tempfile.TemporaryDirectory() as directory:
state = Path(directory) / "state.json"
state.write_text(json.dumps({"harnesses": {"food": {"status": "pass"}, "datafix#1": {"status": "pass"},
"datafix#2": {"status": "fail"}}}))
self.assertEqual(port.harness_results(state), {"food": "pass", "datafix": "fail"})
def test_class_names_drop_moved_old_names(self):
self.assertEqual(port.class_names({"a/X": 0, "a/Y": 0}, {"b/X": 0, "a/Y": 0}, {"a/X": "b/X"}),
["a/Y", "b/X"])
if __name__ == "__main__":
unittest.main()
+322
View File
@@ -0,0 +1,322 @@
#!/usr/bin/env python3
"""The port report: a version diff (tools/version_port.py diff) turned into ranked, area-sized work items
(docs/version-ports.md, "The port report").
Ranks:
1 blocks the switch: version facts, packets, the save format (DataFixers), data component types, report
formats, generated tables to regenerate
2 a harness failed against the new version (with --results), else changed vanilla code a harness proves:
rerun it
3 changed vanilla code Horizon ports and no harness proves
4 new content: registry entries, and new classes in the vanilla packages Horizon ports
5 changed and new vanilla code Horizon doesn't port (two summary items)
Used through `version_port.py report OLD NEW`; the functions here take plain data so tests can build it.
"""
from collections import Counter, defaultdict
from pathlib import Path
import re
import subprocess
try:
from . import class_index, vanilla_map
except ImportError:
import class_index
import vanilla_map
ROOT = Path(__file__).resolve().parents[1]
PROTOCOL = "net/minecraft/network/protocol/"
DATAFIX = "net/minecraft/util/datafix/"
# Generated tables, what makes them stale, and how to regenerate them (docs/version-ports.md, "Regenerate").
GENERATORS = [
("crates/horizon-protocol/src/ids.rs", "python3 tools/generate_packet_ids.py",
lambda d: bool(d["packets"])),
("crates/horizon-client/src/lan/tracking.rs", "python3 tools/generate_entity_tracking.py",
lambda d: "minecraft:entity_type" in d["registries"]["changed"]
or "net/minecraft/world/entity/EntityType" in d["classes"]["changed"]),
("crates/horizon-client/src/lan/command_suggestions.rs", "python3 tools/generate_command_suggestions.py",
lambda d: bool(d["commands"]["added"] or d["commands"]["removed"] or d["commands"]["changed"])),
("crates/horizon-client/src/lan/command_order.rs", "python3 tools/generate_lan_command_order.py",
lambda d: bool(d["commands"]["added"] or d["commands"]["removed"])),
("crates/horizon-client/src/command/vanilla_order.rs", "python3 tools/generate_command_order.py",
lambda d: bool(d["commands"]["added"] or d["commands"]["removed"])),
("crates/horizon-world/src/datafix/block_state_data.txt", "python3 tools/gen_block_state_data.py",
lambda d: "net/minecraft/util/datafix/fixes/BlockStateData" in d["classes"]["changed"]),
]
# Registries whose new entries are new things to port.
CONTENT = ["minecraft:block", "minecraft:item", "minecraft:entity_type", "minecraft:block_entity_type",
"minecraft:particle_type", "minecraft:menu", "minecraft:worldgen/feature", "minecraft:sound_event"]
def version_sites(facts, root=ROOT):
"""file:line of every place the code states these version facts (not tests)."""
pattern = ["-e", str(facts["protocol_version"]), "-e", str(facts["world_version"])]
found = []
for command in (["git", "grep", "-n", "-w"] + pattern, ["git", "grep", "-n", "-F", f'"{facts["id"]}"']):
result = subprocess.run(command + ["--", "crates/*.rs", "launcher/*.rs", "tools/*.py"],
cwd=root, capture_output=True, text=True)
found += [line.split(":", 2)[0] + ":" + line.split(":", 2)[1] for line in result.stdout.splitlines()]
return sorted({site for site in found if not re.search(r"(^|/)tests?(/|\.rs)|/test_|_tests?\.rs", site)})
def files_containing(text, root=ROOT):
"""Rust and Python files that contain this text (a report's file name)."""
command = ["git", "grep", "-l", "-F", text, "--", "crates/*.rs", "launcher/*.rs", "tools/*.py"]
return subprocess.run(command, cwd=root, capture_output=True, text=True).stdout.split()
def files_naming(words, root=ROOT):
"""Rust files that contain any of these words (component or packet names)."""
if not words:
return []
command = ["git", "grep", "-l", "-w"] + [arg for word in words for arg in ("-e", word)] + ["--", "crates/*.rs"]
return subprocess.run(command, cwd=root, capture_output=True, text=True).stdout.split()
def snake(name):
"""LivingEntity -> living_entity."""
return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
def primary_file(entry, members, cls):
"""The Horizon file that ports a changed class: the one whose comments name the most of its changed
methods, else the one named after it, else one of the few files that name it. None for a class many
files name in passing (Entity, BlockPos) when none names a changed method."""
files = entry["rust"]
scored = sorted((-len(set(named) & members), path) for path, named in files.items() if set(named) & members)
if scored:
return scored[0][1]
stem = snake(cls.rsplit("/", 1)[-1].split("$")[0])
named_after = sorted(path for path in files if Path(path).stem == stem)
if named_after:
return named_after[0]
return sorted(files)[0] if 0 < len(files) <= 3 else None
def method_names(change):
methods = change.get("methods", {})
return {m.split("(")[0] for kind in ("added", "removed", "changed") for m in methods.get(kind, [])}
def item(rank, kind, title, **details):
return {"rank": rank, "kind": kind, "title": title, **details}
def blocking_items(diff, mapping, pinned_facts, root):
items = []
if diff["facts"]:
changes = ", ".join(f"{k} {a} -> {b}" for k, (a, b) in diff["facts"].items())
items.append(item(1, "facts", f"Version facts: {changes}", files=version_sites(pinned_facts, root),
details=diff["facts"]))
if diff["packets"]:
classes = diff["classes"]
packets = {"added": [c for c in classes["added"] if c.startswith(PROTOCOL)],
"removed": [c for c in classes["removed"] if c.startswith(PROTOCOL)],
"changed": [c for c in classes["changed"] if c.startswith(PROTOCOL)]}
names = sorted({n.split(":")[1] for flow in diff["packets"].values()
for n in list(flow["added"]) + list(flow["removed"])})
constants = [n.upper().replace("/", "_") for n in names]
counts = ", ".join(f"{flow} +{len(c['added'])} -{len(c['removed'])} ~{len(c['renumbered'])}"
for flow, c in diff["packets"].items())
items.append(item(1, "protocol", f"Protocol: packets {counts}",
files=sorted(set(files_naming(constants, root)) | {"crates/horizon-protocol/src/ids.rs"}),
details={"packets": diff["packets"], "classes": packets}))
if "world_version" in diff["facts"]:
classes = diff["classes"]
added = [c for c in classes["added"] if c.startswith(DATAFIX)]
changed = [c for c in classes["changed"] if c.startswith(DATAFIX)]
schemas = sorted(c for c in added if "/schemas/V" in c and "$" not in c)
fixes = sorted(c for c in added if "/fixes/" in c and "$" not in c)
horizon = vanilla_map.lookup(mapping, "net/minecraft/util/datafix/DataFixers")
named = {f for c in schemas + fixes for f in mapping["classes"].get(c, {}).get("rust", {})}
a, b = diff["facts"]["world_version"]
items.append(item(1, "save-format", f"Save format: DataFixers {a} -> {b}: {len(schemas)} schemas, "
f"{len(fixes)} fixes added, {len(changed)} datafix classes changed",
files=sorted(set(horizon["rust"]) | named), harnesses=horizon["harnesses"],
details={"schemas": schemas, "fixes": fixes, "changed": changed}))
registry = diff["registries"]["changed"].get("minecraft:data_component_type", {})
if registry.get("added") or registry.get("removed"):
names = [n.split(":")[1] for n in registry.get("added", []) + registry.get("removed", [])]
items.append(item(1, "components", f"Data component types: +{len(registry.get('added', []))} "
f"-{len(registry.get('removed', []))} (sent on the wire and saved)",
files=files_naming(names, root),
details={"types": registry, "items": diff["items"]["components"]}))
if diff["blocks"].get("format"):
items.append(item(1, "report-format", f"blocks.json format: {', '.join(diff['blocks']['format'])}",
files=files_containing("blocks.json", root),
details={"fields": diff["blocks"]["format"]}))
for output, command, stale in GENERATORS:
if stale(diff):
items.append(item(1, "regenerate", f"Regenerate {output}", files=[output], command=command))
return items
def item_harnesses(path, classes, mapping, limit=8):
"""The harnesses for an item: those whose probes reach its file, else those covering the most of its
classes, the most specific first."""
harnesses = mapping["harnesses"]
reaching = sorted(h for h, data in harnesses.items() if path in data["rust"])
if reaching:
return reaching
covering = Counter(h for c in classes for h in vanilla_map.lookup(mapping, c)["harnesses"])
return sorted(sorted(covering, key=lambda h: (-covering[h], len(harnesses[h]["classes"]), h))[:limit])
def code_items(diff, mapping, results=None):
"""Ranks 2, 3 and 5 from the changed and removed vanilla classes, one item per Horizon file that ports
them (classes many files name in passing go by vanilla package)."""
classes = diff["classes"]
groups = defaultdict(lambda: {"classes": {}, "files": set(), "precise": 0, "methods": 0})
unported = Counter()
for name in list(classes["changed"]) + classes["removed"]:
if name.startswith((PROTOCOL, DATAFIX)):
continue # The protocol and save-format items have them.
entry = classes["horizon"].get(name)
if not entry:
unported[name.rsplit("/", 1)[0]] += 1
continue
removed = name not in classes["changed"]
change = classes["changed"].get(name, {})
members = method_names(change)
path = primary_file(entry, members, name) if entry["rust"] else None
key = path or "widely named: " + name.rsplit("/", 1)[0].removeprefix("net/minecraft/")
group = groups[key]
precise = sorted({m for named in entry["rust"].values() for m in named} & members)
methods = 1 if removed else class_index.changed_methods(change)
group["classes"][name] = {"methods": methods, "named": precise, "removed": removed}
group["files"].update([path] if path else entry["rust"])
group["precise"] += len(precise)
group["methods"] += methods
failing = {h for h, status in (results or {}).items() if status != "pass"}
items = []
for key, group in groups.items():
harnesses = item_harnesses(key, group["classes"], mapping)
failed = sorted(set(harnesses) & failing)
rank, kind = (2, "harness-failed") if failed else (2, "rerun") if harnesses else (3, "unproved")
top = sorted(group["classes"], key=lambda c: (-group["classes"][c]["methods"], c))
title = (f"{key}: {len(group['classes'])} vanilla classes changed "
f"({group['methods']} methods, {group['precise']} named in Horizon)")
items.append(item(rank, kind, title, module=key, files=sorted(group["files"]), harnesses=harnesses,
failed=failed, classes={c: group["classes"][c] for c in top},
score=group["methods"] + 10 * group["precise"] + (1000 if failed else 0)))
if unported:
items.append(item(5, "unported", f"{sum(unported.values())} changed or removed vanilla classes Horizon "
f"doesn't port, in {len(unported)} packages", packages=dict(unported.most_common())))
return items
def content_items(diff, mapping):
"""Rank 4: new registry entries, and new classes by vanilla package where Horizon ports other classes of
that package (with those Horizon already names, after a port); new classes elsewhere go in one summary."""
items = []
for registry in CONTENT:
entries = diff["registries"]["changed"].get(registry, {}).get("added", [])
if entries:
items.append(item(4, "content", f"New {registry.split(':')[1]}: {len(entries)} entries", entries=entries))
ported_packages = Counter(c.rsplit("/", 1)[0] for c, e in mapping["classes"].items() if e["rust"])
by_package = defaultdict(list)
for name in diff["classes"]["added"]:
if not re.search(r"\$\d", name) and not name.startswith((PROTOCOL, DATAFIX)):
by_package[name.rsplit("/", 1)[0]].append(name)
elsewhere = Counter()
for package, names in sorted(by_package.items()):
if package not in ported_packages:
elsewhere[package] += len(names)
continue
ported = sorted(c for c in names if mapping["classes"].get(c, {}).get("rust"))
nearby = Counter(f for c, e in mapping["classes"].items() if c.rsplit("/", 1)[0] == package for f in e["rust"])
files = sorted({f for c in ported for f in mapping["classes"][c]["rust"]}) or [f for f, _ in nearby.most_common(5)]
items.append(item(4, "new-classes", f"New in {package.removeprefix('net/minecraft/')}: {len(names)} classes"
f" ({len(ported)} already named in Horizon)", classes=sorted(names), ported=ported,
files=files, score=len(names)))
if elsewhere:
items.append(item(5, "new-unported", f"{sum(elsewhere.values())} new vanilla classes in {len(elsewhere)} "
f"packages Horizon doesn't port", packages=dict(elsewhere.most_common())))
return items
def build(diff, mapping, old, new, pinned_facts, results=None, root=ROOT):
"""The report: summary, ranked items and what came out clean. pinned_facts: the facts of the version the
code states today (to find where it states them); results: {harness: "pass" | "fail"} against NEW."""
items = blocking_items(diff, mapping, pinned_facts, root)
items += code_items(diff, mapping, results) + content_items(diff, mapping)
items.sort(key=lambda i: (i["rank"], -i.get("score", 0), i["title"]))
for number, entry in enumerate(items, 1):
entry["id"] = number
clean = [kind for kind in ("packets", "commands") if not diff[kind]]
clean += [kind for kind in ("blocks", "items") if not (diff[kind]["added"] or diff[kind]["removed"]
or diff[kind]["changed"])]
classes = diff["classes"]
return {
"old": old, "new": new,
"summary": {
"facts": diff["facts"], "ranks": dict(sorted(Counter(i["rank"] for i in items).items())),
"classes": {"added": len(classes["added"]), "removed": len(classes["removed"]),
"moved": len(classes["moved"]), "changed": len(classes["changed"]),
"horizon": len(classes["horizon"])},
"generators": [i["files"][0] for i in items if i["kind"] == "regenerate"],
},
"items": items, "clean": clean,
}
def short(name):
return name.rsplit("/", 1)[-1].replace("$", ".")
def markdown(report, limit=12):
"""report.md: what a person or an agent reads first."""
lines = [f"# Port report: Minecraft {report['old']} -> {report['new']}", ""]
summary = report["summary"]
for key, (a, b) in summary["facts"].items():
lines.append(f"- {key}: {a} -> {b}")
c = summary["classes"]
lines.append(f"- classes: +{c['added']} -{c['removed']}, {c['moved']} moved, {c['changed']} changed; "
f"{c['horizon']} changed or removed classes concern Horizon code or harnesses")
lines.append(f"- work items by rank: " + ", ".join(f"{r}: {n}" for r, n in summary["ranks"].items()))
if report["clean"]:
lines.append(f"- unchanged: {', '.join(report['clean'])}")
names = {1: "Blocks the switch", 2: "Proved by a harness: rerun it", 3: "Ported, no harness",
4: "New content", 5: "Not ported"}
for rank in sorted({i["rank"] for i in report["items"]}):
lines += ["", f"## {rank}. {names[rank]}", ""]
for entry in [i for i in report["items"] if i["rank"] == rank]:
lines.append(f"### #{entry['id']} {entry['title']}")
if entry.get("command"):
lines.append(f"- run: `{entry['command']}`")
if entry.get("failed"):
lines.append(f"- failing harnesses: {', '.join(entry['failed'])}")
if entry.get("harnesses"):
lines.append(f"- harnesses: {', '.join(entry['harnesses'][:limit])}"
+ (f" (+{len(entry['harnesses']) - limit})" if len(entry["harnesses"]) > limit else ""))
if entry.get("files"):
lines.append(f"- Horizon: {', '.join(f'`{f}`' for f in entry['files'][:limit])}"
+ (f" (+{len(entry['files']) - limit})" if len(entry["files"]) > limit else ""))
if entry.get("classes") and isinstance(entry["classes"], dict):
shown = list(entry["classes"].items())[:limit]
lines.append("- vanilla: " + ", ".join(
f"{short(n)} ({'removed' if v['removed'] else str(v['methods']) + ' methods'}"
+ (f"; {', '.join(v['named'])}" if v["named"] else "") + ")" for n, v in shown)
+ (f" (+{len(entry['classes']) - limit})" if len(entry["classes"]) > limit else ""))
elif entry.get("classes"):
lines.append(f"- new classes: {', '.join(short(n) for n in entry['classes'][:limit])}"
+ (f" (+{len(entry['classes']) - limit})" if len(entry["classes"]) > limit else ""))
if entry.get("entries"):
lines.append(f"- entries: {', '.join(n.split(':')[-1] for n in entry['entries'][:limit])}"
+ (f" (+{len(entry['entries']) - limit})" if len(entry["entries"]) > limit else ""))
if entry.get("ported"):
lines.append(f"- already in Horizon: {', '.join(short(n) for n in entry['ported'][:limit])}")
details = entry.get("details", {})
for key in ("schemas", "fixes"):
if details.get(key):
lines.append(f"- {key}: {', '.join(short(n) for n in details[key])}")
if entry["kind"] == "protocol":
for flow, change in details["packets"].items():
lines.append(f"- {flow}: added {', '.join(change['added']) or 'none'}; "
f"removed {', '.join(change['removed']) or 'none'}")
if entry["kind"] in ("unported", "new-unported"):
lines.append("- " + ", ".join(f"{p.removeprefix('net/minecraft/')} {n}"
for p, n in list(entry["packages"].items())[:limit]))
lines.append("")
return "\n".join(lines).rstrip() + "\n"
+68 -12
View File
@@ -8,6 +8,8 @@
version_port.py diff OLD NEW facts, packets, registries, blocks, items, commands,
data packs, assets, lang and classes, with the Horizon
code each changed class concerns: diff-OLD-NEW/diff.json
version_port.py report OLD NEW [--results S] the ranked port report: diff-OLD-NEW/report.md and .json
version_port.py port NEW [--old OLD] prepare both, diff, report
version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep
The downloads are for development only: they go to the version store (~/code/horizon-parity/versions,
@@ -31,10 +33,11 @@ import urllib.request
import zipfile
try:
from . import class_index, lab, vanilla_map
from . import class_index, lab, port_report, vanilla_map
except ImportError:
import class_index
import lab
import port_report
import vanilla_map
MANIFEST_URL = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"
@@ -438,13 +441,17 @@ def read_reports(reports):
}
def diff_classes(old, new, mapping):
"""The class diff, each changed or removed class with the Horizon files and harnesses that concern it
(looked up under its old name: Horizon's code names the pinned version's classes)."""
result = class_index.diff(old, new)
def class_names(old, new, moved):
"""Every class name of either version, but a moved class only under its new name, so that its simple
name stays unambiguous."""
return sorted(set(new) | (set(old) - set(moved)))
def diff_classes(result, mapping):
"""The class diff, each changed or removed class with the Horizon files and harnesses that concern it."""
horizon = {}
for name, change in result["changed"].items():
entry = vanilla_map.lookup(mapping, change.get("moved_from", name))
for name in result["changed"]:
entry = vanilla_map.lookup(mapping, name)
if entry["rust"] or entry["harnesses"]:
horizon[name] = entry
for name in result["removed"]:
@@ -455,9 +462,9 @@ def diff_classes(old, new, mapping):
return result
def diff_versions(old_paths, new_paths, old_facts, new_facts, old_classes=None, new_classes=None, mapping=None):
def diff_versions(old_paths, new_paths, old_facts, new_facts, classes=None, mapping=None):
a, b = read_reports(old_paths["reports"]), read_reports(new_paths["reports"])
classes = diff_classes(old_classes, new_classes, mapping) if old_classes is not None else None
classes = diff_classes(classes, mapping) if classes is not None else None
return {
"facts": diff_facts(old_facts, new_facts),
"packets": diff_packets(a["packets"], b["packets"]),
@@ -526,16 +533,53 @@ def load(version, name):
def diff(args):
directory = diff_dir(args.old, args.new)
old_classes, new_classes = load(args.old, "classes.json"), load(args.new, "classes.json")
mapping = vanilla_map.build(old_classes, dependencies=vanilla_map.recorded_dependencies(vanilla_map.default_shared()))
classes = class_index.diff(old_classes, new_classes)
mapping = vanilla_map.build(class_names(old_classes, new_classes, classes["moved"]),
dependencies=vanilla_map.recorded_dependencies(vanilla_map.default_shared()))
lab.write_json(directory / "vanilla_map.json", mapping)
result = diff_versions(paths(args.old), paths(args.new), load(args.old, "facts.json"), load(args.new, "facts.json"),
old_classes, new_classes, mapping)
classes, mapping)
lab.write_json(directory / "diff.json", result)
print(summary(args.old, args.new, result))
print(f"\n{directory / 'diff.json'}")
return 0
# --- report ------------------------------------------------------------------------------------------------
def harness_results(state_path):
"""{harness: "pass" | "fail"} from tools/parity.py's state; a harness fails if one of its shards does."""
found = {}
for name, record in json.loads(Path(state_path).read_text()).get("harnesses", {}).items():
harness = name.split("#")[0]
found[harness] = "fail" if record.get("status") != "pass" or found.get(harness) == "fail" else "pass"
return found
def report(args):
directory = diff_dir(args.old, args.new)
if not (directory / "diff.json").exists():
raise SystemExit(f"No diff yet; run version_port.py diff {args.old} {args.new}")
result = json.loads((directory / "diff.json").read_text())
mapping = json.loads((directory / "vanilla_map.json").read_text())
results = harness_results(args.results) if args.results else None
built = port_report.build(result, mapping, args.old, args.new, load(pinned(), "facts.json"), results)
lab.write_json(directory / "report.json", built)
(directory / "report.md").write_text(port_report.markdown(built))
print("\n".join(port_report.markdown(built).splitlines()[:args.lines]))
print(f"\n{directory / 'report.md'} ({len(built['items'])} items)")
return 0
def port(args):
"""prepare both, diff, report."""
old = args.old or pinned()
for version in (old, args.new):
prepare(argparse.Namespace(version=version, download=False, jobs=args.jobs))
diff(argparse.Namespace(old=old, new=args.new))
return report(argparse.Namespace(old=old, new=args.new, results=args.results, lines=args.lines))
# --- gc ----------------------------------------------------------------------------------------------------
def to_remove(entries, pin, keep):
@@ -579,12 +623,24 @@ def main():
command = commands.add_parser("diff", help="diff two prepared versions")
command.add_argument("old")
command.add_argument("new")
command = commands.add_parser("report", help="the ranked port report of a diff")
command.add_argument("old")
command.add_argument("new")
command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW")
command.add_argument("--lines", type=int, default=60, help="how much of report.md to print")
command = commands.add_parser("port", help="prepare, diff and report in one go")
command.add_argument("new")
command.add_argument("--old", help="default: the pinned version")
command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW")
command.add_argument("--lines", type=int, default=60)
command.add_argument("-j", "--jobs", type=int, default=int(os.environ.get("FLEET_LOAD_JOBS", 0)) or None)
command = commands.add_parser("gc", help="delete versions and scratch the store doesn't need")
command.add_argument("--keep", nargs="*", default=[])
command.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
STORE.mkdir(parents=True, exist_ok=True)
return {"detect": detect, "prepare": prepare, "diff": diff, "gc": gc}[args.command](args)
return {"detect": detect, "prepare": prepare, "diff": diff, "report": report, "port": port,
"gc": gc}[args.command](args)
if __name__ == "__main__":