Merge pull request #325 from ludovic111/feat/version-port-report
Port report: ranked work items from a Minecraft version diff
This commit is contained in:
+25
-2
@@ -256,7 +256,28 @@ Manual, by agents (and Ludovic's review):
|
||||
- **New JDK**: a fleet toolchain install, once per Java bump.
|
||||
- **Judgment on the switch**: merging the pin bump and releasing stay Ludovic's call.
|
||||
|
||||
The replay of 26.2 → 26.3 below measures how much that is for one release.
|
||||
## Replay: 26.2 → 26.3
|
||||
|
||||
Horizon was written against 26.3 from its first commit, so its code is the ground truth of what porting 26.2 to 26.3
|
||||
takes. On the Mac mini, `prepare 26.2` took 9 s (136 MB), `diff 26.2 26.3` 6 s and `report` under a second.
|
||||
|
||||
| What 26.3 needed (in Horizon's code) | What the report found |
|
||||
| --- | --- |
|
||||
| Protocol 777, DataVersion 5023, pack formats | rank 1: protocol 776 → 777, world version 4903 → 5023, resource pack 88.0 → 97.1, data pack 107.1 → 121.0, with every site stating them (`PROTOCOL_VERSION`, both `DATA_VERSION`s, the `"26.3"` names) |
|
||||
| `ids.rs`: `ADD_TRANSIENT_BLOCK`, `POST_EFFECTS` (configuration and play), `SWING_ANIMATION`, serverbound `PUNCH` instead of `SWING`; handlers in `packets/play.rs`, `lan/hub.rs` | rank 1: exactly those packets, 131 renumbered, those files, and "regenerate `ids.rs`". Regenerating from 26.3's report reproduces the committed file byte for byte (after rustfmt); from 26.2's it differs by 274 lines |
|
||||
| DataFixers 4996-5016 in `datafix/fixes_c.rs` | rank 1: schemas V4996, V4996_1, V4997, V5000 and 16 new fixes (`MergeTerrainChunkStatusFix`, `ExplorerMapItemFix`, `SwingAnimationComponentSplitFix`...), with `datafix/` |
|
||||
| `swing_animation` split into `attack_animation` and `interact_animation` (`first_person.rs`, `game/hand.rs`, `item_model.rs`) | rank 1: data component types +13 -2, those files among the ones naming them; per component, 1,537 items affected |
|
||||
| `lan/tracking.rs` and the command tables | rank 1: regenerate `tracking.rs`, `command_suggestions.rs`, `command_order.rs`, `vanilla_order.rs` |
|
||||
| 133 methods that exist only in 26.3 and that Horizon's comments name (`Minecraft.startAttack`, `MultiPlayerGameMode.piercingAttack`, `SkyRenderer.buildEndSky`, `GameRenderer.render3dHud`...) | all 133: 132 in rank 2-3 items with the method named, on the file that ports it; 1 in the save-format item |
|
||||
| 49 classes that exist only in 26.3 and that Horizon names (`ComputeCommand`, `PostEffectCommand`, `LivingEntity.SwingState`, `BlockTransformer`, `StrawBedBlock`...) | all 49, in rank 4 "New in <package>" items |
|
||||
|
||||
The report has 499 items: 10 of rank 1, 311 of rank 2, 96 of rank 3, 80 of rank 4, and 2 rank-5 summaries. Rank 1 is
|
||||
exact. Ranks 2 and 3 are leads, not all work: 173 of their 407 items have a changed method that Horizon's comments
|
||||
name. The rest are classes whose changed methods Horizon doesn't name. Many of those are mechanical (26.3 renamed
|
||||
`GuiGraphics` to `GuiGraphicsExtractor` and reworked the render pipeline), and 105 are "widely named" buckets
|
||||
(classes such as `Entity` that many files mention in passing). Running every harness against the new version (with
|
||||
`--results`) is what turns a rank-2 lead into a confirmed failure; the replay cannot do that, because the oracles are
|
||||
written against 26.3.
|
||||
|
||||
## Implementation status
|
||||
|
||||
@@ -265,6 +286,8 @@ python3 tools/version_port.py detect --snapshots # exit 10: new release, 11:
|
||||
python3 tools/version_port.py prepare 26.3 # the pinned version: indexes .local, downloads nothing
|
||||
python3 tools/version_port.py prepare 26.4 # downloads, verifies, runs the data generator (about 10 s)
|
||||
python3 tools/version_port.py diff 26.3 26.4 # summary; the full diff in versions/diff-26.3-26.4/diff.json
|
||||
python3 tools/version_port.py report 26.3 26.4 [--results .local/parity-runner/state.json] # report.md, report.json
|
||||
python3 tools/version_port.py port 26.4 # prepare both, diff, report
|
||||
python3 tools/version_port.py gc # keep the pinned version and the newest two others
|
||||
python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food.FoodData # its Rust files and harnesses
|
||||
```
|
||||
@@ -274,5 +297,5 @@ python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food
|
||||
| This design | this page |
|
||||
| `detect`, `prepare`, `diff` (facts, packets, registries, blocks, items, commands, data, assets), `gc` | done |
|
||||
| Class index (`tools/class_index.py`), class diff with moves, `vanilla_map.json` (`tools/vanilla_map.py`) | done |
|
||||
| Port report (`report`), `port` | next PR |
|
||||
| Port report (`tools/port_report.py`, `report`), `port` | done |
|
||||
| Changed-class decompilation, `regen`, `version_consts` refactor, candidate harness run, `publish`, launchd agent | later |
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from tools import port_report as report
|
||||
from tools import version_port as port
|
||||
|
||||
|
||||
def mapping():
|
||||
return {
|
||||
"classes": {
|
||||
"net/minecraft/world/food/FoodData": {
|
||||
"rust": {"crates/w/src/food.rs": ["tick"], "crates/c/src/hud.rs": []}, "harnesses": ["food"]},
|
||||
"net/minecraft/world/entity/Entity": {
|
||||
"rust": {f"crates/c/src/f{i}.rs": [] for i in range(5)}, "harnesses": ["food", "move"]},
|
||||
"net/minecraft/client/Sky": {"rust": {"crates/c/src/sky.rs": []}, "harnesses": []},
|
||||
"net/minecraft/world/entity/NewMob": {"rust": {"crates/c/src/new_mob.rs": []}, "harnesses": []},
|
||||
"net/minecraft/util/datafix/DataFixers": {"rust": {"crates/w/src/datafix/mod.rs": []},
|
||||
"harnesses": ["datafix"]},
|
||||
},
|
||||
"harnesses": {
|
||||
"food": {"rust": ["crates/w/src/food.rs"], "classes": ["net/minecraft/world/food/FoodData"]},
|
||||
"move": {"rust": [], "classes": ["net/minecraft/world/entity/Entity"]},
|
||||
"datafix": {"rust": [], "classes": []},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def diff():
|
||||
def methods(**kinds):
|
||||
return {"methods": {k: kinds.get(k, []) for k in ("added", "removed", "changed")}}
|
||||
changed = {
|
||||
"net/minecraft/world/food/FoodData": methods(changed=["tick()V", "eat(I)V"]),
|
||||
"net/minecraft/world/entity/Entity": methods(changed=["move()V"]),
|
||||
"net/minecraft/client/Sky": methods(added=["stars()V"]),
|
||||
"net/minecraft/util/datafix/DataFixers": methods(changed=["addFixers()V"]),
|
||||
"net/minecraft/server/Unported": methods(changed=["run()V"]),
|
||||
}
|
||||
m = mapping()
|
||||
return {
|
||||
"facts": {"protocol_version": [776, 777], "world_version": [4903, 5023]},
|
||||
"packets": {"play/serverbound": {"added": {"minecraft:punch": 46}, "removed": {"minecraft:swing": 63},
|
||||
"renumbered": {}, "count": [69, 69]}},
|
||||
"registries": {"added": {}, "removed": [], "changed": {
|
||||
"minecraft:entity_type": {"added": ["minecraft:new_mob"]},
|
||||
"minecraft:data_component_type": {"added": ["minecraft:attack_animation"],
|
||||
"removed": ["minecraft:swing_animation"]}}},
|
||||
"blocks": {"added": [], "removed": [], "changed": {}, "format": ["definition"]},
|
||||
"items": {"added": [], "removed": [], "changed": {}, "components": {}},
|
||||
"commands": {"added": [], "removed": [], "changed": {}},
|
||||
"classes": {
|
||||
"added": ["net/minecraft/world/entity/NewMob", "net/minecraft/util/datafix/schemas/V4996",
|
||||
"net/minecraft/util/datafix/fixes/SwingSplitFix", "net/minecraft/network/protocol/game/Punch",
|
||||
"net/minecraft/elsewhere/Thing"],
|
||||
"removed": ["net/minecraft/network/protocol/game/Swing"], "moved": {},
|
||||
"changed": changed,
|
||||
"horizon": {name: m["classes"][name] for name in changed if name in m["classes"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class PortReportTests(unittest.TestCase):
|
||||
def build(self, results=None):
|
||||
with mock.patch.object(report, "version_sites", return_value=["crates/p/src/lib.rs:20"]), \
|
||||
mock.patch.object(report, "files_naming", return_value=["crates/p/src/play.rs"]), \
|
||||
mock.patch.object(report, "files_containing", return_value=["crates/w/src/blocks.rs"]):
|
||||
return report.build(diff(), mapping(), "26.2", "26.3", {"id": "26.3"}, results)
|
||||
|
||||
def test_ranks_and_kinds(self):
|
||||
built = self.build()
|
||||
kinds = [(i["rank"], i["kind"]) for i in built["items"]]
|
||||
self.assertEqual(sorted(k for r, k in kinds if r == 1),
|
||||
["components", "facts", "protocol", "regenerate", "regenerate", "report-format",
|
||||
"save-format"])
|
||||
self.assertEqual([i["id"] for i in built["items"]], list(range(1, len(kinds) + 1)))
|
||||
self.assertEqual(kinds, sorted(kinds, key=lambda k: k[0]))
|
||||
regenerated = {i["files"][0] for i in built["items"] if i["kind"] == "regenerate"}
|
||||
self.assertEqual(regenerated, {"crates/horizon-protocol/src/ids.rs", "crates/horizon-client/src/lan/tracking.rs"})
|
||||
|
||||
def test_save_format_lists_new_schemas_and_fixes(self):
|
||||
save = next(i for i in self.build()["items"] if i["kind"] == "save-format")
|
||||
self.assertEqual(save["details"]["schemas"], ["net/minecraft/util/datafix/schemas/V4996"])
|
||||
self.assertEqual(save["details"]["fixes"], ["net/minecraft/util/datafix/fixes/SwingSplitFix"])
|
||||
self.assertEqual(save["files"], ["crates/w/src/datafix/mod.rs"])
|
||||
|
||||
def test_code_items_by_file(self):
|
||||
items = {i["module"]: i for i in self.build()["items"] if "module" in i}
|
||||
food = items["crates/w/src/food.rs"]
|
||||
self.assertEqual((food["rank"], food["kind"], food["harnesses"]), (2, "rerun", ["food"]))
|
||||
self.assertEqual(food["classes"]["net/minecraft/world/food/FoodData"]["named"], ["tick"])
|
||||
sky = items["crates/c/src/sky.rs"]
|
||||
self.assertEqual((sky["rank"], sky["kind"]), (3, "unproved"))
|
||||
# Named by many files and none names the changed method: grouped by package, harnesses by coverage.
|
||||
widely = items["widely named: world/entity"]
|
||||
self.assertEqual(widely["harnesses"], ["food", "move"])
|
||||
unported = next(i for i in self.build()["items"] if i["kind"] == "unported")
|
||||
self.assertEqual(unported["packages"], {"net/minecraft/server": 1})
|
||||
|
||||
def test_failed_harness_ranks_first_in_rank_two(self):
|
||||
items = self.build({"food": "fail", "move": "pass"})["items"]
|
||||
first = next(i for i in items if i["rank"] == 2)
|
||||
self.assertEqual((first["kind"], first["failed"]), ("harness-failed", ["food"]))
|
||||
|
||||
def test_new_content(self):
|
||||
items = self.build()["items"]
|
||||
self.assertIn("minecraft:new_mob", next(i for i in items if i["kind"] == "content")["entries"])
|
||||
new = next(i for i in items if i["kind"] == "new-classes")
|
||||
self.assertEqual((new["classes"], new["ported"]),
|
||||
(["net/minecraft/world/entity/NewMob"], ["net/minecraft/world/entity/NewMob"]))
|
||||
self.assertEqual(next(i for i in items if i["kind"] == "new-unported")["packages"],
|
||||
{"net/minecraft/elsewhere": 1})
|
||||
|
||||
def test_markdown(self):
|
||||
text = report.markdown(self.build())
|
||||
self.assertTrue(text.startswith("# Port report: Minecraft 26.2 -> 26.3"))
|
||||
self.assertIn("## 1. Blocks the switch", text)
|
||||
self.assertIn("play/serverbound: added minecraft:punch; removed minecraft:swing", text)
|
||||
self.assertIn("FoodData (2 methods; tick)", text)
|
||||
|
||||
def test_helpers(self):
|
||||
self.assertEqual(report.snake("LivingEntity"), "living_entity")
|
||||
entry = {"rust": {"a/x.rs": ["tick"], "a/living_entity.rs": [], "a/y.rs": ["tick", "eat"]}}
|
||||
self.assertEqual(report.primary_file(entry, {"tick", "eat"}, "n/LivingEntity"), "a/y.rs")
|
||||
self.assertEqual(report.primary_file(entry, {"other"}, "n/LivingEntity"), "a/living_entity.rs")
|
||||
self.assertEqual(report.primary_file(entry, set(), "n/Other"), "a/living_entity.rs")
|
||||
self.assertIsNone(report.primary_file({"rust": {f"{i}.rs": [] for i in range(4)}}, set(), "n/Other"))
|
||||
|
||||
|
||||
class VersionPortReportTests(unittest.TestCase):
|
||||
def test_harness_results_merge_shards(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
state = Path(directory) / "state.json"
|
||||
state.write_text(json.dumps({"harnesses": {"food": {"status": "pass"}, "datafix#1": {"status": "pass"},
|
||||
"datafix#2": {"status": "fail"}}}))
|
||||
self.assertEqual(port.harness_results(state), {"food": "pass", "datafix": "fail"})
|
||||
|
||||
def test_class_names_drop_moved_old_names(self):
|
||||
self.assertEqual(port.class_names({"a/X": 0, "a/Y": 0}, {"b/X": 0, "a/Y": 0}, {"a/X": "b/X"}),
|
||||
["a/Y", "b/X"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The port report: a version diff (tools/version_port.py diff) turned into ranked, area-sized work items
|
||||
(docs/version-ports.md, "The port report").
|
||||
|
||||
Ranks:
|
||||
1 blocks the switch: version facts, packets, the save format (DataFixers), data component types, report
|
||||
formats, generated tables to regenerate
|
||||
2 a harness failed against the new version (with --results), else changed vanilla code a harness proves:
|
||||
rerun it
|
||||
3 changed vanilla code Horizon ports and no harness proves
|
||||
4 new content: registry entries, and new classes in the vanilla packages Horizon ports
|
||||
5 changed and new vanilla code Horizon doesn't port (two summary items)
|
||||
|
||||
Used through `version_port.py report OLD NEW`; the functions here take plain data so tests can build it.
|
||||
"""
|
||||
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
from . import class_index, vanilla_map
|
||||
except ImportError:
|
||||
import class_index
|
||||
import vanilla_map
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROTOCOL = "net/minecraft/network/protocol/"
|
||||
DATAFIX = "net/minecraft/util/datafix/"
|
||||
# Generated tables, what makes them stale, and how to regenerate them (docs/version-ports.md, "Regenerate").
|
||||
GENERATORS = [
|
||||
("crates/horizon-protocol/src/ids.rs", "python3 tools/generate_packet_ids.py",
|
||||
lambda d: bool(d["packets"])),
|
||||
("crates/horizon-client/src/lan/tracking.rs", "python3 tools/generate_entity_tracking.py",
|
||||
lambda d: "minecraft:entity_type" in d["registries"]["changed"]
|
||||
or "net/minecraft/world/entity/EntityType" in d["classes"]["changed"]),
|
||||
("crates/horizon-client/src/lan/command_suggestions.rs", "python3 tools/generate_command_suggestions.py",
|
||||
lambda d: bool(d["commands"]["added"] or d["commands"]["removed"] or d["commands"]["changed"])),
|
||||
("crates/horizon-client/src/lan/command_order.rs", "python3 tools/generate_lan_command_order.py",
|
||||
lambda d: bool(d["commands"]["added"] or d["commands"]["removed"])),
|
||||
("crates/horizon-client/src/command/vanilla_order.rs", "python3 tools/generate_command_order.py",
|
||||
lambda d: bool(d["commands"]["added"] or d["commands"]["removed"])),
|
||||
("crates/horizon-world/src/datafix/block_state_data.txt", "python3 tools/gen_block_state_data.py",
|
||||
lambda d: "net/minecraft/util/datafix/fixes/BlockStateData" in d["classes"]["changed"]),
|
||||
]
|
||||
# Registries whose new entries are new things to port.
|
||||
CONTENT = ["minecraft:block", "minecraft:item", "minecraft:entity_type", "minecraft:block_entity_type",
|
||||
"minecraft:particle_type", "minecraft:menu", "minecraft:worldgen/feature", "minecraft:sound_event"]
|
||||
|
||||
|
||||
def version_sites(facts, root=ROOT):
|
||||
"""file:line of every place the code states these version facts (not tests)."""
|
||||
pattern = ["-e", str(facts["protocol_version"]), "-e", str(facts["world_version"])]
|
||||
found = []
|
||||
for command in (["git", "grep", "-n", "-w"] + pattern, ["git", "grep", "-n", "-F", f'"{facts["id"]}"']):
|
||||
result = subprocess.run(command + ["--", "crates/*.rs", "launcher/*.rs", "tools/*.py"],
|
||||
cwd=root, capture_output=True, text=True)
|
||||
found += [line.split(":", 2)[0] + ":" + line.split(":", 2)[1] for line in result.stdout.splitlines()]
|
||||
return sorted({site for site in found if not re.search(r"(^|/)tests?(/|\.rs)|/test_|_tests?\.rs", site)})
|
||||
|
||||
|
||||
def files_containing(text, root=ROOT):
|
||||
"""Rust and Python files that contain this text (a report's file name)."""
|
||||
command = ["git", "grep", "-l", "-F", text, "--", "crates/*.rs", "launcher/*.rs", "tools/*.py"]
|
||||
return subprocess.run(command, cwd=root, capture_output=True, text=True).stdout.split()
|
||||
|
||||
|
||||
def files_naming(words, root=ROOT):
|
||||
"""Rust files that contain any of these words (component or packet names)."""
|
||||
if not words:
|
||||
return []
|
||||
command = ["git", "grep", "-l", "-w"] + [arg for word in words for arg in ("-e", word)] + ["--", "crates/*.rs"]
|
||||
return subprocess.run(command, cwd=root, capture_output=True, text=True).stdout.split()
|
||||
|
||||
|
||||
def snake(name):
|
||||
"""LivingEntity -> living_entity."""
|
||||
return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
|
||||
|
||||
|
||||
def primary_file(entry, members, cls):
|
||||
"""The Horizon file that ports a changed class: the one whose comments name the most of its changed
|
||||
methods, else the one named after it, else one of the few files that name it. None for a class many
|
||||
files name in passing (Entity, BlockPos) when none names a changed method."""
|
||||
files = entry["rust"]
|
||||
scored = sorted((-len(set(named) & members), path) for path, named in files.items() if set(named) & members)
|
||||
if scored:
|
||||
return scored[0][1]
|
||||
stem = snake(cls.rsplit("/", 1)[-1].split("$")[0])
|
||||
named_after = sorted(path for path in files if Path(path).stem == stem)
|
||||
if named_after:
|
||||
return named_after[0]
|
||||
return sorted(files)[0] if 0 < len(files) <= 3 else None
|
||||
|
||||
|
||||
def method_names(change):
|
||||
methods = change.get("methods", {})
|
||||
return {m.split("(")[0] for kind in ("added", "removed", "changed") for m in methods.get(kind, [])}
|
||||
|
||||
|
||||
def item(rank, kind, title, **details):
|
||||
return {"rank": rank, "kind": kind, "title": title, **details}
|
||||
|
||||
|
||||
def blocking_items(diff, mapping, pinned_facts, root):
|
||||
items = []
|
||||
if diff["facts"]:
|
||||
changes = ", ".join(f"{k} {a} -> {b}" for k, (a, b) in diff["facts"].items())
|
||||
items.append(item(1, "facts", f"Version facts: {changes}", files=version_sites(pinned_facts, root),
|
||||
details=diff["facts"]))
|
||||
if diff["packets"]:
|
||||
classes = diff["classes"]
|
||||
packets = {"added": [c for c in classes["added"] if c.startswith(PROTOCOL)],
|
||||
"removed": [c for c in classes["removed"] if c.startswith(PROTOCOL)],
|
||||
"changed": [c for c in classes["changed"] if c.startswith(PROTOCOL)]}
|
||||
names = sorted({n.split(":")[1] for flow in diff["packets"].values()
|
||||
for n in list(flow["added"]) + list(flow["removed"])})
|
||||
constants = [n.upper().replace("/", "_") for n in names]
|
||||
counts = ", ".join(f"{flow} +{len(c['added'])} -{len(c['removed'])} ~{len(c['renumbered'])}"
|
||||
for flow, c in diff["packets"].items())
|
||||
items.append(item(1, "protocol", f"Protocol: packets {counts}",
|
||||
files=sorted(set(files_naming(constants, root)) | {"crates/horizon-protocol/src/ids.rs"}),
|
||||
details={"packets": diff["packets"], "classes": packets}))
|
||||
if "world_version" in diff["facts"]:
|
||||
classes = diff["classes"]
|
||||
added = [c for c in classes["added"] if c.startswith(DATAFIX)]
|
||||
changed = [c for c in classes["changed"] if c.startswith(DATAFIX)]
|
||||
schemas = sorted(c for c in added if "/schemas/V" in c and "$" not in c)
|
||||
fixes = sorted(c for c in added if "/fixes/" in c and "$" not in c)
|
||||
horizon = vanilla_map.lookup(mapping, "net/minecraft/util/datafix/DataFixers")
|
||||
named = {f for c in schemas + fixes for f in mapping["classes"].get(c, {}).get("rust", {})}
|
||||
a, b = diff["facts"]["world_version"]
|
||||
items.append(item(1, "save-format", f"Save format: DataFixers {a} -> {b}: {len(schemas)} schemas, "
|
||||
f"{len(fixes)} fixes added, {len(changed)} datafix classes changed",
|
||||
files=sorted(set(horizon["rust"]) | named), harnesses=horizon["harnesses"],
|
||||
details={"schemas": schemas, "fixes": fixes, "changed": changed}))
|
||||
registry = diff["registries"]["changed"].get("minecraft:data_component_type", {})
|
||||
if registry.get("added") or registry.get("removed"):
|
||||
names = [n.split(":")[1] for n in registry.get("added", []) + registry.get("removed", [])]
|
||||
items.append(item(1, "components", f"Data component types: +{len(registry.get('added', []))} "
|
||||
f"-{len(registry.get('removed', []))} (sent on the wire and saved)",
|
||||
files=files_naming(names, root),
|
||||
details={"types": registry, "items": diff["items"]["components"]}))
|
||||
if diff["blocks"].get("format"):
|
||||
items.append(item(1, "report-format", f"blocks.json format: {', '.join(diff['blocks']['format'])}",
|
||||
files=files_containing("blocks.json", root),
|
||||
details={"fields": diff["blocks"]["format"]}))
|
||||
for output, command, stale in GENERATORS:
|
||||
if stale(diff):
|
||||
items.append(item(1, "regenerate", f"Regenerate {output}", files=[output], command=command))
|
||||
return items
|
||||
|
||||
|
||||
def item_harnesses(path, classes, mapping, limit=8):
|
||||
"""The harnesses for an item: those whose probes reach its file, else those covering the most of its
|
||||
classes, the most specific first."""
|
||||
harnesses = mapping["harnesses"]
|
||||
reaching = sorted(h for h, data in harnesses.items() if path in data["rust"])
|
||||
if reaching:
|
||||
return reaching
|
||||
covering = Counter(h for c in classes for h in vanilla_map.lookup(mapping, c)["harnesses"])
|
||||
return sorted(sorted(covering, key=lambda h: (-covering[h], len(harnesses[h]["classes"]), h))[:limit])
|
||||
|
||||
|
||||
def code_items(diff, mapping, results=None):
|
||||
"""Ranks 2, 3 and 5 from the changed and removed vanilla classes, one item per Horizon file that ports
|
||||
them (classes many files name in passing go by vanilla package)."""
|
||||
classes = diff["classes"]
|
||||
groups = defaultdict(lambda: {"classes": {}, "files": set(), "precise": 0, "methods": 0})
|
||||
unported = Counter()
|
||||
for name in list(classes["changed"]) + classes["removed"]:
|
||||
if name.startswith((PROTOCOL, DATAFIX)):
|
||||
continue # The protocol and save-format items have them.
|
||||
entry = classes["horizon"].get(name)
|
||||
if not entry:
|
||||
unported[name.rsplit("/", 1)[0]] += 1
|
||||
continue
|
||||
removed = name not in classes["changed"]
|
||||
change = classes["changed"].get(name, {})
|
||||
members = method_names(change)
|
||||
path = primary_file(entry, members, name) if entry["rust"] else None
|
||||
key = path or "widely named: " + name.rsplit("/", 1)[0].removeprefix("net/minecraft/")
|
||||
group = groups[key]
|
||||
precise = sorted({m for named in entry["rust"].values() for m in named} & members)
|
||||
methods = 1 if removed else class_index.changed_methods(change)
|
||||
group["classes"][name] = {"methods": methods, "named": precise, "removed": removed}
|
||||
group["files"].update([path] if path else entry["rust"])
|
||||
group["precise"] += len(precise)
|
||||
group["methods"] += methods
|
||||
failing = {h for h, status in (results or {}).items() if status != "pass"}
|
||||
items = []
|
||||
for key, group in groups.items():
|
||||
harnesses = item_harnesses(key, group["classes"], mapping)
|
||||
failed = sorted(set(harnesses) & failing)
|
||||
rank, kind = (2, "harness-failed") if failed else (2, "rerun") if harnesses else (3, "unproved")
|
||||
top = sorted(group["classes"], key=lambda c: (-group["classes"][c]["methods"], c))
|
||||
title = (f"{key}: {len(group['classes'])} vanilla classes changed "
|
||||
f"({group['methods']} methods, {group['precise']} named in Horizon)")
|
||||
items.append(item(rank, kind, title, module=key, files=sorted(group["files"]), harnesses=harnesses,
|
||||
failed=failed, classes={c: group["classes"][c] for c in top},
|
||||
score=group["methods"] + 10 * group["precise"] + (1000 if failed else 0)))
|
||||
if unported:
|
||||
items.append(item(5, "unported", f"{sum(unported.values())} changed or removed vanilla classes Horizon "
|
||||
f"doesn't port, in {len(unported)} packages", packages=dict(unported.most_common())))
|
||||
return items
|
||||
|
||||
|
||||
def content_items(diff, mapping):
|
||||
"""Rank 4: new registry entries, and new classes by vanilla package where Horizon ports other classes of
|
||||
that package (with those Horizon already names, after a port); new classes elsewhere go in one summary."""
|
||||
items = []
|
||||
for registry in CONTENT:
|
||||
entries = diff["registries"]["changed"].get(registry, {}).get("added", [])
|
||||
if entries:
|
||||
items.append(item(4, "content", f"New {registry.split(':')[1]}: {len(entries)} entries", entries=entries))
|
||||
ported_packages = Counter(c.rsplit("/", 1)[0] for c, e in mapping["classes"].items() if e["rust"])
|
||||
by_package = defaultdict(list)
|
||||
for name in diff["classes"]["added"]:
|
||||
if not re.search(r"\$\d", name) and not name.startswith((PROTOCOL, DATAFIX)):
|
||||
by_package[name.rsplit("/", 1)[0]].append(name)
|
||||
elsewhere = Counter()
|
||||
for package, names in sorted(by_package.items()):
|
||||
if package not in ported_packages:
|
||||
elsewhere[package] += len(names)
|
||||
continue
|
||||
ported = sorted(c for c in names if mapping["classes"].get(c, {}).get("rust"))
|
||||
nearby = Counter(f for c, e in mapping["classes"].items() if c.rsplit("/", 1)[0] == package for f in e["rust"])
|
||||
files = sorted({f for c in ported for f in mapping["classes"][c]["rust"]}) or [f for f, _ in nearby.most_common(5)]
|
||||
items.append(item(4, "new-classes", f"New in {package.removeprefix('net/minecraft/')}: {len(names)} classes"
|
||||
f" ({len(ported)} already named in Horizon)", classes=sorted(names), ported=ported,
|
||||
files=files, score=len(names)))
|
||||
if elsewhere:
|
||||
items.append(item(5, "new-unported", f"{sum(elsewhere.values())} new vanilla classes in {len(elsewhere)} "
|
||||
f"packages Horizon doesn't port", packages=dict(elsewhere.most_common())))
|
||||
return items
|
||||
|
||||
|
||||
def build(diff, mapping, old, new, pinned_facts, results=None, root=ROOT):
|
||||
"""The report: summary, ranked items and what came out clean. pinned_facts: the facts of the version the
|
||||
code states today (to find where it states them); results: {harness: "pass" | "fail"} against NEW."""
|
||||
items = blocking_items(diff, mapping, pinned_facts, root)
|
||||
items += code_items(diff, mapping, results) + content_items(diff, mapping)
|
||||
items.sort(key=lambda i: (i["rank"], -i.get("score", 0), i["title"]))
|
||||
for number, entry in enumerate(items, 1):
|
||||
entry["id"] = number
|
||||
clean = [kind for kind in ("packets", "commands") if not diff[kind]]
|
||||
clean += [kind for kind in ("blocks", "items") if not (diff[kind]["added"] or diff[kind]["removed"]
|
||||
or diff[kind]["changed"])]
|
||||
classes = diff["classes"]
|
||||
return {
|
||||
"old": old, "new": new,
|
||||
"summary": {
|
||||
"facts": diff["facts"], "ranks": dict(sorted(Counter(i["rank"] for i in items).items())),
|
||||
"classes": {"added": len(classes["added"]), "removed": len(classes["removed"]),
|
||||
"moved": len(classes["moved"]), "changed": len(classes["changed"]),
|
||||
"horizon": len(classes["horizon"])},
|
||||
"generators": [i["files"][0] for i in items if i["kind"] == "regenerate"],
|
||||
},
|
||||
"items": items, "clean": clean,
|
||||
}
|
||||
|
||||
|
||||
def short(name):
|
||||
return name.rsplit("/", 1)[-1].replace("$", ".")
|
||||
|
||||
|
||||
def markdown(report, limit=12):
|
||||
"""report.md: what a person or an agent reads first."""
|
||||
lines = [f"# Port report: Minecraft {report['old']} -> {report['new']}", ""]
|
||||
summary = report["summary"]
|
||||
for key, (a, b) in summary["facts"].items():
|
||||
lines.append(f"- {key}: {a} -> {b}")
|
||||
c = summary["classes"]
|
||||
lines.append(f"- classes: +{c['added']} -{c['removed']}, {c['moved']} moved, {c['changed']} changed; "
|
||||
f"{c['horizon']} changed or removed classes concern Horizon code or harnesses")
|
||||
lines.append(f"- work items by rank: " + ", ".join(f"{r}: {n}" for r, n in summary["ranks"].items()))
|
||||
if report["clean"]:
|
||||
lines.append(f"- unchanged: {', '.join(report['clean'])}")
|
||||
names = {1: "Blocks the switch", 2: "Proved by a harness: rerun it", 3: "Ported, no harness",
|
||||
4: "New content", 5: "Not ported"}
|
||||
for rank in sorted({i["rank"] for i in report["items"]}):
|
||||
lines += ["", f"## {rank}. {names[rank]}", ""]
|
||||
for entry in [i for i in report["items"] if i["rank"] == rank]:
|
||||
lines.append(f"### #{entry['id']} {entry['title']}")
|
||||
if entry.get("command"):
|
||||
lines.append(f"- run: `{entry['command']}`")
|
||||
if entry.get("failed"):
|
||||
lines.append(f"- failing harnesses: {', '.join(entry['failed'])}")
|
||||
if entry.get("harnesses"):
|
||||
lines.append(f"- harnesses: {', '.join(entry['harnesses'][:limit])}"
|
||||
+ (f" (+{len(entry['harnesses']) - limit})" if len(entry["harnesses"]) > limit else ""))
|
||||
if entry.get("files"):
|
||||
lines.append(f"- Horizon: {', '.join(f'`{f}`' for f in entry['files'][:limit])}"
|
||||
+ (f" (+{len(entry['files']) - limit})" if len(entry["files"]) > limit else ""))
|
||||
if entry.get("classes") and isinstance(entry["classes"], dict):
|
||||
shown = list(entry["classes"].items())[:limit]
|
||||
lines.append("- vanilla: " + ", ".join(
|
||||
f"{short(n)} ({'removed' if v['removed'] else str(v['methods']) + ' methods'}"
|
||||
+ (f"; {', '.join(v['named'])}" if v["named"] else "") + ")" for n, v in shown)
|
||||
+ (f" (+{len(entry['classes']) - limit})" if len(entry["classes"]) > limit else ""))
|
||||
elif entry.get("classes"):
|
||||
lines.append(f"- new classes: {', '.join(short(n) for n in entry['classes'][:limit])}"
|
||||
+ (f" (+{len(entry['classes']) - limit})" if len(entry["classes"]) > limit else ""))
|
||||
if entry.get("entries"):
|
||||
lines.append(f"- entries: {', '.join(n.split(':')[-1] for n in entry['entries'][:limit])}"
|
||||
+ (f" (+{len(entry['entries']) - limit})" if len(entry["entries"]) > limit else ""))
|
||||
if entry.get("ported"):
|
||||
lines.append(f"- already in Horizon: {', '.join(short(n) for n in entry['ported'][:limit])}")
|
||||
details = entry.get("details", {})
|
||||
for key in ("schemas", "fixes"):
|
||||
if details.get(key):
|
||||
lines.append(f"- {key}: {', '.join(short(n) for n in details[key])}")
|
||||
if entry["kind"] == "protocol":
|
||||
for flow, change in details["packets"].items():
|
||||
lines.append(f"- {flow}: added {', '.join(change['added']) or 'none'}; "
|
||||
f"removed {', '.join(change['removed']) or 'none'}")
|
||||
if entry["kind"] in ("unported", "new-unported"):
|
||||
lines.append("- " + ", ".join(f"{p.removeprefix('net/minecraft/')} {n}"
|
||||
for p, n in list(entry["packages"].items())[:limit]))
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
+68
-12
@@ -8,6 +8,8 @@
|
||||
version_port.py diff OLD NEW facts, packets, registries, blocks, items, commands,
|
||||
data packs, assets, lang and classes, with the Horizon
|
||||
code each changed class concerns: diff-OLD-NEW/diff.json
|
||||
version_port.py report OLD NEW [--results S] the ranked port report: diff-OLD-NEW/report.md and .json
|
||||
version_port.py port NEW [--old OLD] prepare both, diff, report
|
||||
version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep
|
||||
|
||||
The downloads are for development only: they go to the version store (~/code/horizon-parity/versions,
|
||||
@@ -31,10 +33,11 @@ import urllib.request
|
||||
import zipfile
|
||||
|
||||
try:
|
||||
from . import class_index, lab, vanilla_map
|
||||
from . import class_index, lab, port_report, vanilla_map
|
||||
except ImportError:
|
||||
import class_index
|
||||
import lab
|
||||
import port_report
|
||||
import vanilla_map
|
||||
|
||||
MANIFEST_URL = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"
|
||||
@@ -438,13 +441,17 @@ def read_reports(reports):
|
||||
}
|
||||
|
||||
|
||||
def diff_classes(old, new, mapping):
|
||||
"""The class diff, each changed or removed class with the Horizon files and harnesses that concern it
|
||||
(looked up under its old name: Horizon's code names the pinned version's classes)."""
|
||||
result = class_index.diff(old, new)
|
||||
def class_names(old, new, moved):
|
||||
"""Every class name of either version, but a moved class only under its new name, so that its simple
|
||||
name stays unambiguous."""
|
||||
return sorted(set(new) | (set(old) - set(moved)))
|
||||
|
||||
|
||||
def diff_classes(result, mapping):
|
||||
"""The class diff, each changed or removed class with the Horizon files and harnesses that concern it."""
|
||||
horizon = {}
|
||||
for name, change in result["changed"].items():
|
||||
entry = vanilla_map.lookup(mapping, change.get("moved_from", name))
|
||||
for name in result["changed"]:
|
||||
entry = vanilla_map.lookup(mapping, name)
|
||||
if entry["rust"] or entry["harnesses"]:
|
||||
horizon[name] = entry
|
||||
for name in result["removed"]:
|
||||
@@ -455,9 +462,9 @@ def diff_classes(old, new, mapping):
|
||||
return result
|
||||
|
||||
|
||||
def diff_versions(old_paths, new_paths, old_facts, new_facts, old_classes=None, new_classes=None, mapping=None):
|
||||
def diff_versions(old_paths, new_paths, old_facts, new_facts, classes=None, mapping=None):
|
||||
a, b = read_reports(old_paths["reports"]), read_reports(new_paths["reports"])
|
||||
classes = diff_classes(old_classes, new_classes, mapping) if old_classes is not None else None
|
||||
classes = diff_classes(classes, mapping) if classes is not None else None
|
||||
return {
|
||||
"facts": diff_facts(old_facts, new_facts),
|
||||
"packets": diff_packets(a["packets"], b["packets"]),
|
||||
@@ -526,16 +533,53 @@ def load(version, name):
|
||||
def diff(args):
|
||||
directory = diff_dir(args.old, args.new)
|
||||
old_classes, new_classes = load(args.old, "classes.json"), load(args.new, "classes.json")
|
||||
mapping = vanilla_map.build(old_classes, dependencies=vanilla_map.recorded_dependencies(vanilla_map.default_shared()))
|
||||
classes = class_index.diff(old_classes, new_classes)
|
||||
mapping = vanilla_map.build(class_names(old_classes, new_classes, classes["moved"]),
|
||||
dependencies=vanilla_map.recorded_dependencies(vanilla_map.default_shared()))
|
||||
lab.write_json(directory / "vanilla_map.json", mapping)
|
||||
result = diff_versions(paths(args.old), paths(args.new), load(args.old, "facts.json"), load(args.new, "facts.json"),
|
||||
old_classes, new_classes, mapping)
|
||||
classes, mapping)
|
||||
lab.write_json(directory / "diff.json", result)
|
||||
print(summary(args.old, args.new, result))
|
||||
print(f"\n{directory / 'diff.json'}")
|
||||
return 0
|
||||
|
||||
|
||||
# --- report ------------------------------------------------------------------------------------------------
|
||||
|
||||
def harness_results(state_path):
|
||||
"""{harness: "pass" | "fail"} from tools/parity.py's state; a harness fails if one of its shards does."""
|
||||
found = {}
|
||||
for name, record in json.loads(Path(state_path).read_text()).get("harnesses", {}).items():
|
||||
harness = name.split("#")[0]
|
||||
found[harness] = "fail" if record.get("status") != "pass" or found.get(harness) == "fail" else "pass"
|
||||
return found
|
||||
|
||||
|
||||
def report(args):
|
||||
directory = diff_dir(args.old, args.new)
|
||||
if not (directory / "diff.json").exists():
|
||||
raise SystemExit(f"No diff yet; run version_port.py diff {args.old} {args.new}")
|
||||
result = json.loads((directory / "diff.json").read_text())
|
||||
mapping = json.loads((directory / "vanilla_map.json").read_text())
|
||||
results = harness_results(args.results) if args.results else None
|
||||
built = port_report.build(result, mapping, args.old, args.new, load(pinned(), "facts.json"), results)
|
||||
lab.write_json(directory / "report.json", built)
|
||||
(directory / "report.md").write_text(port_report.markdown(built))
|
||||
print("\n".join(port_report.markdown(built).splitlines()[:args.lines]))
|
||||
print(f"\n{directory / 'report.md'} ({len(built['items'])} items)")
|
||||
return 0
|
||||
|
||||
|
||||
def port(args):
|
||||
"""prepare both, diff, report."""
|
||||
old = args.old or pinned()
|
||||
for version in (old, args.new):
|
||||
prepare(argparse.Namespace(version=version, download=False, jobs=args.jobs))
|
||||
diff(argparse.Namespace(old=old, new=args.new))
|
||||
return report(argparse.Namespace(old=old, new=args.new, results=args.results, lines=args.lines))
|
||||
|
||||
|
||||
# --- gc ----------------------------------------------------------------------------------------------------
|
||||
|
||||
def to_remove(entries, pin, keep):
|
||||
@@ -579,12 +623,24 @@ def main():
|
||||
command = commands.add_parser("diff", help="diff two prepared versions")
|
||||
command.add_argument("old")
|
||||
command.add_argument("new")
|
||||
command = commands.add_parser("report", help="the ranked port report of a diff")
|
||||
command.add_argument("old")
|
||||
command.add_argument("new")
|
||||
command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW")
|
||||
command.add_argument("--lines", type=int, default=60, help="how much of report.md to print")
|
||||
command = commands.add_parser("port", help="prepare, diff and report in one go")
|
||||
command.add_argument("new")
|
||||
command.add_argument("--old", help="default: the pinned version")
|
||||
command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW")
|
||||
command.add_argument("--lines", type=int, default=60)
|
||||
command.add_argument("-j", "--jobs", type=int, default=int(os.environ.get("FLEET_LOAD_JOBS", 0)) or None)
|
||||
command = commands.add_parser("gc", help="delete versions and scratch the store doesn't need")
|
||||
command.add_argument("--keep", nargs="*", default=[])
|
||||
command.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
STORE.mkdir(parents=True, exist_ok=True)
|
||||
return {"detect": detect, "prepare": prepare, "diff": diff, "gc": gc}[args.command](args)
|
||||
return {"detect": detect, "prepare": prepare, "diff": diff, "report": report, "port": port,
|
||||
"gc": gc}[args.command](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user