Diff vanilla classes method by method and map them to Horizon

tools/class_index.py hashes every method's bytecode with constant-pool references
resolved and branch targets as instruction indices, lambdas folded into their
declaring method; class diffs list moved classes. tools/vanilla_map.py maps vanilla
classes to the Rust files whose comments name them and the harnesses whose oracles
or probes reach them. version_port.py prepare indexes, diff reports both.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-15 13:44:26 +02:00
parent 58ad81674b
commit 93bb113e0d
5 changed files with 773 additions and 7 deletions
+2 -1
View File
@@ -266,12 +266,13 @@ python3 tools/version_port.py prepare 26.3 # the pinned version: index
python3 tools/version_port.py prepare 26.4 # downloads, verifies, runs the data generator (about 10 s)
python3 tools/version_port.py diff 26.3 26.4 # summary; the full diff in versions/diff-26.3-26.4/diff.json
python3 tools/version_port.py gc # keep the pinned version and the newest two others
python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food.FoodData # its Rust files and harnesses
```
| Slice | Status |
| --- | --- |
| This design | this page |
| `detect`, `prepare`, `diff` (facts, packets, registries, blocks, items, commands, data, assets), `gc` | done |
| Class index, class diff, `vanilla_map.json` | next PR |
| Class index (`tools/class_index.py`), class diff with moves, `vanilla_map.json` (`tools/vanilla_map.py`) | done |
| Port report (`report`), `port` | next PR |
| Changed-class decompilation, `regen`, `version_consts` refactor, candidate harness run, `publish`, launchd agent | later |
+170
View File
@@ -0,0 +1,170 @@
import struct
import unittest
from tools import class_index, vanilla_map
class Pool:
"""A constant pool built in the order entries are asked for; `pad` shifts every index."""
def __init__(self, pad=0):
self.entries, self.index = [], {}
for i in range(pad):
self.utf8(f"padding{i}")
def add(self, key, data):
if key not in self.index:
self.entries.append(data)
self.index[key] = len(self.entries)
return self.index[key]
def utf8(self, text):
raw = text.encode()
return self.add(("u", text), b"\x01" + struct.pack(">H", len(raw)) + raw)
def cls(self, name):
return self.add(("c", name), b"\x07" + struct.pack(">H", self.utf8(name)))
def string(self, text):
return self.add(("s", text), b"\x08" + struct.pack(">H", self.utf8(text)))
def method(self, owner, name, descriptor):
nat = self.add(("nt", name, descriptor),
b"\x0c" + struct.pack(">HH", self.utf8(name), self.utf8(descriptor)))
return self.add(("m", owner, name, descriptor), b"\x0a" + struct.pack(">HH", self.cls(owner), nat))
def bytes(self):
return struct.pack(">H", len(self.entries) + 1) + b"".join(self.entries)
def class_file(methods, name="a/B", pad=0, wide=False):
"""A class with the given methods: {name: (descriptor, [instructions])}, each instruction
("ldc", text) | ("call", owner, name, descriptor) | ("ifnull", +instructions) | (opcode,)."""
pool = Pool(pad)
this, parent = pool.cls(name), pool.cls("java/lang/Object")
code_name = pool.utf8("Code")
encoded = []
for method, (descriptor, instructions) in methods.items():
sizes = [3 if i[0] in ("call", "ifnull") or (i[0] == "ldc" and wide) else 2 if i[0] == "ldc" else 1
for i in instructions]
body = b""
for at, instruction in enumerate(instructions):
if instruction[0] == "ldc":
index = pool.string(instruction[1])
body += bytes([19]) + struct.pack(">H", index) if wide else bytes([18, index])
elif instruction[0] == "call":
body += bytes([184]) + struct.pack(">H", pool.method(*instruction[1:]))
elif instruction[0] == "ifnull":
offset = sum(sizes[at:at + instruction[1]])
body += bytes([198]) + struct.pack(">h", offset)
else:
body += bytes([instruction[0]])
code = struct.pack(">HHI", 2, 1, len(body)) + body + struct.pack(">HH", 0, 0)
encoded.append(struct.pack(">HHHH", 0x0009, pool.utf8(method), pool.utf8(descriptor), 1)
+ struct.pack(">HI", code_name, len(code)) + code)
return (b"\xca\xfe\xba\xbe" + struct.pack(">HH", 0, 69) + pool.bytes()
+ struct.pack(">HHHH", 0x0021, this, parent, 0) + struct.pack(">H", 0)
+ struct.pack(">H", len(encoded)) + b"".join(encoded) + struct.pack(">H", 0))
def index(methods, **options):
return class_index.ClassFile(class_file(methods, **options)).index()
GREET = {"greet": ("()Ljava/lang/String;", [("ldc", "hi"), ("ifnull", 2), (1,), (176,)])}
class ClassIndexTests(unittest.TestCase):
def test_constant_pool_layout_and_ldc_width_do_not_change_hashes(self):
plain = index(GREET)
self.assertEqual(index(GREET, pad=3), plain)
self.assertEqual(index(GREET, pad=300, wide=True), plain)
def test_a_changed_literal_or_call_changes_the_hash(self):
plain = index(GREET)["methods"]
other = index({"greet": ("()Ljava/lang/String;", [("ldc", "hello"), ("ifnull", 2), (1,), (176,)])})
self.assertNotEqual(other["methods"], plain)
call = {"run": ("()V", [("call", "a/C", "go", "()V"), (177,)])}
moved = {"run": ("()V", [("call", "b/C", "go", "()V"), (177,)])}
self.assertNotEqual(index(call)["methods"], index(moved)["methods"])
def test_lambdas_fold_into_their_declaring_method(self):
body = ("()V", [(177,)])
first = index({"run": body, "lambda$run$0": body, "lambda$run$1": ("()V", [(1,), (87,), (177,)])})
renumbered = index({"run": body, "lambda$run$4": ("()V", [(1,), (87,), (177,)]), "lambda$run$5": body})
self.assertEqual(set(first["methods"]), {"run()V", "lambda$run"})
self.assertEqual(first["methods"], renumbered["methods"])
def test_diff_with_moves(self):
def entry(**methods):
return {"access": 33, "super": "java/lang/Object", "interfaces": [], "fields": {}, "methods": methods}
old = {"a/Same": entry(f="1"), "a/Edit": entry(f="1", g="2"), "a/Gone": entry(),
"a/Moved": entry(f="1"), "a/x/Twin": entry(), "a/y/Twin": entry()}
new = {"a/Same": entry(f="1"), "a/Edit": entry(f="9", h="3"), "b/Moved": entry(f="2"), "a/New": entry(),
"b/Twin": entry()}
result = class_index.diff(old, new)
self.assertEqual(result["moved"], {"a/Moved": "b/Moved"})
self.assertEqual(result["added"], ["a/New", "b/Twin"])
self.assertEqual(result["removed"], ["a/Gone", "a/x/Twin", "a/y/Twin"])
self.assertEqual(result["changed"]["a/Edit"],
{"methods": {"added": ["h"], "removed": ["g"], "changed": ["f"]}})
self.assertEqual(result["changed"]["b/Moved"]["moved_from"], "a/Moved")
self.assertEqual(class_index.changed_methods(result["changed"]["a/Edit"]), 3)
CLASSES = ["net/minecraft/world/food/FoodData", "net/minecraft/world/level/Level",
"com/mojang/realmsclient/Level", "net/minecraft/world/entity/Entity",
"net/minecraft/world/entity/Entity$RemovalReason", "net/minecraft/a/Thing$1",
"net/minecraft/server/level/ServerLevel"]
class VanillaMapTests(unittest.TestCase):
def setUp(self):
self.names = vanilla_map.Names(CLASSES)
def test_resolve(self):
self.assertEqual(self.names.resolve("FoodData"), "net/minecraft/world/food/FoodData")
self.assertEqual(self.names.resolve("Level"), "net/minecraft/world/level/Level")
self.assertEqual(self.names.resolve("RemovalReason"), "net/minecraft/world/entity/Entity$RemovalReason")
self.assertEqual(self.names.resolve("Entity.RemovalReason"), "net/minecraft/world/entity/Entity$RemovalReason")
self.assertIsNone(self.names.resolve("Missing"))
self.assertEqual(self.names.java("net.minecraft.world.entity.Entity.RemovalReason"),
"net/minecraft/world/entity/Entity$RemovalReason")
def test_rust_comments(self):
files = {"crates/x/src/food.rs": "/// FoodData.tick, as vanilla\nfn tick() {} // ServerLevel too\n"
"let FoodData = 1;\n// a Level, a Level.tick() and Entity.RemovalReason\n"}
found = vanilla_map.rust_references(files, self.names, read=files.get)
self.assertEqual(found["net/minecraft/world/food/FoodData"], {"crates/x/src/food.rs": ["tick"]})
self.assertEqual(found["net/minecraft/server/level/ServerLevel"], {"crates/x/src/food.rs": []})
# A one-word name counts only with a member: "a Level" alone is English.
self.assertEqual(found["net/minecraft/world/level/Level"], {"crates/x/src/food.rs": ["tick"]})
self.assertIn("net/minecraft/world/entity/Entity$RemovalReason", found)
def test_java_references(self):
text = ("import net.minecraft.world.food.*;import net.minecraft.world.entity.Entity;\n"
"import static net.minecraft.world.level.Level.OVERWORLD;\n"
"class X { FoodData f; net.minecraft.server.level.ServerLevel l; }")
self.assertEqual(vanilla_map.java_references(text, self.names), {
"net/minecraft/world/food/FoodData", "net/minecraft/world/entity/Entity",
"net/minecraft/world/level/Level", "net/minecraft/server/level/ServerLevel"})
def test_probe_modules(self):
sources = {"crates/horizon-world/src/food.rs", "crates/horizon-world/src/level/mod.rs",
"crates/horizon-world/src/level/tick.rs"}
text = "use horizon_world::food::FoodData;\nuse horizon_world::level::tick::{a, b};\nuse crate::level;\n"
self.assertEqual(vanilla_map.module_files("crates/horizon-world/examples/p.rs", text, sources),
{"crates/horizon-world/src/food.rs", "crates/horizon-world/src/level/tick.rs",
"crates/horizon-world/src/level/mod.rs"})
self.assertEqual(vanilla_map.probe_files("run(['food_fixture'])", {"food_fixture": "e.rs", "food": "f.rs"}),
{"food_fixture": "e.rs"})
self.assertEqual(vanilla_map.harness_name("tools/test_food_parity.py"), "food")
def test_lookup_falls_back_to_the_outer_class(self):
mapping = {"classes": {"a/B": {"rust": {"x.rs": []}, "harnesses": ["h"]}}}
self.assertEqual(vanilla_map.lookup(mapping, "a/B$C")["harnesses"], ["h"])
self.assertEqual(vanilla_map.lookup(mapping, "a/D"), {"rust": {}, "harnesses": []})
if __name__ == "__main__":
unittest.main()
+327
View File
@@ -0,0 +1,327 @@
#!/usr/bin/env python3
"""Index a game jar's classes for version diffs (docs/version-ports.md).
Each class gets its superclass, interfaces, fields and methods; each method a hash of its bytecode in which
every constant-pool reference is replaced by what it names and every branch target by an instruction index.
Recompiling unchanged code (a constant pool reordered, ldc widened to ldc_w) then hashes the same. Lambda
bodies (lambda$name$N) are folded into one entry per declaring method, `lambda$name`, so lambdas renumbered by
an unrelated change don't show as changes.
class_index.py JAR [JAR...] -o classes.json
"""
import argparse
from concurrent.futures import ProcessPoolExecutor
import hashlib
import json
import re
import struct
import sys
import zipfile
LAMBDA = re.compile(r"^lambda\$(.*)\$\d+$")
# Operand bytes per opcode, for the fixed-length instructions. None: tableswitch, lookupswitch, wide.
OPERANDS = [0] * 202
for _op, _size in [(16, 1), (17, 2), (18, 1), (19, 2), (20, 2), (169, 1), (188, 1), (185, 4), (186, 4), (197, 3)]:
OPERANDS[_op] = _size
for _op in list(range(21, 26)) + list(range(54, 59)):
OPERANDS[_op] = 1 # Local variable loads and stores.
OPERANDS[132] = 2 # iinc.
for _op in list(range(153, 169)) + [198, 199]:
OPERANDS[_op] = 2 # Branches.
OPERANDS[200] = OPERANDS[201] = 4 # goto_w, jsr_w.
for _op in [178, 179, 180, 181, 182, 183, 184, 187, 189, 192, 193]:
OPERANDS[_op] = 2 # Constant-pool references.
OPERANDS[170] = OPERANDS[171] = OPERANDS[196] = None
CP_OPERAND = {18, 19, 20, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 192, 193, 197}
BRANCH = set(range(153, 169)) | {198, 199, 200, 201}
# ldc_w loads what ldc loads; goto_w jumps where goto jumps.
CANONICAL = {19: 18, 200: 167, 201: 168}
def lambda_owner(name):
match = LAMBDA.match(name)
return f"lambda${match[1]}" if match else name
class ClassFile:
def __init__(self, data):
self.data = data
self.offset = 0
self.parse()
def u(self, size):
value = int.from_bytes(self.data[self.offset:self.offset + size], "big")
self.offset += size
return value
def raw(self, size):
value = self.data[self.offset:self.offset + size]
self.offset += size
return value
def parse(self):
if self.raw(4) != b"\xca\xfe\xba\xbe":
raise ValueError("Invalid class file magic")
self.raw(4)
count = self.u(2)
self.pool = [None] * count
index = 1
while index < count:
tag = self.u(1)
if tag == 1:
entry = (1, self.raw(self.u(2)).decode("utf-8", errors="replace"))
elif tag in (3, 4):
entry = (tag, self.raw(4))
elif tag in (5, 6):
entry = (tag, self.raw(8))
elif tag in (7, 8, 16, 19, 20):
entry = (tag, self.u(2))
elif tag in (9, 10, 11, 12, 17, 18):
entry = (tag, self.u(2), self.u(2))
elif tag == 15:
entry = (tag, self.u(1), self.u(2))
else:
raise ValueError(f"Unknown constant pool tag {tag}")
self.pool[index] = entry
index += 2 if tag in (5, 6) else 1
self.access = self.u(2)
self.name = self.class_name(self.u(2))
super_index = self.u(2)
self.super = self.class_name(super_index) if super_index else None
self.interfaces = [self.class_name(self.u(2)) for _ in range(self.u(2))]
self.fields = {}
for _ in range(self.u(2)):
access, name, descriptor = self.u(2), self.utf8(self.u(2)), self.utf8(self.u(2))
constant = None
for attribute, data in self.attributes():
if attribute == "ConstantValue":
constant = self.constant(struct.unpack(">H", data)[0])
self.fields[f"{name} {descriptor}"] = [access, constant]
self.methods = []
for _ in range(self.u(2)):
access, name, descriptor = self.u(2), self.utf8(self.u(2)), self.utf8(self.u(2))
code = next((data for attribute, data in self.attributes() if attribute == "Code"), None)
self.methods.append((access, name, descriptor, code))
self.bootstrap = []
for attribute, data in self.attributes():
if attribute == "BootstrapMethods":
reader = ClassFile.__new__(ClassFile)
reader.data, reader.offset = data, 0
for _ in range(reader.u(2)):
method = reader.u(2)
self.bootstrap.append((method, [reader.u(2) for _ in range(reader.u(2))]))
def attributes(self):
return [(self.utf8(self.u(2)), self.raw(self.u(4))) for _ in range(self.u(2))]
def utf8(self, index):
return self.pool[index][1]
def class_name(self, index):
return self.utf8(self.pool[index][1])
def member(self, index):
_, owner, name_and_type = self.pool[index]
_, name, descriptor = self.pool[name_and_type]
return f"{self.class_name(owner)}.{lambda_owner(self.utf8(name))}{self.utf8(descriptor)}"
def constant(self, index):
"""What a constant-pool entry names, independent of where it sits in the pool."""
entry = self.pool[index]
tag = entry[0]
if tag == 1:
return f"u:{entry[1]}"
if tag == 3:
return f"i:{struct.unpack('>i', entry[1])[0]}"
if tag == 4:
return f"f:{entry[1].hex()}"
if tag == 5:
return f"l:{struct.unpack('>q', entry[1])[0]}"
if tag == 6:
return f"d:{entry[1].hex()}"
if tag == 7:
return f"c:{self.utf8(entry[1])}"
if tag == 8:
return f"s:{self.utf8(entry[1])}"
if tag in (9, 10, 11):
return f"m:{self.member(index)}"
if tag == 12:
return f"nt:{lambda_owner(self.utf8(entry[1]))}{self.utf8(entry[2])}"
if tag == 15:
return f"h{entry[1]}:{self.member(entry[2])}"
if tag == 16:
return f"t:{self.utf8(entry[1])}"
if tag in (17, 18):
method, arguments = self.bootstrap[entry[1]]
return (f"y:{self.constant(entry[2])}@{self.constant(method)}"
f"({','.join(self.constant(a) for a in arguments)})")
return f"x{tag}:{entry[1]}"
def code_tokens(self, code):
"""The method's instructions, exception handlers included, as position-independent tokens."""
max_stack, max_locals, length = struct.unpack(">HHI", code[:8])
body = code[8:8 + length]
starts, instructions = {}, []
pc = 0
while pc < length:
starts[pc] = len(instructions)
op = body[pc]
if op == 170 or op == 171: # tableswitch, lookupswitch
at = (pc + 4) & ~3
default = struct.unpack(">i", body[at:at + 4])[0]
if op == 170:
low, high = struct.unpack(">ii", body[at + 4:at + 12])
targets = struct.unpack(f">{high - low + 1}i", body[at + 12:at + 12 + 4 * (high - low + 1)])
keys = list(range(low, high + 1))
end = at + 12 + 4 * len(targets)
else:
count = struct.unpack(">i", body[at + 4:at + 8])[0]
pairs = struct.unpack(f">{2 * count}i", body[at + 8:at + 8 + 8 * count])
keys, targets = pairs[0::2], pairs[1::2]
end = at + 8 + 8 * count
instructions.append(("switch", pc, default, list(zip(keys, targets))))
pc = end
continue
if op == 196: # wide
size = 5 if body[pc + 1] == 132 else 3
instructions.append(("op", pc, 196, body[pc + 1:pc + 1 + size]))
pc += 1 + size
continue
if op >= len(OPERANDS):
raise ValueError(f"Unknown opcode {op}")
size = OPERANDS[op]
instructions.append(("op", pc, op, body[pc + 1:pc + 1 + size]))
pc += 1 + size
tokens = [f"{max_stack},{max_locals}"]
for kind, pc, *rest in instructions:
if kind == "switch":
default, cases = rest
tokens.append(f"switch {starts.get(pc + default)} "
+ " ".join(f"{k}>{starts.get(pc + t)}" for k, t in cases))
continue
op, operand = rest
if op in CP_OPERAND:
index = operand[0] if op == 18 else int.from_bytes(operand[:2], "big")
tokens.append(f"{CANONICAL.get(op, op)} {self.constant(index)}")
elif op in BRANCH:
offset = int.from_bytes(operand, "big", signed=True)
tokens.append(f"{CANONICAL.get(op, op)} >{starts.get(pc + offset)}")
else:
tokens.append(f"{op} {operand.hex()}")
exceptions = 8 + length
for i in range(struct.unpack(">H", code[exceptions:exceptions + 2])[0]):
start, end, handler, kind = struct.unpack(">HHHH", code[exceptions + 2 + 8 * i:exceptions + 10 + 8 * i])
caught = self.class_name(kind) if kind else "*"
tokens.append(f"try {starts.get(start)} {starts.get(end, len(instructions))} {starts.get(handler)} {caught}")
return tokens
def index(self):
methods, lambdas = {}, {}
for access, name, descriptor, code in self.methods:
tokens = [str(access & 0x0FFF)] + (self.code_tokens(code) if code else ["abstract"])
digest = hashlib.sha1("\n".join(tokens).encode()).hexdigest()[:16]
owner = lambda_owner(name)
if owner != name:
lambdas.setdefault(owner, []).append(digest)
else:
methods[f"{name}{descriptor}"] = digest
for owner, digests in lambdas.items():
methods[owner] = hashlib.sha1(" ".join(sorted(digests)).encode()).hexdigest()[:16]
return {"access": self.access, "super": self.super, "interfaces": self.interfaces,
"fields": self.fields, "methods": methods}
def index_entries(jar, names):
with zipfile.ZipFile(jar) as archive:
return {name[:-6]: ClassFile(archive.read(name)).index() for name in names}
def index_jar(jar, jobs=None, prefixes=("net/minecraft/", "com/mojang/")):
"""{internal class name: index} for every game class in a jar."""
with zipfile.ZipFile(jar) as archive:
names = sorted(n for n in archive.namelist() if n.endswith(".class") and n.startswith(prefixes))
batches = [names[i::32] for i in range(32)]
result = {}
with ProcessPoolExecutor(max_workers=jobs) as pool:
for part in pool.map(index_entries, [jar] * len(batches), batches):
result.update(part)
return dict(sorted(result.items()))
def index_jars(jars, jobs=None):
"""Several jars into one index; a class in more than one keeps the first jar's (client before server)."""
classes = {}
for jar in jars:
for name, data in index_jar(jar, jobs).items():
classes.setdefault(name, data)
return classes
def local_name(name):
return name.rsplit("/", 1)[-1]
def moves(added, removed):
"""{old: new} for a removed class whose local name (Outer$Inner) one added class has, uniquely on both sides."""
def unique(names):
by_local = {}
for name in names:
by_local.setdefault(local_name(name), []).append(name)
return {local: found[0] for local, found in by_local.items() if len(found) == 1}
gone, new = unique(removed), unique(added)
return {gone[local]: new[local] for local in sorted(gone.keys() & new.keys())}
def diff(old, new):
"""Classes added, removed, moved (same local name, another package) and changed; per changed class its
methods and fields added, removed and changed. A moved class that changed too is listed under its new name
with moved_from."""
added = sorted(new.keys() - old.keys())
removed = sorted(old.keys() - new.keys())
moved = moves(added, removed)
targets = set(moved.values())
added = [name for name in added if name not in targets]
removed = [name for name in removed if name not in moved]
changed = {}
pairs = [(name, name) for name in sorted(old.keys() & new.keys())] + sorted(moved.items())
for name, target in pairs:
a, b = old[name], new[target]
if a == b:
continue
change = {"moved_from": name} if name != target else {}
for part in ("methods", "fields"):
x, y = a[part], b[part]
part_change = {"added": sorted(y.keys() - x.keys()), "removed": sorted(x.keys() - y.keys()),
"changed": sorted(k for k in x.keys() & y.keys() if x[k] != y[k])}
if any(part_change.values()):
change[part] = part_change
for part in ("super", "interfaces", "access"):
if a[part] != b[part]:
change[part] = [a[part], b[part]]
changed[target] = change
return {"added": added, "removed": removed, "moved": moved, "changed": dict(sorted(changed.items()))}
def changed_methods(change):
"""How many methods a changed class gained, lost or changed."""
methods = change.get("methods", {})
return sum(len(methods.get(k, [])) for k in ("added", "removed", "changed"))
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("jars", nargs="+")
parser.add_argument("-o", "--output", required=True)
parser.add_argument("-j", "--jobs", type=int)
args = parser.parse_args()
classes = index_jars(args.jars, args.jobs)
with open(args.output, "w") as stream:
json.dump(classes, stream, separators=(",", ":"))
print(f"{len(classes)} classes, {sum(len(c['methods']) for c in classes.values())} methods")
return 0
if __name__ == "__main__":
sys.exit(main())
+227
View File
@@ -0,0 +1,227 @@
#!/usr/bin/env python3
"""Map vanilla classes to the Horizon code that ports them and the harnesses that prove it
(docs/version-ports.md).
Three sources, all in the repository:
- Rust comments: a CamelCase name in a `//` comment that is exactly one vanilla class's simple name (or
`Outer.Inner`), or any class name followed by `.member`, `::member` or `#member`, maps that file (and the
member, when named) to the class.
- Java oracles (mods/horizon/src/test/java/dev/horizonlab/parity): the vanilla classes each one imports or
names in full; a harness uses the oracles its script names, and those its recorded parity dependencies list.
- Harness probes: the crate examples and binaries a harness script names, and the modules they `use`; the
harness proves those files, so it covers the classes they name.
vanilla_map.py CLASS_INDEX [CLASS...] print what each class maps to (all classes: --all)
"""
import argparse
from collections import defaultdict
import json
from pathlib import Path
import re
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
ORACLES = "mods/horizon/src/test/java/dev/horizonlab/parity"
COMMENT_NAME = re.compile(r"\b([A-Z][A-Za-z0-9]*)(?:(?:\.|::|#)([A-Za-z][A-Za-z0-9]*))?")
JAVA_IMPORT = re.compile(r"\bimport\s+(static\s+)?((?:net\.minecraft|com\.mojang)\.[\w.]+?)(\.\*)?\s*;")
JAVA_QUALIFIED = re.compile(r"\b((?:net\.minecraft|com\.mojang)(?:\.[a-z][a-z0-9_]*)+(?:\.[A-Z]\w*)+)")
RUST_USE = re.compile(r"\b(horizon_[a-z_]+|crate)((?:::[a-z_][a-z0-9_]*)+)")
def humps(name):
return sum(c.isupper() for c in name)
class Names:
"""Vanilla class names (internal form) by simple name, with a tie-break for the ambiguous ones."""
def __init__(self, classes):
self.classes = set(classes)
self.by_simple = defaultdict(set)
self.by_package = defaultdict(set)
for name in classes:
package, _, local = name.rpartition("/")
parts = local.split("$")
if any(p[:1].isdigit() for p in parts):
continue # Anonymous classes.
self.by_simple[parts[-1]].add(name)
if len(parts) > 1:
self.by_simple[".".join(parts[-2:])].add(name)
else:
self.by_package[package].add(name)
def resolve(self, simple):
"""The one class a simple name means: unique, else the unique top-level one, else the unique
top-level one under net/minecraft."""
candidates = self.by_simple.get(simple, set())
for narrowed in (candidates, {c for c in candidates if "$" not in c},
{c for c in candidates if "$" not in c and c.startswith("net/minecraft/")}):
if len(narrowed) == 1:
return next(iter(narrowed))
return None
def java(self, dotted):
"""net.minecraft.a.B.C -> net/minecraft/a/B$C, if that class exists."""
parts = dotted.split(".")
for split in range(len(parts) - 1, 0, -1):
if not parts[split][:1].isupper() or parts[split - 1][:1].isupper():
continue
name = "/".join(parts[:split]) + "/" + "$".join(parts[split:])
if name in self.classes:
return name
return None
def comment_text(line):
at = line.find("//")
return line[at:] if at >= 0 else ""
def rust_references(files, names, read=lambda f: (ROOT / f).read_text(errors="replace")):
"""{class: {file: [members named]}} from the comments of the given Rust files."""
found = defaultdict(lambda: defaultdict(set))
for path in files:
for line in read(path).splitlines():
for match in COMMENT_NAME.finditer(comment_text(line)):
name, member = match[1], match[2]
if member and member[0].isupper() and names.resolve(f"{name}.{member}"):
name, member = f"{name}.{member}", None
if humps(name.split(".")[-1]) < 2 and not member:
continue
cls = names.resolve(name)
if cls:
found[cls][path].update([member] if member else [])
return {cls: {path: sorted(members) for path, members in files.items()} for cls, files in found.items()}
def java_references(text, names):
"""The vanilla classes a Java source imports or names in full."""
found = set()
words = set(re.findall(r"\b[A-Z]\w*", text))
for static, dotted, wildcard in JAVA_IMPORT.findall(text):
if wildcard and not static:
package = dotted.replace(".", "/")
found.update(c for c in names.by_package.get(package, ()) if c.rsplit("/", 1)[1] in words)
else:
cls = names.java(dotted if not static or wildcard else dotted.rsplit(".", 1)[0])
if cls:
found.add(cls)
for dotted in JAVA_QUALIFIED.findall(text):
cls = names.java(dotted)
if cls:
found.add(cls)
return found
def harness_name(script):
return Path(script).stem.removeprefix("test_").removesuffix("_parity")
def probe_files(script_text, examples):
"""The crate examples and binaries a harness script names: {name: path}."""
words = set(re.findall(r"[A-Za-z0-9_]+", script_text))
return {name: path for name, path in examples.items() if name in words}
def module_files(path, text, sources):
"""The workspace source files the `use` paths of a Rust file reach (one level)."""
crate_dir = path.split("/src/")[0] if "/src/" in path else path.split("/examples/")[0]
reached = set()
for crate, modules in RUST_USE.findall(text):
base = crate_dir if crate == "crate" else "crates/" + crate.replace("_", "-")
parts = [p for p in modules.split("::") if p]
for depth in range(len(parts), 0, -1):
stem = f"{base}/src/{'/'.join(parts[:depth])}"
hit = next((c for c in (f"{stem}.rs", f"{stem}/mod.rs") if c in sources), None)
if hit:
reached.add(hit)
break
return reached
def build(classes, root=ROOT, dependencies=None):
"""{"classes": {class: {"rust": {file: [members]}, "harnesses": [...]}}, "harnesses": {harness: {...}}}.
dependencies: {harness: [repository paths]} recorded by tools/parity.py, when available."""
names = Names(classes)
tracked = subprocess.check_output(["git", "ls-files"], cwd=root, text=True).split()
rust = [f for f in tracked if f.endswith(".rs") and (f.startswith("crates/") or f.startswith("launcher/"))]
sources = set(rust)
read = lambda f: (root / f).read_text(errors="replace") # noqa: E731
by_class = rust_references(rust, names, read)
by_file = defaultdict(set)
for cls, files in by_class.items():
for path in files:
by_file[path].add(cls)
oracles = {Path(f).stem: java_references(read(f), names)
for f in tracked if f.startswith(ORACLES + "/") and f.endswith(".java")}
examples = {Path(f).stem: f for f in rust if "/examples/" in f or "/src/bin/" in f}
harnesses = {}
for script in sorted(f for f in tracked if re.fullmatch(r"tools/test_\w+_parity\.py", f)):
text = read(script)
name = harness_name(script)
used = {o for o in oracles if re.search(rf"\b{o}\b", text)}
used |= {Path(p).stem for p in (dependencies or {}).get(name, []) if Path(p).stem in oracles}
probes = probe_files(text, examples)
files = set(probes.values())
for probe in probes.values():
files |= module_files(probe, read(probe), sources)
covered = set().union(*(oracles[o] for o in used), *(by_file.get(f, ()) for f in files))
harnesses[name] = {"oracles": sorted(used), "rust": sorted(files), "classes": sorted(covered)}
mapped = defaultdict(lambda: {"rust": {}, "harnesses": []})
for cls, files in by_class.items():
mapped[cls]["rust"] = files
for name, harness in harnesses.items():
for cls in harness["classes"]:
mapped[cls]["harnesses"].append(name)
return {"classes": dict(sorted(mapped.items())), "harnesses": harnesses}
def lookup(mapping, cls):
"""What a class maps to, falling back to its outermost class for nested ones."""
entry = mapping["classes"].get(cls)
if entry is None and "$" in cls:
entry = mapping["classes"].get(cls.split("$")[0])
return entry or {"rust": {}, "harnesses": []}
def recorded_dependencies(shared):
"""tools/parity.py's dependency records ({harness: [paths]}), from parity-shared/dependencies."""
directory = Path(shared) / "dependencies"
found = {}
for path in sorted(directory.glob("*.json")) if directory.is_dir() else []:
try:
found[path.stem.split("#")[0]] = json.loads(path.read_text())["dependencies"]
except (ValueError, KeyError):
continue
return found
def default_shared():
return (ROOT / ".local" / "minecraft").resolve().parent / "parity-shared"
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("index", help="a class index (tools/class_index.py, versions/<id>/classes.json)")
parser.add_argument("classes", nargs="*", help="internal or dotted names")
parser.add_argument("--all", action="store_true", help="print the whole map")
args = parser.parse_args()
classes = json.loads(Path(args.index).read_text())
mapping = build(classes, dependencies=recorded_dependencies(default_shared()))
if args.all:
print(json.dumps(mapping, indent=1))
return 0
for cls in args.classes:
cls = cls.replace(".", "/") if "/" not in cls else cls
print(json.dumps({cls: lookup(mapping, cls)}, indent=2))
covered = sum(1 for entry in mapping["classes"].values() if entry["harnesses"])
print(f"{len(mapping['classes'])} vanilla classes mapped, {covered} covered by a harness", file=sys.stderr)
return 0
if __name__ == "__main__":
sys.exit(main())
+47 -6
View File
@@ -3,9 +3,11 @@
(docs/version-ports.md).
version_port.py detect [--snapshots] what Mojang released that Horizon doesn't pin
version_port.py prepare VERSION download, verify, run the data generator, read its facts
version_port.py prepare VERSION download, verify, run the data generator, read its facts,
index its classes (tools/class_index.py)
version_port.py diff OLD NEW facts, packets, registries, blocks, items, commands,
data packs, assets and lang: diff-OLD-NEW/diff.json
data packs, assets, lang and classes, with the Horizon
code each changed class concerns: diff-OLD-NEW/diff.json
version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep
The downloads are for development only: they go to the version store (~/code/horizon-parity/versions,
@@ -29,9 +31,11 @@ import urllib.request
import zipfile
try:
from . import lab
from . import class_index, lab, vanilla_map
except ImportError:
import class_index
import lab
import vanilla_map
MANIFEST_URL = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"
STORE = Path(os.environ.get("HORIZON_VERSIONS", Path.home() / "code" / "horizon-parity" / "versions"))
@@ -216,6 +220,9 @@ def prepare(args):
if facts["id"] != version:
raise ValueError(f"The client jar says it is {facts['id']}, not {version}")
lab.write_json(directory / "facts.json", facts)
classes = class_index.index_jars([found["client"], found["server_game"]], jobs=args.jobs)
(directory / "classes.json").write_text(json.dumps(classes, separators=(",", ":")))
print(f"Indexed {len(classes)} classes, {sum(len(c['methods']) for c in classes.values())} methods")
lab.write_json(directory / "prepared.json", {
"version": version, "pinned": version == pinned() and not args.download,
"paths": {key: str(path) for key, path in found.items()},
@@ -431,8 +438,26 @@ def read_reports(reports):
}
def diff_versions(old_paths, new_paths, old_facts, new_facts):
def diff_classes(old, new, mapping):
"""The class diff, each changed or removed class with the Horizon files and harnesses that concern it
(looked up under its old name: Horizon's code names the pinned version's classes)."""
result = class_index.diff(old, new)
horizon = {}
for name, change in result["changed"].items():
entry = vanilla_map.lookup(mapping, change.get("moved_from", name))
if entry["rust"] or entry["harnesses"]:
horizon[name] = entry
for name in result["removed"]:
entry = vanilla_map.lookup(mapping, name)
if entry["rust"] or entry["harnesses"]:
horizon[name] = entry
result["horizon"] = horizon
return result
def diff_versions(old_paths, new_paths, old_facts, new_facts, old_classes=None, new_classes=None, mapping=None):
a, b = read_reports(old_paths["reports"]), read_reports(new_paths["reports"])
classes = diff_classes(old_classes, new_classes, mapping) if old_classes is not None else None
return {
"facts": diff_facts(old_facts, new_facts),
"packets": diff_packets(a["packets"], b["packets"]),
@@ -443,6 +468,7 @@ def diff_versions(old_paths, new_paths, old_facts, new_facts):
"data": diff_contents(old_paths["client"], new_paths["client"], "data/minecraft/"),
"assets": diff_contents(old_paths["client"], new_paths["client"], "assets/minecraft/"),
"lang": diff_lang(old_paths["client"], new_paths["client"]),
"classes": classes,
}
@@ -480,6 +506,12 @@ def summary(old, new, result):
lines.append(f" {key} files: +{counts['added']} -{counts['removed']} ~{counts['changed']}")
lang = result["lang"]
lines.append(f" en_us keys: +{len(lang['added'])} -{len(lang['removed'])} ~{len(lang['changed'])}")
classes = result.get("classes")
if classes:
methods = sum(class_index.changed_methods(c) for c in classes["changed"].values())
lines.append(f" classes: +{len(classes['added'])} -{len(classes['removed'])}, {len(classes['moved'])} moved, "
f"{len(classes['changed'])} changed ({methods} methods); {len(classes['horizon'])} of the "
f"changed or removed ones are ported or proved by Horizon")
return "\n".join(lines)
@@ -487,10 +519,17 @@ def diff_dir(old, new):
return STORE / f"diff-{safe_version(old)}-{safe_version(new)}"
def load(version, name):
return json.loads((version_dir(version) / name).read_text())
def diff(args):
facts = {v: json.loads((version_dir(v) / "facts.json").read_text()) for v in (args.old, args.new)}
result = diff_versions(paths(args.old), paths(args.new), facts[args.old], facts[args.new])
directory = diff_dir(args.old, args.new)
old_classes, new_classes = load(args.old, "classes.json"), load(args.new, "classes.json")
mapping = vanilla_map.build(old_classes, dependencies=vanilla_map.recorded_dependencies(vanilla_map.default_shared()))
lab.write_json(directory / "vanilla_map.json", mapping)
result = diff_versions(paths(args.old), paths(args.new), load(args.old, "facts.json"), load(args.new, "facts.json"),
old_classes, new_classes, mapping)
lab.write_json(directory / "diff.json", result)
print(summary(args.old, args.new, result))
print(f"\n{directory / 'diff.json'}")
@@ -535,6 +574,8 @@ def main():
command = commands.add_parser("prepare", help="download, verify and run the data generator")
command.add_argument("version")
command.add_argument("--download", action="store_true", help="download even the pinned version")
command.add_argument("-j", "--jobs", type=int, default=int(os.environ.get("FLEET_LOAD_JOBS", 0)) or None,
help="class indexing processes (default: $FLEET_LOAD_JOBS, else the CPU count)")
command = commands.add_parser("diff", help="diff two prepared versions")
command.add_argument("old")
command.add_argument("new")