diff --git a/docs/version-ports.md b/docs/version-ports.md index d625fcda8..719f99a8c 100644 --- a/docs/version-ports.md +++ b/docs/version-ports.md @@ -266,12 +266,13 @@ python3 tools/version_port.py prepare 26.3 # the pinned version: index python3 tools/version_port.py prepare 26.4 # downloads, verifies, runs the data generator (about 10 s) python3 tools/version_port.py diff 26.3 26.4 # summary; the full diff in versions/diff-26.3-26.4/diff.json python3 tools/version_port.py gc # keep the pinned version and the newest two others +python3 tools/vanilla_map.py versions/26.3/classes.json net.minecraft.world.food.FoodData # its Rust files and harnesses ``` | Slice | Status | | --- | --- | | This design | this page | | `detect`, `prepare`, `diff` (facts, packets, registries, blocks, items, commands, data, assets), `gc` | done | -| Class index, class diff, `vanilla_map.json` | next PR | +| Class index (`tools/class_index.py`), class diff with moves, `vanilla_map.json` (`tools/vanilla_map.py`) | done | | Port report (`report`), `port` | next PR | | Changed-class decompilation, `regen`, `version_consts` refactor, candidate harness run, `publish`, launchd agent | later | diff --git a/tests/test_class_index.py b/tests/test_class_index.py new file mode 100644 index 000000000..23aeffec9 --- /dev/null +++ b/tests/test_class_index.py @@ -0,0 +1,170 @@ +import struct +import unittest + +from tools import class_index, vanilla_map + + +class Pool: + """A constant pool built in the order entries are asked for; `pad` shifts every index.""" + + def __init__(self, pad=0): + self.entries, self.index = [], {} + for i in range(pad): + self.utf8(f"padding{i}") + + def add(self, key, data): + if key not in self.index: + self.entries.append(data) + self.index[key] = len(self.entries) + return self.index[key] + + def utf8(self, text): + raw = text.encode() + return self.add(("u", text), b"\x01" + struct.pack(">H", len(raw)) + raw) + + def cls(self, name): + return self.add(("c", name), b"\x07" + struct.pack(">H", self.utf8(name))) + + def string(self, text): + return self.add(("s", text), b"\x08" + struct.pack(">H", self.utf8(text))) + + def method(self, owner, name, descriptor): + nat = self.add(("nt", name, descriptor), + b"\x0c" + struct.pack(">HH", self.utf8(name), self.utf8(descriptor))) + return self.add(("m", owner, name, descriptor), b"\x0a" + struct.pack(">HH", self.cls(owner), nat)) + + def bytes(self): + return struct.pack(">H", len(self.entries) + 1) + b"".join(self.entries) + + +def class_file(methods, name="a/B", pad=0, wide=False): + """A class with the given methods: {name: (descriptor, [instructions])}, each instruction + ("ldc", text) | ("call", owner, name, descriptor) | ("ifnull", +instructions) | (opcode,).""" + pool = Pool(pad) + this, parent = pool.cls(name), pool.cls("java/lang/Object") + code_name = pool.utf8("Code") + encoded = [] + for method, (descriptor, instructions) in methods.items(): + sizes = [3 if i[0] in ("call", "ifnull") or (i[0] == "ldc" and wide) else 2 if i[0] == "ldc" else 1 + for i in instructions] + body = b"" + for at, instruction in enumerate(instructions): + if instruction[0] == "ldc": + index = pool.string(instruction[1]) + body += bytes([19]) + struct.pack(">H", index) if wide else bytes([18, index]) + elif instruction[0] == "call": + body += bytes([184]) + struct.pack(">H", pool.method(*instruction[1:])) + elif instruction[0] == "ifnull": + offset = sum(sizes[at:at + instruction[1]]) + body += bytes([198]) + struct.pack(">h", offset) + else: + body += bytes([instruction[0]]) + code = struct.pack(">HHI", 2, 1, len(body)) + body + struct.pack(">HH", 0, 0) + encoded.append(struct.pack(">HHHH", 0x0009, pool.utf8(method), pool.utf8(descriptor), 1) + + struct.pack(">HI", code_name, len(code)) + code) + return (b"\xca\xfe\xba\xbe" + struct.pack(">HH", 0, 69) + pool.bytes() + + struct.pack(">HHHH", 0x0021, this, parent, 0) + struct.pack(">H", 0) + + struct.pack(">H", len(encoded)) + b"".join(encoded) + struct.pack(">H", 0)) + + +def index(methods, **options): + return class_index.ClassFile(class_file(methods, **options)).index() + + +GREET = {"greet": ("()Ljava/lang/String;", [("ldc", "hi"), ("ifnull", 2), (1,), (176,)])} + + +class ClassIndexTests(unittest.TestCase): + def test_constant_pool_layout_and_ldc_width_do_not_change_hashes(self): + plain = index(GREET) + self.assertEqual(index(GREET, pad=3), plain) + self.assertEqual(index(GREET, pad=300, wide=True), plain) + + def test_a_changed_literal_or_call_changes_the_hash(self): + plain = index(GREET)["methods"] + other = index({"greet": ("()Ljava/lang/String;", [("ldc", "hello"), ("ifnull", 2), (1,), (176,)])}) + self.assertNotEqual(other["methods"], plain) + call = {"run": ("()V", [("call", "a/C", "go", "()V"), (177,)])} + moved = {"run": ("()V", [("call", "b/C", "go", "()V"), (177,)])} + self.assertNotEqual(index(call)["methods"], index(moved)["methods"]) + + def test_lambdas_fold_into_their_declaring_method(self): + body = ("()V", [(177,)]) + first = index({"run": body, "lambda$run$0": body, "lambda$run$1": ("()V", [(1,), (87,), (177,)])}) + renumbered = index({"run": body, "lambda$run$4": ("()V", [(1,), (87,), (177,)]), "lambda$run$5": body}) + self.assertEqual(set(first["methods"]), {"run()V", "lambda$run"}) + self.assertEqual(first["methods"], renumbered["methods"]) + + def test_diff_with_moves(self): + def entry(**methods): + return {"access": 33, "super": "java/lang/Object", "interfaces": [], "fields": {}, "methods": methods} + old = {"a/Same": entry(f="1"), "a/Edit": entry(f="1", g="2"), "a/Gone": entry(), + "a/Moved": entry(f="1"), "a/x/Twin": entry(), "a/y/Twin": entry()} + new = {"a/Same": entry(f="1"), "a/Edit": entry(f="9", h="3"), "b/Moved": entry(f="2"), "a/New": entry(), + "b/Twin": entry()} + result = class_index.diff(old, new) + self.assertEqual(result["moved"], {"a/Moved": "b/Moved"}) + self.assertEqual(result["added"], ["a/New", "b/Twin"]) + self.assertEqual(result["removed"], ["a/Gone", "a/x/Twin", "a/y/Twin"]) + self.assertEqual(result["changed"]["a/Edit"], + {"methods": {"added": ["h"], "removed": ["g"], "changed": ["f"]}}) + self.assertEqual(result["changed"]["b/Moved"]["moved_from"], "a/Moved") + self.assertEqual(class_index.changed_methods(result["changed"]["a/Edit"]), 3) + + +CLASSES = ["net/minecraft/world/food/FoodData", "net/minecraft/world/level/Level", + "com/mojang/realmsclient/Level", "net/minecraft/world/entity/Entity", + "net/minecraft/world/entity/Entity$RemovalReason", "net/minecraft/a/Thing$1", + "net/minecraft/server/level/ServerLevel"] + + +class VanillaMapTests(unittest.TestCase): + def setUp(self): + self.names = vanilla_map.Names(CLASSES) + + def test_resolve(self): + self.assertEqual(self.names.resolve("FoodData"), "net/minecraft/world/food/FoodData") + self.assertEqual(self.names.resolve("Level"), "net/minecraft/world/level/Level") + self.assertEqual(self.names.resolve("RemovalReason"), "net/minecraft/world/entity/Entity$RemovalReason") + self.assertEqual(self.names.resolve("Entity.RemovalReason"), "net/minecraft/world/entity/Entity$RemovalReason") + self.assertIsNone(self.names.resolve("Missing")) + self.assertEqual(self.names.java("net.minecraft.world.entity.Entity.RemovalReason"), + "net/minecraft/world/entity/Entity$RemovalReason") + + def test_rust_comments(self): + files = {"crates/x/src/food.rs": "/// FoodData.tick, as vanilla\nfn tick() {} // ServerLevel too\n" + "let FoodData = 1;\n// a Level, a Level.tick() and Entity.RemovalReason\n"} + found = vanilla_map.rust_references(files, self.names, read=files.get) + self.assertEqual(found["net/minecraft/world/food/FoodData"], {"crates/x/src/food.rs": ["tick"]}) + self.assertEqual(found["net/minecraft/server/level/ServerLevel"], {"crates/x/src/food.rs": []}) + # A one-word name counts only with a member: "a Level" alone is English. + self.assertEqual(found["net/minecraft/world/level/Level"], {"crates/x/src/food.rs": ["tick"]}) + self.assertIn("net/minecraft/world/entity/Entity$RemovalReason", found) + + def test_java_references(self): + text = ("import net.minecraft.world.food.*;import net.minecraft.world.entity.Entity;\n" + "import static net.minecraft.world.level.Level.OVERWORLD;\n" + "class X { FoodData f; net.minecraft.server.level.ServerLevel l; }") + self.assertEqual(vanilla_map.java_references(text, self.names), { + "net/minecraft/world/food/FoodData", "net/minecraft/world/entity/Entity", + "net/minecraft/world/level/Level", "net/minecraft/server/level/ServerLevel"}) + + def test_probe_modules(self): + sources = {"crates/horizon-world/src/food.rs", "crates/horizon-world/src/level/mod.rs", + "crates/horizon-world/src/level/tick.rs"} + text = "use horizon_world::food::FoodData;\nuse horizon_world::level::tick::{a, b};\nuse crate::level;\n" + self.assertEqual(vanilla_map.module_files("crates/horizon-world/examples/p.rs", text, sources), + {"crates/horizon-world/src/food.rs", "crates/horizon-world/src/level/tick.rs", + "crates/horizon-world/src/level/mod.rs"}) + self.assertEqual(vanilla_map.probe_files("run(['food_fixture'])", {"food_fixture": "e.rs", "food": "f.rs"}), + {"food_fixture": "e.rs"}) + self.assertEqual(vanilla_map.harness_name("tools/test_food_parity.py"), "food") + + def test_lookup_falls_back_to_the_outer_class(self): + mapping = {"classes": {"a/B": {"rust": {"x.rs": []}, "harnesses": ["h"]}}} + self.assertEqual(vanilla_map.lookup(mapping, "a/B$C")["harnesses"], ["h"]) + self.assertEqual(vanilla_map.lookup(mapping, "a/D"), {"rust": {}, "harnesses": []}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/class_index.py b/tools/class_index.py new file mode 100644 index 000000000..3575e3535 --- /dev/null +++ b/tools/class_index.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Index a game jar's classes for version diffs (docs/version-ports.md). + +Each class gets its superclass, interfaces, fields and methods; each method a hash of its bytecode in which +every constant-pool reference is replaced by what it names and every branch target by an instruction index. +Recompiling unchanged code (a constant pool reordered, ldc widened to ldc_w) then hashes the same. Lambda +bodies (lambda$name$N) are folded into one entry per declaring method, `lambda$name`, so lambdas renumbered by +an unrelated change don't show as changes. + + class_index.py JAR [JAR...] -o classes.json +""" + +import argparse +from concurrent.futures import ProcessPoolExecutor +import hashlib +import json +import re +import struct +import sys +import zipfile + +LAMBDA = re.compile(r"^lambda\$(.*)\$\d+$") + +# Operand bytes per opcode, for the fixed-length instructions. None: tableswitch, lookupswitch, wide. +OPERANDS = [0] * 202 +for _op, _size in [(16, 1), (17, 2), (18, 1), (19, 2), (20, 2), (169, 1), (188, 1), (185, 4), (186, 4), (197, 3)]: + OPERANDS[_op] = _size +for _op in list(range(21, 26)) + list(range(54, 59)): + OPERANDS[_op] = 1 # Local variable loads and stores. +OPERANDS[132] = 2 # iinc. +for _op in list(range(153, 169)) + [198, 199]: + OPERANDS[_op] = 2 # Branches. +OPERANDS[200] = OPERANDS[201] = 4 # goto_w, jsr_w. +for _op in [178, 179, 180, 181, 182, 183, 184, 187, 189, 192, 193]: + OPERANDS[_op] = 2 # Constant-pool references. +OPERANDS[170] = OPERANDS[171] = OPERANDS[196] = None +CP_OPERAND = {18, 19, 20, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 192, 193, 197} +BRANCH = set(range(153, 169)) | {198, 199, 200, 201} +# ldc_w loads what ldc loads; goto_w jumps where goto jumps. +CANONICAL = {19: 18, 200: 167, 201: 168} + + +def lambda_owner(name): + match = LAMBDA.match(name) + return f"lambda${match[1]}" if match else name + + +class ClassFile: + def __init__(self, data): + self.data = data + self.offset = 0 + self.parse() + + def u(self, size): + value = int.from_bytes(self.data[self.offset:self.offset + size], "big") + self.offset += size + return value + + def raw(self, size): + value = self.data[self.offset:self.offset + size] + self.offset += size + return value + + def parse(self): + if self.raw(4) != b"\xca\xfe\xba\xbe": + raise ValueError("Invalid class file magic") + self.raw(4) + count = self.u(2) + self.pool = [None] * count + index = 1 + while index < count: + tag = self.u(1) + if tag == 1: + entry = (1, self.raw(self.u(2)).decode("utf-8", errors="replace")) + elif tag in (3, 4): + entry = (tag, self.raw(4)) + elif tag in (5, 6): + entry = (tag, self.raw(8)) + elif tag in (7, 8, 16, 19, 20): + entry = (tag, self.u(2)) + elif tag in (9, 10, 11, 12, 17, 18): + entry = (tag, self.u(2), self.u(2)) + elif tag == 15: + entry = (tag, self.u(1), self.u(2)) + else: + raise ValueError(f"Unknown constant pool tag {tag}") + self.pool[index] = entry + index += 2 if tag in (5, 6) else 1 + self.access = self.u(2) + self.name = self.class_name(self.u(2)) + super_index = self.u(2) + self.super = self.class_name(super_index) if super_index else None + self.interfaces = [self.class_name(self.u(2)) for _ in range(self.u(2))] + self.fields = {} + for _ in range(self.u(2)): + access, name, descriptor = self.u(2), self.utf8(self.u(2)), self.utf8(self.u(2)) + constant = None + for attribute, data in self.attributes(): + if attribute == "ConstantValue": + constant = self.constant(struct.unpack(">H", data)[0]) + self.fields[f"{name} {descriptor}"] = [access, constant] + self.methods = [] + for _ in range(self.u(2)): + access, name, descriptor = self.u(2), self.utf8(self.u(2)), self.utf8(self.u(2)) + code = next((data for attribute, data in self.attributes() if attribute == "Code"), None) + self.methods.append((access, name, descriptor, code)) + self.bootstrap = [] + for attribute, data in self.attributes(): + if attribute == "BootstrapMethods": + reader = ClassFile.__new__(ClassFile) + reader.data, reader.offset = data, 0 + for _ in range(reader.u(2)): + method = reader.u(2) + self.bootstrap.append((method, [reader.u(2) for _ in range(reader.u(2))])) + + def attributes(self): + return [(self.utf8(self.u(2)), self.raw(self.u(4))) for _ in range(self.u(2))] + + def utf8(self, index): + return self.pool[index][1] + + def class_name(self, index): + return self.utf8(self.pool[index][1]) + + def member(self, index): + _, owner, name_and_type = self.pool[index] + _, name, descriptor = self.pool[name_and_type] + return f"{self.class_name(owner)}.{lambda_owner(self.utf8(name))}{self.utf8(descriptor)}" + + def constant(self, index): + """What a constant-pool entry names, independent of where it sits in the pool.""" + entry = self.pool[index] + tag = entry[0] + if tag == 1: + return f"u:{entry[1]}" + if tag == 3: + return f"i:{struct.unpack('>i', entry[1])[0]}" + if tag == 4: + return f"f:{entry[1].hex()}" + if tag == 5: + return f"l:{struct.unpack('>q', entry[1])[0]}" + if tag == 6: + return f"d:{entry[1].hex()}" + if tag == 7: + return f"c:{self.utf8(entry[1])}" + if tag == 8: + return f"s:{self.utf8(entry[1])}" + if tag in (9, 10, 11): + return f"m:{self.member(index)}" + if tag == 12: + return f"nt:{lambda_owner(self.utf8(entry[1]))}{self.utf8(entry[2])}" + if tag == 15: + return f"h{entry[1]}:{self.member(entry[2])}" + if tag == 16: + return f"t:{self.utf8(entry[1])}" + if tag in (17, 18): + method, arguments = self.bootstrap[entry[1]] + return (f"y:{self.constant(entry[2])}@{self.constant(method)}" + f"({','.join(self.constant(a) for a in arguments)})") + return f"x{tag}:{entry[1]}" + + def code_tokens(self, code): + """The method's instructions, exception handlers included, as position-independent tokens.""" + max_stack, max_locals, length = struct.unpack(">HHI", code[:8]) + body = code[8:8 + length] + starts, instructions = {}, [] + pc = 0 + while pc < length: + starts[pc] = len(instructions) + op = body[pc] + if op == 170 or op == 171: # tableswitch, lookupswitch + at = (pc + 4) & ~3 + default = struct.unpack(">i", body[at:at + 4])[0] + if op == 170: + low, high = struct.unpack(">ii", body[at + 4:at + 12]) + targets = struct.unpack(f">{high - low + 1}i", body[at + 12:at + 12 + 4 * (high - low + 1)]) + keys = list(range(low, high + 1)) + end = at + 12 + 4 * len(targets) + else: + count = struct.unpack(">i", body[at + 4:at + 8])[0] + pairs = struct.unpack(f">{2 * count}i", body[at + 8:at + 8 + 8 * count]) + keys, targets = pairs[0::2], pairs[1::2] + end = at + 8 + 8 * count + instructions.append(("switch", pc, default, list(zip(keys, targets)))) + pc = end + continue + if op == 196: # wide + size = 5 if body[pc + 1] == 132 else 3 + instructions.append(("op", pc, 196, body[pc + 1:pc + 1 + size])) + pc += 1 + size + continue + if op >= len(OPERANDS): + raise ValueError(f"Unknown opcode {op}") + size = OPERANDS[op] + instructions.append(("op", pc, op, body[pc + 1:pc + 1 + size])) + pc += 1 + size + tokens = [f"{max_stack},{max_locals}"] + for kind, pc, *rest in instructions: + if kind == "switch": + default, cases = rest + tokens.append(f"switch {starts.get(pc + default)} " + + " ".join(f"{k}>{starts.get(pc + t)}" for k, t in cases)) + continue + op, operand = rest + if op in CP_OPERAND: + index = operand[0] if op == 18 else int.from_bytes(operand[:2], "big") + tokens.append(f"{CANONICAL.get(op, op)} {self.constant(index)}") + elif op in BRANCH: + offset = int.from_bytes(operand, "big", signed=True) + tokens.append(f"{CANONICAL.get(op, op)} >{starts.get(pc + offset)}") + else: + tokens.append(f"{op} {operand.hex()}") + exceptions = 8 + length + for i in range(struct.unpack(">H", code[exceptions:exceptions + 2])[0]): + start, end, handler, kind = struct.unpack(">HHHH", code[exceptions + 2 + 8 * i:exceptions + 10 + 8 * i]) + caught = self.class_name(kind) if kind else "*" + tokens.append(f"try {starts.get(start)} {starts.get(end, len(instructions))} {starts.get(handler)} {caught}") + return tokens + + def index(self): + methods, lambdas = {}, {} + for access, name, descriptor, code in self.methods: + tokens = [str(access & 0x0FFF)] + (self.code_tokens(code) if code else ["abstract"]) + digest = hashlib.sha1("\n".join(tokens).encode()).hexdigest()[:16] + owner = lambda_owner(name) + if owner != name: + lambdas.setdefault(owner, []).append(digest) + else: + methods[f"{name}{descriptor}"] = digest + for owner, digests in lambdas.items(): + methods[owner] = hashlib.sha1(" ".join(sorted(digests)).encode()).hexdigest()[:16] + return {"access": self.access, "super": self.super, "interfaces": self.interfaces, + "fields": self.fields, "methods": methods} + + +def index_entries(jar, names): + with zipfile.ZipFile(jar) as archive: + return {name[:-6]: ClassFile(archive.read(name)).index() for name in names} + + +def index_jar(jar, jobs=None, prefixes=("net/minecraft/", "com/mojang/")): + """{internal class name: index} for every game class in a jar.""" + with zipfile.ZipFile(jar) as archive: + names = sorted(n for n in archive.namelist() if n.endswith(".class") and n.startswith(prefixes)) + batches = [names[i::32] for i in range(32)] + result = {} + with ProcessPoolExecutor(max_workers=jobs) as pool: + for part in pool.map(index_entries, [jar] * len(batches), batches): + result.update(part) + return dict(sorted(result.items())) + + +def index_jars(jars, jobs=None): + """Several jars into one index; a class in more than one keeps the first jar's (client before server).""" + classes = {} + for jar in jars: + for name, data in index_jar(jar, jobs).items(): + classes.setdefault(name, data) + return classes + + +def local_name(name): + return name.rsplit("/", 1)[-1] + + +def moves(added, removed): + """{old: new} for a removed class whose local name (Outer$Inner) one added class has, uniquely on both sides.""" + def unique(names): + by_local = {} + for name in names: + by_local.setdefault(local_name(name), []).append(name) + return {local: found[0] for local, found in by_local.items() if len(found) == 1} + gone, new = unique(removed), unique(added) + return {gone[local]: new[local] for local in sorted(gone.keys() & new.keys())} + + +def diff(old, new): + """Classes added, removed, moved (same local name, another package) and changed; per changed class its + methods and fields added, removed and changed. A moved class that changed too is listed under its new name + with moved_from.""" + added = sorted(new.keys() - old.keys()) + removed = sorted(old.keys() - new.keys()) + moved = moves(added, removed) + targets = set(moved.values()) + added = [name for name in added if name not in targets] + removed = [name for name in removed if name not in moved] + changed = {} + pairs = [(name, name) for name in sorted(old.keys() & new.keys())] + sorted(moved.items()) + for name, target in pairs: + a, b = old[name], new[target] + if a == b: + continue + change = {"moved_from": name} if name != target else {} + for part in ("methods", "fields"): + x, y = a[part], b[part] + part_change = {"added": sorted(y.keys() - x.keys()), "removed": sorted(x.keys() - y.keys()), + "changed": sorted(k for k in x.keys() & y.keys() if x[k] != y[k])} + if any(part_change.values()): + change[part] = part_change + for part in ("super", "interfaces", "access"): + if a[part] != b[part]: + change[part] = [a[part], b[part]] + changed[target] = change + return {"added": added, "removed": removed, "moved": moved, "changed": dict(sorted(changed.items()))} + + +def changed_methods(change): + """How many methods a changed class gained, lost or changed.""" + methods = change.get("methods", {}) + return sum(len(methods.get(k, [])) for k in ("added", "removed", "changed")) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("jars", nargs="+") + parser.add_argument("-o", "--output", required=True) + parser.add_argument("-j", "--jobs", type=int) + args = parser.parse_args() + classes = index_jars(args.jars, args.jobs) + with open(args.output, "w") as stream: + json.dump(classes, stream, separators=(",", ":")) + print(f"{len(classes)} classes, {sum(len(c['methods']) for c in classes.values())} methods") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/vanilla_map.py b/tools/vanilla_map.py new file mode 100644 index 000000000..826165773 --- /dev/null +++ b/tools/vanilla_map.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Map vanilla classes to the Horizon code that ports them and the harnesses that prove it +(docs/version-ports.md). + +Three sources, all in the repository: + +- Rust comments: a CamelCase name in a `//` comment that is exactly one vanilla class's simple name (or + `Outer.Inner`), or any class name followed by `.member`, `::member` or `#member`, maps that file (and the + member, when named) to the class. +- Java oracles (mods/horizon/src/test/java/dev/horizonlab/parity): the vanilla classes each one imports or + names in full; a harness uses the oracles its script names, and those its recorded parity dependencies list. +- Harness probes: the crate examples and binaries a harness script names, and the modules they `use`; the + harness proves those files, so it covers the classes they name. + + vanilla_map.py CLASS_INDEX [CLASS...] print what each class maps to (all classes: --all) +""" + +import argparse +from collections import defaultdict +import json +from pathlib import Path +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +ORACLES = "mods/horizon/src/test/java/dev/horizonlab/parity" +COMMENT_NAME = re.compile(r"\b([A-Z][A-Za-z0-9]*)(?:(?:\.|::|#)([A-Za-z][A-Za-z0-9]*))?") +JAVA_IMPORT = re.compile(r"\bimport\s+(static\s+)?((?:net\.minecraft|com\.mojang)\.[\w.]+?)(\.\*)?\s*;") +JAVA_QUALIFIED = re.compile(r"\b((?:net\.minecraft|com\.mojang)(?:\.[a-z][a-z0-9_]*)+(?:\.[A-Z]\w*)+)") +RUST_USE = re.compile(r"\b(horizon_[a-z_]+|crate)((?:::[a-z_][a-z0-9_]*)+)") + + +def humps(name): + return sum(c.isupper() for c in name) + + +class Names: + """Vanilla class names (internal form) by simple name, with a tie-break for the ambiguous ones.""" + + def __init__(self, classes): + self.classes = set(classes) + self.by_simple = defaultdict(set) + self.by_package = defaultdict(set) + for name in classes: + package, _, local = name.rpartition("/") + parts = local.split("$") + if any(p[:1].isdigit() for p in parts): + continue # Anonymous classes. + self.by_simple[parts[-1]].add(name) + if len(parts) > 1: + self.by_simple[".".join(parts[-2:])].add(name) + else: + self.by_package[package].add(name) + + def resolve(self, simple): + """The one class a simple name means: unique, else the unique top-level one, else the unique + top-level one under net/minecraft.""" + candidates = self.by_simple.get(simple, set()) + for narrowed in (candidates, {c for c in candidates if "$" not in c}, + {c for c in candidates if "$" not in c and c.startswith("net/minecraft/")}): + if len(narrowed) == 1: + return next(iter(narrowed)) + return None + + def java(self, dotted): + """net.minecraft.a.B.C -> net/minecraft/a/B$C, if that class exists.""" + parts = dotted.split(".") + for split in range(len(parts) - 1, 0, -1): + if not parts[split][:1].isupper() or parts[split - 1][:1].isupper(): + continue + name = "/".join(parts[:split]) + "/" + "$".join(parts[split:]) + if name in self.classes: + return name + return None + + +def comment_text(line): + at = line.find("//") + return line[at:] if at >= 0 else "" + + +def rust_references(files, names, read=lambda f: (ROOT / f).read_text(errors="replace")): + """{class: {file: [members named]}} from the comments of the given Rust files.""" + found = defaultdict(lambda: defaultdict(set)) + for path in files: + for line in read(path).splitlines(): + for match in COMMENT_NAME.finditer(comment_text(line)): + name, member = match[1], match[2] + if member and member[0].isupper() and names.resolve(f"{name}.{member}"): + name, member = f"{name}.{member}", None + if humps(name.split(".")[-1]) < 2 and not member: + continue + cls = names.resolve(name) + if cls: + found[cls][path].update([member] if member else []) + return {cls: {path: sorted(members) for path, members in files.items()} for cls, files in found.items()} + + +def java_references(text, names): + """The vanilla classes a Java source imports or names in full.""" + found = set() + words = set(re.findall(r"\b[A-Z]\w*", text)) + for static, dotted, wildcard in JAVA_IMPORT.findall(text): + if wildcard and not static: + package = dotted.replace(".", "/") + found.update(c for c in names.by_package.get(package, ()) if c.rsplit("/", 1)[1] in words) + else: + cls = names.java(dotted if not static or wildcard else dotted.rsplit(".", 1)[0]) + if cls: + found.add(cls) + for dotted in JAVA_QUALIFIED.findall(text): + cls = names.java(dotted) + if cls: + found.add(cls) + return found + + +def harness_name(script): + return Path(script).stem.removeprefix("test_").removesuffix("_parity") + + +def probe_files(script_text, examples): + """The crate examples and binaries a harness script names: {name: path}.""" + words = set(re.findall(r"[A-Za-z0-9_]+", script_text)) + return {name: path for name, path in examples.items() if name in words} + + +def module_files(path, text, sources): + """The workspace source files the `use` paths of a Rust file reach (one level).""" + crate_dir = path.split("/src/")[0] if "/src/" in path else path.split("/examples/")[0] + reached = set() + for crate, modules in RUST_USE.findall(text): + base = crate_dir if crate == "crate" else "crates/" + crate.replace("_", "-") + parts = [p for p in modules.split("::") if p] + for depth in range(len(parts), 0, -1): + stem = f"{base}/src/{'/'.join(parts[:depth])}" + hit = next((c for c in (f"{stem}.rs", f"{stem}/mod.rs") if c in sources), None) + if hit: + reached.add(hit) + break + return reached + + +def build(classes, root=ROOT, dependencies=None): + """{"classes": {class: {"rust": {file: [members]}, "harnesses": [...]}}, "harnesses": {harness: {...}}}. + + dependencies: {harness: [repository paths]} recorded by tools/parity.py, when available.""" + names = Names(classes) + tracked = subprocess.check_output(["git", "ls-files"], cwd=root, text=True).split() + rust = [f for f in tracked if f.endswith(".rs") and (f.startswith("crates/") or f.startswith("launcher/"))] + sources = set(rust) + read = lambda f: (root / f).read_text(errors="replace") # noqa: E731 + by_class = rust_references(rust, names, read) + by_file = defaultdict(set) + for cls, files in by_class.items(): + for path in files: + by_file[path].add(cls) + oracles = {Path(f).stem: java_references(read(f), names) + for f in tracked if f.startswith(ORACLES + "/") and f.endswith(".java")} + examples = {Path(f).stem: f for f in rust if "/examples/" in f or "/src/bin/" in f} + harnesses = {} + for script in sorted(f for f in tracked if re.fullmatch(r"tools/test_\w+_parity\.py", f)): + text = read(script) + name = harness_name(script) + used = {o for o in oracles if re.search(rf"\b{o}\b", text)} + used |= {Path(p).stem for p in (dependencies or {}).get(name, []) if Path(p).stem in oracles} + probes = probe_files(text, examples) + files = set(probes.values()) + for probe in probes.values(): + files |= module_files(probe, read(probe), sources) + covered = set().union(*(oracles[o] for o in used), *(by_file.get(f, ()) for f in files)) + harnesses[name] = {"oracles": sorted(used), "rust": sorted(files), "classes": sorted(covered)} + mapped = defaultdict(lambda: {"rust": {}, "harnesses": []}) + for cls, files in by_class.items(): + mapped[cls]["rust"] = files + for name, harness in harnesses.items(): + for cls in harness["classes"]: + mapped[cls]["harnesses"].append(name) + return {"classes": dict(sorted(mapped.items())), "harnesses": harnesses} + + +def lookup(mapping, cls): + """What a class maps to, falling back to its outermost class for nested ones.""" + entry = mapping["classes"].get(cls) + if entry is None and "$" in cls: + entry = mapping["classes"].get(cls.split("$")[0]) + return entry or {"rust": {}, "harnesses": []} + + +def recorded_dependencies(shared): + """tools/parity.py's dependency records ({harness: [paths]}), from parity-shared/dependencies.""" + directory = Path(shared) / "dependencies" + found = {} + for path in sorted(directory.glob("*.json")) if directory.is_dir() else []: + try: + found[path.stem.split("#")[0]] = json.loads(path.read_text())["dependencies"] + except (ValueError, KeyError): + continue + return found + + +def default_shared(): + return (ROOT / ".local" / "minecraft").resolve().parent / "parity-shared" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("index", help="a class index (tools/class_index.py, versions//classes.json)") + parser.add_argument("classes", nargs="*", help="internal or dotted names") + parser.add_argument("--all", action="store_true", help="print the whole map") + args = parser.parse_args() + classes = json.loads(Path(args.index).read_text()) + mapping = build(classes, dependencies=recorded_dependencies(default_shared())) + if args.all: + print(json.dumps(mapping, indent=1)) + return 0 + for cls in args.classes: + cls = cls.replace(".", "/") if "/" not in cls else cls + print(json.dumps({cls: lookup(mapping, cls)}, indent=2)) + covered = sum(1 for entry in mapping["classes"].values() if entry["harnesses"]) + print(f"{len(mapping['classes'])} vanilla classes mapped, {covered} covered by a harness", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/version_port.py b/tools/version_port.py index 4330b7503..c718be0c4 100644 --- a/tools/version_port.py +++ b/tools/version_port.py @@ -3,9 +3,11 @@ (docs/version-ports.md). version_port.py detect [--snapshots] what Mojang released that Horizon doesn't pin - version_port.py prepare VERSION download, verify, run the data generator, read its facts + version_port.py prepare VERSION download, verify, run the data generator, read its facts, + index its classes (tools/class_index.py) version_port.py diff OLD NEW facts, packets, registries, blocks, items, commands, - data packs, assets and lang: diff-OLD-NEW/diff.json + data packs, assets, lang and classes, with the Horizon + code each changed class concerns: diff-OLD-NEW/diff.json version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep The downloads are for development only: they go to the version store (~/code/horizon-parity/versions, @@ -29,9 +31,11 @@ import urllib.request import zipfile try: - from . import lab + from . import class_index, lab, vanilla_map except ImportError: + import class_index import lab + import vanilla_map MANIFEST_URL = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json" STORE = Path(os.environ.get("HORIZON_VERSIONS", Path.home() / "code" / "horizon-parity" / "versions")) @@ -216,6 +220,9 @@ def prepare(args): if facts["id"] != version: raise ValueError(f"The client jar says it is {facts['id']}, not {version}") lab.write_json(directory / "facts.json", facts) + classes = class_index.index_jars([found["client"], found["server_game"]], jobs=args.jobs) + (directory / "classes.json").write_text(json.dumps(classes, separators=(",", ":"))) + print(f"Indexed {len(classes)} classes, {sum(len(c['methods']) for c in classes.values())} methods") lab.write_json(directory / "prepared.json", { "version": version, "pinned": version == pinned() and not args.download, "paths": {key: str(path) for key, path in found.items()}, @@ -431,8 +438,26 @@ def read_reports(reports): } -def diff_versions(old_paths, new_paths, old_facts, new_facts): +def diff_classes(old, new, mapping): + """The class diff, each changed or removed class with the Horizon files and harnesses that concern it + (looked up under its old name: Horizon's code names the pinned version's classes).""" + result = class_index.diff(old, new) + horizon = {} + for name, change in result["changed"].items(): + entry = vanilla_map.lookup(mapping, change.get("moved_from", name)) + if entry["rust"] or entry["harnesses"]: + horizon[name] = entry + for name in result["removed"]: + entry = vanilla_map.lookup(mapping, name) + if entry["rust"] or entry["harnesses"]: + horizon[name] = entry + result["horizon"] = horizon + return result + + +def diff_versions(old_paths, new_paths, old_facts, new_facts, old_classes=None, new_classes=None, mapping=None): a, b = read_reports(old_paths["reports"]), read_reports(new_paths["reports"]) + classes = diff_classes(old_classes, new_classes, mapping) if old_classes is not None else None return { "facts": diff_facts(old_facts, new_facts), "packets": diff_packets(a["packets"], b["packets"]), @@ -443,6 +468,7 @@ def diff_versions(old_paths, new_paths, old_facts, new_facts): "data": diff_contents(old_paths["client"], new_paths["client"], "data/minecraft/"), "assets": diff_contents(old_paths["client"], new_paths["client"], "assets/minecraft/"), "lang": diff_lang(old_paths["client"], new_paths["client"]), + "classes": classes, } @@ -480,6 +506,12 @@ def summary(old, new, result): lines.append(f" {key} files: +{counts['added']} -{counts['removed']} ~{counts['changed']}") lang = result["lang"] lines.append(f" en_us keys: +{len(lang['added'])} -{len(lang['removed'])} ~{len(lang['changed'])}") + classes = result.get("classes") + if classes: + methods = sum(class_index.changed_methods(c) for c in classes["changed"].values()) + lines.append(f" classes: +{len(classes['added'])} -{len(classes['removed'])}, {len(classes['moved'])} moved, " + f"{len(classes['changed'])} changed ({methods} methods); {len(classes['horizon'])} of the " + f"changed or removed ones are ported or proved by Horizon") return "\n".join(lines) @@ -487,10 +519,17 @@ def diff_dir(old, new): return STORE / f"diff-{safe_version(old)}-{safe_version(new)}" +def load(version, name): + return json.loads((version_dir(version) / name).read_text()) + + def diff(args): - facts = {v: json.loads((version_dir(v) / "facts.json").read_text()) for v in (args.old, args.new)} - result = diff_versions(paths(args.old), paths(args.new), facts[args.old], facts[args.new]) directory = diff_dir(args.old, args.new) + old_classes, new_classes = load(args.old, "classes.json"), load(args.new, "classes.json") + mapping = vanilla_map.build(old_classes, dependencies=vanilla_map.recorded_dependencies(vanilla_map.default_shared())) + lab.write_json(directory / "vanilla_map.json", mapping) + result = diff_versions(paths(args.old), paths(args.new), load(args.old, "facts.json"), load(args.new, "facts.json"), + old_classes, new_classes, mapping) lab.write_json(directory / "diff.json", result) print(summary(args.old, args.new, result)) print(f"\n{directory / 'diff.json'}") @@ -535,6 +574,8 @@ def main(): command = commands.add_parser("prepare", help="download, verify and run the data generator") command.add_argument("version") command.add_argument("--download", action="store_true", help="download even the pinned version") + command.add_argument("-j", "--jobs", type=int, default=int(os.environ.get("FLEET_LOAD_JOBS", 0)) or None, + help="class indexing processes (default: $FLEET_LOAD_JOBS, else the CPU count)") command = commands.add_parser("diff", help="diff two prepared versions") command.add_argument("old") command.add_argument("new")