chexit/horizon-airdef
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

version_port.py harnesses: horizon-prepare and every harness in the candidate, newly failing ones in the report

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-15 18:16:57 +02:00
parent b3bc3c0768
commit ec0725aeb0
5 changed files with 335 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_port_harness.py
+62
View File
@@ -0,0 +1,62 @@
from pathlib import Path
import tempfile
import unittest
from tools import port_harness as harness
JAVAC = """/x/mods/horizon/src/test/java/dev/horizonlab/parity/ParityData.java:45: error: cannot find symbol
symbol: variable WORLD
location: class RegistryLayer
1 error
"""
def state(**statuses):
return {"harnesses": {name.replace("_1", "#1"): {"status": status} for name, status in statuses.items()}}
class ClassifyTests(unittest.TestCase):
def setUp(self):
self.directory = tempfile.TemporaryDirectory()
self.logs = Path(self.directory.name)
(self.logs / "food.log").write_text(JAVAC)
(self.logs / "food_extra.log").write_text("not food's log")
(self.logs / "light.log").write_text("first difference at (3, 64, 2)")
def tearDown(self):
self.directory.cleanup()
def test_pass_oracle_fail_and_new(self):
now = state(food="fail", light="fail", mob_ai="fail", move="pass", move_1="fail")
baseline = state(food="pass", light="pass", mob_ai="fail", move="pass")
found = harness.classify(now, self.logs, baseline)
self.assertEqual(found["food"], {"status": "oracle", "baseline": "pass", "new": True, "compile_errors": [
"ParityData.java:45: cannot find symbol: variable WORLD (class RegistryLayer)"]})
self.assertEqual(found["light"], {"status": "fail", "baseline": "pass", "new": True})
self.assertEqual(found["mob_ai"], {"status": "fail", "baseline": "fail", "new": False})
self.assertEqual(found["move"]["status"], "fail") # one shard failed
self.assertEqual(harness.summary(found), {"pass": 0, "oracle": 1, "fail": 3, "new": 3, "known": 1})
def test_without_a_baseline_every_failure_is_new(self):
found = harness.classify(state(light="fail", food="pass"), self.logs)
self.assertTrue(found["light"]["new"])
self.assertFalse(found["food"]["new"])
def test_named_harnesses_only(self):
self.assertEqual(list(harness.classify(state(light="fail", food="pass"), self.logs, names=["food"])), ["food"])
def test_logs_of_one_harness(self):
(self.logs / "food#2.log").write_text("")
self.assertEqual([p.name for p in harness.harness_logs(self.logs, "food")], ["food#2.log", "food.log"])
class AssetTests(unittest.TestCase):
def test_sounds_and_panorama_only(self):
self.assertTrue(harness.wanted_asset("minecraft/sounds.json"))
self.assertTrue(harness.wanted_asset("minecraft/sounds/mob/cow/say1.ogg"))
self.assertTrue(harness.wanted_asset("minecraft/textures/gui/title/background/panorama_overlay.png"))
self.assertFalse(harness.wanted_asset("minecraft/lang/fr_fr.json"))
if __name__ == "__main__":
unittest.main()
tests/test_port_report.py
+22 -2
View File
@@ -62,11 +62,31 @@ def diff():
class PortReportTests(unittest.TestCase):
def build(self, results=None, regen=None):
def build(self, results=None, regen=None, harnesses=None):
with mock.patch.object(report, "version_sites", return_value=["crates/p/src/lib.rs:20"]), \
mock.patch.object(report, "files_naming", return_value=["crates/p/src/play.rs"]), \
mock.patch.object(report, "files_containing", return_value=["crates/w/src/blocks.rs"]):
return report.build(diff(), mapping(), "26.2", "26.3", {"id": "26.3"}, results, regen=regen)
return report.build(diff(), mapping(), "26.2", "26.3", {"id": "26.3"}, results, regen=regen,
harnesses=harnesses)
def test_harness_results_count_only_new_failures(self):
ran = {"prepare": {"status": "failed", "error": "javac exited 1",
"compile_errors": ["VanillaBlockExporter.java:9: cannot find symbol: class Foo"]},
"summary": {"pass": 1, "oracle": 1, "fail": 2, "new": 2, "known": 1},
"harnesses": {"food": {"status": "oracle", "baseline": "pass", "new": True, "compile_errors": ["X"]},
"move": {"status": "fail", "baseline": "fail", "new": False},
"light": {"status": "fail", "baseline": "pass", "new": True},
"sky": {"status": "pass", "baseline": "pass", "new": False}}}
built = self.build(harnesses=ran)
[prepare] = [i for i in built["items"] if i["kind"] == "prepare-failed"]
self.assertEqual(prepare["rank"], 1)
failed = {h: i for i in built["items"] for h in i.get("failed", [])}
self.assertEqual(failed["food"]["oracle"], ["food"])
self.assertNotIn("move", failed) # It fails on the old version too.
self.assertEqual(failed["light"]["title"], "Harness light: fails on 26.3")
text = report.markdown(built)
self.assertIn("1 pass, 1 whose oracle no longer compiles, 2 fail; 2 newly failing, 1 failing on 26.2 too", text)
self.assertIn("- javac: `VanillaBlockExporter.java:9: cannot find symbol: class Foo`", text)
def test_regen_results_replace_the_stale_tables(self):
regen = {"new": "26.3", "candidate": "/ports/26.3", "commit": "0123456789abcdef", "generators": {
tools/port_harness.py
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""Run every parity harness against a new version, in its candidate worktree (docs/version-ports.md, "Run every
harness against the new version").
Used through `version_port.py harnesses NEW` after `regen NEW`:
1. horizon-prepare in the candidate (.local/horizon-client/NEW and its exporters' outputs), then its `sounds` and
`panorama` steps from the version store's launcher-shaped copy, whose asset index and the objects those steps
copy are downloaded once per version (sha1 checked);
2. tools/parity.py there: every harness, or the ones named;
3. each harness classified: pass, oracle (its Java oracle no longer compiles against NEW: javac names what
changed), or fail; and, against a baseline run on the pinned version (a tools/parity.py state.json), whether it
newly fails, so the known failures of main don't count.
The result goes to diff-OLD-NEW/harnesses.json, which `version_port.py report` reads.
"""
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import shutil
import subprocess
try:
from . import lab, port_regen
except ImportError:
import lab
import port_regen
RESOURCES_URL = "https://resources.download.minecraft.net"
# What horizon-prepare's sounds and panorama steps copy from the launcher's asset store
# (crates/horizon-prepare/src/sounds.rs is_sound, is_panorama).
PANORAMA = [f"minecraft/textures/gui/title/background/panorama_{i}.png" for i in range(6)] + [
"minecraft/textures/gui/title/background/panorama_overlay.png"]
def wanted_asset(name):
return name == "minecraft/sounds.json" or name.startswith("minecraft/sounds/") or name in PANORAMA
def launcher_assets(tree, manifest, jobs=8):
"""The asset index and the objects horizon-prepare's sounds and panorama steps read, in the launcher-shaped
tree (assets/indexes, assets/objects/xx/HASH)."""
info = manifest["assetIndex"]
index = lab.download(info["url"], tree / "assets" / "indexes" / f"{info['id']}.json", info["sha1"])
objects = json.loads(Path(index).read_text())["objects"]
hashes = sorted({entry["hash"] for name, entry in objects.items() if wanted_asset(name)})
def fetch(checksum):
lab.download(f"{RESOURCES_URL}/{checksum[:2]}/{checksum}", tree / "assets" / "objects" / checksum[:2] / checksum,
checksum)
with ThreadPoolExecutor(jobs) as pool:
list(pool.map(fetch, hashes))
return len(hashes)
def prepare_resources(root, version, store):
"""horizon-prepare against the candidate's version: {status: ok | failed, error, compile_errors}."""
log = root / ".local" / "port-regen" / "horizon-prepare.log"
log.parent.mkdir(parents=True, exist_ok=True)
env = dict(os.environ, CARGO_TARGET_DIR=str(root / ".local" / "port-target"))
jobs = os.environ.get("FLEET_LOAD_JOBS")
try:
port_regen.run(["cargo", "build", "--release", "-p", "horizon-prepare", *(["-j", jobs] if jobs else [])],
root, env=env)
binary = root / ".local/port-target/release/horizon-prepare"
port_regen.run([binary, "--root", root], root, log=log)
found = json.loads((store / "prepared.json").read_text())["paths"]
manifest = json.loads(Path(found["version_json"]).read_text())
tree = store / "launcher"
if tree.exists(): # A downloaded version; the pinned one's sounds come from the player's launcher.
launcher_assets(tree, manifest)
for step in ("sounds", "panorama"):
port_regen.run([binary, step, "--root", root, "--minecraft-dir", tree], root,
log=log.with_name(f"horizon-prepare-{step}.log"))
return {"status": "ok"}
except (RuntimeError, OSError, ValueError, KeyError) as error:
output = getattr(error, "output", str(error))
return {"status": "failed", "error": str(error)[-2500:], "compile_errors": port_regen.compile_errors(output)}
def run_harnesses(root, names=None, timeout=None):
"""tools/parity.py in the candidate, every harness from scratch (or the ones named); its state.json."""
command = [port_regen.python(), "tools/parity.py", *(names or ["--all"])]
if timeout:
command += ["--timeout", str(timeout)]
env = dict(os.environ, PARITY_HOSTS="")
log = root / ".local" / "port-regen" / "parity.log"
log.parent.mkdir(parents=True, exist_ok=True)
with log.open("w") as stream:
subprocess.run(command, cwd=root, stdout=stream, stderr=subprocess.STDOUT, env=env)
return root / ".local" / "parity-runner" / "state.json"
def harness_logs(logs, harness):
"""A harness's logs: its own, and its shards' (NAME#N.log)."""
return sorted([*logs.glob(f"{harness}.log"), *logs.glob(f"{harness}#*.log")])
def statuses(state):
"""{harness: "pass" | "fail"} from a tools/parity.py state; a harness fails if one of its shards does."""
found = {}
for name, record in state.get("harnesses", {}).items():
harness = name.split("#")[0]
found[harness] = "fail" if record.get("status") != "pass" or found.get(harness) == "fail" else "pass"
return found
def classify(state, logs, baseline=None, names=None):
"""{harness: {status: pass | oracle | fail, baseline, new, compile_errors}} for the harnesses that ran.
oracle: the run's log has javac errors (the oracle no longer compiles against the new version). new: it does
not pass now and passed in the baseline (or there is no baseline)."""
now = statuses(state)
before = statuses(baseline) if baseline else {}
result = {}
for harness, status in sorted(now.items()):
if names and harness not in names:
continue
entry = {"status": status, "baseline": before.get(harness)}
if status != "pass":
text = "".join(path.read_text(errors="replace")[-(1 << 20):] for path in harness_logs(logs, harness))
errors = port_regen.compile_errors(text)
if errors:
entry["status"] = "oracle"
entry["compile_errors"] = errors
entry["new"] = entry["status"] != "pass" and entry["baseline"] != "fail"
result[harness] = entry
return result
def summary(results):
counts = {"pass": 0, "oracle": 0, "fail": 0, "new": 0, "known": 0}
for entry in results.values():
counts[entry["status"]] += 1
if entry["status"] != "pass":
counts["new" if entry["new"] else "known"] += 1
return counts
def harnesses(version, store, root, baseline=None, names=None, timeout=None, previous=None):
"""The whole stage: {candidate, commit, prepare, harnesses, summary}. previous: an earlier result to classify
again (against another baseline) without preparing or running anything."""
root = Path(root)
if not (root / ".git").exists():
raise SystemExit(f"No candidate at {root}; run version_port.py regen {version} first")
pinned = json.loads((root / "research-lock.json").read_text())["minecraft"]["version"]
if pinned != version:
raise SystemExit(f"{root} pins {pinned}, not {version}")
if previous:
prepared, names = previous["prepare"], names or list(previous["harnesses"])
state = json.loads((root / ".local" / "parity-runner" / "state.json").read_text())
else:
print(f"horizon-prepare in {root}", flush=True)
prepared = prepare_resources(root, version, store)
print(f" {prepared['status']}", flush=True)
print("tools/parity.py " + (" ".join(names) if names else "--all"), flush=True)
state = json.loads(run_harnesses(root, names, timeout).read_text())
base = json.loads(Path(baseline).read_text()) if baseline else None
results = classify(state, root / ".local" / "parity-runner" / "logs", base, names)
return {"candidate": str(root), "commit": port_regen.git("rev-parse", "HEAD", cwd=root).strip(),
"baseline": str(baseline) if baseline else None, "prepare": prepared, "harnesses": results,
"summary": summary(results)}
def keep_logs(result, destination):
"""Copy the logs of the harnesses that don't pass beside harnesses.json (the candidate may be removed)."""
logs = Path(result["candidate"]) / ".local" / "parity-runner" / "logs"
destination.mkdir(parents=True, exist_ok=True)
for harness, entry in result["harnesses"].items():
if entry["status"] != "pass":
for path in harness_logs(logs, harness):
shutil.copyfile(path, destination / path.name)
tools/port_report.py
+37 -6
View File
@@ -5,8 +5,8 @@
Ranks:
1 blocks the switch: version facts, packets, the save format (DataFixers), data component types, report
formats, generated tables to regenerate
2 a harness failed against the new version (with --results), else changed vanilla code a harness proves:
rerun it
2 a harness newly fails against the new version, or its oracle no longer compiles (version_port.py
harnesses, or --results), else changed vanilla code a harness proves: rerun it
3 changed vanilla code Horizon ports and no harness proves
4 new content: registry entries, and new classes in the vanilla packages Horizon ports
5 changed and new vanilla code Horizon doesn't port (two summary items)
@@ -203,11 +203,12 @@ def code_items(diff, mapping, results=None):
harnesses = item_harnesses(key, group["classes"], mapping)
failed = sorted(set(harnesses) & failing)
rank, kind = (2, "harness-failed") if failed else (2, "rerun") if harnesses else (3, "unproved")
oracle = sorted(h for h in failed if results[h] == "oracle")
top = sorted(group["classes"], key=lambda c: (-group["classes"][c]["methods"], c))
title = (f"{key}: {len(group['classes'])} vanilla classes changed "
f"({group['methods']} methods, {group['precise']} named in Horizon)")
items.append(item(rank, kind, title, module=key, files=sorted(group["files"]), harnesses=harnesses,
failed=failed, classes={c: group["classes"][c] for c in top},
failed=failed, oracle=oracle, classes={c: group["classes"][c] for c in top},
score=group["methods"] + 10 * group["precise"] + (1000 if failed else 0)))
if unported:
items.append(item(5, "unported", f"{sum(unported.values())} changed or removed vanilla classes Horizon "
@@ -245,12 +246,36 @@ def content_items(diff, mapping):
return items
def build(diff, mapping, old, new, pinned_facts, results=None, root=ROOT, regen=None):
def harness_items(ran, items, new):
"""Rank 1 when horizon-prepare fails on NEW (its exporters no longer compile, or a format broke); rank 2 for
each newly failing harness that no code item names, so none is lost."""
found = []
prepared = ran["prepare"]
if prepared["status"] != "ok":
what = "its exporters no longer compile" if prepared.get("compile_errors") else "it fails"
found.append(item(1, "prepare-failed", f"horizon-prepare on {new}: {what}", files=["crates/horizon-prepare"],
error=prepared["error"], compile_errors=prepared.get("compile_errors", [])))
named = {h for i in items for h in i.get("failed", [])}
for harness, entry in ran["harnesses"].items():
if entry["new"] and harness not in named:
what = "its oracle no longer compiles" if entry["status"] == "oracle" else "fails"
found.append(item(2, "harness-failed", f"Harness {harness}: {what} on {new}", harnesses=[harness],
failed=[harness], oracle=[harness] if entry["status"] == "oracle" else [],
compile_errors=entry.get("compile_errors", []), score=1000))
return found
def build(diff, mapping, old, new, pinned_facts, results=None, root=ROOT, regen=None, harnesses=None):
"""The report: summary, ranked items and what came out clean. pinned_facts: the facts of the version the
code states today (to find where it states them); results: {harness: "pass" | "fail"} against NEW; regen:
version_port.py regen's result (without it, the tables a diff makes stale are listed to regenerate)."""
version_port.py regen's result (without it, the tables a diff makes stale are listed to regenerate);
harnesses: version_port.py harnesses' result (only newly failing harnesses count)."""
if harnesses:
results = {h: e["status"] if e["new"] else "pass" for h, e in harnesses["harnesses"].items()}
items = blocking_items(diff, mapping, pinned_facts, root, regen)
items += code_items(diff, mapping, results) + content_items(diff, mapping)
if harnesses:
items += harness_items(harnesses, items, new)
items.sort(key=lambda i: (i["rank"], -i.get("score", 0), i["title"]))
for number, entry in enumerate(items, 1):
entry["id"] = number
@@ -268,6 +293,7 @@ def build(diff, mapping, old, new, pinned_facts, results=None, root=ROOT, regen=
"generators": ({name: e["status"] for name, e in regen["generators"].items()} if regen
else {i["files"][0]: "stale" for i in items if i["kind"] == "regenerate"}),
},
"harnesses": harnesses["summary"] if harnesses else None,
"regen": {"candidate": regen["candidate"], "commit": regen["commit"],
"stat": {p: s for e in regen["generators"].values() for p, s in e.get("stat", {}).items()}}
if regen else None,
@@ -298,6 +324,10 @@ def markdown(report, limit=12):
for s in ("regenerated", "unchanged", "failed")))
for path, (added, removed) in report["regen"]["stat"].items():
lines.append(f" - regenerated `{path}` (+{added} -{removed})")
if report.get("harnesses"):
h = report["harnesses"]
lines.append(f"- harnesses on {report['new']}: {h['pass']} pass, {h['oracle']} whose oracle no longer compiles, "
f"{h['fail']} fail; {h['new']} newly failing, {h['known']} failing on {report['old']} too")
names = {1: "Blocks the switch", 2: "Proved by a harness: rerun it", 3: "Ported, no harness",
4: "New content", 5: "Not ported"}
for rank in sorted({i["rank"] for i in report["items"]}):
@@ -311,7 +341,8 @@ def markdown(report, limit=12):
elif entry.get("error"):
lines.append("- error: `" + entry["error"].strip().splitlines()[-1][:300] + "`")
if entry.get("failed"):
lines.append(f"- failing harnesses: {', '.join(entry['failed'])}")
lines.append(f"- failing harnesses: {', '.join(entry['failed'])}"
+ (f" (oracle no longer compiles: {', '.join(entry['oracle'])})" if entry.get("oracle") else ""))
if entry.get("harnesses"):
lines.append(f"- harnesses: {', '.join(entry['harnesses'][:limit])}"
+ (f" (+{len(entry['harnesses']) - limit})" if len(entry["harnesses"]) > limit else ""))
tools/version_port.py
+40 -6
View File
@@ -10,6 +10,9 @@
code each changed class concerns: diff-OLD-NEW/diff.json
version_port.py regen NEW [--base REF] a port/NEW worktree pinned to NEW with every generated table
regenerated (tools/port_regen.py): diff-OLD-NEW/regen.json
version_port.py harnesses NEW [--baseline S] horizon-prepare and every harness in the candidate, each pass,
oracle (no longer compiles) or fail, and newly failing against
a run on the pinned version (tools/port_harness.py)
version_port.py report OLD NEW [--results S] the ranked port report: diff-OLD-NEW/report.md and .json
version_port.py port NEW [--old OLD] prepare both, diff, regen, report
version_port.py gc [--keep VERSION...] keep the pinned, the newest two others and --keep
@@ -36,10 +39,11 @@ import urllib.request
import zipfile
try:
from . import class_index, lab, port_regen, port_report, vanilla_map
from . import class_index, lab, port_harness, port_regen, port_report, vanilla_map
except ImportError:
import class_index
import lab
import port_harness
import port_regen
import port_report
import vanilla_map
@@ -564,6 +568,25 @@ def regen(args):
return 1 if any(e["status"] == "failed" for e in result["generators"].values()) else 0
# --- harnesses ---------------------------------------------------------------------------------------------
def harnesses(args):
version, old = safe_version(args.new), args.old or pinned()
directory = diff_dir(old, version)
previous = json.loads((directory / "harnesses.json").read_text()) if args.no_run else None
result = port_harness.harnesses(version, version_dir(version),
Path(args.worktree) if args.worktree else PORTS / version,
args.baseline, args.names or None, args.timeout, previous)
lab.write_json(directory / "harnesses.json", result)
port_harness.keep_logs(result, directory / "harness-logs")
counts = result["summary"]
print(f"horizon-prepare: {result['prepare']['status']}")
print(f"harnesses: {counts['pass']} pass, {counts['oracle']} oracle no longer compiles, {counts['fail']} fail; "
f"{counts['new']} newly failing, {counts['known']} failing on the baseline too")
print(f"{directory / 'harnesses.json'}")
return 0
# --- report ------------------------------------------------------------------------------------------------
def harness_results(state_path):
@@ -582,10 +605,11 @@ def report(args):
result = json.loads((directory / "diff.json").read_text())
mapping = json.loads((directory / "vanilla_map.json").read_text())
results = harness_results(args.results) if args.results else None
regenerated = directory / "regen.json"
regenerated, ran = directory / "regen.json", directory / "harnesses.json"
regenerated = json.loads(regenerated.read_text()) if regenerated.exists() else None
ran = json.loads(ran.read_text()) if ran.exists() and not args.results else None
built = port_report.build(result, mapping, args.old, args.new, load(pinned(), "facts.json"), results,
regen=regenerated)
regen=regenerated, harnesses=ran)
lab.write_json(directory / "report.json", built)
(directory / "report.md").write_text(port_report.markdown(built))
print("\n".join(port_report.markdown(built).splitlines()[:args.lines]))
@@ -652,10 +676,20 @@ def main():
command.add_argument("--base", default="origin/main", help="what port/NEW starts from (default origin/main)")
command.add_argument("--worktree", help="where (default: $HORIZON_PORTS/NEW)")
command.add_argument("--only", nargs="*", help="these generators only (names in tools/port_regen.py)")
command = commands.add_parser("harnesses", help="horizon-prepare and every harness in the candidate")
command.add_argument("new")
command.add_argument("names", nargs="*", help="these harnesses only (default: all)")
command.add_argument("--old", help="the version the port starts from (default: the pinned one)")
command.add_argument("--worktree", help="the candidate (default: $HORIZON_PORTS/NEW)")
command.add_argument("--baseline", help="tools/parity.py's state.json from a run on the pinned version: what "
"fails there too is not newly failing")
command.add_argument("--timeout", type=int, help="per harness, seconds (tools/parity.py --timeout)")
command.add_argument("--no-run", action="store_true", help="classify the last run again (another --baseline)")
command = commands.add_parser("report", help="the ranked port report of a diff")
command.add_argument("old")
command.add_argument("new")
command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW")
command.add_argument("--results", help="tools/parity.py's state.json from a run against NEW (default: "
"harnesses.json, when version_port.py harnesses ran)")
command.add_argument("--lines", type=int, default=60, help="how much of report.md to print")
command = commands.add_parser("port", help="prepare, diff and report in one go")
command.add_argument("new")
@@ -670,8 +704,8 @@ def main():
command.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
STORE.mkdir(parents=True, exist_ok=True)
return {"detect": detect, "prepare": prepare, "diff": diff, "regen": regen, "report": report, "port": port,
"gc": gc}[args.command](args)
return {"detect": detect, "prepare": prepare, "diff": diff, "regen": regen, "harnesses": harnesses,
"report": report, "port": port, "gc": gc}[args.command](args)
if __name__ == "__main__":