diff --git a/Cargo.toml b/Cargo.toml index 610e7817e..386c67304 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,11 @@ type_complexity = "allow" needless_range_loop = "allow" large_enum_variant = "allow" +# Line tables keep file:line in backtraces; full debug info made every +# example and test binary hundreds of megabytes and slow to link. +[profile.dev] +debug = "line-tables-only" + [profile.release] lto = "thin" codegen-units = 1 diff --git a/crates/horizon-client/tests/lan_bot.rs b/crates/horizon-client/tests/lan_bot.rs index d82bf4858..49cec6509 100644 --- a/crates/horizon-client/tests/lan_bot.rs +++ b/crates/horizon-client/tests/lan_bot.rs @@ -333,10 +333,9 @@ fn status_and_join() { })); teleported = true; } - C::SetTime(t) => { - assert!(!t.clocks.is_empty()); - time = true; - } + // The join's full clock sync; MinecraftServer's 20-tick + // forceGameTimeSynchronization sends no clocks and may come first. + C::SetTime(t) if !t.clocks.is_empty() => time = true, C::PlayerInfoUpdate(u) => { info |= u.entries.iter().any(|e| e.name.as_deref() == Some("Host")); } diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md new file mode 100644 index 000000000..8db55783f --- /dev/null +++ b/docs/parity-workflow.md @@ -0,0 +1,45 @@ +# Parity workflow + +How parity work is run, checked and merged. The harnesses themselves are described in [Rust rewrite status](rust-parity.md). + +## Running the vanilla harnesses + +`tools/parity.py` runs the `tools/test_*_parity.py` harnesses in parallel and reruns only what a change can affect: + +```sh +python3 tools/parity.py # harnesses whose inputs changed, or that never passed +python3 tools/parity.py food light # these harnesses, whatever their state +python3 tools/parity.py --failed # the ones that failed last time +python3 tools/parity.py --all # everything (about 20 minutes on ludodesktop) +python3 tools/parity.py --list # what would run +python3 tools/parity.py --status # last result of each harness +``` + +Each harness runs unmodified under a tracer that records the files it depended on: its Python modules, its Java oracle and the oracle sources javac pulled in, other repository files it read, and the Rust sources of the probes it ran (from Cargo's `.d` dep-info files). A pass stands until one of those files, the pinned vanilla artifacts, `Cargo.lock` or the Rust/Java toolchains change. State and logs are in `.local/parity-runner/`; each harness's own outputs stay in its `.local//` directory. + +Probes build into `.local/parity-target/` with release optimisation but no LTO and 16 codegen units. On ludodesktop that rebuilds every `horizon-world` probe in 3 seconds after a one-line change, against 122 seconds with the release profile. Rust gives these settings identical results (no fast-math; overflow checks as in release), and the full harness suite passes under them. Harnesses must locate binaries through `cargo metadata` or `lab.cargo_target()`, never a hard-coded `target/`. + +Parallel harnesses are admitted by memory: each harness's recorded peak is reserved, within three quarters of the memory available at the start. `-j` caps the count (default: a third of the CPU threads). + +## The gate + +`tools/gate.py` is what a branch must pass before it is integrated: the public-tree check, rustfmt, clippy with warnings denied, the Rust and Python tests, then the harnesses whose inputs changed. It stops at the first failure and writes `.local/gate.json`. `--known-failures a,b` lets harnesses that already fail on the integration branch keep failing. `--quick` checks formatting, lints and Python tests only; `--no-parity` skips the harnesses. + +## Parallel work: worktrees and the integration branch + +Parity work happens in one worktree and branch per area, next to an integration worktree that has the integration branch (`parity-base`, pushed as `t3/achieve-full-parity`) checked out. All worktrees share the vanilla data through `.local/` links to a `shared-local/` directory beside them. + +```sh +python3 tools/supervise.py status # branches, progress, unpushed work, last gate +python3 tools/supervise.py new dispensers # new worktree and branch from the integration branch +python3 tools/supervise.py save # commit and push unfinished work before agents stop +python3 tools/supervise.py integrate parity/dispensers # merge, gate, push; undo the merge if the gate fails +``` + +An agent working an area merges the integration branch first, uses `tools/parity.py` on its harnesses while it works, runs `tools/gate.py` before its last commit and pushes its branch. Integration happens only through `supervise.py integrate`, which runs the gate with the harnesses already failing on the integration branch as known failures: a branch may leave those failing, or fix them, but must not make any other harness fail. + +On ludodesktop, source `~/code/horizon-parity/env.sh` first: it sets the ALSA build path and the Cargo job cap. Debug builds carry line tables only (`[profile.dev]` in `Cargo.toml`), which keeps backtraces readable while keeping each worktree's build small; a fresh worktree builds `horizon-client` and its tests in under a minute. + +## Reading vanilla + +Besides the Vineflower sources in `.local/minecraft//sources/`, `tools/rea_vanilla.py` queries the shipped bytecode through REA (callers through lambdas, overloads, a second decompiler); see [decompilation status](decompilation-status.md#querying-the-shipped-bytecode-with-rea). diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py new file mode 100644 index 000000000..01ff01be1 --- /dev/null +++ b/tests/test_parity_runner.py @@ -0,0 +1,49 @@ +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from tools import parity + + +class ParityRunnerTests(unittest.TestCase): + def test_dep_info_keeps_escaped_spaces(self): + with tempfile.TemporaryDirectory() as temp: + binary = Path(temp) / "probe" + Path(temp, "probe.d").write_text(f"{binary}: /a/x.rs /b/with\\ space.rs\n\n/a/x.rs:\n") + self.assertEqual(parity.dep_info(binary), ["/a/x.rs", "/b/with space.rs"]) + + def test_artifacts_follow_profile_and_stop_at_program_arguments(self): + self.assertEqual(parity.artifacts_of(["cargo", "build", "--release", "--example", "a", "--bin", "b"]), + [["example", "a", "release"], ["bin", "b", "release"]]) + self.assertEqual(parity.artifacts_of(["cargo", "run", "--profile", "fast", "--example", "c", "--", "--example", "x"]), + [["example", "c", "fast"]]) + self.assertEqual(parity.artifacts_of(["cargo", "build", "--example", "d"]), [["example", "d", "dev"]]) + + def test_java_sources_include_classes_pulled_from_the_sourcepath(self): + with tempfile.TemporaryDirectory() as temp: + sources, classes = Path(temp, "src"), Path(temp, "classes") + (sources / "p").mkdir(parents=True) + (classes / "p").mkdir(parents=True) + for name in ("Oracle", "Helper"): + (sources / "p" / f"{name}.java").write_text("") + for name in ("Oracle", "Helper$Inner"): + (classes / "p" / f"{name}.class").write_bytes(b"") + found = parity.java_sources(["javac", "-sourcepath", str(sources), "-d", str(classes), str(sources / "p/Oracle.java")]) + self.assertEqual(sorted(Path(path).name for path in found), ["Helper.java", "Oracle.java", "Oracle.java"]) + + def test_a_pass_goes_stale_when_a_dependency_changes(self): + with tempfile.TemporaryDirectory() as temp, mock.patch.object(parity, "ROOT", Path(temp)): + source = Path(temp, "a.rs") + source.write_text("one") + state = {"harnesses": {"x": {"status": "pass", "dependencies": ["a.rs"]}}, "hashes": {}} + hasher = parity.Hasher(state["hashes"]) + state["harnesses"]["x"]["key"] = parity.key(state["harnesses"]["x"], hasher, "env") + self.assertFalse(parity.stale("x", state, hasher, "env")) + self.assertTrue(parity.stale("x", state, hasher, "other toolchain")) + source.write_text("two!") + self.assertTrue(parity.stale("x", state, hasher, "env")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/compare_loot_fill.py b/tools/compare_loot_fill.py index 41941ffe6..7ad307420 100644 --- a/tools/compare_loot_fill.py +++ b/tools/compare_loot_fill.py @@ -66,7 +66,7 @@ def main(): names = sorted({k[0] for k in vanilla} | {k[0] for k in errors}) subprocess.run(['cargo', 'build', '--profile', 'fast', '-q', '-p', 'horizon-world', '--example', 'loot_fill'], check=True, cwd=lab.ROOT) - rust_out = subprocess.run([str(lab.ROOT/'target/fast/examples/loot_fill'), resources, ','.join(names), *seeds], + rust_out = subprocess.run([str(lab.cargo_target()/'fast/examples/loot_fill'), resources, ','.join(names), *seeds], check=True, capture_output=True, text=True).stdout rust, _ = parse(rust_out.splitlines()) matched = differ = 0 diff --git a/tools/compare_structure_blocks.py b/tools/compare_structure_blocks.py index 92966b21d..51d71f57b 100644 --- a/tools/compare_structure_blocks.py +++ b/tools/compare_structure_blocks.py @@ -77,7 +77,7 @@ def main(): vanilla = parse(out.read_text().splitlines()) subprocess.run(['cargo', 'build', '--profile', 'fast', '-q', '-p', 'horizon-world', '--example', 'structure_blocks'], check=True, cwd=lab.ROOT) - rust_out = subprocess.run([str(lab.ROOT/'target/fast/examples/structure_blocks'), resources, seed, dimension, *targets], + rust_out = subprocess.run([str(lab.cargo_target()/'fast/examples/structure_blocks'), resources, seed, dimension, *targets], check=True, capture_output=True, text=True).stdout (build_dir()/'rust.txt').write_text(rust_out) rust = parse(rust_out.splitlines()) diff --git a/tools/gate.py b/tools/gate.py new file mode 100644 index 000000000..2edf17913 --- /dev/null +++ b/tools/gate.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""The check a branch must pass before it merges into the integration branch. + + python3 tools/gate.py # everything + python3 tools/gate.py --no-parity # skip the vanilla harnesses + python3 tools/gate.py --quick # formatting, lints and Python tests only + python3 tools/gate.py --known-failures mob_ai,light # harnesses already failing on the base + +Runs, in order, stopping at the first failure: the public-tree check, +rustfmt, clippy with warnings denied, the Rust tests, the Python tests, and +then every vanilla harness whose inputs changed (tools/parity.py). Harnesses +listed with --known-failures may still fail (they fail on the integration +branch too); any other failing harness fails the gate. CI runs the +same Rust and Python checks; the harnesses are the part CI cannot run. +The result is written to .local/gate.json. +""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +RESULT = ROOT / ".local" / "gate.json" + + +def steps(args): + yield "public tree", [sys.executable, "tools/check_public_tree.py"] + yield "rustfmt", ["cargo", "+stable", "fmt", "--all", "--", "--check"] + yield "clippy", ["cargo", "+stable", "clippy", "--locked", "--workspace", "--all-targets", "--", "-D", "warnings"] + if args.quick: + yield "python tests", [sys.executable, "-m", "unittest", "discover", "-s", "tests", "-q"] + return + # Clippy has already compiled the examples; building them again for the + # test run is most of the time and disk `cargo test --workspace` takes. + yield "rust tests", ["cargo", "+stable", "test", "--locked", "--workspace", "--lib", "--bins", "--tests"] + yield "doc tests", ["cargo", "+stable", "test", "--locked", "--workspace", "--doc"] + yield "python tests", [sys.executable, "-m", "unittest", "discover", "-s", "tests", "-q"] + if not args.no_parity: + yield "vanilla harnesses", [sys.executable, "tools/parity.py", "-j", str(args.jobs)] + + +def head(): + return subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--no-parity", action="store_true", help="skip the vanilla harnesses") + parser.add_argument("--quick", action="store_true", help="only formatting, lints and Python tests") + parser.add_argument("--known-failures", default="", help="comma-separated harnesses allowed to keep failing") + parser.add_argument("-j", "--jobs", type=int, default=max(1, (os.cpu_count() or 4) // 3), help="parallel harnesses") + args = parser.parse_args() + logs = ROOT / ".local" / "gate" + logs.mkdir(parents=True, exist_ok=True) + commit, dirty = head(), bool(subprocess.run(["git", "status", "--porcelain"], cwd=ROOT, capture_output=True, text=True).stdout.strip()) + results = [] + passed = True + for name, command in steps(args): + started = time.monotonic() + log = logs / (name.replace(" ", "-") + ".log") + print(f"{name}...", end=" ", flush=True) + if name == "vanilla harnesses": + # Stream the runner's own progress; it keeps per-harness logs. + status = subprocess.run(command, cwd=ROOT).returncode + if status: + known = set(filter(None, args.known_failures.split(","))) + state = json.loads((ROOT / ".local/parity-runner/state.json").read_text())["harnesses"] + failing = {harness for harness, record in state.items() if record["status"] != "pass"} + new = sorted(failing - known) + print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}") + status = 1 if new else 0 + else: + with log.open("w") as stream: + status = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT).returncode + seconds = round(time.monotonic() - started) + results.append({"step": name, "passed": status == 0, "seconds": seconds}) + if status: + passed = False + print(f"FAILED ({seconds}s)") + if name != "vanilla harnesses": + print("".join(log.read_text(errors="replace").splitlines(keepends=True)[-40:])) + print(f"Full log: {log}") + break + print(f"ok ({seconds}s)") + RESULT.write_text(json.dumps({"commit": commit, "dirty": dirty, "passed": passed, "quick": args.quick, + "parity": not (args.quick or args.no_parity), "steps": results, + "finished": time.strftime("%Y-%m-%dT%H:%M:%S")}, indent=1)) + print("Gate passed." if passed else "Gate failed.") + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/lab.py b/tools/lab.py index 77d2b8a87..3443b2a8a 100644 --- a/tools/lab.py +++ b/tools/lab.py @@ -6,6 +6,7 @@ from collections import Counter from concurrent.futures import ThreadPoolExecutor import hashlib import json +import os from pathlib import Path, PurePosixPath import re import shutil @@ -21,6 +22,11 @@ LOCAL = ROOT / ".local" LOCK = ROOT / "research-lock.json" +def cargo_target(): + """Cargo's target directory, honouring CARGO_TARGET_DIR (tools/parity.py sets it).""" + return ROOT / os.environ.get("CARGO_TARGET_DIR", "target") + + def write_json(path, data): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") diff --git a/tools/parity.py b/tools/parity.py new file mode 100644 index 000000000..bcc554bd9 --- /dev/null +++ b/tools/parity.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""Run the vanilla differential harnesses (tools/test_*_parity.py) quickly. + + python3 tools/parity.py # harnesses whose inputs changed, or that never passed + python3 tools/parity.py --all # every harness + python3 tools/parity.py food light # these harnesses (names without test_/_parity.py) + python3 tools/parity.py --failed # the ones that failed last time + python3 tools/parity.py --list # what would run, without running it + python3 tools/parity.py --status # last result of every harness + +Each harness runs unmodified, in parallel, under a tracer that records what it +read: its Python modules, Java oracle sources, other repository files, and the +Rust sources of every probe it executed (from Cargo's dep-info). A harness +reruns only when one of those files, the pinned vanilla artifacts or the +toolchains change; otherwise its last pass stands. + +Harness probes build into .local/parity-target with release optimisations but +without LTO and with 16 codegen units (`PROFILE` below), which builds in +seconds instead of minutes. Rust does not let those settings change results: +there is no fast-math, and overflow checks stay as in release. Probes the +recorded harnesses need are built in one Cargo run before the harnesses start, +so parallel harnesses don't queue on Cargo's lock. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +LOCAL = ROOT / ".local" +STATE_DIR = LOCAL / "parity-runner" +STATE = STATE_DIR / "state.json" +TARGET = LOCAL / "parity-target" +PROFILE = { + "CARGO_PROFILE_RELEASE_LTO": "false", + "CARGO_PROFILE_RELEASE_CODEGEN_UNITS": "16", + "CARGO_PROFILE_RELEASE_INCREMENTAL": "true", +} +DEFAULT_MEMORY = 3 << 30 # assumed peak for a harness that never ran here + + +def harnesses(): + return {path.name[len("test_"):-len("_parity.py")]: path for path in sorted((ROOT / "tools").glob("test_*_parity.py"))} + + +def load_state(): + return json.loads(STATE.read_text()) if STATE.exists() else {"harnesses": {}, "hashes": {}} + + +def save_state(state): + STATE_DIR.mkdir(parents=True, exist_ok=True) + temporary = STATE.with_suffix(".tmp") + temporary.write_text(json.dumps(state, indent=1, sort_keys=True)) + temporary.replace(STATE) + + +class Hasher: + """Content hashes, reused while a file's size and mtime stay the same.""" + + def __init__(self, cache): + self.cache = cache + + def __call__(self, path): + try: + stat = os.stat(path) + except FileNotFoundError: + return "missing" + stamp = [stat.st_size, stat.st_mtime_ns] + cached = self.cache.get(str(path)) + if cached and cached[0] == stamp: + return cached[1] + with open(path, "rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + self.cache[str(path)] = [stamp, digest] + return digest + + +def environment_key(hasher): + """Everything outside the recorded dependencies that can change a result.""" + def version(command): + try: + result = subprocess.run(command, capture_output=True, text=True, cwd=ROOT) + return (result.stdout + result.stderr).strip() + except FileNotFoundError: + return "missing" + lock = json.loads((ROOT / "research-lock.json").read_text()) + version_dir = LOCAL / "minecraft" / lock["minecraft"]["version"] + parts = [version(["rustc", "-V"]), version(["java", "-version"]), version(["javac", "-version"]), + json.dumps(PROFILE, sort_keys=True), hasher(ROOT / "research-lock.json"), hasher(ROOT / "Cargo.lock"), + hasher(version_dir / "prepared.json"), hasher(version_dir / "manifest.json")] + return hashlib.sha256("\n".join(parts).encode()).hexdigest() + + +def key(record, hasher, environment): + digest = hashlib.sha256(environment.encode()) + for path in sorted(record["dependencies"]): + digest.update(f"{path}\0{hasher(ROOT / path)}\n".encode()) + return digest.hexdigest() + + +def inside(path, root): + try: + Path(path).resolve().relative_to(root) + return True + except (ValueError, OSError): + return False + + +def dep_info(binary): + """Source files Cargo recorded for a built binary (its .d file, spaces escaped as "\\ ").""" + info = Path(str(binary) + ".d") + if not info.exists(): + return [] + sources = [] + for line in info.read_text().splitlines(): + _, separator, rest = line.partition(": ") + if separator: + sources += [part.replace("\\ ", " ") for part in re.split(r"(?= jobs: + break + expected = records.get(name, {}).get("peak_bytes", DEFAULT_MEMORY) + if running and in_use + expected > memory_budget: + continue + queue.remove(name) + trace = traces / f"{name}.json" + trace.unlink(missing_ok=True) + log = (logs / f"{name}.log").open("w") + process = subprocess.Popen( + [sys.executable, str(Path(__file__)), "--trace", str(trace), str(selected[name])], + cwd=ROOT, env=run_environment(), stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + running[name] = (process, time.monotonic(), log, expected) + in_use += expected + time.sleep(0.2) + for name, (process, started, log, _) in list(running.items()): + elapsed = time.monotonic() - started + if process.poll() is None and elapsed < timeout: + continue + if process.poll() is None: + os.killpg(process.pid, 9) + process.wait() + print(f"--- {name}: timed out after {timeout}s", file=log) + log.close() + del running[name] + passed = process.returncode == 0 + trace_path = traces / f"{name}.json" + record = {"status": "pass" if passed else "fail", "seconds": round(elapsed, 1), + "finished": time.strftime("%Y-%m-%dT%H:%M:%S")} + if trace_path.exists(): + trace = json.loads(trace_path.read_text()) + record["dependencies"], record["artifacts"] = dependencies(trace) + record["peak_bytes"] = trace["peak_bytes"] + record["key"] = key(record, hasher, environment_digest) + else: + record["dependencies"], record["artifacts"] = [], [] + records[name] = record + results[name] = record + save_state(state) + tail = "" if passed else f" ({logs / (name + '.log')})" + print(f"{'PASS' if passed else 'FAIL'} {name} {elapsed:.0f}s{tail}", flush=True) + return results, time.monotonic() - started_all + + +def stale(name, state, hasher, environment): + record = state["harnesses"].get(name) + if not record or record.get("status") != "pass" or "key" not in record: + return True + return key(record, hasher, environment) != record["key"] + + +def trace_main(output, script, arguments): + """Run one harness and record what it read and executed.""" + import atexit + import resource + import runpy + read, commands = set(), [] + + def audit(event, details): + if event == "open": + path, mode = details[0], details[1] + if isinstance(path, (str, bytes, os.PathLike)) and (mode is None or "r" in str(mode) and "+" not in str(mode)): + read.add(os.fsdecode(path)) + elif event == "subprocess.Popen": + arguments_ = details[1] + arguments_ = [arguments_] if isinstance(arguments_, (str, bytes, os.PathLike)) else list(arguments_ or []) + commands.append([os.fsdecode(a) if isinstance(a, (bytes, os.PathLike)) else str(a) for a in arguments_]) + + def write(): + modules = [getattr(module, "__file__", None) for module in list(sys.modules.values())] + peak = max(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss) + Path(output).write_text(json.dumps({ + "modules": sorted({m for m in modules if m and inside(m, ROOT)}), + "read": sorted(path for path in read if os.path.isabs(path) or not path.startswith("<")), + "commands": commands, + "peak_bytes": peak * 1024, + })) + + atexit.register(write) + sys.addaudithook(audit) + sys.argv = [script, *arguments] + sys.path.insert(0, str(Path(script).parent)) + runpy.run_path(script, run_name="__main__") + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] == "--trace": + return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:]) + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("names", nargs="*", help="harness names, e.g. food or block_update") + parser.add_argument("--all", action="store_true", help="run every harness, ignoring earlier passes") + parser.add_argument("--failed", action="store_true", help="rerun the harnesses that failed last time") + parser.add_argument("--list", action="store_true", help="print what would run, then stop") + parser.add_argument("--status", action="store_true", help="print each harness's last result") + parser.add_argument("-j", "--jobs", type=int, default=max(1, (os.cpu_count() or 4) // 3)) + parser.add_argument("--timeout", type=int, default=3600, help="seconds per harness") + args = parser.parse_args() + + found = harnesses() + state = load_state() + hasher = Hasher(state["hashes"]) + environment = environment_key(hasher) + records = state["harnesses"] + unknown = [name for name in args.names if name not in found] + if unknown: + parser.error("unknown harnesses: " + ", ".join(unknown)) + if args.status: + for name in found: + record = records.get(name) + status = "never run" if not record else record["status"] + (" (stale)" if stale(name, state, hasher, environment) and record["status"] == "pass" else "") + seconds = f"{record['seconds']:>7.0f}s" if record else "" + print(f"{name:<40} {status:<15} {seconds}") + save_state(state) + return 0 + if args.names: + chosen = args.names + elif args.all: + chosen = list(found) + elif args.failed: + chosen = [name for name in found if records.get(name, {}).get("status") == "fail"] + else: + chosen = [name for name in found if stale(name, state, hasher, environment)] + selected = {name: found[name] for name in chosen} + if args.list or not selected: + for name in selected: + print(name) + print(f"{len(selected)} of {len(found)} harnesses would run" if args.list else "Nothing to run: every harness passed with its current inputs.") + save_state(state) + return 0 + + jobs = max(1, args.jobs) + # Harnesses that never ran here have no recorded probes: build them all. + artifacts = [["all", "", "release"]] if any("artifacts" not in records.get(name, {}) for name in selected) else [] + for name in selected: + for artifact in records.get(name, {}).get("artifacts", []): + if artifact not in artifacts: + artifacts.append(artifact) + print("Building probes...", flush=True) + started = time.monotonic() + built, errors = prebuild(artifacts, os.cpu_count() or 8) + if not built: + print(errors[-4000:]) + print("Probe build failed; harnesses will report their own build errors.", flush=True) + print(f"Probes built in {time.monotonic() - started:.0f}s. Running {len(selected)} harnesses, {jobs} at a time...", flush=True) + results, elapsed = run(selected, state, hasher, environment_key(hasher), jobs, args.timeout, int(memory_available() * 0.75)) + save_state(state) + failed = sorted(name for name, record in results.items() if record["status"] != "pass") + print(f"\n{len(results) - len(failed)} passed, {len(failed)} failed in {elapsed / 60:.1f} min.") + for name in failed: + print(f" {name}: {STATE_DIR / 'logs' / (name + '.log')}") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/supervise.py b/tools/supervise.py new file mode 100644 index 000000000..954cc76aa --- /dev/null +++ b/tools/supervise.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Manage parallel parity work: one worktree and branch per area, one integration branch. + + python3 tools/supervise.py status # every worktree: branch, progress, last gate + python3 tools/supervise.py new dispensers # worktree ../dispensers on parity/dispensers, from the integration branch + python3 tools/supervise.py save [AREA ...] # commit and push unfinished work (before stopping agents) + python3 tools/supervise.py integrate parity/dispensers # merge into the integration branch if the gate passes + +The integration branch (`parity-base` by default) is checked out in its own +worktree; `integrate` merges there, runs tools/gate.py and pushes the +integration branch only when the gate passes, allowing only the harnesses +that already fail on the integration branch to keep failing. Otherwise it undoes the merge +and leaves the branch for its owner to fix. Worktrees live next to the +integration worktree and share its vanilla data: their `.local/` entries +link to `shared-local/` in the same directory. +""" + +import argparse +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def git(*args, cwd=ROOT, check=True): + result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) + if check and result.returncode: + raise SystemExit(f"git {' '.join(args)} failed in {cwd}:\n{result.stderr.strip()}") + return result.stdout.strip() + + +def worktrees(): + found, current = [], {} + for line in git("worktree", "list", "--porcelain").splitlines() + [""]: + if not line: + if current: + found.append(current) + current = {} + elif line.startswith("worktree "): + current["path"] = Path(line[len("worktree "):]) + elif line.startswith("branch "): + current["branch"] = line[len("branch refs/heads/"):] + return found + + +def integration(base): + for tree in worktrees(): + if tree.get("branch") == base: + return tree["path"] + raise SystemExit(f"No worktree has {base} checked out; create one with: git worktree add {base}") + + +def count(range_, cwd): + return int(git("rev-list", "--count", range_, cwd=cwd, check=False) or 0) + + +def status(args): + base_tree = integration(args.base) + print(f"{'worktree':<16} {'branch':<26} {'ahead':>5} {'behind':>6} {'dirty':>5} {'unpushed':>8} gate") + for tree in worktrees(): + path, branch = tree["path"], tree.get("branch", "(detached)") + if path.parent != base_tree.parent and path != base_tree: + continue + dirty = len(git("status", "--porcelain", cwd=path).splitlines()) + remote = f"origin/{args.push_to if branch == args.base else branch}" + unpushed = count(f"{remote}..HEAD", path) if git("rev-parse", "--verify", "--quiet", remote, cwd=path, check=False) else "new" + gate = path / ".local" / "gate.json" + verdict = "never run" + if gate.exists(): + result = json.loads(gate.read_text()) + verdict = ("passed" if result["passed"] else "failed") + ("" if result["parity"] else " (no harnesses)") + if result["commit"] != git("rev-parse", "HEAD", cwd=path) or result["dirty"]: + verdict += ", older commit" + print(f"{path.name:<16} {branch:<26} {count(f'{args.base}..HEAD', path):>5} {count(f'HEAD..{args.base}', path):>6}" + f" {dirty:>5} {unpushed:>8} {verdict}") + + +def new(args): + base_tree = integration(args.base) + path = base_tree.parent / args.area + branch = f"parity/{args.area}" + if path.exists(): + raise SystemExit(f"{path} already exists") + git("worktree", "add", "-b", branch, str(path), args.base) + shared = base_tree.parent / "shared-local" + (path / ".local").mkdir() + for entry in sorted(shared.iterdir()) if shared.is_dir() else []: + (path / ".local" / entry.name).symlink_to(entry) + print(f"{path} on {branch}, from {args.base} at {git('rev-parse', '--short', 'HEAD', cwd=path)}") + + +def save(args): + base_tree = integration(args.base) + for tree in worktrees(): + path, branch = tree["path"], tree.get("branch") + if not branch or not branch.startswith("parity/") or path.parent != base_tree.parent: + continue + if args.areas and path.name not in args.areas: + continue + if git("status", "--porcelain", cwd=path): + git("add", "-A", cwd=path) + git("commit", "-q", "-m", f"WIP: unfinished {path.name} parity work\n\nWork in progress saved when its agent " + "stopped; may not build or pass its harnesses yet.\n\n" + "Co-Authored-By: Claude Opus 5.5 ", cwd=path) + print(f"{path.name}: committed unfinished work") + git("push", "-q", "origin", f"HEAD:{branch}", cwd=path) + print(f"{path.name}: {branch} pushed at {git('rev-parse', '--short', 'HEAD', cwd=path)}") + + +def integrate(args): + base_tree = integration(args.base) + if git("status", "--porcelain", "--untracked-files=no", cwd=base_tree): + raise SystemExit(f"{base_tree} has uncommitted changes; commit or stash them first") + before = git("rev-parse", "HEAD", cwd=base_tree) + state = base_tree / ".local/parity-runner/state.json" + known = sorted(name for name, record in json.loads(state.read_text())["harnesses"].items() + if record["status"] != "pass") if state.exists() else [] + merge = subprocess.run(["git", "merge", "--no-ff", "--no-edit", args.branch], cwd=base_tree, capture_output=True, text=True) + if merge.returncode: + conflicts = git("diff", "--name-only", "--diff-filter=U", cwd=base_tree, check=False) + git("merge", "--abort", cwd=base_tree, check=False) + print(f"{args.branch} does not merge cleanly into {args.base}. Conflicts:\n{conflicts or merge.stdout + merge.stderr}") + print(f"Merge {args.base} into {args.branch}, resolve them there, and integrate again.") + return 2 + if git("rev-parse", "HEAD", cwd=base_tree) == before: + print(f"{args.branch} has nothing new for {args.base}.") + return 0 + gate = [sys.executable, "tools/gate.py", "-j", str(args.jobs), "--known-failures", ",".join(known)] + gate += ["--no-parity"] if args.no_parity else [] + if subprocess.run(gate, cwd=base_tree).returncode: + git("reset", "--hard", before, cwd=base_tree) + print(f"The gate failed, so {args.branch} was not integrated; {args.base} is back at {before[:9]}.") + return 1 + git("push", "-q", "origin", f"{args.base}:{args.push_to}", cwd=base_tree) + print(f"Integrated {args.branch}; {args.base} pushed to origin/{args.push_to} at {git('rev-parse', '--short', 'HEAD', cwd=base_tree)}.") + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--base", default="parity-base", help="the integration branch") + parser.add_argument("--push-to", default="t3/achieve-full-parity", help="remote branch the integration branch is pushed to") + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("status").set_defaults(run=status) + command = commands.add_parser("new") + command.add_argument("area") + command.set_defaults(run=new) + command = commands.add_parser("save") + command.add_argument("areas", nargs="*") + command.set_defaults(run=save) + command = commands.add_parser("integrate") + command.add_argument("branch") + command.add_argument("-j", "--jobs", type=int, default=8) + command.add_argument("--no-parity", action="store_true", help="skip the vanilla harnesses in the gate") + command.set_defaults(run=integrate) + args = parser.parse_args() + return args.run(args) or 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/test_mob_ai_parity.py b/tools/test_mob_ai_parity.py index b6b75ec3a..bee2af785 100644 --- a/tools/test_mob_ai_parity.py +++ b/tools/test_mob_ai_parity.py @@ -9,7 +9,7 @@ motion, rotations, running goals and target goals, attack target, navigation pat controls, movement input, brain activities and memories) and the sounds played; this harness reports, per scenario and mob, how many ticks match before the first divergence and what differs. - python3 tools/test_mob_ai_parity.py [--only NAME ...] [--rust-only] [--java-only] [--strict] + python3 tools/test_mob_ai_parity.py [--only NAME ...] [--rust-only] [--java-only] [--report-only] """ import argparse import os @@ -220,7 +220,7 @@ def main(): parser.add_argument('--only', nargs='*') parser.add_argument('--java-only', action='store_true') parser.add_argument('--rust-only', action='store_true', help='reuse the last vanilla output') - parser.add_argument('--strict', action='store_true', help='fail on any divergence') + parser.add_argument('--report-only', action='store_true', help='print divergences without failing') args = parser.parse_args() build = lab.LOCAL / 'mob-ai-parity' build.mkdir(parents=True, exist_ok=True) @@ -237,7 +237,7 @@ def main(): if not args.java_only: oracle_harness.run_logged(['cargo', 'build', '--locked', '--release', '-p', 'horizon-client', '--example', 'mob_ai_probe'], build, 'compile-rust') - binary = lab.ROOT / 'target/release/examples/mob_ai_probe' + binary = lab.cargo_target() / 'release/examples/mob_ai_probe' oracle_harness.run_logged([binary, lab.ROOT / '.local/horizon-client/26.3', inputs, build / 'horizon.txt'], build, 'horizon') if args.java_only: return @@ -248,7 +248,7 @@ def main(): print('\n'.join(report)) print(f"{totals['clean']}/{totals['scenarios']} scenarios match entirely; " f"{totals['matched']}/{totals['mob_ticks']} mob-ticks match before each mob's first divergence") - if args.strict and totals['clean'] != totals['scenarios']: + if not args.report_only and totals['clean'] != totals['scenarios']: raise SystemExit(1) diff --git a/tools/test_mob_attributes_parity.py b/tools/test_mob_attributes_parity.py index da0cfdeb2..9328d223e 100755 --- a/tools/test_mob_attributes_parity.py +++ b/tools/test_mob_attributes_parity.py @@ -63,7 +63,7 @@ def main(): run(['javac', '--release', lock['minecraft']['java_major'], '-proc:none', '-cp', str(jars['server']) + os.pathsep + deps, '-d', classes, oracle], 'compile-java') run(['java', '-Xmx1G', '-cp', os.pathsep.join([str(classes), str(jars['server']), deps]), 'dev.horizonlab.parity.VanillaMobAttributesOracle', build / 'vanilla.txt'], 'java') run(['cargo', 'build', '--release', '-p', 'horizon-client', '--example', 'mob_attributes_fixture'], 'compile-rust') - binary = lab.ROOT / 'target/release/examples/mob_attributes_fixture' + binary = lab.cargo_target() / 'release/examples/mob_attributes_fixture' resources = lab.ROOT / '.local/horizon-client/26.3' run([binary, resources, build / 'rust.txt'], 'rust') vanilla, rust = parse(build / 'vanilla.txt'), parse(build / 'rust.txt') diff --git a/tools/test_save_roundtrip_parity.py b/tools/test_save_roundtrip_parity.py index 18bea7465..126d39b01 100644 --- a/tools/test_save_roundtrip_parity.py +++ b/tools/test_save_roundtrip_parity.py @@ -501,7 +501,7 @@ def summarize(report): def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--keep', action='store_true') - parser.add_argument('--bin-dir', default=str(ROOT / 'target' / 'fast')) + parser.add_argument('--bin-dir', help='prebuilt horizon-client and horizon-ctl (default: build them with the fast profile)') parser.add_argument('--vanilla', help='an existing vanilla world folder (skips generation)') parser.add_argument('--horizon', help='an existing Horizon-saved world folder (skips the Horizon run)') parser.add_argument('--skip-vanilla-load', action='store_true') @@ -524,7 +524,11 @@ def main(): if args.horizon: h = Path(args.horizon) else: - h = horizon_resave(v0, RUN / 'h', Path(args.bin_dir)) + bin_dir = Path(args.bin_dir) if args.bin_dir else None + if bin_dir is None: + subprocess.run(['cargo', 'build', '--locked', '--profile', 'fast', '-p', 'horizon-client', '--bins'], cwd=ROOT, check=True) + bin_dir = ROOT / os.environ.get('CARGO_TARGET_DIR', 'target') / 'fast' + h = horizon_resave(v0, RUN / 'h', bin_dir) print('comparing', v0, 'with', h) report = compare_worlds(v0, h) failures += summarize(report)