diff --git a/crates/horizon-client/src/audio.rs b/crates/horizon-client/src/audio.rs index 5d99c4858..5853a4d7f 100644 --- a/crates/horizon-client/src/audio.rs +++ b/crates/horizon-client/src/audio.rs @@ -360,6 +360,11 @@ impl Audio { } } fn open(mixer: Arc>) -> Result { + // Test runs (harnesses, agents) launch the game on a virtual screen; + // keep them off the user's speakers. + if std::env::var_os("HORIZON_NO_AUDIO").is_some() { + return Err("audio output disabled by HORIZON_NO_AUDIO".into()); + } let host = cpal::default_host(); let device = host .default_output_device() diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index 8db55783f..d072feec7 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -10,14 +10,17 @@ How parity work is run, checked and merged. The harnesses themselves are describ python3 tools/parity.py # harnesses whose inputs changed, or that never passed python3 tools/parity.py food light # these harnesses, whatever their state python3 tools/parity.py --failed # the ones that failed last time -python3 tools/parity.py --all # everything (about 20 minutes on ludodesktop) +python3 tools/parity.py --all # everything, from scratch: the audit (about 20 minutes on ludodesktop) +python3 tools/parity.py --fresh food # don't reuse vanilla oracle results python3 tools/parity.py --list # what would run python3 tools/parity.py --status # last result of each harness ``` Each harness runs unmodified under a tracer that records the files it depended on: its Python modules, its Java oracle and the oracle sources javac pulled in, other repository files it read, and the Rust sources of the probes it ran (from Cargo's `.d` dep-info files). A pass stands until one of those files, the pinned vanilla artifacts, `Cargo.lock` or the Rust/Java toolchains change. State and logs are in `.local/parity-runner/`; each harness's own outputs stay in its `.local//` directory. -Probes build into `.local/parity-target/` with release optimisation but no LTO and 16 codegen units. On ludodesktop that rebuilds every `horizon-world` probe in 3 seconds after a one-line change, against 122 seconds with the release profile. Rust gives these settings identical results (no fast-math; overflow checks as in release), and the full harness suite passes under them. Harnesses must locate binaries through `cargo metadata` or `lab.cargo_target()`, never a hard-coded `target/`. +Vanilla oracle runs are reused as well. `tools/parity_shim.py` stands in for `java` and `javac`: it records each oracle run's outputs and the content of every file under the paths its arguments name, and replays the run while those are unchanged. After a Rust-only change a harness pays only for its Rust side (`block_update`: 53 s to 5 s). Files an oracle writes outside its named paths aren't tracked, so a harness that fails while reusing oracle results is rerun fresh, and stops reusing them if it then passes. `--all` and `--fresh` never reuse them. + +Probes build into `.local/parity-target/` with release optimisation but no LTO and 16 codegen units. On ludodesktop that rebuilds every `horizon-world` probe in 3 seconds after a one-line change, against 122 seconds with the release profile. Rust gives these settings identical results (no fast-math; overflow checks as in release), and the full harness suite passes under them. Probes that harnesses build in the dev profile get `opt-level = 3` too, keeping their overflow checks and debug assertions. Harnesses must locate binaries through `cargo metadata` or `lab.cargo_target()`, never a hard-coded `target/`. Parallel harnesses are admitted by memory: each harness's recorded peak is reserved, within three quarters of the memory available at the start. `-j` caps the count (default: a third of the CPU threads). diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py index 01ff01be1..4692a6efd 100644 --- a/tests/test_parity_runner.py +++ b/tests/test_parity_runner.py @@ -45,5 +45,52 @@ class ParityRunnerTests(unittest.TestCase): self.assertTrue(parity.stale("x", state, hasher, "env")) +class OracleReuseTests(unittest.TestCase): + """tools/parity_shim.py replays a run only when its inputs and outputs are as recorded.""" + + def setUp(self): + import os, subprocess, sys + self.temp = tempfile.TemporaryDirectory() + root = Path(self.temp.name) + self.bin, self.shim = root / "bin", root / "shim" + self.bin.mkdir(); self.shim.mkdir() + # A fake oracle: copies its input to its output and counts its runs. + fake = self.bin / "java" + fake.write_text(f"#!{sys.executable}\nimport sys, pathlib\n" + "runs = pathlib.Path(sys.argv[1]).parent / 'runs'\n" + "runs.write_text(str(int(runs.read_text() or 0) + 1) if runs.exists() else '1')\n" + "pathlib.Path(sys.argv[2]).write_text(pathlib.Path(sys.argv[1]).read_text())\nprint('done')\n") + fake.chmod(0o755) + (self.shim / "java").symlink_to(Path(parity.__file__).with_name("parity_shim.py")) + self.input, self.output = root / "work" / "in.txt", root / "work" / "out.txt" + self.input.parent.mkdir() + self.input.write_text("a") + python = Path(sys.executable).parent + self.env = dict(os.environ, PATH=f"{self.shim}:{self.bin}:{python}", PARITY_SHIM_REAL_PATH=str(self.bin), + PARITY_SHIM_CACHE=str(root / "cache")) + self.subprocess = subprocess + + def tearDown(self): + self.temp.cleanup() + + def oracle(self): + result = self.subprocess.run(["java", str(self.input), str(self.output)], env=self.env, capture_output=True, text=True, + stdin=self.subprocess.DEVNULL, check=True) + self.assertEqual(result.stdout, "done\n") + return int((self.input.parent / "runs").read_text()) + + def test_reuses_until_an_input_or_output_changes(self): + self.assertEqual(self.oracle(), 1) + self.assertEqual(self.oracle(), 1) # replayed, stdout included + self.input.write_text("a") # same content, new mtime: still reusable + self.assertEqual(self.oracle(), 1) + self.input.write_text("b") + self.assertEqual(self.oracle(), 2) + self.assertEqual(self.output.read_text(), "b") + self.output.write_text("tampered") + self.assertEqual(self.oracle(), 3) + self.assertEqual(self.output.read_text(), "b") + + if __name__ == "__main__": unittest.main() diff --git a/tools/parity.py b/tools/parity.py index bcc554bd9..91dd02e04 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -2,7 +2,8 @@ """Run the vanilla differential harnesses (tools/test_*_parity.py) quickly. python3 tools/parity.py # harnesses whose inputs changed, or that never passed - python3 tools/parity.py --all # every harness + python3 tools/parity.py --all # every harness, from scratch (the audit) + python3 tools/parity.py --fresh ... # don't reuse vanilla oracle results python3 tools/parity.py food light # these harnesses (names without test_/_parity.py) python3 tools/parity.py --failed # the ones that failed last time python3 tools/parity.py --list # what would run, without running it @@ -14,10 +15,17 @@ Rust sources of every probe it executed (from Cargo's dep-info). A harness reruns only when one of those files, the pinned vanilla artifacts or the toolchains change; otherwise its last pass stands. +Vanilla oracles are reused too: tools/parity_shim.py stands in for java and +javac and replays an oracle run whose inputs are unchanged, so a Rust-only +change costs only the Rust side. A harness that fails while reusing oracle +results is rerun fresh, and stops reusing them if it then passes. --all and +--fresh never reuse them. + Harness probes build into .local/parity-target with release optimisations but without LTO and with 16 codegen units (`PROFILE` below), which builds in seconds instead of minutes. Rust does not let those settings change results: -there is no fast-math, and overflow checks stay as in release. Probes the +there is no fast-math, and overflow checks stay as in release. Probes built in +the dev profile are optimised too, keeping their overflow checks. Probes the recorded harnesses need are built in one Cargo run before the harnesses start, so parallel harnesses don't queue on Cargo's lock. """ @@ -41,7 +49,11 @@ PROFILE = { "CARGO_PROFILE_RELEASE_LTO": "false", "CARGO_PROFILE_RELEASE_CODEGEN_UNITS": "16", "CARGO_PROFILE_RELEASE_INCREMENTAL": "true", + # Probes built in the dev profile keep its overflow checks and debug + # assertions, but run optimised. + "CARGO_PROFILE_DEV_OPT_LEVEL": "3", } +SHIM = STATE_DIR / "shim" DEFAULT_MEMORY = 3 << 30 # assumed peak for a harness that never ran here @@ -189,9 +201,18 @@ def java_sources(arguments): return sources -def run_environment(): - environment = dict(os.environ, CARGO_TARGET_DIR=str(TARGET), PYTHONDONTWRITEBYTECODE="1", **PROFILE) +def run_environment(reuse_oracles=False): + environment = dict(os.environ, CARGO_TARGET_DIR=str(TARGET), PYTHONDONTWRITEBYTECODE="1", HORIZON_NO_AUDIO="1", **PROFILE) environment.pop("CARGO_BUILD_JOBS", None) + if reuse_oracles: + # tools/parity_shim.py stands in for java and javac and replays unchanged oracle runs. + SHIM.mkdir(parents=True, exist_ok=True) + for name in ("java", "javac"): + link = SHIM / name + if not link.is_symlink(): + link.symlink_to(ROOT / "tools" / "parity_shim.py") + environment.update(PARITY_SHIM_REAL_PATH=os.environ.get("PATH", ""), PARITY_SHIM_CACHE=str(STATE_DIR / "oracle-cache"), + PATH=f"{SHIM}{os.pathsep}{os.environ.get('PATH', '')}") return environment @@ -217,7 +238,7 @@ def memory_available(): return 16 << 30 -def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budget): +def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budget, reuse_oracles): logs = STATE_DIR / "logs" traces = STATE_DIR / "traces" logs.mkdir(parents=True, exist_ok=True) @@ -227,6 +248,7 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge queue = sorted(selected, key=lambda name: -records.get(name, {}).get("seconds", 1e9)) running = {} results = {} + retried = set() started_all = time.monotonic() while queue or running: in_use = sum(item[3] for item in running.values()) @@ -237,16 +259,17 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge if running and in_use + expected > memory_budget: continue queue.remove(name) + reuse = reuse_oracles and records.get(name, {}).get("oracle_reuse", True) and name not in retried trace = traces / f"{name}.json" trace.unlink(missing_ok=True) log = (logs / f"{name}.log").open("w") process = subprocess.Popen( [sys.executable, str(Path(__file__)), "--trace", str(trace), str(selected[name])], - cwd=ROOT, env=run_environment(), stdout=log, stderr=subprocess.STDOUT, start_new_session=True) - running[name] = (process, time.monotonic(), log, expected) + cwd=ROOT, env=run_environment(reuse), stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + running[name] = (process, time.monotonic(), log, expected, reuse) in_use += expected time.sleep(0.2) - for name, (process, started, log, _) in list(running.items()): + for name, (process, started, log, _, reused) in list(running.items()): elapsed = time.monotonic() - started if process.poll() is None and elapsed < timeout: continue @@ -257,9 +280,17 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge log.close() del running[name] passed = process.returncode == 0 + if not passed and reused: + # Maybe a reused oracle result was wrong for it: run it again from scratch. + retried.add(name) + queue.insert(0, name) + print(f"retry {name} without reused oracle results", flush=True) + continue trace_path = traces / f"{name}.json" record = {"status": "pass" if passed else "fail", "seconds": round(elapsed, 1), "finished": time.strftime("%Y-%m-%dT%H:%M:%S")} + # A harness that only passes fresh no longer reuses oracle results. + record["oracle_reuse"] = records.get(name, {}).get("oracle_reuse", True) and not (passed and name in retried) if trace_path.exists(): trace = json.loads(trace_path.read_text()) record["dependencies"], record["artifacts"] = dependencies(trace) @@ -321,7 +352,8 @@ def main(): return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:]) parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("names", nargs="*", help="harness names, e.g. food or block_update") - parser.add_argument("--all", action="store_true", help="run every harness, ignoring earlier passes") + parser.add_argument("--all", action="store_true", help="run every harness from scratch, ignoring earlier passes") + parser.add_argument("--fresh", action="store_true", help="run the vanilla oracles even if their inputs are unchanged") parser.add_argument("--failed", action="store_true", help="rerun the harnesses that failed last time") parser.add_argument("--list", action="store_true", help="print what would run, then stop") parser.add_argument("--status", action="store_true", help="print each harness's last result") @@ -375,7 +407,9 @@ def main(): print(errors[-4000:]) print("Probe build failed; harnesses will report their own build errors.", flush=True) print(f"Probes built in {time.monotonic() - started:.0f}s. Running {len(selected)} harnesses, {jobs} at a time...", flush=True) - results, elapsed = run(selected, state, hasher, environment_key(hasher), jobs, args.timeout, int(memory_available() * 0.75)) + reuse_oracles = not (args.fresh or args.all) + results, elapsed = run(selected, state, hasher, environment_key(hasher), jobs, args.timeout, int(memory_available() * 0.75), + reuse_oracles) save_state(state) failed = sorted(name for name, record in results.items() if record["status"] != "pass") print(f"\n{len(results) - len(failed)} passed, {len(failed)} failed in {elapsed / 60:.1f} min.") diff --git a/tools/parity_shim.py b/tools/parity_shim.py new file mode 100755 index 000000000..8cff84714 --- /dev/null +++ b/tools/parity_shim.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Reuse a vanilla oracle's last results when nothing it reads has changed. + +tools/parity.py puts `java` and `javac` links to this script first on PATH. +A run is identified by its command line, stdin when it is a file, and the +real program. After a successful run, the files it created or changed under +the paths its arguments name (classpath and sourcepath entries included) are +recorded as its outputs, with their size and mtime and its stdout and stderr; +every other file under those paths is recorded as an input with its content +hash. The next run of the same command replays the recorded output instead +of running the oracle if every input has the same content, no file was added +beside them, and the outputs are exactly as it left them. On a Rust-only +change a harness then pays only for its Rust side. + +Files an oracle writes outside its named paths are not tracked. A harness +that depends on them fails on reuse; tools/parity.py then reruns it fresh and +stops reusing its oracle results. + +A run that reads stdin from a pipe, or fails, is never cached. +PARITY_JAVA_CACHE=0 disables reuse (tools/parity.py --fresh). +""" + +import hashlib +import json +import os +from pathlib import Path +import shutil +import sqlite3 +import subprocess +import sys +import threading + +CACHE = Path(os.environ.get("PARITY_SHIM_CACHE", ".local/parity-runner/oracle-cache")) +PATH_OPTIONS = {"-cp", "-classpath", "--class-path", "-sourcepath", "--source-path", "-p", "--module-path"} + + +def real(name): + found = shutil.which(name, path=os.environ.get("PARITY_SHIM_REAL_PATH", "")) + if not found: + sys.exit(f"parity shim: no real {name} on PARITY_SHIM_REAL_PATH") + return found + + +class Hashes: + """Content hashes keyed by path, size and mtime, shared by parallel harnesses.""" + + def __init__(self): + CACHE.mkdir(parents=True, exist_ok=True) + self.db = sqlite3.connect(CACHE / "hashes.sqlite", timeout=60) + self.db.execute("pragma journal_mode=wal") + self.db.execute("create table if not exists hashes (path text primary key, size int, mtime int, sha text)") + + def file(self, path): + stat = os.stat(path) + row = self.db.execute("select size, mtime, sha from hashes where path = ?", (path,)).fetchone() + if row and row[0] == stat.st_size and row[1] == stat.st_mtime_ns: + return row[2] + with open(path, "rb") as stream: + sha = hashlib.file_digest(stream, "sha256").hexdigest() + self.db.execute("insert or replace into hashes values (?, ?, ?, ?)", (path, stat.st_size, stat.st_mtime_ns, sha)) + self.db.commit() + return sha + + +def tree(root): + for directory, _, files in os.walk(root): + for name in files: + yield os.path.join(directory, name) + + +def named_paths(arguments): + """Every filesystem path the arguments name, with classpath lists split.""" + paths = [] + for index, argument in enumerate(arguments): + if index and arguments[index - 1] in PATH_OPTIONS: + paths += [entry for entry in argument.split(os.pathsep) if entry] + elif argument.startswith("-D") and "=" in argument: + paths.append(argument.split("=", 1)[1]) + elif not argument.startswith("-"): + paths.append(argument) + return [os.path.abspath(path) for path in paths if path] + + +def snapshot(paths): + state = {} + for path in paths: + files = tree(path) if os.path.isdir(path) else [path] + for file in files: + try: + stat = os.stat(file) + state[file] = [stat.st_size, stat.st_mtime_ns] + except FileNotFoundError: + state[file] = None + return state + + +def stdin_identity(hashes): + try: + stat = os.fstat(0) + except OSError: + return "closed" + if os.path.samestat(stat, os.stat(os.devnull)) if os.path.exists(os.devnull) else False: + return "null" + import stat as modes + if modes.S_ISREG(stat.st_mode): + return "f:" + hashes.file(os.readlink("/proc/self/fd/0")) + if modes.S_ISCHR(stat.st_mode): + return "tty" + return None # a pipe or socket: its content is unknown, so never cache + + +def run_and_tee(command): + """Run the real program, forwarding and recording its stdout and stderr.""" + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + captured = {"stdout": bytearray(), "stderr": bytearray()} + + def pump(source, sink, buffer): + for chunk in iter(lambda: source.read1(65536), b""): + buffer += chunk + sink.write(chunk) + sink.flush() + + threads = [threading.Thread(target=pump, args=(process.stdout, sys.stdout.buffer, captured["stdout"])), + threading.Thread(target=pump, args=(process.stderr, sys.stderr.buffer, captured["stderr"]))] + for thread in threads: + thread.start() + status = process.wait() + for thread in threads: + thread.join() + return status, captured + + +def files_under(paths): + found = set() + for path in paths: + if os.path.isdir(path): + found.update(tree(path)) + elif os.path.isfile(path): + found.add(path) + return found + + +def reusable(saved, paths, hashes): + """The recorded inputs are unchanged, nothing was added beside them, and the outputs are as left.""" + outputs = saved["outputs"] + if files_under(paths) - set(outputs) != set(saved["inputs"]): + return False + if any(snapshot([path]).get(path) != stamp for path, stamp in outputs.items()): + return False + return all(hashes.file(path) == sha for path, sha in saved["inputs"].items()) + + +def main(): + name = os.path.basename(sys.argv[0]) + arguments = sys.argv[1:] + program = real(name) + if os.environ.get("PARITY_JAVA_CACHE") == "0" or "-version" in arguments or "--version" in arguments: + os.execv(program, [program, *arguments]) + hashes = Hashes() + stdin = stdin_identity(hashes) + if stdin is None: + os.execv(program, [program, *arguments]) + paths = named_paths(arguments) + command = {"program": os.path.realpath(program), "arguments": arguments, "cwd": os.getcwd(), "stdin": stdin} + key = hashlib.sha256(json.dumps(command, sort_keys=True).encode()).hexdigest() + record = CACHE / f"{key}.json" + if record.exists() and reusable(json.loads(record.read_text()), paths, hashes): + sys.stdout.buffer.write((CACHE / f"{key}.stdout").read_bytes()) + sys.stderr.buffer.write((CACHE / f"{key}.stderr").read_bytes()) + return 0 + before = snapshot(paths) + inputs = {path: hashes.file(path) for path in files_under(paths)} + status, captured = run_and_tee([program, *arguments]) + if status != 0: + return status + after = snapshot(paths) + outputs = {path: stamp for path, stamp in after.items() if stamp is not None and before.get(path) != stamp} + for path in outputs: + inputs.pop(path, None) + (CACHE / f"{key}.stdout").write_bytes(captured["stdout"]) + (CACHE / f"{key}.stderr").write_bytes(captured["stderr"]) + temporary = record.with_suffix(f".{os.getpid()}.tmp") + temporary.write_text(json.dumps({"command": command, "inputs": inputs, "outputs": outputs})) + temporary.replace(record) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/test_chunk_generation_parity.py b/tools/test_chunk_generation_parity.py index 8b831d3cd..38e50f9b2 100644 --- a/tools/test_chunk_generation_parity.py +++ b/tools/test_chunk_generation_parity.py @@ -17,6 +17,7 @@ usage: test_chunk_generation_parity.py [--seeds N] [--per-structure K] [--filter REGEX] [--dimension overworld|the_nether|the_end] [--keep] """ import argparse +from concurrent.futures import ThreadPoolExecutor import json import os import random @@ -321,10 +322,13 @@ def main(): tag = f'{dimension}-{seed}-{args.phase}' + ('-iso' if os.environ.get('HORIZON_ISOLATED') else '') + ('-rect' if args.rect else '') vanilla_path, rust_path = build / f'vanilla-{tag}.txt', build / f'rust-{tag}.txt' t0 = time.time() - if not (os.environ.get('REUSE_VANILLA') and vanilla_path.exists()): - oracle.run(vanilla_path, ['dump', seed, dimension, *rects]) - t1 = time.time() - stderr = run_rust(binary, rust_path, seed, dimension, rects) + # Vanilla and Horizon generate the same chunks independently: run them side by side. + with ThreadPoolExecutor(2) as pool: + rust_run = pool.submit(run_rust, binary, rust_path, seed, dimension, rects) + if not (os.environ.get('REUSE_VANILLA') and vanilla_path.exists()): + oracle.run(vanilla_path, ['dump', seed, dimension, *rects]) + t1 = time.time() + stderr = rust_run.result() t2 = time.time() vanilla, rust = parse(vanilla_path), parse(rust_path) for line in stderr.splitlines()[:5]: @@ -342,7 +346,7 @@ def main(): for chunk in vanilla.values(): for row in chunk['q'].values(): biome_seen.update(row) - print(f'{tag}: {len(rects)} sites, {len(vanilla)} chunks, differing chunks {counts[0]} (blocks {counts[1]}, biome cells {counts[2]}, block entities {counts[3]}); vanilla {t1 - t0:.0f}s, horizon {t2 - t1:.0f}s', flush=True) + print(f'{tag}: {len(rects)} sites, {len(vanilla)} chunks, differing chunks {counts[0]} (blocks {counts[1]}, biome cells {counts[2]}, block entities {counts[3]}); vanilla {t1 - t0:.0f}s, both sides {t2 - t0:.0f}s', flush=True) print(f'Compared {totals["compared"]} chunks across {len(biome_seen)} biomes and {sum(structure_seen.values())} structure sites in {time.time() - started:.0f}s') print('structure sites:', dict(sorted(structure_seen.items()))) print(f'Differences: {totals["chunks"]} chunks, {totals["blocks"]} blocks, {totals["biomes"]} biome cells, {totals["entities"]} block entities')