From e8f2ccd403fd147d0851cf0e7d97f7df2adeb453 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:56:28 +0200 Subject: [PATCH 1/3] Fail the gate when the harness runner dies, and clear large harness outputs tools/gate.py took a crashed tools/parity.py for a pass with no new failures. The runner now records each completed run, and the gate fails unless its own run finished. A missing or empty trace (the disk was full) no longer crashes the runner. After a harness passes, the runner deletes its large regenerated outputs (64 MB or more), keeping the vanilla oracle outputs that reusing an oracle run needs. Full suites left 2-4 GB files in every worktree and filled the disk. Co-Authored-By: Claude Opus 5.5 --- tools/gate.py | 17 +++++++++---- tools/parity.py | 60 ++++++++++++++++++++++++++++++++++++++++++-- tools/parity_shim.py | 13 +++++++++- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/tools/gate.py b/tools/gate.py index 2edf17913..538a87892 100644 --- a/tools/gate.py +++ b/tools/gate.py @@ -22,6 +22,7 @@ from pathlib import Path import subprocess import sys import time +import uuid ROOT = Path(__file__).resolve().parents[1] RESULT = ROOT / ".local" / "gate.json" @@ -65,13 +66,19 @@ def main(): print(f"{name}...", end=" ", flush=True) if name == "vanilla harnesses": # Stream the runner's own progress; it keeps per-harness logs. - status = subprocess.run(command, cwd=ROOT).returncode - if status: + run_id = uuid.uuid4().hex + subprocess.run(command, cwd=ROOT, env=dict(os.environ, PARITY_RUN_ID=run_id)) + summary = ROOT / ".local/parity-runner/last-run.json" + finished = json.loads(summary.read_text()) if summary.exists() else {} + if finished.get("run") != run_id: + print("tools/parity.py stopped before finishing its run") + status = 1 + else: known = set(filter(None, args.known_failures.split(","))) - state = json.loads((ROOT / ".local/parity-runner/state.json").read_text())["harnesses"] - failing = {harness for harness, record in state.items() if record["status"] != "pass"} + failing = set(finished["failed"]) new = sorted(failing - known) - print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}") + if failing: + print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}") status = 1 if new else 0 else: with log.open("w") as stream: diff --git a/tools/parity.py b/tools/parity.py index c2b3bd196..cf763bb4c 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -236,6 +236,46 @@ def prebuild(artifacts, jobs): return not errors, errors +LARGE_OUTPUT = 64 << 20 +OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"} + + +def outputs_listing(name, fresh=False): + path = STATE_DIR / "oracle-outputs" / f"{name}.txt" + path.parent.mkdir(parents=True, exist_ok=True) + if fresh: + path.write_text("") + return path + + +def clear_large_outputs(name, trace): + """After a pass, delete the large files a harness regenerates on every run. + + Harness outputs reach gigabytes and used to stay in every worktree. Vanilla + oracle outputs are kept, since reusing an oracle run needs them in place; + everything else large under the harness's .local directories is rebuilt on + its next run.""" + local = LOCAL.resolve() + keep = {os.path.realpath(line) for line in outputs_listing(name).read_text().splitlines() if line} + directories = set() + for command in trace["commands"]: + for argument in command[1:]: + path = Path(os.path.realpath(argument if os.path.isabs(argument) else ROOT / argument)) + try: + top = path.relative_to(local).parts[0] + except (ValueError, IndexError): + continue + if top not in OWN_DIRECTORIES: + directories.add(local / top) + for directory in directories: + for file in (directory.rglob("*") if directory.is_dir() else []): + try: + if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep: + file.unlink() + except OSError: + pass + + def publish(name, record): """Share a pass, so any worktree with the same inputs can reuse it.""" if record["status"] != "pass" or "key" not in record: @@ -298,9 +338,11 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge trace = traces / f"{name}.json" trace.unlink(missing_ok=True) log = (logs / f"{name}.log").open("w") + environment = run_environment(reuse) + environment["PARITY_SHIM_OUTPUTS"] = str(outputs_listing(name, fresh=True)) process = subprocess.Popen( [sys.executable, str(Path(__file__)), "--trace", str(trace), str(selected[name])], - cwd=ROOT, env=run_environment(reuse), stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) running[name] = (process, time.monotonic(), log, expected, reuse, slot) in_use += expected time.sleep(0.2) @@ -327,11 +369,16 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge "finished": time.strftime("%Y-%m-%dT%H:%M:%S")} # A harness that only passes fresh no longer reuses oracle results. record["oracle_reuse"] = records.get(name, {}).get("oracle_reuse", True) and not (passed and name in retried) - if trace_path.exists(): + try: trace = json.loads(trace_path.read_text()) + except (OSError, ValueError): + trace = None # killed before writing it, or the disk was full: the pass can't be keyed + if trace: record["dependencies"], record["artifacts"] = dependencies(trace) record["peak_bytes"] = trace["peak_bytes"] record["key"] = key(record, hasher, environment_digest) + if passed: + clear_large_outputs(name, trace) else: record["dependencies"], record["artifacts"] = [], [] records[name] = record @@ -394,6 +441,12 @@ def trace_main(output, script, arguments): runpy.run_path(script, run_name="__main__") +def finished_run(failed): + """Record that this run completed, for tools/gate.py: a crashed runner never writes it.""" + STATE_DIR.mkdir(parents=True, exist_ok=True) + (STATE_DIR / "last-run.json").write_text(json.dumps({"run": os.environ.get("PARITY_RUN_ID"), "failed": failed})) + + def main(): if len(sys.argv) > 1 and sys.argv[1] == "--trace": return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:]) @@ -439,6 +492,8 @@ def main(): print(name) print(f"{len(selected)} of {len(found)} harnesses would run" if args.list else "Nothing to run: every harness passed with its current inputs.") save_state(state) + if not args.list: + finished_run([]) return 0 jobs = max(1, args.jobs) @@ -460,6 +515,7 @@ def main(): reuse_oracles) save_state(state) failed = sorted(name for name, record in results.items() if record["status"] != "pass") + finished_run(failed) print(f"\n{len(results) - len(failed)} passed, {len(failed)} failed in {elapsed / 60:.1f} min.") for name in failed: print(f" {name}: {STATE_DIR / 'logs' / (name + '.log')}") diff --git a/tools/parity_shim.py b/tools/parity_shim.py index 8cff84714..f69c71faa 100755 --- a/tools/parity_shim.py +++ b/tools/parity_shim.py @@ -140,6 +140,14 @@ def files_under(paths): return found +def note_outputs(outputs): + """Tell tools/parity.py which files are oracle outputs, which it keeps when clearing large outputs.""" + listing = os.environ.get("PARITY_SHIM_OUTPUTS") + if listing: + with open(listing, "a") as stream: + stream.writelines(f"{path}\n" for path in outputs) + + def reusable(saved, paths, hashes): """The recorded inputs are unchanged, nothing was added beside them, and the outputs are as left.""" outputs = saved["outputs"] @@ -164,7 +172,9 @@ def main(): command = {"program": os.path.realpath(program), "arguments": arguments, "cwd": os.getcwd(), "stdin": stdin} key = hashlib.sha256(json.dumps(command, sort_keys=True).encode()).hexdigest() record = CACHE / f"{key}.json" - if record.exists() and reusable(json.loads(record.read_text()), paths, hashes): + saved = json.loads(record.read_text()) if record.exists() else None + if saved and reusable(saved, paths, hashes): + note_outputs(saved["outputs"]) sys.stdout.buffer.write((CACHE / f"{key}.stdout").read_bytes()) sys.stderr.buffer.write((CACHE / f"{key}.stderr").read_bytes()) return 0 @@ -182,6 +192,7 @@ def main(): temporary = record.with_suffix(f".{os.getpid()}.tmp") temporary.write_text(json.dumps({"command": command, "inputs": inputs, "outputs": outputs})) temporary.replace(record) + note_outputs(outputs) return 0 From 5ede186d4074813cad181b1bad1d922fa0873f91 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:56:43 +0200 Subject: [PATCH 2/3] Integrate several branches behind one gate Co-Authored-By: Claude Opus 5.5 --- tools/supervise.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tools/supervise.py b/tools/supervise.py index 954cc76aa..213150aba 100644 --- a/tools/supervise.py +++ b/tools/supervise.py @@ -5,6 +5,7 @@ python3 tools/supervise.py new dispensers # worktree ../dispensers on parity/dispensers, from the integration branch python3 tools/supervise.py save [AREA ...] # commit and push unfinished work (before stopping agents) python3 tools/supervise.py integrate parity/dispensers # merge into the integration branch if the gate passes + python3 tools/supervise.py integrate parity/a parity/b # several branches, one gate The integration branch (`parity-base` by default) is checked out in its own worktree; `integrate` merges there, runs tools/gate.py and pushes the @@ -117,24 +118,27 @@ def integrate(args): state = base_tree / ".local/parity-runner/state.json" known = sorted(name for name, record in json.loads(state.read_text())["harnesses"].items() if record["status"] != "pass") if state.exists() else [] - merge = subprocess.run(["git", "merge", "--no-ff", "--no-edit", args.branch], cwd=base_tree, capture_output=True, text=True) - if merge.returncode: - conflicts = git("diff", "--name-only", "--diff-filter=U", cwd=base_tree, check=False) - git("merge", "--abort", cwd=base_tree, check=False) - print(f"{args.branch} does not merge cleanly into {args.base}. Conflicts:\n{conflicts or merge.stdout + merge.stderr}") - print(f"Merge {args.base} into {args.branch}, resolve them there, and integrate again.") - return 2 + for branch in args.branches: + merge = subprocess.run(["git", "merge", "--no-ff", "--no-edit", branch], cwd=base_tree, capture_output=True, text=True) + if merge.returncode: + conflicts = git("diff", "--name-only", "--diff-filter=U", cwd=base_tree, check=False) + git("merge", "--abort", cwd=base_tree, check=False) + git("reset", "--hard", before, cwd=base_tree) + print(f"{branch} does not merge cleanly into {args.base}. Conflicts:\n{conflicts or merge.stdout + merge.stderr}") + print(f"Merge {args.base} into {branch}, resolve them there, and integrate again. Nothing was integrated.") + return 2 + names = ", ".join(args.branches) if git("rev-parse", "HEAD", cwd=base_tree) == before: - print(f"{args.branch} has nothing new for {args.base}.") + print(f"{names}: nothing new for {args.base}.") return 0 gate = [sys.executable, "tools/gate.py", "-j", str(args.jobs), "--known-failures", ",".join(known)] gate += ["--no-parity"] if args.no_parity else [] if subprocess.run(gate, cwd=base_tree).returncode: git("reset", "--hard", before, cwd=base_tree) - print(f"The gate failed, so {args.branch} was not integrated; {args.base} is back at {before[:9]}.") + print(f"The gate failed, so {names} were not integrated; {args.base} is back at {before[:9]}.") return 1 git("push", "-q", "origin", f"{args.base}:{args.push_to}", cwd=base_tree) - print(f"Integrated {args.branch}; {args.base} pushed to origin/{args.push_to} at {git('rev-parse', '--short', 'HEAD', cwd=base_tree)}.") + print(f"Integrated {names}; {args.base} pushed to origin/{args.push_to} at {git('rev-parse', '--short', 'HEAD', cwd=base_tree)}.") return 0 @@ -151,7 +155,7 @@ def main(): command.add_argument("areas", nargs="*") command.set_defaults(run=save) command = commands.add_parser("integrate") - command.add_argument("branch") + command.add_argument("branches", nargs="+", help="merged in order, then gated together") command.add_argument("-j", "--jobs", type=int, default=8) command.add_argument("--no-parity", action="store_true", help="skip the vanilla harnesses in the gate") command.set_defaults(run=integrate) From d572cd726262994875bba2c6dc3cba2a64d31389 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:38:16 +0200 Subject: [PATCH 3/3] Link only the vanilla data into new worktrees Co-Authored-By: Claude Opus 5.5 --- tools/supervise.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/supervise.py b/tools/supervise.py index 213150aba..6886a15ca 100644 --- a/tools/supervise.py +++ b/tools/supervise.py @@ -87,8 +87,10 @@ def new(args): git("worktree", "add", "-b", branch, str(path), args.base) shared = base_tree.parent / "shared-local" (path / ".local").mkdir() + # Link the vanilla data, not the shared runner and datafix state beside it. for entry in sorted(shared.iterdir()) if shared.is_dir() else []: - (path / ".local" / entry.name).symlink_to(entry) + if entry.is_dir() and entry.name not in {"parity-shared", "datafix"}: + (path / ".local" / entry.name).symlink_to(entry) print(f"{path} on {branch}, from {args.base} at {git('rev-parse', '--short', 'HEAD', cwd=path)}")