From e8f2ccd403fd147d0851cf0e7d97f7df2adeb453 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:56:28 +0200 Subject: [PATCH] Fail the gate when the harness runner dies, and clear large harness outputs tools/gate.py took a crashed tools/parity.py for a pass with no new failures. The runner now records each completed run, and the gate fails unless its own run finished. A missing or empty trace (the disk was full) no longer crashes the runner. After a harness passes, the runner deletes its large regenerated outputs (64 MB or more), keeping the vanilla oracle outputs that reusing an oracle run needs. Full suites left 2-4 GB files in every worktree and filled the disk. Co-Authored-By: Claude Opus 5.5 --- tools/gate.py | 17 +++++++++---- tools/parity.py | 60 ++++++++++++++++++++++++++++++++++++++++++-- tools/parity_shim.py | 13 +++++++++- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/tools/gate.py b/tools/gate.py index 2edf17913..538a87892 100644 --- a/tools/gate.py +++ b/tools/gate.py @@ -22,6 +22,7 @@ from pathlib import Path import subprocess import sys import time +import uuid ROOT = Path(__file__).resolve().parents[1] RESULT = ROOT / ".local" / "gate.json" @@ -65,13 +66,19 @@ def main(): print(f"{name}...", end=" ", flush=True) if name == "vanilla harnesses": # Stream the runner's own progress; it keeps per-harness logs. - status = subprocess.run(command, cwd=ROOT).returncode - if status: + run_id = uuid.uuid4().hex + subprocess.run(command, cwd=ROOT, env=dict(os.environ, PARITY_RUN_ID=run_id)) + summary = ROOT / ".local/parity-runner/last-run.json" + finished = json.loads(summary.read_text()) if summary.exists() else {} + if finished.get("run") != run_id: + print("tools/parity.py stopped before finishing its run") + status = 1 + else: known = set(filter(None, args.known_failures.split(","))) - state = json.loads((ROOT / ".local/parity-runner/state.json").read_text())["harnesses"] - failing = {harness for harness, record in state.items() if record["status"] != "pass"} + failing = set(finished["failed"]) new = sorted(failing - known) - print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}") + if failing: + print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}") status = 1 if new else 0 else: with log.open("w") as stream: diff --git a/tools/parity.py b/tools/parity.py index c2b3bd196..cf763bb4c 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -236,6 +236,46 @@ def prebuild(artifacts, jobs): return not errors, errors +LARGE_OUTPUT = 64 << 20 +OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"} + + +def outputs_listing(name, fresh=False): + path = STATE_DIR / "oracle-outputs" / f"{name}.txt" + path.parent.mkdir(parents=True, exist_ok=True) + if fresh: + path.write_text("") + return path + + +def clear_large_outputs(name, trace): + """After a pass, delete the large files a harness regenerates on every run. + + Harness outputs reach gigabytes and used to stay in every worktree. Vanilla + oracle outputs are kept, since reusing an oracle run needs them in place; + everything else large under the harness's .local directories is rebuilt on + its next run.""" + local = LOCAL.resolve() + keep = {os.path.realpath(line) for line in outputs_listing(name).read_text().splitlines() if line} + directories = set() + for command in trace["commands"]: + for argument in command[1:]: + path = Path(os.path.realpath(argument if os.path.isabs(argument) else ROOT / argument)) + try: + top = path.relative_to(local).parts[0] + except (ValueError, IndexError): + continue + if top not in OWN_DIRECTORIES: + directories.add(local / top) + for directory in directories: + for file in (directory.rglob("*") if directory.is_dir() else []): + try: + if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep: + file.unlink() + except OSError: + pass + + def publish(name, record): """Share a pass, so any worktree with the same inputs can reuse it.""" if record["status"] != "pass" or "key" not in record: @@ -298,9 +338,11 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge trace = traces / f"{name}.json" trace.unlink(missing_ok=True) log = (logs / f"{name}.log").open("w") + environment = run_environment(reuse) + environment["PARITY_SHIM_OUTPUTS"] = str(outputs_listing(name, fresh=True)) process = subprocess.Popen( [sys.executable, str(Path(__file__)), "--trace", str(trace), str(selected[name])], - cwd=ROOT, env=run_environment(reuse), stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) running[name] = (process, time.monotonic(), log, expected, reuse, slot) in_use += expected time.sleep(0.2) @@ -327,11 +369,16 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge "finished": time.strftime("%Y-%m-%dT%H:%M:%S")} # A harness that only passes fresh no longer reuses oracle results. record["oracle_reuse"] = records.get(name, {}).get("oracle_reuse", True) and not (passed and name in retried) - if trace_path.exists(): + try: trace = json.loads(trace_path.read_text()) + except (OSError, ValueError): + trace = None # killed before writing it, or the disk was full: the pass can't be keyed + if trace: record["dependencies"], record["artifacts"] = dependencies(trace) record["peak_bytes"] = trace["peak_bytes"] record["key"] = key(record, hasher, environment_digest) + if passed: + clear_large_outputs(name, trace) else: record["dependencies"], record["artifacts"] = [], [] records[name] = record @@ -394,6 +441,12 @@ def trace_main(output, script, arguments): runpy.run_path(script, run_name="__main__") +def finished_run(failed): + """Record that this run completed, for tools/gate.py: a crashed runner never writes it.""" + STATE_DIR.mkdir(parents=True, exist_ok=True) + (STATE_DIR / "last-run.json").write_text(json.dumps({"run": os.environ.get("PARITY_RUN_ID"), "failed": failed})) + + def main(): if len(sys.argv) > 1 and sys.argv[1] == "--trace": return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:]) @@ -439,6 +492,8 @@ def main(): print(name) print(f"{len(selected)} of {len(found)} harnesses would run" if args.list else "Nothing to run: every harness passed with its current inputs.") save_state(state) + if not args.list: + finished_run([]) return 0 jobs = max(1, args.jobs) @@ -460,6 +515,7 @@ def main(): reuse_oracles) save_state(state) failed = sorted(name for name, record in results.items() if record["status"] != "pass") + finished_run(failed) print(f"\n{len(results) - len(failed)} passed, {len(failed)} failed in {elapsed / 60:.1f} min.") for name in failed: print(f" {name}: {STATE_DIR / 'logs' / (name + '.log')}") diff --git a/tools/parity_shim.py b/tools/parity_shim.py index 8cff84714..f69c71faa 100755 --- a/tools/parity_shim.py +++ b/tools/parity_shim.py @@ -140,6 +140,14 @@ def files_under(paths): return found +def note_outputs(outputs): + """Tell tools/parity.py which files are oracle outputs, which it keeps when clearing large outputs.""" + listing = os.environ.get("PARITY_SHIM_OUTPUTS") + if listing: + with open(listing, "a") as stream: + stream.writelines(f"{path}\n" for path in outputs) + + def reusable(saved, paths, hashes): """The recorded inputs are unchanged, nothing was added beside them, and the outputs are as left.""" outputs = saved["outputs"] @@ -164,7 +172,9 @@ def main(): command = {"program": os.path.realpath(program), "arguments": arguments, "cwd": os.getcwd(), "stdin": stdin} key = hashlib.sha256(json.dumps(command, sort_keys=True).encode()).hexdigest() record = CACHE / f"{key}.json" - if record.exists() and reusable(json.loads(record.read_text()), paths, hashes): + saved = json.loads(record.read_text()) if record.exists() else None + if saved and reusable(saved, paths, hashes): + note_outputs(saved["outputs"]) sys.stdout.buffer.write((CACHE / f"{key}.stdout").read_bytes()) sys.stderr.buffer.write((CACHE / f"{key}.stderr").read_bytes()) return 0 @@ -182,6 +192,7 @@ def main(): temporary = record.with_suffix(f".{os.getpid()}.tmp") temporary.write_text(json.dumps({"command": command, "inputs": inputs, "outputs": outputs})) temporary.replace(record) + note_outputs(outputs) return 0