chexit/horizon-airdef
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fail the gate when the harness runner dies, and clear large harness outputs

Browse Source 
tools/gate.py took a crashed tools/parity.py for a pass with no new
failures. The runner now records each completed run, and the gate fails
unless its own run finished. A missing or empty trace (the disk was full)
no longer crashes the runner.

After a harness passes, the runner deletes its large regenerated outputs
(64 MB or more), keeping the vanilla oracle outputs that reusing an oracle
run needs. Full suites left 2-4 GB files in every worktree and filled the
disk.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-11 18:56:28 +02:00
parent 168bc1d0a3
commit e8f2ccd403
3 changed files with 82 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/gate.py
+12 -5
View File
@@ -22,6 +22,7 @@ from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
RESULT = ROOT / ".local" / "gate.json"
@@ -65,13 +66,19 @@ def main():
print(f"{name}...", end=" ", flush=True)
if name == "vanilla harnesses":
# Stream the runner's own progress; it keeps per-harness logs.
status = subprocess.run(command, cwd=ROOT).returncode
if status:
run_id = uuid.uuid4().hex
subprocess.run(command, cwd=ROOT, env=dict(os.environ, PARITY_RUN_ID=run_id))
summary = ROOT / ".local/parity-runner/last-run.json"
finished = json.loads(summary.read_text()) if summary.exists() else {}
if finished.get("run") != run_id:
print("tools/parity.py stopped before finishing its run")
status = 1
else:
known = set(filter(None, args.known_failures.split(",")))
state = json.loads((ROOT / ".local/parity-runner/state.json").read_text())["harnesses"]
failing = {harness for harness, record in state.items() if record["status"] != "pass"}
failing = set(finished["failed"])
new = sorted(failing - known)
print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}")
if failing:
print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}")
status = 1 if new else 0
else:
with log.open("w") as stream:
tools/parity.py
+58 -2
View File
@@ -236,6 +236,46 @@ def prebuild(artifacts, jobs):
return not errors, errors
LARGE_OUTPUT = 64 << 20
OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"}
def outputs_listing(name, fresh=False):
path = STATE_DIR / "oracle-outputs" / f"{name}.txt"
path.parent.mkdir(parents=True, exist_ok=True)
if fresh:
path.write_text("")
return path
def clear_large_outputs(name, trace):
"""After a pass, delete the large files a harness regenerates on every run.
Harness outputs reach gigabytes and used to stay in every worktree. Vanilla
oracle outputs are kept, since reusing an oracle run needs them in place;
everything else large under the harness's .local directories is rebuilt on
its next run."""
local = LOCAL.resolve()
keep = {os.path.realpath(line) for line in outputs_listing(name).read_text().splitlines() if line}
directories = set()
for command in trace["commands"]:
for argument in command[1:]:
path = Path(os.path.realpath(argument if os.path.isabs(argument) else ROOT / argument))
try:
top = path.relative_to(local).parts[0]
except (ValueError, IndexError):
continue
if top not in OWN_DIRECTORIES:
directories.add(local / top)
for directory in directories:
for file in (directory.rglob("*") if directory.is_dir() else []):
try:
if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep:
file.unlink()
except OSError:
pass
def publish(name, record):
"""Share a pass, so any worktree with the same inputs can reuse it."""
if record["status"] != "pass" or "key" not in record:
@@ -298,9 +338,11 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
trace = traces / f"{name}.json"
trace.unlink(missing_ok=True)
log = (logs / f"{name}.log").open("w")
environment = run_environment(reuse)
environment["PARITY_SHIM_OUTPUTS"] = str(outputs_listing(name, fresh=True))
process = subprocess.Popen(
[sys.executable, str(Path(__file__)), "--trace", str(trace), str(selected[name])],
cwd=ROOT, env=run_environment(reuse), stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
running[name] = (process, time.monotonic(), log, expected, reuse, slot)
in_use += expected
time.sleep(0.2)
@@ -327,11 +369,16 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
"finished": time.strftime("%Y-%m-%dT%H:%M:%S")}
# A harness that only passes fresh no longer reuses oracle results.
record["oracle_reuse"] = records.get(name, {}).get("oracle_reuse", True) and not (passed and name in retried)
if trace_path.exists():
try:
trace = json.loads(trace_path.read_text())
except (OSError, ValueError):
trace = None # killed before writing it, or the disk was full: the pass can't be keyed
if trace:
record["dependencies"], record["artifacts"] = dependencies(trace)
record["peak_bytes"] = trace["peak_bytes"]
record["key"] = key(record, hasher, environment_digest)
if passed:
clear_large_outputs(name, trace)
else:
record["dependencies"], record["artifacts"] = [], []
records[name] = record
@@ -394,6 +441,12 @@ def trace_main(output, script, arguments):
runpy.run_path(script, run_name="__main__")
def finished_run(failed):
"""Record that this run completed, for tools/gate.py: a crashed runner never writes it."""
STATE_DIR.mkdir(parents=True, exist_ok=True)
(STATE_DIR / "last-run.json").write_text(json.dumps({"run": os.environ.get("PARITY_RUN_ID"), "failed": failed}))
def main():
if len(sys.argv) > 1 and sys.argv[1] == "--trace":
return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:])
@@ -439,6 +492,8 @@ def main():
print(name)
print(f"{len(selected)} of {len(found)} harnesses would run" if args.list else "Nothing to run: every harness passed with its current inputs.")
save_state(state)
if not args.list:
finished_run([])
return 0
jobs = max(1, args.jobs)
@@ -460,6 +515,7 @@ def main():
reuse_oracles)
save_state(state)
failed = sorted(name for name, record in results.items() if record["status"] != "pass")
finished_run(failed)
print(f"\n{len(results) - len(failed)} passed, {len(failed)} failed in {elapsed / 60:.1f} min.")
for name in failed:
print(f" {name}: {STATE_DIR / 'logs' / (name + '.log')}")
tools/parity_shim.py
+12 -1
View File
@@ -140,6 +140,14 @@ def files_under(paths):
return found
def note_outputs(outputs):
"""Tell tools/parity.py which files are oracle outputs, which it keeps when clearing large outputs."""
listing = os.environ.get("PARITY_SHIM_OUTPUTS")
if listing:
with open(listing, "a") as stream:
stream.writelines(f"{path}\n" for path in outputs)
def reusable(saved, paths, hashes):
"""The recorded inputs are unchanged, nothing was added beside them, and the outputs are as left."""
outputs = saved["outputs"]
@@ -164,7 +172,9 @@ def main():
command = {"program": os.path.realpath(program), "arguments": arguments, "cwd": os.getcwd(), "stdin": stdin}
key = hashlib.sha256(json.dumps(command, sort_keys=True).encode()).hexdigest()
record = CACHE / f"{key}.json"
if record.exists() and reusable(json.loads(record.read_text()), paths, hashes):
saved = json.loads(record.read_text()) if record.exists() else None
if saved and reusable(saved, paths, hashes):
note_outputs(saved["outputs"])
sys.stdout.buffer.write((CACHE / f"{key}.stdout").read_bytes())
sys.stderr.buffer.write((CACHE / f"{key}.stderr").read_bytes())
return 0
@@ -182,6 +192,7 @@ def main():
temporary = record.with_suffix(f".{os.getpid()}.tmp")
temporary.write_text(json.dumps({"command": command, "inputs": inputs, "outputs": outputs}))
temporary.replace(record)
note_outputs(outputs)
return 0