Merge remote-tracking branch 'origin/t3/achieve-full-parity' into parity/entity-ai

This commit is contained in:
Ludovic Marie
2026-10-12 12:43:08 +02:00
4 changed files with 100 additions and 20 deletions
+12 -5
View File
@@ -22,6 +22,7 @@ from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
RESULT = ROOT / ".local" / "gate.json"
@@ -65,13 +66,19 @@ def main():
print(f"{name}...", end=" ", flush=True)
if name == "vanilla harnesses":
# Stream the runner's own progress; it keeps per-harness logs.
status = subprocess.run(command, cwd=ROOT).returncode
if status:
run_id = uuid.uuid4().hex
subprocess.run(command, cwd=ROOT, env=dict(os.environ, PARITY_RUN_ID=run_id))
summary = ROOT / ".local/parity-runner/last-run.json"
finished = json.loads(summary.read_text()) if summary.exists() else {}
if finished.get("run") != run_id:
print("tools/parity.py stopped before finishing its run")
status = 1
else:
known = set(filter(None, args.known_failures.split(",")))
state = json.loads((ROOT / ".local/parity-runner/state.json").read_text())["harnesses"]
failing = {harness for harness, record in state.items() if record["status"] != "pass"}
failing = set(finished["failed"])
new = sorted(failing - known)
print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}")
if failing:
print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}")
status = 1 if new else 0
else:
with log.open("w") as stream:
+58 -2
View File
@@ -236,6 +236,46 @@ def prebuild(artifacts, jobs):
return not errors, errors
LARGE_OUTPUT = 64 << 20
OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"}
def outputs_listing(name, fresh=False):
path = STATE_DIR / "oracle-outputs" / f"{name}.txt"
path.parent.mkdir(parents=True, exist_ok=True)
if fresh:
path.write_text("")
return path
def clear_large_outputs(name, trace):
"""After a pass, delete the large files a harness regenerates on every run.
Harness outputs reach gigabytes and used to stay in every worktree. Vanilla
oracle outputs are kept, since reusing an oracle run needs them in place;
everything else large under the harness's .local directories is rebuilt on
its next run."""
local = LOCAL.resolve()
keep = {os.path.realpath(line) for line in outputs_listing(name).read_text().splitlines() if line}
directories = set()
for command in trace["commands"]:
for argument in command[1:]:
path = Path(os.path.realpath(argument if os.path.isabs(argument) else ROOT / argument))
try:
top = path.relative_to(local).parts[0]
except (ValueError, IndexError):
continue
if top not in OWN_DIRECTORIES:
directories.add(local / top)
for directory in directories:
for file in (directory.rglob("*") if directory.is_dir() else []):
try:
if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep:
file.unlink()
except OSError:
pass
def publish(name, record):
"""Share a pass, so any worktree with the same inputs can reuse it."""
if record["status"] != "pass" or "key" not in record:
@@ -298,9 +338,11 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
trace = traces / f"{name}.json"
trace.unlink(missing_ok=True)
log = (logs / f"{name}.log").open("w")
environment = run_environment(reuse)
environment["PARITY_SHIM_OUTPUTS"] = str(outputs_listing(name, fresh=True))
process = subprocess.Popen(
[sys.executable, str(Path(__file__)), "--trace", str(trace), str(selected[name])],
cwd=ROOT, env=run_environment(reuse), stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
running[name] = (process, time.monotonic(), log, expected, reuse, slot)
in_use += expected
time.sleep(0.2)
@@ -327,11 +369,16 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
"finished": time.strftime("%Y-%m-%dT%H:%M:%S")}
# A harness that only passes fresh no longer reuses oracle results.
record["oracle_reuse"] = records.get(name, {}).get("oracle_reuse", True) and not (passed and name in retried)
if trace_path.exists():
try:
trace = json.loads(trace_path.read_text())
except (OSError, ValueError):
trace = None # killed before writing it, or the disk was full: the pass can't be keyed
if trace:
record["dependencies"], record["artifacts"] = dependencies(trace)
record["peak_bytes"] = trace["peak_bytes"]
record["key"] = key(record, hasher, environment_digest)
if passed:
clear_large_outputs(name, trace)
else:
record["dependencies"], record["artifacts"] = [], []
records[name] = record
@@ -394,6 +441,12 @@ def trace_main(output, script, arguments):
runpy.run_path(script, run_name="__main__")
def finished_run(failed):
"""Record that this run completed, for tools/gate.py: a crashed runner never writes it."""
STATE_DIR.mkdir(parents=True, exist_ok=True)
(STATE_DIR / "last-run.json").write_text(json.dumps({"run": os.environ.get("PARITY_RUN_ID"), "failed": failed}))
def main():
if len(sys.argv) > 1 and sys.argv[1] == "--trace":
return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:])
@@ -439,6 +492,8 @@ def main():
print(name)
print(f"{len(selected)} of {len(found)} harnesses would run" if args.list else "Nothing to run: every harness passed with its current inputs.")
save_state(state)
if not args.list:
finished_run([])
return 0
jobs = max(1, args.jobs)
@@ -460,6 +515,7 @@ def main():
reuse_oracles)
save_state(state)
failed = sorted(name for name, record in results.items() if record["status"] != "pass")
finished_run(failed)
print(f"\n{len(results) - len(failed)} passed, {len(failed)} failed in {elapsed / 60:.1f} min.")
for name in failed:
print(f" {name}: {STATE_DIR / 'logs' / (name + '.log')}")
+12 -1
View File
@@ -140,6 +140,14 @@ def files_under(paths):
return found
def note_outputs(outputs):
"""Tell tools/parity.py which files are oracle outputs, which it keeps when clearing large outputs."""
listing = os.environ.get("PARITY_SHIM_OUTPUTS")
if listing:
with open(listing, "a") as stream:
stream.writelines(f"{path}\n" for path in outputs)
def reusable(saved, paths, hashes):
"""The recorded inputs are unchanged, nothing was added beside them, and the outputs are as left."""
outputs = saved["outputs"]
@@ -164,7 +172,9 @@ def main():
command = {"program": os.path.realpath(program), "arguments": arguments, "cwd": os.getcwd(), "stdin": stdin}
key = hashlib.sha256(json.dumps(command, sort_keys=True).encode()).hexdigest()
record = CACHE / f"{key}.json"
if record.exists() and reusable(json.loads(record.read_text()), paths, hashes):
saved = json.loads(record.read_text()) if record.exists() else None
if saved and reusable(saved, paths, hashes):
note_outputs(saved["outputs"])
sys.stdout.buffer.write((CACHE / f"{key}.stdout").read_bytes())
sys.stderr.buffer.write((CACHE / f"{key}.stderr").read_bytes())
return 0
@@ -182,6 +192,7 @@ def main():
temporary = record.with_suffix(f".{os.getpid()}.tmp")
temporary.write_text(json.dumps({"command": command, "inputs": inputs, "outputs": outputs}))
temporary.replace(record)
note_outputs(outputs)
return 0
+18 -12
View File
@@ -5,6 +5,7 @@
python3 tools/supervise.py new dispensers # worktree ../dispensers on parity/dispensers, from the integration branch
python3 tools/supervise.py save [AREA ...] # commit and push unfinished work (before stopping agents)
python3 tools/supervise.py integrate parity/dispensers # merge into the integration branch if the gate passes
python3 tools/supervise.py integrate parity/a parity/b # several branches, one gate
The integration branch (`parity-base` by default) is checked out in its own
worktree; `integrate` merges there, runs tools/gate.py and pushes the
@@ -86,8 +87,10 @@ def new(args):
git("worktree", "add", "-b", branch, str(path), args.base)
shared = base_tree.parent / "shared-local"
(path / ".local").mkdir()
# Link the vanilla data, not the shared runner and datafix state beside it.
for entry in sorted(shared.iterdir()) if shared.is_dir() else []:
(path / ".local" / entry.name).symlink_to(entry)
if entry.is_dir() and entry.name not in {"parity-shared", "datafix"}:
(path / ".local" / entry.name).symlink_to(entry)
print(f"{path} on {branch}, from {args.base} at {git('rev-parse', '--short', 'HEAD', cwd=path)}")
@@ -117,24 +120,27 @@ def integrate(args):
state = base_tree / ".local/parity-runner/state.json"
known = sorted(name for name, record in json.loads(state.read_text())["harnesses"].items()
if record["status"] != "pass") if state.exists() else []
merge = subprocess.run(["git", "merge", "--no-ff", "--no-edit", args.branch], cwd=base_tree, capture_output=True, text=True)
if merge.returncode:
conflicts = git("diff", "--name-only", "--diff-filter=U", cwd=base_tree, check=False)
git("merge", "--abort", cwd=base_tree, check=False)
print(f"{args.branch} does not merge cleanly into {args.base}. Conflicts:\n{conflicts or merge.stdout + merge.stderr}")
print(f"Merge {args.base} into {args.branch}, resolve them there, and integrate again.")
return 2
for branch in args.branches:
merge = subprocess.run(["git", "merge", "--no-ff", "--no-edit", branch], cwd=base_tree, capture_output=True, text=True)
if merge.returncode:
conflicts = git("diff", "--name-only", "--diff-filter=U", cwd=base_tree, check=False)
git("merge", "--abort", cwd=base_tree, check=False)
git("reset", "--hard", before, cwd=base_tree)
print(f"{branch} does not merge cleanly into {args.base}. Conflicts:\n{conflicts or merge.stdout + merge.stderr}")
print(f"Merge {args.base} into {branch}, resolve them there, and integrate again. Nothing was integrated.")
return 2
names = ", ".join(args.branches)
if git("rev-parse", "HEAD", cwd=base_tree) == before:
print(f"{args.branch} has nothing new for {args.base}.")
print(f"{names}: nothing new for {args.base}.")
return 0
gate = [sys.executable, "tools/gate.py", "-j", str(args.jobs), "--known-failures", ",".join(known)]
gate += ["--no-parity"] if args.no_parity else []
if subprocess.run(gate, cwd=base_tree).returncode:
git("reset", "--hard", before, cwd=base_tree)
print(f"The gate failed, so {args.branch} was not integrated; {args.base} is back at {before[:9]}.")
print(f"The gate failed, so {names} were not integrated; {args.base} is back at {before[:9]}.")
return 1
git("push", "-q", "origin", f"{args.base}:{args.push_to}", cwd=base_tree)
print(f"Integrated {args.branch}; {args.base} pushed to origin/{args.push_to} at {git('rev-parse', '--short', 'HEAD', cwd=base_tree)}.")
print(f"Integrated {names}; {args.base} pushed to origin/{args.push_to} at {git('rev-parse', '--short', 'HEAD', cwd=base_tree)}.")
return 0
@@ -151,7 +157,7 @@ def main():
command.add_argument("areas", nargs="*")
command.set_defaults(run=save)
command = commands.add_parser("integrate")
command.add_argument("branch")
command.add_argument("branches", nargs="+", help="merged in order, then gated together")
command.add_argument("-j", "--jobs", type=int, default=8)
command.add_argument("--no-parity", action="store_true", help="skip the vanilla harnesses in the gate")
command.set_defaults(run=integrate)