Merge remote-tracking branch 'origin/t3/achieve-full-parity' into parity/entity-ai
This commit is contained in:
+12
-5
@@ -22,6 +22,7 @@ from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
RESULT = ROOT / ".local" / "gate.json"
|
||||
@@ -65,13 +66,19 @@ def main():
|
||||
print(f"{name}...", end=" ", flush=True)
|
||||
if name == "vanilla harnesses":
|
||||
# Stream the runner's own progress; it keeps per-harness logs.
|
||||
status = subprocess.run(command, cwd=ROOT).returncode
|
||||
if status:
|
||||
run_id = uuid.uuid4().hex
|
||||
subprocess.run(command, cwd=ROOT, env=dict(os.environ, PARITY_RUN_ID=run_id))
|
||||
summary = ROOT / ".local/parity-runner/last-run.json"
|
||||
finished = json.loads(summary.read_text()) if summary.exists() else {}
|
||||
if finished.get("run") != run_id:
|
||||
print("tools/parity.py stopped before finishing its run")
|
||||
status = 1
|
||||
else:
|
||||
known = set(filter(None, args.known_failures.split(",")))
|
||||
state = json.loads((ROOT / ".local/parity-runner/state.json").read_text())["harnesses"]
|
||||
failing = {harness for harness, record in state.items() if record["status"] != "pass"}
|
||||
failing = set(finished["failed"])
|
||||
new = sorted(failing - known)
|
||||
print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}")
|
||||
if failing:
|
||||
print(f"Failing harnesses: {', '.join(sorted(failing))}; new: {', '.join(new) or 'none'}")
|
||||
status = 1 if new else 0
|
||||
else:
|
||||
with log.open("w") as stream:
|
||||
|
||||
+58
-2
@@ -236,6 +236,46 @@ def prebuild(artifacts, jobs):
|
||||
return not errors, errors
|
||||
|
||||
|
||||
LARGE_OUTPUT = 64 << 20
|
||||
OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"}
|
||||
|
||||
|
||||
def outputs_listing(name, fresh=False):
|
||||
path = STATE_DIR / "oracle-outputs" / f"{name}.txt"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if fresh:
|
||||
path.write_text("")
|
||||
return path
|
||||
|
||||
|
||||
def clear_large_outputs(name, trace):
|
||||
"""After a pass, delete the large files a harness regenerates on every run.
|
||||
|
||||
Harness outputs reach gigabytes and used to stay in every worktree. Vanilla
|
||||
oracle outputs are kept, since reusing an oracle run needs them in place;
|
||||
everything else large under the harness's .local directories is rebuilt on
|
||||
its next run."""
|
||||
local = LOCAL.resolve()
|
||||
keep = {os.path.realpath(line) for line in outputs_listing(name).read_text().splitlines() if line}
|
||||
directories = set()
|
||||
for command in trace["commands"]:
|
||||
for argument in command[1:]:
|
||||
path = Path(os.path.realpath(argument if os.path.isabs(argument) else ROOT / argument))
|
||||
try:
|
||||
top = path.relative_to(local).parts[0]
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
if top not in OWN_DIRECTORIES:
|
||||
directories.add(local / top)
|
||||
for directory in directories:
|
||||
for file in (directory.rglob("*") if directory.is_dir() else []):
|
||||
try:
|
||||
if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep:
|
||||
file.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def publish(name, record):
|
||||
"""Share a pass, so any worktree with the same inputs can reuse it."""
|
||||
if record["status"] != "pass" or "key" not in record:
|
||||
@@ -298,9 +338,11 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
|
||||
trace = traces / f"{name}.json"
|
||||
trace.unlink(missing_ok=True)
|
||||
log = (logs / f"{name}.log").open("w")
|
||||
environment = run_environment(reuse)
|
||||
environment["PARITY_SHIM_OUTPUTS"] = str(outputs_listing(name, fresh=True))
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, str(Path(__file__)), "--trace", str(trace), str(selected[name])],
|
||||
cwd=ROOT, env=run_environment(reuse), stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
running[name] = (process, time.monotonic(), log, expected, reuse, slot)
|
||||
in_use += expected
|
||||
time.sleep(0.2)
|
||||
@@ -327,11 +369,16 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
|
||||
"finished": time.strftime("%Y-%m-%dT%H:%M:%S")}
|
||||
# A harness that only passes fresh no longer reuses oracle results.
|
||||
record["oracle_reuse"] = records.get(name, {}).get("oracle_reuse", True) and not (passed and name in retried)
|
||||
if trace_path.exists():
|
||||
try:
|
||||
trace = json.loads(trace_path.read_text())
|
||||
except (OSError, ValueError):
|
||||
trace = None # killed before writing it, or the disk was full: the pass can't be keyed
|
||||
if trace:
|
||||
record["dependencies"], record["artifacts"] = dependencies(trace)
|
||||
record["peak_bytes"] = trace["peak_bytes"]
|
||||
record["key"] = key(record, hasher, environment_digest)
|
||||
if passed:
|
||||
clear_large_outputs(name, trace)
|
||||
else:
|
||||
record["dependencies"], record["artifacts"] = [], []
|
||||
records[name] = record
|
||||
@@ -394,6 +441,12 @@ def trace_main(output, script, arguments):
|
||||
runpy.run_path(script, run_name="__main__")
|
||||
|
||||
|
||||
def finished_run(failed):
|
||||
"""Record that this run completed, for tools/gate.py: a crashed runner never writes it."""
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(STATE_DIR / "last-run.json").write_text(json.dumps({"run": os.environ.get("PARITY_RUN_ID"), "failed": failed}))
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "--trace":
|
||||
return trace_main(sys.argv[2], sys.argv[3], sys.argv[4:])
|
||||
@@ -439,6 +492,8 @@ def main():
|
||||
print(name)
|
||||
print(f"{len(selected)} of {len(found)} harnesses would run" if args.list else "Nothing to run: every harness passed with its current inputs.")
|
||||
save_state(state)
|
||||
if not args.list:
|
||||
finished_run([])
|
||||
return 0
|
||||
|
||||
jobs = max(1, args.jobs)
|
||||
@@ -460,6 +515,7 @@ def main():
|
||||
reuse_oracles)
|
||||
save_state(state)
|
||||
failed = sorted(name for name, record in results.items() if record["status"] != "pass")
|
||||
finished_run(failed)
|
||||
print(f"\n{len(results) - len(failed)} passed, {len(failed)} failed in {elapsed / 60:.1f} min.")
|
||||
for name in failed:
|
||||
print(f" {name}: {STATE_DIR / 'logs' / (name + '.log')}")
|
||||
|
||||
+12
-1
@@ -140,6 +140,14 @@ def files_under(paths):
|
||||
return found
|
||||
|
||||
|
||||
def note_outputs(outputs):
|
||||
"""Tell tools/parity.py which files are oracle outputs, which it keeps when clearing large outputs."""
|
||||
listing = os.environ.get("PARITY_SHIM_OUTPUTS")
|
||||
if listing:
|
||||
with open(listing, "a") as stream:
|
||||
stream.writelines(f"{path}\n" for path in outputs)
|
||||
|
||||
|
||||
def reusable(saved, paths, hashes):
|
||||
"""The recorded inputs are unchanged, nothing was added beside them, and the outputs are as left."""
|
||||
outputs = saved["outputs"]
|
||||
@@ -164,7 +172,9 @@ def main():
|
||||
command = {"program": os.path.realpath(program), "arguments": arguments, "cwd": os.getcwd(), "stdin": stdin}
|
||||
key = hashlib.sha256(json.dumps(command, sort_keys=True).encode()).hexdigest()
|
||||
record = CACHE / f"{key}.json"
|
||||
if record.exists() and reusable(json.loads(record.read_text()), paths, hashes):
|
||||
saved = json.loads(record.read_text()) if record.exists() else None
|
||||
if saved and reusable(saved, paths, hashes):
|
||||
note_outputs(saved["outputs"])
|
||||
sys.stdout.buffer.write((CACHE / f"{key}.stdout").read_bytes())
|
||||
sys.stderr.buffer.write((CACHE / f"{key}.stderr").read_bytes())
|
||||
return 0
|
||||
@@ -182,6 +192,7 @@ def main():
|
||||
temporary = record.with_suffix(f".{os.getpid()}.tmp")
|
||||
temporary.write_text(json.dumps({"command": command, "inputs": inputs, "outputs": outputs}))
|
||||
temporary.replace(record)
|
||||
note_outputs(outputs)
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
+18
-12
@@ -5,6 +5,7 @@
|
||||
python3 tools/supervise.py new dispensers # worktree ../dispensers on parity/dispensers, from the integration branch
|
||||
python3 tools/supervise.py save [AREA ...] # commit and push unfinished work (before stopping agents)
|
||||
python3 tools/supervise.py integrate parity/dispensers # merge into the integration branch if the gate passes
|
||||
python3 tools/supervise.py integrate parity/a parity/b # several branches, one gate
|
||||
|
||||
The integration branch (`parity-base` by default) is checked out in its own
|
||||
worktree; `integrate` merges there, runs tools/gate.py and pushes the
|
||||
@@ -86,8 +87,10 @@ def new(args):
|
||||
git("worktree", "add", "-b", branch, str(path), args.base)
|
||||
shared = base_tree.parent / "shared-local"
|
||||
(path / ".local").mkdir()
|
||||
# Link the vanilla data, not the shared runner and datafix state beside it.
|
||||
for entry in sorted(shared.iterdir()) if shared.is_dir() else []:
|
||||
(path / ".local" / entry.name).symlink_to(entry)
|
||||
if entry.is_dir() and entry.name not in {"parity-shared", "datafix"}:
|
||||
(path / ".local" / entry.name).symlink_to(entry)
|
||||
print(f"{path} on {branch}, from {args.base} at {git('rev-parse', '--short', 'HEAD', cwd=path)}")
|
||||
|
||||
|
||||
@@ -117,24 +120,27 @@ def integrate(args):
|
||||
state = base_tree / ".local/parity-runner/state.json"
|
||||
known = sorted(name for name, record in json.loads(state.read_text())["harnesses"].items()
|
||||
if record["status"] != "pass") if state.exists() else []
|
||||
merge = subprocess.run(["git", "merge", "--no-ff", "--no-edit", args.branch], cwd=base_tree, capture_output=True, text=True)
|
||||
if merge.returncode:
|
||||
conflicts = git("diff", "--name-only", "--diff-filter=U", cwd=base_tree, check=False)
|
||||
git("merge", "--abort", cwd=base_tree, check=False)
|
||||
print(f"{args.branch} does not merge cleanly into {args.base}. Conflicts:\n{conflicts or merge.stdout + merge.stderr}")
|
||||
print(f"Merge {args.base} into {args.branch}, resolve them there, and integrate again.")
|
||||
return 2
|
||||
for branch in args.branches:
|
||||
merge = subprocess.run(["git", "merge", "--no-ff", "--no-edit", branch], cwd=base_tree, capture_output=True, text=True)
|
||||
if merge.returncode:
|
||||
conflicts = git("diff", "--name-only", "--diff-filter=U", cwd=base_tree, check=False)
|
||||
git("merge", "--abort", cwd=base_tree, check=False)
|
||||
git("reset", "--hard", before, cwd=base_tree)
|
||||
print(f"{branch} does not merge cleanly into {args.base}. Conflicts:\n{conflicts or merge.stdout + merge.stderr}")
|
||||
print(f"Merge {args.base} into {branch}, resolve them there, and integrate again. Nothing was integrated.")
|
||||
return 2
|
||||
names = ", ".join(args.branches)
|
||||
if git("rev-parse", "HEAD", cwd=base_tree) == before:
|
||||
print(f"{args.branch} has nothing new for {args.base}.")
|
||||
print(f"{names}: nothing new for {args.base}.")
|
||||
return 0
|
||||
gate = [sys.executable, "tools/gate.py", "-j", str(args.jobs), "--known-failures", ",".join(known)]
|
||||
gate += ["--no-parity"] if args.no_parity else []
|
||||
if subprocess.run(gate, cwd=base_tree).returncode:
|
||||
git("reset", "--hard", before, cwd=base_tree)
|
||||
print(f"The gate failed, so {args.branch} was not integrated; {args.base} is back at {before[:9]}.")
|
||||
print(f"The gate failed, so {names} were not integrated; {args.base} is back at {before[:9]}.")
|
||||
return 1
|
||||
git("push", "-q", "origin", f"{args.base}:{args.push_to}", cwd=base_tree)
|
||||
print(f"Integrated {args.branch}; {args.base} pushed to origin/{args.push_to} at {git('rev-parse', '--short', 'HEAD', cwd=base_tree)}.")
|
||||
print(f"Integrated {names}; {args.base} pushed to origin/{args.push_to} at {git('rev-parse', '--short', 'HEAD', cwd=base_tree)}.")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -151,7 +157,7 @@ def main():
|
||||
command.add_argument("areas", nargs="*")
|
||||
command.set_defaults(run=save)
|
||||
command = commands.add_parser("integrate")
|
||||
command.add_argument("branch")
|
||||
command.add_argument("branches", nargs="+", help="merged in order, then gated together")
|
||||
command.add_argument("-j", "--jobs", type=int, default=8)
|
||||
command.add_argument("--no-parity", action="store_true", help="skip the vanilla harnesses in the gate")
|
||||
command.set_defaults(run=integrate)
|
||||
|
||||
Reference in New Issue
Block a user