Share harness passes between worktrees and cap harnesses machine-wide
A pass is keyed by the content of the harness's dependencies, so it is also valid in any worktree with the same files. Passes are published to parity-shared/ beside the shared vanilla data, and a worktree adopts one whose key matches its own files instead of running the harness: an agent's gate only runs what its own changes affect, instead of the whole suite from scratch. Every runner on the machine also shares PARITY_MACHINE_JOBS slots, so several agents' gates can't overload it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -22,7 +22,9 @@ Vanilla oracle runs are reused as well. `tools/parity_shim.py` stands in for `ja
|
||||
|
||||
Probes build into `.local/parity-target/` with release optimisation but no LTO and 16 codegen units. On ludodesktop that rebuilds every `horizon-world` probe in 3 seconds after a one-line change, against 122 seconds with the release profile. Rust gives these settings identical results (no fast-math; overflow checks as in release), and the full harness suite passes under them. Probes that harnesses build in the dev profile get `opt-level = 3` too, keeping their overflow checks and debug assertions. Harnesses must locate binaries through `cargo metadata` or `lab.cargo_target()`, never a hard-coded `target/`.
|
||||
|
||||
Parallel harnesses are admitted by memory: each harness's recorded peak is reserved, within three quarters of the memory available at the start. `-j` caps the count (default: a third of the CPU threads).
|
||||
Passes are shared between worktrees. A harness's key depends only on the content of its dependencies, not on where they are, so every pass is also written to `parity-shared/` beside the shared vanilla data (`PARITY_SHARED` overrides it). A worktree whose files match those a harness passed with, in any worktree, counts that pass without running the harness: after merging the integration branch, an agent's gate only runs the harnesses its own changes affect.
|
||||
|
||||
Every runner on the machine shares `PARITY_MACHINE_JOBS` slots (default: half the CPU threads), so several agents' runs together can't overload it. Parallel harnesses are also admitted by memory: each harness's recorded peak is reserved, within three quarters of the memory available at the start. `-j` caps the count (default: a third of the CPU threads).
|
||||
|
||||
## The gate
|
||||
|
||||
|
||||
@@ -44,6 +44,25 @@ class ParityRunnerTests(unittest.TestCase):
|
||||
source.write_text("two!")
|
||||
self.assertTrue(parity.stale("x", state, hasher, "env"))
|
||||
|
||||
def test_a_pass_from_another_worktree_counts_only_for_the_same_content(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
elsewhere, here, shared = Path(temp, "a"), Path(temp, "b"), Path(temp, "shared")
|
||||
for root in (elsewhere, here):
|
||||
root.mkdir()
|
||||
(root / "x.rs").write_text("same")
|
||||
with mock.patch.object(parity, "SHARED", shared):
|
||||
with mock.patch.object(parity, "ROOT", elsewhere):
|
||||
record = {"status": "pass", "dependencies": ["x.rs"], "artifacts": []}
|
||||
record["key"] = parity.key(record, parity.Hasher({}), "env")
|
||||
parity.publish("x", record)
|
||||
with mock.patch.object(parity, "ROOT", here):
|
||||
state = {"harnesses": {}, "hashes": {}}
|
||||
self.assertFalse(parity.stale("x", state, parity.Hasher(state["hashes"]), "env"))
|
||||
self.assertTrue(state["harnesses"]["x"]["shared"])
|
||||
(here / "x.rs").write_text("changed")
|
||||
state = {"harnesses": {}, "hashes": {}}
|
||||
self.assertTrue(parity.stale("x", state, parity.Hasher(state["hashes"]), "env"))
|
||||
|
||||
|
||||
class OracleReuseTests(unittest.TestCase):
|
||||
"""tools/parity_shim.py replays a run only when its inputs and outputs are as recorded."""
|
||||
|
||||
+54
-6
@@ -54,6 +54,11 @@ PROFILE = {
|
||||
"CARGO_PROFILE_DEV_OPT_LEVEL": "3",
|
||||
}
|
||||
SHIM = STATE_DIR / "shim"
|
||||
# Results shared by every worktree using the same vanilla data (.local/minecraft
|
||||
# links to a shared directory in parallel worktrees): passes keyed by content,
|
||||
# so a harness whose inputs match one that passed elsewhere doesn't run again.
|
||||
SHARED = Path(os.environ.get("PARITY_SHARED") or (LOCAL / "minecraft").resolve().parent / "parity-shared")
|
||||
MACHINE_JOBS = int(os.environ.get("PARITY_MACHINE_JOBS") or max(1, (os.cpu_count() or 4) // 2))
|
||||
DEFAULT_MEMORY = 3 << 30 # assumed peak for a harness that never ran here
|
||||
|
||||
|
||||
@@ -231,6 +236,33 @@ def prebuild(artifacts, jobs):
|
||||
return not errors, errors
|
||||
|
||||
|
||||
def publish(name, record):
|
||||
"""Share a pass, so any worktree with the same inputs can reuse it."""
|
||||
if record["status"] != "pass" or "key" not in record:
|
||||
return
|
||||
for path, data in ((SHARED / "passes" / name / f"{record['key']}.json", record),
|
||||
(SHARED / "dependencies" / f"{name}.json", {k: record[k] for k in ("dependencies", "artifacts")})):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp")
|
||||
temporary.write_text(json.dumps(data))
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def machine_slot():
|
||||
"""One of MACHINE_JOBS lock files shared by every runner on this machine, or None if all are taken."""
|
||||
import fcntl
|
||||
slots = SHARED / "slots"
|
||||
slots.mkdir(parents=True, exist_ok=True)
|
||||
for index in range(MACHINE_JOBS):
|
||||
handle = open(slots / f"{index}.lock", "w")
|
||||
try:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
return handle
|
||||
except OSError:
|
||||
handle.close()
|
||||
return None
|
||||
|
||||
|
||||
def memory_available():
|
||||
for line in Path("/proc/meminfo").read_text().splitlines():
|
||||
if line.startswith("MemAvailable:"):
|
||||
@@ -258,6 +290,9 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
|
||||
expected = records.get(name, {}).get("peak_bytes", DEFAULT_MEMORY)
|
||||
if running and in_use + expected > memory_budget:
|
||||
continue
|
||||
slot = machine_slot()
|
||||
if slot is None:
|
||||
break # other runners on this machine hold every slot
|
||||
queue.remove(name)
|
||||
reuse = reuse_oracles and records.get(name, {}).get("oracle_reuse", True) and name not in retried
|
||||
trace = traces / f"{name}.json"
|
||||
@@ -266,10 +301,10 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, str(Path(__file__)), "--trace", str(trace), str(selected[name])],
|
||||
cwd=ROOT, env=run_environment(reuse), stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
running[name] = (process, time.monotonic(), log, expected, reuse)
|
||||
running[name] = (process, time.monotonic(), log, expected, reuse, slot)
|
||||
in_use += expected
|
||||
time.sleep(0.2)
|
||||
for name, (process, started, log, _, reused) in list(running.items()):
|
||||
for name, (process, started, log, _, reused, slot) in list(running.items()):
|
||||
elapsed = time.monotonic() - started
|
||||
if process.poll() is None and elapsed < timeout:
|
||||
continue
|
||||
@@ -278,6 +313,7 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
|
||||
process.wait()
|
||||
print(f"--- {name}: timed out after {timeout}s", file=log)
|
||||
log.close()
|
||||
slot.close()
|
||||
del running[name]
|
||||
passed = process.returncode == 0
|
||||
if not passed and reused and records.get(name, {}).get("status") != "fail":
|
||||
@@ -301,6 +337,7 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
|
||||
records[name] = record
|
||||
results[name] = record
|
||||
save_state(state)
|
||||
publish(name, record)
|
||||
tail = "" if passed else f" ({logs / (name + '.log')})"
|
||||
print(f"{'PASS' if passed else 'FAIL'} {name} {elapsed:.0f}s{tail}", flush=True)
|
||||
return results, time.monotonic() - started_all
|
||||
@@ -308,9 +345,19 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge
|
||||
|
||||
def stale(name, state, hasher, environment):
|
||||
record = state["harnesses"].get(name)
|
||||
if not record or record.get("status") != "pass" or "key" not in record:
|
||||
return True
|
||||
return key(record, hasher, environment) != record["key"]
|
||||
if record and record.get("status") == "pass" and "key" in record and key(record, hasher, environment) == record["key"]:
|
||||
return False
|
||||
# Passed elsewhere with the same inputs? Try this worktree's and the shared dependency lists.
|
||||
candidates = [record] if record and record.get("dependencies") else []
|
||||
template = SHARED / "dependencies" / f"{name}.json"
|
||||
if template.exists():
|
||||
candidates.append(json.loads(template.read_text()))
|
||||
for candidate in candidates:
|
||||
shared = SHARED / "passes" / name / f"{key(candidate, hasher, environment)}.json"
|
||||
if shared.exists():
|
||||
state["harnesses"][name] = dict(json.loads(shared.read_text()), shared=True)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def trace_main(output, script, arguments):
|
||||
@@ -357,7 +404,8 @@ def main():
|
||||
parser.add_argument("--failed", action="store_true", help="rerun the harnesses that failed last time")
|
||||
parser.add_argument("--list", action="store_true", help="print what would run, then stop")
|
||||
parser.add_argument("--status", action="store_true", help="print each harness's last result")
|
||||
parser.add_argument("-j", "--jobs", type=int, default=max(1, (os.cpu_count() or 4) // 3))
|
||||
parser.add_argument("-j", "--jobs", type=int, default=max(1, (os.cpu_count() or 4) // 3),
|
||||
help="harnesses at once in this run; every runner on the machine shares PARITY_MACHINE_JOBS slots")
|
||||
parser.add_argument("--timeout", type=int, default=3600, help="seconds per harness")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user