diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index d072feec7..5e283c23e 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -22,7 +22,9 @@ Vanilla oracle runs are reused as well. `tools/parity_shim.py` stands in for `ja Probes build into `.local/parity-target/` with release optimisation but no LTO and 16 codegen units. On ludodesktop that rebuilds every `horizon-world` probe in 3 seconds after a one-line change, against 122 seconds with the release profile. Rust gives these settings identical results (no fast-math; overflow checks as in release), and the full harness suite passes under them. Probes that harnesses build in the dev profile get `opt-level = 3` too, keeping their overflow checks and debug assertions. Harnesses must locate binaries through `cargo metadata` or `lab.cargo_target()`, never a hard-coded `target/`. -Parallel harnesses are admitted by memory: each harness's recorded peak is reserved, within three quarters of the memory available at the start. `-j` caps the count (default: a third of the CPU threads). +Passes are shared between worktrees. A harness's key depends only on the content of its dependencies, not on where they are, so every pass is also written to `parity-shared/` beside the shared vanilla data (`PARITY_SHARED` overrides it). A worktree whose files match those a harness passed with, in any worktree, counts that pass without running the harness: after merging the integration branch, an agent's gate only runs the harnesses its own changes affect. + +Every runner on the machine shares `PARITY_MACHINE_JOBS` slots (default: half the CPU threads), so several agents' runs together can't overload it. Parallel harnesses are also admitted by memory: each harness's recorded peak is reserved, within three quarters of the memory available at the start. `-j` caps the count (default: a third of the CPU threads). ## The gate diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py index 4692a6efd..50aec62dd 100644 --- a/tests/test_parity_runner.py +++ b/tests/test_parity_runner.py @@ -44,6 +44,25 @@ class ParityRunnerTests(unittest.TestCase): source.write_text("two!") self.assertTrue(parity.stale("x", state, hasher, "env")) + def test_a_pass_from_another_worktree_counts_only_for_the_same_content(self): + with tempfile.TemporaryDirectory() as temp: + elsewhere, here, shared = Path(temp, "a"), Path(temp, "b"), Path(temp, "shared") + for root in (elsewhere, here): + root.mkdir() + (root / "x.rs").write_text("same") + with mock.patch.object(parity, "SHARED", shared): + with mock.patch.object(parity, "ROOT", elsewhere): + record = {"status": "pass", "dependencies": ["x.rs"], "artifacts": []} + record["key"] = parity.key(record, parity.Hasher({}), "env") + parity.publish("x", record) + with mock.patch.object(parity, "ROOT", here): + state = {"harnesses": {}, "hashes": {}} + self.assertFalse(parity.stale("x", state, parity.Hasher(state["hashes"]), "env")) + self.assertTrue(state["harnesses"]["x"]["shared"]) + (here / "x.rs").write_text("changed") + state = {"harnesses": {}, "hashes": {}} + self.assertTrue(parity.stale("x", state, parity.Hasher(state["hashes"]), "env")) + class OracleReuseTests(unittest.TestCase): """tools/parity_shim.py replays a run only when its inputs and outputs are as recorded.""" diff --git a/tools/parity.py b/tools/parity.py index e5bad7978..c2b3bd196 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -54,6 +54,11 @@ PROFILE = { "CARGO_PROFILE_DEV_OPT_LEVEL": "3", } SHIM = STATE_DIR / "shim" +# Results shared by every worktree using the same vanilla data (.local/minecraft +# links to a shared directory in parallel worktrees): passes keyed by content, +# so a harness whose inputs match one that passed elsewhere doesn't run again. +SHARED = Path(os.environ.get("PARITY_SHARED") or (LOCAL / "minecraft").resolve().parent / "parity-shared") +MACHINE_JOBS = int(os.environ.get("PARITY_MACHINE_JOBS") or max(1, (os.cpu_count() or 4) // 2)) DEFAULT_MEMORY = 3 << 30 # assumed peak for a harness that never ran here @@ -231,6 +236,33 @@ def prebuild(artifacts, jobs): return not errors, errors +def publish(name, record): + """Share a pass, so any worktree with the same inputs can reuse it.""" + if record["status"] != "pass" or "key" not in record: + return + for path, data in ((SHARED / "passes" / name / f"{record['key']}.json", record), + (SHARED / "dependencies" / f"{name}.json", {k: record[k] for k in ("dependencies", "artifacts")})): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(data)) + temporary.replace(path) + + +def machine_slot(): + """One of MACHINE_JOBS lock files shared by every runner on this machine, or None if all are taken.""" + import fcntl + slots = SHARED / "slots" + slots.mkdir(parents=True, exist_ok=True) + for index in range(MACHINE_JOBS): + handle = open(slots / f"{index}.lock", "w") + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + return handle + except OSError: + handle.close() + return None + + def memory_available(): for line in Path("/proc/meminfo").read_text().splitlines(): if line.startswith("MemAvailable:"): @@ -258,6 +290,9 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge expected = records.get(name, {}).get("peak_bytes", DEFAULT_MEMORY) if running and in_use + expected > memory_budget: continue + slot = machine_slot() + if slot is None: + break # other runners on this machine hold every slot queue.remove(name) reuse = reuse_oracles and records.get(name, {}).get("oracle_reuse", True) and name not in retried trace = traces / f"{name}.json" @@ -266,10 +301,10 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge process = subprocess.Popen( [sys.executable, str(Path(__file__)), "--trace", str(trace), str(selected[name])], cwd=ROOT, env=run_environment(reuse), stdout=log, stderr=subprocess.STDOUT, start_new_session=True) - running[name] = (process, time.monotonic(), log, expected, reuse) + running[name] = (process, time.monotonic(), log, expected, reuse, slot) in_use += expected time.sleep(0.2) - for name, (process, started, log, _, reused) in list(running.items()): + for name, (process, started, log, _, reused, slot) in list(running.items()): elapsed = time.monotonic() - started if process.poll() is None and elapsed < timeout: continue @@ -278,6 +313,7 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge process.wait() print(f"--- {name}: timed out after {timeout}s", file=log) log.close() + slot.close() del running[name] passed = process.returncode == 0 if not passed and reused and records.get(name, {}).get("status") != "fail": @@ -301,6 +337,7 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge records[name] = record results[name] = record save_state(state) + publish(name, record) tail = "" if passed else f" ({logs / (name + '.log')})" print(f"{'PASS' if passed else 'FAIL'} {name} {elapsed:.0f}s{tail}", flush=True) return results, time.monotonic() - started_all @@ -308,9 +345,19 @@ def run(selected, state, hasher, environment_digest, jobs, timeout, memory_budge def stale(name, state, hasher, environment): record = state["harnesses"].get(name) - if not record or record.get("status") != "pass" or "key" not in record: - return True - return key(record, hasher, environment) != record["key"] + if record and record.get("status") == "pass" and "key" in record and key(record, hasher, environment) == record["key"]: + return False + # Passed elsewhere with the same inputs? Try this worktree's and the shared dependency lists. + candidates = [record] if record and record.get("dependencies") else [] + template = SHARED / "dependencies" / f"{name}.json" + if template.exists(): + candidates.append(json.loads(template.read_text())) + for candidate in candidates: + shared = SHARED / "passes" / name / f"{key(candidate, hasher, environment)}.json" + if shared.exists(): + state["harnesses"][name] = dict(json.loads(shared.read_text()), shared=True) + return False + return True def trace_main(output, script, arguments): @@ -357,7 +404,8 @@ def main(): parser.add_argument("--failed", action="store_true", help="rerun the harnesses that failed last time") parser.add_argument("--list", action="store_true", help="print what would run, then stop") parser.add_argument("--status", action="store_true", help="print each harness's last result") - parser.add_argument("-j", "--jobs", type=int, default=max(1, (os.cpu_count() or 4) // 3)) + parser.add_argument("-j", "--jobs", type=int, default=max(1, (os.cpu_count() or 4) // 3), + help="harnesses at once in this run; every runner on the machine shares PARITY_MACHINE_JOBS slots") parser.add_argument("--timeout", type=int, default=3600, help="seconds per harness") args = parser.parse_args()