diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index b48be1d05..1fc31d8cb 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -52,7 +52,14 @@ jobs: - name: The prepared output is the one the preparer's OUTPUT_VERSION says env: BASE_SHA: ${{ inputs.base }} - run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA" + run: | + # The job runs main's workflow on the head's files: a head made before OUTPUT_VERSION (#1947) has neither + # the tool nor the number, so there is nothing to check. + if [ ! -f tools/prepare_output.py ] || ! grep -q OUTPUT_VERSION crates/horizon-prepare/src/lib.rs; then + echo "This head's preparer has no OUTPUT_VERSION: nothing to check" + exit 0 + fi + python3 tools/prepare_output.py check --cache --base "$BASE_SHA" - name: The version constants are the pinned jar's run: python3 tools/gen_version_consts.py --check - name: The harnesses this change can affect diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index e8e030c83..846a24e18 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -53,7 +53,7 @@ A harness can declare, at the top level of its script: ### Which harnesses a change affects -`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `.local/parity-runner/reach-cache.json`. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses. +`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `reach-cache.json` of the shared parity folder (beside `passes/`), so a new worktree starts warm; a harness's search stops as soon as it reaches a changed file, and after `PARITY_REACH_BUDGET` seconds (default 180) the remaining harnesses count every source their probes link, so choosing never holds a slow machine for long. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses. ## The gate diff --git a/tests/test_parity_reach.py b/tests/test_parity_reach.py index 9b7464b47..07e3df9e3 100644 --- a/tests/test_parity_reach.py +++ b/tests/test_parity_reach.py @@ -101,6 +101,16 @@ class ReachTests(unittest.TestCase): (self.root / "crates/a/src/unused.rs").unlink() self.assertIn("crates/a/src/unused.rs", self.narrowed()) + def test_a_search_told_to_stop_still_keeps_the_file_it_stopped_at(self): + graph = parity_reach.SourceGraph(self.root, parity.rust_code) + kept = parity_reach.narrow(self.dependencies, PROBE, graph, {"crates/a/src/used.rs"}, {"crates/a/src/used.rs"}) + self.assertEqual(kept["crates/a/src/used.rs"], (True, set())) + self.assertFalse(graph.results, "a partial search is not remembered") + # A stop file outside the harness's sources doesn't end its search. + full = parity_reach.narrow(self.dependencies, PROBE, graph, (), {"crates/b/src/lib.rs"}) + self.assertIn("crates/a/src/display.rs", full) + self.assertTrue(graph.results) + def test_unknown_probes_keep_everything(self): self.assertEqual(set(self.narrowed(artifacts=[])), set(self.dependencies)) self.assertEqual(set(self.narrowed(artifacts=[["example", "missing", "release", ["-p", "a"], True]])), diff --git a/tools/parity.py b/tools/parity.py index 3f18b922d..285e7268b 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -1245,14 +1245,20 @@ def affected_harnesses(found, records, changed, reach=True, base=None): commit), only declarations of modules the harness doesn't reach changed (tools/parity_reach.py).""" if changed & SHARED_INPUTS: return set(found) - graph = None + graph, stop = None, frozenset() + # Parses are keyed by content, so every worktree sharing the vanilla data shares them. + cache_path = (SHARED if SHARED.is_dir() else STATE_DIR) / "reach-cache.json" if reach and any(path.startswith("crates/") for path in changed): - cache_path = STATE_DIR / "reach-cache.json" try: cache = json.loads(cache_path.read_text(encoding="utf-8")) except (OSError, ValueError): cache = {} graph = parity_reach.SourceGraph(ROOT, rust_code, cache) + # A changed file that differs in more than its `mod` declarations counts as soon as a harness reaches it, + # so the search for that harness can stop there. + stop = frozenset(path for path in changed if parity_reach.is_workspace_rust(path) + and parity_reach.declarations_changed(ROOT, path, (), base, True)) + budget, started, over = float(os.environ.get("PARITY_REACH_BUDGET") or 180), time.monotonic(), False affected = set() for name, script in found.items(): record = records.get(name) or {} @@ -1265,16 +1271,31 @@ def affected_harnesses(found, records, changed, reach=True, base=None): affected.add(name) continue plain = {path.split(KEEP)[0] for path in dependencies} - if graph is not None and plain & changed: - narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed) - if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed( - ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()): - affected.add(name) - elif plain & changed: + hit = plain & changed + if not hit: + continue + if graph is None or any(not path.startswith("crates/") for path in hit): + affected.add(name) # no narrowing, or a file narrowing always keeps (tools/, Java oracles) + continue + if not over and time.monotonic() - started > budget: + over = True + print(f"note: choosing harnesses took over {budget:.0f} s (PARITY_REACH_BUDGET): the rest count every " + "source their probes link", flush=True) + if over: + affected.add(name) + continue + narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed, stop) + if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed( + ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()): affected.add(name) if graph is not None: - STATE_DIR.mkdir(parents=True, exist_ok=True) - cache_path.write_text(json.dumps(graph.saved_cache()), encoding="utf-8") + saved = graph.saved_cache() + if len(cache) + len(saved) < 4 * max(len(saved), 1): + saved = {**cache, **saved} # other worktrees' versions of the files too, within bounds + cache_path.parent.mkdir(parents=True, exist_ok=True) + temporary = cache_path.with_suffix(f".{os.getpid()}.tmp") + temporary.write_text(json.dumps(saved), encoding="utf-8") + temporary.replace(cache_path) return affected diff --git a/tools/parity_reach.py b/tools/parity_reach.py index 0c3051b86..dc61f6215 100644 --- a/tools/parity_reach.py +++ b/tools/parity_reach.py @@ -25,8 +25,9 @@ assets, fonts) that only unreached sources name; a dependency that no longer exi a change adds take part. """ +import functools import hashlib -from pathlib import Path, PurePosixPath +from pathlib import Path import re import subprocess @@ -478,9 +479,10 @@ class SourceGraph: seeds=seeds, node_sets={}) return self.tables - def reachable(self, entries, universe, reasons=None): + def reachable(self, entries, universe, reasons=None, stop=frozenset()): """The files of universe (workspace Rust sources) holding an item reachable from entries (by the rules in - this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging).""" + this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging). The + search ends early, with only part of the answer, once it reaches a file of stop.""" # Searching every source of the crates involved (not only those Cargo listed) only adds reach, and lets # harnesses with the same entries (the game binary's) share one search. crates = frozenset(crate_of(path) for path in universe) @@ -488,11 +490,12 @@ class SourceGraph: if reasons is None and key in self.results: return self.results[key] & set(universe) everything = {path for path in self.files if crate_of(path) in crates} | set(universe) - files = self.search(entries, everything, reasons) - self.results[key] = files + files, complete = self.search(entries, everything, reasons, stop) + if complete: + self.results[key] = files return files & set(universe) - def search(self, entries, universe, reasons): + def search(self, entries, universe, reasons, stop=frozenset()): targets = {self.target(path) for path in entries} universe = {path for path in universe if (self.is_library(path) or self.target(path) in targets) and path not in self.test_only} @@ -600,71 +603,80 @@ class SourceGraph: files.add(path) if reasons is not None: reasons[path] = why + if path in stop: + raise Stop - for path in universe: - if path.endswith("/src/lib.rs") or path in entries or path not in self.parent: - open_module(path, "crate root or entry") - for node in seeds: - reach(node, "entry" if node[0] in entries else "always counted") - for path in entries & universe: - files.add(path) - if reasons is not None: - reasons.setdefault(path, "entry") - while queue: - node = queue.pop() - path, index, member = node - item = item_of(node) - crate = crate_of(path) - label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"] - source = sets_of(node, item) - why = f"{path} ({label})" - # Plain mentions reach items of this file and of the modules above it. - ancestor, new = path, source["mentions"] - while ancestor is not None and new: - known = local.setdefault(ancestor, set()) - new = new - known - known |= new + try: + for path in universe: + if path.endswith("/src/lib.rs") or path in entries or path not in self.parent: + open_module(path, "crate root or entry") + for node in seeds: + reach(node, "entry" if node[0] in entries else "always counted") + for path in entries & universe: + files.add(path) + if reasons is not None: + reasons.setdefault(path, "entry") + while queue: + node = queue.pop() + path, index, member = node + item = item_of(node) + crate = crate_of(path) + label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"] + source = sets_of(node, item) + why = f"{path} ({label})" + # Plain mentions reach items of this file and of the modules above it. + ancestor, new = path, source["mentions"] + while ancestor is not None and new: + known = local.setdefault(ancestor, set()) + new = new - known + known |= new + for name in new: + wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}") + ancestor = self.parent.get(ancestor) + if item["kind"] == "use" and item["glob"]: + for name in item["names"]: + for target in sees.get(crate, {crate}): + for module in list(by_module.get((target, name), ())) + \ + ([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []): + if module not in globbed and module in universe: + globbed.add(module) + wake([(module, index, None) for index in range(len(self.parse(module)["items"]))] + if self.parse(module) else [], f"{why} glob-imports {name}") + visible_to = sees.get(crate, {crate}) + new = source["paths"] - done["paths"].setdefault(crate, set()) + done["paths"][crate] |= new for name in new: - wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}") - ancestor = self.parent.get(ancestor) - if item["kind"] == "use" and item["glob"]: - for name in item["names"]: - for target in sees.get(crate, {crate}): - for module in list(by_module.get((target, name), ())) + \ - ([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []): - if module not in globbed and module in universe: - globbed.add(module) - wake([(module, index, None) for index in range(len(self.parse(module)["items"]))] - if self.parse(module) else [], f"{why} glob-imports {name}") - visible_to = sees.get(crate, {crate}) - new = source["paths"] - done["paths"].setdefault(crate, set()) - done["paths"][crate] |= new - for name in new: - for target in visible_to: - path_names.setdefault(target, set()).add(name) - for module in by_module.get((target, name), ()): - if module in universe and self.parent[module] in opened: - open_module(module, f"{why} names {name}") - for key, table, wanted in (("qualified", qualified, lambda other: True), - ("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"), - ("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)): - new = source[key] - done[key].setdefault(crate, set()) - done[key][crate] |= new - for name in new: - table.setdefault(name, set()).add(crate) - wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)], - f"{why} {key} {name}") - if item["kind"] in TYPE_KINDS and member is None: - own = types.setdefault(crate, set()) - local_types.setdefault(path, set()).update(names(node)) - if not names(node) <= own: - own |= names(node) - visible_cache.clear() - seeing = seen_by.get(crate, {crate}) - for name in names(node): # also when the crate has one by that name already: impls of this one - wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing], - f"implements for {name}, defined in {path}") - return files + for target in visible_to: + path_names.setdefault(target, set()).add(name) + for module in by_module.get((target, name), ()): + if module in universe and self.parent[module] in opened: + open_module(module, f"{why} names {name}") + for key, table, wanted in (("qualified", qualified, lambda other: True), + ("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"), + ("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)): + new = source[key] - done[key].setdefault(crate, set()) + done[key][crate] |= new + for name in new: + table.setdefault(name, set()).add(crate) + wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)], + f"{why} {key} {name}") + if item["kind"] in TYPE_KINDS and member is None: + own = types.setdefault(crate, set()) + local_types.setdefault(path, set()).update(names(node)) + if not names(node) <= own: + own |= names(node) + visible_cache.clear() + seeing = seen_by.get(crate, {crate}) + for name in names(node): # also when the crate has one by that name already: impls of this one + wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing], + f"implements for {name}, defined in {path}") + except Stop: + return files, False + return files, True + + +class Stop(Exception): + """A search reached a file it was told to stop at.""" def impl_subject(header): @@ -682,13 +694,19 @@ def impl_subject(header): return parameters, re.split(r"\bwhere\b", header)[0] +_CRATES = {} + + def crate_of(path): - return path.split("/", 2)[1] + """The crate folder of a crates//... path (called millions of times by a search: cached).""" + crate = _CRATES.get(path) + if crate is None: + crate = _CRATES[path] = path.split("/", 2)[1] + return crate def is_workspace_rust(path): - parts = PurePosixPath(path).parts - return len(parts) > 2 and parts[0] == "crates" and path.endswith(".rs") + return path.startswith("crates/") and path.endswith(".rs") and path.count("/") >= 2 DECLARATION_LINE = re.compile(rf"\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+({IDENT})\s*;") @@ -718,6 +736,11 @@ def declarations_changed(root, path, children, base, whole): missing.""" if base is None: return True + return _declarations_changed(str(root), path, frozenset(children), base, whole) + + +@functools.lru_cache(maxsize=None) +def _declarations_changed(root, path, children, base, whole): try: head = (Path(root) / path).read_text(encoding="utf-8", errors="replace") before = subprocess.run(["git", "show", f"{base}:{path}"], cwd=root, capture_output=True, text=True, check=True).stdout @@ -726,36 +749,37 @@ def declarations_changed(root, path, children, base, whole): return declarations(head, children, whole) != declarations(before, children, whole) -def narrow(dependencies, artifacts, graph, changed=()): +def narrow(dependencies, artifacts, graph, changed=(), stop=frozenset()): """A harness's dependencies narrowed to what its entry points can reach: {dependency: (whole, children)}, where whole says the file counts for its own content (else only through its `mod` declarations of children, as a module ancestor of a reached file) and children are its reached child modules. Without known entry points (a - probe or binary the harness built that isn't found here), every dependency is kept whole.""" + probe or binary the harness built that isn't found here), every dependency is kept whole. The search may end + as soon as it reaches a file of stop (changed files that count once reached): then only part is returned.""" plain = [dependency.split("#keep:")[0] for dependency in dependencies] - crates = {PurePosixPath(path).parts[1] for path in plain if path.startswith("crates/")} + crates = {crate_of(path) for path in plain if path.startswith("crates/") and path.count("/") >= 2} rust = {path for path in plain if is_workspace_rust(path)} - added = {path for path in changed if is_workspace_rust(path) and PurePosixPath(path).parts[1] in crates + added = {path for path in changed if is_workspace_rust(path) and crate_of(path) in crates and path not in rust and (graph.root / path).is_file()} entries = graph.entries(artifacts, rust) if entries is None: return {dependency: (True, None) for dependency in dependencies} missing = {path for path in rust if graph.parse(path) is None} - reached = graph.reachable(entries, rust | added | entries) | missing + reached = graph.reachable(entries, rust | added | entries, stop=frozenset(stop) & (rust | added)) | missing named, named_in_crate = set(), {} for path in rust | added: parsed = graph.parse(path) file_names = set(parsed["file_names"]) if parsed else set() if path in reached: named |= file_names - named_in_crate.setdefault(PurePosixPath(path).parts[1], set()).update(file_names) + named_in_crate.setdefault(crate_of(path), set()).update(file_names) kept = {} for dependency, path in zip(dependencies, plain): if is_workspace_rust(path): if path in reached: kept[dependency] = (True, set()) elif not path.startswith("crates/") or path.endswith(("Cargo.toml", "build.rs")) \ - or PurePosixPath(path).name in named \ - or PurePosixPath(path).name not in named_in_crate.get(PurePosixPath(path).parts[1], ()): + or path.rsplit("/", 1)[-1] in named \ + or path.rsplit("/", 1)[-1] not in named_in_crate.get(crate_of(path), ()): kept[dependency] = (True, None) for path in sorted(added & reached): kept[path] = (True, set())