Merge pull request #1443 from ludovic111/fix/parity-prebuild-selection

🤖 Agent: merged after verify passed. The parity probe prebuild repeats each harness's own -p and features selection, so harness builds after the prebuild compile nothing (0 of 10, was 10 of 10). The full harness set ran on it locally in #1453's before/after diff.
This commit is contained in:
Ludovic Marie
2026-10-23 17:39:09 +02:00
committed by GitHub
2 changed files with 117 additions and 24 deletions
+31 -3
View File
@@ -16,10 +16,38 @@ class ParityRunnerTests(unittest.TestCase):
def test_artifacts_follow_profile_and_stop_at_program_arguments(self):
self.assertEqual(parity.artifacts_of(["cargo", "build", "--release", "--example", "a", "--bin", "b"]),
[["example", "a", "release"], ["bin", "b", "release"]])
[["example", "a", "release", [], True], ["bin", "b", "release", [], True]])
self.assertEqual(parity.artifacts_of(["cargo", "run", "--profile", "fast", "--example", "c", "--", "--example", "x"]),
[["example", "c", "fast"]])
self.assertEqual(parity.artifacts_of(["cargo", "build", "--example", "d"]), [["example", "d", "dev"]])
[["example", "c", "fast", [], True]])
self.assertEqual(parity.artifacts_of(["cargo", "build", "--example", "d"]), [["example", "d", "dev", [], True]])
def test_artifacts_keep_the_package_and_feature_selection(self):
self.assertEqual(parity.artifacts_of(["cargo", "build", "--locked", "--profile=fast", "-j", "4", "--package",
"horizon-client", "--bins", "-F", "x"]),
[["bins", "", "fast", ["-p", "horizon-client", "--features", "x"], False]])
self.assertEqual(parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--bin=horizon-world"]),
[["bin", "horizon-world", "release", ["-p", "horizon-world"], False]])
def test_prebuild_repeats_each_harness_selection(self):
artifacts = [
*parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--example", "a"]),
*parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--example", "b"]),
*parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--bin", "horizon-world"]),
*parity.artifacts_of(["cargo", "build", "--release", "--bin", "horizon-world"]),
*parity.artifacts_of(["cargo", "run", "--profile", "fast", "-p", "horizon-client", "--example", "c"]),
*parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--example", "renamed"]),
["example", "old", "release"], # recorded before selections were kept: its own crate's -p
["example", "unknown", "release"], # no longer in the workspace: left to its harness
]
packages = {("example", "a"): "horizon-world", ("example", "b"): "horizon-world", ("example", "c"): "horizon-client",
("bin", "horizon-world"): "horizon-world", ("example", "old"): "horizon-world"}
self.assertEqual(parity.prebuild_commands(artifacts, packages), [
["cargo", "build", "--locked", "--profile", "fast", "-p", "horizon-client", "--example", "c"],
["cargo", "build", "--locked", "--profile", "release", "--bin", "horizon-world"],
["cargo", "build", "--locked", "--profile", "release", "-p", "horizon-world", "--bin", "horizon-world"],
["cargo", "build", "--locked", "--profile", "release", "-p", "horizon-world", "--example", "a", "--example", "b",
"--example", "old"],
])
def test_java_sources_include_classes_pulled_from_the_sourcepath(self):
with tempfile.TemporaryDirectory() as temp:
+86 -21
View File
@@ -27,9 +27,10 @@ Harness probes build into .local/parity-target with release optimisations but
without LTO and with 16 codegen units (`PROFILE` below), which builds in
seconds instead of minutes. Rust does not let those settings change results:
there is no fast-math, and overflow checks stay as in release. Probes built in
the dev profile are optimised too, keeping their overflow checks. Probes the
recorded harnesses need are built in one Cargo run before the harnesses start,
so parallel harnesses don't queue on Cargo's lock.
the dev profile are optimised too, keeping their overflow checks. The probes the
selected harnesses built last time are built before they start, so parallel
harnesses don't queue on Cargo's lock, with each harness's own package and
feature flags so that its Cargo run then finds them fresh.
"""
import argparse
@@ -210,18 +211,42 @@ def artifact_path(kind, name, profile):
return TARGET / directory / ("examples" if kind == "example" else "") / name
SELECTION_FLAGS = {"-p", "--package", "-F", "--features"} # followed by a value
SELECTION_SWITCHES = {"--workspace", "--all", "--all-features", "--no-default-features"}
def artifacts_of(arguments):
"""The [kind, name, profile] targets a cargo build/run command line names."""
"""The [kind, name, profile, selection, examples] targets a cargo build/run command line names.
kind is example, bin, or examples/bins (every one, name ""). selection is the command's package and feature
flags, and examples whether the command builds any example (which turns on dev-dependencies): together they
decide how Cargo unifies features, so the probe prebuild must repeat them exactly. A `--workspace` build of
the same example gives its dependencies other features, and the harness's own `-p` build then compiles
everything again."""
arguments = arguments[:arguments.index("--")] if "--" in arguments else arguments
profile = "release" if "--release" in arguments else "dev"
if "--profile" in arguments:
profile = arguments[arguments.index("--profile") + 1]
selection = []
found = []
for flag, kind in (("--example", "example"), ("--bin", "bin")):
for index, argument in enumerate(arguments[:-1]):
if argument == flag:
found.append([kind, arguments[index + 1], profile])
return found
index = 1
while index < len(arguments):
argument = arguments[index]
flag, _, value = argument.partition("=")
if flag in SELECTION_FLAGS | {"--profile", "--example", "--bin"} and not value and index + 1 < len(arguments):
value = arguments[index + 1]
index += 1
if flag == "--profile":
profile = value
elif flag in SELECTION_FLAGS:
selection += ["-p" if flag == "--package" else "--features" if flag == "-F" else flag, value]
elif flag in SELECTION_SWITCHES:
selection.append("--workspace" if flag == "--all" else flag)
elif flag in ("--example", "--bin"):
found.append([flag[2:], value])
elif flag in ("--examples", "--bins"):
found.append([flag[2:], ""])
index += 1
examples = any(kind in ("example", "examples") for kind, _ in found)
return [[kind, name, profile, selection, examples] for kind, name in found]
RUST_PATH = re.compile(r"(?<![\w:])((?:crate|self|super)(?:::super)*|[a-z_][a-z0-9_]*)::((?:[A-Za-z_]\w*::)*[A-Za-z_]\w*)")
@@ -457,8 +482,9 @@ def dependencies(trace, scope=None):
for argument in arguments[1:]:
if isinstance(argument, str) and os.path.isfile(argument):
found.add(argument)
for kind, name, profile in artifacts:
found.update(dep_info(artifact_path(kind, name, profile)))
for kind, name, profile, *_ in artifacts:
if name:
found.update(dep_info(artifact_path(kind, name, profile)))
kept = set()
for path in found:
path = Path(path) if os.path.isabs(path) else ROOT / path
@@ -515,15 +541,49 @@ def default_build_jobs():
return jobs
def target_packages():
"""{(kind, name): package} for the workspace's examples and binaries."""
result = subprocess.run(["cargo", "metadata", "--no-deps", "--format-version", "1"], cwd=ROOT, capture_output=True,
text=True)
if result.returncode:
return {}
return {(kind, target["name"]): package["name"] for package in json.loads(result.stdout)["packages"]
for target in package["targets"] for kind in target["kind"] if kind in ("example", "bin")}
def prebuild_commands(artifacts, packages=None):
"""The Cargo runs that build the recorded probes exactly as their harnesses do: one per profile, package and
feature selection, and whether examples are among the command's targets (see artifacts_of).
packages: target_packages(). Records of targets that no longer exist (a probe renamed or deleted since a harness
ran it) are left out, since one unknown target fails its whole Cargo run. A [kind, name, profile] record from
before selections were kept is taken as built with `-p` and its own crate, as nearly every harness does; the
harness's next run records the real one."""
groups = {}
for artifact in artifacts:
if packages and artifact[1] and tuple(artifact[:2]) not in packages:
continue
if len(artifact) == 3 and (packages or {}).get(tuple(artifact[:2])):
kind, name, profile = artifact
artifact = [kind, name, profile, ["-p", packages[(kind, name)]], kind == "example"]
if len(artifact) < 5:
continue
kind, name, profile, selection, examples = artifact
targets = groups.setdefault((profile, tuple(selection), examples), [])
target = [f"--{kind}", name] if name else [f"--{kind}"]
if target not in targets:
targets.append(target)
return [["cargo", "build", "--locked", "--profile", profile, *selection, *(part for target in targets for part in target)]
for (profile, selection, _), targets in sorted(groups.items())]
def prebuild(artifacts, jobs):
"""Build every recorded probe in one Cargo run so harnesses don't serialise on its lock."""
"""Build the recorded probes before the harnesses start, so they don't serialise on Cargo's lock."""
environment = run_environment()
environment["CARGO_BUILD_JOBS"] = str(jobs)
errors = ""
for profile in sorted({profile for _, _, profile in artifacts}):
command = ["cargo", "build", "--locked", "--workspace", "--profile", profile]
for kind, name, _ in (artifact for artifact in artifacts if artifact[2] == profile):
command += ["--examples", "--bins"] if kind == "all" else [f"--{kind}", name]
packages = target_packages()
for command in prebuild_commands(artifacts, packages):
result = subprocess.run(command, cwd=ROOT, env=environment, capture_output=True, text=True)
if result.returncode:
errors += result.stderr
@@ -1217,10 +1277,15 @@ def main():
finished_run([])
return 0
# Harnesses that never ran here have no recorded probes: build them all.
artifacts = [["all", "", "release"]] if any("artifacts" not in records.get(name, {}) for name in selected) else []
# The probes the selected harnesses built last time, here or (for those that never ran here) in another
# worktree. A harness with no record builds its own: building every probe for it would be ~275 binaries.
artifacts = []
for name in selected:
for artifact in records.get(name, {}).get("artifacts", []):
record = records.get(name)
if not record or "artifacts" not in record:
template = SHARED / "dependencies" / f"{name}.json"
record = json.loads(template.read_text(encoding="utf-8")) if template.exists() else {}
for artifact in record.get("artifacts", []):
if artifact not in artifacts:
artifacts.append(artifact)
config = host_config()