From ac54227465703e3e18c5462eb22735fa44fd1439 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:08:47 +0200 Subject: [PATCH 1/2] parity: prebuild probes with each harness's own package selection The probe prebuild ran `cargo build --workspace --examples --bins` (or named targets with --workspace). The harnesses build with `-p `, and Cargo unifies features over the selected packages: under --workspace, flate2, libc, memchr, syn, thiserror, postcard and digest get other features for horizon-world, and horizon-mods (pack) and horizon-telemetry (upload) for horizon-client. So every probe the prebuild made was a variant no harness used, and each harness rebuilt its crate and its probe again while the others waited on Cargo's lock. Record each probe with its command's package and feature flags and whether that command builds examples (which turns dev-dependencies on), and prebuild one Cargo run per such selection. --bins and --examples are recorded too, so the fast-profile horizon-client build that save_roundtrip and world_reopen run is prebuilt as well. A harness that never ran here takes its probes from the shared dependency lists instead of a build of all ~275 examples. Co-Authored-By: Claude Opus 5.5 --- tests/test_parity_runner.py | 31 +++++++++-- tools/parity.py | 103 ++++++++++++++++++++++++++++-------- 2 files changed, 110 insertions(+), 24 deletions(-) diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py index ffec8d3d5..beffb557d 100644 --- a/tests/test_parity_runner.py +++ b/tests/test_parity_runner.py @@ -16,10 +16,35 @@ class ParityRunnerTests(unittest.TestCase): def test_artifacts_follow_profile_and_stop_at_program_arguments(self): self.assertEqual(parity.artifacts_of(["cargo", "build", "--release", "--example", "a", "--bin", "b"]), - [["example", "a", "release"], ["bin", "b", "release"]]) + [["example", "a", "release", [], True], ["bin", "b", "release", [], True]]) self.assertEqual(parity.artifacts_of(["cargo", "run", "--profile", "fast", "--example", "c", "--", "--example", "x"]), - [["example", "c", "fast"]]) - self.assertEqual(parity.artifacts_of(["cargo", "build", "--example", "d"]), [["example", "d", "dev"]]) + [["example", "c", "fast", [], True]]) + self.assertEqual(parity.artifacts_of(["cargo", "build", "--example", "d"]), [["example", "d", "dev", [], True]]) + + def test_artifacts_keep_the_package_and_feature_selection(self): + self.assertEqual(parity.artifacts_of(["cargo", "build", "--locked", "--profile=fast", "-j", "4", "--package", + "horizon-client", "--bins", "-F", "x"]), + [["bins", "", "fast", ["-p", "horizon-client", "--features", "x"], False]]) + self.assertEqual(parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--bin=horizon-world"]), + [["bin", "horizon-world", "release", ["-p", "horizon-world"], False]]) + + def test_prebuild_repeats_each_harness_selection(self): + artifacts = [ + *parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--example", "a"]), + *parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--example", "b"]), + *parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--bin", "horizon-world"]), + *parity.artifacts_of(["cargo", "build", "--release", "--bin", "horizon-world"]), + *parity.artifacts_of(["cargo", "run", "--profile", "fast", "-p", "horizon-client", "--example", "c"]), + ["example", "old", "release"], # recorded before selections were kept: its own crate's -p + ["example", "unknown", "release"], # no longer in the workspace: left to its harness + ] + self.assertEqual(parity.prebuild_commands(artifacts, {("example", "old"): "horizon-world"}), [ + ["cargo", "build", "--locked", "--profile", "fast", "-p", "horizon-client", "--example", "c"], + ["cargo", "build", "--locked", "--profile", "release", "--bin", "horizon-world"], + ["cargo", "build", "--locked", "--profile", "release", "-p", "horizon-world", "--bin", "horizon-world"], + ["cargo", "build", "--locked", "--profile", "release", "-p", "horizon-world", "--example", "a", "--example", "b", + "--example", "old"], + ]) def test_java_sources_include_classes_pulled_from_the_sourcepath(self): with tempfile.TemporaryDirectory() as temp: diff --git a/tools/parity.py b/tools/parity.py index 6ba4c0d90..5dbf13554 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -27,9 +27,10 @@ Harness probes build into .local/parity-target with release optimisations but without LTO and with 16 codegen units (`PROFILE` below), which builds in seconds instead of minutes. Rust does not let those settings change results: there is no fast-math, and overflow checks stay as in release. Probes built in -the dev profile are optimised too, keeping their overflow checks. Probes the -recorded harnesses need are built in one Cargo run before the harnesses start, -so parallel harnesses don't queue on Cargo's lock. +the dev profile are optimised too, keeping their overflow checks. The probes the +selected harnesses built last time are built before they start, so parallel +harnesses don't queue on Cargo's lock, with each harness's own package and +feature flags so that its Cargo run then finds them fresh. """ import argparse @@ -210,18 +211,42 @@ def artifact_path(kind, name, profile): return TARGET / directory / ("examples" if kind == "example" else "") / name +SELECTION_FLAGS = {"-p", "--package", "-F", "--features"} # followed by a value +SELECTION_SWITCHES = {"--workspace", "--all", "--all-features", "--no-default-features"} + + def artifacts_of(arguments): - """The [kind, name, profile] targets a cargo build/run command line names.""" + """The [kind, name, profile, selection, examples] targets a cargo build/run command line names. + + kind is example, bin, or examples/bins (every one, name ""). selection is the command's package and feature + flags, and examples whether the command builds any example (which turns on dev-dependencies): together they + decide how Cargo unifies features, so the probe prebuild must repeat them exactly. A `--workspace` build of + the same example gives its dependencies other features, and the harness's own `-p` build then compiles + everything again.""" arguments = arguments[:arguments.index("--")] if "--" in arguments else arguments profile = "release" if "--release" in arguments else "dev" - if "--profile" in arguments: - profile = arguments[arguments.index("--profile") + 1] + selection = [] found = [] - for flag, kind in (("--example", "example"), ("--bin", "bin")): - for index, argument in enumerate(arguments[:-1]): - if argument == flag: - found.append([kind, arguments[index + 1], profile]) - return found + index = 1 + while index < len(arguments): + argument = arguments[index] + flag, _, value = argument.partition("=") + if flag in SELECTION_FLAGS | {"--profile", "--example", "--bin"} and not value and index + 1 < len(arguments): + value = arguments[index + 1] + index += 1 + if flag == "--profile": + profile = value + elif flag in SELECTION_FLAGS: + selection += ["-p" if flag == "--package" else "--features" if flag == "-F" else flag, value] + elif flag in SELECTION_SWITCHES: + selection.append("--workspace" if flag == "--all" else flag) + elif flag in ("--example", "--bin"): + found.append([flag[2:], value]) + elif flag in ("--examples", "--bins"): + found.append([flag[2:], ""]) + index += 1 + examples = any(kind in ("example", "examples") for kind, _ in found) + return [[kind, name, profile, selection, examples] for kind, name in found] RUST_PATH = re.compile(r"(? Date: Fri, 23 Oct 2026 16:30:27 +0200 Subject: [PATCH 2/2] parity: leave out of the prebuild probes that no longer exist A record naming an example since renamed or deleted made the whole Cargo run fail ("no example target named ..."), so no probe was prebuilt and every harness built its own: the probe binaries rename ~250 of them. Co-Authored-By: Claude Opus 5.5 --- tests/test_parity_runner.py | 5 ++++- tools/parity.py | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py index beffb557d..a9b7ff595 100644 --- a/tests/test_parity_runner.py +++ b/tests/test_parity_runner.py @@ -35,10 +35,13 @@ class ParityRunnerTests(unittest.TestCase): *parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--bin", "horizon-world"]), *parity.artifacts_of(["cargo", "build", "--release", "--bin", "horizon-world"]), *parity.artifacts_of(["cargo", "run", "--profile", "fast", "-p", "horizon-client", "--example", "c"]), + *parity.artifacts_of(["cargo", "build", "--release", "-p", "horizon-world", "--example", "renamed"]), ["example", "old", "release"], # recorded before selections were kept: its own crate's -p ["example", "unknown", "release"], # no longer in the workspace: left to its harness ] - self.assertEqual(parity.prebuild_commands(artifacts, {("example", "old"): "horizon-world"}), [ + packages = {("example", "a"): "horizon-world", ("example", "b"): "horizon-world", ("example", "c"): "horizon-client", + ("bin", "horizon-world"): "horizon-world", ("example", "old"): "horizon-world"} + self.assertEqual(parity.prebuild_commands(artifacts, packages), [ ["cargo", "build", "--locked", "--profile", "fast", "-p", "horizon-client", "--example", "c"], ["cargo", "build", "--locked", "--profile", "release", "--bin", "horizon-world"], ["cargo", "build", "--locked", "--profile", "release", "-p", "horizon-world", "--bin", "horizon-world"], diff --git a/tools/parity.py b/tools/parity.py index 5dbf13554..4113cc791 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -555,10 +555,14 @@ def prebuild_commands(artifacts, packages=None): """The Cargo runs that build the recorded probes exactly as their harnesses do: one per profile, package and feature selection, and whether examples are among the command's targets (see artifacts_of). - A [kind, name, profile] record from before selections were kept is taken as built with `-p` and its own crate, - as nearly every harness does (packages: target_packages()); the harness's next run records the real one.""" + packages: target_packages(). Records of targets that no longer exist (a probe renamed or deleted since a harness + ran it) are left out, since one unknown target fails its whole Cargo run. A [kind, name, profile] record from + before selections were kept is taken as built with `-p` and its own crate, as nearly every harness does; the + harness's next run records the real one.""" groups = {} for artifact in artifacts: + if packages and artifact[1] and tuple(artifact[:2]) not in packages: + continue if len(artifact) == 3 and (packages or {}).get(tuple(artifact[:2])): kind, name, profile = artifact artifact = [kind, name, profile, ["-p", packages[(kind, name)]], kind == "example"] @@ -578,7 +582,7 @@ def prebuild(artifacts, jobs): environment = run_environment() environment["CARGO_BUILD_JOBS"] = str(jobs) errors = "" - packages = target_packages() if any(len(artifact) == 3 for artifact in artifacts) else None + packages = target_packages() for command in prebuild_commands(artifacts, packages): result = subprocess.run(command, cwd=ROOT, env=environment, capture_output=True, text=True) if result.returncode: