From d4b2e24535111e7b8e1b839bcc16c4ef126fa663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Dr=C3=A4ger?= <20014538+sdraeger@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:21:23 -0700 Subject: [PATCH 01/13] Add mod command capability and Predator Missile example --- AGENTS.md | 2 +- Cargo.lock | 7 + Cargo.toml | 2 +- crates/horizon-client/src/game/mods.rs | 13 ++ .../examples/predator-missile/Cargo.toml | 16 ++ .../predator-missile/horizon-mod.toml | 8 + .../examples/predator-missile/src/lib.rs | 161 ++++++++++++++++++ crates/horizon-mod-sdk/src/lib.rs | 4 +- crates/horizon-mod-sdk/wit/horizon.wit | 9 + crates/horizon-mods/src/manifest.rs | 12 +- crates/horizon-mods/src/runtime.rs | 17 +- crates/horizon-mods/tests/sandbox.rs | 45 ++++- docs/api/mods-reference.md | 13 +- docs/mods.md | 3 +- 14 files changed, 300 insertions(+), 12 deletions(-) create mode 100644 crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml create mode 100644 crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml create mode 100644 crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs diff --git a/AGENTS.md b/AGENTS.md index f6f6b463b..fda05c1e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,7 +80,7 @@ python3 tools/agent_control_e2e.py # horizon-ctl / MCP end to end ~25, a file of the client lib ~103, of `horizon-world` ~103–109, of `horizon-core` ~119. Plan for it. - Rust integration tests: `crates/horizon-client/tests/` (`lan_bot.rs`, `generation_saves.rs`, `streaming_meshes.rs`). - **macOS:** `cargo build/test --workspace` may fail to link the two wasm guest cdylibs (undefined - `horizon:mods/events` symbols) until #1446 lands: add `--exclude coords-hud --exclude horizon-mods-test-guest`. + `horizon:mods/events` symbols) until #1446 lands: add `--exclude coords-hud --exclude predator-missile --exclude horizon-mods-test-guest`. Their real check is `cargo test -p horizon-mods` (builds them for `wasm32-unknown-unknown`). - Linux needs `libasound2-dev libudev-dev`; `--no-default-features` drops audio and controllers (the server build). - Run and drive the game (starts the `horizon-client` beside `horizon-ctl`; other arguments go to the game): diff --git a/Cargo.lock b/Cargo.lock index 27eaad0c4..1468327ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2620,6 +2620,13 @@ dependencies = [ "zerovec", ] +[[package]] +name = "predator-missile" +version = "0.1.0" +dependencies = [ + "horizon-mod-sdk", +] + [[package]] name = "presser" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index f197c34fe..a0feaba87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] exclude = ["launcher", "crates/horizon-telemetry-server", "third_party/wgpu-hal"] -members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"] +members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "crates/horizon-mod-sdk/examples/predator-missile", "xtask"] resolver = "2" [workspace.package] diff --git a/crates/horizon-client/src/game/mods.rs b/crates/horizon-client/src/game/mods.rs index 572f9fe2b..c203d0ca2 100644 --- a/crates/horizon-client/src/game/mods.rs +++ b/crates/horizon-client/src/game/mods.rs @@ -342,6 +342,19 @@ impl Game<'_> { } Output::Feedback { text, error: false } => Text::literal(text), Output::Feedback { text, error: true } => Text::literal(text).with_color(text::RED), + // Vanilla's commands only: a mod running a mod command could loop. + Output::Command { line } => { + let name = line.split(' ').next().unwrap_or_default(); + // As handle_chat_input, without the chat history. + if !vanilla_command(name) { + // dropped + } else if self.remote.is_some() { + self.send_chat(&format!("/{line}")); + } else { + self.run_command(&line); + } + continue; + } Output::Stopped { name, reason } => { self.list_mods(); self.sync_mod_commands(); diff --git a/crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml b/crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml new file mode 100644 index 000000000..a3a77ce0b --- /dev/null +++ b/crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "predator-missile" +version = "0.1.0" +edition.workspace = true +license.workspace = true +publish = false +description = "Example Horizon mod: a missile you launch with /missile and steer with the mouse" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +horizon-mod-sdk = { path = "../.." } + +[lints] +workspace = true diff --git a/crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml b/crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml new file mode 100644 index 000000000..3baaf7d87 --- /dev/null +++ b/crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml @@ -0,0 +1,8 @@ +id = "predator_missile" +name = "Predator Missile" +version = "0.1.0" +description = "/missile launches a guided missile 60 blocks above you. Steer it with the mouse; it explodes on whatever it hits. Needs cheats (it runs vanilla commands as you)." +authors = ["Horizon"] +api = "0.1" +side = "client" +capabilities = ["game-read", "hud", "chat", "command"] diff --git a/crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs b/crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs new file mode 100644 index 000000000..75ebbbdec --- /dev/null +++ b/crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs @@ -0,0 +1,161 @@ +//! Example Horizon mod (`docs/mods.md`): `/missile` turns you into a guided +//! missile 60 blocks up. You fly along your own look, so the mouse steers; +//! on a block or a mob a stack of primed TNT goes off, and you are put back +//! where you launched. Built from vanilla commands (capability `command`), +//! so it needs cheats. +use horizon_mod_sdk::{ + Mod, Registration, + command::run, + game::{self, Player}, + hud::{self, Corner, Line}, +}; + +/// Blocks per tick (a fired arrow does about 3). +const SPEED: f64 = 1.5; +const ALTITUDE: i32 = 60; +/// Ticks of flight before the missile detonates on its own. +const FUEL: u32 = 30 * 20; +/// Ticks to watch the blast before going back. +const AFTERMATH: u32 = 40; +/// Primed TNT per blast: the stack's explosions add up. +const WARHEAD: usize = 4; +/// What the impact checks print; the mod hears it through the chat hook. +const IMPACT: &str = "Predator: impact"; + +enum State { + Idle, + Flying { from: Player, ticks: u32 }, + Aftermath { from: Player, ticks: u32 }, +} + +struct Predator { + state: State, +} + +fn launch() { + run("gamerule send_command_feedback false"); + run("execute if entity @s[gamemode=creative] run tag @s add pm_creative"); + run("gamemode spectator"); + run(&format!("tp @s ~ ~{ALTITUDE} ~ ~ 60")); + run("effect give @s night_vision 60 0 true"); + // The thermal camera: every mob outlined. + run("effect give @e[type=!player,distance=..256] glowing 60 0 true"); +} + +/// One tick of flight: stop at the surface (the next step would enter a +/// block or a mob), else move one step along the look. +fn fly() { + run(&format!( + "execute at @s unless block ^ ^ ^{SPEED} #minecraft:air run tellraw @s \"{IMPACT}\"" + )); + run(&format!( + "execute at @s positioned ^ ^ ^{SPEED} if entity @e[type=!player,distance=..2] run tellraw @s \"{IMPACT}\"" + )); + run(&format!( + "execute at @s if block ^ ^ ^{SPEED} #minecraft:air run tp @s ^ ^ ^{SPEED}" + )); +} + +fn detonate() { + for _ in 0..WARHEAD { + run("execute at @s run summon tnt ^ ^ ^1 {fuse:0s}"); + } +} + +fn land(from: &Player) { + run(&format!( + "tp @s {} {} {} {} {}", + from.x, from.y, from.z, from.yaw, from.pitch + )); + run("execute if entity @s[tag=pm_creative] run gamemode creative"); + run("execute unless entity @s[tag=pm_creative] run gamemode survival"); + run("execute if entity @s[tag=pm_creative] run tag @s remove pm_creative"); + run("effect clear @s night_vision"); + // shortcut: assumes feedback was on before launch, as in a new world. + run("gamerule send_command_feedback true"); +} + +fn panel(p: &Player, ticks: u32) -> Vec { + let green = 0x7CFF7C; + let left = (FUEL - ticks.min(FUEL)) / 20; + [ + "PREDATOR".to_string(), + format!("ALT {:.0}", p.y), + format!("FUEL {left}s"), + "Mouse to steer, /missile to detonate".to_string(), + ] + .into_iter() + .map(|text| Line { text, color: green }) + .collect() +} + +impl Predator { + fn impact(&mut self) { + if let State::Flying { from, .. } = &self.state { + detonate(); + self.state = State::Aftermath { + from: from.clone(), + ticks: 0, + }; + } + } +} + +impl Mod for Predator { + fn init() -> Result<(Self, Registration), String> { + let hooks = Registration::new().command("missile").on_tick().on_chat(); + Ok((Predator { state: State::Idle }, hooks)) + } + + fn tick(&mut self, _tick: u64) { + match &mut self.state { + State::Idle => {} + State::Flying { ticks, .. } => { + *ticks += 1; + let ticks = *ticks; + if ticks >= FUEL { + return self.impact(); + } + // The first tick waits for the launch teleport. + if ticks > 1 { + fly(); + } + if let Some(p) = game::get_player() { + hud::set_panel(Corner::TopLeft, &panel(&p, ticks)); + } + } + State::Aftermath { from, ticks } => { + *ticks += 1; + if *ticks >= AFTERMATH { + land(from); + hud::clear(); + self.state = State::Idle; + } + } + } + } + + fn chat(&mut self, text: &str) { + if text.contains(IMPACT) { + self.impact(); + } + } + + fn command(&mut self, _name: &str, _args: &str) -> Result, String> { + match self.state { + State::Idle => { + let from = game::get_player().ok_or("Not in a world")?; + launch(); + self.state = State::Flying { from, ticks: 0 }; + Ok(Some("Missile away".into())) + } + State::Flying { .. } => { + self.impact(); + Ok(None) + } + State::Aftermath { .. } => Err("Wait for the blast to clear".into()), + } + } +} + +horizon_mod_sdk::export_mod!(Predator); diff --git a/crates/horizon-mod-sdk/src/lib.rs b/crates/horizon-mod-sdk/src/lib.rs index 71c36cf80..7efcf20b3 100644 --- a/crates/horizon-mod-sdk/src/lib.rs +++ b/crates/horizon-mod-sdk/src/lib.rs @@ -21,7 +21,7 @@ //! ``` //! //! Every interface but `log` is a capability: list the ones the mod uses in -//! `horizon-mod.toml` (`game-read`, `chat`, `hud`, `storage`). +//! `horizon-mod.toml` (`game-read`, `chat`, `hud`, `storage`, `command`). wit_bindgen::generate!({ path: "wit", @@ -32,7 +32,7 @@ wit_bindgen::generate!({ }); pub use exports::horizon::mods::events::{Guest as __Guest, Registration}; -pub use horizon::mods::{chat, game, hud, log, storage}; +pub use horizon::mods::{chat, command, game, hud, log, storage}; /// The game's calls into a mod. Only the hooks [`Registration`] asks for /// are called; the others may stay as their defaults. diff --git a/crates/horizon-mod-sdk/wit/horizon.wit b/crates/horizon-mod-sdk/wit/horizon.wit index 9f00a12b1..e14913782 100644 --- a/crates/horizon-mod-sdk/wit/horizon.wit +++ b/crates/horizon-mod-sdk/wit/horizon.wit @@ -86,6 +86,14 @@ interface storage { delete: func(key: string); } +/// Capability `command`: vanilla commands run as the player, with the +/// player's permissions, after the hook call returns (8 per call). A mod's +/// own commands are not run this way. +interface command { + /// `line` without its slash; its feedback goes to the chat as if typed. + run: func(line: string); +} + /// What the game calls. Only the hooks `init` asks for are ever called. interface events { record registration { @@ -117,5 +125,6 @@ world horizon-mod { import chat; import hud; import storage; + import command; export events; } diff --git a/crates/horizon-mods/src/manifest.rs b/crates/horizon-mods/src/manifest.rs index a71210883..6f1674e05 100644 --- a/crates/horizon-mods/src/manifest.rs +++ b/crates/horizon-mods/src/manifest.rs @@ -69,16 +69,19 @@ pub enum Capability { Chat, Hud, Storage, + /// Runs vanilla commands as the player, with the player's permissions. + Command, /// New blocks and items from the mod's `data/` (API 0.2): no WIT /// interface, the game reads them before the world loads. Registries, } impl Capability { - pub const ALL: [Capability; 5] = [ + pub const ALL: [Capability; 6] = [ Capability::GameRead, Capability::Chat, Capability::Hud, Capability::Storage, + Capability::Command, Capability::Registries, ]; /// The WIT interface it grants, in `horizon:mods`. @@ -88,6 +91,7 @@ impl Capability { Capability::Chat => "chat", Capability::Hud => "hud", Capability::Storage => "storage", + Capability::Command => "command", Capability::Registries => "registries", } } @@ -98,6 +102,7 @@ impl Capability { Capability::Chat => "show you chat messages", Capability::Hud => "draw text on your screen", Capability::Storage => "save its own settings to disk", + Capability::Command => "run commands as you, with your permissions", Capability::Registries => "add blocks and items to your worlds", } } @@ -128,6 +133,7 @@ impl fmt::Display for Capability { Capability::Chat => "chat", Capability::Hud => "hud", Capability::Storage => "storage", + Capability::Command => "command", Capability::Registries => "registries", }; f.write_str(name) @@ -366,6 +372,10 @@ capabilities = ["game-read", "hud"] storage, Err("uses capability storage, missing from horizon-mod.toml".into()) ); + assert_eq!( + m.check_imports(["horizon:mods/command@0.1.0"]), + Err("uses capability command, missing from horizon-mod.toml".into()) + ); let with_storage = EXAMPLE.replace("\"hud\"", "\"hud\", \"storage\""); let m = Manifest::parse(&with_storage).unwrap(); assert_eq!(m.check_imports(["horizon:mods/storage@0.1.0"]), Ok(())); diff --git a/crates/horizon-mods/src/runtime.rs b/crates/horizon-mods/src/runtime.rs index 9dd2eb54e..07be81909 100644 --- a/crates/horizon-mods/src/runtime.rs +++ b/crates/horizon-mods/src/runtime.rs @@ -23,7 +23,7 @@ pub use bindings::horizon::mods::{ use bindings::{ HorizonMod, exports::horizon::mods::events::Registration, - horizon::mods::{chat, game, hud, log as mod_log, storage}, + horizon::mods::{chat, command, game, hud, log as mod_log, storage}, }; /// A mod's linear memory cap. @@ -67,6 +67,9 @@ pub enum Output { Chat { name: String, text: String }, /// A mod command's feedback, red when `error`. Feedback { text: String, error: bool }, + /// `command.run`: a vanilla command line, without its slash, to run as + /// the player. + Command { line: String }, /// The mod was turned off: it trapped, overran or failed to start. Stopped { name: String, reason: String }, } @@ -514,6 +517,18 @@ impl chat::Host for State { } } } +impl command::Host for State { + fn run(&mut self, line: String) { + let queued = self + .out + .iter() + .filter(|o| matches!(o, Output::Command { .. })) + .count(); + if queued < CHAT_PER_CALL { + self.out.push(Output::Command { line: clip(line) }); + } + } +} impl hud::Host for State { fn set_panel(&mut self, corner: Corner, lines: Vec) { let lines = lines diff --git a/crates/horizon-mods/tests/sandbox.rs b/crates/horizon-mods/tests/sandbox.rs index a246c07b0..c8e4fb883 100644 --- a/crates/horizon-mods/tests/sandbox.rs +++ b/crates/horizon-mods/tests/sandbox.rs @@ -1,4 +1,5 @@ -//! Loads real mods built with the SDK: the coords-hud example and a guest +//! Loads real mods built with the SDK: the coords-hud and predator-missile +//! examples and a guest //! that misbehaves on command. Needs `rustup target add wasm32-unknown-unknown`. use horizon_mods::{ Clock, Corner, Mods, Output, Package, Player, Snapshot, Status, manifest::componentize, @@ -18,7 +19,7 @@ api = "0.1" capabilities = ["chat", "storage"] "#; -/// Builds both mods once and writes their `.hmod` files; returns the folder. +/// Builds the mods once and writes their `.hmod` files; returns the folder. fn mods_dir() -> &'static Path { static DIR: OnceLock = OnceLock::new(); DIR.get_or_init(|| { @@ -31,6 +32,8 @@ fn mods_dir() -> &'static Path { "-p", "coords-hud", "-p", + "predator-missile", + "-p", "horizon-mods-test-guest", "--target-dir", ]) @@ -44,9 +47,13 @@ fn mods_dir() -> &'static Path { let built = target.join("wasm32-unknown-unknown/release"); let dir = target.join("mods"); std::fs::create_dir_all(&dir).unwrap(); - let example = root.join("crates/horizon-mod-sdk/examples/coords-hud/horizon-mod.toml"); + let example = |name: &str| { + let path = format!("crates/horizon-mod-sdk/examples/{name}/horizon-mod.toml"); + std::fs::read_to_string(root.join(path)).unwrap() + }; let mods = [ - ("coords_hud", std::fs::read_to_string(example).unwrap()), + ("coords_hud", example("coords-hud")), + ("predator_missile", example("predator-missile")), ("horizon_mods_test_guest", GUEST_MANIFEST.to_string()), ]; for (name, manifest) in mods { @@ -81,6 +88,36 @@ fn player() -> Snapshot { } } +/// Capability `command`: `/missile` queues vanilla commands, and the chat +/// line its impact check prints sets off the warhead. +#[test] +fn predator_missile_runs_commands() { + let mut mods = load("predator_missile.hmod"); + assert_eq!(mods.entries()[0].status, Status::Running); + mods.set_snapshot(player()); + let commands = |mods: &mut Mods| -> Vec { + mods.drain() + .into_iter() + .filter_map(|o| match o { + Output::Command { line } => Some(line), + _ => None, + }) + .collect() + }; + assert!(mods.command("missile")); + let launch = commands(&mut mods); + assert_eq!(launch.len(), 6); + assert!(launch.contains(&"gamemode spectator".to_string())); + mods.tick(1); + mods.tick(2); + let flight = commands(&mut mods); + assert!(flight.iter().any(|l| l.ends_with("run tp @s ^ ^ ^1.5"))); + mods.chat("Predator: impact"); + let blast = commands(&mut mods); + assert_eq!(blast.len(), 4); + assert!(blast.iter().all(|l| l.contains("summon tnt"))); +} + #[test] fn coords_hud_draws_and_answers() { let mut mods = load("coords_hud.hmod"); diff --git a/docs/api/mods-reference.md b/docs/api/mods-reference.md index ca2fe9d7a..28703e784 100644 --- a/docs/api/mods-reference.md +++ b/docs/api/mods-reference.md @@ -11,11 +11,12 @@ The contract between Horizon and its mods: the WIT package `horizon:mods@0.1.0`, | `chat` | [`chat`](#chat) | show you chat messages | | `hud` | [`hud`](#hud) | draw text on your screen | | `storage` | [`storage`](#storage) | save its own settings to disk | +| `command` | [`command`](#command) | run commands as you, with your permissions | | `registries` | data files (`assets/`, `data/`), no interface | add blocks and items to your worlds | ## World `horizon-mod` -- Imports (what the game offers): `log`, `game`, `chat`, `hud`, `storage` +- Imports (what the game offers): `log`, `game`, `chat`, `hud`, `storage`, `command` - Exports (what the mod provides): `events` ## `log` @@ -117,6 +118,16 @@ Capability `storage`: a per-mod folder of small key/value files, quota'd (docs/m | `set` | `func(key: string, value: list) -> result<_, string>` | Errs on a malformed key or over quota (bytes for the whole mod). | | `delete` | `func(key: string)` | No error if `key` was already absent. | +## `command` + +Access: capability `command`. + +Capability `command`: vanilla commands run as the player, with the player's permissions, after the hook call returns (8 per call). A mod's own commands are not run this way. + +| Function | Signature | What it does | +| --- | --- | --- | +| `run` | `func(line: string)` | `line` without its slash; its feedback goes to the chat as if typed. | + ## `events` Access: exported by the mod: the game calls it. diff --git a/docs/mods.md b/docs/mods.md index d8ea925b3..5d386720b 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -13,7 +13,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. |---|---|---| | The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage; tick, frame, chat and command hooks | | SDK for mod authors | `crates/horizon-mod-sdk` | 0.1 | -| Example mod | `crates/horizon-mod-sdk/examples/coords-hud` | coordinates, facing and time HUD, `/where`, `/coords` | +| Example mods | `crates/horizon-mod-sdk/examples/coords-hud`, `examples/predator-missile` | coordinates, facing and time HUD, `/where`, `/coords`; `/missile`, a guided missile from vanilla commands | | Runtime | `crates/horizon-mods` | wasmtime 36 LTS, component model | | Authoring tool | `crates/cargo-horizon-mod` | `cargo horizon-mod new / build / run` | | In the game | `crates/horizon-client/src/game/mods.rs`, Options > Mods | client mods | @@ -84,6 +84,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | `chat` | `chat` | `show(text)`: a line only the local player sees, after the mod's name (8 per call) | | `hud` | `hud` | `set-panel(corner, lines)`, `clear()`: up to 16 lines of 256 characters | | `storage` | `storage` | `get(key)`, `set(key, value)`, `delete(key)`: a per-mod folder of up to 64 files, 4 MiB total, at most 1 MiB written per call | +| `command` | `command` | `run(line)`: a vanilla command run as the player, with the player's permissions, after the hook returns (8 per call); mods' own commands are not run | | Hook (export) | When | |---|---| From c2c8df4542b857911ec396e0430da5b46fdffdb1 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:47:54 +0100 Subject: [PATCH 02/13] Mods: run a mod's commands on the next tick Run while the mod's output was shown, a command's chat line reached the mods again inside that call: a mod that answers a line with a command that prints one recursed without end, and when a control-socket command set the hook off, the mod's commands ran with the console's level. Queue them for the next tick instead, where the player's own level applies (the server already checks it on a server). Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game/mods.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/horizon-client/src/game/mods.rs b/crates/horizon-client/src/game/mods.rs index c203d0ca2..8364a4d58 100644 --- a/crates/horizon-client/src/game/mods.rs +++ b/crates/horizon-client/src/game/mods.rs @@ -39,6 +39,11 @@ pub(super) struct ModsState { /// The chat's command tree while a running mod owns commands. #[cfg(feature = "mods")] tree: Option, + /// `command.run` lines for this window's own world, run on the next tick: + /// run at once, their chat lines would reach the mods while their output + /// is still being shown, and a mod answering its own lines would recurse. + #[cfg(feature = "mods")] + commands: Vec, } impl ModsState { /// The chat's command tree: vanilla's, plus the running mods' commands. @@ -283,6 +288,11 @@ impl Game<'_> { } /// A game tick. pub(super) fn tick_mods(&mut self) { + // Here, at the player's level: shown inside a console call (a control + // socket command set the hook off), they would run as the console. + for line in std::mem::take(&mut self.mods.commands) { + self.run_command(&line); + } if !self.mods.mods.is_running() { return; } @@ -345,13 +355,13 @@ impl Game<'_> { // Vanilla's commands only: a mod running a mod command could loop. Output::Command { line } => { let name = line.split(' ').next().unwrap_or_default(); - // As handle_chat_input, without the chat history. if !vanilla_command(name) { - // dropped + // Dropped. } else if self.remote.is_some() { + // The server checks the player's permissions. self.send_chat(&format!("/{line}")); } else { - self.run_command(&line); + self.mods.commands.push(line); } continue; } From 5262effa7e4b13cd4737563dabeda3dc03e2f8af Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:48:35 +0100 Subject: [PATCH 03/13] cargo horizon-mod check: list the commands a mod runs Main's check reports every output; the command capability adds one. Co-Authored-By: Claude Opus 5.5 --- crates/cargo-horizon-mod/src/check.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/cargo-horizon-mod/src/check.rs b/crates/cargo-horizon-mod/src/check.rs index 0294bec98..a1cb44c9e 100644 --- a/crates/cargo-horizon-mod/src/check.rs +++ b/crates/cargo-horizon-mod/src/check.rs @@ -214,6 +214,7 @@ fn play(path: &Path, report: &mut Report) -> Result<(), String> { Output::Feedback { text, error } => { format!("{}: {text}", if error { "error" } else { "feedback" }) } + Output::Command { line } => format!("command: /{line}"), Output::Stopped { reason, .. } => return Err(format!("it stopped: {reason}")), }); } From 39904c8051546efdb808f0fa9a8ccb68043f7dd2 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:33:08 +0100 Subject: [PATCH 04/13] docs/mods.md: the contract lists the command interface Co-Authored-By: Claude Opus 5.5 --- docs/mods.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mods.md b/docs/mods.md index 2f7105398..a021d420b 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -12,7 +12,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | Part | Where | State | |---|---|---| -| The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage; tick, frame, chat and command hooks | +| The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage, command; tick, frame, chat and command hooks | | SDK for mod authors | `crates/horizon-mod-sdk` | 0.1 | | Example mods | `crates/horizon-mod-sdk/examples/coords-hud`, `examples/predator-missile` | coordinates, facing and time HUD, `/where`, `/coords`; `/missile`, a guided missile from vanilla commands | | Runtime | `crates/horizon-mods` | wasmtime 36 LTS, component model | From 1437537cbfd16e349ee39eef7ec544fb6cb3fe01 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:39:29 +0100 Subject: [PATCH 05/13] Worker threads: leave one core, as vanilla does, not two Graphics::worker_threads sizes the worker pools (the server thread's, which generates and lights, and the window's, which meshes). Its automatic count left two cores to the render and server threads; vanilla's executor leaves one (Util.maxAllowedExecutorThreads). The workers run behind those threads (niced on Linux; above four cores, raised server and render threads on macOS and Windows), so the second core mostly idled: on a 4-thread laptop only 2 cores generated while a world loaded. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/graphics.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/horizon-client/src/graphics.rs b/crates/horizon-client/src/graphics.rs index 3cf1402d2..291bf8a80 100644 --- a/crates/horizon-client/src/graphics.rs +++ b/crates/horizon-client/src/graphics.rs @@ -376,14 +376,18 @@ impl Graphics { if self.workers > 0 { return self.workers as usize; } - // The pool never takes every core. The render thread and the - // integrated server thread (single player runs on its own server - // thread by default) each need one, and a pool that fills every core - // starves them: the frame rate collapses to single digits while the - // workers saturate the CPU and its memory bandwidth. + // Vanilla's count (Util.maxAllowedExecutorThreads: every core but + // one). The pool never takes every core: a pool that fills them all + // starves the render and server threads, and the frame rate collapses + // while the workers saturate the CPU and its memory bandwidth. One + // core left is enough with the workers behind those two threads + // (horizon_threads: niced on Linux; on macOS and Windows the server + // and render threads are raised above four cores, where vanilla + // raises its server thread). Leaving two idled a core of a 4-thread + // laptop while a world loaded: a teleport there took ~15% longer. let auto = std::thread::available_parallelism() .map_or(2, |n| n.get()) - .saturating_sub(2) + .saturating_sub(1) .max(1); if self.battery { auto.div_ceil(2) } else { auto } } @@ -546,11 +550,11 @@ mod tests { } #[test] - fn automatic_workers_leave_cores_for_the_render_and_server_threads() { + fn automatic_workers_leave_a_core_as_vanilla_does() { let g = Graphics::default(); assert_eq!(g.workers, 0); let cores = std::thread::available_parallelism().map_or(2, |n| n.get()); - assert_eq!(g.worker_threads(), cores.saturating_sub(2).max(1)); + assert_eq!(g.worker_threads(), cores.saturating_sub(1).max(1)); // A manual count is the player's own choice, kept as asked. let manual = Graphics { workers: 3, ..g }; assert_eq!(manual.worker_threads(), 3); From 40c77c02f0332188ae7622dd7b2ae842df5becb7 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:15:19 +0100 Subject: [PATCH 06/13] PalettedContainer: look up a word's filled bits, not divide for them `locate` found an entry's word by multiplication (the MAGIC table) but still divided `64 / bits` at run time for the shift, on every block read and write of a section stored in 5, 6 or 7 bits. A division by a value the compiler does not know costs tens of cycles on x86: `perf annotate` of a 10-bot server on the ThinkPad put 12% of `PalettedContainer::get`, the server thread's top function, on that `div`. The table now also holds the bits a word's entries fill, so the shift is `i * bits - word * filled`. A test checks `locate` against the division for every width and entry. ThinkPad (i5-7200U), 4M random reads, 5-7 bit palettes: 28.5-36.9 -> 24.6-32.4 ns over 2,000 sections, 6.7-7.1 -> 5.4-5.8 ns within one; 4 and 8 bits (shifts, unchanged) the same. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-core/src/paletted.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/crates/horizon-core/src/paletted.rs b/crates/horizon-core/src/paletted.rs index 802eac225..9aa4aec6a 100644 --- a/crates/horizon-core/src/paletted.rs +++ b/crates/horizon-core/src/paletted.rs @@ -106,14 +106,16 @@ enum Palette { List(Vec), Global, } -/// `n / per_word` for `n < 4096` by multiplication: `MAGIC[bits] = ceil(2^20 / -/// (64 / bits))` is exact while `n * (64 / bits) < 2^20`. -const MAGIC: [u32; 33] = { - let mut table = [0; 33]; +/// `n / per_word` for `n < 4096` by multiplication: `MAGIC[bits].0 = ceil(2^20 / +/// (64 / bits))` is exact while `n * (64 / bits) < 2^20`. `.1` is the bits a +/// word's entries fill (`64 / bits * bits`): looked up, not divided for, as a +/// division by a run-time value costs tens of cycles on x86. +const MAGIC: [(u32, u32); 33] = { + let mut table = [(0, 0); 33]; let mut bits = 1; while bits <= 32 { let per = 64 / bits as u32; - table[bits] = (1u32 << 20).div_ceil(per); + table[bits] = ((1u32 << 20).div_ceil(per), per * bits as u32); bits += 1; } table @@ -134,9 +136,9 @@ fn locate(bits: u8, i: usize) -> (usize, u32) { let word = i >> (6 - log); (word, ((i & ((64 >> log) - 1)) as u32) << log) } else { - let per = 64 / u32::from(bits); - let word = (i as u32 * MAGIC[usize::from(bits)]) >> 20; - (word as usize, (i as u32 - word * per) * u32::from(bits)) + let (magic, filled) = MAGIC[usize::from(bits)]; + let word = (i as u32 * magic) >> 20; + (word as usize, i as u32 * u32::from(bits) - word * filled) } } #[inline] @@ -720,6 +722,17 @@ mod tests { use super::*; const BLOCKS: Strategy = Strategy::blocks(16); const BIOMES: Strategy = Strategy::biomes(7); + + #[test] + fn locate_matches_division_for_every_width() { + for bits in 1..=32u8 { + let per = 64 / usize::from(bits); + for i in 0..4096 { + let expected = (i / per, ((i % per) * usize::from(bits)) as u32); + assert_eq!(locate(bits, i), expected, "{bits} bits, entry {i}"); + } + } + } fn rng(state: &mut u64) -> u64 { *state ^= *state << 13; *state ^= *state >> 7; From c5a640dd56c0ac7607fc056934ef7471a6eb60a6 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:59:56 +0100 Subject: [PATCH 07/13] Endermen, endermites, blazes, eyes of ender, dried ghasts and end portals make their ambient effects The client-side branches vanilla runs on its own level: Enderman and Endermite aiStep (two portal particles a tick), Blaze aiStep (two large smoke particles, the burn sound one tick in 24), EyeOfEnder.tick (a portal particle behind it, four bubbles in water), and the DriedGhastBlock and EndPortalBlock animateTick. They run on the window's game in-frame and as a server's client, never on a server. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/animate_tick.rs | 50 +++++++++++++++++ crates/horizon-client/src/game.rs | 2 + .../horizon-client/src/game/client_ambient.rs | 37 +++++++++++++ crates/horizon-client/src/game/end.rs | 36 +++++++++++- crates/horizon-client/src/game/remote.rs | 1 + crates/horizon-client/src/mobs.rs | 1 + crates/horizon-client/src/mobs/ambient.rs | 55 +++++++++++++++++++ 7 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 crates/horizon-client/src/game/client_ambient.rs create mode 100644 crates/horizon-client/src/mobs/ambient.rs diff --git a/crates/horizon-client/src/animate_tick.rs b/crates/horizon-client/src/animate_tick.rs index ce34df510..48e8f16f4 100644 --- a/crates/horizon-client/src/animate_tick.rs +++ b/crates/horizon-client/src/animate_tick.rs @@ -63,6 +63,10 @@ enum Animate { SculkSensor, /// Potent sulfur that is not dry. PotentSulfur, + DriedGhast { + waterlogged: bool, + }, + EndPortal, } fn step(facing: &str) -> [i32; 3] { match facing { @@ -137,6 +141,10 @@ fn classify(name: &str, prop: &dyn Fn(&str) -> Option) -> Animate { drag: prop("drag").as_deref() == Some("true"), }, "fire" | "soul_fire" => Animate::Fire, + "dried_ghast" => Animate::DriedGhast { + waterlogged: prop("waterlogged").as_deref() == Some("true"), + }, + "end_portal" => Animate::EndPortal, "sculk_sensor" | "calibrated_sculk_sensor" if prop("sculk_sensor_phase").as_deref() == Some("active") => { @@ -180,6 +188,8 @@ pub struct AnimateTables { impermeable: Vec, power_provider: Vec, power_transmitter: Vec, + /// `#triggers_ambient_dried_ghast_block_sounds` (soul sand, soul soil). + dried_ghast_sounds: Vec, /// Biome id -> ambient particles and probabilities (`minecraft:visual/ambient_particles`). ambient: Vec>, } @@ -237,6 +247,7 @@ impl AnimateTables { impermeable: by_state("minecraft:impermeable"), power_provider: by_state("minecraft:enchantment_power_provider"), power_transmitter: by_state("minecraft:enchantment_power_transmitter"), + dried_ghast_sounds: by_state("minecraft:triggers_ambient_dried_ghast_block_sounds"), ambient, }) } @@ -869,6 +880,45 @@ fn block_animate_tick( ]; add(engine, T::Smoke, p, [0.0; 3]); } + Animate::DriedGhast { waterlogged } => { + // DriedGhastBlock.animateTick. + let c = [fx + 0.5, fy + 0.5, fz + 0.5]; + if waterlogged { + if next_int(engine.random(), 40) == 0 { + let name = "minecraft:block.dried_ghast.ambient_water"; + sound(engine, name, c, 1.0, 1.0); + } + let r = engine.random(); + if next_int(r, 6) == 0 { + let x = c[0] + f64::from((r.next_float() * 2.0 - 1.0) / 3.0); + let y = c[1] + 0.4; + let z = c[2] + f64::from((r.next_float() * 2.0 - 1.0) / 3.0); + let v = [0.0, f64::from(r.next_float()), 0.0]; + add(engine, T::HappyVillager, [x, y, z], v); + } + } else { + let below = ctx.state([pos[0], pos[1] - 1, pos[2]]); + let triggers = || { + tables + .dried_ghast_sounds + .get(below.0 as usize) + .copied() + .unwrap_or(false) + }; + if next_int(engine.random(), 40) == 0 && triggers() { + sound(engine, "minecraft:block.dried_ghast.ambient", c, 1.0, 1.0); + } + if next_int(engine.random(), 6) == 0 { + add(engine, T::WhiteSmoke, c, [0.0, 0.02, 0.0]); + } + } + } + Animate::EndPortal => { + // EndPortalBlock.animateTick: one smoke particle above the surface. + let r = engine.random(); + let p = [fx + r.next_double(), fy + 0.8, fz + r.next_double()]; + add(engine, T::Smoke, p, [0.0; 3]); + } Animate::EnderChest => { for _ in 0..3 { let r = engine.random(); diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index 699468651..46621082e 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -20,6 +20,7 @@ mod bossbars; mod breezes; mod cartography; mod chat; +mod client_ambient; mod clipboard; mod combat; mod command_blocks; @@ -4447,6 +4448,7 @@ impl Game<'_> { self.tick_block_entity_anims() }); horizon_client::spans::time("tick_particles", || self.tick_particles()); + self.tick_client_ambient(); } // ServerBossEvent broadcasts, after this level's entities have ticked. self.publish_boss_bars(); diff --git a/crates/horizon-client/src/game/client_ambient.rs b/crates/horizon-client/src/game/client_ambient.rs new file mode 100644 index 000000000..a10c6da49 --- /dev/null +++ b/crates/horizon-client/src/game/client_ambient.rs @@ -0,0 +1,37 @@ +//! The window's own ambient effects of entities, which a client makes for +//! itself (`level().isClientSide()` branches): endermen's and endermites' +//! portal particles, blazes' smoke and burn, eyes of ender's trail. They run +//! on the game that has the window, in-frame or following a server, never on +//! a server: nothing of them goes over the network. +use super::Game; +use horizon_client::mobs::{MobEvent, ambient}; +use horizon_client::particles::{ParticleOptions, ParticleType}; + +impl Game<'_> { + pub(super) fn tick_client_ambient(&mut self) { + if self.serving.is_some() || self.options.dedicated.is_some() { + return; + } + let mut events = Vec::new(); + for m in self.mobs.iter_mut().filter(|m| !m.removed) { + ambient::client_ai_step(m, &mut events); + } + for event in events { + match event { + MobEvent::ParticleAt { kind, at, velocity } => { + if let Some(ty) = ParticleType::from_name(kind) { + self.particles + .add(&ParticleOptions::simple(ty), at, velocity); + } + } + // Each guest of a LAN host makes its own. + MobEvent::LocalSound(name, at, volume, pitch) => { + self.audio + .play_unshared(&format!("minecraft:{name}"), Some(at), volume, pitch) + } + _ => {} + } + } + self.eye_particles(); + } +} diff --git a/crates/horizon-client/src/game/end.rs b/crates/horizon-client/src/game/end.rs index 599158e3e..655971a5c 100644 --- a/crates/horizon-client/src/game/end.rs +++ b/crates/horizon-client/src/game/end.rs @@ -4,7 +4,12 @@ //! inactive until the dragon dies. use super::{Dim, Game}; use horizon_client::{ - Result, environment::Environment, items::ItemStack, mesher::Vertex, motion::PlayerMotion, + Result, + environment::Environment, + items::ItemStack, + mesher::Vertex, + motion::PlayerMotion, + particles::{ParticleOptions, ParticleType}, vanillagen::VanillaTerrain, }; use horizon_core::{ @@ -1122,6 +1127,35 @@ impl Game<'_> { self.end.eyes = eyes; } + /// `EyeOfEnder.tick` on a client: behind each eye (a quarter of its last + /// move back), four bubbles in water, else a portal particle. + pub(super) fn eye_particles(&mut self) { + for i in 0..self.end.eyes.len() { + let e = &self.end.eyes[i]; + let movement = (e.position - e.previous).to_array(); + let origin = (e.position - (e.position - e.previous) * 0.25).to_array(); + let block = origin.map(|v| v.floor() as i32); + let water = self + .res + .facts() + .get(self.world.block(block[0], block[1], block[2])) + .fluid + == horizon_client::blocks::Fluid::Water; + if water { + let bubble = ParticleOptions::simple(ParticleType::Bubble); + for _ in 0..4 { + self.particles.add(&bubble, origin, movement); + } + } else { + let r = self.particles.random(); + let x = origin[0] + r.next_double() * 0.6 - 0.3; + let z = origin[2] + r.next_double() * 0.6 - 0.3; + let portal = ParticleOptions::simple(ParticleType::Portal); + self.particles + .add(&portal, [x, origin[1] - 0.5, z], movement); + } + } + } /// ThrownItemRenderer: the ender eye item facing the camera. fn eye_geometry(&self, camera: glam::DVec3, out: &mut Vec) { let Some(r) = self.gui.atlas.sprites.get("item/ender_eye").copied() else { diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index 044e43835..711753869 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -3187,6 +3187,7 @@ impl Game<'_> { self.tick_block_entity_anims(); self.tick_hand(); self.tick_particles(); + self.tick_client_ambient(); self.tick_client_effects(); // Player.tick's sleep counter (the darkening and the wake-up fade). self.sleep_timer = super::tick_sleep_timer(self.sleep_timer, self.sleeping.is_some()); diff --git a/crates/horizon-client/src/mobs.rs b/crates/horizon-client/src/mobs.rs index 4e3879fe5..6dee6a829 100644 --- a/crates/horizon-client/src/mobs.rs +++ b/crates/horizon-client/src/mobs.rs @@ -7,6 +7,7 @@ //! behaviour, models, animations and sounds. pub mod ai; pub mod allay; +pub mod ambient; pub mod anim; pub mod animal_brain; pub mod aquatic; diff --git a/crates/horizon-client/src/mobs/ambient.rs b/crates/horizon-client/src/mobs/ambient.rs new file mode 100644 index 000000000..cb4146e08 --- /dev/null +++ b/crates/horizon-client/src/mobs/ambient.rs @@ -0,0 +1,55 @@ +//! What a client's copy of a mob makes on its own, in the +//! `level().isClientSide()` branches of `aiStep`: nothing of it reaches the +//! server or goes over the network, so the window's game runs it, whether it +//! simulates the world (in-frame) or follows a server. +use super::{Mob, MobEvent, Species}; +use horizon_core::random::RandomSource; + +/// `Enderman.aiStep`, `Endermite.aiStep` and `Blaze.aiStep` on a client: +/// two portal particles a tick around endermen and endermites; two large +/// smoke particles a tick around blazes and, one tick in 24, their burn. +pub fn client_ai_step(m: &mut Mob, out: &mut Vec) { + let species = m.species(); + if !matches!( + species, + Species::Enderman | Species::Endermite | Species::Blaze + ) { + return; + } + let (w, h) = (f64::from(m.width()), f64::from(m.height())); + let p = m.position; + if species == Species::Blaze && m.random.next_int_bound(24).unwrap_or(1) == 0 && !m.silent { + let volume = 1.0 + m.random.next_float(); + let pitch = m.random.next_float() * 0.7 + 0.3; + out.push(MobEvent::LocalSound( + "entity.blaze.burn", + [p.x + 0.5, p.y + 0.5, p.z + 0.5], + volume, + pitch, + )); + } + for _ in 0..2 { + // Entity.getRandomX(0.5), getRandomY(), getRandomZ(0.5). + let r = &mut m.random; + let x = p.x + w * (2.0 * r.next_double() - 1.0) * 0.5; + let y = p.y + h * r.next_double(); + let z = p.z + w * (2.0 * r.next_double() - 1.0) * 0.5; + let (kind, at, velocity) = if species == Species::Blaze { + ("large_smoke", [x, y, z], [0.0; 3]) + } else { + // The enderman's are a quarter block lower. + let y = if species == Species::Enderman { + y - 0.25 + } else { + y + }; + let velocity = [ + (r.next_double() - 0.5) * 2.0, + -r.next_double(), + (r.next_double() - 0.5) * 2.0, + ]; + ("portal", [x, y, z], velocity) + }; + out.push(MobEvent::ParticleAt { kind, at, velocity }); + } +} From 116a7f7dcafc4e36d14b75600f402446ec8056ac Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:48:43 +0100 Subject: [PATCH 08/13] horizon-prepare: OUTPUT_VERSION, the number of what it prepares, checked by CI Launchers copy a player's prepared resources to a new release while its preparer declares the same OUTPUT_VERSION (the launcher side is its own pull request). The number is bumped by hand; CI keeps it honest: the parity job already prepares with every change to the preparer, and tools/prepare_output.py check now fails when that preparation's digest is not the one crates/horizon-prepare/output-versions.txt gives the current number, or when the file changed other than by appending a version. The digest canonicalises what differs between two runs of the same preparer (JSON key order, creative tab tag order, lambda class addresses, logs, the sounds): 10 preparations by 5 preparer versions on the three Mac runners give one digest, and a Linux preparation by an older preparer gives the Mac runners' digest for it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parity.yml | 4 + crates/horizon-prepare/output-versions.txt | 3 + crates/horizon-prepare/src/lib.rs | 32 ++++ docs/release-flow.md | 4 + tests/test_prepare_output.py | 64 ++++++++ tools/prepare_output.py | 176 +++++++++++++++++++++ 6 files changed, 283 insertions(+) create mode 100644 crates/horizon-prepare/output-versions.txt create mode 100644 tests/test_prepare_output.py create mode 100755 tools/prepare_output.py diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index a18fc5019..0588c12a3 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -37,6 +37,10 @@ jobs: run: bash tools/ci/runner-env.sh - name: Vanilla data, and resources prepared by this preparer run: bash tools/ci/vanilla-data.sh + - name: The prepared output is the one the preparer's OUTPUT_VERSION says + env: + BASE_SHA: ${{ inputs.base }} + run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA" - name: The version constants are the pinned jar's run: python3 tools/gen_version_consts.py --check - name: The harnesses this change can affect diff --git a/crates/horizon-prepare/output-versions.txt b/crates/horizon-prepare/output-versions.txt new file mode 100644 index 000000000..71a531fab --- /dev/null +++ b/crates/horizon-prepare/output-versions.txt @@ -0,0 +1,3 @@ +# OUTPUT_VERSION (src/lib.rs) and the digest of what it prepares (python3 tools/prepare_output.py digest), +# one line per version, never edited: a change to the prepared files appends the next version. +1 0a608bf4115a8ed2e2c4f5f2b2609dc10c54e63479af8e56fabaa10af1060389 diff --git a/crates/horizon-prepare/src/lib.rs b/crates/horizon-prepare/src/lib.rs index c7e443c7d..5ea1121df 100644 --- a/crates/horizon-prepare/src/lib.rs +++ b/crates/horizon-prepare/src/lib.rs @@ -31,6 +31,15 @@ pub type Log<'a> = &'a dyn Fn(&str); /// The manifest's `schema`: the prepared files' layout, which the launcher compares. pub const SCHEMA: i64 = 7; +/// What this preparer makes. The launcher copies a prepared version's resources to a newly installed +/// version whose preparer has the same number (and the same pinned jars), so a release that changes +/// this crate without changing its output asks nobody to prepare again. Bump it, and append the new +/// digest to output-versions.txt, when the prepared files change: CI prepares with every change to +/// the preparer and fails when the output is not the one output-versions.txt gives this number +/// (`python3 tools/prepare_output.py check`). The launcher reads this line from the version +/// folder's sources: keep it on one line, as it is. +pub const OUTPUT_VERSION: u32 = 1; + /// Digest of this crate's sources (build.rs): the launcher prepares a version in-process when /// that version ships these same sources, and runs the version's own binary otherwise. pub const SOURCES_SHA256: &str = env!("HORIZON_PREPARE_SOURCES_SHA256"); @@ -1209,6 +1218,29 @@ mod tests { fs::remove_dir_all(dir).unwrap(); } + #[test] + fn output_versions_end_at_the_output_version() { + let text = include_str!("../output-versions.txt"); + let versions: Vec<(u32, &str)> = text + .lines() + .filter(|l| !l.trim().is_empty() && !l.starts_with('#')) + .map(|l| { + let (n, digest) = l.split_once(' ').unwrap(); + (n.parse().unwrap(), digest.trim()) + }) + .collect(); + let numbers: Vec = versions.iter().map(|v| v.0).collect(); + assert_eq!(numbers, (1..=OUTPUT_VERSION).collect::>()); + for (i, (_, digest)) in versions.iter().enumerate() { + assert_eq!(digest.len(), 64); + assert!(!versions[..i].iter().any(|(_, d)| d == digest)); + } + // The launcher finds the number with this line in a version folder's sources. + let lib = include_str!("lib.rs"); + let line = format!("pub const OUTPUT_VERSION: u32 = {OUTPUT_VERSION};"); + assert!(lib.lines().any(|l| l == line)); + } + #[test] fn the_source_digest_is_the_one_build_rs_computed() { assert_eq!( diff --git a/docs/release-flow.md b/docs/release-flow.md index 4fb0209d9..1413cde01 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -39,6 +39,10 @@ copy is: the Mac mini. Its runner runs as the maintainer's account and keeps its depend only on the Minecraft version, are kept once in `shared-local/sounds//` (copied from a folder `horizon-prepare sounds` made from the maintainer's own game; the Mac mini has no official launcher of its own) and cloned into each preparation (not linked: vanilla's packs refuse symbolic links), for the audio harnesses. + `tools/prepare_output.py check` then compares what was prepared with `crates/horizon-prepare/output-versions.txt`: + launchers keep a player's prepared resources across releases whose preparer declares the same `OUTPUT_VERSION` + (`crates/horizon-prepare/src/lib.rs`), so a change to the prepared files fails here until it bumps the number + and appends the new digest (the error gives both lines). 3. `tools/parity.py --changed ` picks the harnesses whose traced dependencies (their script, Python modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to diff --git a/tests/test_prepare_output.py b/tests/test_prepare_output.py new file mode 100644 index 000000000..00c489fbb --- /dev/null +++ b/tests/test_prepare_output.py @@ -0,0 +1,64 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from tools import prepare_output + + +class PrepareOutputTests(unittest.TestCase): + def preparation(self, root: Path, files: dict[str, bytes]) -> Path: + for rel, data in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return root + + def test_what_varies_between_runs_does_not_count(self): + manifest = {'schema': 7, 'data_version': 5023, 'render_files_sha256': {'a.json': 'x'}} + a = { + 'horizon-resources.json': json.dumps(manifest).encode(), + 'render-definitions.json': b'{"a": 1, "b": [1, 2]}', + 'creative-inventory.json': b'{"tags": [["b", "a"]]}', + 'block-behaviours.json': b'{"c": "Foo$$Lambda/0x0000012a"}', + 'export.log': b'run 1', + 'assets/minecraft/sounds/a.ogg': b'one', + } + manifest['render_files_sha256'] = {'a.json': 'y'} + b = { + 'horizon-resources.json': json.dumps(manifest).encode(), + 'render-definitions.json': b'{"b": [1, 2], "a": 1}', + 'creative-inventory.json': b'{"tags": [["a", "b"]]}', + 'block-behaviours.json': b'{"c": "Foo$$Lambda/0x00000ff0"}', + 'export.log': b'run 2', + 'assets/minecraft/sounds/a.ogg': b'two', + } + with tempfile.TemporaryDirectory() as tmp: + first = prepare_output.digest(self.preparation(Path(tmp) / 'a', a)) + second = prepare_output.digest(self.preparation(Path(tmp) / 'b', b)) + self.assertEqual(first, second) + # The data does, and so do the manifest's schema and the files there are. + (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [2, 1], "a": 1}') + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [1, 2], "a": 1}') + (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps({**manifest, 'schema': 8})) + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps(manifest)) + (Path(tmp) / 'b/new.nbt').write_bytes(b'') + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + + def test_the_versions_file_ends_at_the_constant(self): + versions = prepare_output.parse_versions('# comment\n1 aa\n2 bb\n') + self.assertEqual(versions, [(1, 'aa'), (2, 'bb')]) + self.assertEqual(prepare_output.problems(versions, 2), []) + self.assertTrue(prepare_output.problems(versions, 1)) + self.assertTrue(prepare_output.problems([(1, 'aa'), (3, 'bb')], 3)) + self.assertTrue(prepare_output.problems([(1, 'aa'), (2, 'aa')], 2)) + root = Path(prepare_output.__file__).resolve().parent.parent + current = prepare_output.parse_versions((root / prepare_output.VERSIONS).read_text()) + constant = prepare_output.CONSTANT.search((root / prepare_output.LIB).read_text()) + self.assertEqual(prepare_output.problems(current, int(constant.group(1))), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/prepare_output.py b/tools/prepare_output.py new file mode 100755 index 000000000..81ed156cb --- /dev/null +++ b/tools/prepare_output.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""The prepared output's version: what keeps `OUTPUT_VERSION` (crates/horizon-prepare/src/lib.rs) honest. + +The launcher copies a prepared version's resources (`.local/horizon-client/`) to a newly installed +one when both pin the same Minecraft jars and their preparers declare the same `OUTPUT_VERSION`, so a release +that changes the preparer's code without changing what it makes asks nobody to prepare again. The number is +bumped by hand; this tool catches a change to the output without a bump. + +crates/horizon-prepare/output-versions.txt lists every `OUTPUT_VERSION` with the digest of what it prepares. +It only grows: the last line is the current version, and a change to the output appends a line. + + python3 tools/prepare_output.py digest [RESOURCES] the digest of a preparation + python3 tools/prepare_output.py check [--base SHA] [--cache] [RESOURCES] + fails when RESOURCES (default: this checkout's .local/horizon-client/, as CI prepares it) is + not what the last line says, or when the file changed other than by appending since SHA. --cache keeps + the digest beside RESOURCES (CI: a preparation never changes once made; it takes ~30 s to compute) + +The digest covers every file of the preparation, canonicalised where the preparer is not deterministic between +runs (verified across the CI runners): JSON is compared as data (key order, the order of each `tags` list of +creative-inventory.json, and the addresses in `$$Lambda/0x...` class names don't count); logs, compiled classes, +the sounds (copied from the player's launcher by another step) and the manifest's file hashes (raw bytes) don't +count. The manifest's own scalar fields (schema, data version, jar hashes, file count) do. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +VERSIONS = 'crates/horizon-prepare/output-versions.txt' +LIB = 'crates/horizon-prepare/src/lib.rs' +MANIFEST = 'horizon-resources.json' +LAMBDA = re.compile(r'\$\$Lambda/0x[0-9a-fA-F]+') +CONSTANT = re.compile(r'^pub const OUTPUT_VERSION: u32 = (\d+);', re.M) + + +def canonical_json(value, key=''): + if isinstance(value, dict): + return {k: canonical_json(v, k) for k, v in value.items()} + if isinstance(value, list): + items = [canonical_json(v, key) for v in value] + if key == 'tags' and all(isinstance(x, list) and all(isinstance(t, str) for t in x) for x in items): + items = [sorted(x) for x in items] + return items + if isinstance(value, str): + return LAMBDA.sub('$$Lambda', value) + return value + + +def canonical(rel: str, data: bytes) -> bytes: + if rel == MANIFEST: + manifest = json.loads(data) + scalars = {k: v for k, v in manifest.items() if not isinstance(v, (dict, list))} + return json.dumps(scalars, sort_keys=True).encode() + if rel.endswith('.json'): + try: + value = json.loads(data) + except ValueError: + return data + return json.dumps(canonical_json(value), sort_keys=True).encode() + return data + + +def skipped(rel: str) -> bool: + parts = rel.split('/') + return rel.endswith(('.log', '.class')) or parts[:3] == ['assets', 'minecraft', 'sounds'] \ + or rel == 'assets/minecraft/sounds.json' + + +def digest(resources: Path) -> tuple[str, int]: + """The digest of a preparation, and how many files it covers.""" + if not (resources / MANIFEST).is_file(): + raise SystemExit(f'{resources} is not a preparation: no {MANIFEST}') + files = [] + for dirpath, dirs, names in os.walk(resources): + dirs.sort() + for name in names: + path = Path(dirpath) / name + rel = path.relative_to(resources).as_posix() + if path.is_symlink() or skipped(rel): + continue + files.append(rel) + total = hashlib.sha256() + for rel in sorted(files): + data = hashlib.sha256(canonical(rel, (resources / rel).read_bytes())).hexdigest() + total.update(f'{rel}\0{data}\n'.encode()) + return total.hexdigest(), len(files) + + +def parse_versions(text: str) -> list[tuple[int, str]]: + out = [] + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + number, value = line.split() + out.append((int(number), value)) + return out + + +def problems(versions: list[tuple[int, str]], constant: int | None) -> list[str]: + found = [] + if not versions: + return [f'{VERSIONS} lists no version'] + if [n for n, _ in versions] != list(range(1, len(versions) + 1)): + found.append(f'{VERSIONS}: the versions must be 1, 2, 3... in order') + if len({d for _, d in versions}) != len(versions): + found.append(f'{VERSIONS}: two versions with the same digest') + if constant != versions[-1][0]: + found.append(f'{LIB} says OUTPUT_VERSION = {constant}, {VERSIONS} ends at {versions[-1][0]}') + return found + + +def default_resources() -> Path: + lock = json.loads((ROOT / 'research-lock.json').read_text()) + return ROOT / '.local/horizon-client' / lock['minecraft']['version'] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + parser.add_argument('command', choices=['digest', 'check']) + parser.add_argument('resources', nargs='?', type=Path) + parser.add_argument('--base', help='the commit the change is based on (check: the file only grew since)') + parser.add_argument('--cache', action='store_true', help='keep the digest beside the preparation') + args = parser.parse_args() + resources = (args.resources or default_resources()).resolve() + tool = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()[:16] + cache = resources.parent / f'.output-digest-{resources.name}-{tool}' + if args.cache and cache.is_file(): + value, count = cache.read_text().split() + else: + value, count = digest(resources) + if args.cache: + cache.write_text(f'{value} {count}\n') + if args.command == 'digest': + print(value) + return 0 + + versions = parse_versions((ROOT / VERSIONS).read_text()) + match = CONSTANT.search((ROOT / LIB).read_text()) + found = problems(versions, int(match.group(1)) if match else None) + if args.base: + shown = subprocess.run(['git', 'show', f'{args.base}:{VERSIONS}'], cwd=ROOT, capture_output=True, text=True) + if shown.returncode == 0: + before = parse_versions(shown.stdout) + if versions[:len(before)] != before: + found.append(f'{VERSIONS} changed lines it already had (since {args.base[:12]}): ' + 'a new output gets a new version, appended') + current = versions[-1] if versions else (0, '') + if value != current[1]: + known = [n for n, d in versions if d == value] + if known: + found.append(f'the preparation is OUTPUT_VERSION {known[0]}\'s output, but OUTPUT_VERSION is {current[0]}') + else: + found.append( + f'the prepared output changed ({count} files, digest {value}) but OUTPUT_VERSION is still ' + f'{current[0]}. Players would keep resources prepared by the old preparer. Bump it:\n' + f' set `pub const OUTPUT_VERSION: u32 = {current[0] + 1};` in {LIB}\n' + f' append `{current[0] + 1} {value}` to {VERSIONS}\n' + ' (if the change is only a nondeterminism this tool should ignore, teach canonical() instead)') + for problem in found: + print(f'::error::{problem}' if os.environ.get('GITHUB_ACTIONS') else f'error: {problem}', file=sys.stderr) + if not found: + print(f'prepared output: OUTPUT_VERSION {current[0]} ({count} files, {value[:16]})') + return 1 if found else 0 + + +if __name__ == '__main__': + sys.exit(main()) From 21d8e530346f3e05c0256e2c085bceb0103f823d Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:14:51 +0100 Subject: [PATCH 09/13] parity: pick harnesses by the Rust items their probes reach, not whole crates `--changed` / `--changed-files` now narrow each harness's recorded Rust sources to those its probes can reach (tools/parity_reach.py): an item-level reachability walk from the probe modules a harness ran (or the binary it built) over free items, inherent methods, trait impls and macros, matched by name so it only over-approximates. Parent modules count through their `mod` declarations, compared with the merge base. A harness whose probe can't be placed keeps every source; `--by-crate` turns the narrowing off. On the last 20 PRs merged on 2026-10-25: 2255 -> 1061 harnesses picked, 2384 -> 1592 harness-minutes. Replaying the week's 61 failed CI parity runs, every failed harness the old selection picked is still picked; LLVM coverage of six harnesses' probes executes only kept files. Co-Authored-By: Claude Opus 5.5 --- docs/parity-workflow.md | 4 + tests/test_parity_reach.py | 111 ++++++ tools/parity.py | 49 ++- tools/parity_reach.py | 768 +++++++++++++++++++++++++++++++++++++ 4 files changed, 925 insertions(+), 7 deletions(-) create mode 100644 tests/test_parity_reach.py create mode 100644 tools/parity_reach.py diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index f9f222071..d794b9c71 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -49,6 +49,10 @@ A harness can declare, at the top level of its script: - `PARITY_SHARDS = N`: it runs as `NAME#1` to `NAME#N`, in parallel, each told its share in `PARITY_SHARD=i/N` (`test_datafix_parity.py` upgrades every sixth world). `NAME` on the command line or in `--known-failures` means all of them. - `PARITY_RUST_SCOPE = [...]`: the Rust sources its probes can reach, as path prefixes. Cargo's dep-info names every file of every crate a probe links, so without it a probe of `horizon-world` depends on all 350 of its files. The runner keeps only the scope's files if the scope is closed: nothing in it names a module, crate-root re-export or macro outside it (`crate::`, `super::`, other workspace crates; comments and strings ignored), declares a module it doesn't cover, or includes source, and nothing outside it implements anything for a type it defines. Crate roots then count without their declarations of other modules, so adding a module elsewhere doesn't rerun the harness. If the scope isn't closed the runner says why and keeps every file. `datafix` depends on 41 files this way, not 354. +### Which harnesses a change affects + +`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `.local/parity-runner/reach-cache.json`. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses. + ## The gate `tools/gate.py` is what a branch must pass before it is integrated: the public-tree check, rustfmt, clippy with warnings denied, the Rust and Python tests, then the harnesses whose inputs changed. It stops at the first failure and writes `.local/gate.json`. `--known-failures a,b` lets harnesses that already fail on the integration branch keep failing. `--quick` checks formatting, lints and Python tests only; `--no-parity` skips the harnesses. diff --git a/tests/test_parity_reach.py b/tests/test_parity_reach.py new file mode 100644 index 000000000..9b7464b47 --- /dev/null +++ b/tests/test_parity_reach.py @@ -0,0 +1,111 @@ +"""tools/parity_reach.py: which workspace sources a harness's probe can reach.""" +from pathlib import Path +import tempfile +import textwrap +import unittest + +from tools import parity, parity_reach + +PROBE = [["example", "world_probe", "release", ["-p", "a"], True]] + + +def workspace(root, files): + for path, text in files.items(): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_text(textwrap.dedent(text), encoding="utf-8") + + +class ReachTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.root = Path(self.directory.name) + workspace(self.root, { + "crates/a/Cargo.toml": '[package]\nname = "a"\n\n[[example]]\nname = "world_probe"\npath = "examples/probe/main.rs"\n', + "crates/a/src/lib.rs": """ + pub mod used; + pub mod unused; + mod methods; + mod display; + mod table; + #[cfg(test)] + mod tests; + """, + "crates/a/src/used.rs": """ + pub struct Thing; + pub fn helper() -> u32 { 1 } + """, + # Same names as used.rs, but nothing names this module. + "crates/a/src/unused.rs": "pub struct Thing;\npub fn helper() -> u32 { 2 }\n", + "crates/a/src/methods.rs": """ + use crate::used::Thing; + impl Thing { + pub fn method(&self) -> u32 { 3 } + pub fn never_called(&self) -> u32 { 4 } + } + """, + "crates/a/src/display.rs": """ + impl std::fmt::Display for crate::used::Thing { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "thing") } + } + """, + "crates/a/src/table.rs": """ + macro_rules! table { () => { pub struct Table; }; } + table!(); + """, + "crates/a/src/tests.rs": "use crate::used::Thing;\nimpl Thing { pub fn method_in_tests(&self) {} }\n", + "crates/a/examples/probe/main.rs": "mod dispatch;\nmod p;\n", + "crates/a/examples/probe/dispatch.rs": "pub fn args() {}\n", + "crates/a/examples/probe/p.rs": """ + use a::used::{helper, Thing}; + pub fn main() { println!("{} {} {}", helper(), Thing.method(), Thing); } + """, + }) + self.dependencies = sorted(str(path.relative_to(self.root)) for path in self.root.rglob("*") if path.is_file()) + self.dependencies.remove("crates/a/examples/probe/main.rs") # parity.only_probes_run leaves it out + + def tearDown(self): + self.directory.cleanup() + + def narrowed(self, changed=(), artifacts=PROBE): + graph = parity_reach.SourceGraph(self.root, parity.rust_code) + return parity_reach.narrow(self.dependencies, artifacts, graph, set(changed)) + + def test_keeps_what_the_probe_reaches(self): + kept = {path for path, (whole, _) in self.narrowed().items() if whole} + self.assertIn("crates/a/src/used.rs", kept) + self.assertIn("crates/a/src/methods.rs", kept) # Thing.method() + self.assertIn("crates/a/src/display.rs", kept) # std's Display for a reached type + self.assertIn("crates/a/Cargo.toml", kept) + self.assertNotIn("crates/a/src/unused.rs", kept) # its names match, but nothing names its module + self.assertNotIn("crates/a/src/table.rs", kept) + self.assertNotIn("crates/a/src/tests.rs", kept) # compiled into tests only + + def test_module_parents_count_through_their_declarations(self): + self.assertEqual(self.narrowed()["crates/a/src/lib.rs"], (False, {"used", "methods", "display"})) + text = (self.root / "crates/a/src/lib.rs").read_text() + self.assertEqual(parity_reach.declarations(text, {"used"}, False), "pub mod used;") + self.assertEqual(parity_reach.declarations(text, {"tests"}, False), "#[cfg(test)]\nmod tests;") + # Adding or dropping a module beside the reached ones changes neither view. + other = text.replace("mod table;", "mod table2;\nmod more;") + self.assertEqual(parity_reach.declarations(other, {"used"}, False), "pub mod used;") + self.assertEqual(parity_reach.declarations(other, {"used"}, True), parity_reach.declarations(text, {"used"}, True)) + self.assertNotEqual(parity_reach.declarations(text + "pub fn f() {}\n", {"used"}, True), + parity_reach.declarations(text, {"used"}, True)) + + def test_a_new_file_counts_once_it_is_reached(self): + (self.root / "crates/a/src/lib.rs").write_text("pub mod used;\nmod extra;\n") + workspace(self.root, {"crates/a/src/extra.rs": "impl crate::used::Thing { pub fn method(&self) -> u32 { 5 } }\n"}) + self.assertIn("crates/a/src/extra.rs", self.narrowed(changed={"crates/a/src/extra.rs"})) + + def test_a_deleted_dependency_counts(self): + (self.root / "crates/a/src/unused.rs").unlink() + self.assertIn("crates/a/src/unused.rs", self.narrowed()) + + def test_unknown_probes_keep_everything(self): + self.assertEqual(set(self.narrowed(artifacts=[])), set(self.dependencies)) + self.assertEqual(set(self.narrowed(artifacts=[["example", "missing", "release", ["-p", "a"], True]])), + set(self.dependencies)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/parity.py b/tools/parity.py index 503a5f3bf..dc5fdb1c9 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -45,6 +45,11 @@ import subprocess import sys import time +try: + from . import parity_reach +except ImportError: + import parity_reach + ROOT = Path(__file__).resolve().parents[1] LOCAL = ROOT / ".local" STATE_DIR = LOCAL / "parity-runner" @@ -1224,20 +1229,44 @@ def changed_harnesses(found, records, ref): return found_changed -def affected_harnesses(found, records, changed): +def affected_harnesses(found, records, changed, reach=True, base=None): """Harnesses the changed files can affect: their script or a recorded dependency (here, or in the shared - dependency lists) changed, a file every harness depends on changed, or nothing is recorded about them yet.""" + dependency lists) changed, a file every harness depends on changed, or nothing is recorded about them yet. + With reach, a harness's workspace Rust sources count only if its probes can reach them (tools/parity_reach.py); + a harness whose probes can't be placed keeps them all. A changed file counts unless, compared with base (a + commit), only declarations of modules the harness doesn't reach changed (tools/parity_reach.py).""" if changed & SHARED_INPUTS: return set(found) + graph = None + if reach and any(path.startswith("crates/") for path in changed): + cache_path = STATE_DIR / "reach-cache.json" + try: + cache = json.loads(cache_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + cache = {} + graph = parity_reach.SourceGraph(ROOT, rust_code, cache) affected = set() for name, script in found.items(): - dependencies = records.get(name, {}).get("dependencies") + record = records.get(name) or {} + dependencies = record.get("dependencies") template = SHARED / "dependencies" / f"{name}.json" if dependencies is None and template.exists(): - dependencies = json.loads(template.read_text(encoding="utf-8")).get("dependencies") - if dependencies is None or Path(script).relative_to(ROOT).as_posix() in changed or \ - any(path.split(KEEP)[0] in changed for path in dependencies): + record = json.loads(template.read_text(encoding="utf-8")) + dependencies = record.get("dependencies") + if dependencies is None or Path(script).relative_to(ROOT).as_posix() in changed: affected.add(name) + continue + plain = {path.split(KEEP)[0] for path in dependencies} + if graph is not None and plain & changed: + narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed) + if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed( + ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()): + affected.add(name) + elif plain & changed: + affected.add(name) + if graph is not None: + STATE_DIR.mkdir(parents=True, exist_ok=True) + cache_path.write_text(json.dumps(graph.saved_cache()), encoding="utf-8") return affected @@ -1280,6 +1309,8 @@ def main(): parser.add_argument("--flake-rounds", type=int, default=3, help="runs required by --flake-check (default 3)") parser.add_argument("--changed", metavar="REF", help="only harnesses the changes since the merge base with REF can " "affect (and that have not passed with their current inputs): the pull-request check") + parser.add_argument("--by-crate", action="store_true", help="with --changed: count every Rust source a probe " + "links, not only those it can reach (tools/parity_reach.py)") parser.add_argument("--changed-files", metavar="FILE", help="like --changed, for the changed paths listed in FILE, " "one per line: for a copy without Git history (tools/pr_ready.py on another machine)") args = parser.parse_args() @@ -1316,7 +1347,11 @@ def main(): changed, since = changed_files(args.changed), f" since the merge base with {args.changed}" if changed is None: parser.error(f"no merge base with {args.changed}") - affected = affected_harnesses(found, records, changed) + base = None + if args.changed: + merge_base = subprocess.run(["git", "merge-base", "HEAD", args.changed], cwd=ROOT, capture_output=True, text=True) + base = merge_base.stdout.strip() or None + affected = affected_harnesses(found, records, changed, reach=not args.by_crate, base=base) chosen = [name for name in found if name in affected and stale(name, state, hasher, environment)] print(f"{len(changed)} files changed{since}: {len(affected)} harnesses " f"depend on them, {len(chosen)} without a pass for their current inputs.", flush=True) diff --git a/tools/parity_reach.py b/tools/parity_reach.py new file mode 100644 index 000000000..0c3051b86 --- /dev/null +++ b/tools/parity_reach.py @@ -0,0 +1,768 @@ +"""Which of a harness's Rust sources its probes can reach, for `parity.py --changed`. + +Cargo's dep-info names every source file of every crate a probe links, so a change anywhere in horizon-client +selected every harness whose probe links horizon-client. This module narrows a harness's recorded dependencies to +the files holding an item its entry points (the probe modules it ran, the binaries it built) can reach, over a graph +of the workspace's current sources whose nodes are items: + +- everything in an entry is reached, and so is everything in a Rust file the module tree doesn't place (an + `include!`d source); +- a module is open when it is a crate root, holds reached code, or is named in a path (`name::`, `use ...name`) by + reached code while its parent is open; +- a free item (fn, struct, enum, union, trait, type, const, static, an item-level macro call's items) is reached when + reached code in its own module or below it mentions its name, or when its module is open and reached code names + it in a path or an import, or when its module is glob-imported (`use m::*`) and its name is mentioned; +- an inherent impl's methods and constants are reached one by one, by name, once their type is; a trait impl once + its trait and its type are (a std or outside name always is, a generic parameter needs nothing); a macro once + invoked; a `pub use` like a free item named by everything it imports; +- "a crate's reached code" covers the crates it names (`horizon_world::`), directly or through one that does, and + code under `#[cfg(test)]` or `#[test]` is left out (probes are never test builds). + +Names match by spelling, so a file can be kept for a name it never meant: the graph only over-approximates. A +reached file's parent modules count through their `mod` declarations of it (parity.py compares those with the merge +base). Everything that isn't a workspace Rust source stays a dependency, except a crate's non-Rust files (shaders, +assets, fonts) that only unreached sources name; a dependency that no longer exists counts as changed, and Rust files +a change adds take part. +""" + +import hashlib +from pathlib import Path, PurePosixPath +import re +import subprocess + +IDENT = r"[A-Za-z_]\w*" +WORD = re.compile(rf"\b({IDENT})\b") +MOD_DECLARATION = re.compile(rf"\bmod\s+({IDENT})\s*;") +INLINE_MOD = re.compile(rf"\bmod\s+({IDENT})\s*\{{") +PATH_ATTRIBUTE = re.compile(r'#\[\s*path\s*=\s*"([^"]+)"\s*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?mod\s+\w+\s*;$') +BRACE = re.compile(r"[{}]") +TEST_ONLY = re.compile(r"#\[\s*(?:cfg\s*\(\s*test\s*\)|test)\s*\]") +TEST_MODULE = re.compile(r"#\[\s*cfg\s*\(\s*test\s*\)\s*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?mod\s+\w+\s*;$") +SPLIT = re.compile(r"[{}()\[\];]") +ITEM = re.compile(rf"(?:pub(?:\s*\([^)]*\))?\s+)?(?:(?:unsafe|async|const|extern|default|auto)\s+)*" + rf"(fn|struct|enum|union|trait|type|const|static|impl|mod|use|macro_rules\s*!|extern\s+crate)" + rf"(?!\w)\s*(?:mut\s+)?({IDENT})?") +PATH_BEFORE = re.compile(rf"\b({IDENT})\s*::") +PATH_AFTER = re.compile(rf"::\s*({IDENT})") +MACRO_USE = re.compile(rf"\b({IDENT})\s*!\s*[(\[{{]") +MACRO_CALL = re.compile(rf"({IDENT}(?:\s*::\s*{IDENT})*)\s*!") +DEFINED_INSIDE = re.compile(rf"\b(?:fn|struct|enum|union|trait|type|const|static)\s+({IDENT})") +NESTED_IMPL = re.compile(r"^[ \t]*(?:unsafe\s+)?impl\b([^{;]*)\{", re.MULTILINE) +QUALIFIED = re.compile(rf"{IDENT}(?:\s*::\s*{IDENT})*") +GLOB_IMPORT = re.compile(r"\buse\s[^;]*::\s*\*") +# File names a source may name in a literal (include_str!, include_bytes!, a path it reads). A file of another kind +# is never "named", so it always counts. +FILE_NAME = re.compile(r"[\w-]+\.(?:wgsl|glsl|json|png|ttf|otf|svg|txt|toml|ogg|wav|nbt|bin|dat|csv|mcmeta|wit|wasm)\b") +TYPE_KINDS = {"struct", "enum", "union", "trait", "type", "macro_call"} +# Names an impl header can take from outside the workspace: std's prelude and primitive types, and the std and serde +# traits a glob import may bring. A glob-imported name outside this list is taken to be the workspace's. +STD_NAMES = {"Display", "Error", "Read", "Write", "Seek", "BufRead", "Deref", "DerefMut", "Index", "IndexMut", "Add", + "Sub", "Mul", "Div", "Rem", "Neg", "Not", "BitAnd", "BitOr", "BitXor", "Shl", "Shr", "AddAssign", + "SubAssign", "MulAssign", "DivAssign", "RemAssign", "BitAndAssign", "BitOrAssign", "BitXorAssign", + "Borrow", "BorrowMut", "Future", "Any", "FromStr", "Hasher", "BuildHasher", "Formatter", "Ordering", + "Serialize", "Deserialize", "Serializer", "Deserializer", "Visitor", "Termination", "Sum", "Product", + "Wake", "Cow", "Rc", "Arc", "Mutex", "RwLock", "Cell", "RefCell", "HashMap", "HashSet", "BTreeMap", + "BTreeSet", "VecDeque"} +PRELUDE = {"Option", "Some", "None", "Result", "Ok", "Err", "Vec", "String", "Box", "ToString", "ToOwned", "Clone", + "Copy", "Send", "Sync", "Sized", "Unpin", "Drop", "Fn", "FnMut", "FnOnce", "From", "Into", "TryFrom", + "TryInto", "AsRef", "AsMut", "Default", "Eq", "PartialEq", "Ord", "PartialOrd", "Iterator", "Extend", + "IntoIterator", "DoubleEndedIterator", "ExactSizeIterator", "FromIterator", "Debug", "Hash", + "bool", "char", "str", "u8", "u16", "u32", "u64", "u128", "usize", "i8", "i16", "i32", "i64", "i128", + "isize", "f32", "f64"} +STD_NAMES |= PRELUDE +KEYWORDS = {"crate", "self", "super", "Self", "pub", "in", "as", "use", "for", "where", "dyn", "mut", "const", "fn", + "unsafe", "impl", "std", "core", "alloc", "let", "if", "else", "match", "return", "struct", "enum", "mod", + "type", "static", "trait", "true", "false", "loop", "while", "move", "ref", "break", "continue", "async", + "await", "extern", "union"} + + +def segments(code, start, end): + """The item-level pieces of code[start:end]: each ends at a `;` or a `}` outside any bracket.""" + found, begin, depth = [], start, 0 + for match in SPLIT.finditer(code, start, end): + character = match.group() + if character in "{([": + depth += 1 + elif character in "})]": + depth -= 1 + if character == "}" and depth == 0: + found.append((begin, match.end())) + begin = match.end() + elif depth == 0: + found.append((begin, match.end())) + begin = match.end() + if code[begin:end].strip(): + found.append((begin, end)) + return found + + +def skip_attributes(code, index, end): + """The offset of the first non-space character after the attributes (`#[...]`, `#![...]`) starting at index.""" + while True: + while index < end and code[index].isspace(): + index += 1 + if not code.startswith("#", index) or not re.match(r"#!?\[", code[index:index + 3]): + return index + depth = 0 + for position in range(code.index("[", index), end): + depth += {"[": 1, "]": -1}.get(code[position], 0) + if depth == 0: + index = position + 1 + break + else: + return end + + +def body_of(code, start, end): + """The range inside an item's outermost `{ }` block, or None.""" + opening = code.find("{", start, end) + closing = code.rfind("}", start, end) + return (opening + 1, closing) if 0 <= opening < closing else None + + +def words(text): + return set(WORD.findall(text)) - KEYWORDS + + +def qualified_words(text): + """Names a piece of code reaches through a path (`a::b`: b), the way items of other modules are named.""" + return set(PATH_AFTER.findall(text)) - KEYWORDS + + +def path_words(text): + """Names a piece of code uses as path segments (`a::`, `::b`), where module names appear.""" + return set(PATH_BEFORE.findall(text)) | set(PATH_AFTER.findall(text)) + + +def parse_items(code, start, end, items): + """Append the items of code[start:end] (a file or an inline module's body) to items, as dicts.""" + for begin, finish in segments(code, start, end): + head = skip_attributes(code, begin, finish) + text = code[head:finish] + if not text.strip(" \n\t;") or TEST_ONLY.search(code[begin:head]): + continue # compiled into tests only, never into a probe + match = ITEM.match(text) + kind = re.sub(r"\s+", "", match.group(1)) if match else None + name = match.group(2) if match else None + if kind == "mod": + body = body_of(code, head, finish) + if body: + parse_items(code, body[0], body[1], items) + continue + if kind in ("externcrate", None) and not MACRO_CALL.match(text): + if kind is None: + items.append({"kind": "glue", "mentions": sorted(words(text)), "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + continue + if kind is None: # an item-level macro call: it may define items + items.append({"kind": "macro_call", "names": sorted(set(DEFINED_INSIDE.findall(text))), + "macro": MACRO_CALL.match(text).group(1).split("::")[-1].strip(), + "mentions": sorted(words(text)), "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + elif kind == "use": + head_word = re.match(rf"\s*(?:::\s*)?({IDENT})", text[match.end(1):]) + items.append({"kind": "use", "public": text.startswith("pub"), "names": sorted(words(text)), + "head": head_word.group(1) if head_word else "", "glob": bool(re.search(r"::\s*\*", text)), + "mentions": sorted(words(text)), "paths": sorted(words(text)), "qualified": sorted(words(text)), + "invokes": []}) + elif kind == "impl": + body = body_of(code, head, finish) + subject = code[head + match.end(1) - match.start(): body[0] - 1 if body else finish] + inherent = not re.search(r"\bfor\b", impl_subject(subject)[1]) + if inherent and body: + members = [] + for member_begin, member_end in segments(code, body[0], body[1]): + member = code[skip_attributes(code, member_begin, member_end):member_end] + found = ITEM.match(member) + members.append({"name": found.group(2) if found else None, "mentions": sorted(words(member)), + "paths": sorted(path_words(member)), "qualified": sorted(qualified_words(member)), "invokes": sorted(set(MACRO_USE.findall(member)))}) + items.append({"kind": "impl", "subject": subject, "inherent": True, "members": members, + "mentions": sorted(words(subject)), "paths": sorted(path_words(subject)), "qualified": sorted(qualified_words(subject)), "invokes": sorted(set(MACRO_USE.findall(subject)))}) + else: + items.append({"kind": "impl", "subject": subject, "inherent": False, "mentions": sorted(words(text)), + "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + else: + kind = "macro" if kind.startswith("macro_rules") else kind + items.append({"kind": kind, "names": [name] if name else [], "mentions": sorted(words(text)), + "defines": sorted(set(DEFINED_INSIDE.findall(text))) if kind == "macro" else [], + "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + if kind in ("fn", "const", "static"): + # An impl inside a function body still holds for the whole program. + for header in NESTED_IMPL.findall(text): + items.append({"kind": "impl", "subject": header, "inherent": False, + "mentions": sorted(words(text)), "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + + +def crate_roots(crate_dir): + """A crate's root files: its library, binaries and examples.""" + roots = [crate_dir / "src" / "lib.rs", crate_dir / "src" / "main.rs"] + for folder in (crate_dir / "src" / "bin", crate_dir / "examples"): + if folder.is_dir(): + roots += sorted(folder.glob("*.rs")) + sorted(folder.glob("*/main.rs")) + return [root for root in roots if root.is_file()] + + +class SourceGraph: + """The workspace's Rust sources, parsed once: what each file names, defines and declares.""" + + def __init__(self, root, rust_code, cache=None): + """cache: a dict of earlier parses by content digest, which the graph reuses and adds to (parity.py keeps it + in .local/parity-runner/reach-cache.json).""" + self.root = Path(root) + self.rust_code = rust_code + self.cache = {} if cache is None else cache + self.used = set() + self.files = {} + self.parent = {} # module file -> the file declaring it + self.module_name = {} + self.roots = set() + self.target_cache = {} + self.workspace_heads = None + self.test_only = set() # modules compiled into tests only + self.tables = None + self.root_of = {} # source -> the root of its target (lib.rs, a bin's or an example's main file) + self.results = {} # (entries, universe) -> reachable files + for manifest in sorted((self.root / "crates").glob("*/Cargo.toml")): + for crate_root in crate_roots(manifest.parent): + path = self.relative(crate_root) + self.roots.add(path) + self.root_of[path] = path + self.place(path, crate_root.parent, set()) + + def entries(self, artifacts, rust): + """The root files of the probes and binaries a harness built (its record's artifacts), among its Rust + dependencies: for a crate's probe binary, the probe modules it ran (parity.only_probes_run kept only those). + None when an artifact can't be placed.""" + found = set() + for kind, name, *rest in artifacts: + selection = rest[1] if len(rest) > 1 else [] + packages = [value for flag, value in zip(selection, selection[1:]) if flag == "-p"] or \ + [manifest.parent.name for manifest in sorted((self.root / "crates").glob("*/Cargo.toml"))] + roots = set() + for package in packages: + crate = self.root / "crates" / package + targets = self.targets(crate) + if kind in ("example", "examples"): + candidates = [path for target, (target_kind, path) in targets.items() + if target_kind == "example" and (not name or target == name)] + else: + candidates = [path for target, (target_kind, path) in targets.items() + if target_kind == "bin" and (not name or target == name)] + for path in candidates: + if path.endswith("/examples/probe/main.rs"): + # The probes it ran, and the dispatch and support modules they use (only_probes_run kept + # those); the binary's root, which only declares them, so their own submodules are placed. + folder = path.removesuffix("main.rs") + roots |= {dependency for dependency in rust if dependency.startswith(folder) + and "/" not in dependency.removeprefix(folder).removeprefix("support/")} + if roots: + roots.add(path) + elif path in rust: + roots.add(path) + if not roots: + return None + found |= roots + return found or None + + def targets(self, crate): + """A crate's bins and examples: {target name: (kind, root file)}, from its Cargo.toml and Cargo's defaults.""" + if crate in self.target_cache: + return self.target_cache[crate] + found = {} + manifest = (crate / "Cargo.toml").read_text(encoding="utf-8") if (crate / "Cargo.toml").is_file() else "" + package = re.search(r'^\[package\][^\[]*?^name\s*=\s*"([^"]+)"', manifest, re.MULTILINE | re.DOTALL) + if (crate / "src" / "main.rs").is_file() and package: + found[package.group(1)] = ("bin", self.relative(crate / "src" / "main.rs")) + for kind, folder in (("bin", crate / "src" / "bin"), ("example", crate / "examples")): + for path in sorted(folder.glob("*.rs")) + sorted(folder.glob("*/main.rs")): + found[path.stem if path.name != "main.rs" else path.parent.name] = (kind, self.relative(path)) + for kind, table in re.findall(r"^\[\[(bin|example)\]\]([^\[]*)", manifest, re.MULTILINE): + name = re.search(r'^name\s*=\s*"([^"]+)"', table, re.MULTILINE) + path = re.search(r'^path\s*=\s*"([^"]+)"', table, re.MULTILINE) + if name and path and (crate / path.group(1)).is_file(): + found[name.group(1)] = (kind, self.relative(crate / path.group(1))) + self.target_cache[crate] = found + return found + + def target(self, path): + """The Cargo target a source belongs to: the root file of the library, bin or example whose module tree + holds it (a file nothing places is its own).""" + return self.root_of.get(path, path) + + def is_library(self, path): + return self.target(path).endswith("/src/lib.rs") + + def relative(self, path): + return Path(path).resolve().relative_to(self.root.resolve()).as_posix() + + def place(self, path, module_dir, seen): + """Walk the module tree from a file: record each declared module's file, parent and name.""" + if path in seen: + return + seen.add(path) + parsed = self.parse(path) + if parsed is None: + return + for name, nesting, attribute, test in parsed["declarations"]: + if attribute is not None: + if nesting: + continue # a #[path] inside an inline module: unplaced, so always kept + candidates = [(self.root / path).parent / attribute] + else: + folder = module_dir.joinpath(*nesting) + candidates = [folder / f"{name}.rs", folder / name / "mod.rs"] + for candidate in candidates: + if candidate.is_file(): + child = self.relative(candidate) + self.parent.setdefault(child, path) + self.module_name.setdefault(child, name) + self.root_of.setdefault(child, self.root_of[path]) + if test or path in self.test_only: + self.test_only.add(child) + child_dir = candidate.parent if candidate.name == "mod.rs" or attribute is not None \ + else candidate.parent / candidate.stem + self.place(child, child_dir, seen) + break + + def parse(self, path): + if path in self.files: + return self.files[path] + try: + data = (self.root / path).read_bytes() + except OSError: + self.files[path] = None + return None + digest = hashlib.sha256(data).hexdigest() + self.used.add(digest) + if digest in self.cache: + self.files[path] = parsed = self.cache[digest] + return parsed + raw = data.decode("utf-8", errors="replace") + code = self.rust_code(raw) # same length as raw: literals and comments blanked + opening = {match.end() - 1: match.group(1) for match in INLINE_MOD.finditer(code)} + events = sorted([(match.start(), match) for match in BRACE.finditer(code)] + + [(match.start(), match) for match in MOD_DECLARATION.finditer(code)], key=lambda event: event[0]) + declarations, stack, depth = [], [], 0 + for index, match in events: + if match.group() == "{": + depth += 1 + if index in opening: + stack.append((opening[index], depth)) + elif match.group() == "}": + if stack and stack[-1][1] == depth: + stack.pop() + depth -= 1 + else: + window = " ".join(raw[max(0, index - 300):match.end()].split()) + attribute = PATH_ATTRIBUTE.search(window) + declarations.append([match.group(1), [name for name, _ in stack], attribute.group(1) if attribute else None, + bool(TEST_MODULE.search(window))]) + items = [] + parse_items(code, 0, len(code), items) + self.files[path] = parsed = { + "declarations": declarations, + "items": items, + "glob": bool(GLOB_IMPORT.search(code)), + "file_names": sorted(set(FILE_NAME.findall(raw))), + } + self.cache[digest] = parsed + return parsed + + def saved_cache(self): + """The parses of the files this graph read, to keep for the next run.""" + return {digest: self.cache[digest] for digest in self.used if digest in self.cache} + + def requirement(self, path, item): + """For an impl: for its trait and for its type, [names the file defines, other workspace names], one of + which must be a reached type or trait (of this file, of a crate it sees) before the impl can matter; both + empty: nothing of the workspace is needed. A name is the workspace's when the path naming it starts in the + workspace, or the file defines it, imports it from the workspace or may glob-import it; otherwise (std's, + another crate's, the prelude's) it needs nothing.""" + if "requires" in item: + return item["requires"] + if self.workspace_heads is None: + self.workspace_heads = {"crate", "self", "super"} | set(self.module_name.values()) | { + manifest.parent.name.replace("-", "_") for manifest in (self.root / "crates").glob("*/Cargo.toml")} + parsed = self.parse(path) + defined = {name for other in parsed["items"] for name in other.get("names", ()) if other["kind"] != "use"} + workspace_imports = {name for other in parsed["items"] if other["kind"] == "use" + and other["head"] in self.workspace_heads for name in other["names"]} + parameters, subject = impl_subject(item["subject"]) + parts = [] + for part in re.split(r"\bfor\b", subject, maxsplit=1): + names, own = set(), set() + for match in QUALIFIED.finditer(part): + segments_ = [segment.strip() for segment in match.group().split("::")] + name = segments_[-1] + if name in parameters or name in KEYWORDS: + continue + if len(segments_) > 1: + if segments_[0] in self.workspace_heads: + names.add(name) + elif name in defined: + own.add(name) + elif name in workspace_imports or (parsed["glob"] and name not in STD_NAMES): + names.add(name) + parts.append([sorted(own), sorted(names)]) + item["requires"] = parts + return parts + + def index(self): + """Lookup tables over every placed source, built once: which crates each crate sees, the nodes (items, an + inherent impl's members) by the names they define and the types they need, the modules by name.""" + if self.tables is not None: + return self.tables + idents = {manifest.parent.name.replace("-", "_"): manifest.parent.name + for manifest in (self.root / "crates").glob("*/Cargo.toml")} + paths = [path for path in self.files if self.files[path] is not None and path not in self.test_only] + sees = {} + for path in paths: + named = {word for item in self.files[path]["items"] for word in item["mentions"]} & idents.keys() + sees.setdefault(crate_of(path), {crate_of(path)}).update(idents[name] for name in named) + changed = True + while changed: + changed = False + for crate, seen in sees.items(): + wider = set().union(*(sees.get(other, {other}) for other in seen)) + if not wider <= seen: + seen |= wider + changed = True + seen_by = {} # crate -> the crates that see it + for crate, seen in sees.items(): + for other in seen: + seen_by.setdefault(other, set()).add(crate) + # What a call of a workspace macro defines: what its definition's body defines. + expands = {} + for path in paths: + for item in self.files[path]["items"]: + if item["kind"] == "macro": + for name in item["names"]: + expands.setdefault(name, set()).update(item["defines"]) + names_of, by_name, by_file_name, by_type, by_module, children, seeds = {}, {}, {}, {}, {}, {}, [] + for path in paths: + crate = crate_of(path) + if path in self.module_name: + by_module.setdefault((crate, self.module_name[path]), []).append(path) + children.setdefault(self.parent[path], []).append(path) + for index, item in enumerate(self.files[path]["items"]): + node = (path, index, None) + names = set(item.get("names", ())) + if item["kind"] == "macro_call": + names |= expands.get(item["macro"], set()) + names_of[node] = names + if item["kind"] == "glue": + seeds.append(node) + elif item["kind"] == "macro_call" and not names: + # A macro call defining nothing by name may still expand to impls: it counts once a type or + # trait it mentions is reached. + for name in item["mentions"]: + by_type.setdefault(name, []).append(node) + elif item["kind"] == "impl": + required = {name for own, foreign in self.requirement(path, item) for name in own + foreign} + members = [(path, index, member) for member in range(len(item["members"]))] if item["inherent"] else [] + for member_node in members: + name = item["members"][member_node[2]]["name"] + names_of[member_node] = set() + if name: + by_name.setdefault(name, []).append(member_node) + for name in required: + for other in [node] + members: + by_type.setdefault(name, []).append(other) + if not required: + seeds.append(node) + seeds += [member_node for member_node in members if not item["members"][member_node[2]]["name"]] + else: + for name in names: + by_name.setdefault(name, []).append(node) + by_file_name.setdefault((path, name), []).append(node) + self.tables = dict(idents=idents, sees=sees, seen_by=seen_by, names_of=names_of, by_name=by_name, + by_file_name=by_file_name, by_type=by_type, by_module=by_module, children=children, + seeds=seeds, node_sets={}) + return self.tables + + def reachable(self, entries, universe, reasons=None): + """The files of universe (workspace Rust sources) holding an item reachable from entries (by the rules in + this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging).""" + # Searching every source of the crates involved (not only those Cargo listed) only adds reach, and lets + # harnesses with the same entries (the game binary's) share one search. + crates = frozenset(crate_of(path) for path in universe) + key = (frozenset(entries), crates) + if reasons is None and key in self.results: + return self.results[key] & set(universe) + everything = {path for path in self.files if crate_of(path) in crates} | set(universe) + files = self.search(entries, everything, reasons) + self.results[key] = files + return files & set(universe) + + def search(self, entries, universe, reasons): + targets = {self.target(path) for path in entries} + universe = {path for path in universe if (self.is_library(path) or self.target(path) in targets) + and path not in self.test_only} + tables = self.index() + idents, sees, seen_by, names_of = tables["idents"], tables["sees"], tables["seen_by"], tables["names_of"] + node_sets = tables["node_sets"] + by_name, by_file_name, by_type = tables["by_name"], tables["by_file_name"], tables["by_type"] + by_module, children = tables["by_module"], tables["children"] + seeds = [node for node in tables["seeds"] if node[0] in universe] + for path in universe: + if (path in entries or (path not in self.parent and path not in self.roots)) and self.parse(path): + seeds += [(path, index, None) for index in range(len(self.parse(path)["items"]))] + + def names(node): + return names_of.get(node, set()) + + def sets_of(node, item): + """A node's mentions, path words, qualified names and macro calls, as sets (parses keep lists).""" + if node not in node_sets: + source = item["members"][node[2]] if node[2] is not None else item + node_sets[node] = {key: set(source.get(key, ())) for key in ("mentions", "paths", "qualified", "invokes")} + return node_sets[node] + + def item_of(node): + return self.parse(node[0])["items"][node[1]] + + reached, opened, globbed, files, queue = set(), set(), set(), set(), [] + # name -> the crates whose reached code mentions it plainly, names it in a path or an import, invokes it. + mentioned, qualified, invoked = {}, {}, {} + done = {"mentions": {}, "qualified": {}, "invokes": {}, "paths": {}} # crate -> names already handled + path_names, types = {}, {} # by crate + local = {} # file -> plain mentions by reached code in it or below it (child modules may `use super::*`) + local_types = {} # file -> its reached types and traits + + def heard(table, name, crate): + """Whether a crate that can see crate's items has name in table.""" + sources = table.get(name) + return bool(sources) and not sources.isdisjoint(seen_by.get(crate, {crate})) + + def ready(node): + path, index, member = node + item = item_of(node) + crate = crate_of(path) + if item["kind"] == "impl": + visible = visible_types(crate) + here = local_types.get(path, set()) + if not all((not own and not foreign) or here.intersection(own) or visible.intersection(foreign) + for own, foreign in self.requirement(path, item)): + return False + name = item["members"][member]["name"] if member is not None else None + return name is None or heard(mentioned, name, crate) # a method: called by its name + if item["kind"] == "macro_call" and not names(node): + return not visible_types(crate).isdisjoint(item["mentions"]) + names_ = names(node) + if item["kind"] == "macro": + return any(heard(invoked, name, crate) for name in names_) + if item["kind"] == "use" and item["glob"]: + return path in opened + nearby = local.get(path, ()) + if any(name in nearby for name in names_): + return True # named in its own module, or below it + if item["kind"] == "use" and not item["public"]: + return False + if path in opened and any(heard(qualified, name, crate) for name in names_): + return True # named through a path, or imported + return path in globbed and any(heard(mentioned, name, crate) for name in names_) + + visible_cache = {} + + def visible_types(crate): + """The reached types and traits of the crates a crate sees (cached until a type is reached).""" + if crate not in visible_cache: + visible_cache[crate] = set().union(*(types.get(other, set()) for other in sees.get(crate, {crate}))) + return visible_cache[crate] + + def wake(nodes, why): + for other in nodes: + if other not in reached and other[0] in universe and ready(other): + reach(other, why) + + def open_module(path, why): + pending = [path] + while pending: + current = pending.pop() + if current in opened: + continue + opened.add(current) + crate = crate_of(current) + pending += [child for child in children.get(current, ()) + if child in universe and self.module_name[child] in path_names.get(crate, ())] + parsed = self.parse(current) + wake([(current, index, None) for index, item in enumerate(parsed["items"] if parsed else ()) + if item["kind"] != "impl"], f"{current} opened ({why})") + + def reach(node, why): + if node in reached: + return + reached.add(node) + queue.append(node) + path, _, member = node + item = item_of(node) + if (item["kind"] == "use" and not item["public"]) or (item["kind"] == "impl" and item["inherent"] and member is None): + return + if path not in files: + files.add(path) + if reasons is not None: + reasons[path] = why + + for path in universe: + if path.endswith("/src/lib.rs") or path in entries or path not in self.parent: + open_module(path, "crate root or entry") + for node in seeds: + reach(node, "entry" if node[0] in entries else "always counted") + for path in entries & universe: + files.add(path) + if reasons is not None: + reasons.setdefault(path, "entry") + while queue: + node = queue.pop() + path, index, member = node + item = item_of(node) + crate = crate_of(path) + label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"] + source = sets_of(node, item) + why = f"{path} ({label})" + # Plain mentions reach items of this file and of the modules above it. + ancestor, new = path, source["mentions"] + while ancestor is not None and new: + known = local.setdefault(ancestor, set()) + new = new - known + known |= new + for name in new: + wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}") + ancestor = self.parent.get(ancestor) + if item["kind"] == "use" and item["glob"]: + for name in item["names"]: + for target in sees.get(crate, {crate}): + for module in list(by_module.get((target, name), ())) + \ + ([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []): + if module not in globbed and module in universe: + globbed.add(module) + wake([(module, index, None) for index in range(len(self.parse(module)["items"]))] + if self.parse(module) else [], f"{why} glob-imports {name}") + visible_to = sees.get(crate, {crate}) + new = source["paths"] - done["paths"].setdefault(crate, set()) + done["paths"][crate] |= new + for name in new: + for target in visible_to: + path_names.setdefault(target, set()).add(name) + for module in by_module.get((target, name), ()): + if module in universe and self.parent[module] in opened: + open_module(module, f"{why} names {name}") + for key, table, wanted in (("qualified", qualified, lambda other: True), + ("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"), + ("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)): + new = source[key] - done[key].setdefault(crate, set()) + done[key][crate] |= new + for name in new: + table.setdefault(name, set()).add(crate) + wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)], + f"{why} {key} {name}") + if item["kind"] in TYPE_KINDS and member is None: + own = types.setdefault(crate, set()) + local_types.setdefault(path, set()).update(names(node)) + if not names(node) <= own: + own |= names(node) + visible_cache.clear() + seeing = seen_by.get(crate, {crate}) + for name in names(node): # also when the crate has one by that name already: impls of this one + wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing], + f"implements for {name}, defined in {path}") + return files + + +def impl_subject(header): + """An impl header (the text after `impl`): its generic parameters' names, and the rest without its where clause + (the trait and the type it is for).""" + header, parameters = header.strip(), set() + if header.startswith("<"): + depth = 0 + for index, character in enumerate(header): + depth += {"<": 1, ">": -1}.get(character, 0) + if depth == 0: + generics, header = header[1:index], header[index + 1:] + parameters = {match.group(1) for match in re.finditer(rf"(?:^|,)\s*(?:const\s+)?({IDENT})", generics)} + break + return parameters, re.split(r"\bwhere\b", header)[0] + + +def crate_of(path): + return path.split("/", 2)[1] + + +def is_workspace_rust(path): + parts = PurePosixPath(path).parts + return len(parts) > 2 and parts[0] == "crates" and path.endswith(".rs") + + +DECLARATION_LINE = re.compile(rf"\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+({IDENT})\s*;") + + +def declarations(text, names, keep_rest): + """A Rust file's `mod` declarations of names, each with the attribute lines just above it; with keep_rest, the + whole text except the declarations of other modules instead.""" + lines, found, attributes = text.splitlines(), [], [] + for line in lines: + if line.strip().startswith("#[") and not line.strip().startswith("#!["): + attributes.append(line.strip()) + continue + match = DECLARATION_LINE.match(line) + if match and match.group(1) in names: + found += attributes + [line.strip()] + elif keep_rest and not match: + found += attributes + [line] + attributes = [] + return "\n".join(found + attributes) + + +def declarations_changed(root, path, children, base, whole): + """Whether a changed file differs from the commit base in what a harness depends on: for a module parent, its + `mod` declarations of the reached children; for a whole file (whole), everything but its declarations of other + modules (adding a module beside reached ones changes nothing). True without a base, or if either side is + missing.""" + if base is None: + return True + try: + head = (Path(root) / path).read_text(encoding="utf-8", errors="replace") + before = subprocess.run(["git", "show", f"{base}:{path}"], cwd=root, capture_output=True, text=True, check=True).stdout + except (OSError, subprocess.CalledProcessError): + return True + return declarations(head, children, whole) != declarations(before, children, whole) + + +def narrow(dependencies, artifacts, graph, changed=()): + """A harness's dependencies narrowed to what its entry points can reach: {dependency: (whole, children)}, where + whole says the file counts for its own content (else only through its `mod` declarations of children, as a + module ancestor of a reached file) and children are its reached child modules. Without known entry points (a + probe or binary the harness built that isn't found here), every dependency is kept whole.""" + plain = [dependency.split("#keep:")[0] for dependency in dependencies] + crates = {PurePosixPath(path).parts[1] for path in plain if path.startswith("crates/")} + rust = {path for path in plain if is_workspace_rust(path)} + added = {path for path in changed if is_workspace_rust(path) and PurePosixPath(path).parts[1] in crates + and path not in rust and (graph.root / path).is_file()} + entries = graph.entries(artifacts, rust) + if entries is None: + return {dependency: (True, None) for dependency in dependencies} + missing = {path for path in rust if graph.parse(path) is None} + reached = graph.reachable(entries, rust | added | entries) | missing + named, named_in_crate = set(), {} + for path in rust | added: + parsed = graph.parse(path) + file_names = set(parsed["file_names"]) if parsed else set() + if path in reached: + named |= file_names + named_in_crate.setdefault(PurePosixPath(path).parts[1], set()).update(file_names) + kept = {} + for dependency, path in zip(dependencies, plain): + if is_workspace_rust(path): + if path in reached: + kept[dependency] = (True, set()) + elif not path.startswith("crates/") or path.endswith(("Cargo.toml", "build.rs")) \ + or PurePosixPath(path).name in named \ + or PurePosixPath(path).name not in named_in_crate.get(PurePosixPath(path).parts[1], ()): + kept[dependency] = (True, None) + for path in sorted(added & reached): + kept[path] = (True, set()) + for path in sorted(reached): + child = path + while child in graph.parent: + parent = graph.parent[child] + kept.setdefault(parent, (False, set()))[1].add(graph.module_name[child]) + child = parent + return kept From 836c1ee3b999823ab5d0db767ffbb6b71db4ab1e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:27:32 +0100 Subject: [PATCH 10/13] CI parity: pick harnesses with main's selection code, run them at the head The parity job checks out the PR head, so every branch made before #1964 still selected with the old per-crate rule. The step now runs main's tools/parity.py and tools/parity_reach.py with --changed --list, restores the head's files, and runs the named harnesses with the head's parity.py. A branch that edits either file, or a failure of main's selection, falls back to the head's own `--changed` run. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parity.yml | 32 +++++++++++++++++++++++++++++++- docs/release-flow.md | 3 +++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index a18fc5019..be87bc965 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -43,7 +43,37 @@ jobs: env: PARITY_HOSTS: '' BASE_SHA: ${{ inputs.base }} - run: python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400 # world_spawn takes ~1480 s on a busy runner + run: | + # main's selection (tools/parity.py --changed --list with main's parity.py and parity_reach.py), so a + # branch made before a better selection landed gets it too; the head's own runner then runs those + # harnesses. A branch that changes the selection itself selects with its own code. + set -euo pipefail + own() { python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400; } # world_spawn takes ~1480 s on a busy runner + fork_point=$(git merge-base "$BASE_SHA" HEAD) + if ! git rev-parse --verify -q origin/main >/dev/null || ! git diff --quiet "$fork_point" HEAD -- tools/parity.py tools/parity_reach.py; then + own + exit + fi + had_reach=$(git ls-files tools/parity_reach.py) + git show origin/main:tools/parity.py > tools/parity.py + git show origin/main:tools/parity_reach.py > tools/parity_reach.py + status=0 + python3 tools/parity.py --changed "$BASE_SHA" --list > .local/parity-selection.txt || status=$? + git checkout -- tools/parity.py + if [ -n "$had_reach" ]; then git checkout -- tools/parity_reach.py; else rm -f tools/parity_reach.py; fi + cat .local/parity-selection.txt + if [ "$status" != 0 ]; then + echo "main's selection failed: the head's own selection runs" + own + exit + fi + names=$(sed '$d' .local/parity-selection.txt | grep -E '^[A-Za-z0-9_#]+$' || true) + if [ -z "$names" ]; then + echo "Nothing to run." + exit 0 + fi + # shellcheck disable=SC2086 + python3 tools/parity.py $names --timeout 2400 - name: Logs of the failed harnesses if: failure() uses: actions/upload-artifact@v4 diff --git a/docs/release-flow.md b/docs/release-flow.md index 4fb0209d9..3024e0f53 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -43,6 +43,9 @@ copy is: the Mac mini. Its runner runs as the maintainer's account and keeps its modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to `Cargo.lock` or `research-lock.json` affects every harness. Each harness has 25 minutes, the job 45. + The selection runs with main's `tools/parity.py` and `tools/parity_reach.py` (`--changed --list`), so a + branch made before a better selection landed gets it too, and the head's own `parity.py` then runs the named + harnesses. A branch that changes either file selects with its own code. Failed harnesses' logs are uploaded as the `parity-logs` artifact. From 73da2f04ddd56cb2ca1f480f54ab0aad31478223 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:35:28 +0100 Subject: [PATCH 11/13] CI: parity on main, a batch of merges at a time; releases wait for it parity-main runs the vanilla harnesses on every push to main, one run at a time: merges that land during a run wait in one pending run, which checks every merge since main's last green parity-main (tools/parity_main.py base, then parity.py --changed). A failure is bisected over the batch's first-parent merges, running only the failing harnesses, and the breaking merge's pull request gets a comment. Game releases ship only a commit both checks and parity-main passed on (release_train.py; release.yml also starts when parity-main finishes; a forced release doesn't wait). The per-PR parity job now runs only for a PR that changes a harness, an oracle, a probe or the runner, or carries run-parity. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 34 +++++++- .github/workflows/parity-main.yml | 70 ++++++++++++++++ .github/workflows/release.yml | 10 ++- docs/parity-workflow.md | 2 + docs/release-flow.md | 30 ++++++- tests/test_parity_main.py | 43 ++++++++++ tests/test_release_train.py | 21 +++++ tools/parity_main.py | 129 ++++++++++++++++++++++++++++++ tools/release_train.py | 34 ++++++-- 9 files changed, 356 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/parity-main.yml create mode 100644 tests/test_parity_main.py create mode 100644 tools/parity_main.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98d5b9093..eff82fa99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,8 @@ name: checks # check main requires, so it never has a condition: a skipped job counts as passed, and every run of this workflow # must report a real verify (a pull request that changes no Rust finishes it in seconds instead). # parity: the vanilla harnesses a pull request's changes can affect, on the Mac mini's runner, which holds the -# maintainer's own Minecraft files (Horizon never downloads the game); .github/workflows/parity.yml. A fork's pull +# maintainer's own Minecraft files (Horizon never downloads the game); .github/workflows/parity.yml. Only for a pull +# request that changes a harness, oracle, probe or the runner: the rest are checked on main by parity-main. A fork's pull # request runs it only when a maintainer adds the run-parity label (.github/workflows/run-parity.yml), so labels # never start this workflow. See docs/release-flow.md. on: @@ -72,8 +73,37 @@ jobs: run: | cargo +stable test --workspace --lib --bins --tests --features horizon-client/mods,horizon-mods/pack cargo +stable test --workspace --doc --features horizon-client/mods,horizon-mods/pack - parity: + # Most pull requests leave the harnesses to parity-main, which runs them on main a batch of merges at a time + # (.github/workflows/parity-main.yml; releases wait for it). A pull request still runs them itself when it changes + # a harness, an oracle, a probe or the runner, or carries the run-parity label (run-parity.yml, when it is added). + parity-scope: if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + outputs: + run: ${{ steps.scope.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + - id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + LABELS: ${{ join(github.event.pull_request.labels.*.name, ' ') }} + run: | + changed=$(git diff --name-only "$(git merge-base "$BASE_SHA" HEAD)" HEAD) + harness=$(grep -E '^(tools/test_[a-z0-9_]+_parity\.py|tools/(parity[a-z_]*|lab)\.py|tools/parity_hosts\.json|mods/horizon/src/test/|crates/[^/]+/examples/|crates/horizon-client/src/game/[a-z_]*(fixture|harness)\.rs|\.github/workflows/parity[a-z-]*\.yml)' <<< "$changed" || true) + if [ -n "$harness" ] || [[ " $LABELS " == *" run-parity "* ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + echo "The pull request runs its harnesses: ${harness:-label run-parity}" + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "No harness, oracle, probe or runner change: parity-main checks this on main" + fi + parity: + needs: parity-scope + if: needs.parity-scope.outputs.run == 'true' uses: ./.github/workflows/parity.yml with: base: ${{ github.event.pull_request.base.sha }} diff --git a/.github/workflows/parity-main.yml b/.github/workflows/parity-main.yml new file mode 100644 index 000000000..1ddfb24bb --- /dev/null +++ b/.github/workflows/parity-main.yml @@ -0,0 +1,70 @@ +name: parity-main +# The vanilla harnesses on main, a batch of merges at a time (docs/release-flow.md#parity-on-main), on the Mac mini's +# runner. One run at a time: merges that land while a run goes on wait in one pending run (GitHub keeps only the +# newest), which checks every merge since main's last green parity-main (tools/parity_main.py base). On a failure it +# bisects the failing harnesses over those merges and comments on the pull request that broke them. Releases ship +# only a commit whose parity-main passed (tools/release_train.py), so nothing a harness catches reaches players. +on: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read + # The comment on the merge that broke a harness. + pull-requests: write + # tools/parity_main.py base reads earlier runs. + actions: read +concurrency: + group: parity-main + cancel-in-progress: false +env: + # Test runs never reach the telemetry server: no live-count ping, no uploads (docs/telemetry.md). + HORIZON_TELEMETRY_URL: "off" + +jobs: + parity-main: + runs-on: [self-hosted, horizon-game] + timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + # Keeps .local/ (parity state, probe builds, oracle results) and target/ between runs. + clean: false + - uses: dtolnay/rust-toolchain@stable + - name: Tools from the runner + run: bash tools/ci/runner-env.sh + - name: Vanilla data, and resources prepared by this preparer + run: bash tools/ci/vanilla-data.sh + - name: Main's last green parity-main + id: base + env: + GH_TOKEN: ${{ github.token }} + run: echo "sha=$(python3 tools/parity_main.py base)" >> "$GITHUB_OUTPUT" + - name: The harnesses the merges since then can affect + env: + PARITY_HOSTS: '' + BASE_SHA: ${{ steps.base.outputs.sha }} + run: | + echo "Merges since ${BASE_SHA}:" + git log --first-parent --format=' %h %s' "$BASE_SHA..HEAD" + python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400 + - name: The merge that broke them + if: failure() + env: + GH_TOKEN: ${{ github.token }} + PARITY_HOSTS: '' + BASE_SHA: ${{ steps.base.outputs.sha }} + run: | + rm -rf .local/parity-main-logs && cp -R .local/parity-runner/logs .local/parity-main-logs 2>/dev/null || true + python3 tools/parity_main.py bisect "$BASE_SHA" --comment + - name: Logs of the failed harnesses + if: failure() + uses: actions/upload-artifact@v4 + with: + name: parity-main-logs + path: .local/parity-main-logs/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4cba851fa..05f590096 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,7 @@ name: release # Automatic game releases (docs/release-flow.md). After every green `checks` run on main, every 10 minutes (so a -# release waiting for merges to settle goes out without another merge) and by hand, tools/release_train.py decides: +# release waiting for merges to settle goes out without another merge) and by hand, tools/release_train.py decides, +# for a commit whose checks and parity-main (the vanilla harnesses on main) both passed: # a release when 5 pull requests merged since the last one changed what players run and the newest merged 5 minutes # ago, or when the oldest has waited 6 hours, or at once for a pull request labelled release-now. # RELEASES_PAUSED=1 (repository variable) stops them; so does an open issue labelled release-blocker, forced runs too. @@ -18,7 +19,8 @@ name: release # gh workflow run release -f dry_run=false -f force=true # on main: release now, whatever the rules say on: workflow_run: - workflows: [checks] + # A game release ships a commit both passed on (parity-main runs on batches: the newest commit it passed on). + workflows: [checks, parity-main] types: [completed] branches: [main] schedule: @@ -64,8 +66,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASES_PAUSED: ${{ vars.RELEASES_PAUSED }} - # The commit whose checks just passed; a dry run builds the branch it was started on; otherwise - # main's newest commit with green checks. + # The commit whose checks or parity-main just passed; a dry run builds the branch it was started on; + # otherwise main's newest commit with green checks and parity-main. HEAD_SHA: ${{ github.event.workflow_run.head_sha || (steps.mode.outputs.dry_run == 'true' && github.sha) || '' }} FORCE: ${{ inputs.force && 'yes' || '' }} run: | diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index d794b9c71..d64ec3087 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -20,6 +20,8 @@ python3 tools/parity.py --changed origin/main # those a branch's changes ca python3 tools/parity.py --changed-files paths.txt # the same for listed paths, in a copy without Git (tools/pr_ready.py) ``` +On CI, a pull request runs the parity job only when it changes a harness, an oracle, a probe or the runner; everything else is checked on main by `parity-main`, a batch of merges at a time, which bisects a failure to the merge that caused it (`tools/parity_main.py`; [release-flow.md](release-flow.md#parity-on-main)). Game releases wait for it. + Each harness runs unmodified under a tracer that records the files it depended on: its Python modules, its Java oracle and the oracle sources javac pulled in, other repository files it read, and the Rust sources of the probes it ran (from Cargo's `.d` dep-info files). A pass stands until one of those files, the pinned vanilla artifacts, `Cargo.lock` or the Rust/Java toolchains change. State and logs are in `.local/parity-runner/`; each harness's own outputs stay in its `.local//` directory. Vanilla oracle runs are reused as well. `tools/parity_shim.py` stands in for `java` and `javac`: it records each oracle run's outputs and the content of every file under the paths its arguments name, and replays the run while those are unchanged. After a Rust-only change a harness pays only for its Rust side (`block_update`: 53 s to 5 s). Files an oracle writes outside its named paths aren't tracked, so a harness that fails while reusing oracle results is rerun fresh, and stops reusing them if it then passes. `--all` and `--fresh` never reuse them. diff --git a/docs/release-flow.md b/docs/release-flow.md index 4fb0209d9..b551bad21 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -13,7 +13,7 @@ reopened; labels never start it): | Job | Runs on | What | Time | | --- | --- | --- | --- | | `verify` | GitHub `ubuntu-latest` | Public-tree check, Python tests, rustfmt, clippy with warnings denied, Rust tests. Dependencies and the workspace build come from a cache saved by main's runs | ~4 min cold, less warm | -| `parity / parity` | The Mac mini's runner (`ludovics-mac-mini`, label `horizon-game`) | The vanilla harnesses the change can affect (`tools/parity.py --changed `) | seconds when nothing is affected; minutes per affected harness | +| `parity / parity` | The Mac mini's runner (`ludovics-mac-mini`, label `horizon-game`) | Only for a change to a harness, an oracle, a probe or the runner (or the `run-parity` label): the vanilla harnesses the change can affect (`tools/parity.py --changed `). Other pull requests skip it: [parity-main](#parity-on-main) checks them on main | seconds when nothing is affected; minutes per affected harness | Pushes to main run `verify` only; the nightly audit covers main's harnesses. @@ -80,10 +80,34 @@ gh api -X POST repos/ludovic111/horizon/branches/main/protection/required_status A skipped `parity` (a fork's pull request without the label) counts as passed, so requiring it never blocks a fork's pull request; the maintainer decides whether to run it. +## Parity on main + +Most pull requests merge on `verify` alone and leave the vanilla harnesses to `.github/workflows/parity-main.yml`, +which runs them on main a batch of merges at a time, on the Mac mini's runner: + +- It starts on every push to main, one run at a time (`concurrency: parity-main`). Merges that land while a run + goes on wait in one pending run: GitHub keeps only the newest and cancels the others, so the next run takes them + all. +- `tools/parity_main.py base` gives the newest commit main contains whose parity-main passed (else the last + merge's parent), and the run is `tools/parity.py --changed `: the union of the harnesses every merge + since then can affect, once, with every earlier pass reused. +- On a failure, `tools/parity_main.py bisect` checks out the batch's first-parent merges by halves and runs only + the failing harnesses, names the first merge they fail on in the run's summary and comments on its pull request + (`🤖 Agent: parity-main found that this merge breaks ...`). Fix it with a pull request as usual; the next + run checks from the last green commit again. +- Game releases ship only a commit both `checks` and `parity-main` passed on (`tools/release_train.py`, see + below), so a broken merge holds releases instead of reaching players. `release.yml` also starts when a + parity-main run finishes. A forced release (`force=true`) doesn't wait for parity-main. + +A pull request still runs its own parity job when it changes a harness script, an oracle +(`mods/horizon/src/test/`), a probe (`crates/*/examples/`, `game/*_fixture.rs`, `game/*_harness.rs`), the runner +(`tools/parity*.py`, `tools/lab.py`, `tools/parity_hosts.json`, `parity*.yml`), or carries the `run-parity` label +(`run-parity.yml` runs it when the label is added). + ## Game releases -`.github/workflows/release.yml` releases the game by itself. It starts after every green `checks` run of a push -to main, every 10 minutes (so a release waiting for merges to settle goes out without another merge), and by hand. +`.github/workflows/release.yml` releases the game by itself. It starts after every green `checks` or `parity-main` +run of a push to main (it releases a commit only once both passed on it), every 10 minutes (so a release waiting for merges to settle goes out without another merge), and by hand. ```mermaid flowchart LR diff --git a/tests/test_parity_main.py b/tests/test_parity_main.py new file mode 100644 index 000000000..7a72ecae5 --- /dev/null +++ b/tests/test_parity_main.py @@ -0,0 +1,43 @@ +"""tools/parity_main.py: the base of a parity-main run, and the bisect after a failure.""" +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from tools import parity_main + + +class ParityMainTests(unittest.TestCase): + def test_first_bad_finds_the_first_failing_merge(self): + commits = list("abcdefgh") # h failed; the base before a passed + for culprit in commits: + tried = [] + fails = lambda commit: tried.append(commit) or commit >= culprit + self.assertEqual(parity_main.first_bad(commits, fails), culprit) + self.assertLessEqual(len(tried), 3, "log2 of 8 merges") + self.assertEqual(parity_main.first_bad(["a"], lambda commit: self.fail("nothing to try")), "a") + + def test_base_is_the_newest_green_run_head_contains(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.object(parity_main, "ROOT", Path(tmp)): + git = lambda *arguments: parity_main.git(*arguments).stdout.strip() + git("init", "-q") + for n in range(3): + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", str(n)) + head, parent, grandparent = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^1"), git("rev-parse", "HEAD~2") + git("checkout", "-q", "-b", "side", grandparent) + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "side") + side = git("rev-parse", "HEAD") + git("checkout", "-q", head) + self.assertEqual(parity_main.base(runs=[side, grandparent, parent]), grandparent, + "a run on a commit HEAD doesn't contain is skipped") + self.assertEqual(parity_main.base(runs=[]), parent, "no green run yet: the last merge") + self.assertEqual(parity_main.base(runs=[head]), head) + + def test_a_harness_a_commit_does_not_have_cannot_fail_there(self): + with mock.patch.object(parity_main.subprocess, "run") as run: + self.assertFalse(parity_main.run_harnesses(["not_a_harness_yet"], 60)) + run.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_train.py b/tests/test_release_train.py index 56064fa42..ee1fbcae5 100644 --- a/tests/test_release_train.py +++ b/tests/test_release_train.py @@ -2,6 +2,7 @@ import datetime as dt from pathlib import Path import tempfile import unittest +from unittest import mock from tools import release_train as train @@ -147,6 +148,26 @@ class ReleaseTrainTests(unittest.TestCase): self.assertFalse(train.decision("game", [], NOW, new_line="0.3", blockers=[7])[0]) self.assertTrue(train.decision("game", [pr(1)], NOW, blockers=[])[0]) + def test_game_releases_ship_only_what_parity_main_passed(self): + def runs(parity, checks): + def gh_json(*arguments): + table = parity if "parity-main" in arguments else checks + if "--commit" in arguments: + sha = arguments[arguments.index("--commit") + 1] + return [{"status": s, "conclusion": c} for h, s, c in table if h == sha] + return [{"headSha": h, "status": s, "conclusion": c} for h, s, c in table] + return mock.patch.object(train, "gh_json", gh_json) + checks = [("c", "in_progress", ""), ("b", "completed", "success"), ("a", "completed", "success")] + with runs([("c", "queued", ""), ("b", "completed", "cancelled"), ("a", "completed", "success")], checks): + self.assertEqual(train.green_main_head(), "b", "checks alone: the newest green checks") + self.assertEqual(train.green_main_head(parity=True), "a", "a superseded parity-main run counts for nothing") + self.assertTrue(train.passed_on("parity-main", "a")) + self.assertFalse(train.passed_on("parity-main", "b")) + with runs([("b", "completed", "failure"), ("a", "completed", "success")], checks): + self.assertIsNone(train.green_main_head(parity=True), "a failed batch holds releases until fixed") + with runs([("c", "completed", "success")], checks): + self.assertIsNone(train.green_main_head(parity=True), "its checks must pass too") + def test_a_release_is_published_only_after_its_smoke_test_and_with_no_blocker(self): drew = {"error": None, "frames": 12710, "chunks": 81, "meshed_sections": 550, "player": [160.5, 71.0, 156.5]} self.assertIsNone(train.gate(drew, [])) diff --git a/tools/parity_main.py b/tools/parity_main.py new file mode 100644 index 000000000..ff319aed0 --- /dev/null +++ b/tools/parity_main.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""parity-main (.github/workflows/parity-main.yml): the vanilla harnesses run on main, a batch of merges at a time. + + python3 tools/parity_main.py base # the commit this run checks from: main's last green parity-main + python3 tools/parity_main.py bisect BASE # after a failure: the first merge since BASE that broke a harness + +`base` prints the newest commit on main whose parity-main run passed and that HEAD contains, else HEAD's first parent, +so a run tests every merge since the last green one (`tools/parity.py --changed `), however many landed while +the previous run went on. `bisect` reads the harnesses that failed (.local/parity-runner/last-run.json), checks out +the first-parent merges between BASE and HEAD by halves and runs only those harnesses, then names the first merge they +fail on (written to $GITHUB_STEP_SUMMARY and, with --comment, on its pull request). HEAD is checked out again at the end. +""" + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +REPO = os.environ.get("GITHUB_REPOSITORY", "ludovic111/horizon") +LAST_RUN = ROOT / ".local" / "parity-runner" / "last-run.json" + + +def git(*arguments, check=True): + result = subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, text=True) + if check and result.returncode: + sys.exit(f"git {' '.join(arguments)}: {result.stderr.strip()}") + return result + + +def green_runs(): + """Head commits of main's passing parity-main runs, newest first.""" + result = subprocess.run(["gh", "run", "list", "--repo", REPO, "--workflow", "parity-main", "--branch", "main", + "--status", "success", "--limit", "30", "--json", "headSha"], capture_output=True, text=True) + if result.returncode: + print(f"note: no parity-main history ({result.stderr.strip()})", file=sys.stderr) + return [] + return [run["headSha"] for run in json.loads(result.stdout or "[]")] + + +def base(head="HEAD", runs=None): + for sha in green_runs() if runs is None else runs: + if git("merge-base", "--is-ancestor", sha, head, check=False).returncode == 0: + return git("rev-parse", sha).stdout.strip() + return git("rev-parse", f"{head}^1").stdout.strip() + + +def first_bad(commits, fails): + """The first of commits (oldest first, the last known to fail; the one before them known to pass) where fails().""" + good, bad = -1, len(commits) - 1 + while bad - good > 1: + middle = (good + bad) // 2 + if fails(commits[middle]): + bad = middle + else: + good = middle + return commits[bad] + + +def harness_exists(name): + return (ROOT / "tools" / f"test_{name.split('#')[0]}_parity.py").exists() + + +def run_harnesses(names, timeout): + """Whether any of the harnesses fails at the checked-out commit (those it doesn't have yet can't).""" + present = [name for name in names if harness_exists(name)] + if not present: + return False + return subprocess.run([sys.executable, "tools/parity.py", *present, "--timeout", str(timeout)], cwd=ROOT).returncode != 0 + + +def bisect(since, timeout, comment): + failed = [] + if LAST_RUN.exists(): + failed = json.loads(LAST_RUN.read_text(encoding="utf-8")).get("failed") or [] + if not failed: + print("No failed harness recorded: nothing to bisect.") + return 0 + head = git("rev-parse", "HEAD").stdout.strip() + commits = git("rev-list", "--first-parent", "--reverse", f"{since}..{head}").stdout.split() + if not commits: + print("No merge since the base.") + return 0 + + def fails(commit): + print(f"--- {commit[:9]} {git('log', '-1', '--format=%s', commit).stdout.strip()}", flush=True) + git("checkout", "-q", "--detach", commit) + return run_harnesses(failed, timeout) + + try: + culprit = first_bad(commits, fails) if len(commits) > 1 else commits[0] + finally: + git("checkout", "-q", "--detach", head) + subject = git("log", "-1", "--format=%s", culprit).stdout.strip() + number = re.search(r"Merge pull request #(\d+)", subject) + line = (f"parity-main: {', '.join(failed)} first fail at {culprit[:9]} ({subject}), " + f"of {len(commits)} merges since {since[:9]}.") + print(line) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write(f"### {line}\n") + if comment and number: + url = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{REPO}/actions/runs/{os.environ.get('GITHUB_RUN_ID', '')}" + body = (f"🤖 Agent: parity-main found that this merge breaks {', '.join(f'`{name}`' for name in failed)} on main " + f"(bisected over {len(commits)} merges: {url}). Releases wait until parity-main passes again.") + subprocess.run(["gh", "pr", "comment", number.group(1), "--repo", REPO, "--body", body]) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("base") + bisecting = commands.add_parser("bisect") + bisecting.add_argument("since", help="the last commit known to pass (the run's base)") + bisecting.add_argument("--timeout", type=int, default=2400, help="seconds per harness") + bisecting.add_argument("--comment", action="store_true", help="comment on the breaking merge's pull request") + args = parser.parse_args() + if args.command == "base": + print(base()) + return 0 + return bisect(args.since, args.timeout, args.comment) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/release_train.py b/tools/release_train.py index 48d4d46cf..72d81ce57 100644 --- a/tools/release_train.py +++ b/tools/release_train.py @@ -414,13 +414,26 @@ def gather(kind, head, releases=None): return last, prs -def green_main_head(): - """The newest commit on main whose checks passed: what an automatic release builds.""" - runs = gh_json("run", "list", "--repo", REPO, "--workflow", "checks", "--branch", "main", "--event", "push", +def passed_on(workflow, sha): + """Whether the newest finished push run of workflow on main for commit sha passed.""" + runs = gh_json("run", "list", "--repo", REPO, "--workflow", workflow, "--branch", "main", "--event", "push", + "--commit", sha, "--limit", "5", "--json", "status,conclusion") + done = [entry for entry in runs if entry["status"] == "completed" and entry["conclusion"] != "cancelled"] + return bool(done) and done[0]["conclusion"] == "success" + + +def green_main_head(parity=False): + """The newest commit on main whose checks passed: what an automatic release builds. With parity (game + releases), the newest commit parity-main passed on, if its checks passed too: parity-main runs on batches of + merges, so the commits a run skipped are covered by the next one, and a superseded run counts for nothing.""" + workflow = "parity-main" if parity else "checks" + runs = gh_json("run", "list", "--repo", REPO, "--workflow", workflow, "--branch", "main", "--event", "push", "--limit", "10", "--json", "headSha,status,conclusion") for entry in runs: - if entry["status"] == "completed": - return entry["headSha"] if entry["conclusion"] == "success" else None + if entry["status"] == "completed" and entry["conclusion"] != "cancelled": + if entry["conclusion"] != "success": + return None + return entry["headSha"] if not parity or passed_on("checks", entry["headSha"]) else None return None @@ -437,7 +450,7 @@ def main(): parser.add_argument("--kind", choices=["game", "launcher"], required=True) parser.add_argument("--head", help="the commit to release (default: the newest commit on main whose checks passed)") parser.add_argument("--tag", help="notes: the release's tag") - parser.add_argument("--force", action="store_true", help="release even without player-facing changes, before merges settle or while paused") + parser.add_argument("--force", action="store_true", help="release even without player-facing changes, before merges settle, while paused or before parity-main passed") parser.add_argument("--out", type=Path, help="notes: write title.txt and body.md here") parser.add_argument("--now", help="the time to decide at (ISO 8601; default now)") parser.add_argument("--smoke", type=Path, help="gate: the release smoke test's report (smoke-test.json)") @@ -456,9 +469,12 @@ def main(): summary.write(f"### {line}\n") return 1 if problem else 0 - head = args.head or green_main_head() + # Game releases ship only a commit the vanilla harnesses passed on (parity-main, docs/release-flow.md). + parity = args.kind == "game" and args.command == "decide" and not args.force + head = args.head or green_main_head(parity) if not head: - result = {"release": False, "reason": "main's latest checks did not pass"} + result = {"release": False, "reason": "main's latest checks did not pass" + ( + " or parity-main did not pass on them" if parity else "")} print(json.dumps(result, indent=1)) write_outputs({"release": "false", "reason": result["reason"]}) return 0 @@ -482,6 +498,8 @@ def main(): blockers = open_blockers() if args.kind == "game" else [] release, reason = decision(args.kind, prs, now, os.environ.get("RELEASES_PAUSED") == "1", args.force, new_line, blockers) + if release and args.head and parity and not (passed_on("parity-main", head) and passed_on("checks", head)): + release, reason = False, f"waiting for checks and parity-main to pass on {head[:9]}" minor = any("feature" in pr["labels"] for pr in prs if pr["paths"]) tag = next_game_tag(tags, now.date(), line) if args.kind == "game" else next_launcher_tag(tags, minor, line) result = { From 40c08c2d12ebfb8d0a56e9bc2f71b72460cfff0c Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:38:47 +0100 Subject: [PATCH 12/13] streaming: check every fourth cell of the cover test in debug builds the_grid_answers_as_the_scan compared the grid with the scan on every cell of 60 cases and took 20 of the ~30 s of `cargo test -p horizon-client` on the ThinkPad. A debug build now checks every fourth cell of each row, shifted by row and case, so the cases together still meet every cell position relative to the areas; a release build checks every cell. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/streaming.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/horizon-client/src/streaming.rs b/crates/horizon-client/src/streaming.rs index 55a2f0086..13b19fefc 100644 --- a/crates/horizon-client/src/streaming.rs +++ b/crates/horizon-client/src/streaming.rs @@ -2148,8 +2148,13 @@ mod cover_tests { } /// Every question streaming asks gets the same answer with the grid. + /// + /// A debug build checks every fourth cell of each row, shifted by row and case so the cases still cover every + /// cell's position relative to the areas: the full check took 20 of the ~30 s of the crate's tests. + /// `cargo test --release -p horizon-client --lib the_grid_answers_as_the_scan` checks every cell. #[test] fn the_grid_answers_as_the_scan() { + let stride = if cfg!(debug_assertions) { 4 } else { 1 }; let mut seed = 0x9E37_79B9_7F4A_7C15u64; let forced: HashSet<(i32, i32)> = [(40, -40)].into(); for case in 0..60 { @@ -2165,7 +2170,8 @@ mod cover_tests { let (scan, grid) = (ranges(None), ranges(Some(&cover))); let span = [4, 30, 120][case % 3] + 40; for z in -span..=span { - for x in -span..=span { + let first = -span + (z + case as i32).rem_euclid(stride); + for x in (first..=span).step_by(stride as usize) { for m in 0..=ExtraCover::REACH + 2 { assert_eq!( scan.in_extra(x, z, m), From 5034e4557dba8c54ebd406a50ef4449b8d7ec2ea Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:44:22 +0100 Subject: [PATCH 13/13] parity: clear rust--parity outputs after a pass, reuse oracles on CI Disk: clear_large_outputs only looked at .local directories named after the harness, so the 174 harnesses writing to rust--parity kept every dump (86 GB in one runner checkout, 13 GB for state_provider alone). It now clears those too, keeps vanilla outputs only up to 256 MB (larger ones cost more disk than rerunning their oracle), and handles shard names. collision, world and geometry dropped one mkdtemp run directory per run (11 GB): each run now removes the earlier ones. Time: harnesses inherited the runner's stdin, a pipe on GitHub's runner, and parity_shim never reuses an oracle run whose stdin is a pipe, so oracle reuse never worked on CI. Harnesses now get /dev/null. CI: parity.yml falls back to the head's own selection when the names main's `--list` printed don't match its summary count. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parity.yml | 11 ++++++++++- tests/test_parity_runner.py | 14 ++++++++++++++ tools/parity.py | 20 ++++++++++++++------ tools/test_collision_parity.py | 3 +++ tools/test_geometry_parity.py | 3 +++ tools/test_world_parity.py | 3 +++ 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index be87bc965..9dcb21775 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -68,7 +68,16 @@ jobs: exit fi names=$(sed '$d' .local/parity-selection.txt | grep -E '^[A-Za-z0-9_#]+$' || true) - if [ -z "$names" ]; then + # The summary line says how many would run: a count that doesn't match the names read (a format change) + # must not let the job pass untested. + expected=$(tail -1 .local/parity-selection.txt | sed -nE 's/^([0-9]+) of [0-9]+ harnesses would run$/\1/p') + count=$(printf '%s' "$names" | grep -c . || true) + if [ -z "$expected" ] || [ "$expected" != "$count" ]; then + echo "main's selection listed $count names, its summary says '${expected:-?}': the head's own selection runs" + own + exit + fi + if [ "$count" = 0 ]; then echo "Nothing to run." exit 0 fi diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py index d10fde854..2d16e7179 100644 --- a/tests/test_parity_runner.py +++ b/tests/test_parity_runner.py @@ -300,3 +300,17 @@ class LargeOutputTests(unittest.TestCase): self.assertFalse((own / "dump.txt").exists()) # chunk_generation may still be writing there. self.assertTrue((shared / "dump.txt").exists()) + + def test_rust_named_directories_and_big_oracle_outputs_go_too(self): + with tempfile.TemporaryDirectory() as tmp: + local, state = Path(tmp) / ".local", Path(tmp) / "state" + own = local / "rust-state-provider-parity" + own.mkdir(parents=True) + for name, size in (("rust.txt", 16), ("server.txt", 16), ("client.txt", 64), ("report.json", 4)): + (own / name).write_bytes(b"x" * size) + trace = {"commands": [["java", "Oracle", str(own)], ["probe", "fixture", str(own)]]} + with mock.patch.multiple(parity, LOCAL=local, STATE_DIR=state, LARGE_OUTPUT=8, KEPT_ORACLE_OUTPUT=32): + listing = parity.outputs_listing("state_provider#2", fresh=True) + listing.write_text(f"{own / 'server.txt'}\n{own / 'client.txt'}\n") + parity.clear_large_outputs("state_provider#2", trace) + self.assertEqual(sorted(path.name for path in own.iterdir()), ["report.json", "server.txt"]) diff --git a/tools/parity.py b/tools/parity.py index dc5fdb1c9..3f18b922d 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -657,6 +657,9 @@ def prebuild(artifacts, jobs): LARGE_OUTPUT = 64 << 20 +# Vanilla outputs above this are deleted after a pass too: rerunning those few oracles (12 harnesses on the Mac +# mini's runner) costs less than the 26 GB they held in every checkout; smaller ones stay for oracle reuse. +KEPT_ORACLE_OUTPUT = 256 << 20 OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"} @@ -672,11 +675,12 @@ def clear_large_outputs(name, trace): """After a pass, delete the large files a harness regenerates on every run. Harness outputs reach gigabytes and used to stay in every worktree. Vanilla - oracle outputs are kept, since reusing an oracle run needs them in place; - everything else large under the harness's .local directories is rebuilt on - its next run.""" + oracle outputs under KEPT_ORACLE_OUTPUT are kept, since reusing an oracle + run needs them in place; everything else large under the harness's .local + directories (`...` or `rust--parity`) is rebuilt on its next run.""" local = LOCAL.resolve() keep = {os.path.realpath(line) for line in outputs_listing(name).read_text(encoding="utf-8").splitlines() if line} + own = name.split("#")[0].replace("_", "-") directories = set() for command in trace["commands"]: for argument in command[1:]: @@ -688,12 +692,13 @@ def clear_large_outputs(name, trace): # Only the harness's own directory: another harness running beside it may be writing # large files in a directory this one merely read (decorated_region compiles its oracle # into chunk-generation-parity while chunk_generation writes its dumps there). - if top not in OWN_DIRECTORIES and top.startswith(name.replace("_", "-")): + if top not in OWN_DIRECTORIES and (top.startswith(own) or top == f"rust-{own}-parity"): directories.add(local / top) for directory in directories: for file in (directory.rglob("*") if directory.is_dir() else []): try: - if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep: + size = file.stat().st_size if file.is_file() and not file.is_symlink() else 0 + if size >= LARGE_OUTPUT and (str(file) not in keep or size > KEPT_ORACLE_OUTPUT): file.unlink() except OSError: pass @@ -828,8 +833,11 @@ class LocalExecutor: environment.pop("PARITY_SHARD", None) if shard: environment["PARITY_SHARD"] = shard + # stdin from /dev/null: tools/parity_shim.py never reuses an oracle run that reads a pipe, and CI runners + # (and --worker's ssh) give the runner one, which kept oracle reuse off on CI. process = subprocess.Popen([sys.executable, str(Path(__file__)), "--trace", str(trace), str(script)], - cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + cwd=ROOT, env=environment, stdin=subprocess.DEVNULL, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) self.running[name] = {"process": process, "started": time.monotonic(), "log": log, "expected": expected, "slot": slot, "script": script} return True diff --git a/tools/test_collision_parity.py b/tools/test_collision_parity.py index ad99ab960..26f7f25df 100644 --- a/tools/test_collision_parity.py +++ b/tools/test_collision_parity.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path import platform +import shutil import subprocess import tempfile @@ -37,6 +38,8 @@ def main(): if args.prepare: lab.prepare() lock, jars, libraries = test_rust_parity.verify_artifacts() + for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks + shutil.rmtree(earlier, ignore_errors=True) base = Path(tempfile.mkdtemp(prefix="run-", dir=directory)) oracle = lab.ROOT / "mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java" sources = [oracle, Path(__file__).resolve(), lab.ROOT / "tools/test_rust_parity.py", lab.ROOT / "Cargo.toml", lab.ROOT / "Cargo.lock", lab.ROOT / "crates/horizon-core/Cargo.toml", lab.ROOT / "crates/horizon-world/Cargo.toml", *sorted((lab.ROOT / "crates/horizon-core/src").rglob("*.rs")), *sorted((lab.ROOT / "crates/horizon-world/src").rglob("*.rs")), lab.ROOT / "crates/horizon-world/examples/probe/collision_fixture.rs"] diff --git a/tools/test_geometry_parity.py b/tools/test_geometry_parity.py index 54ebc21ae..fa58bf66f 100644 --- a/tools/test_geometry_parity.py +++ b/tools/test_geometry_parity.py @@ -9,6 +9,7 @@ from pathlib import Path import platform import random import struct +import shutil import subprocess import tempfile @@ -273,6 +274,8 @@ def main(): if args.prepare: lab.prepare() lock, jars, libraries = test_rust_parity.verify_artifacts() + for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks + shutil.rmtree(earlier, ignore_errors=True) base = Path(tempfile.mkdtemp(prefix="run-", dir=directory)) commands = list(corpus()) inputs = base / "inputs.txt" diff --git a/tools/test_world_parity.py b/tools/test_world_parity.py index 0b4a3deb7..9344a47b0 100644 --- a/tools/test_world_parity.py +++ b/tools/test_world_parity.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path import platform +import shutil import subprocess import tempfile @@ -35,6 +36,8 @@ def main(): if args.prepare: lab.prepare() lock, jars, libraries = test_rust_parity.verify_artifacts() + for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks + shutil.rmtree(earlier, ignore_errors=True) base = Path(tempfile.mkdtemp(prefix="run-", dir=directory)) classes = base / "classes" classes.mkdir()