diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98d5b9093..eff82fa99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,8 @@ name: checks # check main requires, so it never has a condition: a skipped job counts as passed, and every run of this workflow # must report a real verify (a pull request that changes no Rust finishes it in seconds instead). # parity: the vanilla harnesses a pull request's changes can affect, on the Mac mini's runner, which holds the -# maintainer's own Minecraft files (Horizon never downloads the game); .github/workflows/parity.yml. A fork's pull +# maintainer's own Minecraft files (Horizon never downloads the game); .github/workflows/parity.yml. Only for a pull +# request that changes a harness, oracle, probe or the runner: the rest are checked on main by parity-main. A fork's pull # request runs it only when a maintainer adds the run-parity label (.github/workflows/run-parity.yml), so labels # never start this workflow. See docs/release-flow.md. on: @@ -72,8 +73,37 @@ jobs: run: | cargo +stable test --workspace --lib --bins --tests --features horizon-client/mods,horizon-mods/pack cargo +stable test --workspace --doc --features horizon-client/mods,horizon-mods/pack - parity: + # Most pull requests leave the harnesses to parity-main, which runs them on main a batch of merges at a time + # (.github/workflows/parity-main.yml; releases wait for it). A pull request still runs them itself when it changes + # a harness, an oracle, a probe or the runner, or carries the run-parity label (run-parity.yml, when it is added). + parity-scope: if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + outputs: + run: ${{ steps.scope.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + - id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + LABELS: ${{ join(github.event.pull_request.labels.*.name, ' ') }} + run: | + changed=$(git diff --name-only "$(git merge-base "$BASE_SHA" HEAD)" HEAD) + harness=$(grep -E '^(tools/test_[a-z0-9_]+_parity\.py|tools/(parity[a-z_]*|lab)\.py|tools/parity_hosts\.json|mods/horizon/src/test/|crates/[^/]+/examples/|crates/horizon-client/src/game/[a-z_]*(fixture|harness)\.rs|\.github/workflows/parity[a-z-]*\.yml)' <<< "$changed" || true) + if [ -n "$harness" ] || [[ " $LABELS " == *" run-parity "* ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + echo "The pull request runs its harnesses: ${harness:-label run-parity}" + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "No harness, oracle, probe or runner change: parity-main checks this on main" + fi + parity: + needs: parity-scope + if: needs.parity-scope.outputs.run == 'true' uses: ./.github/workflows/parity.yml with: base: ${{ github.event.pull_request.base.sha }} diff --git a/.github/workflows/parity-main.yml b/.github/workflows/parity-main.yml new file mode 100644 index 000000000..1ddfb24bb --- /dev/null +++ b/.github/workflows/parity-main.yml @@ -0,0 +1,70 @@ +name: parity-main +# The vanilla harnesses on main, a batch of merges at a time (docs/release-flow.md#parity-on-main), on the Mac mini's +# runner. One run at a time: merges that land while a run goes on wait in one pending run (GitHub keeps only the +# newest), which checks every merge since main's last green parity-main (tools/parity_main.py base). On a failure it +# bisects the failing harnesses over those merges and comments on the pull request that broke them. Releases ship +# only a commit whose parity-main passed (tools/release_train.py), so nothing a harness catches reaches players. +on: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read + # The comment on the merge that broke a harness. + pull-requests: write + # tools/parity_main.py base reads earlier runs. + actions: read +concurrency: + group: parity-main + cancel-in-progress: false +env: + # Test runs never reach the telemetry server: no live-count ping, no uploads (docs/telemetry.md). + HORIZON_TELEMETRY_URL: "off" + +jobs: + parity-main: + runs-on: [self-hosted, horizon-game] + timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + # Keeps .local/ (parity state, probe builds, oracle results) and target/ between runs. + clean: false + - uses: dtolnay/rust-toolchain@stable + - name: Tools from the runner + run: bash tools/ci/runner-env.sh + - name: Vanilla data, and resources prepared by this preparer + run: bash tools/ci/vanilla-data.sh + - name: Main's last green parity-main + id: base + env: + GH_TOKEN: ${{ github.token }} + run: echo "sha=$(python3 tools/parity_main.py base)" >> "$GITHUB_OUTPUT" + - name: The harnesses the merges since then can affect + env: + PARITY_HOSTS: '' + BASE_SHA: ${{ steps.base.outputs.sha }} + run: | + echo "Merges since ${BASE_SHA}:" + git log --first-parent --format=' %h %s' "$BASE_SHA..HEAD" + python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400 + - name: The merge that broke them + if: failure() + env: + GH_TOKEN: ${{ github.token }} + PARITY_HOSTS: '' + BASE_SHA: ${{ steps.base.outputs.sha }} + run: | + rm -rf .local/parity-main-logs && cp -R .local/parity-runner/logs .local/parity-main-logs 2>/dev/null || true + python3 tools/parity_main.py bisect "$BASE_SHA" --comment + - name: Logs of the failed harnesses + if: failure() + uses: actions/upload-artifact@v4 + with: + name: parity-main-logs + path: .local/parity-main-logs/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index a18fc5019..577380338 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -37,13 +37,56 @@ jobs: run: bash tools/ci/runner-env.sh - name: Vanilla data, and resources prepared by this preparer run: bash tools/ci/vanilla-data.sh + - name: The prepared output is the one the preparer's OUTPUT_VERSION says + env: + BASE_SHA: ${{ inputs.base }} + run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA" - name: The version constants are the pinned jar's run: python3 tools/gen_version_consts.py --check - name: The harnesses this change can affect env: PARITY_HOSTS: '' BASE_SHA: ${{ inputs.base }} - run: python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400 # world_spawn takes ~1480 s on a busy runner + run: | + # main's selection (tools/parity.py --changed --list with main's parity.py and parity_reach.py), so a + # branch made before a better selection landed gets it too; the head's own runner then runs those + # harnesses. A branch that changes the selection itself selects with its own code. + set -euo pipefail + own() { python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400; } # world_spawn takes ~1480 s on a busy runner + fork_point=$(git merge-base "$BASE_SHA" HEAD) + if ! git rev-parse --verify -q origin/main >/dev/null || ! git diff --quiet "$fork_point" HEAD -- tools/parity.py tools/parity_reach.py; then + own + exit + fi + had_reach=$(git ls-files tools/parity_reach.py) + git show origin/main:tools/parity.py > tools/parity.py + git show origin/main:tools/parity_reach.py > tools/parity_reach.py + status=0 + python3 tools/parity.py --changed "$BASE_SHA" --list > .local/parity-selection.txt || status=$? + git checkout -- tools/parity.py + if [ -n "$had_reach" ]; then git checkout -- tools/parity_reach.py; else rm -f tools/parity_reach.py; fi + cat .local/parity-selection.txt + if [ "$status" != 0 ]; then + echo "main's selection failed: the head's own selection runs" + own + exit + fi + names=$(sed '$d' .local/parity-selection.txt | grep -E '^[A-Za-z0-9_#]+$' || true) + # The summary line says how many would run: a count that doesn't match the names read (a format change) + # must not let the job pass untested. + expected=$(tail -1 .local/parity-selection.txt | sed -nE 's/^([0-9]+) of [0-9]+ harnesses would run$/\1/p') + count=$(printf '%s' "$names" | grep -c . || true) + if [ -z "$expected" ] || [ "$expected" != "$count" ]; then + echo "main's selection listed $count names, its summary says '${expected:-?}': the head's own selection runs" + own + exit + fi + if [ "$count" = 0 ]; then + echo "Nothing to run." + exit 0 + fi + # shellcheck disable=SC2086 + python3 tools/parity.py $names --timeout 2400 - name: Logs of the failed harnesses if: failure() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4cba851fa..05f590096 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,7 @@ name: release # Automatic game releases (docs/release-flow.md). After every green `checks` run on main, every 10 minutes (so a -# release waiting for merges to settle goes out without another merge) and by hand, tools/release_train.py decides: +# release waiting for merges to settle goes out without another merge) and by hand, tools/release_train.py decides, +# for a commit whose checks and parity-main (the vanilla harnesses on main) both passed: # a release when 5 pull requests merged since the last one changed what players run and the newest merged 5 minutes # ago, or when the oldest has waited 6 hours, or at once for a pull request labelled release-now. # RELEASES_PAUSED=1 (repository variable) stops them; so does an open issue labelled release-blocker, forced runs too. @@ -18,7 +19,8 @@ name: release # gh workflow run release -f dry_run=false -f force=true # on main: release now, whatever the rules say on: workflow_run: - workflows: [checks] + # A game release ships a commit both passed on (parity-main runs on batches: the newest commit it passed on). + workflows: [checks, parity-main] types: [completed] branches: [main] schedule: @@ -64,8 +66,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASES_PAUSED: ${{ vars.RELEASES_PAUSED }} - # The commit whose checks just passed; a dry run builds the branch it was started on; otherwise - # main's newest commit with green checks. + # The commit whose checks or parity-main just passed; a dry run builds the branch it was started on; + # otherwise main's newest commit with green checks and parity-main. HEAD_SHA: ${{ github.event.workflow_run.head_sha || (steps.mode.outputs.dry_run == 'true' && github.sha) || '' }} FORCE: ${{ inputs.force && 'yes' || '' }} run: | diff --git a/AGENTS.md b/AGENTS.md index 2d883b33b..549fa5c7f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,7 +82,7 @@ python3 tools/agent_control_e2e.py # horizon-ctl / MCP end to end ~25, a file of the client lib ~103, of `horizon-world` ~103–109, of `horizon-core` ~119. Plan for it. - Rust integration tests: `crates/horizon-client/tests/` (`lan_bot.rs`, `generation_saves.rs`, `streaming_meshes.rs`). - **macOS:** `cargo build/test --workspace` may fail to link the two wasm guest cdylibs (undefined - `horizon:mods/events` symbols) until #1446 lands: add `--exclude coords-hud --exclude horizon-mods-test-guest --exclude horizon-mods-test-ui-guest`. + `horizon:mods/events` symbols) until #1446 lands: add `--exclude coords-hud --exclude predator-missile --exclude horizon-mods-test-guest --exclude horizon-mods-test-ui-guest`. Their real check is `cargo test -p horizon-mods` (builds them for `wasm32-unknown-unknown`). - Linux needs `libasound2-dev libudev-dev`; `--no-default-features` drops audio and controllers (the server build). - Run and drive the game (starts the `horizon-client` beside `horizon-ctl`; other arguments go to the game): diff --git a/Cargo.lock b/Cargo.lock index 4d749add6..cb5f2f554 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2733,6 +2733,13 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "predator-missile" +version = "0.1.0" +dependencies = [ + "horizon-mod-sdk", +] + [[package]] name = "presser" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 4076b9079..fff062e3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] exclude = ["launcher", "crates/horizon-telemetry-server", "crates/horizon-mod-registry", "third_party/wgpu-hal"] -members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/horizon-mods/tests/ui_guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-shaders", "crates/horizon-shader-api", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"] +members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/horizon-mods/tests/ui_guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-shaders", "crates/horizon-shader-api", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "crates/horizon-mod-sdk/examples/predator-missile", "xtask"] resolver = "2" [workspace.package] diff --git a/crates/cargo-horizon-mod/src/check.rs b/crates/cargo-horizon-mod/src/check.rs index 8e1b2bc9b..cefddf966 100644 --- a/crates/cargo-horizon-mod/src/check.rs +++ b/crates/cargo-horizon-mod/src/check.rs @@ -224,6 +224,7 @@ fn play(path: &Path, report: &mut Report) -> Result<(), String> { Output::Feedback { text, error } => { format!("{}: {text}", if error { "error" } else { "feedback" }) } + Output::Command { line } => format!("command: /{line}"), Output::Stopped { reason, .. } => return Err(format!("it stopped: {reason}")), Output::OpenScreen { .. } => "opened a screen".to_string(), Output::CloseScreen { .. } => "closed its screen".to_string(), diff --git a/crates/horizon-client/src/animate_tick.rs b/crates/horizon-client/src/animate_tick.rs index ce34df510..48e8f16f4 100644 --- a/crates/horizon-client/src/animate_tick.rs +++ b/crates/horizon-client/src/animate_tick.rs @@ -63,6 +63,10 @@ enum Animate { SculkSensor, /// Potent sulfur that is not dry. PotentSulfur, + DriedGhast { + waterlogged: bool, + }, + EndPortal, } fn step(facing: &str) -> [i32; 3] { match facing { @@ -137,6 +141,10 @@ fn classify(name: &str, prop: &dyn Fn(&str) -> Option) -> Animate { drag: prop("drag").as_deref() == Some("true"), }, "fire" | "soul_fire" => Animate::Fire, + "dried_ghast" => Animate::DriedGhast { + waterlogged: prop("waterlogged").as_deref() == Some("true"), + }, + "end_portal" => Animate::EndPortal, "sculk_sensor" | "calibrated_sculk_sensor" if prop("sculk_sensor_phase").as_deref() == Some("active") => { @@ -180,6 +188,8 @@ pub struct AnimateTables { impermeable: Vec, power_provider: Vec, power_transmitter: Vec, + /// `#triggers_ambient_dried_ghast_block_sounds` (soul sand, soul soil). + dried_ghast_sounds: Vec, /// Biome id -> ambient particles and probabilities (`minecraft:visual/ambient_particles`). ambient: Vec>, } @@ -237,6 +247,7 @@ impl AnimateTables { impermeable: by_state("minecraft:impermeable"), power_provider: by_state("minecraft:enchantment_power_provider"), power_transmitter: by_state("minecraft:enchantment_power_transmitter"), + dried_ghast_sounds: by_state("minecraft:triggers_ambient_dried_ghast_block_sounds"), ambient, }) } @@ -869,6 +880,45 @@ fn block_animate_tick( ]; add(engine, T::Smoke, p, [0.0; 3]); } + Animate::DriedGhast { waterlogged } => { + // DriedGhastBlock.animateTick. + let c = [fx + 0.5, fy + 0.5, fz + 0.5]; + if waterlogged { + if next_int(engine.random(), 40) == 0 { + let name = "minecraft:block.dried_ghast.ambient_water"; + sound(engine, name, c, 1.0, 1.0); + } + let r = engine.random(); + if next_int(r, 6) == 0 { + let x = c[0] + f64::from((r.next_float() * 2.0 - 1.0) / 3.0); + let y = c[1] + 0.4; + let z = c[2] + f64::from((r.next_float() * 2.0 - 1.0) / 3.0); + let v = [0.0, f64::from(r.next_float()), 0.0]; + add(engine, T::HappyVillager, [x, y, z], v); + } + } else { + let below = ctx.state([pos[0], pos[1] - 1, pos[2]]); + let triggers = || { + tables + .dried_ghast_sounds + .get(below.0 as usize) + .copied() + .unwrap_or(false) + }; + if next_int(engine.random(), 40) == 0 && triggers() { + sound(engine, "minecraft:block.dried_ghast.ambient", c, 1.0, 1.0); + } + if next_int(engine.random(), 6) == 0 { + add(engine, T::WhiteSmoke, c, [0.0, 0.02, 0.0]); + } + } + } + Animate::EndPortal => { + // EndPortalBlock.animateTick: one smoke particle above the surface. + let r = engine.random(); + let p = [fx + r.next_double(), fy + 0.8, fz + r.next_double()]; + add(engine, T::Smoke, p, [0.0; 3]); + } Animate::EnderChest => { for _ in 0..3 { let r = engine.random(); diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index f5803ad08..1e6158bc4 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -20,6 +20,7 @@ mod bossbars; mod breezes; mod cartography; mod chat; +mod client_ambient; mod clipboard; mod combat; mod command_blocks; @@ -4481,6 +4482,7 @@ impl Game<'_> { self.tick_block_entity_anims() }); horizon_client::spans::time("tick_particles", || self.tick_particles()); + self.tick_client_ambient(); } // ServerBossEvent broadcasts, after this level's entities have ticked. self.publish_boss_bars(); diff --git a/crates/horizon-client/src/game/client_ambient.rs b/crates/horizon-client/src/game/client_ambient.rs new file mode 100644 index 000000000..a10c6da49 --- /dev/null +++ b/crates/horizon-client/src/game/client_ambient.rs @@ -0,0 +1,37 @@ +//! The window's own ambient effects of entities, which a client makes for +//! itself (`level().isClientSide()` branches): endermen's and endermites' +//! portal particles, blazes' smoke and burn, eyes of ender's trail. They run +//! on the game that has the window, in-frame or following a server, never on +//! a server: nothing of them goes over the network. +use super::Game; +use horizon_client::mobs::{MobEvent, ambient}; +use horizon_client::particles::{ParticleOptions, ParticleType}; + +impl Game<'_> { + pub(super) fn tick_client_ambient(&mut self) { + if self.serving.is_some() || self.options.dedicated.is_some() { + return; + } + let mut events = Vec::new(); + for m in self.mobs.iter_mut().filter(|m| !m.removed) { + ambient::client_ai_step(m, &mut events); + } + for event in events { + match event { + MobEvent::ParticleAt { kind, at, velocity } => { + if let Some(ty) = ParticleType::from_name(kind) { + self.particles + .add(&ParticleOptions::simple(ty), at, velocity); + } + } + // Each guest of a LAN host makes its own. + MobEvent::LocalSound(name, at, volume, pitch) => { + self.audio + .play_unshared(&format!("minecraft:{name}"), Some(at), volume, pitch) + } + _ => {} + } + } + self.eye_particles(); + } +} diff --git a/crates/horizon-client/src/game/end.rs b/crates/horizon-client/src/game/end.rs index 599158e3e..655971a5c 100644 --- a/crates/horizon-client/src/game/end.rs +++ b/crates/horizon-client/src/game/end.rs @@ -4,7 +4,12 @@ //! inactive until the dragon dies. use super::{Dim, Game}; use horizon_client::{ - Result, environment::Environment, items::ItemStack, mesher::Vertex, motion::PlayerMotion, + Result, + environment::Environment, + items::ItemStack, + mesher::Vertex, + motion::PlayerMotion, + particles::{ParticleOptions, ParticleType}, vanillagen::VanillaTerrain, }; use horizon_core::{ @@ -1122,6 +1127,35 @@ impl Game<'_> { self.end.eyes = eyes; } + /// `EyeOfEnder.tick` on a client: behind each eye (a quarter of its last + /// move back), four bubbles in water, else a portal particle. + pub(super) fn eye_particles(&mut self) { + for i in 0..self.end.eyes.len() { + let e = &self.end.eyes[i]; + let movement = (e.position - e.previous).to_array(); + let origin = (e.position - (e.position - e.previous) * 0.25).to_array(); + let block = origin.map(|v| v.floor() as i32); + let water = self + .res + .facts() + .get(self.world.block(block[0], block[1], block[2])) + .fluid + == horizon_client::blocks::Fluid::Water; + if water { + let bubble = ParticleOptions::simple(ParticleType::Bubble); + for _ in 0..4 { + self.particles.add(&bubble, origin, movement); + } + } else { + let r = self.particles.random(); + let x = origin[0] + r.next_double() * 0.6 - 0.3; + let z = origin[2] + r.next_double() * 0.6 - 0.3; + let portal = ParticleOptions::simple(ParticleType::Portal); + self.particles + .add(&portal, [x, origin[1] - 0.5, z], movement); + } + } + } /// ThrownItemRenderer: the ender eye item facing the camera. fn eye_geometry(&self, camera: glam::DVec3, out: &mut Vec) { let Some(r) = self.gui.atlas.sprites.get("item/ender_eye").copied() else { diff --git a/crates/horizon-client/src/game/mods.rs b/crates/horizon-client/src/game/mods.rs index 2884ed21d..931418931 100644 --- a/crates/horizon-client/src/game/mods.rs +++ b/crates/horizon-client/src/game/mods.rs @@ -42,6 +42,11 @@ pub(super) struct ModsState { /// The chat's command tree while a running mod owns commands. #[cfg(feature = "mods")] tree: Option, + /// `command.run` lines for this window's own world, run on the next tick: + /// run at once, their chat lines would reach the mods while their output + /// is still being shown, and a mod answering its own lines would recurse. + #[cfg(feature = "mods")] + commands: Vec, /// The server's command tree with the running mods' commands added. #[cfg(feature = "mods")] remote_tree: Option, @@ -337,6 +342,11 @@ impl Game<'_> { } /// A game tick. pub(super) fn tick_mods(&mut self) { + // Here, at the player's level: shown inside a console call (a control + // socket command set the hook off), they would run as the console. + for line in std::mem::take(&mut self.mods.commands) { + self.run_command(&line); + } if !self.mods.mods.is_running() { return; } @@ -396,6 +406,19 @@ impl Game<'_> { } Output::Feedback { text, error: false } => Text::literal(text), Output::Feedback { text, error: true } => Text::literal(text).with_color(text::RED), + // Vanilla's commands only: a mod running a mod command could loop. + Output::Command { line } => { + let name = line.split(' ').next().unwrap_or_default(); + if !vanilla_command(name) { + // Dropped. + } else if self.remote.is_some() { + // The server checks the player's permissions. + self.send_chat(&format!("/{line}")); + } else { + self.mods.commands.push(line); + } + continue; + } Output::Stopped { name, reason } => { // Its screen goes with it: the entry's run is gone, so // `mod_ui_view` finds no widgets and the screen closes. diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index a34bc71ba..03b48df3d 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -3060,6 +3060,7 @@ impl Game<'_> { self.tick_block_entity_anims(); self.tick_hand(); self.tick_particles(); + self.tick_client_ambient(); self.tick_client_effects(); // Player.tick's sleep counter (the darkening and the wake-up fade). self.sleep_timer = super::tick_sleep_timer(self.sleep_timer, self.sleeping.is_some()); diff --git a/crates/horizon-client/src/graphics.rs b/crates/horizon-client/src/graphics.rs index 21d38539e..d3318aab2 100644 --- a/crates/horizon-client/src/graphics.rs +++ b/crates/horizon-client/src/graphics.rs @@ -385,14 +385,18 @@ impl Graphics { if self.workers > 0 { return self.workers as usize; } - // The pool never takes every core. The render thread and the - // integrated server thread (single player runs on its own server - // thread by default) each need one, and a pool that fills every core - // starves them: the frame rate collapses to single digits while the - // workers saturate the CPU and its memory bandwidth. + // Vanilla's count (Util.maxAllowedExecutorThreads: every core but + // one). The pool never takes every core: a pool that fills them all + // starves the render and server threads, and the frame rate collapses + // while the workers saturate the CPU and its memory bandwidth. One + // core left is enough with the workers behind those two threads + // (horizon_threads: niced on Linux; on macOS and Windows the server + // and render threads are raised above four cores, where vanilla + // raises its server thread). Leaving two idled a core of a 4-thread + // laptop while a world loaded: a teleport there took ~15% longer. let auto = std::thread::available_parallelism() .map_or(2, |n| n.get()) - .saturating_sub(2) + .saturating_sub(1) .max(1); if self.battery { auto.div_ceil(2) } else { auto } } @@ -555,11 +559,11 @@ mod tests { } #[test] - fn automatic_workers_leave_cores_for_the_render_and_server_threads() { + fn automatic_workers_leave_a_core_as_vanilla_does() { let g = Graphics::default(); assert_eq!(g.workers, 0); let cores = std::thread::available_parallelism().map_or(2, |n| n.get()); - assert_eq!(g.worker_threads(), cores.saturating_sub(2).max(1)); + assert_eq!(g.worker_threads(), cores.saturating_sub(1).max(1)); // A manual count is the player's own choice, kept as asked. let manual = Graphics { workers: 3, ..g }; assert_eq!(manual.worker_threads(), 3); diff --git a/crates/horizon-client/src/mobs.rs b/crates/horizon-client/src/mobs.rs index 1a124b6de..46834e9d1 100644 --- a/crates/horizon-client/src/mobs.rs +++ b/crates/horizon-client/src/mobs.rs @@ -7,6 +7,7 @@ //! behaviour, models, animations and sounds. pub mod ai; pub mod allay; +pub mod ambient; pub mod anim; pub mod animal_brain; pub mod aquatic; diff --git a/crates/horizon-client/src/mobs/ambient.rs b/crates/horizon-client/src/mobs/ambient.rs new file mode 100644 index 000000000..cb4146e08 --- /dev/null +++ b/crates/horizon-client/src/mobs/ambient.rs @@ -0,0 +1,55 @@ +//! What a client's copy of a mob makes on its own, in the +//! `level().isClientSide()` branches of `aiStep`: nothing of it reaches the +//! server or goes over the network, so the window's game runs it, whether it +//! simulates the world (in-frame) or follows a server. +use super::{Mob, MobEvent, Species}; +use horizon_core::random::RandomSource; + +/// `Enderman.aiStep`, `Endermite.aiStep` and `Blaze.aiStep` on a client: +/// two portal particles a tick around endermen and endermites; two large +/// smoke particles a tick around blazes and, one tick in 24, their burn. +pub fn client_ai_step(m: &mut Mob, out: &mut Vec) { + let species = m.species(); + if !matches!( + species, + Species::Enderman | Species::Endermite | Species::Blaze + ) { + return; + } + let (w, h) = (f64::from(m.width()), f64::from(m.height())); + let p = m.position; + if species == Species::Blaze && m.random.next_int_bound(24).unwrap_or(1) == 0 && !m.silent { + let volume = 1.0 + m.random.next_float(); + let pitch = m.random.next_float() * 0.7 + 0.3; + out.push(MobEvent::LocalSound( + "entity.blaze.burn", + [p.x + 0.5, p.y + 0.5, p.z + 0.5], + volume, + pitch, + )); + } + for _ in 0..2 { + // Entity.getRandomX(0.5), getRandomY(), getRandomZ(0.5). + let r = &mut m.random; + let x = p.x + w * (2.0 * r.next_double() - 1.0) * 0.5; + let y = p.y + h * r.next_double(); + let z = p.z + w * (2.0 * r.next_double() - 1.0) * 0.5; + let (kind, at, velocity) = if species == Species::Blaze { + ("large_smoke", [x, y, z], [0.0; 3]) + } else { + // The enderman's are a quarter block lower. + let y = if species == Species::Enderman { + y - 0.25 + } else { + y + }; + let velocity = [ + (r.next_double() - 0.5) * 2.0, + -r.next_double(), + (r.next_double() - 0.5) * 2.0, + ]; + ("portal", [x, y, z], velocity) + }; + out.push(MobEvent::ParticleAt { kind, at, velocity }); + } +} diff --git a/crates/horizon-client/src/streaming.rs b/crates/horizon-client/src/streaming.rs index 55a2f0086..13b19fefc 100644 --- a/crates/horizon-client/src/streaming.rs +++ b/crates/horizon-client/src/streaming.rs @@ -2148,8 +2148,13 @@ mod cover_tests { } /// Every question streaming asks gets the same answer with the grid. + /// + /// A debug build checks every fourth cell of each row, shifted by row and case so the cases still cover every + /// cell's position relative to the areas: the full check took 20 of the ~30 s of the crate's tests. + /// `cargo test --release -p horizon-client --lib the_grid_answers_as_the_scan` checks every cell. #[test] fn the_grid_answers_as_the_scan() { + let stride = if cfg!(debug_assertions) { 4 } else { 1 }; let mut seed = 0x9E37_79B9_7F4A_7C15u64; let forced: HashSet<(i32, i32)> = [(40, -40)].into(); for case in 0..60 { @@ -2165,7 +2170,8 @@ mod cover_tests { let (scan, grid) = (ranges(None), ranges(Some(&cover))); let span = [4, 30, 120][case % 3] + 40; for z in -span..=span { - for x in -span..=span { + let first = -span + (z + case as i32).rem_euclid(stride); + for x in (first..=span).step_by(stride as usize) { for m in 0..=ExtraCover::REACH + 2 { assert_eq!( scan.in_extra(x, z, m), diff --git a/crates/horizon-core/src/paletted.rs b/crates/horizon-core/src/paletted.rs index 802eac225..9aa4aec6a 100644 --- a/crates/horizon-core/src/paletted.rs +++ b/crates/horizon-core/src/paletted.rs @@ -106,14 +106,16 @@ enum Palette { List(Vec), Global, } -/// `n / per_word` for `n < 4096` by multiplication: `MAGIC[bits] = ceil(2^20 / -/// (64 / bits))` is exact while `n * (64 / bits) < 2^20`. -const MAGIC: [u32; 33] = { - let mut table = [0; 33]; +/// `n / per_word` for `n < 4096` by multiplication: `MAGIC[bits].0 = ceil(2^20 / +/// (64 / bits))` is exact while `n * (64 / bits) < 2^20`. `.1` is the bits a +/// word's entries fill (`64 / bits * bits`): looked up, not divided for, as a +/// division by a run-time value costs tens of cycles on x86. +const MAGIC: [(u32, u32); 33] = { + let mut table = [(0, 0); 33]; let mut bits = 1; while bits <= 32 { let per = 64 / bits as u32; - table[bits] = (1u32 << 20).div_ceil(per); + table[bits] = ((1u32 << 20).div_ceil(per), per * bits as u32); bits += 1; } table @@ -134,9 +136,9 @@ fn locate(bits: u8, i: usize) -> (usize, u32) { let word = i >> (6 - log); (word, ((i & ((64 >> log) - 1)) as u32) << log) } else { - let per = 64 / u32::from(bits); - let word = (i as u32 * MAGIC[usize::from(bits)]) >> 20; - (word as usize, (i as u32 - word * per) * u32::from(bits)) + let (magic, filled) = MAGIC[usize::from(bits)]; + let word = (i as u32 * magic) >> 20; + (word as usize, i as u32 * u32::from(bits) - word * filled) } } #[inline] @@ -720,6 +722,17 @@ mod tests { use super::*; const BLOCKS: Strategy = Strategy::blocks(16); const BIOMES: Strategy = Strategy::biomes(7); + + #[test] + fn locate_matches_division_for_every_width() { + for bits in 1..=32u8 { + let per = 64 / usize::from(bits); + for i in 0..4096 { + let expected = (i / per, ((i % per) * usize::from(bits)) as u32); + assert_eq!(locate(bits, i), expected, "{bits} bits, entry {i}"); + } + } + } fn rng(state: &mut u64) -> u64 { *state ^= *state << 13; *state ^= *state >> 7; diff --git a/crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml b/crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml new file mode 100644 index 000000000..a3a77ce0b --- /dev/null +++ b/crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "predator-missile" +version = "0.1.0" +edition.workspace = true +license.workspace = true +publish = false +description = "Example Horizon mod: a missile you launch with /missile and steer with the mouse" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +horizon-mod-sdk = { path = "../.." } + +[lints] +workspace = true diff --git a/crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml b/crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml new file mode 100644 index 000000000..3baaf7d87 --- /dev/null +++ b/crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml @@ -0,0 +1,8 @@ +id = "predator_missile" +name = "Predator Missile" +version = "0.1.0" +description = "/missile launches a guided missile 60 blocks above you. Steer it with the mouse; it explodes on whatever it hits. Needs cheats (it runs vanilla commands as you)." +authors = ["Horizon"] +api = "0.1" +side = "client" +capabilities = ["game-read", "hud", "chat", "command"] diff --git a/crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs b/crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs new file mode 100644 index 000000000..75ebbbdec --- /dev/null +++ b/crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs @@ -0,0 +1,161 @@ +//! Example Horizon mod (`docs/mods.md`): `/missile` turns you into a guided +//! missile 60 blocks up. You fly along your own look, so the mouse steers; +//! on a block or a mob a stack of primed TNT goes off, and you are put back +//! where you launched. Built from vanilla commands (capability `command`), +//! so it needs cheats. +use horizon_mod_sdk::{ + Mod, Registration, + command::run, + game::{self, Player}, + hud::{self, Corner, Line}, +}; + +/// Blocks per tick (a fired arrow does about 3). +const SPEED: f64 = 1.5; +const ALTITUDE: i32 = 60; +/// Ticks of flight before the missile detonates on its own. +const FUEL: u32 = 30 * 20; +/// Ticks to watch the blast before going back. +const AFTERMATH: u32 = 40; +/// Primed TNT per blast: the stack's explosions add up. +const WARHEAD: usize = 4; +/// What the impact checks print; the mod hears it through the chat hook. +const IMPACT: &str = "Predator: impact"; + +enum State { + Idle, + Flying { from: Player, ticks: u32 }, + Aftermath { from: Player, ticks: u32 }, +} + +struct Predator { + state: State, +} + +fn launch() { + run("gamerule send_command_feedback false"); + run("execute if entity @s[gamemode=creative] run tag @s add pm_creative"); + run("gamemode spectator"); + run(&format!("tp @s ~ ~{ALTITUDE} ~ ~ 60")); + run("effect give @s night_vision 60 0 true"); + // The thermal camera: every mob outlined. + run("effect give @e[type=!player,distance=..256] glowing 60 0 true"); +} + +/// One tick of flight: stop at the surface (the next step would enter a +/// block or a mob), else move one step along the look. +fn fly() { + run(&format!( + "execute at @s unless block ^ ^ ^{SPEED} #minecraft:air run tellraw @s \"{IMPACT}\"" + )); + run(&format!( + "execute at @s positioned ^ ^ ^{SPEED} if entity @e[type=!player,distance=..2] run tellraw @s \"{IMPACT}\"" + )); + run(&format!( + "execute at @s if block ^ ^ ^{SPEED} #minecraft:air run tp @s ^ ^ ^{SPEED}" + )); +} + +fn detonate() { + for _ in 0..WARHEAD { + run("execute at @s run summon tnt ^ ^ ^1 {fuse:0s}"); + } +} + +fn land(from: &Player) { + run(&format!( + "tp @s {} {} {} {} {}", + from.x, from.y, from.z, from.yaw, from.pitch + )); + run("execute if entity @s[tag=pm_creative] run gamemode creative"); + run("execute unless entity @s[tag=pm_creative] run gamemode survival"); + run("execute if entity @s[tag=pm_creative] run tag @s remove pm_creative"); + run("effect clear @s night_vision"); + // shortcut: assumes feedback was on before launch, as in a new world. + run("gamerule send_command_feedback true"); +} + +fn panel(p: &Player, ticks: u32) -> Vec { + let green = 0x7CFF7C; + let left = (FUEL - ticks.min(FUEL)) / 20; + [ + "PREDATOR".to_string(), + format!("ALT {:.0}", p.y), + format!("FUEL {left}s"), + "Mouse to steer, /missile to detonate".to_string(), + ] + .into_iter() + .map(|text| Line { text, color: green }) + .collect() +} + +impl Predator { + fn impact(&mut self) { + if let State::Flying { from, .. } = &self.state { + detonate(); + self.state = State::Aftermath { + from: from.clone(), + ticks: 0, + }; + } + } +} + +impl Mod for Predator { + fn init() -> Result<(Self, Registration), String> { + let hooks = Registration::new().command("missile").on_tick().on_chat(); + Ok((Predator { state: State::Idle }, hooks)) + } + + fn tick(&mut self, _tick: u64) { + match &mut self.state { + State::Idle => {} + State::Flying { ticks, .. } => { + *ticks += 1; + let ticks = *ticks; + if ticks >= FUEL { + return self.impact(); + } + // The first tick waits for the launch teleport. + if ticks > 1 { + fly(); + } + if let Some(p) = game::get_player() { + hud::set_panel(Corner::TopLeft, &panel(&p, ticks)); + } + } + State::Aftermath { from, ticks } => { + *ticks += 1; + if *ticks >= AFTERMATH { + land(from); + hud::clear(); + self.state = State::Idle; + } + } + } + } + + fn chat(&mut self, text: &str) { + if text.contains(IMPACT) { + self.impact(); + } + } + + fn command(&mut self, _name: &str, _args: &str) -> Result, String> { + match self.state { + State::Idle => { + let from = game::get_player().ok_or("Not in a world")?; + launch(); + self.state = State::Flying { from, ticks: 0 }; + Ok(Some("Missile away".into())) + } + State::Flying { .. } => { + self.impact(); + Ok(None) + } + State::Aftermath { .. } => Err("Wait for the blast to clear".into()), + } + } +} + +horizon_mod_sdk::export_mod!(Predator); diff --git a/crates/horizon-mod-sdk/src/lib.rs b/crates/horizon-mod-sdk/src/lib.rs index aea52339f..11da3e897 100644 --- a/crates/horizon-mod-sdk/src/lib.rs +++ b/crates/horizon-mod-sdk/src/lib.rs @@ -21,7 +21,7 @@ //! ``` //! //! Every interface but `log` is a capability: list the ones the mod uses in -//! `horizon-mod.toml` (`game-read`, `chat`, `hud`, `storage`, `ui`). +//! `horizon-mod.toml` (`game-read`, `chat`, `hud`, `storage`, `command`, `ui`). //! //! Widgets (capability `ui`): a mod shows a screen it owns with //! [`ui::open_screen`] and a page in Options > Mods with @@ -38,7 +38,7 @@ wit_bindgen::generate!({ pub use exports::horizon::mods::events::{Guest as __Guest, Registration}; pub use exports::horizon::mods::ui_events::Guest as __UiEventsGuest; -pub use horizon::mods::{chat, game, hud, log, storage, ui}; +pub use horizon::mods::{chat, command, game, hud, log, storage, ui}; /// The game's calls into a mod. Only the hooks [`Registration`] asks for /// are called; the others may stay as their defaults. diff --git a/crates/horizon-mod-sdk/wit/horizon.wit b/crates/horizon-mod-sdk/wit/horizon.wit index 1e49919c3..a46cb133a 100644 --- a/crates/horizon-mod-sdk/wit/horizon.wit +++ b/crates/horizon-mod-sdk/wit/horizon.wit @@ -86,6 +86,14 @@ interface storage { delete: func(key: string); } +/// Capability `command`: vanilla commands run as the player, with the +/// player's permissions, after the hook call returns (8 per call). A mod's +/// own commands are not run this way. +interface command { + /// `line` without its slash; its feedback goes to the chat as if typed. + run: func(line: string); +} + /// Capability `ui`: widgets the game draws with its own menus' widgets, for /// a screen the mod owns and a settings page in Options > Mods. Layout and /// drawing are the game's: a mod says what its widgets hold, never where @@ -194,6 +202,7 @@ world horizon-mod { import chat; import hud; import storage; + import command; export events; } diff --git a/crates/horizon-mods/src/manifest.rs b/crates/horizon-mods/src/manifest.rs index 6070f99bd..ccb90a9c6 100644 --- a/crates/horizon-mods/src/manifest.rs +++ b/crates/horizon-mods/src/manifest.rs @@ -78,6 +78,8 @@ pub enum Capability { Chat, Hud, Storage, + /// Runs vanilla commands as the player, with the player's permissions. + Command, /// New blocks and items from the mod's `data/` (API 0.2): no WIT /// interface, the game reads them before the world loads. Registries, @@ -86,11 +88,12 @@ pub enum Capability { Ui, } impl Capability { - pub const ALL: [Capability; 6] = [ + pub const ALL: [Capability; 7] = [ Capability::GameRead, Capability::Chat, Capability::Hud, Capability::Storage, + Capability::Command, Capability::Registries, Capability::Ui, ]; @@ -101,6 +104,7 @@ impl Capability { Capability::Chat => "chat", Capability::Hud => "hud", Capability::Storage => "storage", + Capability::Command => "command", Capability::Registries => "registries", Capability::Ui => "ui", } @@ -117,6 +121,7 @@ impl Capability { Capability::Chat => "show you chat messages", Capability::Hud => "draw text on your screen", Capability::Storage => "save its own settings to disk", + Capability::Command => "run commands as you, with your permissions", Capability::Registries => "add blocks and items to your worlds", Capability::Ui => "show its own screens and a settings page", } @@ -148,6 +153,7 @@ impl fmt::Display for Capability { Capability::Chat => "chat", Capability::Hud => "hud", Capability::Storage => "storage", + Capability::Command => "command", Capability::Registries => "registries", Capability::Ui => "ui", }; @@ -397,6 +403,10 @@ capabilities = ["game-read", "hud"] storage, Err("uses capability storage, missing from horizon-mod.toml".into()) ); + assert_eq!( + m.check_imports(["horizon:mods/command@0.1.0"]), + Err("uses capability command, missing from horizon-mod.toml".into()) + ); let with_storage = EXAMPLE.replace("\"hud\"", "\"hud\", \"storage\""); let m = Manifest::parse(&with_storage).unwrap(); assert_eq!(m.check_imports(["horizon:mods/storage@0.1.0"]), Ok(())); diff --git a/crates/horizon-mods/src/runtime.rs b/crates/horizon-mods/src/runtime.rs index 1b1795037..dfda140e1 100644 --- a/crates/horizon-mods/src/runtime.rs +++ b/crates/horizon-mods/src/runtime.rs @@ -32,7 +32,7 @@ pub use bindings::horizon::mods::{ use bindings::{ HorizonMod, exports::horizon::mods::events::Registration, - horizon::mods::{chat, game, hud, log as mod_log, storage}, + horizon::mods::{chat, command, game, hud, log as mod_log, storage}, }; /// The widget model a `ui` screen or settings page shows (WIT `interface /// ui`), for the game to draw with its menus' own widgets. @@ -79,6 +79,9 @@ pub enum Output { Chat { name: String, text: String }, /// A mod command's feedback, red when `error`. Feedback { text: String, error: bool }, + /// `command.run`: a vanilla command line, without its slash, to run as + /// the player. + Command { line: String }, /// The mod was turned off: it trapped, overran or failed to start. Stopped { name: String, reason: String }, /// `ui.open-screen`: the mod's entry in `entries()`; what it shows is @@ -668,6 +671,18 @@ impl chat::Host for State { } } } +impl command::Host for State { + fn run(&mut self, line: String) { + let queued = self + .out + .iter() + .filter(|o| matches!(o, Output::Command { .. })) + .count(); + if queued < CHAT_PER_CALL { + self.out.push(Output::Command { line: clip(line) }); + } + } +} impl hud::Host for State { fn set_panel(&mut self, corner: Corner, lines: Vec) { let lines = lines diff --git a/crates/horizon-mods/tests/sandbox.rs b/crates/horizon-mods/tests/sandbox.rs index 77f0975f3..eb4034a37 100644 --- a/crates/horizon-mods/tests/sandbox.rs +++ b/crates/horizon-mods/tests/sandbox.rs @@ -1,4 +1,5 @@ -//! Loads real mods built with the SDK: the coords-hud example and a guest +//! Loads real mods built with the SDK: the coords-hud and predator-missile +//! examples and a guest //! that misbehaves on command. Needs `rustup target add wasm32-unknown-unknown`. use horizon_mods::{ Clock, Corner, Mods, Output, Package, Player, Snapshot, Status, manifest::componentize, @@ -26,7 +27,7 @@ capabilities = ["chat", "ui"] experimental = ["ui"] "#; -/// Builds both mods once and writes their `.hmod` files; returns the folder. +/// Builds the mods once and writes their `.hmod` files; returns the folder. fn mods_dir() -> &'static Path { static DIR: OnceLock = OnceLock::new(); DIR.get_or_init(|| { @@ -39,6 +40,8 @@ fn mods_dir() -> &'static Path { "-p", "coords-hud", "-p", + "predator-missile", + "-p", "horizon-mods-test-guest", "-p", "horizon-mods-test-ui-guest", @@ -54,9 +57,13 @@ fn mods_dir() -> &'static Path { let built = target.join("wasm32-unknown-unknown/release"); let dir = target.join("mods"); std::fs::create_dir_all(&dir).unwrap(); - let example = root.join("crates/horizon-mod-sdk/examples/coords-hud/horizon-mod.toml"); + let example = |name: &str| { + let path = format!("crates/horizon-mod-sdk/examples/{name}/horizon-mod.toml"); + std::fs::read_to_string(root.join(path)).unwrap() + }; let mods = [ - ("coords_hud", std::fs::read_to_string(example).unwrap()), + ("coords_hud", example("coords-hud")), + ("predator_missile", example("predator-missile")), ("horizon_mods_test_guest", GUEST_MANIFEST.to_string()), ("horizon_mods_test_ui_guest", UI_MANIFEST.to_string()), ]; @@ -92,6 +99,36 @@ fn player() -> Snapshot { } } +/// Capability `command`: `/missile` queues vanilla commands, and the chat +/// line its impact check prints sets off the warhead. +#[test] +fn predator_missile_runs_commands() { + let mut mods = load("predator_missile.hmod"); + assert_eq!(mods.entries()[0].status, Status::Running); + mods.set_snapshot(player()); + let commands = |mods: &mut Mods| -> Vec { + mods.drain() + .into_iter() + .filter_map(|o| match o { + Output::Command { line } => Some(line), + _ => None, + }) + .collect() + }; + assert!(mods.command("missile")); + let launch = commands(&mut mods); + assert_eq!(launch.len(), 6); + assert!(launch.contains(&"gamemode spectator".to_string())); + mods.tick(1); + mods.tick(2); + let flight = commands(&mut mods); + assert!(flight.iter().any(|l| l.ends_with("run tp @s ^ ^ ^1.5"))); + mods.chat("Predator: impact"); + let blast = commands(&mut mods); + assert_eq!(blast.len(), 4); + assert!(blast.iter().all(|l| l.contains("summon tnt"))); +} + #[test] fn coords_hud_draws_and_answers() { let mut mods = load("coords_hud.hmod"); diff --git a/crates/horizon-prepare/output-versions.txt b/crates/horizon-prepare/output-versions.txt new file mode 100644 index 000000000..71a531fab --- /dev/null +++ b/crates/horizon-prepare/output-versions.txt @@ -0,0 +1,3 @@ +# OUTPUT_VERSION (src/lib.rs) and the digest of what it prepares (python3 tools/prepare_output.py digest), +# one line per version, never edited: a change to the prepared files appends the next version. +1 0a608bf4115a8ed2e2c4f5f2b2609dc10c54e63479af8e56fabaa10af1060389 diff --git a/crates/horizon-prepare/src/lib.rs b/crates/horizon-prepare/src/lib.rs index c7e443c7d..5ea1121df 100644 --- a/crates/horizon-prepare/src/lib.rs +++ b/crates/horizon-prepare/src/lib.rs @@ -31,6 +31,15 @@ pub type Log<'a> = &'a dyn Fn(&str); /// The manifest's `schema`: the prepared files' layout, which the launcher compares. pub const SCHEMA: i64 = 7; +/// What this preparer makes. The launcher copies a prepared version's resources to a newly installed +/// version whose preparer has the same number (and the same pinned jars), so a release that changes +/// this crate without changing its output asks nobody to prepare again. Bump it, and append the new +/// digest to output-versions.txt, when the prepared files change: CI prepares with every change to +/// the preparer and fails when the output is not the one output-versions.txt gives this number +/// (`python3 tools/prepare_output.py check`). The launcher reads this line from the version +/// folder's sources: keep it on one line, as it is. +pub const OUTPUT_VERSION: u32 = 1; + /// Digest of this crate's sources (build.rs): the launcher prepares a version in-process when /// that version ships these same sources, and runs the version's own binary otherwise. pub const SOURCES_SHA256: &str = env!("HORIZON_PREPARE_SOURCES_SHA256"); @@ -1209,6 +1218,29 @@ mod tests { fs::remove_dir_all(dir).unwrap(); } + #[test] + fn output_versions_end_at_the_output_version() { + let text = include_str!("../output-versions.txt"); + let versions: Vec<(u32, &str)> = text + .lines() + .filter(|l| !l.trim().is_empty() && !l.starts_with('#')) + .map(|l| { + let (n, digest) = l.split_once(' ').unwrap(); + (n.parse().unwrap(), digest.trim()) + }) + .collect(); + let numbers: Vec = versions.iter().map(|v| v.0).collect(); + assert_eq!(numbers, (1..=OUTPUT_VERSION).collect::>()); + for (i, (_, digest)) in versions.iter().enumerate() { + assert_eq!(digest.len(), 64); + assert!(!versions[..i].iter().any(|(_, d)| d == digest)); + } + // The launcher finds the number with this line in a version folder's sources. + let lib = include_str!("lib.rs"); + let line = format!("pub const OUTPUT_VERSION: u32 = {OUTPUT_VERSION};"); + assert!(lib.lines().any(|l| l == line)); + } + #[test] fn the_source_digest_is_the_one_build_rs_computed() { assert_eq!( diff --git a/docs/api/mods-reference.md b/docs/api/mods-reference.md index 0812cde54..8acf450c0 100644 --- a/docs/api/mods-reference.md +++ b/docs/api/mods-reference.md @@ -11,19 +11,20 @@ The contract between Horizon and its mods: the WIT package `horizon:mods@0.1.0`, | `chat` | [`chat`](#chat) | show you chat messages | | `hud` | [`hud`](#hud) | draw text on your screen | | `storage` | [`storage`](#storage) | save its own settings to disk | +| `command` | [`command`](#command) | run commands as you, with your permissions | | `registries` | data files (`assets/`, `data/`), no interface | add blocks and items to your worlds | | `ui` | [`ui`](#ui) | show its own screens and a settings page | ## World `horizon-mod` -- Imports (what the game offers): `log`, `game`, `chat`, `hud`, `storage` +- Imports (what the game offers): `log`, `game`, `chat`, `hud`, `storage`, `command` - Exports (what the mod provides): `events` ## World `horizon-ui-mod` A mod with widgets (capability `ui`): the SDK builds every mod against this world. Mods built for `horizon-mod` alone (no `ui-events` export) still load and simply hear nothing of their widgets. -- Imports (what the game offers): `ui`, `log`, `game`, `chat`, `hud`, `storage` +- Imports (what the game offers): `ui`, `log`, `game`, `chat`, `hud`, `storage`, `command` - Exports (what the mod provides): `ui-events`, `events` ## World `ui-host` @@ -132,6 +133,16 @@ Capability `storage`: a per-mod folder of small key/value files, quota'd (docs/m | `set` | `func(key: string, value: list) -> result<_, string>` | Errs on a malformed key or over quota (bytes for the whole mod). | | `delete` | `func(key: string)` | No error if `key` was already absent. | +## `command` + +Access: capability `command`. + +Capability `command`: vanilla commands run as the player, with the player's permissions, after the hook call returns (8 per call). A mod's own commands are not run this way. + +| Function | Signature | What it does | +| --- | --- | --- | +| `run` | `func(line: string)` | `line` without its slash; its feedback goes to the chat as if typed. | + ## `ui` Access: capability `ui`. diff --git a/docs/api/stability.md b/docs/api/stability.md index ffa9d9b11..9aa9684f6 100644 --- a/docs/api/stability.md +++ b/docs/api/stability.md @@ -86,7 +86,7 @@ Deprecations and breaking changes are in bold. The release notes of the game rel | Part | Level | Version | | --- | --- | --- | -| Mod API: `log`, `game`, `chat`, `hud`, `storage`, `events` | preview | 0.1.0 | +| Mod API: `log`, `game`, `chat`, `hud`, `storage`, `command`, `events` | preview | 0.1.0 | | Mod API: `ui`, `ui-events` (a mod's screens and settings page) | experimental | manifest `experimental = ["ui"]` | | Content mods (`registries`: data-driven blocks and items) | preview | manifest `api = "0.2"` | | Server mods (events, world, server commands) | not started | manifest `api = "0.3"`, experimental first | diff --git a/docs/mods.md b/docs/mods.md index d47c3f372..e22228e0b 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -12,9 +12,9 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | Part | Where | State | |---|---|---| -| The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage, ui; tick, frame, chat and command hooks | +| The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage, command, ui; tick, frame, chat and command hooks | | SDK for mod authors | `crates/horizon-mod-sdk` | 0.1 | -| Example mod | `crates/horizon-mod-sdk/examples/coords-hud` | coordinates, facing and time HUD, `/where`, `/coords` | +| Example mods | `crates/horizon-mod-sdk/examples/coords-hud`, `examples/predator-missile` | coordinates, facing and time HUD, `/where`, `/coords`; `/missile`, a guided missile from vanilla commands | | Runtime | `crates/horizon-mods` | wasmtime 36 LTS, component model | | Authoring tool | `crates/cargo-horizon-mod` | `cargo horizon-mod new [--template code\|content] / build / run / check`, and the registry's CI (`registry build / review / list`) | | In the game | `crates/horizon-client/src/game/mods.rs`, Options > Mods | client mods | @@ -86,6 +86,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | `chat` | `chat` | `show(text)`: a line only the local player sees, after the mod's name (8 per call) | | `hud` | `hud` | `set-panel(corner, lines)`, `clear()`: up to 16 lines of 256 characters | | `storage` | `storage` | `get(key)`, `set(key, value)`, `delete(key)`: a per-mod folder of up to 64 files, 4 MiB total, at most 1 MiB written per call | +| `command` | `command` | `run(line)`: a vanilla command run as the player, with the player's permissions, after the hook returns (8 per call); mods' own commands are not run | | `ui` | `ui` | `open-screen`, `close-screen`, `set-settings`: a screen the mod owns and a page in Options > Mods, of text, item and list widgets the game lays out and draws with its own menus' code. **Experimental:** the manifest needs `experimental = ["ui"]` | | Hook (export) | When | diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index 7eecdd8bb..c1aa979fc 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -20,6 +20,8 @@ python3 tools/parity.py --changed origin/main # those a branch's changes ca python3 tools/parity.py --changed-files paths.txt # the same for listed paths, in a copy without Git (tools/pr_ready.py) ``` +On CI, a pull request runs the parity job only when it changes a harness, an oracle, a probe or the runner; everything else is checked on main by `parity-main`, a batch of merges at a time, which bisects a failure to the merge that caused it (`tools/parity_main.py`; [release-flow.md](release-flow.md#parity-on-main)). Game releases wait for it. + Each harness runs unmodified under a tracer that records the files it depended on: its Python modules, its Java oracle and the oracle sources javac pulled in, other repository files it read, and the Rust sources of the probes it ran (from Cargo's `.d` dep-info files). A pass stands until one of those files, the pinned vanilla artifacts, `Cargo.lock` or the Rust/Java toolchains change. State and logs are in `.local/parity-runner/`; each harness's own outputs stay in its `.local//` directory. Vanilla oracle runs are reused as well. `tools/parity_shim.py` stands in for `java` and `javac`: it records each oracle run's outputs and the content of every file under the paths its arguments name, and replays the run while those are unchanged. After a Rust-only change a harness pays only for its Rust side (`block_update`: 53 s to 5 s). Files an oracle writes outside its named paths aren't tracked, so a harness that fails while reusing oracle results is rerun fresh, and stops reusing them if it then passes. `--all` and `--fresh` never reuse them. @@ -49,6 +51,10 @@ A harness can declare, at the top level of its script: - `PARITY_SHARDS = N`: it runs as `NAME#1` to `NAME#N`, in parallel, each told its share in `PARITY_SHARD=i/N` (`test_datafix_parity.py` upgrades every sixth world). `NAME` on the command line or in `--known-failures` means all of them. - `PARITY_RUST_SCOPE = [...]`: the Rust sources its probes can reach, as path prefixes. Cargo's dep-info names every file of every crate a probe links, so without it a probe of `horizon-world` depends on all 350 of its files. The runner keeps only the scope's files if the scope is closed: nothing in it names a module, crate-root re-export or macro outside it (`crate::`, `super::`, other workspace crates; comments and strings ignored), declares a module it doesn't cover, or includes source, and nothing outside it implements anything for a type it defines. Crate roots then count without their declarations of other modules, so adding a module elsewhere doesn't rerun the harness. If the scope isn't closed the runner says why and keeps every file. `datafix` depends on 41 files this way, not 354. +### Which harnesses a change affects + +`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `.local/parity-runner/reach-cache.json`. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses. + ## The gate `tools/gate.py` is what a branch must pass before it is integrated: the public-tree check, rustfmt, clippy with warnings denied, the Rust and Python tests, then the harnesses whose inputs changed. It stops at the first failure and writes `.local/gate.json`. `--known-failures a,b` lets harnesses that already fail on the integration branch keep failing. `--quick` checks formatting, lints and Python tests only; `--no-parity` skips the harnesses. diff --git a/docs/release-flow.md b/docs/release-flow.md index d9d1801aa..28d1feef9 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -13,7 +13,7 @@ reopened; labels never start it): | Job | Runs on | What | Time | | --- | --- | --- | --- | | `verify` | GitHub `ubuntu-latest` | Public-tree check, Python tests, rustfmt, clippy with warnings denied, Rust tests. Dependencies and the workspace build come from a cache saved by main's runs | ~4 min cold, less warm | -| `parity / parity` | A runner with the label `horizon-game`: the Mac mini's two, and the desktop's when no game runs there (below) | The vanilla harnesses the change can affect (`tools/parity.py --changed `) | seconds when nothing is affected; minutes per affected harness | +| `parity / parity` | A runner with the label `horizon-game`: the Mac mini's two, and the desktop's while nobody uses it (below) | Only for a change to a harness, an oracle, a probe or the runner (or the `run-parity` label): the vanilla harnesses the change can affect (`tools/parity.py --changed `). Other pull requests skip it: [parity-main](#parity-on-main) checks them on main | seconds when nothing is affected; minutes per affected harness | Pushes to main run `verify` only; the nightly audit covers main's harnesses. @@ -42,10 +42,17 @@ stand. depend only on the Minecraft version, are kept once in `shared-local/sounds//` (copied from a folder `horizon-prepare sounds` made from the maintainer's own game; the Mac mini has no official launcher of its own) and cloned into each preparation (not linked: vanilla's packs refuse symbolic links), for the audio harnesses. + `tools/prepare_output.py check` then compares what was prepared with `crates/horizon-prepare/output-versions.txt`: + launchers keep a player's prepared resources across releases whose preparer declares the same `OUTPUT_VERSION` + (`crates/horizon-prepare/src/lib.rs`), so a change to the prepared files fails here until it bumps the number + and appends the new digest (the error gives both lines). 3. `tools/parity.py --changed ` picks the harnesses whose traced dependencies (their script, Python modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to `Cargo.lock` or `research-lock.json` affects every harness. Each harness has 25 minutes, the job 45. + The selection runs with main's `tools/parity.py` and `tools/parity_reach.py` (`--changed --list`), so a + branch made before a better selection landed gets it too, and the head's own `parity.py` then runs the named + harnesses. A branch that changes either file selects with its own code. Failed harnesses' logs are uploaded as the `parity-logs` artifact. @@ -102,10 +109,34 @@ gh api -X POST repos/ludovic111/horizon/branches/main/protection/required_status A skipped `parity` (a fork's pull request without the label) counts as passed, so requiring it never blocks a fork's pull request; the maintainer decides whether to run it. +## Parity on main + +Most pull requests merge on `verify` alone and leave the vanilla harnesses to `.github/workflows/parity-main.yml`, +which runs them on main a batch of merges at a time, on the Mac mini's runner: + +- It starts on every push to main, one run at a time (`concurrency: parity-main`). Merges that land while a run + goes on wait in one pending run: GitHub keeps only the newest and cancels the others, so the next run takes them + all. +- `tools/parity_main.py base` gives the newest commit main contains whose parity-main passed (else the last + merge's parent), and the run is `tools/parity.py --changed `: the union of the harnesses every merge + since then can affect, once, with every earlier pass reused. +- On a failure, `tools/parity_main.py bisect` checks out the batch's first-parent merges by halves and runs only + the failing harnesses, names the first merge they fail on in the run's summary and comments on its pull request + (`🤖 Agent: parity-main found that this merge breaks ...`). Fix it with a pull request as usual; the next + run checks from the last green commit again. +- Game releases ship only a commit both `checks` and `parity-main` passed on (`tools/release_train.py`, see + below), so a broken merge holds releases instead of reaching players. `release.yml` also starts when a + parity-main run finishes. A forced release (`force=true`) doesn't wait for parity-main. + +A pull request still runs its own parity job when it changes a harness script, an oracle +(`mods/horizon/src/test/`), a probe (`crates/*/examples/`, `game/*_fixture.rs`, `game/*_harness.rs`), the runner +(`tools/parity*.py`, `tools/lab.py`, `tools/parity_hosts.json`, `parity*.yml`), or carries the `run-parity` label +(`run-parity.yml` runs it when the label is added). + ## Game releases -`.github/workflows/release.yml` releases the game by itself. It starts after every green `checks` run of a push -to main, every 10 minutes (so a release waiting for merges to settle goes out without another merge), and by hand. +`.github/workflows/release.yml` releases the game by itself. It starts after every green `checks` or `parity-main` +run of a push to main (it releases a commit only once both passed on it), every 10 minutes (so a release waiting for merges to settle goes out without another merge), and by hand. ```mermaid flowchart LR diff --git a/tests/test_parity_main.py b/tests/test_parity_main.py new file mode 100644 index 000000000..7a72ecae5 --- /dev/null +++ b/tests/test_parity_main.py @@ -0,0 +1,43 @@ +"""tools/parity_main.py: the base of a parity-main run, and the bisect after a failure.""" +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from tools import parity_main + + +class ParityMainTests(unittest.TestCase): + def test_first_bad_finds_the_first_failing_merge(self): + commits = list("abcdefgh") # h failed; the base before a passed + for culprit in commits: + tried = [] + fails = lambda commit: tried.append(commit) or commit >= culprit + self.assertEqual(parity_main.first_bad(commits, fails), culprit) + self.assertLessEqual(len(tried), 3, "log2 of 8 merges") + self.assertEqual(parity_main.first_bad(["a"], lambda commit: self.fail("nothing to try")), "a") + + def test_base_is_the_newest_green_run_head_contains(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.object(parity_main, "ROOT", Path(tmp)): + git = lambda *arguments: parity_main.git(*arguments).stdout.strip() + git("init", "-q") + for n in range(3): + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", str(n)) + head, parent, grandparent = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^1"), git("rev-parse", "HEAD~2") + git("checkout", "-q", "-b", "side", grandparent) + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "side") + side = git("rev-parse", "HEAD") + git("checkout", "-q", head) + self.assertEqual(parity_main.base(runs=[side, grandparent, parent]), grandparent, + "a run on a commit HEAD doesn't contain is skipped") + self.assertEqual(parity_main.base(runs=[]), parent, "no green run yet: the last merge") + self.assertEqual(parity_main.base(runs=[head]), head) + + def test_a_harness_a_commit_does_not_have_cannot_fail_there(self): + with mock.patch.object(parity_main.subprocess, "run") as run: + self.assertFalse(parity_main.run_harnesses(["not_a_harness_yet"], 60)) + run.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parity_reach.py b/tests/test_parity_reach.py new file mode 100644 index 000000000..9b7464b47 --- /dev/null +++ b/tests/test_parity_reach.py @@ -0,0 +1,111 @@ +"""tools/parity_reach.py: which workspace sources a harness's probe can reach.""" +from pathlib import Path +import tempfile +import textwrap +import unittest + +from tools import parity, parity_reach + +PROBE = [["example", "world_probe", "release", ["-p", "a"], True]] + + +def workspace(root, files): + for path, text in files.items(): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_text(textwrap.dedent(text), encoding="utf-8") + + +class ReachTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.root = Path(self.directory.name) + workspace(self.root, { + "crates/a/Cargo.toml": '[package]\nname = "a"\n\n[[example]]\nname = "world_probe"\npath = "examples/probe/main.rs"\n', + "crates/a/src/lib.rs": """ + pub mod used; + pub mod unused; + mod methods; + mod display; + mod table; + #[cfg(test)] + mod tests; + """, + "crates/a/src/used.rs": """ + pub struct Thing; + pub fn helper() -> u32 { 1 } + """, + # Same names as used.rs, but nothing names this module. + "crates/a/src/unused.rs": "pub struct Thing;\npub fn helper() -> u32 { 2 }\n", + "crates/a/src/methods.rs": """ + use crate::used::Thing; + impl Thing { + pub fn method(&self) -> u32 { 3 } + pub fn never_called(&self) -> u32 { 4 } + } + """, + "crates/a/src/display.rs": """ + impl std::fmt::Display for crate::used::Thing { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "thing") } + } + """, + "crates/a/src/table.rs": """ + macro_rules! table { () => { pub struct Table; }; } + table!(); + """, + "crates/a/src/tests.rs": "use crate::used::Thing;\nimpl Thing { pub fn method_in_tests(&self) {} }\n", + "crates/a/examples/probe/main.rs": "mod dispatch;\nmod p;\n", + "crates/a/examples/probe/dispatch.rs": "pub fn args() {}\n", + "crates/a/examples/probe/p.rs": """ + use a::used::{helper, Thing}; + pub fn main() { println!("{} {} {}", helper(), Thing.method(), Thing); } + """, + }) + self.dependencies = sorted(str(path.relative_to(self.root)) for path in self.root.rglob("*") if path.is_file()) + self.dependencies.remove("crates/a/examples/probe/main.rs") # parity.only_probes_run leaves it out + + def tearDown(self): + self.directory.cleanup() + + def narrowed(self, changed=(), artifacts=PROBE): + graph = parity_reach.SourceGraph(self.root, parity.rust_code) + return parity_reach.narrow(self.dependencies, artifacts, graph, set(changed)) + + def test_keeps_what_the_probe_reaches(self): + kept = {path for path, (whole, _) in self.narrowed().items() if whole} + self.assertIn("crates/a/src/used.rs", kept) + self.assertIn("crates/a/src/methods.rs", kept) # Thing.method() + self.assertIn("crates/a/src/display.rs", kept) # std's Display for a reached type + self.assertIn("crates/a/Cargo.toml", kept) + self.assertNotIn("crates/a/src/unused.rs", kept) # its names match, but nothing names its module + self.assertNotIn("crates/a/src/table.rs", kept) + self.assertNotIn("crates/a/src/tests.rs", kept) # compiled into tests only + + def test_module_parents_count_through_their_declarations(self): + self.assertEqual(self.narrowed()["crates/a/src/lib.rs"], (False, {"used", "methods", "display"})) + text = (self.root / "crates/a/src/lib.rs").read_text() + self.assertEqual(parity_reach.declarations(text, {"used"}, False), "pub mod used;") + self.assertEqual(parity_reach.declarations(text, {"tests"}, False), "#[cfg(test)]\nmod tests;") + # Adding or dropping a module beside the reached ones changes neither view. + other = text.replace("mod table;", "mod table2;\nmod more;") + self.assertEqual(parity_reach.declarations(other, {"used"}, False), "pub mod used;") + self.assertEqual(parity_reach.declarations(other, {"used"}, True), parity_reach.declarations(text, {"used"}, True)) + self.assertNotEqual(parity_reach.declarations(text + "pub fn f() {}\n", {"used"}, True), + parity_reach.declarations(text, {"used"}, True)) + + def test_a_new_file_counts_once_it_is_reached(self): + (self.root / "crates/a/src/lib.rs").write_text("pub mod used;\nmod extra;\n") + workspace(self.root, {"crates/a/src/extra.rs": "impl crate::used::Thing { pub fn method(&self) -> u32 { 5 } }\n"}) + self.assertIn("crates/a/src/extra.rs", self.narrowed(changed={"crates/a/src/extra.rs"})) + + def test_a_deleted_dependency_counts(self): + (self.root / "crates/a/src/unused.rs").unlink() + self.assertIn("crates/a/src/unused.rs", self.narrowed()) + + def test_unknown_probes_keep_everything(self): + self.assertEqual(set(self.narrowed(artifacts=[])), set(self.dependencies)) + self.assertEqual(set(self.narrowed(artifacts=[["example", "missing", "release", ["-p", "a"], True]])), + set(self.dependencies)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py index d10fde854..2d16e7179 100644 --- a/tests/test_parity_runner.py +++ b/tests/test_parity_runner.py @@ -300,3 +300,17 @@ class LargeOutputTests(unittest.TestCase): self.assertFalse((own / "dump.txt").exists()) # chunk_generation may still be writing there. self.assertTrue((shared / "dump.txt").exists()) + + def test_rust_named_directories_and_big_oracle_outputs_go_too(self): + with tempfile.TemporaryDirectory() as tmp: + local, state = Path(tmp) / ".local", Path(tmp) / "state" + own = local / "rust-state-provider-parity" + own.mkdir(parents=True) + for name, size in (("rust.txt", 16), ("server.txt", 16), ("client.txt", 64), ("report.json", 4)): + (own / name).write_bytes(b"x" * size) + trace = {"commands": [["java", "Oracle", str(own)], ["probe", "fixture", str(own)]]} + with mock.patch.multiple(parity, LOCAL=local, STATE_DIR=state, LARGE_OUTPUT=8, KEPT_ORACLE_OUTPUT=32): + listing = parity.outputs_listing("state_provider#2", fresh=True) + listing.write_text(f"{own / 'server.txt'}\n{own / 'client.txt'}\n") + parity.clear_large_outputs("state_provider#2", trace) + self.assertEqual(sorted(path.name for path in own.iterdir()), ["report.json", "server.txt"]) diff --git a/tests/test_prepare_output.py b/tests/test_prepare_output.py new file mode 100644 index 000000000..00c489fbb --- /dev/null +++ b/tests/test_prepare_output.py @@ -0,0 +1,64 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from tools import prepare_output + + +class PrepareOutputTests(unittest.TestCase): + def preparation(self, root: Path, files: dict[str, bytes]) -> Path: + for rel, data in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return root + + def test_what_varies_between_runs_does_not_count(self): + manifest = {'schema': 7, 'data_version': 5023, 'render_files_sha256': {'a.json': 'x'}} + a = { + 'horizon-resources.json': json.dumps(manifest).encode(), + 'render-definitions.json': b'{"a": 1, "b": [1, 2]}', + 'creative-inventory.json': b'{"tags": [["b", "a"]]}', + 'block-behaviours.json': b'{"c": "Foo$$Lambda/0x0000012a"}', + 'export.log': b'run 1', + 'assets/minecraft/sounds/a.ogg': b'one', + } + manifest['render_files_sha256'] = {'a.json': 'y'} + b = { + 'horizon-resources.json': json.dumps(manifest).encode(), + 'render-definitions.json': b'{"b": [1, 2], "a": 1}', + 'creative-inventory.json': b'{"tags": [["a", "b"]]}', + 'block-behaviours.json': b'{"c": "Foo$$Lambda/0x00000ff0"}', + 'export.log': b'run 2', + 'assets/minecraft/sounds/a.ogg': b'two', + } + with tempfile.TemporaryDirectory() as tmp: + first = prepare_output.digest(self.preparation(Path(tmp) / 'a', a)) + second = prepare_output.digest(self.preparation(Path(tmp) / 'b', b)) + self.assertEqual(first, second) + # The data does, and so do the manifest's schema and the files there are. + (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [2, 1], "a": 1}') + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [1, 2], "a": 1}') + (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps({**manifest, 'schema': 8})) + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps(manifest)) + (Path(tmp) / 'b/new.nbt').write_bytes(b'') + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + + def test_the_versions_file_ends_at_the_constant(self): + versions = prepare_output.parse_versions('# comment\n1 aa\n2 bb\n') + self.assertEqual(versions, [(1, 'aa'), (2, 'bb')]) + self.assertEqual(prepare_output.problems(versions, 2), []) + self.assertTrue(prepare_output.problems(versions, 1)) + self.assertTrue(prepare_output.problems([(1, 'aa'), (3, 'bb')], 3)) + self.assertTrue(prepare_output.problems([(1, 'aa'), (2, 'aa')], 2)) + root = Path(prepare_output.__file__).resolve().parent.parent + current = prepare_output.parse_versions((root / prepare_output.VERSIONS).read_text()) + constant = prepare_output.CONSTANT.search((root / prepare_output.LIB).read_text()) + self.assertEqual(prepare_output.problems(current, int(constant.group(1))), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_release_train.py b/tests/test_release_train.py index 56064fa42..ee1fbcae5 100644 --- a/tests/test_release_train.py +++ b/tests/test_release_train.py @@ -2,6 +2,7 @@ import datetime as dt from pathlib import Path import tempfile import unittest +from unittest import mock from tools import release_train as train @@ -147,6 +148,26 @@ class ReleaseTrainTests(unittest.TestCase): self.assertFalse(train.decision("game", [], NOW, new_line="0.3", blockers=[7])[0]) self.assertTrue(train.decision("game", [pr(1)], NOW, blockers=[])[0]) + def test_game_releases_ship_only_what_parity_main_passed(self): + def runs(parity, checks): + def gh_json(*arguments): + table = parity if "parity-main" in arguments else checks + if "--commit" in arguments: + sha = arguments[arguments.index("--commit") + 1] + return [{"status": s, "conclusion": c} for h, s, c in table if h == sha] + return [{"headSha": h, "status": s, "conclusion": c} for h, s, c in table] + return mock.patch.object(train, "gh_json", gh_json) + checks = [("c", "in_progress", ""), ("b", "completed", "success"), ("a", "completed", "success")] + with runs([("c", "queued", ""), ("b", "completed", "cancelled"), ("a", "completed", "success")], checks): + self.assertEqual(train.green_main_head(), "b", "checks alone: the newest green checks") + self.assertEqual(train.green_main_head(parity=True), "a", "a superseded parity-main run counts for nothing") + self.assertTrue(train.passed_on("parity-main", "a")) + self.assertFalse(train.passed_on("parity-main", "b")) + with runs([("b", "completed", "failure"), ("a", "completed", "success")], checks): + self.assertIsNone(train.green_main_head(parity=True), "a failed batch holds releases until fixed") + with runs([("c", "completed", "success")], checks): + self.assertIsNone(train.green_main_head(parity=True), "its checks must pass too") + def test_a_release_is_published_only_after_its_smoke_test_and_with_no_blocker(self): drew = {"error": None, "frames": 12710, "chunks": 81, "meshed_sections": 550, "player": [160.5, 71.0, 156.5]} self.assertIsNone(train.gate(drew, [])) diff --git a/tools/parity.py b/tools/parity.py index 503a5f3bf..3f18b922d 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -45,6 +45,11 @@ import subprocess import sys import time +try: + from . import parity_reach +except ImportError: + import parity_reach + ROOT = Path(__file__).resolve().parents[1] LOCAL = ROOT / ".local" STATE_DIR = LOCAL / "parity-runner" @@ -652,6 +657,9 @@ def prebuild(artifacts, jobs): LARGE_OUTPUT = 64 << 20 +# Vanilla outputs above this are deleted after a pass too: rerunning those few oracles (12 harnesses on the Mac +# mini's runner) costs less than the 26 GB they held in every checkout; smaller ones stay for oracle reuse. +KEPT_ORACLE_OUTPUT = 256 << 20 OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"} @@ -667,11 +675,12 @@ def clear_large_outputs(name, trace): """After a pass, delete the large files a harness regenerates on every run. Harness outputs reach gigabytes and used to stay in every worktree. Vanilla - oracle outputs are kept, since reusing an oracle run needs them in place; - everything else large under the harness's .local directories is rebuilt on - its next run.""" + oracle outputs under KEPT_ORACLE_OUTPUT are kept, since reusing an oracle + run needs them in place; everything else large under the harness's .local + directories (`...` or `rust--parity`) is rebuilt on its next run.""" local = LOCAL.resolve() keep = {os.path.realpath(line) for line in outputs_listing(name).read_text(encoding="utf-8").splitlines() if line} + own = name.split("#")[0].replace("_", "-") directories = set() for command in trace["commands"]: for argument in command[1:]: @@ -683,12 +692,13 @@ def clear_large_outputs(name, trace): # Only the harness's own directory: another harness running beside it may be writing # large files in a directory this one merely read (decorated_region compiles its oracle # into chunk-generation-parity while chunk_generation writes its dumps there). - if top not in OWN_DIRECTORIES and top.startswith(name.replace("_", "-")): + if top not in OWN_DIRECTORIES and (top.startswith(own) or top == f"rust-{own}-parity"): directories.add(local / top) for directory in directories: for file in (directory.rglob("*") if directory.is_dir() else []): try: - if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep: + size = file.stat().st_size if file.is_file() and not file.is_symlink() else 0 + if size >= LARGE_OUTPUT and (str(file) not in keep or size > KEPT_ORACLE_OUTPUT): file.unlink() except OSError: pass @@ -823,8 +833,11 @@ class LocalExecutor: environment.pop("PARITY_SHARD", None) if shard: environment["PARITY_SHARD"] = shard + # stdin from /dev/null: tools/parity_shim.py never reuses an oracle run that reads a pipe, and CI runners + # (and --worker's ssh) give the runner one, which kept oracle reuse off on CI. process = subprocess.Popen([sys.executable, str(Path(__file__)), "--trace", str(trace), str(script)], - cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + cwd=ROOT, env=environment, stdin=subprocess.DEVNULL, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) self.running[name] = {"process": process, "started": time.monotonic(), "log": log, "expected": expected, "slot": slot, "script": script} return True @@ -1224,20 +1237,44 @@ def changed_harnesses(found, records, ref): return found_changed -def affected_harnesses(found, records, changed): +def affected_harnesses(found, records, changed, reach=True, base=None): """Harnesses the changed files can affect: their script or a recorded dependency (here, or in the shared - dependency lists) changed, a file every harness depends on changed, or nothing is recorded about them yet.""" + dependency lists) changed, a file every harness depends on changed, or nothing is recorded about them yet. + With reach, a harness's workspace Rust sources count only if its probes can reach them (tools/parity_reach.py); + a harness whose probes can't be placed keeps them all. A changed file counts unless, compared with base (a + commit), only declarations of modules the harness doesn't reach changed (tools/parity_reach.py).""" if changed & SHARED_INPUTS: return set(found) + graph = None + if reach and any(path.startswith("crates/") for path in changed): + cache_path = STATE_DIR / "reach-cache.json" + try: + cache = json.loads(cache_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + cache = {} + graph = parity_reach.SourceGraph(ROOT, rust_code, cache) affected = set() for name, script in found.items(): - dependencies = records.get(name, {}).get("dependencies") + record = records.get(name) or {} + dependencies = record.get("dependencies") template = SHARED / "dependencies" / f"{name}.json" if dependencies is None and template.exists(): - dependencies = json.loads(template.read_text(encoding="utf-8")).get("dependencies") - if dependencies is None or Path(script).relative_to(ROOT).as_posix() in changed or \ - any(path.split(KEEP)[0] in changed for path in dependencies): + record = json.loads(template.read_text(encoding="utf-8")) + dependencies = record.get("dependencies") + if dependencies is None or Path(script).relative_to(ROOT).as_posix() in changed: affected.add(name) + continue + plain = {path.split(KEEP)[0] for path in dependencies} + if graph is not None and plain & changed: + narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed) + if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed( + ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()): + affected.add(name) + elif plain & changed: + affected.add(name) + if graph is not None: + STATE_DIR.mkdir(parents=True, exist_ok=True) + cache_path.write_text(json.dumps(graph.saved_cache()), encoding="utf-8") return affected @@ -1280,6 +1317,8 @@ def main(): parser.add_argument("--flake-rounds", type=int, default=3, help="runs required by --flake-check (default 3)") parser.add_argument("--changed", metavar="REF", help="only harnesses the changes since the merge base with REF can " "affect (and that have not passed with their current inputs): the pull-request check") + parser.add_argument("--by-crate", action="store_true", help="with --changed: count every Rust source a probe " + "links, not only those it can reach (tools/parity_reach.py)") parser.add_argument("--changed-files", metavar="FILE", help="like --changed, for the changed paths listed in FILE, " "one per line: for a copy without Git history (tools/pr_ready.py on another machine)") args = parser.parse_args() @@ -1316,7 +1355,11 @@ def main(): changed, since = changed_files(args.changed), f" since the merge base with {args.changed}" if changed is None: parser.error(f"no merge base with {args.changed}") - affected = affected_harnesses(found, records, changed) + base = None + if args.changed: + merge_base = subprocess.run(["git", "merge-base", "HEAD", args.changed], cwd=ROOT, capture_output=True, text=True) + base = merge_base.stdout.strip() or None + affected = affected_harnesses(found, records, changed, reach=not args.by_crate, base=base) chosen = [name for name in found if name in affected and stale(name, state, hasher, environment)] print(f"{len(changed)} files changed{since}: {len(affected)} harnesses " f"depend on them, {len(chosen)} without a pass for their current inputs.", flush=True) diff --git a/tools/parity_main.py b/tools/parity_main.py new file mode 100644 index 000000000..ff319aed0 --- /dev/null +++ b/tools/parity_main.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""parity-main (.github/workflows/parity-main.yml): the vanilla harnesses run on main, a batch of merges at a time. + + python3 tools/parity_main.py base # the commit this run checks from: main's last green parity-main + python3 tools/parity_main.py bisect BASE # after a failure: the first merge since BASE that broke a harness + +`base` prints the newest commit on main whose parity-main run passed and that HEAD contains, else HEAD's first parent, +so a run tests every merge since the last green one (`tools/parity.py --changed `), however many landed while +the previous run went on. `bisect` reads the harnesses that failed (.local/parity-runner/last-run.json), checks out +the first-parent merges between BASE and HEAD by halves and runs only those harnesses, then names the first merge they +fail on (written to $GITHUB_STEP_SUMMARY and, with --comment, on its pull request). HEAD is checked out again at the end. +""" + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +REPO = os.environ.get("GITHUB_REPOSITORY", "ludovic111/horizon") +LAST_RUN = ROOT / ".local" / "parity-runner" / "last-run.json" + + +def git(*arguments, check=True): + result = subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, text=True) + if check and result.returncode: + sys.exit(f"git {' '.join(arguments)}: {result.stderr.strip()}") + return result + + +def green_runs(): + """Head commits of main's passing parity-main runs, newest first.""" + result = subprocess.run(["gh", "run", "list", "--repo", REPO, "--workflow", "parity-main", "--branch", "main", + "--status", "success", "--limit", "30", "--json", "headSha"], capture_output=True, text=True) + if result.returncode: + print(f"note: no parity-main history ({result.stderr.strip()})", file=sys.stderr) + return [] + return [run["headSha"] for run in json.loads(result.stdout or "[]")] + + +def base(head="HEAD", runs=None): + for sha in green_runs() if runs is None else runs: + if git("merge-base", "--is-ancestor", sha, head, check=False).returncode == 0: + return git("rev-parse", sha).stdout.strip() + return git("rev-parse", f"{head}^1").stdout.strip() + + +def first_bad(commits, fails): + """The first of commits (oldest first, the last known to fail; the one before them known to pass) where fails().""" + good, bad = -1, len(commits) - 1 + while bad - good > 1: + middle = (good + bad) // 2 + if fails(commits[middle]): + bad = middle + else: + good = middle + return commits[bad] + + +def harness_exists(name): + return (ROOT / "tools" / f"test_{name.split('#')[0]}_parity.py").exists() + + +def run_harnesses(names, timeout): + """Whether any of the harnesses fails at the checked-out commit (those it doesn't have yet can't).""" + present = [name for name in names if harness_exists(name)] + if not present: + return False + return subprocess.run([sys.executable, "tools/parity.py", *present, "--timeout", str(timeout)], cwd=ROOT).returncode != 0 + + +def bisect(since, timeout, comment): + failed = [] + if LAST_RUN.exists(): + failed = json.loads(LAST_RUN.read_text(encoding="utf-8")).get("failed") or [] + if not failed: + print("No failed harness recorded: nothing to bisect.") + return 0 + head = git("rev-parse", "HEAD").stdout.strip() + commits = git("rev-list", "--first-parent", "--reverse", f"{since}..{head}").stdout.split() + if not commits: + print("No merge since the base.") + return 0 + + def fails(commit): + print(f"--- {commit[:9]} {git('log', '-1', '--format=%s', commit).stdout.strip()}", flush=True) + git("checkout", "-q", "--detach", commit) + return run_harnesses(failed, timeout) + + try: + culprit = first_bad(commits, fails) if len(commits) > 1 else commits[0] + finally: + git("checkout", "-q", "--detach", head) + subject = git("log", "-1", "--format=%s", culprit).stdout.strip() + number = re.search(r"Merge pull request #(\d+)", subject) + line = (f"parity-main: {', '.join(failed)} first fail at {culprit[:9]} ({subject}), " + f"of {len(commits)} merges since {since[:9]}.") + print(line) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write(f"### {line}\n") + if comment and number: + url = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{REPO}/actions/runs/{os.environ.get('GITHUB_RUN_ID', '')}" + body = (f"🤖 Agent: parity-main found that this merge breaks {', '.join(f'`{name}`' for name in failed)} on main " + f"(bisected over {len(commits)} merges: {url}). Releases wait until parity-main passes again.") + subprocess.run(["gh", "pr", "comment", number.group(1), "--repo", REPO, "--body", body]) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("base") + bisecting = commands.add_parser("bisect") + bisecting.add_argument("since", help="the last commit known to pass (the run's base)") + bisecting.add_argument("--timeout", type=int, default=2400, help="seconds per harness") + bisecting.add_argument("--comment", action="store_true", help="comment on the breaking merge's pull request") + args = parser.parse_args() + if args.command == "base": + print(base()) + return 0 + return bisect(args.since, args.timeout, args.comment) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/parity_reach.py b/tools/parity_reach.py new file mode 100644 index 000000000..0c3051b86 --- /dev/null +++ b/tools/parity_reach.py @@ -0,0 +1,768 @@ +"""Which of a harness's Rust sources its probes can reach, for `parity.py --changed`. + +Cargo's dep-info names every source file of every crate a probe links, so a change anywhere in horizon-client +selected every harness whose probe links horizon-client. This module narrows a harness's recorded dependencies to +the files holding an item its entry points (the probe modules it ran, the binaries it built) can reach, over a graph +of the workspace's current sources whose nodes are items: + +- everything in an entry is reached, and so is everything in a Rust file the module tree doesn't place (an + `include!`d source); +- a module is open when it is a crate root, holds reached code, or is named in a path (`name::`, `use ...name`) by + reached code while its parent is open; +- a free item (fn, struct, enum, union, trait, type, const, static, an item-level macro call's items) is reached when + reached code in its own module or below it mentions its name, or when its module is open and reached code names + it in a path or an import, or when its module is glob-imported (`use m::*`) and its name is mentioned; +- an inherent impl's methods and constants are reached one by one, by name, once their type is; a trait impl once + its trait and its type are (a std or outside name always is, a generic parameter needs nothing); a macro once + invoked; a `pub use` like a free item named by everything it imports; +- "a crate's reached code" covers the crates it names (`horizon_world::`), directly or through one that does, and + code under `#[cfg(test)]` or `#[test]` is left out (probes are never test builds). + +Names match by spelling, so a file can be kept for a name it never meant: the graph only over-approximates. A +reached file's parent modules count through their `mod` declarations of it (parity.py compares those with the merge +base). Everything that isn't a workspace Rust source stays a dependency, except a crate's non-Rust files (shaders, +assets, fonts) that only unreached sources name; a dependency that no longer exists counts as changed, and Rust files +a change adds take part. +""" + +import hashlib +from pathlib import Path, PurePosixPath +import re +import subprocess + +IDENT = r"[A-Za-z_]\w*" +WORD = re.compile(rf"\b({IDENT})\b") +MOD_DECLARATION = re.compile(rf"\bmod\s+({IDENT})\s*;") +INLINE_MOD = re.compile(rf"\bmod\s+({IDENT})\s*\{{") +PATH_ATTRIBUTE = re.compile(r'#\[\s*path\s*=\s*"([^"]+)"\s*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?mod\s+\w+\s*;$') +BRACE = re.compile(r"[{}]") +TEST_ONLY = re.compile(r"#\[\s*(?:cfg\s*\(\s*test\s*\)|test)\s*\]") +TEST_MODULE = re.compile(r"#\[\s*cfg\s*\(\s*test\s*\)\s*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?mod\s+\w+\s*;$") +SPLIT = re.compile(r"[{}()\[\];]") +ITEM = re.compile(rf"(?:pub(?:\s*\([^)]*\))?\s+)?(?:(?:unsafe|async|const|extern|default|auto)\s+)*" + rf"(fn|struct|enum|union|trait|type|const|static|impl|mod|use|macro_rules\s*!|extern\s+crate)" + rf"(?!\w)\s*(?:mut\s+)?({IDENT})?") +PATH_BEFORE = re.compile(rf"\b({IDENT})\s*::") +PATH_AFTER = re.compile(rf"::\s*({IDENT})") +MACRO_USE = re.compile(rf"\b({IDENT})\s*!\s*[(\[{{]") +MACRO_CALL = re.compile(rf"({IDENT}(?:\s*::\s*{IDENT})*)\s*!") +DEFINED_INSIDE = re.compile(rf"\b(?:fn|struct|enum|union|trait|type|const|static)\s+({IDENT})") +NESTED_IMPL = re.compile(r"^[ \t]*(?:unsafe\s+)?impl\b([^{;]*)\{", re.MULTILINE) +QUALIFIED = re.compile(rf"{IDENT}(?:\s*::\s*{IDENT})*") +GLOB_IMPORT = re.compile(r"\buse\s[^;]*::\s*\*") +# File names a source may name in a literal (include_str!, include_bytes!, a path it reads). A file of another kind +# is never "named", so it always counts. +FILE_NAME = re.compile(r"[\w-]+\.(?:wgsl|glsl|json|png|ttf|otf|svg|txt|toml|ogg|wav|nbt|bin|dat|csv|mcmeta|wit|wasm)\b") +TYPE_KINDS = {"struct", "enum", "union", "trait", "type", "macro_call"} +# Names an impl header can take from outside the workspace: std's prelude and primitive types, and the std and serde +# traits a glob import may bring. A glob-imported name outside this list is taken to be the workspace's. +STD_NAMES = {"Display", "Error", "Read", "Write", "Seek", "BufRead", "Deref", "DerefMut", "Index", "IndexMut", "Add", + "Sub", "Mul", "Div", "Rem", "Neg", "Not", "BitAnd", "BitOr", "BitXor", "Shl", "Shr", "AddAssign", + "SubAssign", "MulAssign", "DivAssign", "RemAssign", "BitAndAssign", "BitOrAssign", "BitXorAssign", + "Borrow", "BorrowMut", "Future", "Any", "FromStr", "Hasher", "BuildHasher", "Formatter", "Ordering", + "Serialize", "Deserialize", "Serializer", "Deserializer", "Visitor", "Termination", "Sum", "Product", + "Wake", "Cow", "Rc", "Arc", "Mutex", "RwLock", "Cell", "RefCell", "HashMap", "HashSet", "BTreeMap", + "BTreeSet", "VecDeque"} +PRELUDE = {"Option", "Some", "None", "Result", "Ok", "Err", "Vec", "String", "Box", "ToString", "ToOwned", "Clone", + "Copy", "Send", "Sync", "Sized", "Unpin", "Drop", "Fn", "FnMut", "FnOnce", "From", "Into", "TryFrom", + "TryInto", "AsRef", "AsMut", "Default", "Eq", "PartialEq", "Ord", "PartialOrd", "Iterator", "Extend", + "IntoIterator", "DoubleEndedIterator", "ExactSizeIterator", "FromIterator", "Debug", "Hash", + "bool", "char", "str", "u8", "u16", "u32", "u64", "u128", "usize", "i8", "i16", "i32", "i64", "i128", + "isize", "f32", "f64"} +STD_NAMES |= PRELUDE +KEYWORDS = {"crate", "self", "super", "Self", "pub", "in", "as", "use", "for", "where", "dyn", "mut", "const", "fn", + "unsafe", "impl", "std", "core", "alloc", "let", "if", "else", "match", "return", "struct", "enum", "mod", + "type", "static", "trait", "true", "false", "loop", "while", "move", "ref", "break", "continue", "async", + "await", "extern", "union"} + + +def segments(code, start, end): + """The item-level pieces of code[start:end]: each ends at a `;` or a `}` outside any bracket.""" + found, begin, depth = [], start, 0 + for match in SPLIT.finditer(code, start, end): + character = match.group() + if character in "{([": + depth += 1 + elif character in "})]": + depth -= 1 + if character == "}" and depth == 0: + found.append((begin, match.end())) + begin = match.end() + elif depth == 0: + found.append((begin, match.end())) + begin = match.end() + if code[begin:end].strip(): + found.append((begin, end)) + return found + + +def skip_attributes(code, index, end): + """The offset of the first non-space character after the attributes (`#[...]`, `#![...]`) starting at index.""" + while True: + while index < end and code[index].isspace(): + index += 1 + if not code.startswith("#", index) or not re.match(r"#!?\[", code[index:index + 3]): + return index + depth = 0 + for position in range(code.index("[", index), end): + depth += {"[": 1, "]": -1}.get(code[position], 0) + if depth == 0: + index = position + 1 + break + else: + return end + + +def body_of(code, start, end): + """The range inside an item's outermost `{ }` block, or None.""" + opening = code.find("{", start, end) + closing = code.rfind("}", start, end) + return (opening + 1, closing) if 0 <= opening < closing else None + + +def words(text): + return set(WORD.findall(text)) - KEYWORDS + + +def qualified_words(text): + """Names a piece of code reaches through a path (`a::b`: b), the way items of other modules are named.""" + return set(PATH_AFTER.findall(text)) - KEYWORDS + + +def path_words(text): + """Names a piece of code uses as path segments (`a::`, `::b`), where module names appear.""" + return set(PATH_BEFORE.findall(text)) | set(PATH_AFTER.findall(text)) + + +def parse_items(code, start, end, items): + """Append the items of code[start:end] (a file or an inline module's body) to items, as dicts.""" + for begin, finish in segments(code, start, end): + head = skip_attributes(code, begin, finish) + text = code[head:finish] + if not text.strip(" \n\t;") or TEST_ONLY.search(code[begin:head]): + continue # compiled into tests only, never into a probe + match = ITEM.match(text) + kind = re.sub(r"\s+", "", match.group(1)) if match else None + name = match.group(2) if match else None + if kind == "mod": + body = body_of(code, head, finish) + if body: + parse_items(code, body[0], body[1], items) + continue + if kind in ("externcrate", None) and not MACRO_CALL.match(text): + if kind is None: + items.append({"kind": "glue", "mentions": sorted(words(text)), "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + continue + if kind is None: # an item-level macro call: it may define items + items.append({"kind": "macro_call", "names": sorted(set(DEFINED_INSIDE.findall(text))), + "macro": MACRO_CALL.match(text).group(1).split("::")[-1].strip(), + "mentions": sorted(words(text)), "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + elif kind == "use": + head_word = re.match(rf"\s*(?:::\s*)?({IDENT})", text[match.end(1):]) + items.append({"kind": "use", "public": text.startswith("pub"), "names": sorted(words(text)), + "head": head_word.group(1) if head_word else "", "glob": bool(re.search(r"::\s*\*", text)), + "mentions": sorted(words(text)), "paths": sorted(words(text)), "qualified": sorted(words(text)), + "invokes": []}) + elif kind == "impl": + body = body_of(code, head, finish) + subject = code[head + match.end(1) - match.start(): body[0] - 1 if body else finish] + inherent = not re.search(r"\bfor\b", impl_subject(subject)[1]) + if inherent and body: + members = [] + for member_begin, member_end in segments(code, body[0], body[1]): + member = code[skip_attributes(code, member_begin, member_end):member_end] + found = ITEM.match(member) + members.append({"name": found.group(2) if found else None, "mentions": sorted(words(member)), + "paths": sorted(path_words(member)), "qualified": sorted(qualified_words(member)), "invokes": sorted(set(MACRO_USE.findall(member)))}) + items.append({"kind": "impl", "subject": subject, "inherent": True, "members": members, + "mentions": sorted(words(subject)), "paths": sorted(path_words(subject)), "qualified": sorted(qualified_words(subject)), "invokes": sorted(set(MACRO_USE.findall(subject)))}) + else: + items.append({"kind": "impl", "subject": subject, "inherent": False, "mentions": sorted(words(text)), + "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + else: + kind = "macro" if kind.startswith("macro_rules") else kind + items.append({"kind": kind, "names": [name] if name else [], "mentions": sorted(words(text)), + "defines": sorted(set(DEFINED_INSIDE.findall(text))) if kind == "macro" else [], + "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + if kind in ("fn", "const", "static"): + # An impl inside a function body still holds for the whole program. + for header in NESTED_IMPL.findall(text): + items.append({"kind": "impl", "subject": header, "inherent": False, + "mentions": sorted(words(text)), "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))}) + + +def crate_roots(crate_dir): + """A crate's root files: its library, binaries and examples.""" + roots = [crate_dir / "src" / "lib.rs", crate_dir / "src" / "main.rs"] + for folder in (crate_dir / "src" / "bin", crate_dir / "examples"): + if folder.is_dir(): + roots += sorted(folder.glob("*.rs")) + sorted(folder.glob("*/main.rs")) + return [root for root in roots if root.is_file()] + + +class SourceGraph: + """The workspace's Rust sources, parsed once: what each file names, defines and declares.""" + + def __init__(self, root, rust_code, cache=None): + """cache: a dict of earlier parses by content digest, which the graph reuses and adds to (parity.py keeps it + in .local/parity-runner/reach-cache.json).""" + self.root = Path(root) + self.rust_code = rust_code + self.cache = {} if cache is None else cache + self.used = set() + self.files = {} + self.parent = {} # module file -> the file declaring it + self.module_name = {} + self.roots = set() + self.target_cache = {} + self.workspace_heads = None + self.test_only = set() # modules compiled into tests only + self.tables = None + self.root_of = {} # source -> the root of its target (lib.rs, a bin's or an example's main file) + self.results = {} # (entries, universe) -> reachable files + for manifest in sorted((self.root / "crates").glob("*/Cargo.toml")): + for crate_root in crate_roots(manifest.parent): + path = self.relative(crate_root) + self.roots.add(path) + self.root_of[path] = path + self.place(path, crate_root.parent, set()) + + def entries(self, artifacts, rust): + """The root files of the probes and binaries a harness built (its record's artifacts), among its Rust + dependencies: for a crate's probe binary, the probe modules it ran (parity.only_probes_run kept only those). + None when an artifact can't be placed.""" + found = set() + for kind, name, *rest in artifacts: + selection = rest[1] if len(rest) > 1 else [] + packages = [value for flag, value in zip(selection, selection[1:]) if flag == "-p"] or \ + [manifest.parent.name for manifest in sorted((self.root / "crates").glob("*/Cargo.toml"))] + roots = set() + for package in packages: + crate = self.root / "crates" / package + targets = self.targets(crate) + if kind in ("example", "examples"): + candidates = [path for target, (target_kind, path) in targets.items() + if target_kind == "example" and (not name or target == name)] + else: + candidates = [path for target, (target_kind, path) in targets.items() + if target_kind == "bin" and (not name or target == name)] + for path in candidates: + if path.endswith("/examples/probe/main.rs"): + # The probes it ran, and the dispatch and support modules they use (only_probes_run kept + # those); the binary's root, which only declares them, so their own submodules are placed. + folder = path.removesuffix("main.rs") + roots |= {dependency for dependency in rust if dependency.startswith(folder) + and "/" not in dependency.removeprefix(folder).removeprefix("support/")} + if roots: + roots.add(path) + elif path in rust: + roots.add(path) + if not roots: + return None + found |= roots + return found or None + + def targets(self, crate): + """A crate's bins and examples: {target name: (kind, root file)}, from its Cargo.toml and Cargo's defaults.""" + if crate in self.target_cache: + return self.target_cache[crate] + found = {} + manifest = (crate / "Cargo.toml").read_text(encoding="utf-8") if (crate / "Cargo.toml").is_file() else "" + package = re.search(r'^\[package\][^\[]*?^name\s*=\s*"([^"]+)"', manifest, re.MULTILINE | re.DOTALL) + if (crate / "src" / "main.rs").is_file() and package: + found[package.group(1)] = ("bin", self.relative(crate / "src" / "main.rs")) + for kind, folder in (("bin", crate / "src" / "bin"), ("example", crate / "examples")): + for path in sorted(folder.glob("*.rs")) + sorted(folder.glob("*/main.rs")): + found[path.stem if path.name != "main.rs" else path.parent.name] = (kind, self.relative(path)) + for kind, table in re.findall(r"^\[\[(bin|example)\]\]([^\[]*)", manifest, re.MULTILINE): + name = re.search(r'^name\s*=\s*"([^"]+)"', table, re.MULTILINE) + path = re.search(r'^path\s*=\s*"([^"]+)"', table, re.MULTILINE) + if name and path and (crate / path.group(1)).is_file(): + found[name.group(1)] = (kind, self.relative(crate / path.group(1))) + self.target_cache[crate] = found + return found + + def target(self, path): + """The Cargo target a source belongs to: the root file of the library, bin or example whose module tree + holds it (a file nothing places is its own).""" + return self.root_of.get(path, path) + + def is_library(self, path): + return self.target(path).endswith("/src/lib.rs") + + def relative(self, path): + return Path(path).resolve().relative_to(self.root.resolve()).as_posix() + + def place(self, path, module_dir, seen): + """Walk the module tree from a file: record each declared module's file, parent and name.""" + if path in seen: + return + seen.add(path) + parsed = self.parse(path) + if parsed is None: + return + for name, nesting, attribute, test in parsed["declarations"]: + if attribute is not None: + if nesting: + continue # a #[path] inside an inline module: unplaced, so always kept + candidates = [(self.root / path).parent / attribute] + else: + folder = module_dir.joinpath(*nesting) + candidates = [folder / f"{name}.rs", folder / name / "mod.rs"] + for candidate in candidates: + if candidate.is_file(): + child = self.relative(candidate) + self.parent.setdefault(child, path) + self.module_name.setdefault(child, name) + self.root_of.setdefault(child, self.root_of[path]) + if test or path in self.test_only: + self.test_only.add(child) + child_dir = candidate.parent if candidate.name == "mod.rs" or attribute is not None \ + else candidate.parent / candidate.stem + self.place(child, child_dir, seen) + break + + def parse(self, path): + if path in self.files: + return self.files[path] + try: + data = (self.root / path).read_bytes() + except OSError: + self.files[path] = None + return None + digest = hashlib.sha256(data).hexdigest() + self.used.add(digest) + if digest in self.cache: + self.files[path] = parsed = self.cache[digest] + return parsed + raw = data.decode("utf-8", errors="replace") + code = self.rust_code(raw) # same length as raw: literals and comments blanked + opening = {match.end() - 1: match.group(1) for match in INLINE_MOD.finditer(code)} + events = sorted([(match.start(), match) for match in BRACE.finditer(code)] + + [(match.start(), match) for match in MOD_DECLARATION.finditer(code)], key=lambda event: event[0]) + declarations, stack, depth = [], [], 0 + for index, match in events: + if match.group() == "{": + depth += 1 + if index in opening: + stack.append((opening[index], depth)) + elif match.group() == "}": + if stack and stack[-1][1] == depth: + stack.pop() + depth -= 1 + else: + window = " ".join(raw[max(0, index - 300):match.end()].split()) + attribute = PATH_ATTRIBUTE.search(window) + declarations.append([match.group(1), [name for name, _ in stack], attribute.group(1) if attribute else None, + bool(TEST_MODULE.search(window))]) + items = [] + parse_items(code, 0, len(code), items) + self.files[path] = parsed = { + "declarations": declarations, + "items": items, + "glob": bool(GLOB_IMPORT.search(code)), + "file_names": sorted(set(FILE_NAME.findall(raw))), + } + self.cache[digest] = parsed + return parsed + + def saved_cache(self): + """The parses of the files this graph read, to keep for the next run.""" + return {digest: self.cache[digest] for digest in self.used if digest in self.cache} + + def requirement(self, path, item): + """For an impl: for its trait and for its type, [names the file defines, other workspace names], one of + which must be a reached type or trait (of this file, of a crate it sees) before the impl can matter; both + empty: nothing of the workspace is needed. A name is the workspace's when the path naming it starts in the + workspace, or the file defines it, imports it from the workspace or may glob-import it; otherwise (std's, + another crate's, the prelude's) it needs nothing.""" + if "requires" in item: + return item["requires"] + if self.workspace_heads is None: + self.workspace_heads = {"crate", "self", "super"} | set(self.module_name.values()) | { + manifest.parent.name.replace("-", "_") for manifest in (self.root / "crates").glob("*/Cargo.toml")} + parsed = self.parse(path) + defined = {name for other in parsed["items"] for name in other.get("names", ()) if other["kind"] != "use"} + workspace_imports = {name for other in parsed["items"] if other["kind"] == "use" + and other["head"] in self.workspace_heads for name in other["names"]} + parameters, subject = impl_subject(item["subject"]) + parts = [] + for part in re.split(r"\bfor\b", subject, maxsplit=1): + names, own = set(), set() + for match in QUALIFIED.finditer(part): + segments_ = [segment.strip() for segment in match.group().split("::")] + name = segments_[-1] + if name in parameters or name in KEYWORDS: + continue + if len(segments_) > 1: + if segments_[0] in self.workspace_heads: + names.add(name) + elif name in defined: + own.add(name) + elif name in workspace_imports or (parsed["glob"] and name not in STD_NAMES): + names.add(name) + parts.append([sorted(own), sorted(names)]) + item["requires"] = parts + return parts + + def index(self): + """Lookup tables over every placed source, built once: which crates each crate sees, the nodes (items, an + inherent impl's members) by the names they define and the types they need, the modules by name.""" + if self.tables is not None: + return self.tables + idents = {manifest.parent.name.replace("-", "_"): manifest.parent.name + for manifest in (self.root / "crates").glob("*/Cargo.toml")} + paths = [path for path in self.files if self.files[path] is not None and path not in self.test_only] + sees = {} + for path in paths: + named = {word for item in self.files[path]["items"] for word in item["mentions"]} & idents.keys() + sees.setdefault(crate_of(path), {crate_of(path)}).update(idents[name] for name in named) + changed = True + while changed: + changed = False + for crate, seen in sees.items(): + wider = set().union(*(sees.get(other, {other}) for other in seen)) + if not wider <= seen: + seen |= wider + changed = True + seen_by = {} # crate -> the crates that see it + for crate, seen in sees.items(): + for other in seen: + seen_by.setdefault(other, set()).add(crate) + # What a call of a workspace macro defines: what its definition's body defines. + expands = {} + for path in paths: + for item in self.files[path]["items"]: + if item["kind"] == "macro": + for name in item["names"]: + expands.setdefault(name, set()).update(item["defines"]) + names_of, by_name, by_file_name, by_type, by_module, children, seeds = {}, {}, {}, {}, {}, {}, [] + for path in paths: + crate = crate_of(path) + if path in self.module_name: + by_module.setdefault((crate, self.module_name[path]), []).append(path) + children.setdefault(self.parent[path], []).append(path) + for index, item in enumerate(self.files[path]["items"]): + node = (path, index, None) + names = set(item.get("names", ())) + if item["kind"] == "macro_call": + names |= expands.get(item["macro"], set()) + names_of[node] = names + if item["kind"] == "glue": + seeds.append(node) + elif item["kind"] == "macro_call" and not names: + # A macro call defining nothing by name may still expand to impls: it counts once a type or + # trait it mentions is reached. + for name in item["mentions"]: + by_type.setdefault(name, []).append(node) + elif item["kind"] == "impl": + required = {name for own, foreign in self.requirement(path, item) for name in own + foreign} + members = [(path, index, member) for member in range(len(item["members"]))] if item["inherent"] else [] + for member_node in members: + name = item["members"][member_node[2]]["name"] + names_of[member_node] = set() + if name: + by_name.setdefault(name, []).append(member_node) + for name in required: + for other in [node] + members: + by_type.setdefault(name, []).append(other) + if not required: + seeds.append(node) + seeds += [member_node for member_node in members if not item["members"][member_node[2]]["name"]] + else: + for name in names: + by_name.setdefault(name, []).append(node) + by_file_name.setdefault((path, name), []).append(node) + self.tables = dict(idents=idents, sees=sees, seen_by=seen_by, names_of=names_of, by_name=by_name, + by_file_name=by_file_name, by_type=by_type, by_module=by_module, children=children, + seeds=seeds, node_sets={}) + return self.tables + + def reachable(self, entries, universe, reasons=None): + """The files of universe (workspace Rust sources) holding an item reachable from entries (by the rules in + this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging).""" + # Searching every source of the crates involved (not only those Cargo listed) only adds reach, and lets + # harnesses with the same entries (the game binary's) share one search. + crates = frozenset(crate_of(path) for path in universe) + key = (frozenset(entries), crates) + if reasons is None and key in self.results: + return self.results[key] & set(universe) + everything = {path for path in self.files if crate_of(path) in crates} | set(universe) + files = self.search(entries, everything, reasons) + self.results[key] = files + return files & set(universe) + + def search(self, entries, universe, reasons): + targets = {self.target(path) for path in entries} + universe = {path for path in universe if (self.is_library(path) or self.target(path) in targets) + and path not in self.test_only} + tables = self.index() + idents, sees, seen_by, names_of = tables["idents"], tables["sees"], tables["seen_by"], tables["names_of"] + node_sets = tables["node_sets"] + by_name, by_file_name, by_type = tables["by_name"], tables["by_file_name"], tables["by_type"] + by_module, children = tables["by_module"], tables["children"] + seeds = [node for node in tables["seeds"] if node[0] in universe] + for path in universe: + if (path in entries or (path not in self.parent and path not in self.roots)) and self.parse(path): + seeds += [(path, index, None) for index in range(len(self.parse(path)["items"]))] + + def names(node): + return names_of.get(node, set()) + + def sets_of(node, item): + """A node's mentions, path words, qualified names and macro calls, as sets (parses keep lists).""" + if node not in node_sets: + source = item["members"][node[2]] if node[2] is not None else item + node_sets[node] = {key: set(source.get(key, ())) for key in ("mentions", "paths", "qualified", "invokes")} + return node_sets[node] + + def item_of(node): + return self.parse(node[0])["items"][node[1]] + + reached, opened, globbed, files, queue = set(), set(), set(), set(), [] + # name -> the crates whose reached code mentions it plainly, names it in a path or an import, invokes it. + mentioned, qualified, invoked = {}, {}, {} + done = {"mentions": {}, "qualified": {}, "invokes": {}, "paths": {}} # crate -> names already handled + path_names, types = {}, {} # by crate + local = {} # file -> plain mentions by reached code in it or below it (child modules may `use super::*`) + local_types = {} # file -> its reached types and traits + + def heard(table, name, crate): + """Whether a crate that can see crate's items has name in table.""" + sources = table.get(name) + return bool(sources) and not sources.isdisjoint(seen_by.get(crate, {crate})) + + def ready(node): + path, index, member = node + item = item_of(node) + crate = crate_of(path) + if item["kind"] == "impl": + visible = visible_types(crate) + here = local_types.get(path, set()) + if not all((not own and not foreign) or here.intersection(own) or visible.intersection(foreign) + for own, foreign in self.requirement(path, item)): + return False + name = item["members"][member]["name"] if member is not None else None + return name is None or heard(mentioned, name, crate) # a method: called by its name + if item["kind"] == "macro_call" and not names(node): + return not visible_types(crate).isdisjoint(item["mentions"]) + names_ = names(node) + if item["kind"] == "macro": + return any(heard(invoked, name, crate) for name in names_) + if item["kind"] == "use" and item["glob"]: + return path in opened + nearby = local.get(path, ()) + if any(name in nearby for name in names_): + return True # named in its own module, or below it + if item["kind"] == "use" and not item["public"]: + return False + if path in opened and any(heard(qualified, name, crate) for name in names_): + return True # named through a path, or imported + return path in globbed and any(heard(mentioned, name, crate) for name in names_) + + visible_cache = {} + + def visible_types(crate): + """The reached types and traits of the crates a crate sees (cached until a type is reached).""" + if crate not in visible_cache: + visible_cache[crate] = set().union(*(types.get(other, set()) for other in sees.get(crate, {crate}))) + return visible_cache[crate] + + def wake(nodes, why): + for other in nodes: + if other not in reached and other[0] in universe and ready(other): + reach(other, why) + + def open_module(path, why): + pending = [path] + while pending: + current = pending.pop() + if current in opened: + continue + opened.add(current) + crate = crate_of(current) + pending += [child for child in children.get(current, ()) + if child in universe and self.module_name[child] in path_names.get(crate, ())] + parsed = self.parse(current) + wake([(current, index, None) for index, item in enumerate(parsed["items"] if parsed else ()) + if item["kind"] != "impl"], f"{current} opened ({why})") + + def reach(node, why): + if node in reached: + return + reached.add(node) + queue.append(node) + path, _, member = node + item = item_of(node) + if (item["kind"] == "use" and not item["public"]) or (item["kind"] == "impl" and item["inherent"] and member is None): + return + if path not in files: + files.add(path) + if reasons is not None: + reasons[path] = why + + for path in universe: + if path.endswith("/src/lib.rs") or path in entries or path not in self.parent: + open_module(path, "crate root or entry") + for node in seeds: + reach(node, "entry" if node[0] in entries else "always counted") + for path in entries & universe: + files.add(path) + if reasons is not None: + reasons.setdefault(path, "entry") + while queue: + node = queue.pop() + path, index, member = node + item = item_of(node) + crate = crate_of(path) + label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"] + source = sets_of(node, item) + why = f"{path} ({label})" + # Plain mentions reach items of this file and of the modules above it. + ancestor, new = path, source["mentions"] + while ancestor is not None and new: + known = local.setdefault(ancestor, set()) + new = new - known + known |= new + for name in new: + wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}") + ancestor = self.parent.get(ancestor) + if item["kind"] == "use" and item["glob"]: + for name in item["names"]: + for target in sees.get(crate, {crate}): + for module in list(by_module.get((target, name), ())) + \ + ([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []): + if module not in globbed and module in universe: + globbed.add(module) + wake([(module, index, None) for index in range(len(self.parse(module)["items"]))] + if self.parse(module) else [], f"{why} glob-imports {name}") + visible_to = sees.get(crate, {crate}) + new = source["paths"] - done["paths"].setdefault(crate, set()) + done["paths"][crate] |= new + for name in new: + for target in visible_to: + path_names.setdefault(target, set()).add(name) + for module in by_module.get((target, name), ()): + if module in universe and self.parent[module] in opened: + open_module(module, f"{why} names {name}") + for key, table, wanted in (("qualified", qualified, lambda other: True), + ("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"), + ("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)): + new = source[key] - done[key].setdefault(crate, set()) + done[key][crate] |= new + for name in new: + table.setdefault(name, set()).add(crate) + wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)], + f"{why} {key} {name}") + if item["kind"] in TYPE_KINDS and member is None: + own = types.setdefault(crate, set()) + local_types.setdefault(path, set()).update(names(node)) + if not names(node) <= own: + own |= names(node) + visible_cache.clear() + seeing = seen_by.get(crate, {crate}) + for name in names(node): # also when the crate has one by that name already: impls of this one + wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing], + f"implements for {name}, defined in {path}") + return files + + +def impl_subject(header): + """An impl header (the text after `impl`): its generic parameters' names, and the rest without its where clause + (the trait and the type it is for).""" + header, parameters = header.strip(), set() + if header.startswith("<"): + depth = 0 + for index, character in enumerate(header): + depth += {"<": 1, ">": -1}.get(character, 0) + if depth == 0: + generics, header = header[1:index], header[index + 1:] + parameters = {match.group(1) for match in re.finditer(rf"(?:^|,)\s*(?:const\s+)?({IDENT})", generics)} + break + return parameters, re.split(r"\bwhere\b", header)[0] + + +def crate_of(path): + return path.split("/", 2)[1] + + +def is_workspace_rust(path): + parts = PurePosixPath(path).parts + return len(parts) > 2 and parts[0] == "crates" and path.endswith(".rs") + + +DECLARATION_LINE = re.compile(rf"\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+({IDENT})\s*;") + + +def declarations(text, names, keep_rest): + """A Rust file's `mod` declarations of names, each with the attribute lines just above it; with keep_rest, the + whole text except the declarations of other modules instead.""" + lines, found, attributes = text.splitlines(), [], [] + for line in lines: + if line.strip().startswith("#[") and not line.strip().startswith("#!["): + attributes.append(line.strip()) + continue + match = DECLARATION_LINE.match(line) + if match and match.group(1) in names: + found += attributes + [line.strip()] + elif keep_rest and not match: + found += attributes + [line] + attributes = [] + return "\n".join(found + attributes) + + +def declarations_changed(root, path, children, base, whole): + """Whether a changed file differs from the commit base in what a harness depends on: for a module parent, its + `mod` declarations of the reached children; for a whole file (whole), everything but its declarations of other + modules (adding a module beside reached ones changes nothing). True without a base, or if either side is + missing.""" + if base is None: + return True + try: + head = (Path(root) / path).read_text(encoding="utf-8", errors="replace") + before = subprocess.run(["git", "show", f"{base}:{path}"], cwd=root, capture_output=True, text=True, check=True).stdout + except (OSError, subprocess.CalledProcessError): + return True + return declarations(head, children, whole) != declarations(before, children, whole) + + +def narrow(dependencies, artifacts, graph, changed=()): + """A harness's dependencies narrowed to what its entry points can reach: {dependency: (whole, children)}, where + whole says the file counts for its own content (else only through its `mod` declarations of children, as a + module ancestor of a reached file) and children are its reached child modules. Without known entry points (a + probe or binary the harness built that isn't found here), every dependency is kept whole.""" + plain = [dependency.split("#keep:")[0] for dependency in dependencies] + crates = {PurePosixPath(path).parts[1] for path in plain if path.startswith("crates/")} + rust = {path for path in plain if is_workspace_rust(path)} + added = {path for path in changed if is_workspace_rust(path) and PurePosixPath(path).parts[1] in crates + and path not in rust and (graph.root / path).is_file()} + entries = graph.entries(artifacts, rust) + if entries is None: + return {dependency: (True, None) for dependency in dependencies} + missing = {path for path in rust if graph.parse(path) is None} + reached = graph.reachable(entries, rust | added | entries) | missing + named, named_in_crate = set(), {} + for path in rust | added: + parsed = graph.parse(path) + file_names = set(parsed["file_names"]) if parsed else set() + if path in reached: + named |= file_names + named_in_crate.setdefault(PurePosixPath(path).parts[1], set()).update(file_names) + kept = {} + for dependency, path in zip(dependencies, plain): + if is_workspace_rust(path): + if path in reached: + kept[dependency] = (True, set()) + elif not path.startswith("crates/") or path.endswith(("Cargo.toml", "build.rs")) \ + or PurePosixPath(path).name in named \ + or PurePosixPath(path).name not in named_in_crate.get(PurePosixPath(path).parts[1], ()): + kept[dependency] = (True, None) + for path in sorted(added & reached): + kept[path] = (True, set()) + for path in sorted(reached): + child = path + while child in graph.parent: + parent = graph.parent[child] + kept.setdefault(parent, (False, set()))[1].add(graph.module_name[child]) + child = parent + return kept diff --git a/tools/prepare_output.py b/tools/prepare_output.py new file mode 100755 index 000000000..81ed156cb --- /dev/null +++ b/tools/prepare_output.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""The prepared output's version: what keeps `OUTPUT_VERSION` (crates/horizon-prepare/src/lib.rs) honest. + +The launcher copies a prepared version's resources (`.local/horizon-client/`) to a newly installed +one when both pin the same Minecraft jars and their preparers declare the same `OUTPUT_VERSION`, so a release +that changes the preparer's code without changing what it makes asks nobody to prepare again. The number is +bumped by hand; this tool catches a change to the output without a bump. + +crates/horizon-prepare/output-versions.txt lists every `OUTPUT_VERSION` with the digest of what it prepares. +It only grows: the last line is the current version, and a change to the output appends a line. + + python3 tools/prepare_output.py digest [RESOURCES] the digest of a preparation + python3 tools/prepare_output.py check [--base SHA] [--cache] [RESOURCES] + fails when RESOURCES (default: this checkout's .local/horizon-client/, as CI prepares it) is + not what the last line says, or when the file changed other than by appending since SHA. --cache keeps + the digest beside RESOURCES (CI: a preparation never changes once made; it takes ~30 s to compute) + +The digest covers every file of the preparation, canonicalised where the preparer is not deterministic between +runs (verified across the CI runners): JSON is compared as data (key order, the order of each `tags` list of +creative-inventory.json, and the addresses in `$$Lambda/0x...` class names don't count); logs, compiled classes, +the sounds (copied from the player's launcher by another step) and the manifest's file hashes (raw bytes) don't +count. The manifest's own scalar fields (schema, data version, jar hashes, file count) do. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +VERSIONS = 'crates/horizon-prepare/output-versions.txt' +LIB = 'crates/horizon-prepare/src/lib.rs' +MANIFEST = 'horizon-resources.json' +LAMBDA = re.compile(r'\$\$Lambda/0x[0-9a-fA-F]+') +CONSTANT = re.compile(r'^pub const OUTPUT_VERSION: u32 = (\d+);', re.M) + + +def canonical_json(value, key=''): + if isinstance(value, dict): + return {k: canonical_json(v, k) for k, v in value.items()} + if isinstance(value, list): + items = [canonical_json(v, key) for v in value] + if key == 'tags' and all(isinstance(x, list) and all(isinstance(t, str) for t in x) for x in items): + items = [sorted(x) for x in items] + return items + if isinstance(value, str): + return LAMBDA.sub('$$Lambda', value) + return value + + +def canonical(rel: str, data: bytes) -> bytes: + if rel == MANIFEST: + manifest = json.loads(data) + scalars = {k: v for k, v in manifest.items() if not isinstance(v, (dict, list))} + return json.dumps(scalars, sort_keys=True).encode() + if rel.endswith('.json'): + try: + value = json.loads(data) + except ValueError: + return data + return json.dumps(canonical_json(value), sort_keys=True).encode() + return data + + +def skipped(rel: str) -> bool: + parts = rel.split('/') + return rel.endswith(('.log', '.class')) or parts[:3] == ['assets', 'minecraft', 'sounds'] \ + or rel == 'assets/minecraft/sounds.json' + + +def digest(resources: Path) -> tuple[str, int]: + """The digest of a preparation, and how many files it covers.""" + if not (resources / MANIFEST).is_file(): + raise SystemExit(f'{resources} is not a preparation: no {MANIFEST}') + files = [] + for dirpath, dirs, names in os.walk(resources): + dirs.sort() + for name in names: + path = Path(dirpath) / name + rel = path.relative_to(resources).as_posix() + if path.is_symlink() or skipped(rel): + continue + files.append(rel) + total = hashlib.sha256() + for rel in sorted(files): + data = hashlib.sha256(canonical(rel, (resources / rel).read_bytes())).hexdigest() + total.update(f'{rel}\0{data}\n'.encode()) + return total.hexdigest(), len(files) + + +def parse_versions(text: str) -> list[tuple[int, str]]: + out = [] + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + number, value = line.split() + out.append((int(number), value)) + return out + + +def problems(versions: list[tuple[int, str]], constant: int | None) -> list[str]: + found = [] + if not versions: + return [f'{VERSIONS} lists no version'] + if [n for n, _ in versions] != list(range(1, len(versions) + 1)): + found.append(f'{VERSIONS}: the versions must be 1, 2, 3... in order') + if len({d for _, d in versions}) != len(versions): + found.append(f'{VERSIONS}: two versions with the same digest') + if constant != versions[-1][0]: + found.append(f'{LIB} says OUTPUT_VERSION = {constant}, {VERSIONS} ends at {versions[-1][0]}') + return found + + +def default_resources() -> Path: + lock = json.loads((ROOT / 'research-lock.json').read_text()) + return ROOT / '.local/horizon-client' / lock['minecraft']['version'] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + parser.add_argument('command', choices=['digest', 'check']) + parser.add_argument('resources', nargs='?', type=Path) + parser.add_argument('--base', help='the commit the change is based on (check: the file only grew since)') + parser.add_argument('--cache', action='store_true', help='keep the digest beside the preparation') + args = parser.parse_args() + resources = (args.resources or default_resources()).resolve() + tool = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()[:16] + cache = resources.parent / f'.output-digest-{resources.name}-{tool}' + if args.cache and cache.is_file(): + value, count = cache.read_text().split() + else: + value, count = digest(resources) + if args.cache: + cache.write_text(f'{value} {count}\n') + if args.command == 'digest': + print(value) + return 0 + + versions = parse_versions((ROOT / VERSIONS).read_text()) + match = CONSTANT.search((ROOT / LIB).read_text()) + found = problems(versions, int(match.group(1)) if match else None) + if args.base: + shown = subprocess.run(['git', 'show', f'{args.base}:{VERSIONS}'], cwd=ROOT, capture_output=True, text=True) + if shown.returncode == 0: + before = parse_versions(shown.stdout) + if versions[:len(before)] != before: + found.append(f'{VERSIONS} changed lines it already had (since {args.base[:12]}): ' + 'a new output gets a new version, appended') + current = versions[-1] if versions else (0, '') + if value != current[1]: + known = [n for n, d in versions if d == value] + if known: + found.append(f'the preparation is OUTPUT_VERSION {known[0]}\'s output, but OUTPUT_VERSION is {current[0]}') + else: + found.append( + f'the prepared output changed ({count} files, digest {value}) but OUTPUT_VERSION is still ' + f'{current[0]}. Players would keep resources prepared by the old preparer. Bump it:\n' + f' set `pub const OUTPUT_VERSION: u32 = {current[0] + 1};` in {LIB}\n' + f' append `{current[0] + 1} {value}` to {VERSIONS}\n' + ' (if the change is only a nondeterminism this tool should ignore, teach canonical() instead)') + for problem in found: + print(f'::error::{problem}' if os.environ.get('GITHUB_ACTIONS') else f'error: {problem}', file=sys.stderr) + if not found: + print(f'prepared output: OUTPUT_VERSION {current[0]} ({count} files, {value[:16]})') + return 1 if found else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/release_train.py b/tools/release_train.py index 48d4d46cf..72d81ce57 100644 --- a/tools/release_train.py +++ b/tools/release_train.py @@ -414,13 +414,26 @@ def gather(kind, head, releases=None): return last, prs -def green_main_head(): - """The newest commit on main whose checks passed: what an automatic release builds.""" - runs = gh_json("run", "list", "--repo", REPO, "--workflow", "checks", "--branch", "main", "--event", "push", +def passed_on(workflow, sha): + """Whether the newest finished push run of workflow on main for commit sha passed.""" + runs = gh_json("run", "list", "--repo", REPO, "--workflow", workflow, "--branch", "main", "--event", "push", + "--commit", sha, "--limit", "5", "--json", "status,conclusion") + done = [entry for entry in runs if entry["status"] == "completed" and entry["conclusion"] != "cancelled"] + return bool(done) and done[0]["conclusion"] == "success" + + +def green_main_head(parity=False): + """The newest commit on main whose checks passed: what an automatic release builds. With parity (game + releases), the newest commit parity-main passed on, if its checks passed too: parity-main runs on batches of + merges, so the commits a run skipped are covered by the next one, and a superseded run counts for nothing.""" + workflow = "parity-main" if parity else "checks" + runs = gh_json("run", "list", "--repo", REPO, "--workflow", workflow, "--branch", "main", "--event", "push", "--limit", "10", "--json", "headSha,status,conclusion") for entry in runs: - if entry["status"] == "completed": - return entry["headSha"] if entry["conclusion"] == "success" else None + if entry["status"] == "completed" and entry["conclusion"] != "cancelled": + if entry["conclusion"] != "success": + return None + return entry["headSha"] if not parity or passed_on("checks", entry["headSha"]) else None return None @@ -437,7 +450,7 @@ def main(): parser.add_argument("--kind", choices=["game", "launcher"], required=True) parser.add_argument("--head", help="the commit to release (default: the newest commit on main whose checks passed)") parser.add_argument("--tag", help="notes: the release's tag") - parser.add_argument("--force", action="store_true", help="release even without player-facing changes, before merges settle or while paused") + parser.add_argument("--force", action="store_true", help="release even without player-facing changes, before merges settle, while paused or before parity-main passed") parser.add_argument("--out", type=Path, help="notes: write title.txt and body.md here") parser.add_argument("--now", help="the time to decide at (ISO 8601; default now)") parser.add_argument("--smoke", type=Path, help="gate: the release smoke test's report (smoke-test.json)") @@ -456,9 +469,12 @@ def main(): summary.write(f"### {line}\n") return 1 if problem else 0 - head = args.head or green_main_head() + # Game releases ship only a commit the vanilla harnesses passed on (parity-main, docs/release-flow.md). + parity = args.kind == "game" and args.command == "decide" and not args.force + head = args.head or green_main_head(parity) if not head: - result = {"release": False, "reason": "main's latest checks did not pass"} + result = {"release": False, "reason": "main's latest checks did not pass" + ( + " or parity-main did not pass on them" if parity else "")} print(json.dumps(result, indent=1)) write_outputs({"release": "false", "reason": result["reason"]}) return 0 @@ -482,6 +498,8 @@ def main(): blockers = open_blockers() if args.kind == "game" else [] release, reason = decision(args.kind, prs, now, os.environ.get("RELEASES_PAUSED") == "1", args.force, new_line, blockers) + if release and args.head and parity and not (passed_on("parity-main", head) and passed_on("checks", head)): + release, reason = False, f"waiting for checks and parity-main to pass on {head[:9]}" minor = any("feature" in pr["labels"] for pr in prs if pr["paths"]) tag = next_game_tag(tags, now.date(), line) if args.kind == "game" else next_launcher_tag(tags, minor, line) result = { diff --git a/tools/test_collision_parity.py b/tools/test_collision_parity.py index ad99ab960..26f7f25df 100644 --- a/tools/test_collision_parity.py +++ b/tools/test_collision_parity.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path import platform +import shutil import subprocess import tempfile @@ -37,6 +38,8 @@ def main(): if args.prepare: lab.prepare() lock, jars, libraries = test_rust_parity.verify_artifacts() + for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks + shutil.rmtree(earlier, ignore_errors=True) base = Path(tempfile.mkdtemp(prefix="run-", dir=directory)) oracle = lab.ROOT / "mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java" sources = [oracle, Path(__file__).resolve(), lab.ROOT / "tools/test_rust_parity.py", lab.ROOT / "Cargo.toml", lab.ROOT / "Cargo.lock", lab.ROOT / "crates/horizon-core/Cargo.toml", lab.ROOT / "crates/horizon-world/Cargo.toml", *sorted((lab.ROOT / "crates/horizon-core/src").rglob("*.rs")), *sorted((lab.ROOT / "crates/horizon-world/src").rglob("*.rs")), lab.ROOT / "crates/horizon-world/examples/probe/collision_fixture.rs"] diff --git a/tools/test_geometry_parity.py b/tools/test_geometry_parity.py index 54ebc21ae..fa58bf66f 100644 --- a/tools/test_geometry_parity.py +++ b/tools/test_geometry_parity.py @@ -9,6 +9,7 @@ from pathlib import Path import platform import random import struct +import shutil import subprocess import tempfile @@ -273,6 +274,8 @@ def main(): if args.prepare: lab.prepare() lock, jars, libraries = test_rust_parity.verify_artifacts() + for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks + shutil.rmtree(earlier, ignore_errors=True) base = Path(tempfile.mkdtemp(prefix="run-", dir=directory)) commands = list(corpus()) inputs = base / "inputs.txt" diff --git a/tools/test_world_parity.py b/tools/test_world_parity.py index 0b4a3deb7..9344a47b0 100644 --- a/tools/test_world_parity.py +++ b/tools/test_world_parity.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path import platform +import shutil import subprocess import tempfile @@ -35,6 +36,8 @@ def main(): if args.prepare: lab.prepare() lock, jars, libraries = test_rust_parity.verify_artifacts() + for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks + shutil.rmtree(earlier, ignore_errors=True) base = Path(tempfile.mkdtemp(prefix="run-", dir=directory)) classes = base / "classes" classes.mkdir()