Merge origin/main into ci/linux-runner
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,7 +3,8 @@ name: checks
|
||||
# check main requires, so it never has a condition: a skipped job counts as passed, and every run of this workflow
|
||||
# must report a real verify (a pull request that changes no Rust finishes it in seconds instead).
|
||||
# parity: the vanilla harnesses a pull request's changes can affect, on the Mac mini's runner, which holds the
|
||||
# maintainer's own Minecraft files (Horizon never downloads the game); .github/workflows/parity.yml. A fork's pull
|
||||
# maintainer's own Minecraft files (Horizon never downloads the game); .github/workflows/parity.yml. Only for a pull
|
||||
# request that changes a harness, oracle, probe or the runner: the rest are checked on main by parity-main. A fork's pull
|
||||
# request runs it only when a maintainer adds the run-parity label (.github/workflows/run-parity.yml), so labels
|
||||
# never start this workflow. See docs/release-flow.md.
|
||||
on:
|
||||
@@ -72,8 +73,37 @@ jobs:
|
||||
run: |
|
||||
cargo +stable test --workspace --lib --bins --tests --features horizon-client/mods,horizon-mods/pack
|
||||
cargo +stable test --workspace --doc --features horizon-client/mods,horizon-mods/pack
|
||||
parity:
|
||||
# Most pull requests leave the harnesses to parity-main, which runs them on main a batch of merges at a time
|
||||
# (.github/workflows/parity-main.yml; releases wait for it). A pull request still runs them itself when it changes
|
||||
# a harness, an oracle, a probe or the runner, or carries the run-parity label (run-parity.yml, when it is added).
|
||||
parity-scope:
|
||||
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
run: ${{ steps.scope.outputs.run }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- id: scope
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
LABELS: ${{ join(github.event.pull_request.labels.*.name, ' ') }}
|
||||
run: |
|
||||
changed=$(git diff --name-only "$(git merge-base "$BASE_SHA" HEAD)" HEAD)
|
||||
harness=$(grep -E '^(tools/test_[a-z0-9_]+_parity\.py|tools/(parity[a-z_]*|lab)\.py|tools/parity_hosts\.json|mods/horizon/src/test/|crates/[^/]+/examples/|crates/horizon-client/src/game/[a-z_]*(fixture|harness)\.rs|\.github/workflows/parity[a-z-]*\.yml)' <<< "$changed" || true)
|
||||
if [ -n "$harness" ] || [[ " $LABELS " == *" run-parity "* ]]; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "The pull request runs its harnesses: ${harness:-label run-parity}"
|
||||
else
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No harness, oracle, probe or runner change: parity-main checks this on main"
|
||||
fi
|
||||
parity:
|
||||
needs: parity-scope
|
||||
if: needs.parity-scope.outputs.run == 'true'
|
||||
uses: ./.github/workflows/parity.yml
|
||||
with:
|
||||
base: ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
name: parity-main
|
||||
# The vanilla harnesses on main, a batch of merges at a time (docs/release-flow.md#parity-on-main), on the Mac mini's
|
||||
# runner. One run at a time: merges that land while a run goes on wait in one pending run (GitHub keeps only the
|
||||
# newest), which checks every merge since main's last green parity-main (tools/parity_main.py base). On a failure it
|
||||
# bisects the failing harnesses over those merges and comments on the pull request that broke them. Releases ship
|
||||
# only a commit whose parity-main passed (tools/release_train.py), so nothing a harness catches reaches players.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
contents: read
|
||||
# The comment on the merge that broke a harness.
|
||||
pull-requests: write
|
||||
# tools/parity_main.py base reads earlier runs.
|
||||
actions: read
|
||||
concurrency:
|
||||
group: parity-main
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
# Test runs never reach the telemetry server: no live-count ping, no uploads (docs/telemetry.md).
|
||||
HORIZON_TELEMETRY_URL: "off"
|
||||
|
||||
jobs:
|
||||
parity-main:
|
||||
runs-on: [self-hosted, horizon-game]
|
||||
timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
# Keeps .local/ (parity state, probe builds, oracle results) and target/ between runs.
|
||||
clean: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Tools from the runner
|
||||
run: bash tools/ci/runner-env.sh
|
||||
- name: Vanilla data, and resources prepared by this preparer
|
||||
run: bash tools/ci/vanilla-data.sh
|
||||
- name: Main's last green parity-main
|
||||
id: base
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: echo "sha=$(python3 tools/parity_main.py base)" >> "$GITHUB_OUTPUT"
|
||||
- name: The harnesses the merges since then can affect
|
||||
env:
|
||||
PARITY_HOSTS: ''
|
||||
BASE_SHA: ${{ steps.base.outputs.sha }}
|
||||
run: |
|
||||
echo "Merges since ${BASE_SHA}:"
|
||||
git log --first-parent --format=' %h %s' "$BASE_SHA..HEAD"
|
||||
python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400
|
||||
- name: The merge that broke them
|
||||
if: failure()
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PARITY_HOSTS: ''
|
||||
BASE_SHA: ${{ steps.base.outputs.sha }}
|
||||
run: |
|
||||
rm -rf .local/parity-main-logs && cp -R .local/parity-runner/logs .local/parity-main-logs 2>/dev/null || true
|
||||
python3 tools/parity_main.py bisect "$BASE_SHA" --comment
|
||||
- name: Logs of the failed harnesses
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: parity-main-logs
|
||||
path: .local/parity-main-logs/
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
@@ -37,13 +37,56 @@ jobs:
|
||||
run: bash tools/ci/runner-env.sh
|
||||
- name: Vanilla data, and resources prepared by this preparer
|
||||
run: bash tools/ci/vanilla-data.sh
|
||||
- name: The prepared output is the one the preparer's OUTPUT_VERSION says
|
||||
env:
|
||||
BASE_SHA: ${{ inputs.base }}
|
||||
run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA"
|
||||
- name: The version constants are the pinned jar's
|
||||
run: python3 tools/gen_version_consts.py --check
|
||||
- name: The harnesses this change can affect
|
||||
env:
|
||||
PARITY_HOSTS: ''
|
||||
BASE_SHA: ${{ inputs.base }}
|
||||
run: python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400 # world_spawn takes ~1480 s on a busy runner
|
||||
run: |
|
||||
# main's selection (tools/parity.py --changed --list with main's parity.py and parity_reach.py), so a
|
||||
# branch made before a better selection landed gets it too; the head's own runner then runs those
|
||||
# harnesses. A branch that changes the selection itself selects with its own code.
|
||||
set -euo pipefail
|
||||
own() { python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400; } # world_spawn takes ~1480 s on a busy runner
|
||||
fork_point=$(git merge-base "$BASE_SHA" HEAD)
|
||||
if ! git rev-parse --verify -q origin/main >/dev/null || ! git diff --quiet "$fork_point" HEAD -- tools/parity.py tools/parity_reach.py; then
|
||||
own
|
||||
exit
|
||||
fi
|
||||
had_reach=$(git ls-files tools/parity_reach.py)
|
||||
git show origin/main:tools/parity.py > tools/parity.py
|
||||
git show origin/main:tools/parity_reach.py > tools/parity_reach.py
|
||||
status=0
|
||||
python3 tools/parity.py --changed "$BASE_SHA" --list > .local/parity-selection.txt || status=$?
|
||||
git checkout -- tools/parity.py
|
||||
if [ -n "$had_reach" ]; then git checkout -- tools/parity_reach.py; else rm -f tools/parity_reach.py; fi
|
||||
cat .local/parity-selection.txt
|
||||
if [ "$status" != 0 ]; then
|
||||
echo "main's selection failed: the head's own selection runs"
|
||||
own
|
||||
exit
|
||||
fi
|
||||
names=$(sed '$d' .local/parity-selection.txt | grep -E '^[A-Za-z0-9_#]+$' || true)
|
||||
# The summary line says how many would run: a count that doesn't match the names read (a format change)
|
||||
# must not let the job pass untested.
|
||||
expected=$(tail -1 .local/parity-selection.txt | sed -nE 's/^([0-9]+) of [0-9]+ harnesses would run$/\1/p')
|
||||
count=$(printf '%s' "$names" | grep -c . || true)
|
||||
if [ -z "$expected" ] || [ "$expected" != "$count" ]; then
|
||||
echo "main's selection listed $count names, its summary says '${expected:-?}': the head's own selection runs"
|
||||
own
|
||||
exit
|
||||
fi
|
||||
if [ "$count" = 0 ]; then
|
||||
echo "Nothing to run."
|
||||
exit 0
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
python3 tools/parity.py $names --timeout 2400
|
||||
- name: Logs of the failed harnesses
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
name: release
|
||||
# Automatic game releases (docs/release-flow.md). After every green `checks` run on main, every 10 minutes (so a
|
||||
# release waiting for merges to settle goes out without another merge) and by hand, tools/release_train.py decides:
|
||||
# release waiting for merges to settle goes out without another merge) and by hand, tools/release_train.py decides,
|
||||
# for a commit whose checks and parity-main (the vanilla harnesses on main) both passed:
|
||||
# a release when 5 pull requests merged since the last one changed what players run and the newest merged 5 minutes
|
||||
# ago, or when the oldest has waited 6 hours, or at once for a pull request labelled release-now.
|
||||
# RELEASES_PAUSED=1 (repository variable) stops them; so does an open issue labelled release-blocker, forced runs too.
|
||||
@@ -18,7 +19,8 @@ name: release
|
||||
# gh workflow run release -f dry_run=false -f force=true # on main: release now, whatever the rules say
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [checks]
|
||||
# A game release ships a commit both passed on (parity-main runs on batches: the newest commit it passed on).
|
||||
workflows: [checks, parity-main]
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
schedule:
|
||||
@@ -64,8 +66,8 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASES_PAUSED: ${{ vars.RELEASES_PAUSED }}
|
||||
# The commit whose checks just passed; a dry run builds the branch it was started on; otherwise
|
||||
# main's newest commit with green checks.
|
||||
# The commit whose checks or parity-main just passed; a dry run builds the branch it was started on;
|
||||
# otherwise main's newest commit with green checks and parity-main.
|
||||
HEAD_SHA: ${{ github.event.workflow_run.head_sha || (steps.mode.outputs.dry_run == 'true' && github.sha) || '' }}
|
||||
FORCE: ${{ inputs.force && 'yes' || '' }}
|
||||
run: |
|
||||
|
||||
@@ -82,7 +82,7 @@ python3 tools/agent_control_e2e.py # horizon-ctl / MCP end to end
|
||||
~25, a file of the client lib ~103, of `horizon-world` ~103–109, of `horizon-core` ~119. Plan for it.
|
||||
- Rust integration tests: `crates/horizon-client/tests/` (`lan_bot.rs`, `generation_saves.rs`, `streaming_meshes.rs`).
|
||||
- **macOS:** `cargo build/test --workspace` may fail to link the two wasm guest cdylibs (undefined
|
||||
`horizon:mods/events` symbols) until #1446 lands: add `--exclude coords-hud --exclude horizon-mods-test-guest --exclude horizon-mods-test-ui-guest`.
|
||||
`horizon:mods/events` symbols) until #1446 lands: add `--exclude coords-hud --exclude predator-missile --exclude horizon-mods-test-guest --exclude horizon-mods-test-ui-guest`.
|
||||
Their real check is `cargo test -p horizon-mods` (builds them for `wasm32-unknown-unknown`).
|
||||
- Linux needs `libasound2-dev libudev-dev`; `--no-default-features` drops audio and controllers (the server build).
|
||||
- Run and drive the game (starts the `horizon-client` beside `horizon-ctl`; other arguments go to the game):
|
||||
|
||||
Generated
+7
@@ -2733,6 +2733,13 @@ dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "predator-missile"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"horizon-mod-sdk",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "presser"
|
||||
version = "0.3.1"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[workspace]
|
||||
exclude = ["launcher", "crates/horizon-telemetry-server", "crates/horizon-mod-registry", "third_party/wgpu-hal"]
|
||||
members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/horizon-mods/tests/ui_guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-shaders", "crates/horizon-shader-api", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"]
|
||||
members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/horizon-mods/tests/ui_guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-shaders", "crates/horizon-shader-api", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "crates/horizon-mod-sdk/examples/predator-missile", "xtask"]
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
|
||||
@@ -224,6 +224,7 @@ fn play(path: &Path, report: &mut Report) -> Result<(), String> {
|
||||
Output::Feedback { text, error } => {
|
||||
format!("{}: {text}", if error { "error" } else { "feedback" })
|
||||
}
|
||||
Output::Command { line } => format!("command: /{line}"),
|
||||
Output::Stopped { reason, .. } => return Err(format!("it stopped: {reason}")),
|
||||
Output::OpenScreen { .. } => "opened a screen".to_string(),
|
||||
Output::CloseScreen { .. } => "closed its screen".to_string(),
|
||||
|
||||
@@ -63,6 +63,10 @@ enum Animate {
|
||||
SculkSensor,
|
||||
/// Potent sulfur that is not dry.
|
||||
PotentSulfur,
|
||||
DriedGhast {
|
||||
waterlogged: bool,
|
||||
},
|
||||
EndPortal,
|
||||
}
|
||||
fn step(facing: &str) -> [i32; 3] {
|
||||
match facing {
|
||||
@@ -137,6 +141,10 @@ fn classify(name: &str, prop: &dyn Fn(&str) -> Option<String>) -> Animate {
|
||||
drag: prop("drag").as_deref() == Some("true"),
|
||||
},
|
||||
"fire" | "soul_fire" => Animate::Fire,
|
||||
"dried_ghast" => Animate::DriedGhast {
|
||||
waterlogged: prop("waterlogged").as_deref() == Some("true"),
|
||||
},
|
||||
"end_portal" => Animate::EndPortal,
|
||||
"sculk_sensor" | "calibrated_sculk_sensor"
|
||||
if prop("sculk_sensor_phase").as_deref() == Some("active") =>
|
||||
{
|
||||
@@ -180,6 +188,8 @@ pub struct AnimateTables {
|
||||
impermeable: Vec<bool>,
|
||||
power_provider: Vec<bool>,
|
||||
power_transmitter: Vec<bool>,
|
||||
/// `#triggers_ambient_dried_ghast_block_sounds` (soul sand, soul soil).
|
||||
dried_ghast_sounds: Vec<bool>,
|
||||
/// Biome id -> ambient particles and probabilities (`minecraft:visual/ambient_particles`).
|
||||
ambient: Vec<Vec<(T, f32)>>,
|
||||
}
|
||||
@@ -237,6 +247,7 @@ impl AnimateTables {
|
||||
impermeable: by_state("minecraft:impermeable"),
|
||||
power_provider: by_state("minecraft:enchantment_power_provider"),
|
||||
power_transmitter: by_state("minecraft:enchantment_power_transmitter"),
|
||||
dried_ghast_sounds: by_state("minecraft:triggers_ambient_dried_ghast_block_sounds"),
|
||||
ambient,
|
||||
})
|
||||
}
|
||||
@@ -869,6 +880,45 @@ fn block_animate_tick(
|
||||
];
|
||||
add(engine, T::Smoke, p, [0.0; 3]);
|
||||
}
|
||||
Animate::DriedGhast { waterlogged } => {
|
||||
// DriedGhastBlock.animateTick.
|
||||
let c = [fx + 0.5, fy + 0.5, fz + 0.5];
|
||||
if waterlogged {
|
||||
if next_int(engine.random(), 40) == 0 {
|
||||
let name = "minecraft:block.dried_ghast.ambient_water";
|
||||
sound(engine, name, c, 1.0, 1.0);
|
||||
}
|
||||
let r = engine.random();
|
||||
if next_int(r, 6) == 0 {
|
||||
let x = c[0] + f64::from((r.next_float() * 2.0 - 1.0) / 3.0);
|
||||
let y = c[1] + 0.4;
|
||||
let z = c[2] + f64::from((r.next_float() * 2.0 - 1.0) / 3.0);
|
||||
let v = [0.0, f64::from(r.next_float()), 0.0];
|
||||
add(engine, T::HappyVillager, [x, y, z], v);
|
||||
}
|
||||
} else {
|
||||
let below = ctx.state([pos[0], pos[1] - 1, pos[2]]);
|
||||
let triggers = || {
|
||||
tables
|
||||
.dried_ghast_sounds
|
||||
.get(below.0 as usize)
|
||||
.copied()
|
||||
.unwrap_or(false)
|
||||
};
|
||||
if next_int(engine.random(), 40) == 0 && triggers() {
|
||||
sound(engine, "minecraft:block.dried_ghast.ambient", c, 1.0, 1.0);
|
||||
}
|
||||
if next_int(engine.random(), 6) == 0 {
|
||||
add(engine, T::WhiteSmoke, c, [0.0, 0.02, 0.0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
Animate::EndPortal => {
|
||||
// EndPortalBlock.animateTick: one smoke particle above the surface.
|
||||
let r = engine.random();
|
||||
let p = [fx + r.next_double(), fy + 0.8, fz + r.next_double()];
|
||||
add(engine, T::Smoke, p, [0.0; 3]);
|
||||
}
|
||||
Animate::EnderChest => {
|
||||
for _ in 0..3 {
|
||||
let r = engine.random();
|
||||
|
||||
@@ -20,6 +20,7 @@ mod bossbars;
|
||||
mod breezes;
|
||||
mod cartography;
|
||||
mod chat;
|
||||
mod client_ambient;
|
||||
mod clipboard;
|
||||
mod combat;
|
||||
mod command_blocks;
|
||||
@@ -4481,6 +4482,7 @@ impl Game<'_> {
|
||||
self.tick_block_entity_anims()
|
||||
});
|
||||
horizon_client::spans::time("tick_particles", || self.tick_particles());
|
||||
self.tick_client_ambient();
|
||||
}
|
||||
// ServerBossEvent broadcasts, after this level's entities have ticked.
|
||||
self.publish_boss_bars();
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
//! The window's own ambient effects of entities, which a client makes for
|
||||
//! itself (`level().isClientSide()` branches): endermen's and endermites'
|
||||
//! portal particles, blazes' smoke and burn, eyes of ender's trail. They run
|
||||
//! on the game that has the window, in-frame or following a server, never on
|
||||
//! a server: nothing of them goes over the network.
|
||||
use super::Game;
|
||||
use horizon_client::mobs::{MobEvent, ambient};
|
||||
use horizon_client::particles::{ParticleOptions, ParticleType};
|
||||
|
||||
impl Game<'_> {
|
||||
pub(super) fn tick_client_ambient(&mut self) {
|
||||
if self.serving.is_some() || self.options.dedicated.is_some() {
|
||||
return;
|
||||
}
|
||||
let mut events = Vec::new();
|
||||
for m in self.mobs.iter_mut().filter(|m| !m.removed) {
|
||||
ambient::client_ai_step(m, &mut events);
|
||||
}
|
||||
for event in events {
|
||||
match event {
|
||||
MobEvent::ParticleAt { kind, at, velocity } => {
|
||||
if let Some(ty) = ParticleType::from_name(kind) {
|
||||
self.particles
|
||||
.add(&ParticleOptions::simple(ty), at, velocity);
|
||||
}
|
||||
}
|
||||
// Each guest of a LAN host makes its own.
|
||||
MobEvent::LocalSound(name, at, volume, pitch) => {
|
||||
self.audio
|
||||
.play_unshared(&format!("minecraft:{name}"), Some(at), volume, pitch)
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
self.eye_particles();
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,12 @@
|
||||
//! inactive until the dragon dies.
|
||||
use super::{Dim, Game};
|
||||
use horizon_client::{
|
||||
Result, environment::Environment, items::ItemStack, mesher::Vertex, motion::PlayerMotion,
|
||||
Result,
|
||||
environment::Environment,
|
||||
items::ItemStack,
|
||||
mesher::Vertex,
|
||||
motion::PlayerMotion,
|
||||
particles::{ParticleOptions, ParticleType},
|
||||
vanillagen::VanillaTerrain,
|
||||
};
|
||||
use horizon_core::{
|
||||
@@ -1122,6 +1127,35 @@ impl Game<'_> {
|
||||
self.end.eyes = eyes;
|
||||
}
|
||||
|
||||
/// `EyeOfEnder.tick` on a client: behind each eye (a quarter of its last
|
||||
/// move back), four bubbles in water, else a portal particle.
|
||||
pub(super) fn eye_particles(&mut self) {
|
||||
for i in 0..self.end.eyes.len() {
|
||||
let e = &self.end.eyes[i];
|
||||
let movement = (e.position - e.previous).to_array();
|
||||
let origin = (e.position - (e.position - e.previous) * 0.25).to_array();
|
||||
let block = origin.map(|v| v.floor() as i32);
|
||||
let water = self
|
||||
.res
|
||||
.facts()
|
||||
.get(self.world.block(block[0], block[1], block[2]))
|
||||
.fluid
|
||||
== horizon_client::blocks::Fluid::Water;
|
||||
if water {
|
||||
let bubble = ParticleOptions::simple(ParticleType::Bubble);
|
||||
for _ in 0..4 {
|
||||
self.particles.add(&bubble, origin, movement);
|
||||
}
|
||||
} else {
|
||||
let r = self.particles.random();
|
||||
let x = origin[0] + r.next_double() * 0.6 - 0.3;
|
||||
let z = origin[2] + r.next_double() * 0.6 - 0.3;
|
||||
let portal = ParticleOptions::simple(ParticleType::Portal);
|
||||
self.particles
|
||||
.add(&portal, [x, origin[1] - 0.5, z], movement);
|
||||
}
|
||||
}
|
||||
}
|
||||
/// ThrownItemRenderer: the ender eye item facing the camera.
|
||||
fn eye_geometry(&self, camera: glam::DVec3, out: &mut Vec<Vertex>) {
|
||||
let Some(r) = self.gui.atlas.sprites.get("item/ender_eye").copied() else {
|
||||
|
||||
@@ -42,6 +42,11 @@ pub(super) struct ModsState {
|
||||
/// The chat's command tree while a running mod owns commands.
|
||||
#[cfg(feature = "mods")]
|
||||
tree: Option<Dispatcher>,
|
||||
/// `command.run` lines for this window's own world, run on the next tick:
|
||||
/// run at once, their chat lines would reach the mods while their output
|
||||
/// is still being shown, and a mod answering its own lines would recurse.
|
||||
#[cfg(feature = "mods")]
|
||||
commands: Vec<String>,
|
||||
/// The server's command tree with the running mods' commands added.
|
||||
#[cfg(feature = "mods")]
|
||||
remote_tree: Option<Dispatcher>,
|
||||
@@ -337,6 +342,11 @@ impl Game<'_> {
|
||||
}
|
||||
/// A game tick.
|
||||
pub(super) fn tick_mods(&mut self) {
|
||||
// Here, at the player's level: shown inside a console call (a control
|
||||
// socket command set the hook off), they would run as the console.
|
||||
for line in std::mem::take(&mut self.mods.commands) {
|
||||
self.run_command(&line);
|
||||
}
|
||||
if !self.mods.mods.is_running() {
|
||||
return;
|
||||
}
|
||||
@@ -396,6 +406,19 @@ impl Game<'_> {
|
||||
}
|
||||
Output::Feedback { text, error: false } => Text::literal(text),
|
||||
Output::Feedback { text, error: true } => Text::literal(text).with_color(text::RED),
|
||||
// Vanilla's commands only: a mod running a mod command could loop.
|
||||
Output::Command { line } => {
|
||||
let name = line.split(' ').next().unwrap_or_default();
|
||||
if !vanilla_command(name) {
|
||||
// Dropped.
|
||||
} else if self.remote.is_some() {
|
||||
// The server checks the player's permissions.
|
||||
self.send_chat(&format!("/{line}"));
|
||||
} else {
|
||||
self.mods.commands.push(line);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Output::Stopped { name, reason } => {
|
||||
// Its screen goes with it: the entry's run is gone, so
|
||||
// `mod_ui_view` finds no widgets and the screen closes.
|
||||
|
||||
@@ -3060,6 +3060,7 @@ impl Game<'_> {
|
||||
self.tick_block_entity_anims();
|
||||
self.tick_hand();
|
||||
self.tick_particles();
|
||||
self.tick_client_ambient();
|
||||
self.tick_client_effects();
|
||||
// Player.tick's sleep counter (the darkening and the wake-up fade).
|
||||
self.sleep_timer = super::tick_sleep_timer(self.sleep_timer, self.sleeping.is_some());
|
||||
|
||||
@@ -385,14 +385,18 @@ impl Graphics {
|
||||
if self.workers > 0 {
|
||||
return self.workers as usize;
|
||||
}
|
||||
// The pool never takes every core. The render thread and the
|
||||
// integrated server thread (single player runs on its own server
|
||||
// thread by default) each need one, and a pool that fills every core
|
||||
// starves them: the frame rate collapses to single digits while the
|
||||
// workers saturate the CPU and its memory bandwidth.
|
||||
// Vanilla's count (Util.maxAllowedExecutorThreads: every core but
|
||||
// one). The pool never takes every core: a pool that fills them all
|
||||
// starves the render and server threads, and the frame rate collapses
|
||||
// while the workers saturate the CPU and its memory bandwidth. One
|
||||
// core left is enough with the workers behind those two threads
|
||||
// (horizon_threads: niced on Linux; on macOS and Windows the server
|
||||
// and render threads are raised above four cores, where vanilla
|
||||
// raises its server thread). Leaving two idled a core of a 4-thread
|
||||
// laptop while a world loaded: a teleport there took ~15% longer.
|
||||
let auto = std::thread::available_parallelism()
|
||||
.map_or(2, |n| n.get())
|
||||
.saturating_sub(2)
|
||||
.saturating_sub(1)
|
||||
.max(1);
|
||||
if self.battery { auto.div_ceil(2) } else { auto }
|
||||
}
|
||||
@@ -555,11 +559,11 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automatic_workers_leave_cores_for_the_render_and_server_threads() {
|
||||
fn automatic_workers_leave_a_core_as_vanilla_does() {
|
||||
let g = Graphics::default();
|
||||
assert_eq!(g.workers, 0);
|
||||
let cores = std::thread::available_parallelism().map_or(2, |n| n.get());
|
||||
assert_eq!(g.worker_threads(), cores.saturating_sub(2).max(1));
|
||||
assert_eq!(g.worker_threads(), cores.saturating_sub(1).max(1));
|
||||
// A manual count is the player's own choice, kept as asked.
|
||||
let manual = Graphics { workers: 3, ..g };
|
||||
assert_eq!(manual.worker_threads(), 3);
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
//! behaviour, models, animations and sounds.
|
||||
pub mod ai;
|
||||
pub mod allay;
|
||||
pub mod ambient;
|
||||
pub mod anim;
|
||||
pub mod animal_brain;
|
||||
pub mod aquatic;
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
//! What a client's copy of a mob makes on its own, in the
|
||||
//! `level().isClientSide()` branches of `aiStep`: nothing of it reaches the
|
||||
//! server or goes over the network, so the window's game runs it, whether it
|
||||
//! simulates the world (in-frame) or follows a server.
|
||||
use super::{Mob, MobEvent, Species};
|
||||
use horizon_core::random::RandomSource;
|
||||
|
||||
/// `Enderman.aiStep`, `Endermite.aiStep` and `Blaze.aiStep` on a client:
|
||||
/// two portal particles a tick around endermen and endermites; two large
|
||||
/// smoke particles a tick around blazes and, one tick in 24, their burn.
|
||||
pub fn client_ai_step(m: &mut Mob, out: &mut Vec<MobEvent>) {
|
||||
let species = m.species();
|
||||
if !matches!(
|
||||
species,
|
||||
Species::Enderman | Species::Endermite | Species::Blaze
|
||||
) {
|
||||
return;
|
||||
}
|
||||
let (w, h) = (f64::from(m.width()), f64::from(m.height()));
|
||||
let p = m.position;
|
||||
if species == Species::Blaze && m.random.next_int_bound(24).unwrap_or(1) == 0 && !m.silent {
|
||||
let volume = 1.0 + m.random.next_float();
|
||||
let pitch = m.random.next_float() * 0.7 + 0.3;
|
||||
out.push(MobEvent::LocalSound(
|
||||
"entity.blaze.burn",
|
||||
[p.x + 0.5, p.y + 0.5, p.z + 0.5],
|
||||
volume,
|
||||
pitch,
|
||||
));
|
||||
}
|
||||
for _ in 0..2 {
|
||||
// Entity.getRandomX(0.5), getRandomY(), getRandomZ(0.5).
|
||||
let r = &mut m.random;
|
||||
let x = p.x + w * (2.0 * r.next_double() - 1.0) * 0.5;
|
||||
let y = p.y + h * r.next_double();
|
||||
let z = p.z + w * (2.0 * r.next_double() - 1.0) * 0.5;
|
||||
let (kind, at, velocity) = if species == Species::Blaze {
|
||||
("large_smoke", [x, y, z], [0.0; 3])
|
||||
} else {
|
||||
// The enderman's are a quarter block lower.
|
||||
let y = if species == Species::Enderman {
|
||||
y - 0.25
|
||||
} else {
|
||||
y
|
||||
};
|
||||
let velocity = [
|
||||
(r.next_double() - 0.5) * 2.0,
|
||||
-r.next_double(),
|
||||
(r.next_double() - 0.5) * 2.0,
|
||||
];
|
||||
("portal", [x, y, z], velocity)
|
||||
};
|
||||
out.push(MobEvent::ParticleAt { kind, at, velocity });
|
||||
}
|
||||
}
|
||||
@@ -2148,8 +2148,13 @@ mod cover_tests {
|
||||
}
|
||||
|
||||
/// Every question streaming asks gets the same answer with the grid.
|
||||
///
|
||||
/// A debug build checks every fourth cell of each row, shifted by row and case so the cases still cover every
|
||||
/// cell's position relative to the areas: the full check took 20 of the ~30 s of the crate's tests.
|
||||
/// `cargo test --release -p horizon-client --lib the_grid_answers_as_the_scan` checks every cell.
|
||||
#[test]
|
||||
fn the_grid_answers_as_the_scan() {
|
||||
let stride = if cfg!(debug_assertions) { 4 } else { 1 };
|
||||
let mut seed = 0x9E37_79B9_7F4A_7C15u64;
|
||||
let forced: HashSet<(i32, i32)> = [(40, -40)].into();
|
||||
for case in 0..60 {
|
||||
@@ -2165,7 +2170,8 @@ mod cover_tests {
|
||||
let (scan, grid) = (ranges(None), ranges(Some(&cover)));
|
||||
let span = [4, 30, 120][case % 3] + 40;
|
||||
for z in -span..=span {
|
||||
for x in -span..=span {
|
||||
let first = -span + (z + case as i32).rem_euclid(stride);
|
||||
for x in (first..=span).step_by(stride as usize) {
|
||||
for m in 0..=ExtraCover::REACH + 2 {
|
||||
assert_eq!(
|
||||
scan.in_extra(x, z, m),
|
||||
|
||||
@@ -106,14 +106,16 @@ enum Palette {
|
||||
List(Vec<u32>),
|
||||
Global,
|
||||
}
|
||||
/// `n / per_word` for `n < 4096` by multiplication: `MAGIC[bits] = ceil(2^20 /
|
||||
/// (64 / bits))` is exact while `n * (64 / bits) < 2^20`.
|
||||
const MAGIC: [u32; 33] = {
|
||||
let mut table = [0; 33];
|
||||
/// `n / per_word` for `n < 4096` by multiplication: `MAGIC[bits].0 = ceil(2^20 /
|
||||
/// (64 / bits))` is exact while `n * (64 / bits) < 2^20`. `.1` is the bits a
|
||||
/// word's entries fill (`64 / bits * bits`): looked up, not divided for, as a
|
||||
/// division by a run-time value costs tens of cycles on x86.
|
||||
const MAGIC: [(u32, u32); 33] = {
|
||||
let mut table = [(0, 0); 33];
|
||||
let mut bits = 1;
|
||||
while bits <= 32 {
|
||||
let per = 64 / bits as u32;
|
||||
table[bits] = (1u32 << 20).div_ceil(per);
|
||||
table[bits] = ((1u32 << 20).div_ceil(per), per * bits as u32);
|
||||
bits += 1;
|
||||
}
|
||||
table
|
||||
@@ -134,9 +136,9 @@ fn locate(bits: u8, i: usize) -> (usize, u32) {
|
||||
let word = i >> (6 - log);
|
||||
(word, ((i & ((64 >> log) - 1)) as u32) << log)
|
||||
} else {
|
||||
let per = 64 / u32::from(bits);
|
||||
let word = (i as u32 * MAGIC[usize::from(bits)]) >> 20;
|
||||
(word as usize, (i as u32 - word * per) * u32::from(bits))
|
||||
let (magic, filled) = MAGIC[usize::from(bits)];
|
||||
let word = (i as u32 * magic) >> 20;
|
||||
(word as usize, i as u32 * u32::from(bits) - word * filled)
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
@@ -720,6 +722,17 @@ mod tests {
|
||||
use super::*;
|
||||
const BLOCKS: Strategy = Strategy::blocks(16);
|
||||
const BIOMES: Strategy = Strategy::biomes(7);
|
||||
|
||||
#[test]
|
||||
fn locate_matches_division_for_every_width() {
|
||||
for bits in 1..=32u8 {
|
||||
let per = 64 / usize::from(bits);
|
||||
for i in 0..4096 {
|
||||
let expected = (i / per, ((i % per) * usize::from(bits)) as u32);
|
||||
assert_eq!(locate(bits, i), expected, "{bits} bits, entry {i}");
|
||||
}
|
||||
}
|
||||
}
|
||||
fn rng(state: &mut u64) -> u64 {
|
||||
*state ^= *state << 13;
|
||||
*state ^= *state >> 7;
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "predator-missile"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
publish = false
|
||||
description = "Example Horizon mod: a missile you launch with /missile and steer with the mouse"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
horizon-mod-sdk = { path = "../.." }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,8 @@
|
||||
id = "predator_missile"
|
||||
name = "Predator Missile"
|
||||
version = "0.1.0"
|
||||
description = "/missile launches a guided missile 60 blocks above you. Steer it with the mouse; it explodes on whatever it hits. Needs cheats (it runs vanilla commands as you)."
|
||||
authors = ["Horizon"]
|
||||
api = "0.1"
|
||||
side = "client"
|
||||
capabilities = ["game-read", "hud", "chat", "command"]
|
||||
@@ -0,0 +1,161 @@
|
||||
//! Example Horizon mod (`docs/mods.md`): `/missile` turns you into a guided
|
||||
//! missile 60 blocks up. You fly along your own look, so the mouse steers;
|
||||
//! on a block or a mob a stack of primed TNT goes off, and you are put back
|
||||
//! where you launched. Built from vanilla commands (capability `command`),
|
||||
//! so it needs cheats.
|
||||
use horizon_mod_sdk::{
|
||||
Mod, Registration,
|
||||
command::run,
|
||||
game::{self, Player},
|
||||
hud::{self, Corner, Line},
|
||||
};
|
||||
|
||||
/// Blocks per tick (a fired arrow does about 3).
|
||||
const SPEED: f64 = 1.5;
|
||||
const ALTITUDE: i32 = 60;
|
||||
/// Ticks of flight before the missile detonates on its own.
|
||||
const FUEL: u32 = 30 * 20;
|
||||
/// Ticks to watch the blast before going back.
|
||||
const AFTERMATH: u32 = 40;
|
||||
/// Primed TNT per blast: the stack's explosions add up.
|
||||
const WARHEAD: usize = 4;
|
||||
/// What the impact checks print; the mod hears it through the chat hook.
|
||||
const IMPACT: &str = "Predator: impact";
|
||||
|
||||
enum State {
|
||||
Idle,
|
||||
Flying { from: Player, ticks: u32 },
|
||||
Aftermath { from: Player, ticks: u32 },
|
||||
}
|
||||
|
||||
struct Predator {
|
||||
state: State,
|
||||
}
|
||||
|
||||
fn launch() {
|
||||
run("gamerule send_command_feedback false");
|
||||
run("execute if entity @s[gamemode=creative] run tag @s add pm_creative");
|
||||
run("gamemode spectator");
|
||||
run(&format!("tp @s ~ ~{ALTITUDE} ~ ~ 60"));
|
||||
run("effect give @s night_vision 60 0 true");
|
||||
// The thermal camera: every mob outlined.
|
||||
run("effect give @e[type=!player,distance=..256] glowing 60 0 true");
|
||||
}
|
||||
|
||||
/// One tick of flight: stop at the surface (the next step would enter a
|
||||
/// block or a mob), else move one step along the look.
|
||||
fn fly() {
|
||||
run(&format!(
|
||||
"execute at @s unless block ^ ^ ^{SPEED} #minecraft:air run tellraw @s \"{IMPACT}\""
|
||||
));
|
||||
run(&format!(
|
||||
"execute at @s positioned ^ ^ ^{SPEED} if entity @e[type=!player,distance=..2] run tellraw @s \"{IMPACT}\""
|
||||
));
|
||||
run(&format!(
|
||||
"execute at @s if block ^ ^ ^{SPEED} #minecraft:air run tp @s ^ ^ ^{SPEED}"
|
||||
));
|
||||
}
|
||||
|
||||
fn detonate() {
|
||||
for _ in 0..WARHEAD {
|
||||
run("execute at @s run summon tnt ^ ^ ^1 {fuse:0s}");
|
||||
}
|
||||
}
|
||||
|
||||
fn land(from: &Player) {
|
||||
run(&format!(
|
||||
"tp @s {} {} {} {} {}",
|
||||
from.x, from.y, from.z, from.yaw, from.pitch
|
||||
));
|
||||
run("execute if entity @s[tag=pm_creative] run gamemode creative");
|
||||
run("execute unless entity @s[tag=pm_creative] run gamemode survival");
|
||||
run("execute if entity @s[tag=pm_creative] run tag @s remove pm_creative");
|
||||
run("effect clear @s night_vision");
|
||||
// shortcut: assumes feedback was on before launch, as in a new world.
|
||||
run("gamerule send_command_feedback true");
|
||||
}
|
||||
|
||||
fn panel(p: &Player, ticks: u32) -> Vec<Line> {
|
||||
let green = 0x7CFF7C;
|
||||
let left = (FUEL - ticks.min(FUEL)) / 20;
|
||||
[
|
||||
"PREDATOR".to_string(),
|
||||
format!("ALT {:.0}", p.y),
|
||||
format!("FUEL {left}s"),
|
||||
"Mouse to steer, /missile to detonate".to_string(),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|text| Line { text, color: green })
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Predator {
|
||||
fn impact(&mut self) {
|
||||
if let State::Flying { from, .. } = &self.state {
|
||||
detonate();
|
||||
self.state = State::Aftermath {
|
||||
from: from.clone(),
|
||||
ticks: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Mod for Predator {
|
||||
fn init() -> Result<(Self, Registration), String> {
|
||||
let hooks = Registration::new().command("missile").on_tick().on_chat();
|
||||
Ok((Predator { state: State::Idle }, hooks))
|
||||
}
|
||||
|
||||
fn tick(&mut self, _tick: u64) {
|
||||
match &mut self.state {
|
||||
State::Idle => {}
|
||||
State::Flying { ticks, .. } => {
|
||||
*ticks += 1;
|
||||
let ticks = *ticks;
|
||||
if ticks >= FUEL {
|
||||
return self.impact();
|
||||
}
|
||||
// The first tick waits for the launch teleport.
|
||||
if ticks > 1 {
|
||||
fly();
|
||||
}
|
||||
if let Some(p) = game::get_player() {
|
||||
hud::set_panel(Corner::TopLeft, &panel(&p, ticks));
|
||||
}
|
||||
}
|
||||
State::Aftermath { from, ticks } => {
|
||||
*ticks += 1;
|
||||
if *ticks >= AFTERMATH {
|
||||
land(from);
|
||||
hud::clear();
|
||||
self.state = State::Idle;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn chat(&mut self, text: &str) {
|
||||
if text.contains(IMPACT) {
|
||||
self.impact();
|
||||
}
|
||||
}
|
||||
|
||||
fn command(&mut self, _name: &str, _args: &str) -> Result<Option<String>, String> {
|
||||
match self.state {
|
||||
State::Idle => {
|
||||
let from = game::get_player().ok_or("Not in a world")?;
|
||||
launch();
|
||||
self.state = State::Flying { from, ticks: 0 };
|
||||
Ok(Some("Missile away".into()))
|
||||
}
|
||||
State::Flying { .. } => {
|
||||
self.impact();
|
||||
Ok(None)
|
||||
}
|
||||
State::Aftermath { .. } => Err("Wait for the blast to clear".into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
horizon_mod_sdk::export_mod!(Predator);
|
||||
@@ -21,7 +21,7 @@
|
||||
//! ```
|
||||
//!
|
||||
//! Every interface but `log` is a capability: list the ones the mod uses in
|
||||
//! `horizon-mod.toml` (`game-read`, `chat`, `hud`, `storage`, `ui`).
|
||||
//! `horizon-mod.toml` (`game-read`, `chat`, `hud`, `storage`, `command`, `ui`).
|
||||
//!
|
||||
//! Widgets (capability `ui`): a mod shows a screen it owns with
|
||||
//! [`ui::open_screen`] and a page in Options > Mods with
|
||||
@@ -38,7 +38,7 @@ wit_bindgen::generate!({
|
||||
|
||||
pub use exports::horizon::mods::events::{Guest as __Guest, Registration};
|
||||
pub use exports::horizon::mods::ui_events::Guest as __UiEventsGuest;
|
||||
pub use horizon::mods::{chat, game, hud, log, storage, ui};
|
||||
pub use horizon::mods::{chat, command, game, hud, log, storage, ui};
|
||||
|
||||
/// The game's calls into a mod. Only the hooks [`Registration`] asks for
|
||||
/// are called; the others may stay as their defaults.
|
||||
|
||||
@@ -86,6 +86,14 @@ interface storage {
|
||||
delete: func(key: string);
|
||||
}
|
||||
|
||||
/// Capability `command`: vanilla commands run as the player, with the
|
||||
/// player's permissions, after the hook call returns (8 per call). A mod's
|
||||
/// own commands are not run this way.
|
||||
interface command {
|
||||
/// `line` without its slash; its feedback goes to the chat as if typed.
|
||||
run: func(line: string);
|
||||
}
|
||||
|
||||
/// Capability `ui`: widgets the game draws with its own menus' widgets, for
|
||||
/// a screen the mod owns and a settings page in Options > Mods. Layout and
|
||||
/// drawing are the game's: a mod says what its widgets hold, never where
|
||||
@@ -194,6 +202,7 @@ world horizon-mod {
|
||||
import chat;
|
||||
import hud;
|
||||
import storage;
|
||||
import command;
|
||||
export events;
|
||||
}
|
||||
|
||||
|
||||
@@ -78,6 +78,8 @@ pub enum Capability {
|
||||
Chat,
|
||||
Hud,
|
||||
Storage,
|
||||
/// Runs vanilla commands as the player, with the player's permissions.
|
||||
Command,
|
||||
/// New blocks and items from the mod's `data/` (API 0.2): no WIT
|
||||
/// interface, the game reads them before the world loads.
|
||||
Registries,
|
||||
@@ -86,11 +88,12 @@ pub enum Capability {
|
||||
Ui,
|
||||
}
|
||||
impl Capability {
|
||||
pub const ALL: [Capability; 6] = [
|
||||
pub const ALL: [Capability; 7] = [
|
||||
Capability::GameRead,
|
||||
Capability::Chat,
|
||||
Capability::Hud,
|
||||
Capability::Storage,
|
||||
Capability::Command,
|
||||
Capability::Registries,
|
||||
Capability::Ui,
|
||||
];
|
||||
@@ -101,6 +104,7 @@ impl Capability {
|
||||
Capability::Chat => "chat",
|
||||
Capability::Hud => "hud",
|
||||
Capability::Storage => "storage",
|
||||
Capability::Command => "command",
|
||||
Capability::Registries => "registries",
|
||||
Capability::Ui => "ui",
|
||||
}
|
||||
@@ -117,6 +121,7 @@ impl Capability {
|
||||
Capability::Chat => "show you chat messages",
|
||||
Capability::Hud => "draw text on your screen",
|
||||
Capability::Storage => "save its own settings to disk",
|
||||
Capability::Command => "run commands as you, with your permissions",
|
||||
Capability::Registries => "add blocks and items to your worlds",
|
||||
Capability::Ui => "show its own screens and a settings page",
|
||||
}
|
||||
@@ -148,6 +153,7 @@ impl fmt::Display for Capability {
|
||||
Capability::Chat => "chat",
|
||||
Capability::Hud => "hud",
|
||||
Capability::Storage => "storage",
|
||||
Capability::Command => "command",
|
||||
Capability::Registries => "registries",
|
||||
Capability::Ui => "ui",
|
||||
};
|
||||
@@ -397,6 +403,10 @@ capabilities = ["game-read", "hud"]
|
||||
storage,
|
||||
Err("uses capability storage, missing from horizon-mod.toml".into())
|
||||
);
|
||||
assert_eq!(
|
||||
m.check_imports(["horizon:mods/command@0.1.0"]),
|
||||
Err("uses capability command, missing from horizon-mod.toml".into())
|
||||
);
|
||||
let with_storage = EXAMPLE.replace("\"hud\"", "\"hud\", \"storage\"");
|
||||
let m = Manifest::parse(&with_storage).unwrap();
|
||||
assert_eq!(m.check_imports(["horizon:mods/storage@0.1.0"]), Ok(()));
|
||||
|
||||
@@ -32,7 +32,7 @@ pub use bindings::horizon::mods::{
|
||||
use bindings::{
|
||||
HorizonMod,
|
||||
exports::horizon::mods::events::Registration,
|
||||
horizon::mods::{chat, game, hud, log as mod_log, storage},
|
||||
horizon::mods::{chat, command, game, hud, log as mod_log, storage},
|
||||
};
|
||||
/// The widget model a `ui` screen or settings page shows (WIT `interface
|
||||
/// ui`), for the game to draw with its menus' own widgets.
|
||||
@@ -79,6 +79,9 @@ pub enum Output {
|
||||
Chat { name: String, text: String },
|
||||
/// A mod command's feedback, red when `error`.
|
||||
Feedback { text: String, error: bool },
|
||||
/// `command.run`: a vanilla command line, without its slash, to run as
|
||||
/// the player.
|
||||
Command { line: String },
|
||||
/// The mod was turned off: it trapped, overran or failed to start.
|
||||
Stopped { name: String, reason: String },
|
||||
/// `ui.open-screen`: the mod's entry in `entries()`; what it shows is
|
||||
@@ -668,6 +671,18 @@ impl chat::Host for State {
|
||||
}
|
||||
}
|
||||
}
|
||||
impl command::Host for State {
|
||||
fn run(&mut self, line: String) {
|
||||
let queued = self
|
||||
.out
|
||||
.iter()
|
||||
.filter(|o| matches!(o, Output::Command { .. }))
|
||||
.count();
|
||||
if queued < CHAT_PER_CALL {
|
||||
self.out.push(Output::Command { line: clip(line) });
|
||||
}
|
||||
}
|
||||
}
|
||||
impl hud::Host for State {
|
||||
fn set_panel(&mut self, corner: Corner, lines: Vec<Line>) {
|
||||
let lines = lines
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
//! Loads real mods built with the SDK: the coords-hud example and a guest
|
||||
//! Loads real mods built with the SDK: the coords-hud and predator-missile
|
||||
//! examples and a guest
|
||||
//! that misbehaves on command. Needs `rustup target add wasm32-unknown-unknown`.
|
||||
use horizon_mods::{
|
||||
Clock, Corner, Mods, Output, Package, Player, Snapshot, Status, manifest::componentize,
|
||||
@@ -26,7 +27,7 @@ capabilities = ["chat", "ui"]
|
||||
experimental = ["ui"]
|
||||
"#;
|
||||
|
||||
/// Builds both mods once and writes their `.hmod` files; returns the folder.
|
||||
/// Builds the mods once and writes their `.hmod` files; returns the folder.
|
||||
fn mods_dir() -> &'static Path {
|
||||
static DIR: OnceLock<PathBuf> = OnceLock::new();
|
||||
DIR.get_or_init(|| {
|
||||
@@ -39,6 +40,8 @@ fn mods_dir() -> &'static Path {
|
||||
"-p",
|
||||
"coords-hud",
|
||||
"-p",
|
||||
"predator-missile",
|
||||
"-p",
|
||||
"horizon-mods-test-guest",
|
||||
"-p",
|
||||
"horizon-mods-test-ui-guest",
|
||||
@@ -54,9 +57,13 @@ fn mods_dir() -> &'static Path {
|
||||
let built = target.join("wasm32-unknown-unknown/release");
|
||||
let dir = target.join("mods");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let example = root.join("crates/horizon-mod-sdk/examples/coords-hud/horizon-mod.toml");
|
||||
let example = |name: &str| {
|
||||
let path = format!("crates/horizon-mod-sdk/examples/{name}/horizon-mod.toml");
|
||||
std::fs::read_to_string(root.join(path)).unwrap()
|
||||
};
|
||||
let mods = [
|
||||
("coords_hud", std::fs::read_to_string(example).unwrap()),
|
||||
("coords_hud", example("coords-hud")),
|
||||
("predator_missile", example("predator-missile")),
|
||||
("horizon_mods_test_guest", GUEST_MANIFEST.to_string()),
|
||||
("horizon_mods_test_ui_guest", UI_MANIFEST.to_string()),
|
||||
];
|
||||
@@ -92,6 +99,36 @@ fn player() -> Snapshot {
|
||||
}
|
||||
}
|
||||
|
||||
/// Capability `command`: `/missile` queues vanilla commands, and the chat
|
||||
/// line its impact check prints sets off the warhead.
|
||||
#[test]
|
||||
fn predator_missile_runs_commands() {
|
||||
let mut mods = load("predator_missile.hmod");
|
||||
assert_eq!(mods.entries()[0].status, Status::Running);
|
||||
mods.set_snapshot(player());
|
||||
let commands = |mods: &mut Mods| -> Vec<String> {
|
||||
mods.drain()
|
||||
.into_iter()
|
||||
.filter_map(|o| match o {
|
||||
Output::Command { line } => Some(line),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
assert!(mods.command("missile"));
|
||||
let launch = commands(&mut mods);
|
||||
assert_eq!(launch.len(), 6);
|
||||
assert!(launch.contains(&"gamemode spectator".to_string()));
|
||||
mods.tick(1);
|
||||
mods.tick(2);
|
||||
let flight = commands(&mut mods);
|
||||
assert!(flight.iter().any(|l| l.ends_with("run tp @s ^ ^ ^1.5")));
|
||||
mods.chat("Predator: impact");
|
||||
let blast = commands(&mut mods);
|
||||
assert_eq!(blast.len(), 4);
|
||||
assert!(blast.iter().all(|l| l.contains("summon tnt")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coords_hud_draws_and_answers() {
|
||||
let mut mods = load("coords_hud.hmod");
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
# OUTPUT_VERSION (src/lib.rs) and the digest of what it prepares (python3 tools/prepare_output.py digest),
|
||||
# one line per version, never edited: a change to the prepared files appends the next version.
|
||||
1 0a608bf4115a8ed2e2c4f5f2b2609dc10c54e63479af8e56fabaa10af1060389
|
||||
@@ -31,6 +31,15 @@ pub type Log<'a> = &'a dyn Fn(&str);
|
||||
/// The manifest's `schema`: the prepared files' layout, which the launcher compares.
|
||||
pub const SCHEMA: i64 = 7;
|
||||
|
||||
/// What this preparer makes. The launcher copies a prepared version's resources to a newly installed
|
||||
/// version whose preparer has the same number (and the same pinned jars), so a release that changes
|
||||
/// this crate without changing its output asks nobody to prepare again. Bump it, and append the new
|
||||
/// digest to output-versions.txt, when the prepared files change: CI prepares with every change to
|
||||
/// the preparer and fails when the output is not the one output-versions.txt gives this number
|
||||
/// (`python3 tools/prepare_output.py check`). The launcher reads this line from the version
|
||||
/// folder's sources: keep it on one line, as it is.
|
||||
pub const OUTPUT_VERSION: u32 = 1;
|
||||
|
||||
/// Digest of this crate's sources (build.rs): the launcher prepares a version in-process when
|
||||
/// that version ships these same sources, and runs the version's own binary otherwise.
|
||||
pub const SOURCES_SHA256: &str = env!("HORIZON_PREPARE_SOURCES_SHA256");
|
||||
@@ -1209,6 +1218,29 @@ mod tests {
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_versions_end_at_the_output_version() {
|
||||
let text = include_str!("../output-versions.txt");
|
||||
let versions: Vec<(u32, &str)> = text
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
|
||||
.map(|l| {
|
||||
let (n, digest) = l.split_once(' ').unwrap();
|
||||
(n.parse().unwrap(), digest.trim())
|
||||
})
|
||||
.collect();
|
||||
let numbers: Vec<u32> = versions.iter().map(|v| v.0).collect();
|
||||
assert_eq!(numbers, (1..=OUTPUT_VERSION).collect::<Vec<_>>());
|
||||
for (i, (_, digest)) in versions.iter().enumerate() {
|
||||
assert_eq!(digest.len(), 64);
|
||||
assert!(!versions[..i].iter().any(|(_, d)| d == digest));
|
||||
}
|
||||
// The launcher finds the number with this line in a version folder's sources.
|
||||
let lib = include_str!("lib.rs");
|
||||
let line = format!("pub const OUTPUT_VERSION: u32 = {OUTPUT_VERSION};");
|
||||
assert!(lib.lines().any(|l| l == line));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_source_digest_is_the_one_build_rs_computed() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -11,19 +11,20 @@ The contract between Horizon and its mods: the WIT package `horizon:mods@0.1.0`,
|
||||
| `chat` | [`chat`](#chat) | show you chat messages |
|
||||
| `hud` | [`hud`](#hud) | draw text on your screen |
|
||||
| `storage` | [`storage`](#storage) | save its own settings to disk |
|
||||
| `command` | [`command`](#command) | run commands as you, with your permissions |
|
||||
| `registries` | data files (`assets/`, `data/`), no interface | add blocks and items to your worlds |
|
||||
| `ui` | [`ui`](#ui) | show its own screens and a settings page |
|
||||
|
||||
## World `horizon-mod`
|
||||
|
||||
- Imports (what the game offers): `log`, `game`, `chat`, `hud`, `storage`
|
||||
- Imports (what the game offers): `log`, `game`, `chat`, `hud`, `storage`, `command`
|
||||
- Exports (what the mod provides): `events`
|
||||
|
||||
## World `horizon-ui-mod`
|
||||
|
||||
A mod with widgets (capability `ui`): the SDK builds every mod against this world. Mods built for `horizon-mod` alone (no `ui-events` export) still load and simply hear nothing of their widgets.
|
||||
|
||||
- Imports (what the game offers): `ui`, `log`, `game`, `chat`, `hud`, `storage`
|
||||
- Imports (what the game offers): `ui`, `log`, `game`, `chat`, `hud`, `storage`, `command`
|
||||
- Exports (what the mod provides): `ui-events`, `events`
|
||||
|
||||
## World `ui-host`
|
||||
@@ -132,6 +133,16 @@ Capability `storage`: a per-mod folder of small key/value files, quota'd (docs/m
|
||||
| `set` | `func(key: string, value: list<u8>) -> result<_, string>` | Errs on a malformed key or over quota (bytes for the whole mod). |
|
||||
| `delete` | `func(key: string)` | No error if `key` was already absent. |
|
||||
|
||||
## `command`
|
||||
|
||||
Access: capability `command`.
|
||||
|
||||
Capability `command`: vanilla commands run as the player, with the player's permissions, after the hook call returns (8 per call). A mod's own commands are not run this way.
|
||||
|
||||
| Function | Signature | What it does |
|
||||
| --- | --- | --- |
|
||||
| `run` | `func(line: string)` | `line` without its slash; its feedback goes to the chat as if typed. |
|
||||
|
||||
## `ui`
|
||||
|
||||
Access: capability `ui`.
|
||||
|
||||
@@ -86,7 +86,7 @@ Deprecations and breaking changes are in bold. The release notes of the game rel
|
||||
|
||||
| Part | Level | Version |
|
||||
| --- | --- | --- |
|
||||
| Mod API: `log`, `game`, `chat`, `hud`, `storage`, `events` | preview | 0.1.0 |
|
||||
| Mod API: `log`, `game`, `chat`, `hud`, `storage`, `command`, `events` | preview | 0.1.0 |
|
||||
| Mod API: `ui`, `ui-events` (a mod's screens and settings page) | experimental | manifest `experimental = ["ui"]` |
|
||||
| Content mods (`registries`: data-driven blocks and items) | preview | manifest `api = "0.2"` |
|
||||
| Server mods (events, world, server commands) | not started | manifest `api = "0.3"`, experimental first |
|
||||
|
||||
+3
-2
@@ -12,9 +12,9 @@ folder, loads no runtime and plays exactly like vanilla 26.3.
|
||||
|
||||
| Part | Where | State |
|
||||
|---|---|---|
|
||||
| The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage, ui; tick, frame, chat and command hooks |
|
||||
| The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage, command, ui; tick, frame, chat and command hooks |
|
||||
| SDK for mod authors | `crates/horizon-mod-sdk` | 0.1 |
|
||||
| Example mod | `crates/horizon-mod-sdk/examples/coords-hud` | coordinates, facing and time HUD, `/where`, `/coords` |
|
||||
| Example mods | `crates/horizon-mod-sdk/examples/coords-hud`, `examples/predator-missile` | coordinates, facing and time HUD, `/where`, `/coords`; `/missile`, a guided missile from vanilla commands |
|
||||
| Runtime | `crates/horizon-mods` | wasmtime 36 LTS, component model |
|
||||
| Authoring tool | `crates/cargo-horizon-mod` | `cargo horizon-mod new [--template code\|content] / build / run / check`, and the registry's CI (`registry build / review / list`) |
|
||||
| In the game | `crates/horizon-client/src/game/mods.rs`, Options > Mods | client mods |
|
||||
@@ -86,6 +86,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3.
|
||||
| `chat` | `chat` | `show(text)`: a line only the local player sees, after the mod's name (8 per call) |
|
||||
| `hud` | `hud` | `set-panel(corner, lines)`, `clear()`: up to 16 lines of 256 characters |
|
||||
| `storage` | `storage` | `get(key)`, `set(key, value)`, `delete(key)`: a per-mod folder of up to 64 files, 4 MiB total, at most 1 MiB written per call |
|
||||
| `command` | `command` | `run(line)`: a vanilla command run as the player, with the player's permissions, after the hook returns (8 per call); mods' own commands are not run |
|
||||
| `ui` | `ui` | `open-screen`, `close-screen`, `set-settings`: a screen the mod owns and a page in Options > Mods, of text, item and list widgets the game lays out and draws with its own menus' code. **Experimental:** the manifest needs `experimental = ["ui"]` |
|
||||
|
||||
| Hook (export) | When |
|
||||
|
||||
@@ -20,6 +20,8 @@ python3 tools/parity.py --changed origin/main # those a branch's changes ca
|
||||
python3 tools/parity.py --changed-files paths.txt # the same for listed paths, in a copy without Git (tools/pr_ready.py)
|
||||
```
|
||||
|
||||
On CI, a pull request runs the parity job only when it changes a harness, an oracle, a probe or the runner; everything else is checked on main by `parity-main`, a batch of merges at a time, which bisects a failure to the merge that caused it (`tools/parity_main.py`; [release-flow.md](release-flow.md#parity-on-main)). Game releases wait for it.
|
||||
|
||||
Each harness runs unmodified under a tracer that records the files it depended on: its Python modules, its Java oracle and the oracle sources javac pulled in, other repository files it read, and the Rust sources of the probes it ran (from Cargo's `.d` dep-info files). A pass stands until one of those files, the pinned vanilla artifacts, `Cargo.lock` or the Rust/Java toolchains change. State and logs are in `.local/parity-runner/`; each harness's own outputs stay in its `.local/<name>/` directory.
|
||||
|
||||
Vanilla oracle runs are reused as well. `tools/parity_shim.py` stands in for `java` and `javac`: it records each oracle run's outputs and the content of every file under the paths its arguments name, and replays the run while those are unchanged. After a Rust-only change a harness pays only for its Rust side (`block_update`: 53 s to 5 s). Files an oracle writes outside its named paths aren't tracked, so a harness that fails while reusing oracle results is rerun fresh, and stops reusing them if it then passes. `--all` and `--fresh` never reuse them.
|
||||
@@ -49,6 +51,10 @@ A harness can declare, at the top level of its script:
|
||||
- `PARITY_SHARDS = N`: it runs as `NAME#1` to `NAME#N`, in parallel, each told its share in `PARITY_SHARD=i/N` (`test_datafix_parity.py` upgrades every sixth world). `NAME` on the command line or in `--known-failures` means all of them.
|
||||
- `PARITY_RUST_SCOPE = [...]`: the Rust sources its probes can reach, as path prefixes. Cargo's dep-info names every file of every crate a probe links, so without it a probe of `horizon-world` depends on all 350 of its files. The runner keeps only the scope's files if the scope is closed: nothing in it names a module, crate-root re-export or macro outside it (`crate::`, `super::`, other workspace crates; comments and strings ignored), declares a module it doesn't cover, or includes source, and nothing outside it implements anything for a type it defines. Crate roots then count without their declarations of other modules, so adding a module elsewhere doesn't rerun the harness. If the scope isn't closed the runner says why and keeps every file. `datafix` depends on 41 files this way, not 354.
|
||||
|
||||
### Which harnesses a change affects
|
||||
|
||||
`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `.local/parity-runner/reach-cache.json`. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses.
|
||||
|
||||
## The gate
|
||||
|
||||
`tools/gate.py` is what a branch must pass before it is integrated: the public-tree check, rustfmt, clippy with warnings denied, the Rust and Python tests, then the harnesses whose inputs changed. It stops at the first failure and writes `.local/gate.json`. `--known-failures a,b` lets harnesses that already fail on the integration branch keep failing. `--quick` checks formatting, lints and Python tests only; `--no-parity` skips the harnesses.
|
||||
|
||||
+34
-3
@@ -13,7 +13,7 @@ reopened; labels never start it):
|
||||
| Job | Runs on | What | Time |
|
||||
| --- | --- | --- | --- |
|
||||
| `verify` | GitHub `ubuntu-latest` | Public-tree check, Python tests, rustfmt, clippy with warnings denied, Rust tests. Dependencies and the workspace build come from a cache saved by main's runs | ~4 min cold, less warm |
|
||||
| `parity / parity` | A runner with the label `horizon-game`: the Mac mini's two, and the desktop's when no game runs there (below) | The vanilla harnesses the change can affect (`tools/parity.py --changed <base>`) | seconds when nothing is affected; minutes per affected harness |
|
||||
| `parity / parity` | A runner with the label `horizon-game`: the Mac mini's two, and the desktop's while nobody uses it (below) | Only for a change to a harness, an oracle, a probe or the runner (or the `run-parity` label): the vanilla harnesses the change can affect (`tools/parity.py --changed <base>`). Other pull requests skip it: [parity-main](#parity-on-main) checks them on main | seconds when nothing is affected; minutes per affected harness |
|
||||
|
||||
Pushes to main run `verify` only; the nightly audit covers main's harnesses.
|
||||
|
||||
@@ -42,10 +42,17 @@ stand.
|
||||
depend only on the Minecraft version, are kept once in `shared-local/sounds/<version>/` (copied from a folder
|
||||
`horizon-prepare sounds` made from the maintainer's own game; the Mac mini has no official launcher of its own)
|
||||
and cloned into each preparation (not linked: vanilla's packs refuse symbolic links), for the audio harnesses.
|
||||
`tools/prepare_output.py check` then compares what was prepared with `crates/horizon-prepare/output-versions.txt`:
|
||||
launchers keep a player's prepared resources across releases whose preparer declares the same `OUTPUT_VERSION`
|
||||
(`crates/horizon-prepare/src/lib.rs`), so a change to the prepared files fails here until it bumps the number
|
||||
and appends the new digest (the error gives both lines).
|
||||
3. `tools/parity.py --changed <base sha>` picks the harnesses whose traced dependencies (their script, Python
|
||||
modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any
|
||||
harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to
|
||||
`Cargo.lock` or `research-lock.json` affects every harness. Each harness has 25 minutes, the job 45.
|
||||
The selection runs with main's `tools/parity.py` and `tools/parity_reach.py` (`--changed <base> --list`), so a
|
||||
branch made before a better selection landed gets it too, and the head's own `parity.py` then runs the named
|
||||
harnesses. A branch that changes either file selects with its own code.
|
||||
|
||||
Failed harnesses' logs are uploaded as the `parity-logs` artifact.
|
||||
|
||||
@@ -102,10 +109,34 @@ gh api -X POST repos/ludovic111/horizon/branches/main/protection/required_status
|
||||
A skipped `parity` (a fork's pull request without the label) counts as passed, so requiring it never blocks a
|
||||
fork's pull request; the maintainer decides whether to run it.
|
||||
|
||||
## Parity on main
|
||||
|
||||
Most pull requests merge on `verify` alone and leave the vanilla harnesses to `.github/workflows/parity-main.yml`,
|
||||
which runs them on main a batch of merges at a time, on the Mac mini's runner:
|
||||
|
||||
- It starts on every push to main, one run at a time (`concurrency: parity-main`). Merges that land while a run
|
||||
goes on wait in one pending run: GitHub keeps only the newest and cancels the others, so the next run takes them
|
||||
all.
|
||||
- `tools/parity_main.py base` gives the newest commit main contains whose parity-main passed (else the last
|
||||
merge's parent), and the run is `tools/parity.py --changed <that commit>`: the union of the harnesses every merge
|
||||
since then can affect, once, with every earlier pass reused.
|
||||
- On a failure, `tools/parity_main.py bisect` checks out the batch's first-parent merges by halves and runs only
|
||||
the failing harnesses, names the first merge they fail on in the run's summary and comments on its pull request
|
||||
(`🤖 Agent: parity-main found that this merge breaks ...`). Fix it with a pull request as usual; the next
|
||||
run checks from the last green commit again.
|
||||
- Game releases ship only a commit both `checks` and `parity-main` passed on (`tools/release_train.py`, see
|
||||
below), so a broken merge holds releases instead of reaching players. `release.yml` also starts when a
|
||||
parity-main run finishes. A forced release (`force=true`) doesn't wait for parity-main.
|
||||
|
||||
A pull request still runs its own parity job when it changes a harness script, an oracle
|
||||
(`mods/horizon/src/test/`), a probe (`crates/*/examples/`, `game/*_fixture.rs`, `game/*_harness.rs`), the runner
|
||||
(`tools/parity*.py`, `tools/lab.py`, `tools/parity_hosts.json`, `parity*.yml`), or carries the `run-parity` label
|
||||
(`run-parity.yml` runs it when the label is added).
|
||||
|
||||
## Game releases
|
||||
|
||||
`.github/workflows/release.yml` releases the game by itself. It starts after every green `checks` run of a push
|
||||
to main, every 10 minutes (so a release waiting for merges to settle goes out without another merge), and by hand.
|
||||
`.github/workflows/release.yml` releases the game by itself. It starts after every green `checks` or `parity-main`
|
||||
run of a push to main (it releases a commit only once both passed on it), every 10 minutes (so a release waiting for merges to settle goes out without another merge), and by hand.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""tools/parity_main.py: the base of a parity-main run, and the bisect after a failure."""
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from tools import parity_main
|
||||
|
||||
|
||||
class ParityMainTests(unittest.TestCase):
|
||||
def test_first_bad_finds_the_first_failing_merge(self):
|
||||
commits = list("abcdefgh") # h failed; the base before a passed
|
||||
for culprit in commits:
|
||||
tried = []
|
||||
fails = lambda commit: tried.append(commit) or commit >= culprit
|
||||
self.assertEqual(parity_main.first_bad(commits, fails), culprit)
|
||||
self.assertLessEqual(len(tried), 3, "log2 of 8 merges")
|
||||
self.assertEqual(parity_main.first_bad(["a"], lambda commit: self.fail("nothing to try")), "a")
|
||||
|
||||
def test_base_is_the_newest_green_run_head_contains(self):
|
||||
with tempfile.TemporaryDirectory() as tmp, mock.patch.object(parity_main, "ROOT", Path(tmp)):
|
||||
git = lambda *arguments: parity_main.git(*arguments).stdout.strip()
|
||||
git("init", "-q")
|
||||
for n in range(3):
|
||||
git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", str(n))
|
||||
head, parent, grandparent = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^1"), git("rev-parse", "HEAD~2")
|
||||
git("checkout", "-q", "-b", "side", grandparent)
|
||||
git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "side")
|
||||
side = git("rev-parse", "HEAD")
|
||||
git("checkout", "-q", head)
|
||||
self.assertEqual(parity_main.base(runs=[side, grandparent, parent]), grandparent,
|
||||
"a run on a commit HEAD doesn't contain is skipped")
|
||||
self.assertEqual(parity_main.base(runs=[]), parent, "no green run yet: the last merge")
|
||||
self.assertEqual(parity_main.base(runs=[head]), head)
|
||||
|
||||
def test_a_harness_a_commit_does_not_have_cannot_fail_there(self):
|
||||
with mock.patch.object(parity_main.subprocess, "run") as run:
|
||||
self.assertFalse(parity_main.run_harnesses(["not_a_harness_yet"], 60))
|
||||
run.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,111 @@
|
||||
"""tools/parity_reach.py: which workspace sources a harness's probe can reach."""
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
from tools import parity, parity_reach
|
||||
|
||||
PROBE = [["example", "world_probe", "release", ["-p", "a"], True]]
|
||||
|
||||
|
||||
def workspace(root, files):
|
||||
for path, text in files.items():
|
||||
(root / path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / path).write_text(textwrap.dedent(text), encoding="utf-8")
|
||||
|
||||
|
||||
class ReachTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.directory.name)
|
||||
workspace(self.root, {
|
||||
"crates/a/Cargo.toml": '[package]\nname = "a"\n\n[[example]]\nname = "world_probe"\npath = "examples/probe/main.rs"\n',
|
||||
"crates/a/src/lib.rs": """
|
||||
pub mod used;
|
||||
pub mod unused;
|
||||
mod methods;
|
||||
mod display;
|
||||
mod table;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
""",
|
||||
"crates/a/src/used.rs": """
|
||||
pub struct Thing;
|
||||
pub fn helper() -> u32 { 1 }
|
||||
""",
|
||||
# Same names as used.rs, but nothing names this module.
|
||||
"crates/a/src/unused.rs": "pub struct Thing;\npub fn helper() -> u32 { 2 }\n",
|
||||
"crates/a/src/methods.rs": """
|
||||
use crate::used::Thing;
|
||||
impl Thing {
|
||||
pub fn method(&self) -> u32 { 3 }
|
||||
pub fn never_called(&self) -> u32 { 4 }
|
||||
}
|
||||
""",
|
||||
"crates/a/src/display.rs": """
|
||||
impl std::fmt::Display for crate::used::Thing {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "thing") }
|
||||
}
|
||||
""",
|
||||
"crates/a/src/table.rs": """
|
||||
macro_rules! table { () => { pub struct Table; }; }
|
||||
table!();
|
||||
""",
|
||||
"crates/a/src/tests.rs": "use crate::used::Thing;\nimpl Thing { pub fn method_in_tests(&self) {} }\n",
|
||||
"crates/a/examples/probe/main.rs": "mod dispatch;\nmod p;\n",
|
||||
"crates/a/examples/probe/dispatch.rs": "pub fn args() {}\n",
|
||||
"crates/a/examples/probe/p.rs": """
|
||||
use a::used::{helper, Thing};
|
||||
pub fn main() { println!("{} {} {}", helper(), Thing.method(), Thing); }
|
||||
""",
|
||||
})
|
||||
self.dependencies = sorted(str(path.relative_to(self.root)) for path in self.root.rglob("*") if path.is_file())
|
||||
self.dependencies.remove("crates/a/examples/probe/main.rs") # parity.only_probes_run leaves it out
|
||||
|
||||
def tearDown(self):
|
||||
self.directory.cleanup()
|
||||
|
||||
def narrowed(self, changed=(), artifacts=PROBE):
|
||||
graph = parity_reach.SourceGraph(self.root, parity.rust_code)
|
||||
return parity_reach.narrow(self.dependencies, artifacts, graph, set(changed))
|
||||
|
||||
def test_keeps_what_the_probe_reaches(self):
|
||||
kept = {path for path, (whole, _) in self.narrowed().items() if whole}
|
||||
self.assertIn("crates/a/src/used.rs", kept)
|
||||
self.assertIn("crates/a/src/methods.rs", kept) # Thing.method()
|
||||
self.assertIn("crates/a/src/display.rs", kept) # std's Display for a reached type
|
||||
self.assertIn("crates/a/Cargo.toml", kept)
|
||||
self.assertNotIn("crates/a/src/unused.rs", kept) # its names match, but nothing names its module
|
||||
self.assertNotIn("crates/a/src/table.rs", kept)
|
||||
self.assertNotIn("crates/a/src/tests.rs", kept) # compiled into tests only
|
||||
|
||||
def test_module_parents_count_through_their_declarations(self):
|
||||
self.assertEqual(self.narrowed()["crates/a/src/lib.rs"], (False, {"used", "methods", "display"}))
|
||||
text = (self.root / "crates/a/src/lib.rs").read_text()
|
||||
self.assertEqual(parity_reach.declarations(text, {"used"}, False), "pub mod used;")
|
||||
self.assertEqual(parity_reach.declarations(text, {"tests"}, False), "#[cfg(test)]\nmod tests;")
|
||||
# Adding or dropping a module beside the reached ones changes neither view.
|
||||
other = text.replace("mod table;", "mod table2;\nmod more;")
|
||||
self.assertEqual(parity_reach.declarations(other, {"used"}, False), "pub mod used;")
|
||||
self.assertEqual(parity_reach.declarations(other, {"used"}, True), parity_reach.declarations(text, {"used"}, True))
|
||||
self.assertNotEqual(parity_reach.declarations(text + "pub fn f() {}\n", {"used"}, True),
|
||||
parity_reach.declarations(text, {"used"}, True))
|
||||
|
||||
def test_a_new_file_counts_once_it_is_reached(self):
|
||||
(self.root / "crates/a/src/lib.rs").write_text("pub mod used;\nmod extra;\n")
|
||||
workspace(self.root, {"crates/a/src/extra.rs": "impl crate::used::Thing { pub fn method(&self) -> u32 { 5 } }\n"})
|
||||
self.assertIn("crates/a/src/extra.rs", self.narrowed(changed={"crates/a/src/extra.rs"}))
|
||||
|
||||
def test_a_deleted_dependency_counts(self):
|
||||
(self.root / "crates/a/src/unused.rs").unlink()
|
||||
self.assertIn("crates/a/src/unused.rs", self.narrowed())
|
||||
|
||||
def test_unknown_probes_keep_everything(self):
|
||||
self.assertEqual(set(self.narrowed(artifacts=[])), set(self.dependencies))
|
||||
self.assertEqual(set(self.narrowed(artifacts=[["example", "missing", "release", ["-p", "a"], True]])),
|
||||
set(self.dependencies))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -300,3 +300,17 @@ class LargeOutputTests(unittest.TestCase):
|
||||
self.assertFalse((own / "dump.txt").exists())
|
||||
# chunk_generation may still be writing there.
|
||||
self.assertTrue((shared / "dump.txt").exists())
|
||||
|
||||
def test_rust_named_directories_and_big_oracle_outputs_go_too(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
local, state = Path(tmp) / ".local", Path(tmp) / "state"
|
||||
own = local / "rust-state-provider-parity"
|
||||
own.mkdir(parents=True)
|
||||
for name, size in (("rust.txt", 16), ("server.txt", 16), ("client.txt", 64), ("report.json", 4)):
|
||||
(own / name).write_bytes(b"x" * size)
|
||||
trace = {"commands": [["java", "Oracle", str(own)], ["probe", "fixture", str(own)]]}
|
||||
with mock.patch.multiple(parity, LOCAL=local, STATE_DIR=state, LARGE_OUTPUT=8, KEPT_ORACLE_OUTPUT=32):
|
||||
listing = parity.outputs_listing("state_provider#2", fresh=True)
|
||||
listing.write_text(f"{own / 'server.txt'}\n{own / 'client.txt'}\n")
|
||||
parity.clear_large_outputs("state_provider#2", trace)
|
||||
self.assertEqual(sorted(path.name for path in own.iterdir()), ["report.json", "server.txt"])
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from tools import prepare_output
|
||||
|
||||
|
||||
class PrepareOutputTests(unittest.TestCase):
|
||||
def preparation(self, root: Path, files: dict[str, bytes]) -> Path:
|
||||
for rel, data in files.items():
|
||||
path = root / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
return root
|
||||
|
||||
def test_what_varies_between_runs_does_not_count(self):
|
||||
manifest = {'schema': 7, 'data_version': 5023, 'render_files_sha256': {'a.json': 'x'}}
|
||||
a = {
|
||||
'horizon-resources.json': json.dumps(manifest).encode(),
|
||||
'render-definitions.json': b'{"a": 1, "b": [1, 2]}',
|
||||
'creative-inventory.json': b'{"tags": [["b", "a"]]}',
|
||||
'block-behaviours.json': b'{"c": "Foo$$Lambda/0x0000012a"}',
|
||||
'export.log': b'run 1',
|
||||
'assets/minecraft/sounds/a.ogg': b'one',
|
||||
}
|
||||
manifest['render_files_sha256'] = {'a.json': 'y'}
|
||||
b = {
|
||||
'horizon-resources.json': json.dumps(manifest).encode(),
|
||||
'render-definitions.json': b'{"b": [1, 2], "a": 1}',
|
||||
'creative-inventory.json': b'{"tags": [["a", "b"]]}',
|
||||
'block-behaviours.json': b'{"c": "Foo$$Lambda/0x00000ff0"}',
|
||||
'export.log': b'run 2',
|
||||
'assets/minecraft/sounds/a.ogg': b'two',
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
first = prepare_output.digest(self.preparation(Path(tmp) / 'a', a))
|
||||
second = prepare_output.digest(self.preparation(Path(tmp) / 'b', b))
|
||||
self.assertEqual(first, second)
|
||||
# The data does, and so do the manifest's schema and the files there are.
|
||||
(Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [2, 1], "a": 1}')
|
||||
self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b'))
|
||||
(Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [1, 2], "a": 1}')
|
||||
(Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps({**manifest, 'schema': 8}))
|
||||
self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b'))
|
||||
(Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps(manifest))
|
||||
(Path(tmp) / 'b/new.nbt').write_bytes(b'')
|
||||
self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b'))
|
||||
|
||||
def test_the_versions_file_ends_at_the_constant(self):
|
||||
versions = prepare_output.parse_versions('# comment\n1 aa\n2 bb\n')
|
||||
self.assertEqual(versions, [(1, 'aa'), (2, 'bb')])
|
||||
self.assertEqual(prepare_output.problems(versions, 2), [])
|
||||
self.assertTrue(prepare_output.problems(versions, 1))
|
||||
self.assertTrue(prepare_output.problems([(1, 'aa'), (3, 'bb')], 3))
|
||||
self.assertTrue(prepare_output.problems([(1, 'aa'), (2, 'aa')], 2))
|
||||
root = Path(prepare_output.__file__).resolve().parent.parent
|
||||
current = prepare_output.parse_versions((root / prepare_output.VERSIONS).read_text())
|
||||
constant = prepare_output.CONSTANT.search((root / prepare_output.LIB).read_text())
|
||||
self.assertEqual(prepare_output.problems(current, int(constant.group(1))), [])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -2,6 +2,7 @@ import datetime as dt
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from tools import release_train as train
|
||||
|
||||
@@ -147,6 +148,26 @@ class ReleaseTrainTests(unittest.TestCase):
|
||||
self.assertFalse(train.decision("game", [], NOW, new_line="0.3", blockers=[7])[0])
|
||||
self.assertTrue(train.decision("game", [pr(1)], NOW, blockers=[])[0])
|
||||
|
||||
def test_game_releases_ship_only_what_parity_main_passed(self):
|
||||
def runs(parity, checks):
|
||||
def gh_json(*arguments):
|
||||
table = parity if "parity-main" in arguments else checks
|
||||
if "--commit" in arguments:
|
||||
sha = arguments[arguments.index("--commit") + 1]
|
||||
return [{"status": s, "conclusion": c} for h, s, c in table if h == sha]
|
||||
return [{"headSha": h, "status": s, "conclusion": c} for h, s, c in table]
|
||||
return mock.patch.object(train, "gh_json", gh_json)
|
||||
checks = [("c", "in_progress", ""), ("b", "completed", "success"), ("a", "completed", "success")]
|
||||
with runs([("c", "queued", ""), ("b", "completed", "cancelled"), ("a", "completed", "success")], checks):
|
||||
self.assertEqual(train.green_main_head(), "b", "checks alone: the newest green checks")
|
||||
self.assertEqual(train.green_main_head(parity=True), "a", "a superseded parity-main run counts for nothing")
|
||||
self.assertTrue(train.passed_on("parity-main", "a"))
|
||||
self.assertFalse(train.passed_on("parity-main", "b"))
|
||||
with runs([("b", "completed", "failure"), ("a", "completed", "success")], checks):
|
||||
self.assertIsNone(train.green_main_head(parity=True), "a failed batch holds releases until fixed")
|
||||
with runs([("c", "completed", "success")], checks):
|
||||
self.assertIsNone(train.green_main_head(parity=True), "its checks must pass too")
|
||||
|
||||
def test_a_release_is_published_only_after_its_smoke_test_and_with_no_blocker(self):
|
||||
drew = {"error": None, "frames": 12710, "chunks": 81, "meshed_sections": 550, "player": [160.5, 71.0, 156.5]}
|
||||
self.assertIsNone(train.gate(drew, []))
|
||||
|
||||
+56
-13
@@ -45,6 +45,11 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
from . import parity_reach
|
||||
except ImportError:
|
||||
import parity_reach
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LOCAL = ROOT / ".local"
|
||||
STATE_DIR = LOCAL / "parity-runner"
|
||||
@@ -652,6 +657,9 @@ def prebuild(artifacts, jobs):
|
||||
|
||||
|
||||
LARGE_OUTPUT = 64 << 20
|
||||
# Vanilla outputs above this are deleted after a pass too: rerunning those few oracles (12 harnesses on the Mac
|
||||
# mini's runner) costs less than the 26 GB they held in every checkout; smaller ones stay for oracle reuse.
|
||||
KEPT_ORACLE_OUTPUT = 256 << 20
|
||||
OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"}
|
||||
|
||||
|
||||
@@ -667,11 +675,12 @@ def clear_large_outputs(name, trace):
|
||||
"""After a pass, delete the large files a harness regenerates on every run.
|
||||
|
||||
Harness outputs reach gigabytes and used to stay in every worktree. Vanilla
|
||||
oracle outputs are kept, since reusing an oracle run needs them in place;
|
||||
everything else large under the harness's .local directories is rebuilt on
|
||||
its next run."""
|
||||
oracle outputs under KEPT_ORACLE_OUTPUT are kept, since reusing an oracle
|
||||
run needs them in place; everything else large under the harness's .local
|
||||
directories (`<name>...` or `rust-<name>-parity`) is rebuilt on its next run."""
|
||||
local = LOCAL.resolve()
|
||||
keep = {os.path.realpath(line) for line in outputs_listing(name).read_text(encoding="utf-8").splitlines() if line}
|
||||
own = name.split("#")[0].replace("_", "-")
|
||||
directories = set()
|
||||
for command in trace["commands"]:
|
||||
for argument in command[1:]:
|
||||
@@ -683,12 +692,13 @@ def clear_large_outputs(name, trace):
|
||||
# Only the harness's own directory: another harness running beside it may be writing
|
||||
# large files in a directory this one merely read (decorated_region compiles its oracle
|
||||
# into chunk-generation-parity while chunk_generation writes its dumps there).
|
||||
if top not in OWN_DIRECTORIES and top.startswith(name.replace("_", "-")):
|
||||
if top not in OWN_DIRECTORIES and (top.startswith(own) or top == f"rust-{own}-parity"):
|
||||
directories.add(local / top)
|
||||
for directory in directories:
|
||||
for file in (directory.rglob("*") if directory.is_dir() else []):
|
||||
try:
|
||||
if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep:
|
||||
size = file.stat().st_size if file.is_file() and not file.is_symlink() else 0
|
||||
if size >= LARGE_OUTPUT and (str(file) not in keep or size > KEPT_ORACLE_OUTPUT):
|
||||
file.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
@@ -823,8 +833,11 @@ class LocalExecutor:
|
||||
environment.pop("PARITY_SHARD", None)
|
||||
if shard:
|
||||
environment["PARITY_SHARD"] = shard
|
||||
# stdin from /dev/null: tools/parity_shim.py never reuses an oracle run that reads a pipe, and CI runners
|
||||
# (and --worker's ssh) give the runner one, which kept oracle reuse off on CI.
|
||||
process = subprocess.Popen([sys.executable, str(Path(__file__)), "--trace", str(trace), str(script)],
|
||||
cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
cwd=ROOT, env=environment, stdin=subprocess.DEVNULL, stdout=log,
|
||||
stderr=subprocess.STDOUT, start_new_session=True)
|
||||
self.running[name] = {"process": process, "started": time.monotonic(), "log": log, "expected": expected,
|
||||
"slot": slot, "script": script}
|
||||
return True
|
||||
@@ -1224,20 +1237,44 @@ def changed_harnesses(found, records, ref):
|
||||
return found_changed
|
||||
|
||||
|
||||
def affected_harnesses(found, records, changed):
|
||||
def affected_harnesses(found, records, changed, reach=True, base=None):
|
||||
"""Harnesses the changed files can affect: their script or a recorded dependency (here, or in the shared
|
||||
dependency lists) changed, a file every harness depends on changed, or nothing is recorded about them yet."""
|
||||
dependency lists) changed, a file every harness depends on changed, or nothing is recorded about them yet.
|
||||
With reach, a harness's workspace Rust sources count only if its probes can reach them (tools/parity_reach.py);
|
||||
a harness whose probes can't be placed keeps them all. A changed file counts unless, compared with base (a
|
||||
commit), only declarations of modules the harness doesn't reach changed (tools/parity_reach.py)."""
|
||||
if changed & SHARED_INPUTS:
|
||||
return set(found)
|
||||
graph = None
|
||||
if reach and any(path.startswith("crates/") for path in changed):
|
||||
cache_path = STATE_DIR / "reach-cache.json"
|
||||
try:
|
||||
cache = json.loads(cache_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
cache = {}
|
||||
graph = parity_reach.SourceGraph(ROOT, rust_code, cache)
|
||||
affected = set()
|
||||
for name, script in found.items():
|
||||
dependencies = records.get(name, {}).get("dependencies")
|
||||
record = records.get(name) or {}
|
||||
dependencies = record.get("dependencies")
|
||||
template = SHARED / "dependencies" / f"{name}.json"
|
||||
if dependencies is None and template.exists():
|
||||
dependencies = json.loads(template.read_text(encoding="utf-8")).get("dependencies")
|
||||
if dependencies is None or Path(script).relative_to(ROOT).as_posix() in changed or \
|
||||
any(path.split(KEEP)[0] in changed for path in dependencies):
|
||||
record = json.loads(template.read_text(encoding="utf-8"))
|
||||
dependencies = record.get("dependencies")
|
||||
if dependencies is None or Path(script).relative_to(ROOT).as_posix() in changed:
|
||||
affected.add(name)
|
||||
continue
|
||||
plain = {path.split(KEEP)[0] for path in dependencies}
|
||||
if graph is not None and plain & changed:
|
||||
narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed)
|
||||
if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed(
|
||||
ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()):
|
||||
affected.add(name)
|
||||
elif plain & changed:
|
||||
affected.add(name)
|
||||
if graph is not None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cache_path.write_text(json.dumps(graph.saved_cache()), encoding="utf-8")
|
||||
return affected
|
||||
|
||||
|
||||
@@ -1280,6 +1317,8 @@ def main():
|
||||
parser.add_argument("--flake-rounds", type=int, default=3, help="runs required by --flake-check (default 3)")
|
||||
parser.add_argument("--changed", metavar="REF", help="only harnesses the changes since the merge base with REF can "
|
||||
"affect (and that have not passed with their current inputs): the pull-request check")
|
||||
parser.add_argument("--by-crate", action="store_true", help="with --changed: count every Rust source a probe "
|
||||
"links, not only those it can reach (tools/parity_reach.py)")
|
||||
parser.add_argument("--changed-files", metavar="FILE", help="like --changed, for the changed paths listed in FILE, "
|
||||
"one per line: for a copy without Git history (tools/pr_ready.py on another machine)")
|
||||
args = parser.parse_args()
|
||||
@@ -1316,7 +1355,11 @@ def main():
|
||||
changed, since = changed_files(args.changed), f" since the merge base with {args.changed}"
|
||||
if changed is None:
|
||||
parser.error(f"no merge base with {args.changed}")
|
||||
affected = affected_harnesses(found, records, changed)
|
||||
base = None
|
||||
if args.changed:
|
||||
merge_base = subprocess.run(["git", "merge-base", "HEAD", args.changed], cwd=ROOT, capture_output=True, text=True)
|
||||
base = merge_base.stdout.strip() or None
|
||||
affected = affected_harnesses(found, records, changed, reach=not args.by_crate, base=base)
|
||||
chosen = [name for name in found if name in affected and stale(name, state, hasher, environment)]
|
||||
print(f"{len(changed)} files changed{since}: {len(affected)} harnesses "
|
||||
f"depend on them, {len(chosen)} without a pass for their current inputs.", flush=True)
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""parity-main (.github/workflows/parity-main.yml): the vanilla harnesses run on main, a batch of merges at a time.
|
||||
|
||||
python3 tools/parity_main.py base # the commit this run checks from: main's last green parity-main
|
||||
python3 tools/parity_main.py bisect BASE # after a failure: the first merge since BASE that broke a harness
|
||||
|
||||
`base` prints the newest commit on main whose parity-main run passed and that HEAD contains, else HEAD's first parent,
|
||||
so a run tests every merge since the last green one (`tools/parity.py --changed <base>`), however many landed while
|
||||
the previous run went on. `bisect` reads the harnesses that failed (.local/parity-runner/last-run.json), checks out
|
||||
the first-parent merges between BASE and HEAD by halves and runs only those harnesses, then names the first merge they
|
||||
fail on (written to $GITHUB_STEP_SUMMARY and, with --comment, on its pull request). HEAD is checked out again at the end.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO = os.environ.get("GITHUB_REPOSITORY", "ludovic111/horizon")
|
||||
LAST_RUN = ROOT / ".local" / "parity-runner" / "last-run.json"
|
||||
|
||||
|
||||
def git(*arguments, check=True):
|
||||
result = subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, text=True)
|
||||
if check and result.returncode:
|
||||
sys.exit(f"git {' '.join(arguments)}: {result.stderr.strip()}")
|
||||
return result
|
||||
|
||||
|
||||
def green_runs():
|
||||
"""Head commits of main's passing parity-main runs, newest first."""
|
||||
result = subprocess.run(["gh", "run", "list", "--repo", REPO, "--workflow", "parity-main", "--branch", "main",
|
||||
"--status", "success", "--limit", "30", "--json", "headSha"], capture_output=True, text=True)
|
||||
if result.returncode:
|
||||
print(f"note: no parity-main history ({result.stderr.strip()})", file=sys.stderr)
|
||||
return []
|
||||
return [run["headSha"] for run in json.loads(result.stdout or "[]")]
|
||||
|
||||
|
||||
def base(head="HEAD", runs=None):
|
||||
for sha in green_runs() if runs is None else runs:
|
||||
if git("merge-base", "--is-ancestor", sha, head, check=False).returncode == 0:
|
||||
return git("rev-parse", sha).stdout.strip()
|
||||
return git("rev-parse", f"{head}^1").stdout.strip()
|
||||
|
||||
|
||||
def first_bad(commits, fails):
|
||||
"""The first of commits (oldest first, the last known to fail; the one before them known to pass) where fails()."""
|
||||
good, bad = -1, len(commits) - 1
|
||||
while bad - good > 1:
|
||||
middle = (good + bad) // 2
|
||||
if fails(commits[middle]):
|
||||
bad = middle
|
||||
else:
|
||||
good = middle
|
||||
return commits[bad]
|
||||
|
||||
|
||||
def harness_exists(name):
|
||||
return (ROOT / "tools" / f"test_{name.split('#')[0]}_parity.py").exists()
|
||||
|
||||
|
||||
def run_harnesses(names, timeout):
|
||||
"""Whether any of the harnesses fails at the checked-out commit (those it doesn't have yet can't)."""
|
||||
present = [name for name in names if harness_exists(name)]
|
||||
if not present:
|
||||
return False
|
||||
return subprocess.run([sys.executable, "tools/parity.py", *present, "--timeout", str(timeout)], cwd=ROOT).returncode != 0
|
||||
|
||||
|
||||
def bisect(since, timeout, comment):
|
||||
failed = []
|
||||
if LAST_RUN.exists():
|
||||
failed = json.loads(LAST_RUN.read_text(encoding="utf-8")).get("failed") or []
|
||||
if not failed:
|
||||
print("No failed harness recorded: nothing to bisect.")
|
||||
return 0
|
||||
head = git("rev-parse", "HEAD").stdout.strip()
|
||||
commits = git("rev-list", "--first-parent", "--reverse", f"{since}..{head}").stdout.split()
|
||||
if not commits:
|
||||
print("No merge since the base.")
|
||||
return 0
|
||||
|
||||
def fails(commit):
|
||||
print(f"--- {commit[:9]} {git('log', '-1', '--format=%s', commit).stdout.strip()}", flush=True)
|
||||
git("checkout", "-q", "--detach", commit)
|
||||
return run_harnesses(failed, timeout)
|
||||
|
||||
try:
|
||||
culprit = first_bad(commits, fails) if len(commits) > 1 else commits[0]
|
||||
finally:
|
||||
git("checkout", "-q", "--detach", head)
|
||||
subject = git("log", "-1", "--format=%s", culprit).stdout.strip()
|
||||
number = re.search(r"Merge pull request #(\d+)", subject)
|
||||
line = (f"parity-main: {', '.join(failed)} first fail at {culprit[:9]} ({subject}), "
|
||||
f"of {len(commits)} merges since {since[:9]}.")
|
||||
print(line)
|
||||
if os.environ.get("GITHUB_STEP_SUMMARY"):
|
||||
with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary:
|
||||
summary.write(f"### {line}\n")
|
||||
if comment and number:
|
||||
url = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{REPO}/actions/runs/{os.environ.get('GITHUB_RUN_ID', '')}"
|
||||
body = (f"🤖 Agent: parity-main found that this merge breaks {', '.join(f'`{name}`' for name in failed)} on main "
|
||||
f"(bisected over {len(commits)} merges: {url}). Releases wait until parity-main passes again.")
|
||||
subprocess.run(["gh", "pr", "comment", number.group(1), "--repo", REPO, "--body", body])
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
commands.add_parser("base")
|
||||
bisecting = commands.add_parser("bisect")
|
||||
bisecting.add_argument("since", help="the last commit known to pass (the run's base)")
|
||||
bisecting.add_argument("--timeout", type=int, default=2400, help="seconds per harness")
|
||||
bisecting.add_argument("--comment", action="store_true", help="comment on the breaking merge's pull request")
|
||||
args = parser.parse_args()
|
||||
if args.command == "base":
|
||||
print(base())
|
||||
return 0
|
||||
return bisect(args.since, args.timeout, args.comment)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,768 @@
|
||||
"""Which of a harness's Rust sources its probes can reach, for `parity.py --changed`.
|
||||
|
||||
Cargo's dep-info names every source file of every crate a probe links, so a change anywhere in horizon-client
|
||||
selected every harness whose probe links horizon-client. This module narrows a harness's recorded dependencies to
|
||||
the files holding an item its entry points (the probe modules it ran, the binaries it built) can reach, over a graph
|
||||
of the workspace's current sources whose nodes are items:
|
||||
|
||||
- everything in an entry is reached, and so is everything in a Rust file the module tree doesn't place (an
|
||||
`include!`d source);
|
||||
- a module is open when it is a crate root, holds reached code, or is named in a path (`name::`, `use ...name`) by
|
||||
reached code while its parent is open;
|
||||
- a free item (fn, struct, enum, union, trait, type, const, static, an item-level macro call's items) is reached when
|
||||
reached code in its own module or below it mentions its name, or when its module is open and reached code names
|
||||
it in a path or an import, or when its module is glob-imported (`use m::*`) and its name is mentioned;
|
||||
- an inherent impl's methods and constants are reached one by one, by name, once their type is; a trait impl once
|
||||
its trait and its type are (a std or outside name always is, a generic parameter needs nothing); a macro once
|
||||
invoked; a `pub use` like a free item named by everything it imports;
|
||||
- "a crate's reached code" covers the crates it names (`horizon_world::`), directly or through one that does, and
|
||||
code under `#[cfg(test)]` or `#[test]` is left out (probes are never test builds).
|
||||
|
||||
Names match by spelling, so a file can be kept for a name it never meant: the graph only over-approximates. A
|
||||
reached file's parent modules count through their `mod` declarations of it (parity.py compares those with the merge
|
||||
base). Everything that isn't a workspace Rust source stays a dependency, except a crate's non-Rust files (shaders,
|
||||
assets, fonts) that only unreached sources name; a dependency that no longer exists counts as changed, and Rust files
|
||||
a change adds take part.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
IDENT = r"[A-Za-z_]\w*"
|
||||
WORD = re.compile(rf"\b({IDENT})\b")
|
||||
MOD_DECLARATION = re.compile(rf"\bmod\s+({IDENT})\s*;")
|
||||
INLINE_MOD = re.compile(rf"\bmod\s+({IDENT})\s*\{{")
|
||||
PATH_ATTRIBUTE = re.compile(r'#\[\s*path\s*=\s*"([^"]+)"\s*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?mod\s+\w+\s*;$')
|
||||
BRACE = re.compile(r"[{}]")
|
||||
TEST_ONLY = re.compile(r"#\[\s*(?:cfg\s*\(\s*test\s*\)|test)\s*\]")
|
||||
TEST_MODULE = re.compile(r"#\[\s*cfg\s*\(\s*test\s*\)\s*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?mod\s+\w+\s*;$")
|
||||
SPLIT = re.compile(r"[{}()\[\];]")
|
||||
ITEM = re.compile(rf"(?:pub(?:\s*\([^)]*\))?\s+)?(?:(?:unsafe|async|const|extern|default|auto)\s+)*"
|
||||
rf"(fn|struct|enum|union|trait|type|const|static|impl|mod|use|macro_rules\s*!|extern\s+crate)"
|
||||
rf"(?!\w)\s*(?:mut\s+)?({IDENT})?")
|
||||
PATH_BEFORE = re.compile(rf"\b({IDENT})\s*::")
|
||||
PATH_AFTER = re.compile(rf"::\s*({IDENT})")
|
||||
MACRO_USE = re.compile(rf"\b({IDENT})\s*!\s*[(\[{{]")
|
||||
MACRO_CALL = re.compile(rf"({IDENT}(?:\s*::\s*{IDENT})*)\s*!")
|
||||
DEFINED_INSIDE = re.compile(rf"\b(?:fn|struct|enum|union|trait|type|const|static)\s+({IDENT})")
|
||||
NESTED_IMPL = re.compile(r"^[ \t]*(?:unsafe\s+)?impl\b([^{;]*)\{", re.MULTILINE)
|
||||
QUALIFIED = re.compile(rf"{IDENT}(?:\s*::\s*{IDENT})*")
|
||||
GLOB_IMPORT = re.compile(r"\buse\s[^;]*::\s*\*")
|
||||
# File names a source may name in a literal (include_str!, include_bytes!, a path it reads). A file of another kind
|
||||
# is never "named", so it always counts.
|
||||
FILE_NAME = re.compile(r"[\w-]+\.(?:wgsl|glsl|json|png|ttf|otf|svg|txt|toml|ogg|wav|nbt|bin|dat|csv|mcmeta|wit|wasm)\b")
|
||||
TYPE_KINDS = {"struct", "enum", "union", "trait", "type", "macro_call"}
|
||||
# Names an impl header can take from outside the workspace: std's prelude and primitive types, and the std and serde
|
||||
# traits a glob import may bring. A glob-imported name outside this list is taken to be the workspace's.
|
||||
STD_NAMES = {"Display", "Error", "Read", "Write", "Seek", "BufRead", "Deref", "DerefMut", "Index", "IndexMut", "Add",
|
||||
"Sub", "Mul", "Div", "Rem", "Neg", "Not", "BitAnd", "BitOr", "BitXor", "Shl", "Shr", "AddAssign",
|
||||
"SubAssign", "MulAssign", "DivAssign", "RemAssign", "BitAndAssign", "BitOrAssign", "BitXorAssign",
|
||||
"Borrow", "BorrowMut", "Future", "Any", "FromStr", "Hasher", "BuildHasher", "Formatter", "Ordering",
|
||||
"Serialize", "Deserialize", "Serializer", "Deserializer", "Visitor", "Termination", "Sum", "Product",
|
||||
"Wake", "Cow", "Rc", "Arc", "Mutex", "RwLock", "Cell", "RefCell", "HashMap", "HashSet", "BTreeMap",
|
||||
"BTreeSet", "VecDeque"}
|
||||
PRELUDE = {"Option", "Some", "None", "Result", "Ok", "Err", "Vec", "String", "Box", "ToString", "ToOwned", "Clone",
|
||||
"Copy", "Send", "Sync", "Sized", "Unpin", "Drop", "Fn", "FnMut", "FnOnce", "From", "Into", "TryFrom",
|
||||
"TryInto", "AsRef", "AsMut", "Default", "Eq", "PartialEq", "Ord", "PartialOrd", "Iterator", "Extend",
|
||||
"IntoIterator", "DoubleEndedIterator", "ExactSizeIterator", "FromIterator", "Debug", "Hash",
|
||||
"bool", "char", "str", "u8", "u16", "u32", "u64", "u128", "usize", "i8", "i16", "i32", "i64", "i128",
|
||||
"isize", "f32", "f64"}
|
||||
STD_NAMES |= PRELUDE
|
||||
KEYWORDS = {"crate", "self", "super", "Self", "pub", "in", "as", "use", "for", "where", "dyn", "mut", "const", "fn",
|
||||
"unsafe", "impl", "std", "core", "alloc", "let", "if", "else", "match", "return", "struct", "enum", "mod",
|
||||
"type", "static", "trait", "true", "false", "loop", "while", "move", "ref", "break", "continue", "async",
|
||||
"await", "extern", "union"}
|
||||
|
||||
|
||||
def segments(code, start, end):
|
||||
"""The item-level pieces of code[start:end]: each ends at a `;` or a `}` outside any bracket."""
|
||||
found, begin, depth = [], start, 0
|
||||
for match in SPLIT.finditer(code, start, end):
|
||||
character = match.group()
|
||||
if character in "{([":
|
||||
depth += 1
|
||||
elif character in "})]":
|
||||
depth -= 1
|
||||
if character == "}" and depth == 0:
|
||||
found.append((begin, match.end()))
|
||||
begin = match.end()
|
||||
elif depth == 0:
|
||||
found.append((begin, match.end()))
|
||||
begin = match.end()
|
||||
if code[begin:end].strip():
|
||||
found.append((begin, end))
|
||||
return found
|
||||
|
||||
|
||||
def skip_attributes(code, index, end):
|
||||
"""The offset of the first non-space character after the attributes (`#[...]`, `#![...]`) starting at index."""
|
||||
while True:
|
||||
while index < end and code[index].isspace():
|
||||
index += 1
|
||||
if not code.startswith("#", index) or not re.match(r"#!?\[", code[index:index + 3]):
|
||||
return index
|
||||
depth = 0
|
||||
for position in range(code.index("[", index), end):
|
||||
depth += {"[": 1, "]": -1}.get(code[position], 0)
|
||||
if depth == 0:
|
||||
index = position + 1
|
||||
break
|
||||
else:
|
||||
return end
|
||||
|
||||
|
||||
def body_of(code, start, end):
|
||||
"""The range inside an item's outermost `{ }` block, or None."""
|
||||
opening = code.find("{", start, end)
|
||||
closing = code.rfind("}", start, end)
|
||||
return (opening + 1, closing) if 0 <= opening < closing else None
|
||||
|
||||
|
||||
def words(text):
|
||||
return set(WORD.findall(text)) - KEYWORDS
|
||||
|
||||
|
||||
def qualified_words(text):
|
||||
"""Names a piece of code reaches through a path (`a::b`: b), the way items of other modules are named."""
|
||||
return set(PATH_AFTER.findall(text)) - KEYWORDS
|
||||
|
||||
|
||||
def path_words(text):
|
||||
"""Names a piece of code uses as path segments (`a::`, `::b`), where module names appear."""
|
||||
return set(PATH_BEFORE.findall(text)) | set(PATH_AFTER.findall(text))
|
||||
|
||||
|
||||
def parse_items(code, start, end, items):
|
||||
"""Append the items of code[start:end] (a file or an inline module's body) to items, as dicts."""
|
||||
for begin, finish in segments(code, start, end):
|
||||
head = skip_attributes(code, begin, finish)
|
||||
text = code[head:finish]
|
||||
if not text.strip(" \n\t;") or TEST_ONLY.search(code[begin:head]):
|
||||
continue # compiled into tests only, never into a probe
|
||||
match = ITEM.match(text)
|
||||
kind = re.sub(r"\s+", "", match.group(1)) if match else None
|
||||
name = match.group(2) if match else None
|
||||
if kind == "mod":
|
||||
body = body_of(code, head, finish)
|
||||
if body:
|
||||
parse_items(code, body[0], body[1], items)
|
||||
continue
|
||||
if kind in ("externcrate", None) and not MACRO_CALL.match(text):
|
||||
if kind is None:
|
||||
items.append({"kind": "glue", "mentions": sorted(words(text)), "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))})
|
||||
continue
|
||||
if kind is None: # an item-level macro call: it may define items
|
||||
items.append({"kind": "macro_call", "names": sorted(set(DEFINED_INSIDE.findall(text))),
|
||||
"macro": MACRO_CALL.match(text).group(1).split("::")[-1].strip(),
|
||||
"mentions": sorted(words(text)), "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))})
|
||||
elif kind == "use":
|
||||
head_word = re.match(rf"\s*(?:::\s*)?({IDENT})", text[match.end(1):])
|
||||
items.append({"kind": "use", "public": text.startswith("pub"), "names": sorted(words(text)),
|
||||
"head": head_word.group(1) if head_word else "", "glob": bool(re.search(r"::\s*\*", text)),
|
||||
"mentions": sorted(words(text)), "paths": sorted(words(text)), "qualified": sorted(words(text)),
|
||||
"invokes": []})
|
||||
elif kind == "impl":
|
||||
body = body_of(code, head, finish)
|
||||
subject = code[head + match.end(1) - match.start(): body[0] - 1 if body else finish]
|
||||
inherent = not re.search(r"\bfor\b", impl_subject(subject)[1])
|
||||
if inherent and body:
|
||||
members = []
|
||||
for member_begin, member_end in segments(code, body[0], body[1]):
|
||||
member = code[skip_attributes(code, member_begin, member_end):member_end]
|
||||
found = ITEM.match(member)
|
||||
members.append({"name": found.group(2) if found else None, "mentions": sorted(words(member)),
|
||||
"paths": sorted(path_words(member)), "qualified": sorted(qualified_words(member)), "invokes": sorted(set(MACRO_USE.findall(member)))})
|
||||
items.append({"kind": "impl", "subject": subject, "inherent": True, "members": members,
|
||||
"mentions": sorted(words(subject)), "paths": sorted(path_words(subject)), "qualified": sorted(qualified_words(subject)), "invokes": sorted(set(MACRO_USE.findall(subject)))})
|
||||
else:
|
||||
items.append({"kind": "impl", "subject": subject, "inherent": False, "mentions": sorted(words(text)),
|
||||
"paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))})
|
||||
else:
|
||||
kind = "macro" if kind.startswith("macro_rules") else kind
|
||||
items.append({"kind": kind, "names": [name] if name else [], "mentions": sorted(words(text)),
|
||||
"defines": sorted(set(DEFINED_INSIDE.findall(text))) if kind == "macro" else [],
|
||||
"paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))})
|
||||
if kind in ("fn", "const", "static"):
|
||||
# An impl inside a function body still holds for the whole program.
|
||||
for header in NESTED_IMPL.findall(text):
|
||||
items.append({"kind": "impl", "subject": header, "inherent": False,
|
||||
"mentions": sorted(words(text)), "paths": sorted(path_words(text)), "qualified": sorted(qualified_words(text)), "invokes": sorted(set(MACRO_USE.findall(text)))})
|
||||
|
||||
|
||||
def crate_roots(crate_dir):
|
||||
"""A crate's root files: its library, binaries and examples."""
|
||||
roots = [crate_dir / "src" / "lib.rs", crate_dir / "src" / "main.rs"]
|
||||
for folder in (crate_dir / "src" / "bin", crate_dir / "examples"):
|
||||
if folder.is_dir():
|
||||
roots += sorted(folder.glob("*.rs")) + sorted(folder.glob("*/main.rs"))
|
||||
return [root for root in roots if root.is_file()]
|
||||
|
||||
|
||||
class SourceGraph:
|
||||
"""The workspace's Rust sources, parsed once: what each file names, defines and declares."""
|
||||
|
||||
def __init__(self, root, rust_code, cache=None):
|
||||
"""cache: a dict of earlier parses by content digest, which the graph reuses and adds to (parity.py keeps it
|
||||
in .local/parity-runner/reach-cache.json)."""
|
||||
self.root = Path(root)
|
||||
self.rust_code = rust_code
|
||||
self.cache = {} if cache is None else cache
|
||||
self.used = set()
|
||||
self.files = {}
|
||||
self.parent = {} # module file -> the file declaring it
|
||||
self.module_name = {}
|
||||
self.roots = set()
|
||||
self.target_cache = {}
|
||||
self.workspace_heads = None
|
||||
self.test_only = set() # modules compiled into tests only
|
||||
self.tables = None
|
||||
self.root_of = {} # source -> the root of its target (lib.rs, a bin's or an example's main file)
|
||||
self.results = {} # (entries, universe) -> reachable files
|
||||
for manifest in sorted((self.root / "crates").glob("*/Cargo.toml")):
|
||||
for crate_root in crate_roots(manifest.parent):
|
||||
path = self.relative(crate_root)
|
||||
self.roots.add(path)
|
||||
self.root_of[path] = path
|
||||
self.place(path, crate_root.parent, set())
|
||||
|
||||
def entries(self, artifacts, rust):
|
||||
"""The root files of the probes and binaries a harness built (its record's artifacts), among its Rust
|
||||
dependencies: for a crate's probe binary, the probe modules it ran (parity.only_probes_run kept only those).
|
||||
None when an artifact can't be placed."""
|
||||
found = set()
|
||||
for kind, name, *rest in artifacts:
|
||||
selection = rest[1] if len(rest) > 1 else []
|
||||
packages = [value for flag, value in zip(selection, selection[1:]) if flag == "-p"] or \
|
||||
[manifest.parent.name for manifest in sorted((self.root / "crates").glob("*/Cargo.toml"))]
|
||||
roots = set()
|
||||
for package in packages:
|
||||
crate = self.root / "crates" / package
|
||||
targets = self.targets(crate)
|
||||
if kind in ("example", "examples"):
|
||||
candidates = [path for target, (target_kind, path) in targets.items()
|
||||
if target_kind == "example" and (not name or target == name)]
|
||||
else:
|
||||
candidates = [path for target, (target_kind, path) in targets.items()
|
||||
if target_kind == "bin" and (not name or target == name)]
|
||||
for path in candidates:
|
||||
if path.endswith("/examples/probe/main.rs"):
|
||||
# The probes it ran, and the dispatch and support modules they use (only_probes_run kept
|
||||
# those); the binary's root, which only declares them, so their own submodules are placed.
|
||||
folder = path.removesuffix("main.rs")
|
||||
roots |= {dependency for dependency in rust if dependency.startswith(folder)
|
||||
and "/" not in dependency.removeprefix(folder).removeprefix("support/")}
|
||||
if roots:
|
||||
roots.add(path)
|
||||
elif path in rust:
|
||||
roots.add(path)
|
||||
if not roots:
|
||||
return None
|
||||
found |= roots
|
||||
return found or None
|
||||
|
||||
def targets(self, crate):
|
||||
"""A crate's bins and examples: {target name: (kind, root file)}, from its Cargo.toml and Cargo's defaults."""
|
||||
if crate in self.target_cache:
|
||||
return self.target_cache[crate]
|
||||
found = {}
|
||||
manifest = (crate / "Cargo.toml").read_text(encoding="utf-8") if (crate / "Cargo.toml").is_file() else ""
|
||||
package = re.search(r'^\[package\][^\[]*?^name\s*=\s*"([^"]+)"', manifest, re.MULTILINE | re.DOTALL)
|
||||
if (crate / "src" / "main.rs").is_file() and package:
|
||||
found[package.group(1)] = ("bin", self.relative(crate / "src" / "main.rs"))
|
||||
for kind, folder in (("bin", crate / "src" / "bin"), ("example", crate / "examples")):
|
||||
for path in sorted(folder.glob("*.rs")) + sorted(folder.glob("*/main.rs")):
|
||||
found[path.stem if path.name != "main.rs" else path.parent.name] = (kind, self.relative(path))
|
||||
for kind, table in re.findall(r"^\[\[(bin|example)\]\]([^\[]*)", manifest, re.MULTILINE):
|
||||
name = re.search(r'^name\s*=\s*"([^"]+)"', table, re.MULTILINE)
|
||||
path = re.search(r'^path\s*=\s*"([^"]+)"', table, re.MULTILINE)
|
||||
if name and path and (crate / path.group(1)).is_file():
|
||||
found[name.group(1)] = (kind, self.relative(crate / path.group(1)))
|
||||
self.target_cache[crate] = found
|
||||
return found
|
||||
|
||||
def target(self, path):
|
||||
"""The Cargo target a source belongs to: the root file of the library, bin or example whose module tree
|
||||
holds it (a file nothing places is its own)."""
|
||||
return self.root_of.get(path, path)
|
||||
|
||||
def is_library(self, path):
|
||||
return self.target(path).endswith("/src/lib.rs")
|
||||
|
||||
def relative(self, path):
|
||||
return Path(path).resolve().relative_to(self.root.resolve()).as_posix()
|
||||
|
||||
def place(self, path, module_dir, seen):
|
||||
"""Walk the module tree from a file: record each declared module's file, parent and name."""
|
||||
if path in seen:
|
||||
return
|
||||
seen.add(path)
|
||||
parsed = self.parse(path)
|
||||
if parsed is None:
|
||||
return
|
||||
for name, nesting, attribute, test in parsed["declarations"]:
|
||||
if attribute is not None:
|
||||
if nesting:
|
||||
continue # a #[path] inside an inline module: unplaced, so always kept
|
||||
candidates = [(self.root / path).parent / attribute]
|
||||
else:
|
||||
folder = module_dir.joinpath(*nesting)
|
||||
candidates = [folder / f"{name}.rs", folder / name / "mod.rs"]
|
||||
for candidate in candidates:
|
||||
if candidate.is_file():
|
||||
child = self.relative(candidate)
|
||||
self.parent.setdefault(child, path)
|
||||
self.module_name.setdefault(child, name)
|
||||
self.root_of.setdefault(child, self.root_of[path])
|
||||
if test or path in self.test_only:
|
||||
self.test_only.add(child)
|
||||
child_dir = candidate.parent if candidate.name == "mod.rs" or attribute is not None \
|
||||
else candidate.parent / candidate.stem
|
||||
self.place(child, child_dir, seen)
|
||||
break
|
||||
|
||||
def parse(self, path):
|
||||
if path in self.files:
|
||||
return self.files[path]
|
||||
try:
|
||||
data = (self.root / path).read_bytes()
|
||||
except OSError:
|
||||
self.files[path] = None
|
||||
return None
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
self.used.add(digest)
|
||||
if digest in self.cache:
|
||||
self.files[path] = parsed = self.cache[digest]
|
||||
return parsed
|
||||
raw = data.decode("utf-8", errors="replace")
|
||||
code = self.rust_code(raw) # same length as raw: literals and comments blanked
|
||||
opening = {match.end() - 1: match.group(1) for match in INLINE_MOD.finditer(code)}
|
||||
events = sorted([(match.start(), match) for match in BRACE.finditer(code)]
|
||||
+ [(match.start(), match) for match in MOD_DECLARATION.finditer(code)], key=lambda event: event[0])
|
||||
declarations, stack, depth = [], [], 0
|
||||
for index, match in events:
|
||||
if match.group() == "{":
|
||||
depth += 1
|
||||
if index in opening:
|
||||
stack.append((opening[index], depth))
|
||||
elif match.group() == "}":
|
||||
if stack and stack[-1][1] == depth:
|
||||
stack.pop()
|
||||
depth -= 1
|
||||
else:
|
||||
window = " ".join(raw[max(0, index - 300):match.end()].split())
|
||||
attribute = PATH_ATTRIBUTE.search(window)
|
||||
declarations.append([match.group(1), [name for name, _ in stack], attribute.group(1) if attribute else None,
|
||||
bool(TEST_MODULE.search(window))])
|
||||
items = []
|
||||
parse_items(code, 0, len(code), items)
|
||||
self.files[path] = parsed = {
|
||||
"declarations": declarations,
|
||||
"items": items,
|
||||
"glob": bool(GLOB_IMPORT.search(code)),
|
||||
"file_names": sorted(set(FILE_NAME.findall(raw))),
|
||||
}
|
||||
self.cache[digest] = parsed
|
||||
return parsed
|
||||
|
||||
def saved_cache(self):
|
||||
"""The parses of the files this graph read, to keep for the next run."""
|
||||
return {digest: self.cache[digest] for digest in self.used if digest in self.cache}
|
||||
|
||||
def requirement(self, path, item):
|
||||
"""For an impl: for its trait and for its type, [names the file defines, other workspace names], one of
|
||||
which must be a reached type or trait (of this file, of a crate it sees) before the impl can matter; both
|
||||
empty: nothing of the workspace is needed. A name is the workspace's when the path naming it starts in the
|
||||
workspace, or the file defines it, imports it from the workspace or may glob-import it; otherwise (std's,
|
||||
another crate's, the prelude's) it needs nothing."""
|
||||
if "requires" in item:
|
||||
return item["requires"]
|
||||
if self.workspace_heads is None:
|
||||
self.workspace_heads = {"crate", "self", "super"} | set(self.module_name.values()) | {
|
||||
manifest.parent.name.replace("-", "_") for manifest in (self.root / "crates").glob("*/Cargo.toml")}
|
||||
parsed = self.parse(path)
|
||||
defined = {name for other in parsed["items"] for name in other.get("names", ()) if other["kind"] != "use"}
|
||||
workspace_imports = {name for other in parsed["items"] if other["kind"] == "use"
|
||||
and other["head"] in self.workspace_heads for name in other["names"]}
|
||||
parameters, subject = impl_subject(item["subject"])
|
||||
parts = []
|
||||
for part in re.split(r"\bfor\b", subject, maxsplit=1):
|
||||
names, own = set(), set()
|
||||
for match in QUALIFIED.finditer(part):
|
||||
segments_ = [segment.strip() for segment in match.group().split("::")]
|
||||
name = segments_[-1]
|
||||
if name in parameters or name in KEYWORDS:
|
||||
continue
|
||||
if len(segments_) > 1:
|
||||
if segments_[0] in self.workspace_heads:
|
||||
names.add(name)
|
||||
elif name in defined:
|
||||
own.add(name)
|
||||
elif name in workspace_imports or (parsed["glob"] and name not in STD_NAMES):
|
||||
names.add(name)
|
||||
parts.append([sorted(own), sorted(names)])
|
||||
item["requires"] = parts
|
||||
return parts
|
||||
|
||||
def index(self):
|
||||
"""Lookup tables over every placed source, built once: which crates each crate sees, the nodes (items, an
|
||||
inherent impl's members) by the names they define and the types they need, the modules by name."""
|
||||
if self.tables is not None:
|
||||
return self.tables
|
||||
idents = {manifest.parent.name.replace("-", "_"): manifest.parent.name
|
||||
for manifest in (self.root / "crates").glob("*/Cargo.toml")}
|
||||
paths = [path for path in self.files if self.files[path] is not None and path not in self.test_only]
|
||||
sees = {}
|
||||
for path in paths:
|
||||
named = {word for item in self.files[path]["items"] for word in item["mentions"]} & idents.keys()
|
||||
sees.setdefault(crate_of(path), {crate_of(path)}).update(idents[name] for name in named)
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for crate, seen in sees.items():
|
||||
wider = set().union(*(sees.get(other, {other}) for other in seen))
|
||||
if not wider <= seen:
|
||||
seen |= wider
|
||||
changed = True
|
||||
seen_by = {} # crate -> the crates that see it
|
||||
for crate, seen in sees.items():
|
||||
for other in seen:
|
||||
seen_by.setdefault(other, set()).add(crate)
|
||||
# What a call of a workspace macro defines: what its definition's body defines.
|
||||
expands = {}
|
||||
for path in paths:
|
||||
for item in self.files[path]["items"]:
|
||||
if item["kind"] == "macro":
|
||||
for name in item["names"]:
|
||||
expands.setdefault(name, set()).update(item["defines"])
|
||||
names_of, by_name, by_file_name, by_type, by_module, children, seeds = {}, {}, {}, {}, {}, {}, []
|
||||
for path in paths:
|
||||
crate = crate_of(path)
|
||||
if path in self.module_name:
|
||||
by_module.setdefault((crate, self.module_name[path]), []).append(path)
|
||||
children.setdefault(self.parent[path], []).append(path)
|
||||
for index, item in enumerate(self.files[path]["items"]):
|
||||
node = (path, index, None)
|
||||
names = set(item.get("names", ()))
|
||||
if item["kind"] == "macro_call":
|
||||
names |= expands.get(item["macro"], set())
|
||||
names_of[node] = names
|
||||
if item["kind"] == "glue":
|
||||
seeds.append(node)
|
||||
elif item["kind"] == "macro_call" and not names:
|
||||
# A macro call defining nothing by name may still expand to impls: it counts once a type or
|
||||
# trait it mentions is reached.
|
||||
for name in item["mentions"]:
|
||||
by_type.setdefault(name, []).append(node)
|
||||
elif item["kind"] == "impl":
|
||||
required = {name for own, foreign in self.requirement(path, item) for name in own + foreign}
|
||||
members = [(path, index, member) for member in range(len(item["members"]))] if item["inherent"] else []
|
||||
for member_node in members:
|
||||
name = item["members"][member_node[2]]["name"]
|
||||
names_of[member_node] = set()
|
||||
if name:
|
||||
by_name.setdefault(name, []).append(member_node)
|
||||
for name in required:
|
||||
for other in [node] + members:
|
||||
by_type.setdefault(name, []).append(other)
|
||||
if not required:
|
||||
seeds.append(node)
|
||||
seeds += [member_node for member_node in members if not item["members"][member_node[2]]["name"]]
|
||||
else:
|
||||
for name in names:
|
||||
by_name.setdefault(name, []).append(node)
|
||||
by_file_name.setdefault((path, name), []).append(node)
|
||||
self.tables = dict(idents=idents, sees=sees, seen_by=seen_by, names_of=names_of, by_name=by_name,
|
||||
by_file_name=by_file_name, by_type=by_type, by_module=by_module, children=children,
|
||||
seeds=seeds, node_sets={})
|
||||
return self.tables
|
||||
|
||||
def reachable(self, entries, universe, reasons=None):
|
||||
"""The files of universe (workspace Rust sources) holding an item reachable from entries (by the rules in
|
||||
this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging)."""
|
||||
# Searching every source of the crates involved (not only those Cargo listed) only adds reach, and lets
|
||||
# harnesses with the same entries (the game binary's) share one search.
|
||||
crates = frozenset(crate_of(path) for path in universe)
|
||||
key = (frozenset(entries), crates)
|
||||
if reasons is None and key in self.results:
|
||||
return self.results[key] & set(universe)
|
||||
everything = {path for path in self.files if crate_of(path) in crates} | set(universe)
|
||||
files = self.search(entries, everything, reasons)
|
||||
self.results[key] = files
|
||||
return files & set(universe)
|
||||
|
||||
def search(self, entries, universe, reasons):
|
||||
targets = {self.target(path) for path in entries}
|
||||
universe = {path for path in universe if (self.is_library(path) or self.target(path) in targets)
|
||||
and path not in self.test_only}
|
||||
tables = self.index()
|
||||
idents, sees, seen_by, names_of = tables["idents"], tables["sees"], tables["seen_by"], tables["names_of"]
|
||||
node_sets = tables["node_sets"]
|
||||
by_name, by_file_name, by_type = tables["by_name"], tables["by_file_name"], tables["by_type"]
|
||||
by_module, children = tables["by_module"], tables["children"]
|
||||
seeds = [node for node in tables["seeds"] if node[0] in universe]
|
||||
for path in universe:
|
||||
if (path in entries or (path not in self.parent and path not in self.roots)) and self.parse(path):
|
||||
seeds += [(path, index, None) for index in range(len(self.parse(path)["items"]))]
|
||||
|
||||
def names(node):
|
||||
return names_of.get(node, set())
|
||||
|
||||
def sets_of(node, item):
|
||||
"""A node's mentions, path words, qualified names and macro calls, as sets (parses keep lists)."""
|
||||
if node not in node_sets:
|
||||
source = item["members"][node[2]] if node[2] is not None else item
|
||||
node_sets[node] = {key: set(source.get(key, ())) for key in ("mentions", "paths", "qualified", "invokes")}
|
||||
return node_sets[node]
|
||||
|
||||
def item_of(node):
|
||||
return self.parse(node[0])["items"][node[1]]
|
||||
|
||||
reached, opened, globbed, files, queue = set(), set(), set(), set(), []
|
||||
# name -> the crates whose reached code mentions it plainly, names it in a path or an import, invokes it.
|
||||
mentioned, qualified, invoked = {}, {}, {}
|
||||
done = {"mentions": {}, "qualified": {}, "invokes": {}, "paths": {}} # crate -> names already handled
|
||||
path_names, types = {}, {} # by crate
|
||||
local = {} # file -> plain mentions by reached code in it or below it (child modules may `use super::*`)
|
||||
local_types = {} # file -> its reached types and traits
|
||||
|
||||
def heard(table, name, crate):
|
||||
"""Whether a crate that can see crate's items has name in table."""
|
||||
sources = table.get(name)
|
||||
return bool(sources) and not sources.isdisjoint(seen_by.get(crate, {crate}))
|
||||
|
||||
def ready(node):
|
||||
path, index, member = node
|
||||
item = item_of(node)
|
||||
crate = crate_of(path)
|
||||
if item["kind"] == "impl":
|
||||
visible = visible_types(crate)
|
||||
here = local_types.get(path, set())
|
||||
if not all((not own and not foreign) or here.intersection(own) or visible.intersection(foreign)
|
||||
for own, foreign in self.requirement(path, item)):
|
||||
return False
|
||||
name = item["members"][member]["name"] if member is not None else None
|
||||
return name is None or heard(mentioned, name, crate) # a method: called by its name
|
||||
if item["kind"] == "macro_call" and not names(node):
|
||||
return not visible_types(crate).isdisjoint(item["mentions"])
|
||||
names_ = names(node)
|
||||
if item["kind"] == "macro":
|
||||
return any(heard(invoked, name, crate) for name in names_)
|
||||
if item["kind"] == "use" and item["glob"]:
|
||||
return path in opened
|
||||
nearby = local.get(path, ())
|
||||
if any(name in nearby for name in names_):
|
||||
return True # named in its own module, or below it
|
||||
if item["kind"] == "use" and not item["public"]:
|
||||
return False
|
||||
if path in opened and any(heard(qualified, name, crate) for name in names_):
|
||||
return True # named through a path, or imported
|
||||
return path in globbed and any(heard(mentioned, name, crate) for name in names_)
|
||||
|
||||
visible_cache = {}
|
||||
|
||||
def visible_types(crate):
|
||||
"""The reached types and traits of the crates a crate sees (cached until a type is reached)."""
|
||||
if crate not in visible_cache:
|
||||
visible_cache[crate] = set().union(*(types.get(other, set()) for other in sees.get(crate, {crate})))
|
||||
return visible_cache[crate]
|
||||
|
||||
def wake(nodes, why):
|
||||
for other in nodes:
|
||||
if other not in reached and other[0] in universe and ready(other):
|
||||
reach(other, why)
|
||||
|
||||
def open_module(path, why):
|
||||
pending = [path]
|
||||
while pending:
|
||||
current = pending.pop()
|
||||
if current in opened:
|
||||
continue
|
||||
opened.add(current)
|
||||
crate = crate_of(current)
|
||||
pending += [child for child in children.get(current, ())
|
||||
if child in universe and self.module_name[child] in path_names.get(crate, ())]
|
||||
parsed = self.parse(current)
|
||||
wake([(current, index, None) for index, item in enumerate(parsed["items"] if parsed else ())
|
||||
if item["kind"] != "impl"], f"{current} opened ({why})")
|
||||
|
||||
def reach(node, why):
|
||||
if node in reached:
|
||||
return
|
||||
reached.add(node)
|
||||
queue.append(node)
|
||||
path, _, member = node
|
||||
item = item_of(node)
|
||||
if (item["kind"] == "use" and not item["public"]) or (item["kind"] == "impl" and item["inherent"] and member is None):
|
||||
return
|
||||
if path not in files:
|
||||
files.add(path)
|
||||
if reasons is not None:
|
||||
reasons[path] = why
|
||||
|
||||
for path in universe:
|
||||
if path.endswith("/src/lib.rs") or path in entries or path not in self.parent:
|
||||
open_module(path, "crate root or entry")
|
||||
for node in seeds:
|
||||
reach(node, "entry" if node[0] in entries else "always counted")
|
||||
for path in entries & universe:
|
||||
files.add(path)
|
||||
if reasons is not None:
|
||||
reasons.setdefault(path, "entry")
|
||||
while queue:
|
||||
node = queue.pop()
|
||||
path, index, member = node
|
||||
item = item_of(node)
|
||||
crate = crate_of(path)
|
||||
label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"]
|
||||
source = sets_of(node, item)
|
||||
why = f"{path} ({label})"
|
||||
# Plain mentions reach items of this file and of the modules above it.
|
||||
ancestor, new = path, source["mentions"]
|
||||
while ancestor is not None and new:
|
||||
known = local.setdefault(ancestor, set())
|
||||
new = new - known
|
||||
known |= new
|
||||
for name in new:
|
||||
wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}")
|
||||
ancestor = self.parent.get(ancestor)
|
||||
if item["kind"] == "use" and item["glob"]:
|
||||
for name in item["names"]:
|
||||
for target in sees.get(crate, {crate}):
|
||||
for module in list(by_module.get((target, name), ())) + \
|
||||
([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []):
|
||||
if module not in globbed and module in universe:
|
||||
globbed.add(module)
|
||||
wake([(module, index, None) for index in range(len(self.parse(module)["items"]))]
|
||||
if self.parse(module) else [], f"{why} glob-imports {name}")
|
||||
visible_to = sees.get(crate, {crate})
|
||||
new = source["paths"] - done["paths"].setdefault(crate, set())
|
||||
done["paths"][crate] |= new
|
||||
for name in new:
|
||||
for target in visible_to:
|
||||
path_names.setdefault(target, set()).add(name)
|
||||
for module in by_module.get((target, name), ()):
|
||||
if module in universe and self.parent[module] in opened:
|
||||
open_module(module, f"{why} names {name}")
|
||||
for key, table, wanted in (("qualified", qualified, lambda other: True),
|
||||
("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"),
|
||||
("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)):
|
||||
new = source[key] - done[key].setdefault(crate, set())
|
||||
done[key][crate] |= new
|
||||
for name in new:
|
||||
table.setdefault(name, set()).add(crate)
|
||||
wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)],
|
||||
f"{why} {key} {name}")
|
||||
if item["kind"] in TYPE_KINDS and member is None:
|
||||
own = types.setdefault(crate, set())
|
||||
local_types.setdefault(path, set()).update(names(node))
|
||||
if not names(node) <= own:
|
||||
own |= names(node)
|
||||
visible_cache.clear()
|
||||
seeing = seen_by.get(crate, {crate})
|
||||
for name in names(node): # also when the crate has one by that name already: impls of this one
|
||||
wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing],
|
||||
f"implements for {name}, defined in {path}")
|
||||
return files
|
||||
|
||||
|
||||
def impl_subject(header):
|
||||
"""An impl header (the text after `impl`): its generic parameters' names, and the rest without its where clause
|
||||
(the trait and the type it is for)."""
|
||||
header, parameters = header.strip(), set()
|
||||
if header.startswith("<"):
|
||||
depth = 0
|
||||
for index, character in enumerate(header):
|
||||
depth += {"<": 1, ">": -1}.get(character, 0)
|
||||
if depth == 0:
|
||||
generics, header = header[1:index], header[index + 1:]
|
||||
parameters = {match.group(1) for match in re.finditer(rf"(?:^|,)\s*(?:const\s+)?({IDENT})", generics)}
|
||||
break
|
||||
return parameters, re.split(r"\bwhere\b", header)[0]
|
||||
|
||||
|
||||
def crate_of(path):
|
||||
return path.split("/", 2)[1]
|
||||
|
||||
|
||||
def is_workspace_rust(path):
|
||||
parts = PurePosixPath(path).parts
|
||||
return len(parts) > 2 and parts[0] == "crates" and path.endswith(".rs")
|
||||
|
||||
|
||||
DECLARATION_LINE = re.compile(rf"\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+({IDENT})\s*;")
|
||||
|
||||
|
||||
def declarations(text, names, keep_rest):
|
||||
"""A Rust file's `mod` declarations of names, each with the attribute lines just above it; with keep_rest, the
|
||||
whole text except the declarations of other modules instead."""
|
||||
lines, found, attributes = text.splitlines(), [], []
|
||||
for line in lines:
|
||||
if line.strip().startswith("#[") and not line.strip().startswith("#!["):
|
||||
attributes.append(line.strip())
|
||||
continue
|
||||
match = DECLARATION_LINE.match(line)
|
||||
if match and match.group(1) in names:
|
||||
found += attributes + [line.strip()]
|
||||
elif keep_rest and not match:
|
||||
found += attributes + [line]
|
||||
attributes = []
|
||||
return "\n".join(found + attributes)
|
||||
|
||||
|
||||
def declarations_changed(root, path, children, base, whole):
|
||||
"""Whether a changed file differs from the commit base in what a harness depends on: for a module parent, its
|
||||
`mod` declarations of the reached children; for a whole file (whole), everything but its declarations of other
|
||||
modules (adding a module beside reached ones changes nothing). True without a base, or if either side is
|
||||
missing."""
|
||||
if base is None:
|
||||
return True
|
||||
try:
|
||||
head = (Path(root) / path).read_text(encoding="utf-8", errors="replace")
|
||||
before = subprocess.run(["git", "show", f"{base}:{path}"], cwd=root, capture_output=True, text=True, check=True).stdout
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return True
|
||||
return declarations(head, children, whole) != declarations(before, children, whole)
|
||||
|
||||
|
||||
def narrow(dependencies, artifacts, graph, changed=()):
|
||||
"""A harness's dependencies narrowed to what its entry points can reach: {dependency: (whole, children)}, where
|
||||
whole says the file counts for its own content (else only through its `mod` declarations of children, as a
|
||||
module ancestor of a reached file) and children are its reached child modules. Without known entry points (a
|
||||
probe or binary the harness built that isn't found here), every dependency is kept whole."""
|
||||
plain = [dependency.split("#keep:")[0] for dependency in dependencies]
|
||||
crates = {PurePosixPath(path).parts[1] for path in plain if path.startswith("crates/")}
|
||||
rust = {path for path in plain if is_workspace_rust(path)}
|
||||
added = {path for path in changed if is_workspace_rust(path) and PurePosixPath(path).parts[1] in crates
|
||||
and path not in rust and (graph.root / path).is_file()}
|
||||
entries = graph.entries(artifacts, rust)
|
||||
if entries is None:
|
||||
return {dependency: (True, None) for dependency in dependencies}
|
||||
missing = {path for path in rust if graph.parse(path) is None}
|
||||
reached = graph.reachable(entries, rust | added | entries) | missing
|
||||
named, named_in_crate = set(), {}
|
||||
for path in rust | added:
|
||||
parsed = graph.parse(path)
|
||||
file_names = set(parsed["file_names"]) if parsed else set()
|
||||
if path in reached:
|
||||
named |= file_names
|
||||
named_in_crate.setdefault(PurePosixPath(path).parts[1], set()).update(file_names)
|
||||
kept = {}
|
||||
for dependency, path in zip(dependencies, plain):
|
||||
if is_workspace_rust(path):
|
||||
if path in reached:
|
||||
kept[dependency] = (True, set())
|
||||
elif not path.startswith("crates/") or path.endswith(("Cargo.toml", "build.rs")) \
|
||||
or PurePosixPath(path).name in named \
|
||||
or PurePosixPath(path).name not in named_in_crate.get(PurePosixPath(path).parts[1], ()):
|
||||
kept[dependency] = (True, None)
|
||||
for path in sorted(added & reached):
|
||||
kept[path] = (True, set())
|
||||
for path in sorted(reached):
|
||||
child = path
|
||||
while child in graph.parent:
|
||||
parent = graph.parent[child]
|
||||
kept.setdefault(parent, (False, set()))[1].add(graph.module_name[child])
|
||||
child = parent
|
||||
return kept
|
||||
Executable
+176
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The prepared output's version: what keeps `OUTPUT_VERSION` (crates/horizon-prepare/src/lib.rs) honest.
|
||||
|
||||
The launcher copies a prepared version's resources (`.local/horizon-client/<version>`) to a newly installed
|
||||
one when both pin the same Minecraft jars and their preparers declare the same `OUTPUT_VERSION`, so a release
|
||||
that changes the preparer's code without changing what it makes asks nobody to prepare again. The number is
|
||||
bumped by hand; this tool catches a change to the output without a bump.
|
||||
|
||||
crates/horizon-prepare/output-versions.txt lists every `OUTPUT_VERSION` with the digest of what it prepares.
|
||||
It only grows: the last line is the current version, and a change to the output appends a line.
|
||||
|
||||
python3 tools/prepare_output.py digest [RESOURCES] the digest of a preparation
|
||||
python3 tools/prepare_output.py check [--base SHA] [--cache] [RESOURCES]
|
||||
fails when RESOURCES (default: this checkout's .local/horizon-client/<version>, as CI prepares it) is
|
||||
not what the last line says, or when the file changed other than by appending since SHA. --cache keeps
|
||||
the digest beside RESOURCES (CI: a preparation never changes once made; it takes ~30 s to compute)
|
||||
|
||||
The digest covers every file of the preparation, canonicalised where the preparer is not deterministic between
|
||||
runs (verified across the CI runners): JSON is compared as data (key order, the order of each `tags` list of
|
||||
creative-inventory.json, and the addresses in `$$Lambda/0x...` class names don't count); logs, compiled classes,
|
||||
the sounds (copied from the player's launcher by another step) and the manifest's file hashes (raw bytes) don't
|
||||
count. The manifest's own scalar fields (schema, data version, jar hashes, file count) do.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
VERSIONS = 'crates/horizon-prepare/output-versions.txt'
|
||||
LIB = 'crates/horizon-prepare/src/lib.rs'
|
||||
MANIFEST = 'horizon-resources.json'
|
||||
LAMBDA = re.compile(r'\$\$Lambda/0x[0-9a-fA-F]+')
|
||||
CONSTANT = re.compile(r'^pub const OUTPUT_VERSION: u32 = (\d+);', re.M)
|
||||
|
||||
|
||||
def canonical_json(value, key=''):
|
||||
if isinstance(value, dict):
|
||||
return {k: canonical_json(v, k) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
items = [canonical_json(v, key) for v in value]
|
||||
if key == 'tags' and all(isinstance(x, list) and all(isinstance(t, str) for t in x) for x in items):
|
||||
items = [sorted(x) for x in items]
|
||||
return items
|
||||
if isinstance(value, str):
|
||||
return LAMBDA.sub('$$Lambda', value)
|
||||
return value
|
||||
|
||||
|
||||
def canonical(rel: str, data: bytes) -> bytes:
|
||||
if rel == MANIFEST:
|
||||
manifest = json.loads(data)
|
||||
scalars = {k: v for k, v in manifest.items() if not isinstance(v, (dict, list))}
|
||||
return json.dumps(scalars, sort_keys=True).encode()
|
||||
if rel.endswith('.json'):
|
||||
try:
|
||||
value = json.loads(data)
|
||||
except ValueError:
|
||||
return data
|
||||
return json.dumps(canonical_json(value), sort_keys=True).encode()
|
||||
return data
|
||||
|
||||
|
||||
def skipped(rel: str) -> bool:
|
||||
parts = rel.split('/')
|
||||
return rel.endswith(('.log', '.class')) or parts[:3] == ['assets', 'minecraft', 'sounds'] \
|
||||
or rel == 'assets/minecraft/sounds.json'
|
||||
|
||||
|
||||
def digest(resources: Path) -> tuple[str, int]:
|
||||
"""The digest of a preparation, and how many files it covers."""
|
||||
if not (resources / MANIFEST).is_file():
|
||||
raise SystemExit(f'{resources} is not a preparation: no {MANIFEST}')
|
||||
files = []
|
||||
for dirpath, dirs, names in os.walk(resources):
|
||||
dirs.sort()
|
||||
for name in names:
|
||||
path = Path(dirpath) / name
|
||||
rel = path.relative_to(resources).as_posix()
|
||||
if path.is_symlink() or skipped(rel):
|
||||
continue
|
||||
files.append(rel)
|
||||
total = hashlib.sha256()
|
||||
for rel in sorted(files):
|
||||
data = hashlib.sha256(canonical(rel, (resources / rel).read_bytes())).hexdigest()
|
||||
total.update(f'{rel}\0{data}\n'.encode())
|
||||
return total.hexdigest(), len(files)
|
||||
|
||||
|
||||
def parse_versions(text: str) -> list[tuple[int, str]]:
|
||||
out = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
number, value = line.split()
|
||||
out.append((int(number), value))
|
||||
return out
|
||||
|
||||
|
||||
def problems(versions: list[tuple[int, str]], constant: int | None) -> list[str]:
|
||||
found = []
|
||||
if not versions:
|
||||
return [f'{VERSIONS} lists no version']
|
||||
if [n for n, _ in versions] != list(range(1, len(versions) + 1)):
|
||||
found.append(f'{VERSIONS}: the versions must be 1, 2, 3... in order')
|
||||
if len({d for _, d in versions}) != len(versions):
|
||||
found.append(f'{VERSIONS}: two versions with the same digest')
|
||||
if constant != versions[-1][0]:
|
||||
found.append(f'{LIB} says OUTPUT_VERSION = {constant}, {VERSIONS} ends at {versions[-1][0]}')
|
||||
return found
|
||||
|
||||
|
||||
def default_resources() -> Path:
|
||||
lock = json.loads((ROOT / 'research-lock.json').read_text())
|
||||
return ROOT / '.local/horizon-client' / lock['minecraft']['version']
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
|
||||
parser.add_argument('command', choices=['digest', 'check'])
|
||||
parser.add_argument('resources', nargs='?', type=Path)
|
||||
parser.add_argument('--base', help='the commit the change is based on (check: the file only grew since)')
|
||||
parser.add_argument('--cache', action='store_true', help='keep the digest beside the preparation')
|
||||
args = parser.parse_args()
|
||||
resources = (args.resources or default_resources()).resolve()
|
||||
tool = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()[:16]
|
||||
cache = resources.parent / f'.output-digest-{resources.name}-{tool}'
|
||||
if args.cache and cache.is_file():
|
||||
value, count = cache.read_text().split()
|
||||
else:
|
||||
value, count = digest(resources)
|
||||
if args.cache:
|
||||
cache.write_text(f'{value} {count}\n')
|
||||
if args.command == 'digest':
|
||||
print(value)
|
||||
return 0
|
||||
|
||||
versions = parse_versions((ROOT / VERSIONS).read_text())
|
||||
match = CONSTANT.search((ROOT / LIB).read_text())
|
||||
found = problems(versions, int(match.group(1)) if match else None)
|
||||
if args.base:
|
||||
shown = subprocess.run(['git', 'show', f'{args.base}:{VERSIONS}'], cwd=ROOT, capture_output=True, text=True)
|
||||
if shown.returncode == 0:
|
||||
before = parse_versions(shown.stdout)
|
||||
if versions[:len(before)] != before:
|
||||
found.append(f'{VERSIONS} changed lines it already had (since {args.base[:12]}): '
|
||||
'a new output gets a new version, appended')
|
||||
current = versions[-1] if versions else (0, '')
|
||||
if value != current[1]:
|
||||
known = [n for n, d in versions if d == value]
|
||||
if known:
|
||||
found.append(f'the preparation is OUTPUT_VERSION {known[0]}\'s output, but OUTPUT_VERSION is {current[0]}')
|
||||
else:
|
||||
found.append(
|
||||
f'the prepared output changed ({count} files, digest {value}) but OUTPUT_VERSION is still '
|
||||
f'{current[0]}. Players would keep resources prepared by the old preparer. Bump it:\n'
|
||||
f' set `pub const OUTPUT_VERSION: u32 = {current[0] + 1};` in {LIB}\n'
|
||||
f' append `{current[0] + 1} {value}` to {VERSIONS}\n'
|
||||
' (if the change is only a nondeterminism this tool should ignore, teach canonical() instead)')
|
||||
for problem in found:
|
||||
print(f'::error::{problem}' if os.environ.get('GITHUB_ACTIONS') else f'error: {problem}', file=sys.stderr)
|
||||
if not found:
|
||||
print(f'prepared output: OUTPUT_VERSION {current[0]} ({count} files, {value[:16]})')
|
||||
return 1 if found else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
+26
-8
@@ -414,13 +414,26 @@ def gather(kind, head, releases=None):
|
||||
return last, prs
|
||||
|
||||
|
||||
def green_main_head():
|
||||
"""The newest commit on main whose checks passed: what an automatic release builds."""
|
||||
runs = gh_json("run", "list", "--repo", REPO, "--workflow", "checks", "--branch", "main", "--event", "push",
|
||||
def passed_on(workflow, sha):
|
||||
"""Whether the newest finished push run of workflow on main for commit sha passed."""
|
||||
runs = gh_json("run", "list", "--repo", REPO, "--workflow", workflow, "--branch", "main", "--event", "push",
|
||||
"--commit", sha, "--limit", "5", "--json", "status,conclusion")
|
||||
done = [entry for entry in runs if entry["status"] == "completed" and entry["conclusion"] != "cancelled"]
|
||||
return bool(done) and done[0]["conclusion"] == "success"
|
||||
|
||||
|
||||
def green_main_head(parity=False):
|
||||
"""The newest commit on main whose checks passed: what an automatic release builds. With parity (game
|
||||
releases), the newest commit parity-main passed on, if its checks passed too: parity-main runs on batches of
|
||||
merges, so the commits a run skipped are covered by the next one, and a superseded run counts for nothing."""
|
||||
workflow = "parity-main" if parity else "checks"
|
||||
runs = gh_json("run", "list", "--repo", REPO, "--workflow", workflow, "--branch", "main", "--event", "push",
|
||||
"--limit", "10", "--json", "headSha,status,conclusion")
|
||||
for entry in runs:
|
||||
if entry["status"] == "completed":
|
||||
return entry["headSha"] if entry["conclusion"] == "success" else None
|
||||
if entry["status"] == "completed" and entry["conclusion"] != "cancelled":
|
||||
if entry["conclusion"] != "success":
|
||||
return None
|
||||
return entry["headSha"] if not parity or passed_on("checks", entry["headSha"]) else None
|
||||
return None
|
||||
|
||||
|
||||
@@ -437,7 +450,7 @@ def main():
|
||||
parser.add_argument("--kind", choices=["game", "launcher"], required=True)
|
||||
parser.add_argument("--head", help="the commit to release (default: the newest commit on main whose checks passed)")
|
||||
parser.add_argument("--tag", help="notes: the release's tag")
|
||||
parser.add_argument("--force", action="store_true", help="release even without player-facing changes, before merges settle or while paused")
|
||||
parser.add_argument("--force", action="store_true", help="release even without player-facing changes, before merges settle, while paused or before parity-main passed")
|
||||
parser.add_argument("--out", type=Path, help="notes: write title.txt and body.md here")
|
||||
parser.add_argument("--now", help="the time to decide at (ISO 8601; default now)")
|
||||
parser.add_argument("--smoke", type=Path, help="gate: the release smoke test's report (smoke-test.json)")
|
||||
@@ -456,9 +469,12 @@ def main():
|
||||
summary.write(f"### {line}\n")
|
||||
return 1 if problem else 0
|
||||
|
||||
head = args.head or green_main_head()
|
||||
# Game releases ship only a commit the vanilla harnesses passed on (parity-main, docs/release-flow.md).
|
||||
parity = args.kind == "game" and args.command == "decide" and not args.force
|
||||
head = args.head or green_main_head(parity)
|
||||
if not head:
|
||||
result = {"release": False, "reason": "main's latest checks did not pass"}
|
||||
result = {"release": False, "reason": "main's latest checks did not pass" + (
|
||||
" or parity-main did not pass on them" if parity else "")}
|
||||
print(json.dumps(result, indent=1))
|
||||
write_outputs({"release": "false", "reason": result["reason"]})
|
||||
return 0
|
||||
@@ -482,6 +498,8 @@ def main():
|
||||
blockers = open_blockers() if args.kind == "game" else []
|
||||
release, reason = decision(args.kind, prs, now, os.environ.get("RELEASES_PAUSED") == "1", args.force, new_line,
|
||||
blockers)
|
||||
if release and args.head and parity and not (passed_on("parity-main", head) and passed_on("checks", head)):
|
||||
release, reason = False, f"waiting for checks and parity-main to pass on {head[:9]}"
|
||||
minor = any("feature" in pr["labels"] for pr in prs if pr["paths"])
|
||||
tag = next_game_tag(tags, now.date(), line) if args.kind == "game" else next_launcher_tag(tags, minor, line)
|
||||
result = {
|
||||
|
||||
@@ -5,6 +5,7 @@ import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
@@ -37,6 +38,8 @@ def main():
|
||||
if args.prepare:
|
||||
lab.prepare()
|
||||
lock, jars, libraries = test_rust_parity.verify_artifacts()
|
||||
for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks
|
||||
shutil.rmtree(earlier, ignore_errors=True)
|
||||
base = Path(tempfile.mkdtemp(prefix="run-", dir=directory))
|
||||
oracle = lab.ROOT / "mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java"
|
||||
sources = [oracle, Path(__file__).resolve(), lab.ROOT / "tools/test_rust_parity.py", lab.ROOT / "Cargo.toml", lab.ROOT / "Cargo.lock", lab.ROOT / "crates/horizon-core/Cargo.toml", lab.ROOT / "crates/horizon-world/Cargo.toml", *sorted((lab.ROOT / "crates/horizon-core/src").rglob("*.rs")), *sorted((lab.ROOT / "crates/horizon-world/src").rglob("*.rs")), lab.ROOT / "crates/horizon-world/examples/probe/collision_fixture.rs"]
|
||||
|
||||
@@ -9,6 +9,7 @@ from pathlib import Path
|
||||
import platform
|
||||
import random
|
||||
import struct
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
@@ -273,6 +274,8 @@ def main():
|
||||
if args.prepare:
|
||||
lab.prepare()
|
||||
lock, jars, libraries = test_rust_parity.verify_artifacts()
|
||||
for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks
|
||||
shutil.rmtree(earlier, ignore_errors=True)
|
||||
base = Path(tempfile.mkdtemp(prefix="run-", dir=directory))
|
||||
commands = list(corpus())
|
||||
inputs = base / "inputs.txt"
|
||||
|
||||
@@ -5,6 +5,7 @@ import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
@@ -35,6 +36,8 @@ def main():
|
||||
if args.prepare:
|
||||
lab.prepare()
|
||||
lock, jars, libraries = test_rust_parity.verify_artifacts()
|
||||
for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks
|
||||
shutil.rmtree(earlier, ignore_errors=True)
|
||||
base = Path(tempfile.mkdtemp(prefix="run-", dir=directory))
|
||||
classes = base / "classes"
|
||||
classes.mkdir()
|
||||
|
||||
Reference in New Issue
Block a user