From d4b2e24535111e7b8e1b839bcc16c4ef126fa663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Dr=C3=A4ger?= <20014538+sdraeger@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:21:23 -0700 Subject: [PATCH 01/61] Add mod command capability and Predator Missile example --- AGENTS.md | 2 +- Cargo.lock | 7 + Cargo.toml | 2 +- crates/horizon-client/src/game/mods.rs | 13 ++ .../examples/predator-missile/Cargo.toml | 16 ++ .../predator-missile/horizon-mod.toml | 8 + .../examples/predator-missile/src/lib.rs | 161 ++++++++++++++++++ crates/horizon-mod-sdk/src/lib.rs | 4 +- crates/horizon-mod-sdk/wit/horizon.wit | 9 + crates/horizon-mods/src/manifest.rs | 12 +- crates/horizon-mods/src/runtime.rs | 17 +- crates/horizon-mods/tests/sandbox.rs | 45 ++++- docs/api/mods-reference.md | 13 +- docs/mods.md | 3 +- 14 files changed, 300 insertions(+), 12 deletions(-) create mode 100644 crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml create mode 100644 crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml create mode 100644 crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs diff --git a/AGENTS.md b/AGENTS.md index f6f6b463b..fda05c1e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,7 +80,7 @@ python3 tools/agent_control_e2e.py # horizon-ctl / MCP end to end ~25, a file of the client lib ~103, of `horizon-world` ~103–109, of `horizon-core` ~119. Plan for it. - Rust integration tests: `crates/horizon-client/tests/` (`lan_bot.rs`, `generation_saves.rs`, `streaming_meshes.rs`). - **macOS:** `cargo build/test --workspace` may fail to link the two wasm guest cdylibs (undefined - `horizon:mods/events` symbols) until #1446 lands: add `--exclude coords-hud --exclude horizon-mods-test-guest`. + `horizon:mods/events` symbols) until #1446 lands: add `--exclude coords-hud --exclude predator-missile --exclude horizon-mods-test-guest`. Their real check is `cargo test -p horizon-mods` (builds them for `wasm32-unknown-unknown`). - Linux needs `libasound2-dev libudev-dev`; `--no-default-features` drops audio and controllers (the server build). - Run and drive the game (starts the `horizon-client` beside `horizon-ctl`; other arguments go to the game): diff --git a/Cargo.lock b/Cargo.lock index 27eaad0c4..1468327ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2620,6 +2620,13 @@ dependencies = [ "zerovec", ] +[[package]] +name = "predator-missile" +version = "0.1.0" +dependencies = [ + "horizon-mod-sdk", +] + [[package]] name = "presser" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index f197c34fe..a0feaba87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] exclude = ["launcher", "crates/horizon-telemetry-server", "third_party/wgpu-hal"] -members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"] +members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "crates/horizon-mod-sdk/examples/predator-missile", "xtask"] resolver = "2" [workspace.package] diff --git a/crates/horizon-client/src/game/mods.rs b/crates/horizon-client/src/game/mods.rs index 572f9fe2b..c203d0ca2 100644 --- a/crates/horizon-client/src/game/mods.rs +++ b/crates/horizon-client/src/game/mods.rs @@ -342,6 +342,19 @@ impl Game<'_> { } Output::Feedback { text, error: false } => Text::literal(text), Output::Feedback { text, error: true } => Text::literal(text).with_color(text::RED), + // Vanilla's commands only: a mod running a mod command could loop. + Output::Command { line } => { + let name = line.split(' ').next().unwrap_or_default(); + // As handle_chat_input, without the chat history. + if !vanilla_command(name) { + // dropped + } else if self.remote.is_some() { + self.send_chat(&format!("/{line}")); + } else { + self.run_command(&line); + } + continue; + } Output::Stopped { name, reason } => { self.list_mods(); self.sync_mod_commands(); diff --git a/crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml b/crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml new file mode 100644 index 000000000..a3a77ce0b --- /dev/null +++ b/crates/horizon-mod-sdk/examples/predator-missile/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "predator-missile" +version = "0.1.0" +edition.workspace = true +license.workspace = true +publish = false +description = "Example Horizon mod: a missile you launch with /missile and steer with the mouse" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +horizon-mod-sdk = { path = "../.." } + +[lints] +workspace = true diff --git a/crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml b/crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml new file mode 100644 index 000000000..3baaf7d87 --- /dev/null +++ b/crates/horizon-mod-sdk/examples/predator-missile/horizon-mod.toml @@ -0,0 +1,8 @@ +id = "predator_missile" +name = "Predator Missile" +version = "0.1.0" +description = "/missile launches a guided missile 60 blocks above you. Steer it with the mouse; it explodes on whatever it hits. Needs cheats (it runs vanilla commands as you)." +authors = ["Horizon"] +api = "0.1" +side = "client" +capabilities = ["game-read", "hud", "chat", "command"] diff --git a/crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs b/crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs new file mode 100644 index 000000000..75ebbbdec --- /dev/null +++ b/crates/horizon-mod-sdk/examples/predator-missile/src/lib.rs @@ -0,0 +1,161 @@ +//! Example Horizon mod (`docs/mods.md`): `/missile` turns you into a guided +//! missile 60 blocks up. You fly along your own look, so the mouse steers; +//! on a block or a mob a stack of primed TNT goes off, and you are put back +//! where you launched. Built from vanilla commands (capability `command`), +//! so it needs cheats. +use horizon_mod_sdk::{ + Mod, Registration, + command::run, + game::{self, Player}, + hud::{self, Corner, Line}, +}; + +/// Blocks per tick (a fired arrow does about 3). +const SPEED: f64 = 1.5; +const ALTITUDE: i32 = 60; +/// Ticks of flight before the missile detonates on its own. +const FUEL: u32 = 30 * 20; +/// Ticks to watch the blast before going back. +const AFTERMATH: u32 = 40; +/// Primed TNT per blast: the stack's explosions add up. +const WARHEAD: usize = 4; +/// What the impact checks print; the mod hears it through the chat hook. +const IMPACT: &str = "Predator: impact"; + +enum State { + Idle, + Flying { from: Player, ticks: u32 }, + Aftermath { from: Player, ticks: u32 }, +} + +struct Predator { + state: State, +} + +fn launch() { + run("gamerule send_command_feedback false"); + run("execute if entity @s[gamemode=creative] run tag @s add pm_creative"); + run("gamemode spectator"); + run(&format!("tp @s ~ ~{ALTITUDE} ~ ~ 60")); + run("effect give @s night_vision 60 0 true"); + // The thermal camera: every mob outlined. + run("effect give @e[type=!player,distance=..256] glowing 60 0 true"); +} + +/// One tick of flight: stop at the surface (the next step would enter a +/// block or a mob), else move one step along the look. +fn fly() { + run(&format!( + "execute at @s unless block ^ ^ ^{SPEED} #minecraft:air run tellraw @s \"{IMPACT}\"" + )); + run(&format!( + "execute at @s positioned ^ ^ ^{SPEED} if entity @e[type=!player,distance=..2] run tellraw @s \"{IMPACT}\"" + )); + run(&format!( + "execute at @s if block ^ ^ ^{SPEED} #minecraft:air run tp @s ^ ^ ^{SPEED}" + )); +} + +fn detonate() { + for _ in 0..WARHEAD { + run("execute at @s run summon tnt ^ ^ ^1 {fuse:0s}"); + } +} + +fn land(from: &Player) { + run(&format!( + "tp @s {} {} {} {} {}", + from.x, from.y, from.z, from.yaw, from.pitch + )); + run("execute if entity @s[tag=pm_creative] run gamemode creative"); + run("execute unless entity @s[tag=pm_creative] run gamemode survival"); + run("execute if entity @s[tag=pm_creative] run tag @s remove pm_creative"); + run("effect clear @s night_vision"); + // shortcut: assumes feedback was on before launch, as in a new world. + run("gamerule send_command_feedback true"); +} + +fn panel(p: &Player, ticks: u32) -> Vec { + let green = 0x7CFF7C; + let left = (FUEL - ticks.min(FUEL)) / 20; + [ + "PREDATOR".to_string(), + format!("ALT {:.0}", p.y), + format!("FUEL {left}s"), + "Mouse to steer, /missile to detonate".to_string(), + ] + .into_iter() + .map(|text| Line { text, color: green }) + .collect() +} + +impl Predator { + fn impact(&mut self) { + if let State::Flying { from, .. } = &self.state { + detonate(); + self.state = State::Aftermath { + from: from.clone(), + ticks: 0, + }; + } + } +} + +impl Mod for Predator { + fn init() -> Result<(Self, Registration), String> { + let hooks = Registration::new().command("missile").on_tick().on_chat(); + Ok((Predator { state: State::Idle }, hooks)) + } + + fn tick(&mut self, _tick: u64) { + match &mut self.state { + State::Idle => {} + State::Flying { ticks, .. } => { + *ticks += 1; + let ticks = *ticks; + if ticks >= FUEL { + return self.impact(); + } + // The first tick waits for the launch teleport. + if ticks > 1 { + fly(); + } + if let Some(p) = game::get_player() { + hud::set_panel(Corner::TopLeft, &panel(&p, ticks)); + } + } + State::Aftermath { from, ticks } => { + *ticks += 1; + if *ticks >= AFTERMATH { + land(from); + hud::clear(); + self.state = State::Idle; + } + } + } + } + + fn chat(&mut self, text: &str) { + if text.contains(IMPACT) { + self.impact(); + } + } + + fn command(&mut self, _name: &str, _args: &str) -> Result, String> { + match self.state { + State::Idle => { + let from = game::get_player().ok_or("Not in a world")?; + launch(); + self.state = State::Flying { from, ticks: 0 }; + Ok(Some("Missile away".into())) + } + State::Flying { .. } => { + self.impact(); + Ok(None) + } + State::Aftermath { .. } => Err("Wait for the blast to clear".into()), + } + } +} + +horizon_mod_sdk::export_mod!(Predator); diff --git a/crates/horizon-mod-sdk/src/lib.rs b/crates/horizon-mod-sdk/src/lib.rs index 71c36cf80..7efcf20b3 100644 --- a/crates/horizon-mod-sdk/src/lib.rs +++ b/crates/horizon-mod-sdk/src/lib.rs @@ -21,7 +21,7 @@ //! ``` //! //! Every interface but `log` is a capability: list the ones the mod uses in -//! `horizon-mod.toml` (`game-read`, `chat`, `hud`, `storage`). +//! `horizon-mod.toml` (`game-read`, `chat`, `hud`, `storage`, `command`). wit_bindgen::generate!({ path: "wit", @@ -32,7 +32,7 @@ wit_bindgen::generate!({ }); pub use exports::horizon::mods::events::{Guest as __Guest, Registration}; -pub use horizon::mods::{chat, game, hud, log, storage}; +pub use horizon::mods::{chat, command, game, hud, log, storage}; /// The game's calls into a mod. Only the hooks [`Registration`] asks for /// are called; the others may stay as their defaults. diff --git a/crates/horizon-mod-sdk/wit/horizon.wit b/crates/horizon-mod-sdk/wit/horizon.wit index 9f00a12b1..e14913782 100644 --- a/crates/horizon-mod-sdk/wit/horizon.wit +++ b/crates/horizon-mod-sdk/wit/horizon.wit @@ -86,6 +86,14 @@ interface storage { delete: func(key: string); } +/// Capability `command`: vanilla commands run as the player, with the +/// player's permissions, after the hook call returns (8 per call). A mod's +/// own commands are not run this way. +interface command { + /// `line` without its slash; its feedback goes to the chat as if typed. + run: func(line: string); +} + /// What the game calls. Only the hooks `init` asks for are ever called. interface events { record registration { @@ -117,5 +125,6 @@ world horizon-mod { import chat; import hud; import storage; + import command; export events; } diff --git a/crates/horizon-mods/src/manifest.rs b/crates/horizon-mods/src/manifest.rs index a71210883..6f1674e05 100644 --- a/crates/horizon-mods/src/manifest.rs +++ b/crates/horizon-mods/src/manifest.rs @@ -69,16 +69,19 @@ pub enum Capability { Chat, Hud, Storage, + /// Runs vanilla commands as the player, with the player's permissions. + Command, /// New blocks and items from the mod's `data/` (API 0.2): no WIT /// interface, the game reads them before the world loads. Registries, } impl Capability { - pub const ALL: [Capability; 5] = [ + pub const ALL: [Capability; 6] = [ Capability::GameRead, Capability::Chat, Capability::Hud, Capability::Storage, + Capability::Command, Capability::Registries, ]; /// The WIT interface it grants, in `horizon:mods`. @@ -88,6 +91,7 @@ impl Capability { Capability::Chat => "chat", Capability::Hud => "hud", Capability::Storage => "storage", + Capability::Command => "command", Capability::Registries => "registries", } } @@ -98,6 +102,7 @@ impl Capability { Capability::Chat => "show you chat messages", Capability::Hud => "draw text on your screen", Capability::Storage => "save its own settings to disk", + Capability::Command => "run commands as you, with your permissions", Capability::Registries => "add blocks and items to your worlds", } } @@ -128,6 +133,7 @@ impl fmt::Display for Capability { Capability::Chat => "chat", Capability::Hud => "hud", Capability::Storage => "storage", + Capability::Command => "command", Capability::Registries => "registries", }; f.write_str(name) @@ -366,6 +372,10 @@ capabilities = ["game-read", "hud"] storage, Err("uses capability storage, missing from horizon-mod.toml".into()) ); + assert_eq!( + m.check_imports(["horizon:mods/command@0.1.0"]), + Err("uses capability command, missing from horizon-mod.toml".into()) + ); let with_storage = EXAMPLE.replace("\"hud\"", "\"hud\", \"storage\""); let m = Manifest::parse(&with_storage).unwrap(); assert_eq!(m.check_imports(["horizon:mods/storage@0.1.0"]), Ok(())); diff --git a/crates/horizon-mods/src/runtime.rs b/crates/horizon-mods/src/runtime.rs index 9dd2eb54e..07be81909 100644 --- a/crates/horizon-mods/src/runtime.rs +++ b/crates/horizon-mods/src/runtime.rs @@ -23,7 +23,7 @@ pub use bindings::horizon::mods::{ use bindings::{ HorizonMod, exports::horizon::mods::events::Registration, - horizon::mods::{chat, game, hud, log as mod_log, storage}, + horizon::mods::{chat, command, game, hud, log as mod_log, storage}, }; /// A mod's linear memory cap. @@ -67,6 +67,9 @@ pub enum Output { Chat { name: String, text: String }, /// A mod command's feedback, red when `error`. Feedback { text: String, error: bool }, + /// `command.run`: a vanilla command line, without its slash, to run as + /// the player. + Command { line: String }, /// The mod was turned off: it trapped, overran or failed to start. Stopped { name: String, reason: String }, } @@ -514,6 +517,18 @@ impl chat::Host for State { } } } +impl command::Host for State { + fn run(&mut self, line: String) { + let queued = self + .out + .iter() + .filter(|o| matches!(o, Output::Command { .. })) + .count(); + if queued < CHAT_PER_CALL { + self.out.push(Output::Command { line: clip(line) }); + } + } +} impl hud::Host for State { fn set_panel(&mut self, corner: Corner, lines: Vec) { let lines = lines diff --git a/crates/horizon-mods/tests/sandbox.rs b/crates/horizon-mods/tests/sandbox.rs index a246c07b0..c8e4fb883 100644 --- a/crates/horizon-mods/tests/sandbox.rs +++ b/crates/horizon-mods/tests/sandbox.rs @@ -1,4 +1,5 @@ -//! Loads real mods built with the SDK: the coords-hud example and a guest +//! Loads real mods built with the SDK: the coords-hud and predator-missile +//! examples and a guest //! that misbehaves on command. Needs `rustup target add wasm32-unknown-unknown`. use horizon_mods::{ Clock, Corner, Mods, Output, Package, Player, Snapshot, Status, manifest::componentize, @@ -18,7 +19,7 @@ api = "0.1" capabilities = ["chat", "storage"] "#; -/// Builds both mods once and writes their `.hmod` files; returns the folder. +/// Builds the mods once and writes their `.hmod` files; returns the folder. fn mods_dir() -> &'static Path { static DIR: OnceLock = OnceLock::new(); DIR.get_or_init(|| { @@ -31,6 +32,8 @@ fn mods_dir() -> &'static Path { "-p", "coords-hud", "-p", + "predator-missile", + "-p", "horizon-mods-test-guest", "--target-dir", ]) @@ -44,9 +47,13 @@ fn mods_dir() -> &'static Path { let built = target.join("wasm32-unknown-unknown/release"); let dir = target.join("mods"); std::fs::create_dir_all(&dir).unwrap(); - let example = root.join("crates/horizon-mod-sdk/examples/coords-hud/horizon-mod.toml"); + let example = |name: &str| { + let path = format!("crates/horizon-mod-sdk/examples/{name}/horizon-mod.toml"); + std::fs::read_to_string(root.join(path)).unwrap() + }; let mods = [ - ("coords_hud", std::fs::read_to_string(example).unwrap()), + ("coords_hud", example("coords-hud")), + ("predator_missile", example("predator-missile")), ("horizon_mods_test_guest", GUEST_MANIFEST.to_string()), ]; for (name, manifest) in mods { @@ -81,6 +88,36 @@ fn player() -> Snapshot { } } +/// Capability `command`: `/missile` queues vanilla commands, and the chat +/// line its impact check prints sets off the warhead. +#[test] +fn predator_missile_runs_commands() { + let mut mods = load("predator_missile.hmod"); + assert_eq!(mods.entries()[0].status, Status::Running); + mods.set_snapshot(player()); + let commands = |mods: &mut Mods| -> Vec { + mods.drain() + .into_iter() + .filter_map(|o| match o { + Output::Command { line } => Some(line), + _ => None, + }) + .collect() + }; + assert!(mods.command("missile")); + let launch = commands(&mut mods); + assert_eq!(launch.len(), 6); + assert!(launch.contains(&"gamemode spectator".to_string())); + mods.tick(1); + mods.tick(2); + let flight = commands(&mut mods); + assert!(flight.iter().any(|l| l.ends_with("run tp @s ^ ^ ^1.5"))); + mods.chat("Predator: impact"); + let blast = commands(&mut mods); + assert_eq!(blast.len(), 4); + assert!(blast.iter().all(|l| l.contains("summon tnt"))); +} + #[test] fn coords_hud_draws_and_answers() { let mut mods = load("coords_hud.hmod"); diff --git a/docs/api/mods-reference.md b/docs/api/mods-reference.md index ca2fe9d7a..28703e784 100644 --- a/docs/api/mods-reference.md +++ b/docs/api/mods-reference.md @@ -11,11 +11,12 @@ The contract between Horizon and its mods: the WIT package `horizon:mods@0.1.0`, | `chat` | [`chat`](#chat) | show you chat messages | | `hud` | [`hud`](#hud) | draw text on your screen | | `storage` | [`storage`](#storage) | save its own settings to disk | +| `command` | [`command`](#command) | run commands as you, with your permissions | | `registries` | data files (`assets/`, `data/`), no interface | add blocks and items to your worlds | ## World `horizon-mod` -- Imports (what the game offers): `log`, `game`, `chat`, `hud`, `storage` +- Imports (what the game offers): `log`, `game`, `chat`, `hud`, `storage`, `command` - Exports (what the mod provides): `events` ## `log` @@ -117,6 +118,16 @@ Capability `storage`: a per-mod folder of small key/value files, quota'd (docs/m | `set` | `func(key: string, value: list) -> result<_, string>` | Errs on a malformed key or over quota (bytes for the whole mod). | | `delete` | `func(key: string)` | No error if `key` was already absent. | +## `command` + +Access: capability `command`. + +Capability `command`: vanilla commands run as the player, with the player's permissions, after the hook call returns (8 per call). A mod's own commands are not run this way. + +| Function | Signature | What it does | +| --- | --- | --- | +| `run` | `func(line: string)` | `line` without its slash; its feedback goes to the chat as if typed. | + ## `events` Access: exported by the mod: the game calls it. diff --git a/docs/mods.md b/docs/mods.md index d8ea925b3..5d386720b 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -13,7 +13,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. |---|---|---| | The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage; tick, frame, chat and command hooks | | SDK for mod authors | `crates/horizon-mod-sdk` | 0.1 | -| Example mod | `crates/horizon-mod-sdk/examples/coords-hud` | coordinates, facing and time HUD, `/where`, `/coords` | +| Example mods | `crates/horizon-mod-sdk/examples/coords-hud`, `examples/predator-missile` | coordinates, facing and time HUD, `/where`, `/coords`; `/missile`, a guided missile from vanilla commands | | Runtime | `crates/horizon-mods` | wasmtime 36 LTS, component model | | Authoring tool | `crates/cargo-horizon-mod` | `cargo horizon-mod new / build / run` | | In the game | `crates/horizon-client/src/game/mods.rs`, Options > Mods | client mods | @@ -84,6 +84,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | `chat` | `chat` | `show(text)`: a line only the local player sees, after the mod's name (8 per call) | | `hud` | `hud` | `set-panel(corner, lines)`, `clear()`: up to 16 lines of 256 characters | | `storage` | `storage` | `get(key)`, `set(key, value)`, `delete(key)`: a per-mod folder of up to 64 files, 4 MiB total, at most 1 MiB written per call | +| `command` | `command` | `run(line)`: a vanilla command run as the player, with the player's permissions, after the hook returns (8 per call); mods' own commands are not run | | Hook (export) | When | |---|---| From 0f9de6dbb0ce0e1aeef50e5b960a3040357a6132 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:39:28 +0200 Subject: [PATCH 02/61] Shader packs, clean room 14: terrain drawn by the pack's program With a pack whose gbuffers_terrain (or a fallback) loads, solid and cutout terrain is drawn by it into the pack's buffers, in place, between prepare and deferred, and Horizon's own solid terrain draw is skipped. - depthtex0 is now a Depth32Float attachment: the blit writes the world's depth there (GL window depth, GL rows), terrain tests and writes against it, so entities and terrain occlude each other; later programs sample it as before. - The vertex glue reads Horizon's TerrainVertex on both draw paths (instanced, and flat with the page tables in bind group 1) and gives atlas coordinates, lightmap coordinates in 0-240, the vertex colour, mc_midTexCoord at the sprite's centre and mc_Entity = -1. Normals, tangents and block ids come later. - texture/gtexture/tex sample the block atlas, lightmap the lightmap; atlasSize is set. - Occlusion culling is off while a pack draws terrain (it reads the depth of Horizon's terrain draw), and flat draws stay on. - Uniforms a program reads that are not provided yet are reported. - Program building is shared by quads and terrain (build_pass, bind). - terrain-test: Horizon's own pack for this slice. Co-Authored-By: Claude Opus 5.5 --- .../horizon-client/src/game/shader_packs.rs | 3 +- crates/horizon-client/src/gfx.rs | 106 ++- crates/horizon-client/src/gfx/flat.rs | 3 +- crates/horizon-client/src/gfx/pack_atlas.rs | 69 +- crates/horizon-client/src/gfx/pack_post.rs | 716 +++++++++++------- crates/horizon-client/src/gfx/pack_post.wgsl | 7 +- .../horizon-client/src/gfx/pack_uniforms.rs | 5 + .../tests/packs/terrain-test/README.md | 8 + .../packs/terrain-test/shaders/composite.fsh | 19 + .../packs/terrain-test/shaders/composite.vsh | 8 + .../terrain-test/shaders/gbuffers_terrain.fsh | 22 + .../terrain-test/shaders/gbuffers_terrain.vsh | 16 + 12 files changed, 702 insertions(+), 280 deletions(-) create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/README.md create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/shaders/composite.fsh create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/shaders/composite.vsh create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.vsh diff --git a/crates/horizon-client/src/game/shader_packs.rs b/crates/horizon-client/src/game/shader_packs.rs index 04af297b6..d819eddef 100644 --- a/crates/horizon-client/src/game/shader_packs.rs +++ b/crates/horizon-client/src/game/shader_packs.rs @@ -42,7 +42,8 @@ impl Game<'_> { ); } let folder = dimension_folder(&pack, dimension); - let post = load_post(&pack, &folder, &properties, ¯os, None); + let terrain = renderer.pack_terrain_inputs(); + let post = load_post(&pack, &folder, &properties, ¯os, terrain.as_ref()); if let Some((programs, notes)) = renderer.set_shader_pack(dimension, Some(post)) { for note in notes { diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 9f05adb65..c117b190f 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -471,6 +471,8 @@ pub struct Renderer { pack_post: Option<(i32, Option)>, /// The block sprites as one atlas, while a shader pack is loaded. pack_atlas: Option, + /// The pack's terrain program reads flat draws' vertices (`pack_terrain_inputs`). + pack_terrain_flat: bool, pub hud: HudRenderer, /// Vanilla's menu background blur, made on first use. menu_blur: Option, @@ -2319,6 +2321,7 @@ impl Renderer { world_post, pack_post: None, pack_atlas: None, + pack_terrain_flat: false, hud, menu_blur: None, info, @@ -2744,17 +2747,50 @@ impl Renderer { dimension: i32, pack: Option, ) -> Option<(Vec, Vec)> { + if pack.is_none() { + self.pack_atlas = None; + } + let instance = wgpu::vertex_attr_array![5=>Float32x4]; + let vertex = wgpu::VertexBufferLayout { + array_stride: std::mem::size_of::() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &TERRAIN_ATTRIBUTES, + }; + let instances = wgpu::VertexBufferLayout { + array_stride: 16, + step_mode: wgpu::VertexStepMode::Instance, + attributes: &instance, + }; + let buffers = [Some(vertex), Some(instances)]; + let flat = self.pack_terrain_flat; let post = pack.map(|post| { + let world = pack_post::World { + atlas: self.pack_atlas.as_ref().map(|a| (a.view.clone(), a.size())), + lightmap: self.lightmap_view.clone(), + terrain: self.pack_atlas.as_ref().map(|_| pack_post::Geometry { + buffers: if flat { &buffers[..1] } else { &buffers }, + table: self.flat.as_ref().filter(|_| flat).map(|f| &f.draw_layout), + }), + }; pack_post::PackPost::new( &self.device, &self.queue, post, self.config.format.remove_srgb_suffix(), + world, ) }); - if post.is_none() { - self.pack_atlas = None; - } else if self.pack_atlas.is_none() { + let report = post.as_ref().map(|p| { + let programs = p.programs().into_iter().map(str::to_owned).collect(); + (programs, p.notes.clone()) + }); + self.pack_post = Some((dimension, post)); + report + } + /// What a shader pack's terrain program reads from Horizon's terrain vertices, in the + /// pack's block atlas (made here when missing); `None` when the atlas does not fit. + pub fn pack_terrain_inputs(&mut self) -> Option { + if self.pack_atlas.is_none() { let mut encoder = self.device.create_command_encoder(&Default::default()); let layers = self .sprite_textures @@ -2771,12 +2807,13 @@ impl Renderer { ); self.queue.submit([encoder.finish()]); } - let report = post.as_ref().map(|p| { - let programs = p.programs().into_iter().map(str::to_owned).collect(); - (programs, p.notes.clone()) - }); - self.pack_post = Some((dimension, post)); - report + // Flat draws whenever the device has them (`render` keeps them on with a pack). + self.pack_terrain_flat = self.flat.is_some() && self.flat_on; + Some( + self.pack_atlas + .as_ref()? + .terrain_inputs(self.pack_terrain_flat), + ) } /// The dimension the shader pack was last set for. pub fn shader_pack_dimension(&self) -> Option { @@ -3184,11 +3221,23 @@ impl Renderer { // Occlusion culling leaves the sections its last result did not see // to phase 2, after the others, with instance counts of 0. Whether // the frame has draws enough goes by the last one's. + // A shader pack drawing terrain draws all of it: occlusion culling reads the depth of + // Horizon's own terrain draw. + let pack_terrain = !frame.title + && self + .pack_post + .as_ref() + .and_then(|p| p.1.as_ref()) + .is_some_and(|p| p.has_terrain()); let mut cull = self .cull .as_mut() .filter(|c| { - self.cull_on && !frame.title && self.multi_draw && self.section_draws >= c.min_draws + self.cull_on + && !frame.title + && !pack_terrain + && self.multi_draw + && self.section_draws >= c.min_draws }) .and_then(|cull| cull.begin(&self.device).then_some(cull)); self.phase2_draws.clear(); @@ -3591,7 +3640,9 @@ impl Renderer { } let flat_frame = match &mut self.flat { Some(flat) - if self.flat_on && !self.draws.is_empty() && self.draws.len() >= flat.min_draws => + if self.flat_on + && !self.draws.is_empty() + && (self.draws.len() >= flat.min_draws || pack_terrain) => { let sections = &self.sections; let lods = &self.lods; @@ -3651,6 +3702,8 @@ impl Renderer { } // The world's targets: the screen, or the scaled target of `post`. let pack_post = !frame.title && self.pack_post.as_ref().is_some_and(|p| p.1.is_some()); + // The pack's program reads the vertices this frame's draws bind. + let pack_draws_terrain = pack_terrain && flat_frame == self.pack_terrain_flat; let scaled = self.world_post.prepare( &self.device, [self.config.width, self.config.height], @@ -3739,7 +3792,14 @@ impl Renderer { pass.draw(0..3, 0..1); } } - match self.flat.as_ref().filter(|_| flat_frame) { + match self + .flat + .as_ref() + .filter(|_| flat_frame && !pack_draws_terrain) + { + _ if pack_draws_terrain => { + pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32); + } Some(flat) => { pass.set_pipeline(&flat.solid); let vertices = |p| self.arena.buffer(p); @@ -3947,9 +4007,27 @@ impl Renderer { } if let Some((color, depth)) = &scaled && pack_post - && let Some((_, Some(post))) = &mut self.pack_post + && let Some((dimension, Some(mut post))) = self.pack_post.take() { let size = self.world_post.size().unwrap_or([1, 1]); + let flat = self.flat.as_ref().filter(|_| flat_frame); + let mut terrain = |rp: &mut wgpu::RenderPass<'_>| { + if !pack_draws_terrain { + return; + } + match flat { + Some(flat) => { + flat.draw(rp, &flat.calls[..flat.solid_calls], |p| { + self.arena.buffer(p) + }); + } + None => { + rp.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32); + rp.set_vertex_buffer(1, self.instances.slice(..)); + self.draw_sections(rp, &self.indirect, 0, 0, solid_draws); + } + } + }; post.run( &self.device, &self.queue, @@ -3961,7 +4039,9 @@ impl Renderer { &frame.pack, size, [self.config.width, self.config.height], + &mut terrain, ); + self.pack_post = Some((dimension, Some(post))); } else if scaled.is_some() { self.world_post.finish(&mut encoder, &view_target); } diff --git a/crates/horizon-client/src/gfx/flat.rs b/crates/horizon-client/src/gfx/flat.rs index f6823c5d8..4c337025a 100644 --- a/crates/horizon-client/src/gfx/flat.rs +++ b/crates/horizon-client/src/gfx/flat.rs @@ -73,7 +73,8 @@ pub struct Flat { phase2_call_data: Vec<[u32; 5]>, /// The copy of phase 2's draws, and its ranges. phase2_group: Option<(wgpu::BindGroup, u32)>, - draw_layout: wgpu::BindGroupLayout, + /// The layout of a page's bind group 1 (shader packs' terrain pipelines share it). + pub(super) draw_layout: wgpu::BindGroupLayout, blocks_per_page: u64, pages: Vec>, free_ids: Vec, diff --git a/crates/horizon-client/src/gfx/pack_atlas.rs b/crates/horizon-client/src/gfx/pack_atlas.rs index c90fa6ac1..011dff379 100644 --- a/crates/horizon-client/src/gfx/pack_atlas.rs +++ b/crates/horizon-client/src/gfx/pack_atlas.rs @@ -3,7 +3,6 @@ //! layer is copied into a grid, row by row from the top left, and an animated layer again //! whenever its frame changes. -#[expect(dead_code, reason = "the gbuffers pass binds it (next slice)")] pub(super) struct PackAtlas { texture: wgpu::Texture, pub view: wgpu::TextureView, @@ -64,10 +63,6 @@ impl PackAtlas { } /// The atlas's size in pixels (`atlasSize`). - #[cfg_attr( - not(test), - expect(dead_code, reason = "the gbuffers pass reads it (next slice)") - )] pub fn size(&self) -> [u32; 2] { [self.texture.width(), self.texture.height()] } @@ -118,6 +113,70 @@ impl PackAtlas { } } +impl PackAtlas { + /// The vertex inputs of a gbuffers program drawing terrain (`TerrainVertex`, terrain.wgsl's + /// `vs_terrain` or, for `flat` draws, `vs_terrain_flat`), in atlas coordinates. + pub fn terrain_inputs(&self, flat: bool) -> horizon_shaders::GbuffersInputs { + let mut declarations = String::from( + "layout(location = 0) in vec3 hz_v_pos;\nlayout(location = 1) in uvec2 hz_v_uv;\n\ + layout(location = 2) in uint hz_v_layer;\nlayout(location = 3) in vec4 hz_v_color;\n\ + layout(location = 4) in uvec2 hz_v_light;\n", + ); + let origin = if flat { + declarations.push_str( + "layout(set = 1, binding = 0) readonly buffer HzBlockSlots { uint hz_block_slots[]; };\n\ + layout(set = 1, binding = 1) readonly buffer HzSlotOrigins { vec4 hz_slot_origins[]; };\n", + ); + "hz_slot_origins[hz_block_slots[uint(gl_VertexIndex) >> 8u]]" + } else { + declarations.push_str("layout(location = 5) in vec4 hz_v_origin;\n"); + "hz_v_origin" + }; + let [width, height] = self.size(); + let scale = [ + self.sprite as f32 / width as f32, + self.sprite as f32 / height as f32, + ]; + let columns = self.columns; + // UVs past a sprite (models up to 2, distant tiles) cannot repeat in an atlas: clamped. + let body = format!( + " vec4 hz_origin = {origin};\n \ + uint hz_layer = hz_v_layer & 32767u;\n \ + hz_cell = vec2(float(hz_layer % {columns}u), float(hz_layer / {columns}u));\n \ + hz_scale = vec2({:?}, {:?});\n \ + hz_Vertex = vec4(hz_v_pos + hz_origin.xyz, 1.0);\n \ + hz_MultiTexCoord0 = vec4((hz_cell + clamp(vec2(hz_v_uv) * hz_origin.w, 0.0, 1.0)) * hz_scale, 0.0, 1.0);\n \ + hz_MultiTexCoord1 = vec4(float(hz_v_light.y), float(hz_v_light.x), 0.0, 1.0);\n \ + hz_MultiTexCoord2 = hz_MultiTexCoord1;\n \ + hz_Color = hz_v_color;\n \ + hz_Normal = vec3(0.0, 1.0, 0.0);\n", + scale[0], scale[1], + ); + declarations.push_str("vec2 hz_cell; vec2 hz_scale;\n"); + let attributes = [ + ("mc_Entity", "vec4(-1.0, -1.0, 0.0, 0.0)"), + ( + "mc_midTexCoord", + "vec4((hz_cell + 0.5) * hz_scale, 0.0, 1.0)", + ), + ("at_tangent", "vec4(1.0, 0.0, 0.0, 1.0)"), + ("vaPosition", "hz_Vertex.xyz"), + ("vaUV0", "hz_MultiTexCoord0.xy"), + ("vaUV2", "hz_MultiTexCoord1.xy"), + ("vaColor", "hz_Color"), + ("vaNormal", "hz_Normal"), + ]; + horizon_shaders::GbuffersInputs { + declarations, + body, + attributes: attributes + .iter() + .map(|&(n, v)| (n.to_owned(), v.to_owned())) + .collect(), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index d5aa9fb7c..81a517dba 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -29,6 +29,9 @@ enum Source { Dummy, /// A 1×1 depth of 1.0 behind a comparison sampler, until shadow maps exist: no shadow. Shadow, + /// The block atlas (pack_atlas.rs), in gbuffers programs. + Atlas, + Lightmap, } struct Binding { @@ -66,9 +69,22 @@ struct Targets { depth: wgpu::TextureView, } +/// What a pack's gbuffers programs read and draw from the game's world. +pub(super) struct World<'a> { + /// The block atlas and its size. + pub atlas: Option<(wgpu::TextureView, [u32; 2])>, + pub lightmap: wgpu::TextureView, + /// Terrain's geometry, when the pack draws it. + pub terrain: Option>, +} + pub struct PackPost { pub notes: Vec, passes: Vec, + /// What draws terrain (`gbuffers_terrain` or its fallback), between `prepare` and `deferred`. + terrain: Option, + atlas: Option, + lightmap: wgpu::TextureView, deferred_pre: u16, composite_pre: u16, formats: [wgpu::TextureFormat; BUFFERS], @@ -85,6 +101,7 @@ pub struct PackPost { linear: wgpu::Sampler, nearest: wgpu::Sampler, repeat: wgpu::Sampler, + nearest_repeat: wgpu::Sampler, blit: (wgpu::RenderPipeline, wgpu::BindGroupLayout), /// The blit also writes depthtex0 (some program reads it) and clears these buffers. blit_depth: bool, @@ -133,6 +150,8 @@ fn source(name: &str) -> Source { match name { "depthtex0" | "gdepthtex" | "depthtex1" | "depthtex2" => Source::Depth, "noisetex" => Source::Noise, + "texture" | "gtexture" | "tex" => Source::Atlas, + "lightmap" => Source::Lightmap, n => horizon_shaders::buffer_index(n).map_or(Source::Dummy, Source::Color), } } @@ -228,6 +247,7 @@ fn helper( entry: &str, entries: &[wgpu::BindGroupLayoutEntry], targets: &[Option], + depth: Option, ) -> (wgpu::RenderPipeline, wgpu::BindGroupLayout) { let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("Shader pack helpers"), @@ -252,7 +272,7 @@ fn helper( buffers: &[], }, primitive: Default::default(), - depth_stencil: None, + depth_stencil: depth, multisample: Default::default(), fragment: Some(wgpu::FragmentState { module: &module, @@ -306,6 +326,211 @@ fn attach( }) } +/// The game's geometry a gbuffers program draws: its vertex buffers and, for flat draws +/// (flat.rs), the layout of their bind group 1. +pub(super) struct Geometry<'a> { + pub buffers: &'a [Option>], + pub table: Option<&'a wgpu::BindGroupLayout>, +} + +/// The pass of program `p`: a quad, or `geometry` drawn by the game. `None` (with a note) when +/// wgpu refuses it or it writes nothing. +fn build_pass( + device: &wgpu::Device, + p: horizon_shaders::PostProgram, + formats: &[wgpu::TextureFormat; BUFFERS], + screen: wgpu::TextureFormat, + notes: &mut Vec, + geometry: Option<&Geometry<'_>>, +) -> Option { + let features = device.features(); + let mask = |flips: &[(u32, bool)]| flips.iter().filter(|f| f.1).fold(0u16, |m, f| m | 1 << f.0); + let interface = &p.program.interface; + let mut entries = vec![entry( + 0, + wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + )]; + let mut samplers = Vec::new(); + let mut reads = 0u16; + for s in &interface.samplers { + let mut src = source(&s.name); + if src == Source::Atlas && geometry.is_none() { + src = Source::Dummy; + } + let float = wgpu::TextureSampleType::Float { filterable: true }; + let mut sample_type = match src { + Source::Color(n) => formats[n] + .sample_type(None, Some(features)) + .unwrap_or(float), + Source::Depth => wgpu::TextureSampleType::Float { filterable: false }, + // Unfilterable on WebGPU-level devices; the lightmap is sampled at texel centres. + Source::Lightmap => float, + _ => float, + }; + let int = !matches!(sample_type, wgpu::TextureSampleType::Float { .. }); + if s.ty == "sampler2DShadow" { + (src, sample_type) = (Source::Shadow, wgpu::TextureSampleType::Depth); + } + let wanted = match s.ty.as_str() { + "sampler2DShadow" => true, + "sampler2D" => !int, + "isampler2D" => sample_type == wgpu::TextureSampleType::Sint, + "usampler2D" => sample_type == wgpu::TextureSampleType::Uint, + _ => false, + }; + if !wanted { + notes.push(format!( + "{}: `{} {}` is not provided yet", + p.name, s.ty, s.name + )); + (src, sample_type) = (Source::Dummy, float); + } + let filtering = sample_type == float; + if let Source::Color(n) = src { + reads |= 1 << n; + } + entries.push(texture_entry(s.binding, sample_type)); + entries.push(entry( + s.binding + 1, + wgpu::BindingType::Sampler(if src == Source::Shadow { + wgpu::SamplerBindingType::Comparison + } else if filtering { + wgpu::SamplerBindingType::Filtering + } else { + wgpu::SamplerBindingType::NonFiltering + }), + )); + samplers.push(Binding { + source: src, + binding: s.binding, + filtering, + }); + } + let final_pass = p.group == Group::Final; + let mut targets = Vec::new(); + let mut writes = Vec::new(); + for &location in &interface.outputs { + let l = location as usize; + if targets.len() <= l { + targets.resize(l + 1, None); + } + if final_pass { + if location == 0 { + targets[0] = target(screen); + } + } else if let Some(&buffer) = p.buffers.get(l) { + targets[l] = target(formats[buffer as usize]); + writes.push((location, buffer as usize)); + } + } + if !final_pass && writes.is_empty() { + notes.push(format!("{}: writes no buffer; skipped", p.name)); + return None; + } + let written = writes.iter().fold(0u16, |m, w| m | 1 << w.1); + let kept = p + .flips + .iter() + .filter(|f| !f.1) + .fold(0u16, |m, f| m | 1 << f.0); + // Gbuffers programs write in place (6.1.6). + let flip = if final_pass || geometry.is_some() { + 0 + } else { + (written & !kept) | mask(&p.flips) + }; + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some(&p.name), + entries: &entries, + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some(&p.name), + bind_group_layouts: &[Some(&layout), geometry.and_then(|g| g.table)], + ..Default::default() + }); + let created = checked(device, || { + let module = |m: naga::Module| { + device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some(&p.name), + source: wgpu::ShaderSource::Naga(std::borrow::Cow::Owned(m)), + }) + }; + let (vs, fs) = (module(p.program.vertex), module(p.program.fragment)); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(&p.name), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &vs, + entry_point: Some("main"), + compilation_options: Default::default(), + buffers: geometry.map_or(&[], |g| g.buffers), + }, + primitive: Default::default(), + depth_stencil: geometry.map(|_| wgpu::DepthStencilState { + format: wgpu::TextureFormat::Depth32Float, + depth_write_enabled: Some(true), + depth_compare: Some(wgpu::CompareFunction::LessEqual), + stencil: Default::default(), + bias: Default::default(), + }), + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &fs, + entry_point: Some("main"), + compilation_options: Default::default(), + targets: &targets, + }), + multiview_mask: None, + cache: None, + }) + }); + let pipeline = match created { + Ok(p) => p, + Err(e) => { + notes.push(format!("{}: skipped: {e}", p.name)); + return None; + } + }; + let interface = &p.program.interface; + let missing: Vec<&str> = interface + .uniforms + .iter() + .filter(|u| !u.name.starts_with("hz_") && pack_uniforms::index(&u.name).is_none()) + .map(|u| u.name.as_str()) + .collect(); + if !missing.is_empty() { + notes.push(format!("{}: zero for now: {}", p.name, missing.join(" "))); + } + Some(Pass { + ubo: device.create_buffer(&wgpu::BufferDescriptor { + label: Some(&p.name), + size: u64::from(interface.uniform_size.max(16)), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }), + block: vec![0; interface.uniform_size as usize], + uniforms: interface + .uniforms + .iter() + .map(|u| (pack_uniforms::index(&u.name), u.ty.clone(), u.offset)) + .collect(), + name: p.name, + group: p.group, + pipeline, + layout, + samplers, + writes, + flip, + reads, + written, + groups: HashMap::new(), + }) +} + impl PackPost { /// Builds the pipelines of `post` for a screen of `screen` format. pub fn new( @@ -313,6 +538,7 @@ impl PackPost { queue: &wgpu::Queue, post: Post, screen: wgpu::TextureFormat, + world: World<'_>, ) -> Self { let features = device.features(); let formats: [wgpu::TextureFormat; BUFFERS] = @@ -329,205 +555,56 @@ impl PackPost { let mask = |flips: &[(u32, bool)]| flips.iter().filter(|f| f.1).fold(0u16, |m, f| m | 1 << f.0); let mut used = 1u16; - let mut passes = Vec::new(); - for p in post.programs { - let interface = &p.program.interface; - let mut entries = vec![entry( - 0, - wgpu::BindingType::Buffer { - ty: wgpu::BufferBindingType::Uniform, - has_dynamic_offset: false, - min_binding_size: None, - }, - )]; - let mut samplers = Vec::new(); - let mut reads = 0u16; - for s in &interface.samplers { - let mut src = source(&s.name); - let float = wgpu::TextureSampleType::Float { filterable: true }; - let mut sample_type = match src { - Source::Color(n) => formats[n] - .sample_type(None, Some(features)) - .unwrap_or(float), - Source::Depth => wgpu::TextureSampleType::Float { filterable: false }, - _ => float, - }; - let int = !matches!(sample_type, wgpu::TextureSampleType::Float { .. }); - if s.ty == "sampler2DShadow" { - (src, sample_type) = (Source::Shadow, wgpu::TextureSampleType::Depth); - } - let wanted = match s.ty.as_str() { - "sampler2DShadow" => true, - "sampler2D" => !int, - "isampler2D" => sample_type == wgpu::TextureSampleType::Sint, - "usampler2D" => sample_type == wgpu::TextureSampleType::Uint, - _ => false, - }; - if !wanted { - notes.push(format!( - "{}: `{} {}` is not provided yet", - p.name, s.ty, s.name - )); - (src, sample_type) = (Source::Dummy, float); - } - let filtering = sample_type == float; - if let Source::Color(n) = src { - reads |= 1 << n; - } - entries.push(texture_entry(s.binding, sample_type)); - entries.push(entry( - s.binding + 1, - wgpu::BindingType::Sampler(if src == Source::Shadow { - wgpu::SamplerBindingType::Comparison - } else if filtering { - wgpu::SamplerBindingType::Filtering - } else { - wgpu::SamplerBindingType::NonFiltering - }), - )); - samplers.push(Binding { - source: src, - binding: s.binding, - filtering, - }); - } - let final_pass = p.group == Group::Final; - let mut targets = Vec::new(); - let mut writes = Vec::new(); - for &location in &interface.outputs { - let l = location as usize; - if targets.len() <= l { - targets.resize(l + 1, None); - } - if final_pass { - if location == 0 { - targets[0] = target(screen); - } - } else if let Some(&buffer) = p.buffers.get(l) { - targets[l] = target(formats[buffer as usize]); - writes.push((location, buffer as usize)); - } - } - if !final_pass && writes.is_empty() { - notes.push(format!("{}: writes no buffer; skipped", p.name)); - continue; - } - let written = writes.iter().fold(0u16, |m, w| m | 1 << w.1); - let kept = p - .flips - .iter() - .filter(|f| !f.1) - .fold(0u16, |m, f| m | 1 << f.0); - let flip = if final_pass { - 0 - } else { - (written & !kept) | mask(&p.flips) - }; - used |= reads | written | flip; - let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some(&p.name), - entries: &entries, - }); - let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some(&p.name), - bind_group_layouts: &[Some(&layout)], - ..Default::default() - }); - let created = checked(device, || { - let module = |m: naga::Module| { - device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some(&p.name), - source: wgpu::ShaderSource::Naga(std::borrow::Cow::Owned(m)), - }) - }; - let (vs, fs) = (module(p.program.vertex), module(p.program.fragment)); - device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some(&p.name), - layout: Some(&pipeline_layout), - vertex: wgpu::VertexState { - module: &vs, - entry_point: Some("main"), - compilation_options: Default::default(), - buffers: &[], - }, - primitive: Default::default(), - depth_stencil: None, - multisample: Default::default(), - fragment: Some(wgpu::FragmentState { - module: &fs, - entry_point: Some("main"), - compilation_options: Default::default(), - targets: &targets, - }), - multiview_mask: None, - cache: None, - }) - }); - let pipeline = match created { - Ok(p) => p, - Err(e) => { - notes.push(format!("{}: skipped: {e}", p.name)); - continue; - } - }; - let interface = &p.program.interface; - passes.push(Pass { - ubo: device.create_buffer(&wgpu::BufferDescriptor { - label: Some(&p.name), - size: u64::from(interface.uniform_size.max(16)), - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - mapped_at_creation: false, - }), - block: vec![0; interface.uniform_size as usize], - uniforms: interface - .uniforms - .iter() - .map(|u| (pack_uniforms::index(&u.name), u.ty.clone(), u.offset)) - .collect(), - name: p.name, - group: p.group, - pipeline, - layout, - samplers, - writes, - flip, - reads, - written, - groups: HashMap::new(), - }); + let mut build = |p: horizon_shaders::PostProgram, geometry: Option<&Geometry<'_>>| { + let pass = build_pass(device, p, &formats, screen, &mut notes, geometry)?; + used |= pass.reads | pass.written | pass.flip; + Some(pass) + }; + let terrain = post + .terrain + .zip(world.terrain.as_ref()) + .and_then(|(p, g)| build(p, Some(g))); + let passes: Vec = post + .programs + .into_iter() + .filter_map(|p| build(p, None)) + .collect(); + if let Some(t) = &terrain { + notes.push(format!("terrain draws with {}", t.name)); } // A clear only matters when a program reads the buffer before one writes all of it - // (composite passes cover their whole target); the others are skipped. + // (composite passes cover their whole target, terrain does not); the others are skipped. let mut cleared = Vec::new(); for b in 1..BUFFERS { let bit = 1u16 << b; - let first = passes - .iter() - .find(|p| (p.reads | p.flip | p.written) & bit != 0); - if clears[b].is_some() - && first.is_some_and(|p| p.reads & bit != 0 || p.written & bit == 0) - { + let touched = |p: &&Pass| (p.reads | p.flip | p.written) & bit != 0; + let first = terrain.iter().find(touched).map(|_| true).or_else(|| { + let p = passes.iter().find(touched)?; + Some(p.reads & bit != 0 || p.written & bit == 0) + }); + if clears[b].is_some() && first == Some(true) { cleared.push(b); } } - let blit_depth = passes - .iter() - .any(|p| p.samplers.iter().any(|s| s.source == Source::Depth)); - let room = 8 - 1 - usize::from(blit_depth); + // Terrain tests against the world's depth (entities...), so the blit copies it then. + let blit_depth = terrain.is_some() + || passes + .iter() + .any(|p| p.samplers.iter().any(|s| s.source == Source::Depth)); + let room = 8 - 1; let other_clears = cleared.split_off(room.min(cleared.len())); let float = wgpu::TextureSampleType::Float { filterable: true }; - let mut blit_targets = vec![target(formats[0])]; - if blit_depth { - blit_targets.push(target(wgpu::TextureFormat::R32Float)); - } + let blit_targets = std::iter::once(target(formats[0])); // Cleared by the pass's load operation; the shader writes nothing there. - blit_targets.extend(cleared.iter().map(|&b| { - Some(wgpu::ColorTargetState { - format: formats[b], - blend: None, - write_mask: wgpu::ColorWrites::empty(), - }) - })); + let blit_targets: Vec<_> = blit_targets + .chain(cleared.iter().map(|&b| { + Some(wgpu::ColorTargetState { + format: formats[b], + blend: None, + write_mask: wgpu::ColorWrites::empty(), + }) + })) + .collect(); let blit = helper( device, "vs_full", @@ -541,6 +618,14 @@ impl PackPost { texture_entry(1, wgpu::TextureSampleType::Depth), ], &blit_targets, + // GL's window depth, written for depthtex0 and the terrain's depth test. + blit_depth.then_some(wgpu::DepthStencilState { + format: wgpu::TextureFormat::Depth32Float, + depth_write_enabled: Some(true), + depth_compare: Some(wgpu::CompareFunction::Always), + stencil: Default::default(), + bias: Default::default(), + }), ); let copy = helper( device, @@ -554,11 +639,15 @@ impl PackPost { ), ], &[target(screen)], + None, ); let white = [255u8; 4]; Self { notes, passes, + terrain, + atlas: world.atlas.as_ref().map(|a| a.0.clone()), + lightmap: world.lightmap, deferred_pre: mask(&post.deferred_pre), composite_pre: mask(&post.composite_pre), formats, @@ -608,12 +697,153 @@ impl PackPost { wgpu::AddressMode::ClampToEdge, ), repeat: sampler(device, wgpu::FilterMode::Linear, wgpu::AddressMode::Repeat), + nearest_repeat: sampler(device, wgpu::FilterMode::Nearest, wgpu::AddressMode::Repeat), blit, copy, - uniforms: FrameUniforms::default(), + uniforms: { + let mut u = FrameUniforms::default(); + u.atlas = world.atlas.map_or([0; 2], |a| a.1); + u + }, } } + /// Writes `pass`'s uniforms for a target of `out` pixels; its bind group for the buffers' + /// current mapping. + fn bind( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + pass: &mut Pass, + out: [u32; 2], + ) -> wgpu::BindGroup { + let fog = self.uniforms.fog(); + // Our block members first (`hz_target`, `hz_fog_color`, `hz_fog`), then the pack's. + let ours = [ + [ + out[0] as f32, + out[1] as f32, + 1.0 / out[0] as f32, + 1.0 / out[1] as f32, + ], + fog[0].to_array(), + fog[1].to_array(), + ]; + let head: &[u8] = bytemuck::cast_slice(&ours); + let n = head.len().min(pass.block.len()); + pass.block[..n].copy_from_slice(&head[..n]); + for (index, ty, offset) in &pass.uniforms { + if let Some(v) = index.and_then(|i| self.uniforms.values.get(i)) { + v.write(ty, &mut pass.block, *offset as usize); + } + } + if !pass.block.is_empty() { + queue.write_buffer(&pass.ubo, 0, &pass.block); + } + let targets = self.targets.as_ref().expect("resized"); + let key = self.main & pass.reads; + let group = pass.groups.entry(key).or_insert_with(|| { + let mut entries = vec![wgpu::BindGroupEntry { + binding: 0, + resource: pass.ubo.as_entire_binding(), + }]; + for s in &pass.samplers { + let filter = if s.filtering { + &self.linear + } else { + &self.nearest + }; + let (texture, sampler) = match s.source { + Source::Color(i) => ( + targets.color[i] + .as_ref() + .map_or(&self.dummy, |t| &t[usize::from(key & 1 << i != 0)]), + filter, + ), + Source::Depth => (&targets.depth, &self.nearest), + Source::Noise => (&self.noise, &self.repeat), + Source::Dummy => (&self.dummy, &self.linear), + Source::Shadow => (&self.shadow.0, &self.shadow.1), + // Blocks look as the game draws them: nearest texels. + Source::Atlas => ( + self.atlas.as_ref().unwrap_or(&self.dummy), + &self.nearest_repeat, + ), + Source::Lightmap => (&self.lightmap, &self.linear), + }; + entries.push(wgpu::BindGroupEntry { + binding: s.binding, + resource: wgpu::BindingResource::TextureView(texture), + }); + entries.push(wgpu::BindGroupEntry { + binding: s.binding + 1, + resource: wgpu::BindingResource::Sampler(sampler), + }); + } + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(&pass.name), + layout: &pass.layout, + entries: &entries, + }) + }); + group.clone() + } + + /// Terrain with the pack's program into the buffers it writes, in place, tested against + /// the world's depth; `draw` issues the game's draws. + fn draw_terrain( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + encoder: &mut wgpu::CommandEncoder, + size: [u32; 2], + draw: &mut dyn FnMut(&mut wgpu::RenderPass<'_>), + ) { + let Some(mut pass) = self.terrain.take() else { + return; + }; + let group = self.bind(device, queue, &mut pass, size); + let targets = self.targets.as_ref().expect("resized"); + let load = wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }; + let mut attachments: Vec> = Vec::new(); + for &(location, buffer) in &pass.writes { + let l = location as usize; + if attachments.len() <= l { + attachments.resize(l + 1, None); + } + attachments[l] = targets.color[buffer] + .as_ref() + .and_then(|t| attach(&t[usize::from(self.main & 1 << buffer != 0)], load)); + } + { + let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some(&pass.name), + color_attachments: &attachments, + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &targets.depth, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }), + stencil_ops: None, + }), + ..Default::default() + }); + rp.set_pipeline(&pass.pipeline); + rp.set_bind_group(0, &group, &[]); + draw(&mut rp); + } + self.terrain = Some(pass); + } + + /// Whether the pack draws terrain itself. + pub fn has_terrain(&self) -> bool { + self.terrain.is_some() + } + /// The names of the programs that run, in order. pub fn programs(&self) -> Vec<&str> { self.passes.iter().map(|p| p.name.as_str()).collect() @@ -651,9 +881,9 @@ impl PackPost { self.targets = Some(Targets { size, color, - depth: make("depthtex0", wgpu::TextureFormat::R32Float), + depth: make("depthtex0", wgpu::TextureFormat::Depth32Float), }); - for pass in &mut self.passes { + for pass in self.passes.iter_mut().chain(&mut self.terrain) { pass.groups.clear(); } } @@ -673,6 +903,7 @@ impl PackPost { inputs: &PackInputs, size: [u32; 2], window: [u32; 2], + terrain: &mut dyn FnMut(&mut wgpu::RenderPass<'_>), ) { self.resize(device, size); self.uniforms.update(frame, inputs, size); @@ -751,9 +982,6 @@ impl PackPost { view(self.main, 0, false).expect("colortex0"), overwrite, )]; - if self.blit_depth { - attachments.push(attach(&targets.depth, overwrite)); - } for &b in &self.blit_clears { attachments.push(attach( view(self.main, b, false).expect("used buffer"), @@ -763,6 +991,16 @@ impl PackPost { let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("Shader pack world"), color_attachments: &attachments, + depth_stencil_attachment: self.blit_depth.then_some( + wgpu::RenderPassDepthStencilAttachment { + view: &targets.depth, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Clear(1.0), + store: wgpu::StoreOp::Store, + }), + stencil_ops: None, + }, + ), ..Default::default() }); pass.set_pipeline(&self.blit.0); @@ -770,10 +1008,15 @@ impl PackPost { pass.draw(0..3, 0..1); } - let fog = self.uniforms.fog(); + let mut passes = std::mem::take(&mut self.passes); let mut seen = (false, false); let mut shown = false; - for pass in &mut self.passes { + let mut terrain_drawn = self.terrain.is_none(); + for pass in &mut passes { + if !terrain_drawn && pass.group != Group::Prepare { + terrain_drawn = true; + self.draw_terrain(device, queue, encoder, size, terrain); + } if pass.group == Group::Deferred && !seen.0 { seen.0 = true; self.main ^= self.deferred_pre; @@ -783,63 +1026,13 @@ impl PackPost { self.main ^= self.composite_pre; } let final_pass = pass.group == Group::Final; - let out = if final_pass { window } else { size }; - // Our block members first (`hz_target`, `hz_fog_color`, `hz_fog`), then the pack's. - let ours = [ - [ - out[0] as f32, - out[1] as f32, - 1.0 / out[0] as f32, - 1.0 / out[1] as f32, - ], - fog[0].to_array(), - fog[1].to_array(), - ]; - let head: &[u8] = bytemuck::cast_slice(&ours); - let n = head.len().min(pass.block.len()); - pass.block[..n].copy_from_slice(&head[..n]); - for (index, ty, offset) in &pass.uniforms { - if let Some(v) = index.and_then(|i| self.uniforms.values.get(i)) { - v.write(ty, &mut pass.block, *offset as usize); - } - } - if !pass.block.is_empty() { - queue.write_buffer(&pass.ubo, 0, &pass.block); - } - let key = self.main & pass.reads; - let group = pass.groups.entry(key).or_insert_with(|| { - let mut entries = vec![wgpu::BindGroupEntry { - binding: 0, - resource: pass.ubo.as_entire_binding(), - }]; - for s in &pass.samplers { - let filter = if s.filtering { - &self.linear - } else { - &self.nearest - }; - let (texture, sampler) = match s.source { - Source::Color(i) => (view(key, i, false).unwrap_or(&self.dummy), filter), - Source::Depth => (&targets.depth, &self.nearest), - Source::Noise => (&self.noise, &self.repeat), - Source::Dummy => (&self.dummy, &self.linear), - Source::Shadow => (&self.shadow.0, &self.shadow.1), - }; - entries.push(wgpu::BindGroupEntry { - binding: s.binding, - resource: wgpu::BindingResource::TextureView(texture), - }); - entries.push(wgpu::BindGroupEntry { - binding: s.binding + 1, - resource: wgpu::BindingResource::Sampler(sampler), - }); - } - device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some(&pass.name), - layout: &pass.layout, - entries: &entries, - }) - }); + let group = self.bind(device, queue, pass, if final_pass { window } else { size }); + let targets = self.targets.as_ref().expect("resized"); + let view = |i: usize| { + targets.color[i] + .as_ref() + .map(|t| &t[usize::from(self.main & 1 << i == 0)]) + }; let mut attachments: Vec> = Vec::new(); if final_pass { attachments.push(attach(screen, overwrite)); @@ -850,8 +1043,7 @@ impl PackPost { if attachments.len() <= l { attachments.resize(l + 1, None); } - attachments[l] = - view(self.main, buffer, true).and_then(|v| attach(v, overwrite)); + attachments[l] = view(buffer).and_then(|v| attach(v, overwrite)); } } { @@ -861,11 +1053,21 @@ impl PackPost { ..Default::default() }); rp.set_pipeline(&pass.pipeline); - rp.set_bind_group(0, &*group, &[]); + rp.set_bind_group(0, &group, &[]); rp.draw(0..6, 0..1); } self.main ^= pass.flip; } + self.passes = passes; + if !terrain_drawn { + self.draw_terrain(device, queue, encoder, size, terrain); + } + let targets = self.targets.as_ref().expect("resized"); + let view = |m: u16, i: usize, alt: bool| { + targets.color[i] + .as_ref() + .map(|t| &t[main(m, i) ^ usize::from(alt)]) + }; if !shown { let group = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("Shader pack colortex0"), diff --git a/crates/horizon-client/src/gfx/pack_post.wgsl b/crates/horizon-client/src/gfx/pack_post.wgsl index 30345b00f..75877dddc 100644 --- a/crates/horizon-client/src/gfx/pack_post.wgsl +++ b/crates/horizon-client/src/gfx/pack_post.wgsl @@ -1,5 +1,6 @@ // Helpers of a shader pack's post-processing (pack_post.rs): the world into colortex0 and -// depthtex0, and colortex0 onto the screen for packs without `final`. +// depthtex0 (a depth attachment, for terrain's depth test), and colortex0 onto the screen for +// packs without `final`. @vertex fn vs_full(@builtin(vertex_index) i: u32) -> @builtin(position) vec4 { @@ -12,7 +13,7 @@ fn vs_full(@builtin(vertex_index) i: u32) -> @builtin(position) vec4 { struct World { @location(0) color: vec4, - @location(1) depth: vec4, + @builtin(frag_depth) depth: f32, } // Pack buffers keep GL's rows (row 0 at the bottom), and GL's window depth: Horizon's @@ -23,7 +24,7 @@ fn fs_blit(@builtin(position) p: vec4) -> World { let at = vec2(i32(p.x), size.y - 1 - i32(p.y)); var out: World; out.color = textureLoad(source, at, 0); - out.depth = vec4(1.0 - textureLoad(world_depth, at, 0), 0.0, 0.0, 1.0); + out.depth = 1.0 - textureLoad(world_depth, at, 0); return out; } diff --git a/crates/horizon-client/src/gfx/pack_uniforms.rs b/crates/horizon-client/src/gfx/pack_uniforms.rs index 7d2a73a6a..92eac4a75 100644 --- a/crates/horizon-client/src/gfx/pack_uniforms.rs +++ b/crates/horizon-client/src/gfx/pack_uniforms.rs @@ -140,11 +140,14 @@ pub const NAMES: &[&str] = &[ "gbufferProjectionInverse", "gbufferPreviousModelView", "gbufferPreviousProjection", + "atlasSize", ]; /// Values carried from frame to frame. pub struct FrameUniforms { pub values: Vec, + /// The block atlas's size (`atlasSize`), 0 without one. + pub atlas: [u32; 2], start: std::time::Instant, last: Option, frame: u32, @@ -157,6 +160,7 @@ impl Default for FrameUniforms { fn default() -> Self { Self { values: Vec::new(), + atlas: [0; 2], start: std::time::Instant::now(), last: None, frame: 0, @@ -243,6 +247,7 @@ impl FrameUniforms { "worldDay" => Value::i(&[world_day as i32]), "moonPhase" => Value::i(&[world_day.rem_euclid(8) as i32]), "frameCounter" => Value::i(&[self.frame as i32]), + "atlasSize" => Value::i(&[self.atlas[0] as i32, self.atlas[1] as i32]), "frameTime" => Value::f(&[dt]), "frameTimeCounter" => Value::f(&[seconds]), "viewWidth" => Value::f(&[size[0] as f32]), diff --git a/crates/horizon-shaders/tests/packs/terrain-test/README.md b/crates/horizon-shaders/tests/packs/terrain-test/README.md new file mode 100644 index 000000000..bb8ddd060 --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/README.md @@ -0,0 +1,8 @@ +# terrain-test + +Horizon's own shader pack for testing gbuffers terrain (written for Horizon, no other pack's code). +`gbuffers_terrain` draws blocks from the atlas with their vertex colour and lightmap, tinted by +the lightmap coordinate's sky light (cooler where the sky is hidden); `mc_midTexCoord` marks each +face's centre with a dot. `composite` puts the depth buffer's view-space distance into the blue +channel of the far half. Copy the folder into `shaderpacks/` and run the game with +`HORIZON_SHADER_PACK=terrain-test`. diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/composite.fsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/composite.fsh new file mode 100644 index 000000000..d473ef240 --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/composite.fsh @@ -0,0 +1,19 @@ +#version 120 + +uniform sampler2D colortex0; +uniform sampler2D depthtex0; +uniform mat4 gbufferProjectionInverse; +uniform float far; + +varying vec2 texcoord; + +void main() { + vec3 color = texture2D(colortex0, texcoord).rgb; + if (texcoord.x > 0.5) { + float depth = texture2D(depthtex0, texcoord).r; + vec4 view = gbufferProjectionInverse * vec4(vec3(texcoord, depth) * 2.0 - 1.0, 1.0); + color.b = clamp(length(view.xyz / view.w) / far, 0.0, 1.0); + } + /* DRAWBUFFERS:0 */ + gl_FragData[0] = vec4(color, 1.0); +} diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/composite.vsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/composite.vsh new file mode 100644 index 000000000..38e4e455c --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/composite.vsh @@ -0,0 +1,8 @@ +#version 120 + +varying vec2 texcoord; + +void main() { + gl_Position = ftransform(); + texcoord = gl_MultiTexCoord0.xy; +} diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh new file mode 100644 index 000000000..902cf3525 --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh @@ -0,0 +1,22 @@ +#version 120 + +uniform sampler2D texture; +uniform sampler2D lightmap; +uniform ivec2 atlasSize; + +varying vec2 texcoord; +varying vec2 lmcoord; +varying vec2 mid; +varying vec4 glcolor; + +void main() { + vec4 color = texture2D(texture, texcoord) * glcolor; + if (color.a < 0.1) discard; + color.rgb *= texture2D(lightmap, lmcoord).rgb; + // Less sky light: bluer. + color.rgb = mix(color.rgb * vec3(0.6, 0.7, 1.2), color.rgb, lmcoord.y); + // A dot at the face's centre, one texel wide. + if (length((texcoord - mid) * vec2(atlasSize)) < 1.0) color.rgb = vec3(1.0, 0.0, 1.0); + /* DRAWBUFFERS:0 */ + gl_FragData[0] = color; +} diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.vsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.vsh new file mode 100644 index 000000000..bc16f4aad --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.vsh @@ -0,0 +1,16 @@ +#version 120 + +attribute vec4 mc_midTexCoord; + +varying vec2 texcoord; +varying vec2 lmcoord; +varying vec2 mid; +varying vec4 glcolor; + +void main() { + gl_Position = ftransform(); + texcoord = (gl_TextureMatrix[0] * gl_MultiTexCoord0).xy; + lmcoord = (gl_TextureMatrix[1] * gl_MultiTexCoord1).xy; + mid = mc_midTexCoord.xy; + glcolor = gl_Color; +} From 6c8880fbd887a69af1f12acc5f2f9c3b64cbec85 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:00:40 +0100 Subject: [PATCH 03/61] Shader packs, clean room 15: globals initialized at run time, switch options naga keeps only constant initializers of globals, so every initialized global that is not `const` (and every conditional one) gets its value from a function of ours that main calls first, one per declarator, converting as GLSL's initialization does (`vec2 v = atlasSize`). `vec3 a = x, b = y;` no longer loses `b`. Program-enable expressions (3.3) now see the pack's switch options that are on by default (8.1): `#define NAME` with nothing after it, tested with `#ifdef`/`#ifndef` anywhere in the pack (OQ-13 working answer). BSL's `deferred` (ambient occlusion) and its TAA ran with options off before. Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, sections 3.3, 8.1; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-shaders/src/lib.rs | 2 + crates/horizon-shaders/src/options.rs | 80 +++++++++++++ crates/horizon-shaders/src/pack.rs | 33 ++++++ crates/horizon-shaders/src/post.rs | 23 +++- crates/horizon-shaders/src/translate/mod.rs | 15 +++ crates/horizon-shaders/src/translate/scan.rs | 116 +++++++++++++++---- 6 files changed, 245 insertions(+), 24 deletions(-) create mode 100644 crates/horizon-shaders/src/options.rs diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index e31eaec6f..d29a0a632 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -17,6 +17,7 @@ mod config; mod discover; mod lang; mod macros; +mod options; mod pack; mod post; mod preprocess; @@ -28,6 +29,7 @@ pub use config::{BUFFERS, Buffer, Config, Format, Kind, buffer_index, draw_buffe pub use discover::{Found, discover}; pub use lang::Lang; pub use macros::{MC_VERSION, Macros, RENDER_STAGES}; +pub use options::switches_on; pub use pack::{Pack, PackError}; pub use post::{Group, Post, PostProgram, dimension_folder, load_post}; pub use preprocess::{Diagnostic, Origin, Preprocessed, preprocess}; diff --git a/crates/horizon-shaders/src/options.rs b/crates/horizon-shaders/src/options.rs new file mode 100644 index 000000000..687fd40e0 --- /dev/null +++ b/crates/horizon-shaders/src/options.rs @@ -0,0 +1,80 @@ +//! Switch options (spec 8.1), as far as program-enable expressions (3.3) need them: which are on +//! by default. A switch is `#define NAME` (on) or `//#define NAME` (off) with nothing after the +//! name but a comment, in any stage or included file of the pack. Packs keep their options in +//! included `.glsl` files and test them elsewhere (OQ-13), so the test the spec asks for +//! (`#ifdef NAME` / `#ifndef NAME`) may be in any file of the pack. + +use std::collections::BTreeSet; + +use crate::Pack; + +const SOURCES: [&str; 8] = ["vsh", "fsh", "gsh", "csh", "tcs", "tes", "glsl", "inc"]; + +/// The switch options that are on by default. +pub fn switches_on(pack: &Pack) -> BTreeSet { + let (mut on, mut off, mut tested) = (BTreeSet::new(), BTreeSet::new(), BTreeSet::new()); + for path in pack.all_files() { + let ext = path.rsplit_once('.').map_or("", |(_, e)| e); + if !SOURCES.contains(&ext) { + continue; + } + let Some(text) = pack.read_text(&path) else { + continue; + }; + for line in text.lines() { + let line = line.trim(); + let (commented, rest) = match line.strip_prefix("//") { + Some(rest) => (true, rest.trim_start()), + None => (false, line), + }; + let Some((directive, rest)) = rest + .strip_prefix('#') + .map(str::trim_start) + .and_then(|r| r.split_once(|c: char| c.is_whitespace())) + else { + continue; + }; + let mut words = rest.split_whitespace(); + let Some(name) = words.next() else { continue }; + match directive { + "ifdef" | "ifndef" if !commented => _ = tested.insert(name.to_owned()), + "define" if words.next().is_none_or(|w| w.starts_with("//")) => { + let set = if commented { &mut off } else { &mut on }; + set.insert(name.to_owned()); + } + _ => {} + } + } + } + // Defined both ways is ambiguous (8.1): it keeps its default only where it is on everywhere. + on.retain(|name| tested.contains(name) && !off.contains(name)); + on +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pack::tests::zip; + + #[test] + fn switches_need_a_test_and_no_value() { + let pack = Pack::from_zip_bytes( + "P.zip", + zip(&[ + ( + "shaders/lib/settings.glsl", + " #define AO // ambient occlusion\n//#define BLOOM\n#define LEVEL 2 //[1 2]\n\ + #define SPLIT\n#define UNTESTED\n", + ), + ( + "shaders/world0/deferred.fsh", + "#ifdef AO\n#endif\n#ifndef BLOOM\n#endif\n#ifdef SPLIT\n#endif\n#ifdef LEVEL\n#endif\n", + ), + ("shaders/world-1/deferred.fsh", "//#define SPLIT\n"), + ("shaders/shaders.properties", "#define NOT_SOURCE\n#ifdef NOT_SOURCE\n"), + ]), + ) + .unwrap(); + assert_eq!(switches_on(&pack), BTreeSet::from(["AO".to_owned()])); + } +} diff --git a/crates/horizon-shaders/src/pack.rs b/crates/horizon-shaders/src/pack.rs index ae2230618..77aeded35 100644 --- a/crates/horizon-shaders/src/pack.rs +++ b/crates/horizon-shaders/src/pack.rs @@ -167,6 +167,39 @@ impl Pack { names.sort(); names } + + /// Every file of the pack, by its path inside `shaders`, sorted. + pub fn all_files(&self) -> Vec { + let mut names = match &self.files { + Files::Folder(root) => { + let mut names = Vec::new(); + let mut dirs = vec![String::new()]; + while let Some(dir) = dirs.pop() { + for e in fs::read_dir(root.join(&dir)) + .into_iter() + .flatten() + .flatten() + { + let name = e.file_name().to_string_lossy().into_owned(); + let path = if dir.is_empty() { + name + } else { + format!("{dir}/{name}") + }; + match e.file_type() { + Ok(t) if t.is_dir() => dirs.push(path), + Ok(t) if t.is_file() => names.push(path), + _ => {} + } + } + } + names + } + Files::Zip { entries, .. } => entries.keys().cloned().collect(), + }; + names.sort(); + names + } } /// `a/./b/../c` → `a/c`; `None` for a path that climbs out of `shaders`. diff --git a/crates/horizon-shaders/src/post.rs b/crates/horizon-shaders/src/post.rs index cdf7fae80..c96730f0a 100644 --- a/crates/horizon-shaders/src/post.rs +++ b/crates/horizon-shaders/src/post.rs @@ -10,6 +10,8 @@ //! - A program that fails to translate is reported and treated as missing (2.5). //! - Terrain draws with the first program of its fallback chain that loads (2.3). +use std::collections::BTreeSet; + use crate::config::{self, Config}; use crate::{GbuffersInputs, Macros, Pack, Preprocessed, Program, Properties, Target}; use crate::{eval_condition, preprocess, translate_composite, translate_gbuffers}; @@ -99,6 +101,7 @@ pub fn load_post( folder, properties, macros, + switches: crate::switches_on(pack), post: &mut post, }; for (group, base) in GROUPS { @@ -135,6 +138,7 @@ struct Loader<'a> { folder: &'a str, properties: &'a Properties, macros: &'a Macros, + switches: BTreeSet, post: &'a mut Post, } @@ -156,7 +160,13 @@ impl Loader<'_> { if !self.pack.exists(&fsh) || !self.pack.exists(&vsh) { return None; } - if !enabled(self.properties, self.folder, &name, self.macros) { + if !enabled( + self.properties, + self.folder, + &name, + self.macros, + &self.switches, + ) { return None; } if self.pack.exists(&path("gsh")) { @@ -215,8 +225,14 @@ fn diagnostics(pp: &Preprocessed) -> Vec { } /// `program..enabled` (or `program./.enabled`): a boolean expression over -/// switch options (3.3). An option is on when its macro is defined. -fn enabled(properties: &Properties, folder: &str, name: &str, macros: &Macros) -> bool { +/// switch options (3.3): those on by default (8.1) and the defined macros. +fn enabled( + properties: &Properties, + folder: &str, + name: &str, + macros: &Macros, + switches: &BTreeSet, +) -> bool { let keys = [ format!("program.{folder}/{name}.enabled"), format!("program.{name}.enabled"), @@ -230,6 +246,7 @@ fn enabled(properties: &Properties, folder: &str, name: &str, macros: &Macros) - match token { "true" => condition.push('1'), "false" => condition.push('0'), + t if switches.contains(t) => condition.push('1'), t if t.starts_with(|c: char| c.is_alphabetic() || c == '_') => { condition.push_str(&format!("defined({t})")); } diff --git a/crates/horizon-shaders/src/translate/mod.rs b/crates/horizon-shaders/src/translate/mod.rs index 32eecb3cc..83dcb3af9 100644 --- a/crates/horizon-shaders/src/translate/mod.rs +++ b/crates/horizon-shaders/src/translate/mod.rs @@ -472,6 +472,21 @@ mod tests { assert_eq!(i.outputs, [0, 1]); } + /// naga drops non-constant global initializers: they run in our `main`, one per declarator, + /// converting as GLSL's initialization does. + #[test] + fn global_initializers_read_uniforms() { + let fsh = "#version 120\nuniform float rainStrength;\nuniform ivec2 atlasSize;\n\ + vec2 size = atlasSize;\nvec3 a = vec3(rainStrength), b = a * 2.0, c;\n\ + const float k = rainStrength > 0.5 ? 1.0 : 0.0;\nfloat w[] = float[](rainStrength, 1.0);\n\ + void main() { c = b; gl_FragColor = vec4(a + c, size.x * k + w[0]); }\n"; + let p = program( + &[("composite.vsh", VSH_120), ("composite.fsh", fsh)], + Target::Window, + ); + assert!(p.is_ok(), "{:?}", p.err()); + } + #[test] fn core_programs_translate() { let vsh = "#version 330 core\nin vec3 vaPosition;\nin vec2 vaUV0;\nuniform mat4 modelViewMatrix, projectionMatrix;\n\ diff --git a/crates/horizon-shaders/src/translate/scan.rs b/crates/horizon-shaders/src/translate/scan.rs index 6789893f4..392538f03 100644 --- a/crates/horizon-shaders/src/translate/scan.rs +++ b/crates/horizon-shaders/src/translate/scan.rs @@ -338,41 +338,115 @@ impl<'p> Stage<'p> { } } - /// A global `T name = a ? b : c;`: naga can't evaluate a conditional at global scope, so - /// the initializer becomes a function of ours on the same line (`T name; T hz_init_name() { - /// return a ? b : c; }`), called by our `main` before the pack's. + /// A global `T name = a ? b : c;` or, without `const`, any initialized global: naga only + /// evaluates constant initializers at global scope (a conditional is not one; GL drivers + /// also take uniforms and other globals there), so each initializer becomes a function of + /// ours on the same line (`T a; T hz_init_a() { return T(...); } T b; ...`), called by our + /// `main` before the pack's, in file order. The return converts as GLSL's initialization + /// does (`vec2 v = ivec2(...)`). pub(super) fn initializer(&mut self, words: &[usize], to: usize) { let text = |i: usize| self.tokens[i]; let Some(eq) = words.iter().position(|&i| text(i) == "=") else { return; }; - if !words[eq..].iter().any(|&i| text(i) == "?") - || words[..eq].iter().any(|&i| text(i) == ",") - { + // Declarators, split at top-level commas: (first word, `=` if any, end). + let mut declarators = Vec::new(); + let (mut begin, mut nesting, mut assign) = (0, 0i32, None); + for (n, &i) in words.iter().enumerate() { + match text(i) { + "(" | "[" | "{" => nesting += 1, + ")" | "]" | "}" => nesting -= 1, + "=" if nesting == 0 && assign.is_none() => assign = Some(n), + "," if nesting == 0 => { + declarators.push((begin, assign.take(), n)); + begin = n + 1; + } + _ => {} + } + } + declarators.push((begin, assign, words.len())); + let constant = words[..eq].iter().any(|&i| text(i) == "const"); + let conditional = + |d: &(usize, Option, usize)| words[d.0..d.2].iter().any(|&i| text(i) == "?"); + if constant && !declarators.iter().any(conditional) { return; } - let Some(name_at) = words[..eq].iter().rposition(|&i| is_ident(text(i))) else { + // The first declarator's name ends the type. + let first_end = declarators[0].1.unwrap_or(declarators[0].2); + let Some(name_at) = words[..first_end].iter().rposition(|&i| is_ident(text(i))) else { return; }; - let name = text(words[name_at]).to_owned(); - let array: String = words[name_at + 1..eq].iter().map(|&i| text(i)).collect(); - let ty: Vec<&str> = words[..name_at] - .iter() - .map(|&i| text(i)) - .filter(|&t| t != "const") - .collect(); - let Some(ty) = ty.last() else { return }; + // The type, with a `vec3[]`'s count written in. + let mut ty = String::new(); + for &i in &words[..name_at] { + match text(i) { + "const" => {} + "]" if ty.ends_with('[') => match self.array_length(i) { + Some(n) => write!(ty, "{n}]").unwrap(), + None => return, + }, + t if ty.is_empty() || t == "[" => ty.push_str(t), + // Qualifiers before the type (`highp`...): the type is the last word. + t => ty = t.to_owned(), + } + } + if ty.is_empty() { + return; + } + let mut replaced = Vec::new(); + let mut names = Vec::new(); + for (n, &(begin, assign, end)) in declarators.iter().enumerate() { + let name_at = if n == 0 { name_at } else { begin }; + let name = text(words[name_at]).to_owned(); + if !is_ident(&name) { + return; + } + let stop = assign.unwrap_or(end); + let mut array: String = words[name_at + 1..stop].iter().map(|&i| text(i)).collect(); + if array == "[]" { + // `T x[] = T[](a, b)`: the count is written in, as the rewrite does for the name. + match assign.and_then(|a| self.array_length(words[a - 1])) { + Some(n) => array = format!("[{n}]"), + None => return, + } + } + let next = if n + 1 < declarators.len() { + format!(" {ty}") + } else { + String::new() + }; + let end_token = words.get(end).copied().unwrap_or(to); + match assign { + Some(a) => { + // Scalars, vectors and matrices convert; arrays and structs must match. + let scalar = array.is_empty() + && !ty.contains('[') + && (matches!(ty.as_str(), "float" | "int" | "uint" | "bool") + || ty.contains("vec") + || ty.starts_with("mat")); + let (open, close) = if scalar { + (format!("{ty}("), ")") + } else { + (String::new(), "") + }; + replaced.push(( + words[a], + format!("; {ty}{array} hz_init_{name}() {{ return {open}"), + )); + replaced.push((end_token, format!("{close}; }}{next}"))); + names.push(name); + } + None if n + 1 < declarators.len() => replaced.push((end_token, format!(";{next}"))), + None => {} + } + } for &i in &words[..name_at] { if text(i) == "const" { self.removed.push(i..i + 1); } } - self.replaced.insert( - words[eq], - format!("; {ty}{array} hz_init_{name}() {{ return"), - ); - self.replaced.insert(to, "; }".into()); - self.inits.push(name); + self.replaced.extend(replaced); + self.inits.extend(names); } /// A function definition or prototype: remember which parameters are samplers. From 505e8e960f9956a51bde92a52b737d4f7f413c69 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:01:48 +0100 Subject: [PATCH 04/61] Shader packs, clean room 16: custom uniforms `uniform..` and `variable..` of shaders.properties (3.4): an expression parser with C precedence, the function table (math, vectors, `if`, `between`, `equals`, `smooth`, `random`...), evaluated on the CPU once per frame in file order after the frame's own uniforms. `smooth` keeps its state per call site and halves the distance to its target every fade time (OQ-26 working answer). A name neither the frame nor the pack defines reads 0 and is listed once in the log. Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, section 3.4; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx/pack_post.rs | 36 +- .../horizon-client/src/gfx/pack_uniforms.rs | 41 ++ crates/horizon-shaders/src/custom.rs | 570 ++++++++++++++++++ crates/horizon-shaders/src/lib.rs | 2 + crates/horizon-shaders/src/post.rs | 4 + crates/horizon-shaders/tests/real_packs.rs | 7 + 6 files changed, 656 insertions(+), 4 deletions(-) create mode 100644 crates/horizon-shaders/src/custom.rs diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 81a517dba..6520565ed 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; -use horizon_shaders::{Format, Group, Kind, Post}; +use horizon_shaders::{Format, Group, Kind, Post, Val}; use super::pack_uniforms::{self, FrameUniforms, PackInputs}; @@ -83,6 +83,8 @@ pub struct PackPost { passes: Vec, /// What draws terrain (`gbuffers_terrain` or its fallback), between `prepare` and `deferred`. terrain: Option, + /// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`. + custom: horizon_shaders::Custom, atlas: Option, lightmap: wgpu::TextureView, deferred_pre: u16, @@ -342,8 +344,16 @@ fn build_pass( screen: wgpu::TextureFormat, notes: &mut Vec, geometry: Option<&Geometry<'_>>, + custom: &[String], ) -> Option { let features = device.features(); + // The frame's uniforms, then the pack's custom ones. + let index = |name: &str| { + pack_uniforms::index(name).or_else(|| { + let k = custom.iter().position(|c| c == name)?; + Some(pack_uniforms::NAMES.len() + k) + }) + }; let mask = |flips: &[(u32, bool)]| flips.iter().filter(|f| f.1).fold(0u16, |m, f| m | 1 << f.0); let interface = &p.program.interface; let mut entries = vec![entry( @@ -499,7 +509,7 @@ fn build_pass( let missing: Vec<&str> = interface .uniforms .iter() - .filter(|u| !u.name.starts_with("hz_") && pack_uniforms::index(&u.name).is_none()) + .filter(|u| !u.name.starts_with("hz_") && index(&u.name).is_none()) .map(|u| u.name.as_str()) .collect(); if !missing.is_empty() { @@ -516,7 +526,7 @@ fn build_pass( uniforms: interface .uniforms .iter() - .map(|u| (pack_uniforms::index(&u.name), u.ty.clone(), u.offset)) + .map(|u| (index(&u.name), u.ty.clone(), u.offset)) .collect(), name: p.name, group: p.group, @@ -552,11 +562,27 @@ impl PackPost { }); let mut notes = post.notes; notes.extend(post.config.notes); + let mut custom = post.custom; + let custom_names: Vec = custom.uniforms().map(|u| u.0.to_owned()).collect(); + // A first run finds the names neither the frame nor the pack defines. + custom.eval(0.0, &|n| pack_uniforms::index(n).map(|_| Val::scalar(0.0))); + if !custom.notes.is_empty() { + notes.push(format!("custom uniforms: {}", custom.notes.join("; "))); + custom.notes.clear(); + } let mask = |flips: &[(u32, bool)]| flips.iter().filter(|f| f.1).fold(0u16, |m, f| m | 1 << f.0); let mut used = 1u16; let mut build = |p: horizon_shaders::PostProgram, geometry: Option<&Geometry<'_>>| { - let pass = build_pass(device, p, &formats, screen, &mut notes, geometry)?; + let pass = build_pass( + device, + p, + &formats, + screen, + &mut notes, + geometry, + &custom_names, + )?; used |= pass.reads | pass.written | pass.flip; Some(pass) }; @@ -646,6 +672,7 @@ impl PackPost { notes, passes, terrain, + custom, atlas: world.atlas.as_ref().map(|a| a.0.clone()), lightmap: world.lightmap, deferred_pre: mask(&post.deferred_pre), @@ -907,6 +934,7 @@ impl PackPost { ) { self.resize(device, size); self.uniforms.update(frame, inputs, size); + self.uniforms.eval_custom(&mut self.custom); let Some(targets) = &self.targets else { return }; let main = |m: u16, i: usize| usize::from(m & 1 << i != 0); let view = |m: u16, i: usize, alt: bool| { diff --git a/crates/horizon-client/src/gfx/pack_uniforms.rs b/crates/horizon-client/src/gfx/pack_uniforms.rs index 92eac4a75..5a27e62a0 100644 --- a/crates/horizon-client/src/gfx/pack_uniforms.rs +++ b/crates/horizon-client/src/gfx/pack_uniforms.rs @@ -304,6 +304,47 @@ impl FrameUniforms { } } +impl FrameUniforms { + /// Evaluates the pack's custom uniforms with this frame's values and puts them after + /// `NAMES`' values, in `custom.uniforms()` order. + pub fn eval_custom(&mut self, custom: &mut horizon_shaders::Custom) { + self.values.truncate(NAMES.len()); + let values = &self.values; + let builtin = |name: &str| { + let v = values.get(index(name)?)?; + let n = components(name); + Some(match *v { + Value::F(f) => horizon_shaders::Val::vector(&f.map(f64::from)[..n]), + Value::I(i) => horizon_shaders::Val::vector(&i.map(f64::from)[..n.min(3)]), + }) + }; + let dt = match index("frameTime").and_then(|i| values.get(i)) { + Some(Value::F(f)) => f64::from(f[0]), + _ => 0.0, + }; + let out = custom.eval(dt, &builtin); + self.values + .extend(out.iter().map(|v| Value::f(&v.v.map(|x| x as f32)[..v.n]))); + } +} + +/// How many components the uniform `name` has (16 for a matrix). +fn components(name: &str) -> usize { + match name { + "cameraPosition" + | "previousCameraPosition" + | "upPosition" + | "sunPosition" + | "moonPosition" + | "shadowLightPosition" + | "fogColor" + | "skyColor" => 3, + "eyeBrightness" | "eyeBrightnessSmooth" | "atlasSize" => 2, + n if n.starts_with("gbuffer") || n.starts_with("shadow") && n != "shadowAngle" => 16, + _ => 1, + } +} + /// The position of `name` in [`NAMES`]. pub fn index(name: &str) -> Option { NAMES.iter().position(|n| *n == name) diff --git a/crates/horizon-shaders/src/custom.rs b/crates/horizon-shaders/src/custom.rs new file mode 100644 index 000000000..34510247e --- /dev/null +++ b/crates/horizon-shaders/src/custom.rs @@ -0,0 +1,570 @@ +//! Custom uniforms and variables of `shaders.properties` (spec 3.4): `uniform..` and +//! `variable..` expressions, parsed once and evaluated on the CPU every frame in +//! file order. Uniforms reach every program; variables only later expressions. +//! +//! Decisions (impl-plan.md): +//! - OQ-25: numbers are doubles; booleans are 1 and 0; `%` is C's remainder (`fmod` floors); +//! comparisons and logic read the first component; a value takes its declared type at the +//! end (`int` truncates toward zero, `bool` is "not zero"). Unknown names read 0 and are +//! reported once. +//! - OQ-26: `smooth` runs per frame with the frame time; its fade times are seconds (1 by +//! default) to cover half of the remaining distance, `fadeIn` while rising, `fadeOut` while +//! falling. Each call keeps its own state, so the optional id is not needed. + +use std::collections::{HashMap, HashSet}; + +use crate::Properties; + +/// A value: up to 16 components (matrices only as names, for `m.row.column`). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Val { + pub v: [f64; 16], + /// Components: 1 for a scalar. + pub n: usize, +} + +impl Val { + pub fn scalar(x: f64) -> Self { + let mut v = [0.0; 16]; + v[0] = x; + Self { v, n: 1 } + } + pub fn vector(c: &[f64]) -> Self { + let mut v = [0.0; 16]; + v[..c.len()].copy_from_slice(c); + Self { v, n: c.len() } + } + fn x(&self) -> f64 { + self.v[0] + } + fn truth(&self) -> bool { + self.v[0] != 0.0 + } + /// `f` per component; a scalar operand stretches to the other's size. + fn zip(self, o: Val, f: impl Fn(f64, f64) -> f64) -> Val { + let n = self.n.max(o.n).min(4); + let at = |a: &Val, i: usize| if a.n == 1 { a.v[0] } else { a.v[i] }; + let mut out = Val::scalar(0.0); + out.n = n; + for i in 0..n { + out.v[i] = f(at(&self, i), at(&o, i)); + } + out + } + fn map(self, f: impl Fn(f64) -> f64) -> Val { + let mut out = self; + for i in 0..self.n.min(4) { + out.v[i] = f(self.v[i]); + } + out + } +} + +#[derive(Clone, Debug)] +enum Expr { + Num(f64), + /// A name and its component (`x`, `2`, `2.1` as row 2, column 1). + Name(String, Option<(usize, Option)>), + Call(String, Vec, usize), + Neg(Box), + Not(Box), + Bin(&'static str, Box, Box), +} + +#[derive(Debug)] +struct Item { + name: String, + ty: String, + uniform: bool, + expr: Expr, +} + +/// The pack's custom uniforms and variables, with the state `smooth` and `random` keep. +#[derive(Debug, Default)] +pub struct Custom { + items: Vec, + /// `smooth` state per call site. + smooth: HashMap, + seed: u64, + unknown: HashSet, + /// Expressions that did not parse, and unknown names, as `key: message`. + pub notes: Vec, + calls: usize, +} + +impl Custom { + /// Reads every `uniform.*` and `variable.*` key of `properties`, in file order. + pub fn parse(properties: &Properties) -> Self { + let mut out = Custom { + seed: 0x9e37_79b9_7f4a_7c15, + ..Default::default() + }; + for e in &properties.entries { + let (uniform, rest) = match e.key.split_once('.') { + Some(("uniform", r)) => (true, r), + Some(("variable", r)) => (false, r), + _ => continue, + }; + let Some((ty, name)) = rest.split_once('.') else { + continue; + }; + if !matches!(ty, "float" | "int" | "bool" | "vec2" | "vec3" | "vec4") { + out.notes + .push(format!("{}: type `{ty}` is not supported", e.key)); + continue; + } + match Parser::new(&e.value, &mut out.calls).expr_all() { + Ok(expr) => out.items.push(Item { + name: name.to_owned(), + ty: ty.to_owned(), + uniform, + expr, + }), + Err(m) => out.notes.push(format!("{}: {m}", e.key)), + } + } + out + } + + /// The uniforms' names and declared types, in file order. + pub fn uniforms(&self) -> impl Iterator { + self.items + .iter() + .filter(|i| i.uniform) + .map(|i| (i.name.as_str(), i.ty.as_str())) + } + + /// Evaluates everything for a frame `dt` seconds after the last; `builtin` gives the + /// values of the game's uniforms and constants (`BIOME_*`...). Returns the uniforms' + /// values, in the order of `uniforms`. + pub fn eval(&mut self, dt: f64, builtin: &dyn Fn(&str) -> Option) -> Vec { + let mut vars: HashMap = HashMap::new(); + let mut out = Vec::new(); + let items = std::mem::take(&mut self.items); + for item in &items { + let v = self.value(&item.expr, dt, &vars, builtin); + let v = match item.ty.as_str() { + "int" => Val::scalar(v.x().trunc()), + "bool" => Val::scalar(f64::from(u8::from(v.truth()))), + "float" => Val::scalar(v.x()), + t => { + let n = (t.as_bytes()[3] - b'0') as usize; + let mut v = v; + if v.n == 1 { + let x = v.v[0]; + v.v[1..n].fill(x); + } + v.n = n; + v + } + }; + vars.insert(item.name.clone(), v); + if item.uniform { + out.push(v); + } + } + self.items = items; + out + } + + fn random(&mut self) -> f64 { + self.seed ^= self.seed << 13; + self.seed ^= self.seed >> 7; + self.seed ^= self.seed << 17; + (self.seed >> 11) as f64 / (1u64 << 53) as f64 + } + + fn value( + &mut self, + e: &Expr, + dt: f64, + vars: &HashMap, + builtin: &dyn Fn(&str) -> Option, + ) -> Val { + let ev = |e: &Expr, s: &mut Self| s.value(e, dt, vars, builtin); + match e { + Expr::Num(x) => Val::scalar(*x), + Expr::Name(name, component) => { + let v = match name.as_str() { + "pi" => Some(Val::scalar(std::f64::consts::PI)), + "true" => Some(Val::scalar(1.0)), + "false" => Some(Val::scalar(0.0)), + n => vars.get(n).copied().or_else(|| builtin(n)), + }; + let Some(v) = v else { + if self.unknown.insert(name.clone()) { + self.notes.push(format!("`{name}` is unknown; it reads 0")); + } + return Val::scalar(0.0); + }; + match component { + None => v, + // GLSL's m[column][row] is stored column after column. + Some((row, Some(column))) => Val::scalar(v.v[column * 4 + row]), + Some((i, None)) => Val::scalar(v.v[*i]), + } + } + Expr::Neg(a) => ev(a, self).map(|x| -x), + Expr::Not(a) => Val::scalar(f64::from(u8::from(!ev(a, self).truth()))), + Expr::Bin(op, a, b) => { + let (a, b) = (ev(a, self), ev(b, self)); + let bool_ = |c: bool| Val::scalar(f64::from(u8::from(c))); + match *op { + "+" => a.zip(b, |x, y| x + y), + "-" => a.zip(b, |x, y| x - y), + "*" => a.zip(b, |x, y| x * y), + "/" => a.zip(b, |x, y| x / y), + "%" => a.zip(b, |x, y| x % y), + "<" => bool_(a.x() < b.x()), + "<=" => bool_(a.x() <= b.x()), + ">" => bool_(a.x() > b.x()), + ">=" => bool_(a.x() >= b.x()), + "==" => bool_(a.x() == b.x()), + "!=" => bool_(a.x() != b.x()), + "&&" => bool_(a.truth() && b.truth()), + _ => bool_(a.truth() || b.truth()), + } + } + Expr::Call(f, args, site) => { + let a: Vec = args.iter().map(|e| ev(e, self)).collect(); + let arg = |i: usize| a.get(i).copied().unwrap_or(Val::scalar(0.0)); + let x = arg(0).x(); + let one = |g: fn(f64) -> f64| arg(0).map(g); + match f.as_str() { + "sin" => one(f64::sin), + "cos" => one(f64::cos), + "tan" => one(f64::tan), + "asin" => one(f64::asin), + "acos" => one(f64::acos), + "atan" if a.len() == 2 => Val::scalar(x.atan2(arg(1).x())), + "atan" => one(f64::atan), + "atan2" => Val::scalar(x.atan2(arg(1).x())), + "torad" | "radians" => one(f64::to_radians), + "todeg" | "degrees" => one(f64::to_degrees), + "min" => a + .iter() + .copied() + .reduce(|p, q| p.zip(q, f64::min)) + .unwrap_or(arg(0)), + "max" => a + .iter() + .copied() + .reduce(|p, q| p.zip(q, f64::max)) + .unwrap_or(arg(0)), + "clamp" => arg(0).zip(arg(1), f64::max).zip(arg(2), f64::min), + "abs" => one(f64::abs), + "floor" => one(f64::floor), + "ceil" => one(f64::ceil), + "exp" => one(f64::exp), + "exp2" => one(f64::exp2), + "exp10" => one(|v| 10f64.powf(v)), + "frac" => one(|v| v - v.floor()), + "log" if a.len() == 2 => Val::scalar(arg(1).x().log(x)), + "log" => one(f64::ln), + "log2" => one(f64::log2), + "log10" => one(f64::log10), + "sqrt" => one(f64::sqrt), + "round" => one(f64::round), + "signum" | "sign" => one(|v| if v == 0.0 { 0.0 } else { v.signum() }), + "pow" => arg(0).zip(arg(1), f64::powf), + "fmod" => arg(0).zip(arg(1), |p, q| p - q * (p / q).floor()), + "lerp" => arg(1).zip(arg(2), |p, q| p + (q - p) * x), + "mix" => arg(0).zip(arg(1), |p, q| p + (q - p) * arg(2).x()), + "edge" => Val::scalar(if arg(1).x() < x { 0.0 } else { 1.0 }), + "random" if a.len() == 2 => Val::scalar(x + (arg(1).x() - x) * self.random()), + "random" => Val::scalar(self.random()), + "randomInt" if a.len() == 2 => { + Val::scalar((x + (arg(1).x() - x) * self.random()).floor()) + } + "randomInt" => Val::scalar((self.random() * f64::from(i32::MAX)).floor()), + "if" | "ifb" => { + let mut i = 0; + while i + 1 < a.len() { + if a[i].truth() { + return a[i + 1]; + } + i += 2; + } + a.last().copied().unwrap_or(Val::scalar(0.0)) + } + "smooth" => { + let (val, fade_in, fade_out) = match a.len() { + 4 => (arg(1).x(), arg(2).x(), arg(3).x()), + 3 => (x, arg(1).x(), arg(2).x()), + 2 => (x, arg(1).x(), arg(1).x()), + _ => (x, 1.0, 1.0), + }; + let state = self.smooth.entry(*site).or_insert(val); + let fade = if val > *state { fade_in } else { fade_out }; + if fade > 0.0 { + *state = val + (*state - val) * 0.5f64.powf(dt / fade); + } else { + *state = val; + } + Val::scalar(*state) + } + "print" => arg(2), + "between" => { + Val::scalar(f64::from(u8::from(x >= arg(1).x() && x <= arg(2).x()))) + } + "equals" => { + Val::scalar(f64::from(u8::from((x - arg(1).x()).abs() <= arg(2).x()))) + } + "in" => Val::scalar(f64::from(u8::from(a[1..].iter().any(|v| v.x() == x)))), + "vec2" | "vec3" | "vec4" => { + let n = (f.as_bytes()[3] - b'0') as usize; + let mut c: Vec = a.iter().flat_map(|v| v.v[..v.n].to_vec()).collect(); + if c.len() == 1 { + c = vec![c[0]; n]; + } + c.resize(n, 0.0); + Val::vector(&c) + } + _ => { + if self.unknown.insert(f.clone()) { + self.notes.push(format!("`{f}()` is unknown; it gives 0")); + } + Val::scalar(0.0) + } + } + } + } + } +} + +struct Parser<'a> { + tokens: Vec, + at: usize, + calls: &'a mut usize, +} + +const OPERATORS: [&str; 18] = [ + "||", "&&", "==", "!=", "<=", ">=", "<", ">", "+", "-", "*", "/", "%", "!", "(", ")", ",", ".", +]; + +impl<'a> Parser<'a> { + fn new(text: &str, calls: &'a mut usize) -> Self { + let mut tokens = Vec::new(); + let b = text.as_bytes(); + let mut i = 0; + while i < b.len() { + let c = b[i] as char; + if c.is_whitespace() { + i += 1; + } else if c.is_ascii_digit() + || (c == '.' + && b.get(i + 1).is_some_and(u8::is_ascii_digit) + && !tokens + .last() + .is_some_and(|t: &String| t == "." || is_name(t))) + { + let start = i; + while i < b.len() + && (b[i].is_ascii_alphanumeric() + || b[i] == b'.' + || ((b[i] == b'-' || b[i] == b'+') && matches!(b[i - 1], b'e' | b'E'))) + { + i += 1; + } + tokens.push(text[start..i].to_owned()); + } else if c.is_ascii_alphabetic() || c == '_' { + let start = i; + while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') { + i += 1; + } + tokens.push(text[start..i].to_owned()); + } else { + let op = OPERATORS + .iter() + .find(|o| text[i..].starts_with(**o)) + .map_or(&text[i..i + c.len_utf8()], |o| o); + i += op.len(); + tokens.push(op.to_owned()); + } + } + Parser { + tokens, + at: 0, + calls, + } + } + fn peek(&self) -> Option<&str> { + self.tokens.get(self.at).map(String::as_str) + } + fn eat(&mut self, t: &str) -> bool { + let hit = self.peek() == Some(t); + self.at += usize::from(hit); + hit + } + fn expr_all(&mut self) -> Result { + let e = self.binary(0)?; + match self.peek() { + None => Ok(e), + Some(t) => Err(format!("unexpected `{t}`")), + } + } + /// Precedence levels from the loosest: `||`, `&&`, equality, relations, sums, products. + fn binary(&mut self, level: usize) -> Result { + const LEVELS: [&[&str]; 6] = [ + &["||"], + &["&&"], + &["==", "!="], + &["<", "<=", ">", ">="], + &["+", "-"], + &["*", "/", "%"], + ]; + if level == LEVELS.len() { + return self.unary(); + } + let mut left = self.binary(level + 1)?; + while let Some(op) = self + .peek() + .and_then(|t| LEVELS[level].iter().find(|o| **o == t)) + { + self.at += 1; + let right = self.binary(level + 1)?; + left = Expr::Bin(op, Box::new(left), Box::new(right)); + } + Ok(left) + } + fn unary(&mut self) -> Result { + if self.eat("-") { + return Ok(Expr::Neg(Box::new(self.unary()?))); + } + if self.eat("!") { + return Ok(Expr::Not(Box::new(self.unary()?))); + } + self.eat("+"); + self.primary() + } + fn primary(&mut self) -> Result { + let Some(t) = self.peek().map(str::to_owned) else { + return Err("the expression ends early".into()); + }; + self.at += 1; + if t == "(" { + let e = self.binary(0)?; + return if self.eat(")") { + Ok(e) + } else { + Err("missing `)`".into()) + }; + } + if t.starts_with(|c: char| c.is_ascii_digit() || c == '.') { + return t + .trim_end_matches(['f', 'F']) + .parse() + .map(Expr::Num) + .map_err(|_| format!("bad number `{t}`")); + } + if !is_name(&t) { + return Err(format!("unexpected `{t}`")); + } + if self.eat("(") { + let mut args = Vec::new(); + if !self.eat(")") { + loop { + args.push(self.binary(0)?); + if self.eat(")") { + break; + } + if !self.eat(",") { + return Err(format!("missing `)` after the arguments of `{t}`")); + } + } + } + *self.calls += 1; + return Ok(Expr::Call(t, args, *self.calls)); + } + let component = if self.eat(".") { + let c = self.peek().map(str::to_owned).ok_or("nothing after `.`")?; + self.at += 1; + let index = |s: &str| match s { + "x" | "r" | "s" | "0" => Some(0), + "y" | "g" | "t" | "1" => Some(1), + "z" | "b" | "p" | "2" => Some(2), + "w" | "a" | "q" | "3" => Some(3), + _ => None, + }; + Some(match c.split_once('.') { + Some((r, col)) => ( + index(r).ok_or(format!("bad component `{c}`"))?, + Some(index(col).ok_or(format!("bad component `{c}`"))?), + ), + None => (index(&c).ok_or(format!("bad component `{c}`"))?, None), + }) + } else { + None + }; + Ok(Expr::Name(t, component)) + } +} + +fn is_name(t: &str) -> bool { + t.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_') +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Macros; + + fn custom(text: &str) -> Custom { + Custom::parse(&Properties::parse(text, &Macros::standard())) + } + + #[test] + fn expressions_follow_c_precedence_and_file_order() { + let mut c = custom( + "variable.float.a = 1 + 2 * 3 - 4 % 3\n\ + uniform.float.b = a / 2\n\ + uniform.bool.c = a > 5 && !(b < 1) || false\n\ + uniform.int.d = -7 / 2\n\ + uniform.vec3.e = vec3(a, cameraPosition.y, 2) * 2\n\ + uniform.float.f = if(a < 0, 1, a < 7, 2, 3) + clamp(fmod(-1, 4), 0, 2.5) + min(4, 2, 3)\n\ + uniform.float.g = gbufferModelView.1.0 + frac(sunAngle - 0.25) + pi * 0\n\ + uniform.float.h = unknownThing + in(biome, 3, BIOME_DESERT)\n", + ); + assert!(c.notes.is_empty(), "{:?}", c.notes); + let names: Vec<_> = c.uniforms().map(|u| u.0).collect(); + assert_eq!(names, ["b", "c", "d", "e", "f", "g", "h"]); + let builtin = |n: &str| match n { + "cameraPosition" => Some(Val::vector(&[1.0, 64.0, 2.0])), + "sunAngle" => Some(Val::scalar(0.125)), + "biome" => Some(Val::scalar(5.0)), + "BIOME_DESERT" => Some(Val::scalar(5.0)), + "gbufferModelView" => { + let mut m = Val::scalar(0.0); + m.v[1] = 9.0; // column 0, row 1 + m.n = 16; + Some(m) + } + _ => None, + }; + let v = c.eval(0.016, &builtin); + let x: Vec = v.iter().map(|v| v.v[0]).collect(); + assert_eq!(x, [3.0, 1.0, -3.0, 12.0, 2.0 + 2.5 + 2.0, 9.0 + 0.875, 1.0]); + assert_eq!(&v[3].v[..3], &[12.0, 128.0, 4.0]); + assert_eq!(c.notes, ["`unknownThing` is unknown; it reads 0"]); + } + + #[test] + fn smooth_halves_the_distance_per_fade_time() { + let mut c = custom("uniform.float.s = smooth(1, target, 2, 0.5)\n"); + let mut target = 0.0; + let mut run = |t: f64, dt: f64| { + target = t; + c.eval(dt, &|n: &str| (n == "target").then(|| Val::scalar(target)))[0].v[0] + }; + assert_eq!(run(0.0, 0.0), 0.0); + assert!((run(1.0, 2.0) - 0.5).abs() < 1e-9, "rising: fadeIn"); + assert!((run(0.0, 0.5) - 0.25).abs() < 1e-9, "falling: fadeOut"); + } + + #[test] + fn bad_expressions_are_reported() { + let c = custom("uniform.float.a = (1 + \nuniform.mat4.m = 1\nuniform.float.ok = 2\n"); + assert_eq!(c.uniforms().count(), 1); + assert_eq!(c.notes.len(), 2, "{:?}", c.notes); + } +} diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index d29a0a632..47cbcca11 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -14,6 +14,7 @@ mod condition; mod config; +mod custom; mod discover; mod lang; mod macros; @@ -26,6 +27,7 @@ mod translate; pub use condition::eval_condition; pub use config::{BUFFERS, Buffer, Config, Format, Kind, buffer_index, draw_buffers}; +pub use custom::{Custom, Val}; pub use discover::{Found, discover}; pub use lang::Lang; pub use macros::{MC_VERSION, Macros, RENDER_STAGES}; diff --git a/crates/horizon-shaders/src/post.rs b/crates/horizon-shaders/src/post.rs index c96730f0a..d6142c70a 100644 --- a/crates/horizon-shaders/src/post.rs +++ b/crates/horizon-shaders/src/post.rs @@ -13,6 +13,7 @@ use std::collections::BTreeSet; use crate::config::{self, Config}; +use crate::custom::Custom; use crate::{GbuffersInputs, Macros, Pack, Preprocessed, Program, Properties, Target}; use crate::{eval_condition, preprocess, translate_composite, translate_gbuffers}; @@ -62,6 +63,8 @@ pub struct Post { pub composite_pre: Vec<(u32, bool)>, /// What draws solid and cutout terrain, when the game asked for it and one loads. pub terrain: Option, + /// The custom uniforms of `shaders.properties` (3.4). + pub custom: Custom, pub config: Config, /// Programs skipped and why, as `program: message` or `file:line: message`. pub notes: Vec, @@ -93,6 +96,7 @@ pub fn load_post( deferred_pre: flips(properties, "deferred_pre"), composite_pre: flips(properties, "composite_pre"), terrain: None, + custom: Custom::parse(properties), config: Config::default(), notes: Vec::new(), }; diff --git a/crates/horizon-shaders/tests/real_packs.rs b/crates/horizon-shaders/tests/real_packs.rs index d0a28d71d..78ba97bfb 100644 --- a/crates/horizon-shaders/tests/real_packs.rs +++ b/crates/horizon-shaders/tests/real_packs.rs @@ -182,6 +182,13 @@ fn real_packs_read() { post.notes.len() + post.config.notes.len(), start.elapsed().as_secs_f64() * 1000.0 ); + let mut custom = post.custom; + custom.eval(0.016, &|_| None); + println!( + " {} custom uniforms; notes: {:?}", + custom.uniforms().count(), + custom.notes.iter().take(4).collect::>() + ); for n in post.notes.iter().chain(&post.config.notes).take(12) { println!(" {n}"); } From a2ec3fc791e773c69b852922ca53af1be298cd47 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:03:17 +0100 Subject: [PATCH 05/61] Shader packs, clean room 17: real packs' buffers and samplers - RGBA16 and R11F_G11F_B10F buffers get their own formats where the device has them (16-bit normalized render targets need the adapter's format features): the float stand-in kept NaN and values past 1, and BSL's TAA history went black for good. - Textures share one sampler variable per kind (linear, nearest, repeat, nearest-repeat, comparison): devices have as few as 16 sampler slots, Complementary's terrain program declares 23 textures. - A gbuffers pass reading the depth buffer or a buffer it draws into reads a blank texture instead (a feedback loop: undefined in GL, refused by wgpu, and a crash here). - A bind group layout that fails validation skips the program with the error's details instead of ending the game. - shadowModelView / shadowProjection (and inverses) for programs that read them before the shadow pass exists: an orthographic camera 160 blocks each side, toward the shadow light. - HORIZON_SHADER_PACK_SKIP=a,b leaves programs out (a developers' aid to bisect a pack). Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, sections 6.1, 6.7, 5.2; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx.rs | 9 +- crates/horizon-client/src/gfx/pack_post.rs | 243 ++++++++++++++---- .../horizon-client/src/gfx/pack_uniforms.rs | 16 ++ 3 files changed, 209 insertions(+), 59 deletions(-) diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index c117b190f..a33d54fa3 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -816,12 +816,17 @@ async fn open_device( continue; } // Mappable primary buffers: section pages that no longer fit in - // video memory go to system memory. + // video memory go to system memory. Shader packs' 16-bit normalized buffers (render + // targets with the adapter's own format features) and R11F_G11F_B10F: a float + // stand-in neither clamps nor drops NaN. let features = timing::GpuTimer::features(&adapter) | cull::Cull::features(&adapter) | (adapter.features() & (wgpu::Features::INDIRECT_FIRST_INSTANCE - | wgpu::Features::MAPPABLE_PRIMARY_BUFFERS)); + | wgpu::Features::MAPPABLE_PRIMARY_BUFFERS + | wgpu::Features::TEXTURE_FORMAT_16BIT_NORM + | wgpu::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES + | wgpu::Features::RG11B10UFLOAT_RENDERABLE)); let driver = format!("{} {}", info.driver, info.driver_info); startup_step(format_args!( "requesting the device (driver {}) with features [{features}] and the adapter's \ diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 6520565ed..0201c7976 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -39,6 +39,82 @@ struct Binding { binding: u32, /// A filtering sampler (the texture is filterable). filtering: bool, + /// Where this texture's sampler object is bound: the first texture of each kind binds it + /// (`binding + 1`), the others share it (devices have as few as 16 sampler slots). + sampler: u32, +} + +/// The sampler objects a pass binds, one slot per kind it uses. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +enum SamplerKind { + Linear, + Nearest, + Repeat, + NearestRepeat, + Comparison, +} + +impl SamplerKind { + fn of(source: Source, filtering: bool) -> Self { + match source { + Source::Color(_) | Source::Dummy if !filtering => Self::Nearest, + Source::Depth => Self::Nearest, + Source::Noise => Self::Repeat, + Source::Shadow => Self::Comparison, + // Blocks look as the game draws them: nearest texels. + Source::Atlas => Self::NearestRepeat, + Source::Color(_) | Source::Dummy | Source::Lightmap => Self::Linear, + } + } + + fn binding_type(self) -> wgpu::SamplerBindingType { + match self { + Self::Linear | Self::Repeat => wgpu::SamplerBindingType::Filtering, + Self::Nearest | Self::NearestRepeat => wgpu::SamplerBindingType::NonFiltering, + Self::Comparison => wgpu::SamplerBindingType::Comparison, + } + } +} + +/// Points every sampler object of `module` at the slot `slot` gives its binding, so that +/// samplers sharing a slot become one variable (two variables may not share a binding). +fn share_samplers(module: &mut naga::Module, slot: &HashMap) { + let mut first = HashMap::new(); + let mut same = HashMap::new(); + for (h, g) in module.global_variables.iter_mut() { + let Some(b) = g.binding.as_mut() else { + continue; + }; + if !matches!(module.types[g.ty].inner, naga::TypeInner::Sampler { .. }) { + continue; + } + let Some(&to) = slot.get(&b.binding) else { + continue; + }; + b.binding = to; + match first.entry(to) { + std::collections::hash_map::Entry::Occupied(e) => _ = same.insert(h, *e.get()), + std::collections::hash_map::Entry::Vacant(e) => _ = e.insert(h), + } + } + if same.is_empty() { + return; + } + let fix = |expressions: &mut naga::Arena| { + for (_, e) in expressions.iter_mut() { + if let naga::Expression::GlobalVariable(g) = e + && let Some(&to) = same.get(g) + { + *g = to; + } + } + }; + for (_, f) in module.functions.iter_mut() { + fix(&mut f.expressions); + } + for ep in &mut module.entry_points { + fix(&mut ep.function.expressions); + } } struct Pass { @@ -128,7 +204,12 @@ fn texture_format(f: Format, features: wgpu::Features) -> wgpu::TextureFormat { let half = pick(T::R16Float, T::Rg16Float, T::Rgba16Float); match (f.kind, f.bits) { (Kind::Unorm, 10) => T::Rgb10a2Unorm, - (Kind::Unorm, 16) if features.contains(wgpu::Features::TEXTURE_FORMAT_16BIT_NORM) => { + (Kind::Unorm, 16) + if features.contains( + wgpu::Features::TEXTURE_FORMAT_16BIT_NORM + | wgpu::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES, + ) => + { pick(T::R16Unorm, T::Rg16Unorm, T::Rgba16Unorm) } (Kind::Unorm, b) if b <= 8 => pick(T::R8Unorm, T::Rg8Unorm, T::Rgba8Unorm), @@ -229,8 +310,11 @@ fn checked(device: &wgpu::Device, create: impl FnOnce() -> T) -> Result { - // The details are in the error's sources. - let mut message = e.to_string(); + // The details are in the error's description or its sources. + let mut message = match &e { + wgpu::Error::Validation { source, .. } => format!("{source:?}"), + e => e.to_string(), + }; let mut source = std::error::Error::source(&e); while let Some(s) = source { message = format!("{message}: {s}"); @@ -365,12 +449,22 @@ fn build_pass( }, )]; let mut samplers = Vec::new(); + let (mut kinds, mut slots) = (HashMap::new(), HashMap::new()); let mut reads = 0u16; + // A gbuffers pass draws into its buffers and the depth buffer: reading them as well would + // be a feedback loop (undefined in GL, refused by wgpu), so those read a blank texture. + let in_place = match geometry { + Some(_) => p.buffers.iter().fold(0u16, |m, &b| m | 1 << b), + None => 0, + }; for s in &interface.samplers { let mut src = source(&s.name); - if src == Source::Atlas && geometry.is_none() { - src = Source::Dummy; - } + let blank = match src { + Source::Atlas => geometry.is_none(), + Source::Depth => geometry.is_some(), + Source::Color(n) => in_place & 1 << n != 0, + _ => false, + }; let float = wgpu::TextureSampleType::Float { filterable: true }; let mut sample_type = match src { Source::Color(n) => formats[n] @@ -399,25 +493,32 @@ fn build_pass( )); (src, sample_type) = (Source::Dummy, float); } + if blank { + // The blank texture is float: an integer sampler then fails the pipeline (a note). + src = Source::Dummy; + if int { + sample_type = float; + } + } let filtering = sample_type == float; if let Source::Color(n) = src { reads |= 1 << n; } entries.push(texture_entry(s.binding, sample_type)); - entries.push(entry( - s.binding + 1, - wgpu::BindingType::Sampler(if src == Source::Shadow { - wgpu::SamplerBindingType::Comparison - } else if filtering { - wgpu::SamplerBindingType::Filtering - } else { - wgpu::SamplerBindingType::NonFiltering - }), - )); + let kind = SamplerKind::of(src, filtering); + let sampler = *kinds.entry(kind).or_insert_with(|| { + entries.push(entry( + s.binding + 1, + wgpu::BindingType::Sampler(kind.binding_type()), + )); + s.binding + 1 + }); + slots.insert(s.binding + 1, sampler); samplers.push(Binding { source: src, binding: s.binding, filtering, + sampler, }); } let final_pass = p.group == Group::Final; @@ -453,24 +554,25 @@ fn build_pass( } else { (written & !kept) | mask(&p.flips) }; - let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some(&p.name), - entries: &entries, - }); - let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some(&p.name), - bind_group_layouts: &[Some(&layout), geometry.and_then(|g| g.table)], - ..Default::default() - }); let created = checked(device, || { - let module = |m: naga::Module| { + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some(&p.name), + entries: &entries, + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some(&p.name), + bind_group_layouts: &[Some(&layout), geometry.and_then(|g| g.table)], + ..Default::default() + }); + let module = |mut m: naga::Module| { + share_samplers(&mut m, &slots); device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some(&p.name), source: wgpu::ShaderSource::Naga(std::borrow::Cow::Owned(m)), }) }; let (vs, fs) = (module(p.program.vertex), module(p.program.fragment)); - device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some(&p.name), layout: Some(&pipeline_layout), vertex: wgpu::VertexState { @@ -496,10 +598,11 @@ fn build_pass( }), multiview_mask: None, cache: None, - }) + }); + (layout, pipeline) }); - let pipeline = match created { - Ok(p) => p, + let (layout, pipeline) = match created { + Ok(created) => created, Err(e) => { notes.push(format!("{}: skipped: {e}", p.name)); return None; @@ -590,9 +693,12 @@ impl PackPost { .terrain .zip(world.terrain.as_ref()) .and_then(|(p, g)| build(p, Some(g))); + // Developers' aid: `HORIZON_SHADER_PACK_SKIP=composite,final` leaves programs out. + let skip = std::env::var("HORIZON_SHADER_PACK_SKIP").unwrap_or_default(); let passes: Vec = post .programs .into_iter() + .filter(|p| !skip.split(',').any(|s| s == p.name)) .filter_map(|p| build(p, None)) .collect(); if let Some(t) = &terrain { @@ -775,37 +881,34 @@ impl PackPost { resource: pass.ubo.as_entire_binding(), }]; for s in &pass.samplers { - let filter = if s.filtering { - &self.linear - } else { - &self.nearest + let texture = match s.source { + Source::Color(i) => targets.color[i] + .as_ref() + .map_or(&self.dummy, |t| &t[usize::from(key & 1 << i != 0)]), + Source::Depth => &targets.depth, + Source::Noise => &self.noise, + Source::Dummy => &self.dummy, + Source::Shadow => &self.shadow.0, + Source::Atlas => self.atlas.as_ref().unwrap_or(&self.dummy), + Source::Lightmap => &self.lightmap, }; - let (texture, sampler) = match s.source { - Source::Color(i) => ( - targets.color[i] - .as_ref() - .map_or(&self.dummy, |t| &t[usize::from(key & 1 << i != 0)]), - filter, - ), - Source::Depth => (&targets.depth, &self.nearest), - Source::Noise => (&self.noise, &self.repeat), - Source::Dummy => (&self.dummy, &self.linear), - Source::Shadow => (&self.shadow.0, &self.shadow.1), - // Blocks look as the game draws them: nearest texels. - Source::Atlas => ( - self.atlas.as_ref().unwrap_or(&self.dummy), - &self.nearest_repeat, - ), - Source::Lightmap => (&self.lightmap, &self.linear), + let sampler = match SamplerKind::of(s.source, s.filtering) { + SamplerKind::Linear => &self.linear, + SamplerKind::Nearest => &self.nearest, + SamplerKind::Repeat => &self.repeat, + SamplerKind::NearestRepeat => &self.nearest_repeat, + SamplerKind::Comparison => &self.shadow.1, }; entries.push(wgpu::BindGroupEntry { binding: s.binding, resource: wgpu::BindingResource::TextureView(texture), }); - entries.push(wgpu::BindGroupEntry { - binding: s.binding + 1, - resource: wgpu::BindingResource::Sampler(sampler), - }); + if s.sampler == s.binding + 1 { + entries.push(wgpu::BindGroupEntry { + binding: s.sampler, + resource: wgpu::BindingResource::Sampler(sampler), + }); + } } device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some(&pass.name), @@ -1130,6 +1233,31 @@ mod tests { use super::*; use wgpu::TextureFormat as T; + /// Two textures with a sampler each become two textures sharing one sampler variable. + #[test] + fn samplers_of_a_kind_share_one_variable() { + let mut module = naga::front::wgsl::parse_str( + "@group(0) @binding(1) var a: texture_2d;\n\ + @group(0) @binding(2) var sa: sampler;\n\ + @group(0) @binding(3) var b: texture_2d;\n\ + @group(0) @binding(4) var sb: sampler;\n\ + @fragment fn main() -> @location(0) vec4 {\n\ + return textureSample(a, sa, vec2(0.5)) + textureSample(b, sb, vec2(0.5));\n}\n", + ) + .unwrap(); + share_samplers(&mut module, &HashMap::from([(2, 2), (4, 2)])); + let info = naga::valid::Validator::new(Default::default(), Default::default()) + .validate(&module) + .unwrap(); + let used: Vec = module + .global_variables + .iter() + .filter(|(h, _)| !info.get_entry_point(0)[*h].is_empty()) + .filter_map(|(_, g)| g.binding.as_ref().map(|b| b.binding)) + .collect(); + assert_eq!(used, [1, 2, 3]); + } + #[test] fn pack_formats_map_to_renderable_ones() { let f = |name| texture_format(Format::parse(name).unwrap(), wgpu::Features::empty()); @@ -1142,8 +1270,9 @@ mod tests { assert_eq!(f("RGBA16"), T::Rgba16Float, "16-bit norm needs a feature"); assert_eq!(f("RGBA8_SNORM"), T::Rgba16Float, "snorm is not renderable"); assert_eq!(f("R11F_G11F_B10F"), T::Rgba16Float); - let all = - wgpu::Features::TEXTURE_FORMAT_16BIT_NORM | wgpu::Features::RG11B10UFLOAT_RENDERABLE; + let all = wgpu::Features::TEXTURE_FORMAT_16BIT_NORM + | wgpu::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES + | wgpu::Features::RG11B10UFLOAT_RENDERABLE; let g = |name| texture_format(Format::parse(name).unwrap(), all); assert_eq!(g("RGBA16"), T::Rgba16Unorm); assert_eq!(g("R11F_G11F_B10F"), T::Rg11b10Ufloat); diff --git a/crates/horizon-client/src/gfx/pack_uniforms.rs b/crates/horizon-client/src/gfx/pack_uniforms.rs index 5a27e62a0..c7c4cfd8b 100644 --- a/crates/horizon-client/src/gfx/pack_uniforms.rs +++ b/crates/horizon-client/src/gfx/pack_uniforms.rs @@ -141,6 +141,10 @@ pub const NAMES: &[&str] = &[ "gbufferPreviousModelView", "gbufferPreviousProjection", "atlasSize", + "shadowModelView", + "shadowModelViewInverse", + "shadowProjection", + "shadowProjectionInverse", ]; /// Values carried from frame to frame. @@ -226,6 +230,14 @@ impl FrameUniforms { ), None => (frame.sky.fog_color, frame.distance * 0.9, frame.distance), }; + // The shadow camera (OQ-6, impl-plan.md): orthographic, `shadowDistance` (160) blocks + // each side, 100 blocks from the player toward the shadow light, world +z up. + let light = view + .inverse() + .transform_vector3(if day { sun } else { moon }) + .normalize(); + let shadow_view = Mat4::look_at_rh(light * 100.0, Vec3::ZERO, Vec3::Z); + let shadow_projection = Mat4::orthographic_rh_gl(-160.0, 160.0, -160.0, 160.0, 0.05, 256.0); let day_time = inputs.day_time.rem_euclid(24000) as i32; let world_day = inputs.day_time.div_euclid(24000); let seconds = (now - self.start).as_secs_f32() % 3600.0; @@ -247,6 +259,10 @@ impl FrameUniforms { "worldDay" => Value::i(&[world_day as i32]), "moonPhase" => Value::i(&[world_day.rem_euclid(8) as i32]), "frameCounter" => Value::i(&[self.frame as i32]), + "shadowModelView" => m4(shadow_view), + "shadowModelViewInverse" => m4(shadow_view.inverse()), + "shadowProjection" => m4(shadow_projection), + "shadowProjectionInverse" => m4(shadow_projection.inverse()), "atlasSize" => Value::i(&[self.atlas[0] as i32, self.atlas[1] as i32]), "frameTime" => Value::f(&[dt]), "frameTimeCounter" => Value::f(&[seconds]), From 3e2c30a852452a3ee0a4cf778d85aee55b98c205 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:47:14 +0100 Subject: [PATCH 06/61] Shader packs, clean room 18: translucent terrain drawn by the pack With a pack whose gbuffers_water (or a fallback down gbuffers_terrain's chain) loads, translucent terrain (water, stained glass) is drawn by it after the deferred programs and before the composites, in place, sorted back to front as the game sorts it, and Horizon's own translucent terrain draw is skipped. - blend. (off, or four GL factors); translucent terrain defaults to SRC_ALPHA ONE_MINUS_SRC_ALPHA ONE ONE_MINUS_SRC_ALPHA, and only blendable buffer formats blend. - depthtex1/depthtex2: depth without translucent terrain, a copy taken just before it when a later program reads them (before it, depthtex0 itself). The hand is drawn after the pack, so depthtex2 = depthtex1. Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, sections 2.1, 2.3, 3.2 (blend), 6.2 (depthtex1/2); no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx.rs | 26 ++- crates/horizon-client/src/gfx/pack_post.rs | 213 +++++++++++++++++---- crates/horizon-shaders/src/lib.rs | 4 +- crates/horizon-shaders/src/post.rs | 130 ++++++++++++- crates/horizon-shaders/tests/real_packs.rs | 3 +- 5 files changed, 327 insertions(+), 49 deletions(-) diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index a33d54fa3..9a7356d6e 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -3709,6 +3709,12 @@ impl Renderer { let pack_post = !frame.title && self.pack_post.as_ref().is_some_and(|p| p.1.is_some()); // The pack's program reads the vertices this frame's draws bind. let pack_draws_terrain = pack_terrain && flat_frame == self.pack_terrain_flat; + let pack_draws_water = pack_draws_terrain + && self + .pack_post + .as_ref() + .and_then(|p| p.1.as_ref()) + .is_some_and(|p| p.has_water()); let scaled = self.world_post.prepare( &self.device, [self.config.width, self.config.height], @@ -3940,6 +3946,8 @@ impl Renderer { pass.draw(0..4, debug_depth..debug_depth + debug_top); } match self.flat.as_ref().filter(|_| flat_frame) { + // The pack draws translucent terrain after its deferred programs. + _ if pack_draws_water => {} Some(flat) => { pass.set_pipeline(&flat.translucent); flat.draw(&mut pass, &flat.calls[flat.solid_calls..], |p| { @@ -4016,20 +4024,28 @@ impl Renderer { { let size = self.world_post.size().unwrap_or([1, 1]); let flat = self.flat.as_ref().filter(|_| flat_frame); - let mut terrain = |rp: &mut wgpu::RenderPass<'_>| { + let mut terrain = |rp: &mut wgpu::RenderPass<'_>, translucent: bool| { if !pack_draws_terrain { return; } match flat { Some(flat) => { - flat.draw(rp, &flat.calls[..flat.solid_calls], |p| { - self.arena.buffer(p) - }); + let calls = if translucent { + &flat.calls[flat.solid_calls..] + } else { + &flat.calls[..flat.solid_calls] + }; + flat.draw(rp, calls, |p| self.arena.buffer(p)); } None => { rp.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32); rp.set_vertex_buffer(1, self.instances.slice(..)); - self.draw_sections(rp, &self.indirect, 0, 0, solid_draws); + let (first, count) = if translucent { + (solid_draws, translucent_draws) + } else { + (0, solid_draws) + }; + self.draw_sections(rp, &self.indirect, 0, first, count); } } }; diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 0201c7976..b96da2599 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -24,6 +24,8 @@ const BUFFERS: usize = horizon_shaders::BUFFERS; enum Source { Color(usize), Depth, + /// `depthtex1`/`depthtex2`: depth without translucent terrain (a copy taken before it). + Depth1, Noise, /// A 1×1 stand-in for names this slice does not provide yet (custom textures...). Dummy, @@ -58,7 +60,7 @@ impl SamplerKind { fn of(source: Source, filtering: bool) -> Self { match source { Source::Color(_) | Source::Dummy if !filtering => Self::Nearest, - Source::Depth => Self::Nearest, + Source::Depth | Source::Depth1 => Self::Nearest, Source::Noise => Self::Repeat, Source::Shadow => Self::Comparison, // Blocks look as the game draws them: nearest texels. @@ -143,6 +145,9 @@ struct Targets { /// A and B of each buffer used. color: [Option<[wgpu::TextureView; 2]>; BUFFERS], depth: wgpu::TextureView, + depth_texture: wgpu::Texture, + /// `depthtex1`, when a program after translucent terrain reads it. + depth1: Option<(wgpu::Texture, wgpu::TextureView)>, } /// What a pack's gbuffers programs read and draw from the game's world. @@ -159,6 +164,10 @@ pub struct PackPost { passes: Vec, /// What draws terrain (`gbuffers_terrain` or its fallback), between `prepare` and `deferred`. terrain: Option, + /// What draws translucent terrain (`gbuffers_water` or its fallback), after `deferred`. + water: Option, + /// A program after translucent terrain reads `depthtex1`: it is copied before water. + depth1: bool, /// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`. custom: horizon_shaders::Custom, atlas: Option, @@ -231,7 +240,9 @@ fn texture_format(f: Format, features: wgpu::Features) -> wgpu::TextureFormat { fn source(name: &str) -> Source { match name { - "depthtex0" | "gdepthtex" | "depthtex1" | "depthtex2" => Source::Depth, + "depthtex0" | "gdepthtex" => Source::Depth, + // The hand is drawn after the pack, so `depthtex2` (without it) is `depthtex1`. + "depthtex1" | "depthtex2" => Source::Depth1, "noisetex" => Source::Noise, "texture" | "gtexture" | "tex" => Source::Atlas, "lightmap" => Source::Lightmap, @@ -392,6 +403,34 @@ fn texture_entry(binding: u32, sample_type: wgpu::TextureSampleType) -> wgpu::Bi ) } +/// `blend.`'s factors as wgpu's, adding both terms. +fn blend_state(b: horizon_shaders::Blend) -> wgpu::BlendState { + use horizon_shaders::Factor as F; + use wgpu::BlendFactor as W; + let f = |f| match f { + F::Zero => W::Zero, + F::One => W::One, + F::SrcColor => W::Src, + F::OneMinusSrcColor => W::OneMinusSrc, + F::DstColor => W::Dst, + F::OneMinusDstColor => W::OneMinusDst, + F::SrcAlpha => W::SrcAlpha, + F::OneMinusSrcAlpha => W::OneMinusSrcAlpha, + F::DstAlpha => W::DstAlpha, + F::OneMinusDstAlpha => W::OneMinusDstAlpha, + F::SrcAlphaSaturate => W::SrcAlphaSaturated, + }; + let component = |src, dst| wgpu::BlendComponent { + src_factor: f(src), + dst_factor: f(dst), + operation: wgpu::BlendOperation::Add, + }; + wgpu::BlendState { + color: component(b[0], b[1]), + alpha: component(b[2], b[3]), + } +} + fn target(format: wgpu::TextureFormat) -> Option { Some(wgpu::ColorTargetState { format, @@ -462,6 +501,8 @@ fn build_pass( let blank = match src { Source::Atlas => geometry.is_none(), Source::Depth => geometry.is_some(), + // Translucent terrain reads the copy; opaque terrain would read itself. + Source::Depth1 => p.group == Group::Gbuffers, Source::Color(n) => in_place & 1 << n != 0, _ => false, }; @@ -470,7 +511,7 @@ fn build_pass( Source::Color(n) => formats[n] .sample_type(None, Some(features)) .unwrap_or(float), - Source::Depth => wgpu::TextureSampleType::Float { filterable: false }, + Source::Depth | Source::Depth1 => wgpu::TextureSampleType::Float { filterable: false }, // Unfilterable on WebGPU-level devices; the lightmap is sampled at texel centres. Source::Lightmap => float, _ => float, @@ -534,7 +575,17 @@ fn build_pass( targets[0] = target(screen); } } else if let Some(&buffer) = p.buffers.get(l) { - targets[l] = target(formats[buffer as usize]); + let format = formats[buffer as usize]; + let mut state = target(format); + if let (Some(t), Some(b)) = (&mut state, p.blend) + && format + .guaranteed_format_features(features) + .flags + .contains(wgpu::TextureFormatFeatureFlags::BLENDABLE) + { + t.blend = Some(blend_state(b)); + } + targets[l] = state; writes.push((location, buffer as usize)); } } @@ -693,6 +744,10 @@ impl PackPost { .terrain .zip(world.terrain.as_ref()) .and_then(|(p, g)| build(p, Some(g))); + let water = post + .water + .zip(world.terrain.as_ref()) + .and_then(|(p, g)| build(p, Some(g))); // Developers' aid: `HORIZON_SHADER_PACK_SKIP=composite,final` leaves programs out. let skip = std::env::var("HORIZON_SHADER_PACK_SKIP").unwrap_or_default(); let passes: Vec = post @@ -704,25 +759,43 @@ impl PackPost { if let Some(t) = &terrain { notes.push(format!("terrain draws with {}", t.name)); } + if let Some(w) = &water { + notes.push(format!("translucent terrain draws with {}", w.name)); + } + let depth1 = water.is_some() + && passes + .iter() + .filter(|p| matches!(p.group, Group::Composite | Group::Final)) + .chain(&water) + .any(|p| p.samplers.iter().any(|s| s.source == Source::Depth1)); // A clear only matters when a program reads the buffer before one writes all of it // (composite passes cover their whole target, terrain does not); the others are skipped. let mut cleared = Vec::new(); for b in 1..BUFFERS { let bit = 1u16 << b; let touched = |p: &&Pass| (p.reads | p.flip | p.written) & bit != 0; - let first = terrain.iter().find(touched).map(|_| true).or_else(|| { - let p = passes.iter().find(touched)?; - Some(p.reads & bit != 0 || p.written & bit == 0) - }); + // Gbuffers passes cover part of their targets: a clear is needed if they touch it. + let first = terrain + .iter() + .chain(&water) + .find(touched) + .map(|_| true) + .or_else(|| { + let p = passes.iter().find(touched)?; + Some(p.reads & bit != 0 || p.written & bit == 0) + }); if clears[b].is_some() && first == Some(true) { cleared.push(b); } } // Terrain tests against the world's depth (entities...), so the blit copies it then. let blit_depth = terrain.is_some() - || passes - .iter() - .any(|p| p.samplers.iter().any(|s| s.source == Source::Depth)); + || water.is_some() + || passes.iter().any(|p| { + p.samplers + .iter() + .any(|s| matches!(s.source, Source::Depth | Source::Depth1)) + }); let room = 8 - 1; let other_clears = cleared.split_off(room.min(cleared.len())); let float = wgpu::TextureSampleType::Float { filterable: true }; @@ -778,6 +851,8 @@ impl PackPost { notes, passes, terrain, + water, + depth1, custom, atlas: world.atlas.as_ref().map(|a| a.0.clone()), lightmap: world.lightmap, @@ -886,6 +961,18 @@ impl PackPost { .as_ref() .map_or(&self.dummy, |t| &t[usize::from(key & 1 << i != 0)]), Source::Depth => &targets.depth, + // Before translucent terrain the two are the same. + Source::Depth1 => match &targets.depth1 { + Some(d) + if matches!( + pass.group, + Group::Composite | Group::Final | Group::Translucent + ) => + { + &d.1 + } + _ => &targets.depth, + }, Source::Noise => &self.noise, Source::Dummy => &self.dummy, Source::Shadow => &self.shadow.0, @@ -919,19 +1006,36 @@ impl PackPost { group.clone() } - /// Terrain with the pack's program into the buffers it writes, in place, tested against - /// the world's depth; `draw` issues the game's draws. + /// Terrain (or, `translucent`, translucent terrain) with the pack's program into the + /// buffers it writes, in place, tested against the world's depth; `draw` issues the + /// game's draws. fn draw_terrain( &mut self, device: &wgpu::Device, queue: &wgpu::Queue, encoder: &mut wgpu::CommandEncoder, size: [u32; 2], - draw: &mut dyn FnMut(&mut wgpu::RenderPass<'_>), + translucent: bool, + draw: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), ) { - let Some(mut pass) = self.terrain.take() else { + let slot = if translucent { + &mut self.water + } else { + &mut self.terrain + }; + let Some(mut pass) = slot.take() else { return; }; + if translucent + && let Some(t) = &self.targets + && let Some((copy, _)) = &t.depth1 + { + encoder.copy_texture_to_texture( + t.depth_texture.as_image_copy(), + copy.as_image_copy(), + t.depth_texture.size(), + ); + } let group = self.bind(device, queue, &mut pass, size); let targets = self.targets.as_ref().expect("resized"); let load = wgpu::Operations { @@ -964,9 +1068,13 @@ impl PackPost { }); rp.set_pipeline(&pass.pipeline); rp.set_bind_group(0, &group, &[]); - draw(&mut rp); + draw(&mut rp, translucent); + } + if translucent { + self.water = Some(pass); + } else { + self.terrain = Some(pass); } - self.terrain = Some(pass); } /// Whether the pack draws terrain itself. @@ -974,6 +1082,11 @@ impl PackPost { self.terrain.is_some() } + /// Whether the pack draws translucent terrain itself. + pub fn has_water(&self) -> bool { + self.water.is_some() + } + /// The names of the programs that run, in order. pub fn programs(&self) -> Vec<&str> { self.passes.iter().map(|p| p.name.as_str()).collect() @@ -983,23 +1096,24 @@ impl PackPost { if self.targets.as_ref().is_some_and(|t| t.size == size) { return; } + let texture = |label: &str, format, usage| { + device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: size[0], + height: size[1], + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: usage | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }) + }; let make = |label: &str, format| { - device - .create_texture(&wgpu::TextureDescriptor { - label: Some(label), - size: wgpu::Extent3d { - width: size[0], - height: size[1], - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT - | wgpu::TextureUsages::TEXTURE_BINDING, - view_formats: &[], - }) + texture(label, format, wgpu::TextureUsages::RENDER_ATTACHMENT) .create_view(&Default::default()) }; let color = std::array::from_fn(|i| { @@ -1008,12 +1122,26 @@ impl PackPost { [make(&label, self.formats[i]), make(&label, self.formats[i])] }) }); + let depth32 = wgpu::TextureFormat::Depth32Float; + let depth_texture = texture( + "depthtex0", + depth32, + wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + ); + let depth1 = self.depth1.then(|| { + let t = texture("depthtex1", depth32, wgpu::TextureUsages::COPY_DST); + let view = t.create_view(&Default::default()); + (t, view) + }); self.targets = Some(Targets { size, color, - depth: make("depthtex0", wgpu::TextureFormat::Depth32Float), + depth: depth_texture.create_view(&Default::default()), + depth_texture, + depth1, }); - for pass in self.passes.iter_mut().chain(&mut self.terrain) { + let gbuffers = self.terrain.iter_mut().chain(&mut self.water); + for pass in self.passes.iter_mut().chain(gbuffers) { pass.groups.clear(); } } @@ -1033,7 +1161,7 @@ impl PackPost { inputs: &PackInputs, size: [u32; 2], window: [u32; 2], - terrain: &mut dyn FnMut(&mut wgpu::RenderPass<'_>), + terrain: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), ) { self.resize(device, size); self.uniforms.update(frame, inputs, size); @@ -1143,10 +1271,15 @@ impl PackPost { let mut seen = (false, false); let mut shown = false; let mut terrain_drawn = self.terrain.is_none(); + let mut water_drawn = self.water.is_none(); for pass in &mut passes { if !terrain_drawn && pass.group != Group::Prepare { terrain_drawn = true; - self.draw_terrain(device, queue, encoder, size, terrain); + self.draw_terrain(device, queue, encoder, size, false, terrain); + } + if !water_drawn && matches!(pass.group, Group::Composite | Group::Final) { + water_drawn = true; + self.draw_terrain(device, queue, encoder, size, true, terrain); } if pass.group == Group::Deferred && !seen.0 { seen.0 = true; @@ -1191,7 +1324,10 @@ impl PackPost { } self.passes = passes; if !terrain_drawn { - self.draw_terrain(device, queue, encoder, size, terrain); + self.draw_terrain(device, queue, encoder, size, false, terrain); + } + if !water_drawn { + self.draw_terrain(device, queue, encoder, size, true, terrain); } let targets = self.targets.as_ref().expect("resized"); let view = |m: u16, i: usize, alt: bool| { @@ -1282,6 +1418,7 @@ mod tests { fn sampler_names() { assert!(source("gaux1") == Source::Color(4)); assert!(source("gdepthtex") == Source::Depth); + assert!(source("depthtex2") == Source::Depth1); assert!(source("noisetex") == Source::Noise); assert!(source("shadowcolor0") == Source::Dummy); } diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index 47cbcca11..f5819d435 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -33,7 +33,9 @@ pub use lang::Lang; pub use macros::{MC_VERSION, Macros, RENDER_STAGES}; pub use options::switches_on; pub use pack::{Pack, PackError}; -pub use post::{Group, Post, PostProgram, dimension_folder, load_post}; +pub use post::{ + Blend, Factor, Group, Post, PostProgram, TRANSLUCENT_BLEND, dimension_folder, load_post, +}; pub use preprocess::{Diagnostic, Origin, Preprocessed, preprocess}; pub use properties::{Entry, Note, Properties}; pub use translate::{ diff --git a/crates/horizon-shaders/src/post.rs b/crates/horizon-shaders/src/post.rs index d6142c70a..e50eb4797 100644 --- a/crates/horizon-shaders/src/post.rs +++ b/crates/horizon-shaders/src/post.rs @@ -24,10 +24,67 @@ pub enum Group { Deferred, Composite, Final, - /// A gbuffers program: the game draws its geometry with it. + /// A gbuffers program: the game draws its geometry with it, before deferred. Gbuffers, + /// A gbuffers program drawing translucent geometry, after deferred (2.1). + Translucent, } +/// A blend factor of `blend.` (3.2), as GL names it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Factor { + Zero, + One, + SrcColor, + OneMinusSrcColor, + DstColor, + OneMinusDstColor, + SrcAlpha, + OneMinusSrcAlpha, + DstAlpha, + OneMinusDstAlpha, + SrcAlphaSaturate, +} + +impl Factor { + fn parse(word: &str) -> Option { + Some(match word { + "ZERO" => Self::Zero, + "ONE" => Self::One, + "SRC_COLOR" => Self::SrcColor, + "ONE_MINUS_SRC_COLOR" => Self::OneMinusSrcColor, + "DST_COLOR" => Self::DstColor, + "ONE_MINUS_DST_COLOR" => Self::OneMinusDstColor, + "SRC_ALPHA" => Self::SrcAlpha, + "ONE_MINUS_SRC_ALPHA" => Self::OneMinusSrcAlpha, + "DST_ALPHA" => Self::DstAlpha, + "ONE_MINUS_DST_ALPHA" => Self::OneMinusDstAlpha, + "SRC_ALPHA_SATURATE" => Self::SrcAlphaSaturate, + _ => return None, + }) + } +} + +/// Source and destination factors for colour, then for alpha. +pub type Blend = [Factor; 4]; + +/// The default blend of translucent geometry (3.2). +pub const TRANSLUCENT_BLEND: Blend = [ + Factor::SrcAlpha, + Factor::OneMinusSrcAlpha, + Factor::One, + Factor::OneMinusSrcAlpha, +]; + +/// `gbuffers_water` and its fallbacks (2.3). +const WATER: [&str; 5] = [ + "gbuffers_water", + "gbuffers_terrain", + "gbuffers_textured_lit", + "gbuffers_textured", + "gbuffers_basic", +]; + /// `gbuffers_terrain` and its fallbacks (2.3). const TERRAIN: [&str; 4] = [ "gbuffers_terrain", @@ -52,6 +109,8 @@ pub struct PostProgram { pub buffers: Vec, /// `flip..`: buffers forced to flip (true) or kept (false) after this program. pub flips: Vec<(u32, bool)>, + /// How a gbuffers program's output blends into its buffers; `None` writes them. + pub blend: Option, } pub struct Post { @@ -63,6 +122,8 @@ pub struct Post { pub composite_pre: Vec<(u32, bool)>, /// What draws solid and cutout terrain, when the game asked for it and one loads. pub terrain: Option, + /// What draws translucent terrain (water, stained glass) after deferred. + pub water: Option, /// The custom uniforms of `shaders.properties` (3.4). pub custom: Custom, pub config: Config, @@ -96,6 +157,7 @@ pub fn load_post( deferred_pre: flips(properties, "deferred_pre"), composite_pre: flips(properties, "composite_pre"), terrain: None, + water: None, custom: Custom::parse(properties), config: Config::default(), notes: Vec::new(), @@ -127,11 +189,24 @@ pub fn load_post( } } if let Some(inputs) = terrain { - loader.post.terrain = TERRAIN.iter().find_map(|name| { - loader.load(name.to_string(), Group::Gbuffers, |v, f| { - translate_gbuffers(v, f, Target::Buffers, inputs) + let mut gbuffers = |names: &[&str], group| { + names.iter().find_map(|name| { + loader.load(name.to_string(), group, |v, f| { + translate_gbuffers(v, f, Target::Buffers, inputs) + }) }) + }; + let terrain = gbuffers(&TERRAIN, Group::Gbuffers); + // gbuffers_water falls back to gbuffers_terrain's chain (2.3). + let water = gbuffers(&WATER, Group::Translucent); + loader.post.terrain = terrain; + loader.post.water = water.map(|mut p| { + p.blend = blend(properties, &p.name).unwrap_or(Some(TRANSLUCENT_BLEND)); + p }); + if let Some(p) = &mut loader.post.terrain { + p.blend = blend(properties, &p.name).flatten(); + } } post.config = std::mem::take(&mut post.config).finish(); post @@ -211,6 +286,7 @@ impl Loader<'_> { group, program, buffers, + blend: None, }) } } @@ -260,6 +336,20 @@ fn enabled( eval_condition(&condition, macros).ok() != Some(0) } +/// `blend.`: `Some(None)` for `off`, `Some(factors)`, or `None` when absent or not +/// four known factors. +fn blend(properties: &Properties, program: &str) -> Option> { + let value = properties.get(&format!("blend.{program}"))?.trim(); + if value == "off" { + return Some(None); + } + let factors: Vec = value + .split_whitespace() + .map(Factor::parse) + .collect::>()?; + Some(Some(factors.try_into().ok()?)) +} + /// `flip..=` keys of one program. fn flips(properties: &Properties, program: &str) -> Vec<(u32, bool)> { let prefix = format!("flip.{program}."); @@ -355,6 +445,38 @@ mod tests { assert!(post.programs.iter().any(|p| p.name == "composite3")); } + /// Water falls back to the terrain chain and blends by default; `blend.*` overrides. + #[test] + fn water_falls_back_to_terrain_and_blends() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/packs/terrain-test"); + let pack = Pack::open(&dir).unwrap(); + let macros = Macros::standard(); + let inputs = crate::GbuffersInputs { + declarations: "layout(location = 0) in vec3 hz_v_pos;".into(), + body: " hz_Vertex = vec4(hz_v_pos, 1.0);\n hz_Color = vec4(1.0);\n".into(), + attributes: Vec::new(), + }; + let post = load_post(&pack, "", &Properties::default(), ¯os, Some(&inputs)); + let (terrain, water) = (post.terrain.unwrap(), post.water.unwrap()); + assert_eq!((terrain.group, terrain.blend), (Group::Gbuffers, None)); + assert_eq!(water.name, "gbuffers_terrain"); + assert_eq!( + (water.group, water.blend), + (Group::Translucent, Some(TRANSLUCENT_BLEND)) + ); + let properties = Properties::parse( + "blend.gbuffers_terrain = ONE ONE ZERO ONE\nblend.gbuffers_water = off\n", + ¯os, + ); + let terrain_blend = blend(&properties, "gbuffers_terrain"); + assert_eq!( + terrain_blend, + Some(Some([Factor::One, Factor::One, Factor::Zero, Factor::One])) + ); + assert_eq!(blend(&properties, "gbuffers_water"), Some(None)); + assert_eq!(blend(&properties, "gbuffers_basic"), None); + } + #[test] fn horizons_test_pack_loads() { let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/packs/post-test"); diff --git a/crates/horizon-shaders/tests/real_packs.rs b/crates/horizon-shaders/tests/real_packs.rs index 78ba97bfb..c5d4defac 100644 --- a/crates/horizon-shaders/tests/real_packs.rs +++ b/crates/horizon-shaders/tests/real_packs.rs @@ -176,9 +176,10 @@ fn real_packs_read() { Some(&terrain), ); println!( - " dimension {id} (folder `{folder}`): {} post programs, terrain {:?}, {} notes, {:.0} ms", + " dimension {id} (folder `{folder}`): {} post programs, terrain {:?}, water {:?}, {} notes, {:.0} ms", post.programs.len(), post.terrain.as_ref().map(|t| &t.name), + post.water.as_ref().map(|t| &t.name), post.notes.len() + post.config.notes.len(), start.elapsed().as_secs_f64() * 1000.0 ); From b6345fc61d640de07afc9a2f3695039e02ac3201 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:56:31 +0100 Subject: [PATCH 07/61] Shader packs, clean room 19: GPU time of each pass (developers' aid) HORIZON_SHADER_PACK_TIMING=1 puts timestamps around every pass of a shader pack, one frame in 30, and logs each pass's median every 30 timed frames. The frame after a timed one waits for the GPU before reading them (Metal resolves timestamps without waiting for the passes that write them). Passes overlap on tiled GPUs, so a pass is charged from the later of its beginning and the previous pass's end. This is what attributes a pack's cost to its own programs or to ours when the machine is shared and FPS comparisons are noisy. Clean-room: Horizon's own tooling, no spec section; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx.rs | 1 + crates/horizon-client/src/gfx/pack_post.rs | 11 ++ crates/horizon-client/src/gfx/pack_timing.rs | 147 +++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 crates/horizon-client/src/gfx/pack_timing.rs diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 9a7356d6e..f0987dddd 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -70,6 +70,7 @@ mod lod; mod occlusion; mod pack_atlas; mod pack_post; +mod pack_timing; pub mod pack_uniforms; mod page_runs; pub mod panorama; diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index b96da2599..739688784 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -168,6 +168,8 @@ pub struct PackPost { water: Option, /// A program after translucent terrain reads `depthtex1`: it is copied before water. depth1: bool, + /// `HORIZON_SHADER_PACK_TIMING`: GPU time of each pass, in the log. + timing: Option, /// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`. custom: horizon_shaders::Custom, atlas: Option, @@ -853,6 +855,7 @@ impl PackPost { terrain, water, depth1, + timing: super::pack_timing::PassTiming::new(device, queue), custom, atlas: world.atlas.as_ref().map(|a| a.0.clone()), lightmap: world.lightmap, @@ -1053,9 +1056,11 @@ impl PackPost { .and_then(|t| attach(&t[usize::from(self.main & 1 << buffer != 0)], load)); } { + let timestamp_writes = self.timing.as_mut().and_then(|t| t.writes(&pass.name)); let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some(&pass.name), color_attachments: &attachments, + timestamp_writes, depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { view: &targets.depth, depth_ops: Some(wgpu::Operations { @@ -1164,6 +1169,9 @@ impl PackPost { terrain: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), ) { self.resize(device, size); + if let Some(t) = &mut self.timing { + t.begin(device, queue); + } self.uniforms.update(frame, inputs, size); self.uniforms.eval_custom(&mut self.custom); let Some(targets) = &self.targets else { return }; @@ -1214,6 +1222,7 @@ impl PackPost { encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("Shader pack clear"), color_attachments: &attachments, + timestamp_writes: self.timing.as_mut().and_then(|t| t.writes("clear")), ..Default::default() }); } @@ -1250,6 +1259,7 @@ impl PackPost { let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("Shader pack world"), color_attachments: &attachments, + timestamp_writes: self.timing.as_mut().and_then(|t| t.writes("world")), depth_stencil_attachment: self.blit_depth.then_some( wgpu::RenderPassDepthStencilAttachment { view: &targets.depth, @@ -1314,6 +1324,7 @@ impl PackPost { let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some(&pass.name), color_attachments: &attachments, + timestamp_writes: self.timing.as_mut().and_then(|t| t.writes(&pass.name)), ..Default::default() }); rp.set_pipeline(&pass.pipeline); diff --git a/crates/horizon-client/src/gfx/pack_timing.rs b/crates/horizon-client/src/gfx/pack_timing.rs new file mode 100644 index 000000000..61ab721e5 --- /dev/null +++ b/crates/horizon-client/src/gfx/pack_timing.rs @@ -0,0 +1,147 @@ +//! Developers' aid: `HORIZON_SHADER_PACK_TIMING=1` times every pass of a shader pack on the +//! GPU, one frame in `EVERY`, and logs each pass's median every `REPORT` timed frames. The +//! frame after a timed one waits for the GPU before reading the timestamps (Metal resolves +//! them without waiting for the passes that write them), so timing slows the game. + +const EVERY: u32 = 30; +const REPORT: usize = 30; +/// Passes timed per frame at most (two timestamps each). +const CAPACITY: u32 = 64; + +pub(super) struct PassTiming { + queries: wgpu::QuerySet, + resolve: wgpu::Buffer, + read: wgpu::Buffer, + period_ns: f32, + frame: u32, + /// The passes timed in the frame being recorded or waiting to be read, in order. + names: Vec, + timed: bool, + samples: Vec<(String, Vec)>, + reports: usize, +} + +impl PassTiming { + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Option { + if std::env::var_os("HORIZON_SHADER_PACK_TIMING").is_none() + || !device.features().contains(wgpu::Features::TIMESTAMP_QUERY) + { + return None; + } + let size = u64::from(CAPACITY) * 2 * 8; + let buffer = |label, usage| { + device.create_buffer(&wgpu::BufferDescriptor { + label: Some(label), + size, + usage, + mapped_at_creation: false, + }) + }; + Some(Self { + queries: device.create_query_set(&wgpu::QuerySetDescriptor { + label: Some("Shader pack pass timing"), + ty: wgpu::QueryType::Timestamp, + count: CAPACITY * 2, + }), + resolve: buffer( + "Shader pack timing resolve", + wgpu::BufferUsages::QUERY_RESOLVE | wgpu::BufferUsages::COPY_SRC, + ), + read: buffer( + "Shader pack timing read", + wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + ), + period_ns: queue.get_timestamp_period(), + frame: 0, + names: Vec::new(), + timed: false, + samples: Vec::new(), + reports: 0, + }) + } + + /// At the start of a frame: reads the last timed frame's passes, and decides whether this + /// one is timed. + pub fn begin(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) { + if self.timed && !self.names.is_empty() { + self.collect(device, queue); + } + self.names.clear(); + self.frame += 1; + self.timed = self.frame.is_multiple_of(EVERY); + } + + /// Timestamp writes for the pass `name`, in a timed frame. + pub fn writes(&mut self, name: &str) -> Option> { + let i = self.names.len() as u32; + if !self.timed || i >= CAPACITY { + return None; + } + self.names.push(name.to_owned()); + Some(wgpu::RenderPassTimestampWrites { + query_set: &self.queries, + beginning_of_pass_write_index: Some(2 * i), + end_of_pass_write_index: Some(2 * i + 1), + }) + } + + fn collect(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) { + // The timed frame first, then the resolve, so every timestamp is written. + _ = device.poll(wgpu::PollType::wait_indefinitely()); + let count = self.names.len() as u32 * 2; + let bytes = u64::from(count) * 8; + let mut encoder = device.create_command_encoder(&Default::default()); + encoder.resolve_query_set(&self.queries, 0..count, &self.resolve, 0); + encoder.copy_buffer_to_buffer(&self.resolve, 0, &self.read, 0, bytes); + queue.submit([encoder.finish()]); + let slice = self.read.slice(..bytes); + slice.map_async(wgpu::MapMode::Read, |_| {}); + _ = device.poll(wgpu::PollType::wait_indefinitely()); + let Ok(data) = slice.get_mapped_range() else { + return; + }; + let stamps: Vec = data + .as_chunks::<8>() + .0 + .iter() + .map(|b| u64::from_le_bytes(*b)) + .collect(); + drop(data); + self.read.unmap(); + // Passes overlap on tiled GPUs (a pass begins before the one before it ends): a pass + // costs the time from the later of its beginning and the previous pass's end. + let mut previous_end = 0; + for (name, pair) in self.names.iter().zip(stamps.as_chunks::<2>().0) { + if pair[0] == 0 || pair[1] < pair[0] { + continue; + } + let from = pair[0].max(previous_end).min(pair[1]); + previous_end = pair[1]; + let ms = (pair[1] - from) as f32 * self.period_ns / 1e6; + match self.samples.iter_mut().find(|s| s.0 == *name) { + Some(s) => s.1.push(ms), + None => self.samples.push((name.clone(), vec![ms])), + } + } + self.reports += 1; + if self.reports.is_multiple_of(REPORT) { + self.report(); + } + } + + fn report(&mut self) { + let mut total = 0.0; + let mut parts = Vec::new(); + for (name, v) in &mut self.samples { + v.sort_by(f32::total_cmp); + let median = v[v.len() / 2]; + total += median; + parts.push(format!("{name} {median:.3}")); + v.clear(); + } + log::info!( + "shader pack GPU time per pass (median ms of {REPORT} frames): {} = {total:.3}", + parts.join(", ") + ); + } +} From 5e348aa479aafa0ab2f03320fb637a5596f00900 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:03:54 +0100 Subject: [PATCH 08/61] Shader packs, clean room 20: buffer mipmaps `const bool MipmapEnabled = true;` in a deferred, composite or final program's fragment stage now regenerates that buffer's mip chain right before the program (6.1.7), on the side programs read, and only when the buffer changed since its last chain this frame. Each level is the average of the 2x2 texels above it; levels attach one at a time and programs sample all of them. Unfilterable buffers (integer, 32-bit float) get none, with a note. BSL's bloom read every tap from the full-size buffer before; with its chain composite4 drops from 0.38 to 0.29 ms at 720p on the Mac mini (HORIZON_SHADER_PACK_TIMING, the chains included), and blurs as intended. Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, section 6.1.7; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx/pack_post.rs | 156 +++++++++++++++++-- crates/horizon-client/src/gfx/pack_post.wgsl | 8 + crates/horizon-shaders/src/post.rs | 18 ++- 3 files changed, 172 insertions(+), 10 deletions(-) diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 739688784..2f152b893 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -137,9 +137,19 @@ struct Pass { reads: u16, /// Buffers written. written: u16, + /// Buffers whose mip chain is regenerated right before it (6.1.7). + mipmaps: u16, groups: HashMap, } +/// One side of a buffer with mipmaps. +struct Mips { + /// Every level, for sampling. + all: wgpu::TextureView, + /// From the second level down: its attachment, and a bind group over the level above. + levels: Vec<(wgpu::TextureView, wgpu::BindGroup)>, +} + struct Targets { size: [u32; 2], /// A and B of each buffer used. @@ -148,6 +158,8 @@ struct Targets { depth_texture: wgpu::Texture, /// `depthtex1`, when a program after translucent terrain reads it. depth1: Option<(wgpu::Texture, wgpu::TextureView)>, + /// Both sides' mip chains of the buffers with mipmaps (`color` is their first level). + mips: [Option<[Mips; 2]>; BUFFERS], } /// What a pack's gbuffers programs read and draw from the game's world. @@ -168,6 +180,12 @@ pub struct PackPost { water: Option, /// A program after translucent terrain reads `depthtex1`: it is copied before water. depth1: bool, + /// Buffers with mipmaps (filterable ones a program asks them for). + mipped: u16, + /// Buffers whose mip chain matches their first level, this frame. + mips_valid: u16, + /// The downsampling pipeline of each format with mipmaps. + down: HashMap, /// `HORIZON_SHADER_PACK_TIMING`: GPU time of each pass, in the log. timing: Option, /// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`. @@ -693,6 +711,7 @@ fn build_pass( flip, reads, written, + mipmaps: p.mipmaps, groups: HashMap::new(), }) } @@ -848,6 +867,39 @@ impl PackPost { &[target(screen)], None, ); + let filterable = |b: usize| { + formats[b].sample_type(None, Some(device.features())) + == Some(wgpu::TextureSampleType::Float { filterable: true }) + }; + let asked = passes.iter().fold(0u16, |m, p| m | p.mipmaps); + let mipped = (0..BUFFERS) + .filter(|&b| asked & 1 << b != 0 && filterable(b)) + .fold(0u16, |m, b| m | 1 << b); + if asked & !mipped != 0 { + notes.push(format!( + "mipmaps of unfilterable buffers are not generated (buffers {:#b})", + asked & !mipped + )); + } + let mut down = HashMap::new(); + for b in (0..BUFFERS).filter(|b| mipped & 1 << b != 0) { + down.entry(formats[b]).or_insert_with(|| { + helper( + device, + "vs_full", + "fs_down", + &[ + texture_entry(0, float), + entry( + 2, + wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + ), + ], + &[target(formats[b])], + None, + ) + }); + } let white = [255u8; 4]; Self { notes, @@ -855,6 +907,9 @@ impl PackPost { terrain, water, depth1, + mipped, + mips_valid: 0, + down, timing: super::pack_timing::PassTiming::new(device, queue), custom, atlas: world.atlas.as_ref().map(|a| a.0.clone()), @@ -960,9 +1015,14 @@ impl PackPost { }]; for s in &pass.samplers { let texture = match s.source { - Source::Color(i) => targets.color[i] - .as_ref() - .map_or(&self.dummy, |t| &t[usize::from(key & 1 << i != 0)]), + Source::Color(i) => { + let side = usize::from(key & 1 << i != 0); + match (&targets.mips[i], &targets.color[i]) { + (Some(m), _) => &m[side].all, + (None, Some(t)) => &t[side], + (None, None) => &self.dummy, + } + } Source::Depth => &targets.depth, // Before translucent terrain the two are the same. Source::Depth1 => match &targets.depth1 { @@ -1075,6 +1135,7 @@ impl PackPost { rp.set_bind_group(0, &group, &[]); draw(&mut rp, translucent); } + self.mips_valid &= !pass.written; if translucent { self.water = Some(pass); } else { @@ -1082,6 +1143,32 @@ impl PackPost { } } + /// Regenerates the mip chain of buffer `b`'s side that programs read. + fn mipmap(&self, encoder: &mut wgpu::CommandEncoder, b: usize) { + let (Some(targets), Some((pipeline, _))) = (&self.targets, self.down.get(&self.formats[b])) + else { + return; + }; + let Some(mips) = &targets.mips[b] else { return }; + let side = &mips[usize::from(self.main & 1 << b != 0)]; + for (attachment, group) in &side.levels { + let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Shader pack mipmap"), + color_attachments: &[attach( + attachment, + wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + )], + ..Default::default() + }); + rp.set_pipeline(pipeline); + rp.set_bind_group(0, group, &[]); + rp.draw(0..3, 0..1); + } + } + /// Whether the pack draws terrain itself. pub fn has_terrain(&self) -> bool { self.terrain.is_some() @@ -1101,7 +1188,8 @@ impl PackPost { if self.targets.as_ref().is_some_and(|t| t.size == size) { return; } - let texture = |label: &str, format, usage| { + let full_chain = 32 - size[0].max(size[1]).max(1).leading_zeros(); + let texture_levels = |label: &str, format, usage, levels| { device.create_texture(&wgpu::TextureDescriptor { label: Some(label), size: wgpu::Extent3d { @@ -1109,7 +1197,7 @@ impl PackPost { height: size[1], depth_or_array_layers: 1, }, - mip_level_count: 1, + mip_level_count: levels, sample_count: 1, dimension: wgpu::TextureDimension::D2, format, @@ -1117,14 +1205,55 @@ impl PackPost { view_formats: &[], }) }; - let make = |label: &str, format| { - texture(label, format, wgpu::TextureUsages::RENDER_ATTACHMENT) - .create_view(&Default::default()) + let texture = |label: &str, format, usage| texture_levels(label, format, usage, 1); + let level = |t: &wgpu::Texture, level: u32| { + t.create_view(&wgpu::TextureViewDescriptor { + base_mip_level: level, + mip_level_count: Some(1), + ..Default::default() + }) }; + let mut mips: [Option<[Mips; 2]>; BUFFERS] = Default::default(); let color = std::array::from_fn(|i| { (self.used & 1 << i != 0).then(|| { let label = format!("colortex{i}"); - [make(&label, self.formats[i]), make(&label, self.formats[i])] + let mipped = self.mipped & 1 << i != 0; + let levels = if mipped { full_chain } else { 1 }; + let attachment = wgpu::TextureUsages::RENDER_ATTACHMENT; + let sides = + [0, 1].map(|_| texture_levels(&label, self.formats[i], attachment, levels)); + if mipped && let Some((_, layout)) = self.down.get(&self.formats[i]) { + mips[i] = Some(sides.each_ref().map(|t| { + Mips { + all: t.create_view(&Default::default()), + levels: (1..levels) + .map(|l| { + let group = + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(&label), + layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView( + &level(t, l - 1), + ), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler( + &self.linear, + ), + }, + ], + }); + (level(t, l), group) + }) + .collect(), + } + })); + } + sides.map(|t| level(&t, 0)) }) }); let depth32 = wgpu::TextureFormat::Depth32Float; @@ -1144,6 +1273,7 @@ impl PackPost { depth: depth_texture.create_view(&Default::default()), depth_texture, depth1, + mips, }); let gbuffers = self.terrain.iter_mut().chain(&mut self.water); for pass in self.passes.iter_mut().chain(gbuffers) { @@ -1277,6 +1407,8 @@ impl PackPost { pass.draw(0..3, 0..1); } + // The world and the clears just rewrote the buffers' first levels. + self.mips_valid = 0; let mut passes = std::mem::take(&mut self.passes); let mut seen = (false, false); let mut shown = false; @@ -1299,6 +1431,11 @@ impl PackPost { seen.1 = true; self.main ^= self.composite_pre; } + let stale = pass.mipmaps & self.mipped & !self.mips_valid; + for b in (0..BUFFERS).filter(|b| stale & 1 << b != 0) { + self.mipmap(encoder, b); + } + self.mips_valid |= stale; let final_pass = pass.group == Group::Final; let group = self.bind(device, queue, pass, if final_pass { window } else { size }); let targets = self.targets.as_ref().expect("resized"); @@ -1332,6 +1469,7 @@ impl PackPost { rp.draw(0..6, 0..1); } self.main ^= pass.flip; + self.mips_valid &= !pass.written; } self.passes = passes; if !terrain_drawn { diff --git a/crates/horizon-client/src/gfx/pack_post.wgsl b/crates/horizon-client/src/gfx/pack_post.wgsl index 75877dddc..299e9207b 100644 --- a/crates/horizon-client/src/gfx/pack_post.wgsl +++ b/crates/horizon-client/src/gfx/pack_post.wgsl @@ -53,3 +53,11 @@ fn vs_uv(@builtin(vertex_index) i: u32) -> Full { fn fs_copy(v: Full) -> @location(0) vec4 { return textureSample(source, linear, v.uv); } + +// One level of a buffer's mip chain from the level above (6.1.7): the average of the 2×2 +// texels under each destination texel, by one bilinear sample at their shared corner. +@fragment +fn fs_down(@builtin(position) p: vec4) -> @location(0) vec4 { + let size = max(textureDimensions(source) / 2u, vec2(1u)); + return textureSampleLevel(source, linear, p.xy / vec2(size), 0.0); +} diff --git a/crates/horizon-shaders/src/post.rs b/crates/horizon-shaders/src/post.rs index e50eb4797..e155cefbe 100644 --- a/crates/horizon-shaders/src/post.rs +++ b/crates/horizon-shaders/src/post.rs @@ -111,6 +111,9 @@ pub struct PostProgram { pub flips: Vec<(u32, bool)>, /// How a gbuffers program's output blends into its buffers; `None` writes them. pub blend: Option, + /// Bit n: `colortexMipmapEnabled` in its fragment stage, so buffer n's mip chain is + /// regenerated right before it (6.1.7; not for gbuffers programs). + pub mipmaps: u16, } pub struct Post { @@ -280,6 +283,16 @@ impl Loader<'_> { self.post.config.read(&vertex); self.post.config.read(&fragment); let buffers = config::draw_buffers(&fragment).unwrap_or_else(|| (0..8).collect()); + let mipmaps = match group { + Group::Gbuffers | Group::Translucent => 0, + _ => { + let mut own = Config::default(); + own.read(&fragment); + (own.buffers.iter().enumerate()) + .filter(|b| b.1.mipmap) + .fold(0, |m, (i, _)| m | 1 << i) + } + }; Some(PostProgram { flips: flips(self.properties, &name), name, @@ -287,6 +300,7 @@ impl Loader<'_> { program, buffers, blend: None, + mipmaps, }) } } @@ -391,7 +405,8 @@ mod tests { ("shaders/deferred.vsh", VSH), ( "shaders/deferred.fsh", - "#version 120\nvoid main() { gl_FragColor = vec4(1.0); }\n", + "#version 120\nconst bool colortex5MipmapEnabled = true;\n\ + void main() { gl_FragColor = vec4(1.0); }\n", ), ("shaders/composite1.fsh", FSH), ("shaders/composite3.vsh", VSH), @@ -423,6 +438,7 @@ mod tests { assert_eq!(names, ["deferred", "composite", "composite2", "final"]); let p = &post.programs; assert_eq!(p[0].buffers, (0..8).collect::>()); + assert_eq!((p[0].mipmaps, p[1].mipmaps), (1 << 5, 0)); assert_eq!(p[1].buffers, [3]); assert_eq!(p[1].flips, [(3, false)]); assert_eq!(p[2].flips, [(4, true)]); From b9b47f12dabd79ba8c48998d9aa98974e26a3eb4 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:29:21 +0100 Subject: [PATCH 09/61] Shader packs, clean room 21: per-draw uniforms (renderStage) A gbuffers program's renderStage, entityId, blockEntityId and entityColor (spec 5.3, updated per draw) move to a block of their own at set 2 that the game picks per draw with a dynamic offset: one slot per render stage. Terrain draws as TERRAIN_SOLID, translucent terrain as TERRAIN_TRANSLUCENT. The terrain test pack paints any other stage red. Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, 5.3, 5.4; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx.rs | 1 + crates/horizon-client/src/gfx/pack_draw.rs | 67 +++++++++++++++++++ crates/horizon-client/src/gfx/pack_post.rs | 25 ++++++- crates/horizon-shaders/src/lib.rs | 6 +- crates/horizon-shaders/src/macros.rs | 8 +++ crates/horizon-shaders/src/translate/mod.rs | 47 ++++++++++++- .../terrain-test/shaders/gbuffers_terrain.fsh | 3 + 7 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 crates/horizon-client/src/gfx/pack_draw.rs diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index f0987dddd..354dc8d07 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -69,6 +69,7 @@ pub mod lightning; mod lod; mod occlusion; mod pack_atlas; +mod pack_draw; mod pack_post; mod pack_timing; pub mod pack_uniforms; diff --git a/crates/horizon-client/src/gfx/pack_draw.rs b/crates/horizon-client/src/gfx/pack_draw.rs new file mode 100644 index 000000000..91c77a6bf --- /dev/null +++ b/crates/horizon-client/src/gfx/pack_draw.rs @@ -0,0 +1,67 @@ +//! The uniforms a shader pack's gbuffers programs read per draw (`HzDraw` at set 2, the +//! translator's `DRAW_UNIFORMS`): one slot per render stage of a uniform buffer, picked with a +//! dynamic offset at each draw. + +use horizon_shaders::{RENDER_STAGES, render_stage}; + +/// `HzDraw`'s size: `renderStage`, `entityId`, `blockEntityId`, a pad, `entityColor`. +const SIZE: u32 = 32; + +pub(super) struct DrawUniforms { + pub layout: wgpu::BindGroupLayout, + group: wgpu::BindGroup, + stride: u32, +} + +impl DrawUniforms { + pub fn new(device: &wgpu::Device) -> Self { + let stride = SIZE.max(device.limits().min_uniform_buffer_offset_alignment); + let mut contents = vec![0u8; stride as usize * RENDER_STAGES.len()]; + for (i, slot) in contents.chunks_mut(stride as usize).enumerate() { + slot[..4].copy_from_slice(&(i as i32).to_le_bytes()); + } + let buffer = wgpu::util::DeviceExt::create_buffer_init( + device, + &wgpu::util::BufferInitDescriptor { + label: Some("Shader pack draws"), + contents: &contents, + usage: wgpu::BufferUsages::UNIFORM, + }, + ); + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Shader pack draws"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: true, + min_binding_size: wgpu::BufferSize::new(u64::from(SIZE)), + }, + count: None, + }], + }); + let group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Shader pack draws"), + layout: &layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { + buffer: &buffer, + offset: 0, + size: wgpu::BufferSize::new(u64::from(SIZE)), + }), + }], + }); + Self { + layout, + group, + stride, + } + } + + /// Draws that follow read `renderStage` = `MC_RENDER_STAGE_`, other values zero. + pub fn bind(&self, pass: &mut wgpu::RenderPass<'_>, stage: &str) { + pass.set_bind_group(2, &self.group, &[self.stride * render_stage(stage)]); + } +} diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 2f152b893..08a2dedcc 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use horizon_shaders::{Format, Group, Kind, Post, Val}; +use super::pack_draw::DrawUniforms; use super::pack_uniforms::{self, FrameUniforms, PackInputs}; const BUFFERS: usize = horizon_shaders::BUFFERS; @@ -139,6 +140,8 @@ struct Pass { written: u16, /// Buffers whose mip chain is regenerated right before it (6.1.7). mipmaps: u16, + /// Reads the per-draw block (`DrawUniforms`). + draw: bool, groups: HashMap, } @@ -186,6 +189,8 @@ pub struct PackPost { mips_valid: u16, /// The downsampling pipeline of each format with mipmaps. down: HashMap, + /// The per-draw uniforms of gbuffers programs. + draws: DrawUniforms, /// `HORIZON_SHADER_PACK_TIMING`: GPU time of each pass, in the log. timing: Option, /// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`. @@ -488,6 +493,7 @@ fn build_pass( notes: &mut Vec, geometry: Option<&Geometry<'_>>, custom: &[String], + draws: &wgpu::BindGroupLayout, ) -> Option { let features = device.features(); // The frame's uniforms, then the pack's custom ones. @@ -499,6 +505,7 @@ fn build_pass( }; let mask = |flips: &[(u32, bool)]| flips.iter().filter(|f| f.1).fold(0u16, |m, f| m | 1 << f.0); let interface = &p.program.interface; + let draw = interface.draw; let mut entries = vec![entry( 0, wgpu::BindingType::Buffer { @@ -632,7 +639,11 @@ fn build_pass( }); let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some(&p.name), - bind_group_layouts: &[Some(&layout), geometry.and_then(|g| g.table)], + bind_group_layouts: &[ + Some(&layout), + geometry.and_then(|g| g.table), + draw.then_some(draws), + ], ..Default::default() }); let module = |mut m: naga::Module| { @@ -712,6 +723,7 @@ fn build_pass( reads, written, mipmaps: p.mipmaps, + draw, groups: HashMap::new(), }) } @@ -748,6 +760,7 @@ impl PackPost { let mask = |flips: &[(u32, bool)]| flips.iter().filter(|f| f.1).fold(0u16, |m, f| m | 1 << f.0); let mut used = 1u16; + let draws = DrawUniforms::new(device); let mut build = |p: horizon_shaders::PostProgram, geometry: Option<&Geometry<'_>>| { let pass = build_pass( device, @@ -757,6 +770,7 @@ impl PackPost { &mut notes, geometry, &custom_names, + &draws.layout, )?; used |= pass.reads | pass.written | pass.flip; Some(pass) @@ -910,6 +924,7 @@ impl PackPost { mipped, mips_valid: 0, down, + draws, timing: super::pack_timing::PassTiming::new(device, queue), custom, atlas: world.atlas.as_ref().map(|a| a.0.clone()), @@ -1133,6 +1148,14 @@ impl PackPost { }); rp.set_pipeline(&pass.pipeline); rp.set_bind_group(0, &group, &[]); + if pass.draw { + let stage = if translucent { + "TERRAIN_TRANSLUCENT" + } else { + "TERRAIN_SOLID" + }; + self.draws.bind(&mut rp, stage); + } draw(&mut rp, translucent); } self.mips_valid &= !pass.written; diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index f5819d435..bed86758e 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -30,7 +30,7 @@ pub use config::{BUFFERS, Buffer, Config, Format, Kind, buffer_index, draw_buffe pub use custom::{Custom, Val}; pub use discover::{Found, discover}; pub use lang::Lang; -pub use macros::{MC_VERSION, Macros, RENDER_STAGES}; +pub use macros::{MC_VERSION, Macros, RENDER_STAGES, render_stage}; pub use options::switches_on; pub use pack::{Pack, PackError}; pub use post::{ @@ -39,6 +39,6 @@ pub use post::{ pub use preprocess::{Diagnostic, Origin, Preprocessed, preprocess}; pub use properties::{Entry, Note, Properties}; pub use translate::{ - Error, GbuffersInputs, Interface, Program, Sampler, Target, Uniform, translate_composite, - translate_gbuffers, + DRAW_UNIFORMS, Error, GbuffersInputs, Interface, Program, Sampler, Target, Uniform, + translate_composite, translate_gbuffers, }; diff --git a/crates/horizon-shaders/src/macros.rs b/crates/horizon-shaders/src/macros.rs index d8a184f41..6c7fb2678 100644 --- a/crates/horizon-shaders/src/macros.rs +++ b/crates/horizon-shaders/src/macros.rs @@ -60,6 +60,14 @@ pub const RENDER_STAGES: [&str; 24] = [ "HAND_TRANSLUCENT", ]; +/// The value of `MC_RENDER_STAGE_` (OQ-7: its index in `RENDER_STAGES`). +pub fn render_stage(name: &str) -> u32 { + RENDER_STAGES + .iter() + .position(|s| *s == name) + .unwrap_or_else(|| panic!("no render stage {name}")) as u32 +} + impl Macros { /// The standard macros that need no renderer (spec 1.5.1): `MC_VERSION`, this system's /// `MC_OS_*`, the GL and GLSL versions Horizon's translation accepts, the render stages and diff --git a/crates/horizon-shaders/src/translate/mod.rs b/crates/horizon-shaders/src/translate/mod.rs index 83dcb3af9..fe736e4d8 100644 --- a/crates/horizon-shaders/src/translate/mod.rs +++ b/crates/horizon-shaders/src/translate/mod.rs @@ -45,8 +45,19 @@ pub struct Interface { pub samplers: Vec, /// The fragment stage's output locations. pub outputs: Vec, + /// A gbuffers program reads `DRAW_UNIFORMS`: the block at set 2, binding 0, set per draw. + pub draw: bool, } +/// The uniforms a gbuffers program reads per draw (5.3's "D"), in a block of their own +/// (`HzDraw`, std140: three ints, a pad, a vec4; 32 bytes) the game sets for each draw. +pub const DRAW_UNIFORMS: [(&str, &str); 4] = [ + ("renderStage", "int"), + ("entityId", "int"), + ("blockEntityId", "int"), + ("entityColor", "vec4"), +]; + #[derive(Clone, Debug, PartialEq)] pub struct Uniform { pub name: String, @@ -170,6 +181,17 @@ fn translate( samplers.insert(name.clone(), ty.clone()); } } + // Per-draw values move to their own block; another type than the documented one keeps + // the frame's block (and reads zero). + let mut draw = false; + if gbuffers.is_some() { + for (name, ty) in DRAW_UNIFORMS { + if uniforms.get(name).is_some_and(|t| t == ty) { + uniforms.remove(name); + draw = true; + } + } + } let samplers: Vec = samplers .into_iter() .enumerate() @@ -203,6 +225,12 @@ fn translate( .unwrap(); } h.push_str(" };\n"); + if draw { + h.push_str( + "layout(set = 2, binding = 0) uniform HzDraw { int renderStage; int entityId; \ + int blockEntityId; int hz_draw_pad; vec4 entityColor; };\n", + ); + } for s in &samplers { let (texture, sampler) = split_sampler(&s.ty); writeln!( @@ -327,6 +355,7 @@ fn translate( uniforms, samplers, outputs, + draw, }, }) } @@ -394,7 +423,11 @@ fn compile(stage: &Stage, text: &str, header_lines: usize) -> Result (u32, Vec<(String, u32)>) { for (_, var) in module.global_variables.iter() { - if var.space != naga::AddressSpace::Uniform { + let frame = var + .binding + .as_ref() + .is_some_and(|b| b.group == 0 && b.binding == 0); + if var.space != naga::AddressSpace::Uniform || !frame { continue; } if let naga::TypeInner::Struct { members, span } = &module.types[var.ty].inner { @@ -546,9 +579,9 @@ mod tests { light = (gl_TextureMatrix[1] * gl_MultiTexCoord1).xy;\n color = gl_Color;\n \ normal = normalize(gl_NormalMatrix * gl_Normal);\n id = mc_Entity.x + mc_midTexCoord.x;\n}\n"; let fsh = "#version 120\nuniform sampler2D texture;\nuniform sampler2D lightmap;\nuniform mat4 gbufferModelView;\n\ - varying vec2 uv;\nvarying vec2 light;\nvarying vec4 color;\nvarying vec3 normal;\nvarying float id;\n\ + uniform int renderStage;\nvarying vec2 uv;\nvarying vec2 light;\nvarying vec4 color;\nvarying vec3 normal;\nvarying float id;\n\ void main() {\n vec4 c = texture2D(texture, uv) * texture2D(lightmap, light) * color;\n \ - /* DRAWBUFFERS:02 */\n gl_FragData[0] = c * gl_FragCoord.z;\n \ + /* DRAWBUFFERS:02 */\n gl_FragData[0] = c * gl_FragCoord.z * float(renderStage);\n \ gl_FragData[1] = vec4(normal * 0.5 + 0.5, id);\n}\n"; let files: Vec<(String, &str)> = [("gbuffers_terrain.vsh", vsh), ("gbuffers_terrain.fsh", fsh)] @@ -586,6 +619,14 @@ mod tests { assert!(names.contains(&name), "{name} missing from {names:?}"); } assert_eq!(p.interface.outputs, vec![0, 1]); + // renderStage is set per draw, in its own block. + assert!(p.interface.draw && !names.contains(&"renderStage")); + let per_draw = p.fragment.global_variables.iter().any(|(_, v)| { + v.binding + .as_ref() + .is_some_and(|b| b.group == 2 && b.binding == 0) + }); + assert!(per_draw); let inputs: Vec<&str> = p.vertex.entry_points[0] .function .arguments diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh index 902cf3525..8be409929 100644 --- a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh @@ -3,6 +3,7 @@ uniform sampler2D texture; uniform sampler2D lightmap; uniform ivec2 atlasSize; +uniform int renderStage; varying vec2 texcoord; varying vec2 lmcoord; @@ -17,6 +18,8 @@ void main() { color.rgb = mix(color.rgb * vec3(0.6, 0.7, 1.2), color.rgb, lmcoord.y); // A dot at the face's centre, one texel wide. if (length((texcoord - mid) * vec2(atlasSize)) < 1.0) color.rgb = vec3(1.0, 0.0, 1.0); + // Any other stage than solid terrain (translucent terrain falls back here): red. + if (renderStage != MC_RENDER_STAGE_TERRAIN_SOLID) color.rgb = vec3(1.0, 0.0, 0.0); /* DRAWBUFFERS:0 */ gl_FragData[0] = color; } From 7d6583dc4d088d794398391aad4c0ced94009883 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:45:24 +0100 Subject: [PATCH 10/61] Shader packs, clean room 22: the sky (gbuffers_skybasic) The pack's gbuffers_skybasic (or gbuffers_basic) draws vanilla's sky geometry, as Horizon's own sky builds it: the disc at y 16, the sunset fan while the sun is low, the stars and the void disc, each with its render stage. One static vertex buffer; the frame's colours and rotations sit in a small uniform the vertex inputs read. The sky draws after the world copy and before terrain, at the far plane: only where nothing else drew, without writing depth. It blends with the translucent default unless blend. says otherwise. Horizon's world pass then skips its own sky disc, stars and void. The sun and moon stay Horizon's for now (gbuffers_skytextured is next): the pack's sky covers them. load_post takes the game's inputs per geometry (GameInputs); GbuffersInputs gains statements run after the pack's main. Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, 2.1, 2.3, 5.4; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- .../horizon-client/src/game/shader_packs.rs | 4 +- crates/horizon-client/src/gfx.rs | 46 +++- crates/horizon-client/src/gfx/pack_atlas.rs | 1 + crates/horizon-client/src/gfx/pack_post.rs | 157 +++++++++--- crates/horizon-client/src/gfx/pack_sky.rs | 229 ++++++++++++++++++ crates/horizon-shaders/src/lib.rs | 3 +- crates/horizon-shaders/src/post.rs | 88 +++++-- crates/horizon-shaders/src/translate/mod.rs | 6 +- .../tests/packs/terrain-test/README.md | 4 +- .../shaders/gbuffers_skybasic.fsh | 15 ++ .../shaders/gbuffers_skybasic.vsh | 8 + crates/horizon-shaders/tests/real_packs.rs | 16 +- 12 files changed, 502 insertions(+), 75 deletions(-) create mode 100644 crates/horizon-client/src/gfx/pack_sky.rs create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_skybasic.fsh create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_skybasic.vsh diff --git a/crates/horizon-client/src/game/shader_packs.rs b/crates/horizon-client/src/game/shader_packs.rs index d819eddef..360262640 100644 --- a/crates/horizon-client/src/game/shader_packs.rs +++ b/crates/horizon-client/src/game/shader_packs.rs @@ -42,8 +42,8 @@ impl Game<'_> { ); } let folder = dimension_folder(&pack, dimension); - let terrain = renderer.pack_terrain_inputs(); - let post = load_post(&pack, &folder, &properties, ¯os, terrain.as_ref()); + let inputs = renderer.pack_inputs(); + let post = load_post(&pack, &folder, &properties, ¯os, &inputs); if let Some((programs, notes)) = renderer.set_shader_pack(dimension, Some(post)) { for note in notes { diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 354dc8d07..7444b136a 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -71,6 +71,7 @@ mod occlusion; mod pack_atlas; mod pack_draw; mod pack_post; +mod pack_sky; mod pack_timing; pub mod pack_uniforms; mod page_runs; @@ -88,6 +89,19 @@ const COMPLETE_COLUMN: u32 = (1 << (crate::chunk::HEIGHT / 16)) - 1; /// its fixed-point UVs (`terrain.wgsl`'s `terrain_uv`). /// The world pass again after occlusion culling's test: it loads both /// targets (phase 2 draws over phase 1, reading the test's results). +/// LevelRenderer: no sky inside lava or powder snow (sky fog ends within 2 blocks). +fn sky_open(frame: &Frame) -> bool { + match frame.fog { + Some(f) => f.sky_end > 2.0, + None => !frame.underwater, + } +} + +/// The Overworld-style sky (disc, sunset, stars, void) draws this frame. +fn sky_visible(frame: &Frame) -> bool { + !frame.title && frame.skybox && sky_open(frame) +} + fn world_pass_after_culling<'e>( encoder: &'e mut wgpu::CommandEncoder, view: &wgpu::TextureView, @@ -2777,6 +2791,7 @@ impl Renderer { terrain: self.pack_atlas.as_ref().map(|_| pack_post::Geometry { buffers: if flat { &buffers[..1] } else { &buffers }, table: self.flat.as_ref().filter(|_| flat).map(|f| &f.draw_layout), + depth_write: true, }), }; pack_post::PackPost::new( @@ -2794,9 +2809,16 @@ impl Renderer { self.pack_post = Some((dimension, post)); report } - /// What a shader pack's terrain program reads from Horizon's terrain vertices, in the - /// pack's block atlas (made here when missing); `None` when the atlas does not fit. - pub fn pack_terrain_inputs(&mut self) -> Option { + /// What a shader pack's gbuffers programs read from Horizon's geometry: the sky's, and + /// terrain's in the pack's block atlas (made here when missing; none when it does not fit). + pub fn pack_inputs(&mut self) -> horizon_shaders::GameInputs { + horizon_shaders::GameInputs { + terrain: self.pack_terrain_inputs(), + sky: Some(pack_sky::inputs()), + } + } + + fn pack_terrain_inputs(&mut self) -> Option { if self.pack_atlas.is_none() { let mut encoder = self.device.create_command_encoder(&Default::default()); let layers = self @@ -3711,6 +3733,12 @@ impl Renderer { let pack_post = !frame.title && self.pack_post.as_ref().is_some_and(|p| p.1.is_some()); // The pack's program reads the vertices this frame's draws bind. let pack_draws_terrain = pack_terrain && flat_frame == self.pack_terrain_flat; + let pack_draws_sky = pack_post + && self + .pack_post + .as_ref() + .and_then(|p| p.1.as_ref()) + .is_some_and(|p| p.has_sky()); let pack_draws_water = pack_draws_terrain && self .pack_post @@ -3770,13 +3798,9 @@ impl Renderer { ..Default::default() }); pass.set_bind_group(0, &self.group, &[]); - // LevelRenderer: no sky inside lava or powder snow (sky fog ends within 2 blocks). - let sky_visible = match frame.fog { - Some(f) => f.sky_end > 2.0, - None => !frame.underwater, - }; + let sky_visible = sky_open(frame); if frame.title || (sky_visible && (frame.skybox || frame.end_sky)) { - if !frame.title && frame.skybox { + if !frame.title && frame.skybox && !pack_draws_sky { pass.set_pipeline(&self.sky); pass.draw(0..3, 0..1); } @@ -3796,11 +3820,11 @@ impl Renderer { } pass.set_bind_group(0, &self.group, &[]); } - if !frame.title && frame.stars.is_some() { + if !frame.title && frame.stars.is_some() && !pack_draws_sky { self.stars.draw(&mut pass); pass.set_bind_group(0, &self.group, &[]); } - if !frame.title && frame.dark_disc { + if !frame.title && frame.dark_disc && !pack_draws_sky { pass.set_pipeline(&self.dark_disc); pass.draw(0..3, 0..1); } diff --git a/crates/horizon-client/src/gfx/pack_atlas.rs b/crates/horizon-client/src/gfx/pack_atlas.rs index 011dff379..1a14a8da9 100644 --- a/crates/horizon-client/src/gfx/pack_atlas.rs +++ b/crates/horizon-client/src/gfx/pack_atlas.rs @@ -173,6 +173,7 @@ impl PackAtlas { .iter() .map(|&(n, v)| (n.to_owned(), v.to_owned())) .collect(), + after: String::new(), } } } diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 08a2dedcc..2334b84c9 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use horizon_shaders::{Format, Group, Kind, Post, Val}; use super::pack_draw::DrawUniforms; +use super::pack_sky::PackSky; use super::pack_uniforms::{self, FrameUniforms, PackInputs}; const BUFFERS: usize = horizon_shaders::BUFFERS; @@ -120,6 +121,14 @@ fn share_samplers(module: &mut naga::Module, slot: &HashMap) { } } +/// The game's geometry a gbuffers pass draws. +#[derive(Clone, Copy, PartialEq)] +enum Gbuffers { + Sky, + Terrain, + Water, +} + struct Pass { name: String, group: Group, @@ -181,6 +190,8 @@ pub struct PackPost { terrain: Option, /// What draws translucent terrain (`gbuffers_water` or its fallback), after `deferred`. water: Option, + /// What draws the sky (`gbuffers_skybasic` or its fallback), before terrain, and its vertices. + sky: Option<(Pass, PackSky)>, /// A program after translucent terrain reads `depthtex1`: it is copied before water. depth1: bool, /// Buffers with mipmaps (filterable ones a program asks them for). @@ -481,6 +492,8 @@ fn attach( pub(super) struct Geometry<'a> { pub buffers: &'a [Option>], pub table: Option<&'a wgpu::BindGroupLayout>, + /// Writes depth (the sky does not). + pub depth_write: bool, } /// The pass of program `p`: a quad, or `geometry` drawn by the game. `None` (with a note) when @@ -664,9 +677,9 @@ fn build_pass( buffers: geometry.map_or(&[], |g| g.buffers), }, primitive: Default::default(), - depth_stencil: geometry.map(|_| wgpu::DepthStencilState { + depth_stencil: geometry.map(|g| wgpu::DepthStencilState { format: wgpu::TextureFormat::Depth32Float, - depth_write_enabled: Some(true), + depth_write_enabled: Some(g.depth_write), depth_compare: Some(wgpu::CompareFunction::LessEqual), stencil: Default::default(), bias: Default::default(), @@ -783,6 +796,15 @@ impl PackPost { .water .zip(world.terrain.as_ref()) .and_then(|(p, g)| build(p, Some(g))); + let sky = post.sky.and_then(|p| { + let sky = PackSky::new(device); + let geometry = Geometry { + buffers: &PackSky::buffers(), + table: Some(&sky.layout), + depth_write: false, + }; + Some((build(p, Some(&geometry))?, sky)) + }); // Developers' aid: `HORIZON_SHADER_PACK_SKIP=composite,final` leaves programs out. let skip = std::env::var("HORIZON_SHADER_PACK_SKIP").unwrap_or_default(); let passes: Vec = post @@ -797,6 +819,9 @@ impl PackPost { if let Some(w) = &water { notes.push(format!("translucent terrain draws with {}", w.name)); } + if let Some((s, _)) = &sky { + notes.push(format!("the sky draws with {}", s.name)); + } let depth1 = water.is_some() && passes .iter() @@ -810,8 +835,10 @@ impl PackPost { let bit = 1u16 << b; let touched = |p: &&Pass| (p.reads | p.flip | p.written) & bit != 0; // Gbuffers passes cover part of their targets: a clear is needed if they touch it. - let first = terrain + let first = sky .iter() + .map(|s| &s.0) + .chain(&terrain) .chain(&water) .find(touched) .map(|_| true) @@ -826,6 +853,7 @@ impl PackPost { // Terrain tests against the world's depth (entities...), so the blit copies it then. let blit_depth = terrain.is_some() || water.is_some() + || sky.is_some() || passes.iter().any(|p| { p.samplers .iter() @@ -920,6 +948,7 @@ impl PackPost { passes, terrain, water, + sky, depth1, mipped, mips_valid: 0, @@ -1084,25 +1113,34 @@ impl PackPost { group.clone() } - /// Terrain (or, `translucent`, translucent terrain) with the pack's program into the - /// buffers it writes, in place, tested against the world's depth; `draw` issues the - /// game's draws. - fn draw_terrain( + /// The sky, terrain or translucent terrain with the pack's program into the buffers it + /// writes, in place, tested against the world's depth; `draw` issues the game's terrain + /// draws (`true`: translucent), the sky draws itself. + #[allow(clippy::too_many_arguments)] + fn draw_gbuffers( &mut self, device: &wgpu::Device, queue: &wgpu::Queue, encoder: &mut wgpu::CommandEncoder, size: [u32; 2], - translucent: bool, + frame: &super::Frame, + which: Gbuffers, draw: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), ) { - let slot = if translucent { - &mut self.water - } else { - &mut self.terrain - }; - let Some(mut pass) = slot.take() else { - return; + let translucent = which == Gbuffers::Water; + let (mut pass, sky) = match which { + Gbuffers::Sky => match self.sky.take() { + Some((pass, sky)) => (pass, Some(sky)), + None => return, + }, + Gbuffers::Terrain => match self.terrain.take() { + Some(pass) => (pass, None), + None => return, + }, + Gbuffers::Water => match self.water.take() { + Some(pass) => (pass, None), + None => return, + }, }; if translucent && let Some(t) = &self.targets @@ -1148,24 +1186,58 @@ impl PackPost { }); rp.set_pipeline(&pass.pipeline); rp.set_bind_group(0, &group, &[]); - if pass.draw { - let stage = if translucent { - "TERRAIN_TRANSLUCENT" - } else { - "TERRAIN_SOLID" - }; - self.draws.bind(&mut rp, stage); + let draws = pass.draw.then_some(&self.draws); + match &sky { + Some(sky) => sky.draw(queue, &mut rp, frame, &|rp, stage| { + if let Some(d) = draws { + d.bind(rp, stage); + } + }), + None => { + if let Some(d) = draws { + let stage = if translucent { + "TERRAIN_TRANSLUCENT" + } else { + "TERRAIN_SOLID" + }; + d.bind(&mut rp, stage); + } + draw(&mut rp, translucent); + } } - draw(&mut rp, translucent); } self.mips_valid &= !pass.written; - if translucent { - self.water = Some(pass); - } else { - self.terrain = Some(pass); + match (which, sky) { + (Gbuffers::Sky, Some(sky)) => self.sky = Some((pass, sky)), + (Gbuffers::Water, _) => self.water = Some(pass), + _ => self.terrain = Some(pass), } } + /// The sky, when it shows, then terrain. + fn draw_opaque( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + encoder: &mut wgpu::CommandEncoder, + size: [u32; 2], + frame: &super::Frame, + terrain: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), + ) { + if super::sky_visible(frame) { + self.draw_gbuffers(device, queue, encoder, size, frame, Gbuffers::Sky, terrain); + } + self.draw_gbuffers( + device, + queue, + encoder, + size, + frame, + Gbuffers::Terrain, + terrain, + ); + } + /// Regenerates the mip chain of buffer `b`'s side that programs read. fn mipmap(&self, encoder: &mut wgpu::CommandEncoder, b: usize) { let (Some(targets), Some((pipeline, _))) = (&self.targets, self.down.get(&self.formats[b])) @@ -1197,6 +1269,11 @@ impl PackPost { self.terrain.is_some() } + /// Whether the pack draws the sky itself. + pub fn has_sky(&self) -> bool { + self.sky.is_some() + } + /// Whether the pack draws translucent terrain itself. pub fn has_water(&self) -> bool { self.water.is_some() @@ -1435,16 +1512,24 @@ impl PackPost { let mut passes = std::mem::take(&mut self.passes); let mut seen = (false, false); let mut shown = false; - let mut terrain_drawn = self.terrain.is_none(); + let mut terrain_drawn = self.terrain.is_none() && self.sky.is_none(); let mut water_drawn = self.water.is_none(); for pass in &mut passes { if !terrain_drawn && pass.group != Group::Prepare { terrain_drawn = true; - self.draw_terrain(device, queue, encoder, size, false, terrain); + self.draw_opaque(device, queue, encoder, size, frame, terrain); } if !water_drawn && matches!(pass.group, Group::Composite | Group::Final) { water_drawn = true; - self.draw_terrain(device, queue, encoder, size, true, terrain); + self.draw_gbuffers( + device, + queue, + encoder, + size, + frame, + Gbuffers::Water, + terrain, + ); } if pass.group == Group::Deferred && !seen.0 { seen.0 = true; @@ -1496,10 +1581,18 @@ impl PackPost { } self.passes = passes; if !terrain_drawn { - self.draw_terrain(device, queue, encoder, size, false, terrain); + self.draw_opaque(device, queue, encoder, size, frame, terrain); } if !water_drawn { - self.draw_terrain(device, queue, encoder, size, true, terrain); + self.draw_gbuffers( + device, + queue, + encoder, + size, + frame, + Gbuffers::Water, + terrain, + ); } let targets = self.targets.as_ref().expect("resized"); let view = |m: u16, i: usize, alt: bool| { diff --git a/crates/horizon-client/src/gfx/pack_sky.rs b/crates/horizon-client/src/gfx/pack_sky.rs new file mode 100644 index 000000000..cd99d03b9 --- /dev/null +++ b/crates/horizon-client/src/gfx/pack_sky.rs @@ -0,0 +1,229 @@ +//! The vanilla sky's geometry for a shader pack's `gbuffers_skybasic` (spec 2.3): the sky disc, +//! the sunset fan, the stars and the void disc, as Horizon's own sky draws them (terrain.wgsl, +//! stars.rs), built once into a vertex buffer. The frame's colours and rotations sit in a small +//! uniform (set 1) that the vertex inputs read. + +use std::ops::Range; + +use wgpu::util::DeviceExt; + +/// A sky vertex: a position and what it belongs to (`SKY`...). +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct Vertex { + pos: [f32; 3], + kind: u32, +} + +const SKY: u32 = 0; +const SUNSET_APEX: u32 = 1; +/// Rim of the sunset fan: `pos.xy` is (sin, cos) of its angle. +const SUNSET_RIM: u32 = 2; +const STAR: u32 = 3; +const VOID: u32 = 4; + +/// The frame's sky (std140). +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct Uniform { + /// Camera-relative world from star space. + stars: [[f32; 4]; 4], + /// Sky colour, alpha 1. + color: [f32; 4], + /// Sunset colour and alpha. + sunrise: [f32; 4], + /// cos and sin of the sunset fan's turn, star brightness. + sun: [f32; 4], +} + +pub(super) const ATTRIBUTES: [wgpu::VertexAttribute; 2] = + wgpu::vertex_attr_array![0 => Float32x3, 1 => Uint32]; + +/// What `gbuffers_skybasic` reads from these vertices. +pub fn inputs() -> horizon_shaders::GbuffersInputs { + let declarations = + "layout(location = 0) in vec3 hz_v_pos;\nlayout(location = 1) in uint hz_v_kind;\n\ + layout(set = 1, binding = 0) uniform HzSky { mat4 hz_sky_stars; vec4 hz_sky_color; \ + vec4 hz_sky_sunrise; vec4 hz_sky_sun; };\n" + .to_owned(); + let body = format!( + " vec3 hz_p = hz_v_pos;\n \ + hz_Color = hz_sky_color;\n \ + if (hz_v_kind == {SUNSET_APEX}u || hz_v_kind == {SUNSET_RIM}u) {{\n \ + vec3 hz_l = vec3(0.0, 100.0, 0.0);\n \ + hz_Color = hz_sky_sunrise;\n \ + if (hz_v_kind == {SUNSET_RIM}u) {{\n \ + hz_l = vec3(hz_v_pos.xy * 120.0, -hz_v_pos.y * 40.0 * hz_sky_sunrise.a);\n \ + hz_Color.a = 0.0;\n \ + }}\n \ + vec3 hz_r = vec3(hz_l.x * hz_sky_sun.x - hz_l.y * hz_sky_sun.y, hz_l.x * hz_sky_sun.y + hz_l.y * hz_sky_sun.x, hz_l.z);\n \ + hz_p = vec3(hz_r.x, -hz_r.z, hz_r.y);\n \ + }} else if (hz_v_kind == {STAR}u) {{\n \ + hz_p = (hz_sky_stars * vec4(hz_v_pos, 1.0)).xyz;\n \ + hz_Color = vec4(hz_sky_sun.z);\n \ + }} else if (hz_v_kind == {VOID}u) {{\n \ + hz_Color = vec4(0.0, 0.0, 0.0, 1.0);\n \ + }}\n \ + hz_Vertex = vec4(hz_p, 1.0);\n \ + hz_MultiTexCoord0 = vec4(0.0, 0.0, 0.0, 1.0);\n \ + hz_MultiTexCoord1 = vec4(0.0, 240.0, 0.0, 1.0);\n \ + hz_MultiTexCoord2 = hz_MultiTexCoord1;\n \ + hz_Normal = vec3(0.0, 1.0, 0.0);\n" + ); + horizon_shaders::GbuffersInputs { + declarations, + body, + attributes: vec![ + ("vaPosition".into(), "hz_Vertex.xyz".into()), + ("vaColor".into(), "hz_Color".into()), + ("vaNormal".into(), "hz_Normal".into()), + ], + // At the far plane: the sky shows only where nothing else drew, and it writes no depth. + after: " gl_Position.z = gl_Position.w;\n".into(), + } +} + +/// A disc fan at height `y`, radius 512, as triangles (SkyRenderer's sky and void discs). +fn disc(y: f32, kind: u32, out: &mut Vec) { + let sign = y.signum(); + let rim = |i: i32| { + let a = (i as f32).to_radians(); + Vertex { + pos: [sign * 512.0 * a.cos(), y, 512.0 * a.sin()], + kind, + } + }; + let centre = Vertex { + pos: [0.0, y, 0.0], + kind, + }; + for i in (-180..180).step_by(45) { + out.extend([centre, rim(i), rim(i + 45)]); + } +} + +pub(super) struct PackSky { + vertices: wgpu::Buffer, + uniform: wgpu::Buffer, + pub layout: wgpu::BindGroupLayout, + group: wgpu::BindGroup, + /// Vertex ranges of the sky disc, the sunset fan, the stars and the void. + ranges: [Range; 4], +} + +impl PackSky { + pub fn new(device: &wgpu::Device) -> Self { + let mut v = Vec::new(); + disc(16.0, SKY, &mut v); + let sky = 0..v.len() as u32; + let apex = Vertex { + pos: [0.0; 3], + kind: SUNSET_APEX, + }; + let rim = |i: u32| { + let a = i as f32 * std::f32::consts::TAU / 16.0; + Vertex { + pos: [a.sin(), a.cos(), 0.0], + kind: SUNSET_RIM, + } + }; + for i in 0..16 { + v.extend([apex, rim(i), rim(i + 1)]); + } + let sunset = sky.end..v.len() as u32; + v.extend( + super::stars::star_triangles() + .into_iter() + .map(|pos| Vertex { pos, kind: STAR }), + ); + let stars = sunset.end..v.len() as u32; + disc(-4.0, VOID, &mut v); + let void = stars.end..v.len() as u32; + let vertices = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Shader pack sky"), + contents: bytemuck::cast_slice(&v), + usage: wgpu::BufferUsages::VERTEX, + }); + let uniform = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("Shader pack sky"), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Shader pack sky"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }], + }); + let group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Shader pack sky"), + layout: &layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: uniform.as_entire_binding(), + }], + }); + Self { + vertices, + uniform, + layout, + group, + ranges: [sky, sunset, stars, void], + } + } + + pub fn buffers() -> [Option>; 1] { + [Some(wgpu::VertexBufferLayout { + array_stride: std::mem::size_of::() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &ATTRIBUTES, + })] + } + + /// Draws the frame's sky, each part with its render stage: the disc, the sunset fan while + /// the sun is near the horizon, the stars and, with the camera low, the void. + pub fn draw( + &self, + queue: &wgpu::Queue, + pass: &mut wgpu::RenderPass<'_>, + frame: &super::Frame, + stage: &dyn Fn(&mut wgpu::RenderPass<'_>, &str), + ) { + let sun_angle = frame.sky.sun_angle.to_radians(); + // The fan turns to the other horizon in the afternoon (terrain.wgsl's sunrise_weight). + let flip = if sun_angle.sin() < 0.0 { 180f32 } else { 0.0 }; + let theta = (flip + 90.0).to_radians(); + let (rotation, brightness) = frame.stars.unwrap_or_default(); + let [r, g, b] = frame.sky.sky_color; + let uniform = Uniform { + stars: rotation.to_cols_array_2d(), + color: [r, g, b, 1.0], + sunrise: frame.sky.sunrise_color, + sun: [theta.cos(), theta.sin(), brightness.clamp(0.0, 1.0), 0.0], + }; + queue.write_buffer(&self.uniform, 0, bytemuck::bytes_of(&uniform)); + pass.set_bind_group(1, &self.group, &[]); + pass.set_vertex_buffer(0, self.vertices.slice(..)); + let [sky, sunset, stars, void] = self.ranges.clone(); + let parts = [ + ("SKY", sky, true), + ("SUNSET", sunset, frame.sky.sunrise_color[3] > 0.001), + ("STARS", stars, frame.stars.is_some()), + ("VOID", void, frame.dark_disc), + ]; + for (name, range, shown) in parts { + if shown { + stage(pass, name); + pass.draw(range, 0..1); + } + } + } +} diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index bed86758e..853f2f2ba 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -34,7 +34,8 @@ pub use macros::{MC_VERSION, Macros, RENDER_STAGES, render_stage}; pub use options::switches_on; pub use pack::{Pack, PackError}; pub use post::{ - Blend, Factor, Group, Post, PostProgram, TRANSLUCENT_BLEND, dimension_folder, load_post, + Blend, Factor, GameInputs, Group, Post, PostProgram, TRANSLUCENT_BLEND, dimension_folder, + load_post, }; pub use preprocess::{Diagnostic, Origin, Preprocessed, preprocess}; pub use properties::{Entry, Note, Properties}; diff --git a/crates/horizon-shaders/src/post.rs b/crates/horizon-shaders/src/post.rs index e155cefbe..56303e67b 100644 --- a/crates/horizon-shaders/src/post.rs +++ b/crates/horizon-shaders/src/post.rs @@ -93,6 +93,9 @@ const TERRAIN: [&str; 4] = [ "gbuffers_basic", ]; +/// `gbuffers_skybasic` and its fallback (2.3). +const SKY: [&str; 2] = ["gbuffers_skybasic", "gbuffers_basic"]; + const GROUPS: [(Group, &str); 4] = [ (Group::Prepare, "prepare"), (Group::Deferred, "deferred"), @@ -127,6 +130,8 @@ pub struct Post { pub terrain: Option, /// What draws translucent terrain (water, stained glass) after deferred. pub water: Option, + /// What draws the sky (disc, sunset, stars, void) before terrain. + pub sky: Option, /// The custom uniforms of `shaders.properties` (3.4). pub custom: Custom, pub config: Config, @@ -145,14 +150,22 @@ pub fn dimension_folder(pack: &Pack, id: i32) -> String { } } -/// Loads the post-processing programs of `folder` and, given the game's terrain vertices, the -/// program terrain draws with. +/// The game's geometry a pack may draw with its gbuffers programs: each kind's vertex inputs, +/// when the game can draw it with a pack. +#[derive(Clone, Debug, Default)] +pub struct GameInputs { + pub terrain: Option, + pub sky: Option, +} + +/// Loads the post-processing programs of `folder` and, for the geometry `inputs` describes, the +/// gbuffers programs it draws with. pub fn load_post( pack: &Pack, folder: &str, properties: &Properties, macros: &Macros, - terrain: Option<&GbuffersInputs>, + inputs: &GameInputs, ) -> Post { let mut post = Post { folder: folder.to_owned(), @@ -161,6 +174,7 @@ pub fn load_post( composite_pre: flips(properties, "composite_pre"), terrain: None, water: None, + sky: None, custom: Custom::parse(properties), config: Config::default(), notes: Vec::new(), @@ -191,17 +205,26 @@ pub fn load_post( } } } - if let Some(inputs) = terrain { - let mut gbuffers = |names: &[&str], group| { - names.iter().find_map(|name| { - loader.load(name.to_string(), group, |v, f| { - translate_gbuffers(v, f, Target::Buffers, inputs) - }) + let mut gbuffers = |names: &[&str], group, inputs: &GbuffersInputs| { + names.iter().find_map(|name| { + loader.load(name.to_string(), group, |v, f| { + translate_gbuffers(v, f, Target::Buffers, inputs) }) - }; - let terrain = gbuffers(&TERRAIN, Group::Gbuffers); + }) + }; + // The sky blends as vanilla draws its sunset fan and stars unless `blend.*` says otherwise. + let sky = inputs + .sky + .as_ref() + .and_then(|i| gbuffers(&SKY, Group::Gbuffers, i)); + let sky = sky.map(|mut p| { + p.blend = blend(properties, &p.name).unwrap_or(Some(TRANSLUCENT_BLEND)); + p + }); + if let Some(inputs) = &inputs.terrain { + let terrain = gbuffers(&TERRAIN, Group::Gbuffers, inputs); // gbuffers_water falls back to gbuffers_terrain's chain (2.3). - let water = gbuffers(&WATER, Group::Translucent); + let water = gbuffers(&WATER, Group::Translucent, inputs); loader.post.terrain = terrain; loader.post.water = water.map(|mut p| { p.blend = blend(properties, &p.name).unwrap_or(Some(TRANSLUCENT_BLEND)); @@ -211,6 +234,7 @@ pub fn load_post( p.blend = blend(properties, &p.name).flatten(); } } + loader.post.sky = sky; post.config = std::mem::take(&mut post.config).finish(); post } @@ -433,7 +457,13 @@ mod tests { ); assert_eq!(dimension_folder(&pack, 0), ""); assert_eq!(dimension_folder(&pack, -1), "world-1"); - let post = load_post(&pack, "", &properties, &Macros::standard(), None); + let post = load_post( + &pack, + "", + &properties, + &Macros::standard(), + &GameInputs::default(), + ); let names: Vec<&str> = post.programs.iter().map(|p| p.name.as_str()).collect(); assert_eq!(names, ["deferred", "composite", "composite2", "final"]); let p = &post.programs; @@ -454,14 +484,21 @@ mod tests { let mut macros = Macros::standard(); macros.define("HQ", ""); - let post = load_post(&pack, "world-1", &properties, ¯os, None); + let post = load_post( + &pack, + "world-1", + &properties, + ¯os, + &GameInputs::default(), + ); let names: Vec<&str> = post.programs.iter().map(|p| p.name.as_str()).collect(); assert_eq!(names, ["final"]); - let post = load_post(&pack, "", &properties, ¯os, None); + let post = load_post(&pack, "", &properties, ¯os, &GameInputs::default()); assert!(post.programs.iter().any(|p| p.name == "composite3")); } - /// Water falls back to the terrain chain and blends by default; `blend.*` overrides. + /// Water falls back to the terrain chain and blends by default, as the sky does; `blend.*` + /// overrides. #[test] fn water_falls_back_to_terrain_and_blends() { let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/packs/terrain-test"); @@ -470,12 +507,25 @@ mod tests { let inputs = crate::GbuffersInputs { declarations: "layout(location = 0) in vec3 hz_v_pos;".into(), body: " hz_Vertex = vec4(hz_v_pos, 1.0);\n hz_Color = vec4(1.0);\n".into(), - attributes: Vec::new(), + ..Default::default() }; - let post = load_post(&pack, "", &Properties::default(), ¯os, Some(&inputs)); + let inputs = GameInputs { + terrain: Some(inputs.clone()), + sky: Some(inputs), + }; + let post = load_post(&pack, "", &Properties::default(), ¯os, &inputs); let (terrain, water) = (post.terrain.unwrap(), post.water.unwrap()); assert_eq!((terrain.group, terrain.blend), (Group::Gbuffers, None)); assert_eq!(water.name, "gbuffers_terrain"); + let sky = post.sky.unwrap(); + assert_eq!( + (sky.name.as_str(), sky.group, sky.blend), + ( + "gbuffers_skybasic", + Group::Gbuffers, + Some(TRANSLUCENT_BLEND) + ) + ); assert_eq!( (water.group, water.blend), (Group::Translucent, Some(TRANSLUCENT_BLEND)) @@ -504,7 +554,7 @@ mod tests { &dimension_folder(&pack, 0), &properties, ¯os, - None, + &GameInputs::default(), ); assert!(post.notes.is_empty(), "{:?}", post.notes); let names: Vec<&str> = post.programs.iter().map(|p| p.name.as_str()).collect(); diff --git a/crates/horizon-shaders/src/translate/mod.rs b/crates/horizon-shaders/src/translate/mod.rs index fe736e4d8..35475ddab 100644 --- a/crates/horizon-shaders/src/translate/mod.rs +++ b/crates/horizon-shaders/src/translate/mod.rs @@ -109,6 +109,8 @@ pub struct GbuffersInputs { /// The value of each attribute a pack may declare (`mc_Entity`, `vaPosition`...), as a GLSL /// expression of the declared type's size; an attribute not listed reads zero. pub attributes: Vec<(String, String)>, + /// GLSL statements run last in `main`, with `gl_Position` in wgpu's clip space. + pub after: String, } /// Translates a composite-style program (`composite`, `deferred`, `final`...). @@ -292,7 +294,8 @@ fn translate( } write!( vertex_main, - " hz_main();\n gl_Position = vec4(gl_Position.x, {flip}gl_Position.y, (gl_Position.z + gl_Position.w) * 0.5, gl_Position.w);\n}}\n" + " hz_main();\n gl_Position = vec4(gl_Position.x, {flip}gl_Position.y, (gl_Position.z + gl_Position.w) * 0.5, gl_Position.w);\n{}}}\n", + gbuffers.map_or("", |g| &g.after) ) .unwrap(); let mut fragment_main = String::from("\nvoid main() {\n"); @@ -597,6 +600,7 @@ mod tests { hz_MultiTexCoord1 = vec4(240.0, 240.0, 0.0, 1.0);\n hz_Color = vec4(1.0);\n hz_Normal = vec3(0.0, 1.0, 0.0);\n" .into(), attributes: vec![("mc_midTexCoord".into(), "vec2(game_uv) / 32768.0".into())], + ..Default::default() }; let p = translate_gbuffers( &preprocess(&pack, "gbuffers_terrain.vsh", &m), diff --git a/crates/horizon-shaders/tests/packs/terrain-test/README.md b/crates/horizon-shaders/tests/packs/terrain-test/README.md index bb8ddd060..3fc37bf4d 100644 --- a/crates/horizon-shaders/tests/packs/terrain-test/README.md +++ b/crates/horizon-shaders/tests/packs/terrain-test/README.md @@ -3,6 +3,8 @@ Horizon's own shader pack for testing gbuffers terrain (written for Horizon, no other pack's code). `gbuffers_terrain` draws blocks from the atlas with their vertex colour and lightmap, tinted by the lightmap coordinate's sky light (cooler where the sky is hidden); `mc_midTexCoord` marks each -face's centre with a dot. `composite` puts the depth buffer's view-space distance into the blue +face's centre with a dot, and any stage but solid terrain (translucent terrain falls back to it) +is red. `gbuffers_skybasic` draws the sky in its vertex colour, a little greener, with magenta +stars and a dark red void. `composite` puts the depth buffer's view-space distance into the blue channel of the far half. Copy the folder into `shaderpacks/` and run the game with `HORIZON_SHADER_PACK=terrain-test`. diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_skybasic.fsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_skybasic.fsh new file mode 100644 index 000000000..8d62f3291 --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_skybasic.fsh @@ -0,0 +1,15 @@ +#version 120 + +uniform int renderStage; + +varying vec4 glcolor; + +void main() { + vec4 color = glcolor; + // Stars magenta, the void dark red, the sky disc a little greener: each stage is told apart. + if (renderStage == MC_RENDER_STAGE_STARS) color.rgb = vec3(1.0, 0.0, 1.0); + if (renderStage == MC_RENDER_STAGE_VOID) color.rgb = vec3(0.3, 0.0, 0.0); + if (renderStage == MC_RENDER_STAGE_SKY) color.g = min(color.g + 0.2, 1.0); + /* DRAWBUFFERS:0 */ + gl_FragData[0] = color; +} diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_skybasic.vsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_skybasic.vsh new file mode 100644 index 000000000..7b3bd59ce --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_skybasic.vsh @@ -0,0 +1,8 @@ +#version 120 + +varying vec4 glcolor; + +void main() { + gl_Position = ftransform(); + glcolor = gl_Color; +} diff --git a/crates/horizon-shaders/tests/real_packs.rs b/crates/horizon-shaders/tests/real_packs.rs index c5d4defac..a04a31b15 100644 --- a/crates/horizon-shaders/tests/real_packs.rs +++ b/crates/horizon-shaders/tests/real_packs.rs @@ -167,19 +167,19 @@ fn real_packs_read() { declarations: "layout(location = 0) in vec3 hz_v_pos;".into(), body: " hz_Vertex = vec4(hz_v_pos, 1.0);\n hz_Color = vec4(1.0);\n".into(), attributes: vec![("mc_Entity".into(), "vec4(-1.0)".into())], + ..Default::default() }; - let post = load_post( - &pack, - &folder, - &shaders, - &Macros::standard(), - Some(&terrain), - ); + let inputs = horizon_shaders::GameInputs { + terrain: Some(terrain.clone()), + sky: Some(terrain), + }; + let post = load_post(&pack, &folder, &shaders, &Macros::standard(), &inputs); println!( - " dimension {id} (folder `{folder}`): {} post programs, terrain {:?}, water {:?}, {} notes, {:.0} ms", + " dimension {id} (folder `{folder}`): {} post programs, terrain {:?}, water {:?}, sky {:?}, {} notes, {:.0} ms", post.programs.len(), post.terrain.as_ref().map(|t| &t.name), post.water.as_ref().map(|t| &t.name), + post.sky.as_ref().map(|t| &t.name), post.notes.len() + post.config.notes.len(), start.elapsed().as_secs_f64() * 1000.0 ); From 230730d1f5324ed3fc7a658e384767f6d706e713 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:02:13 +0100 Subject: [PATCH 11/61] Shader packs, clean room 23: the sun and moon (gbuffers_skytextured) gbuffers_skytextured (or gbuffers_textured, gbuffers_basic) draws the frame's sun and moon quads, with their render stages, `texture` reading the shared atlas their sprites live in. The sky's parts draw in vanilla's order (disc, sunset, sun, moon, stars, void), in one render pass while consecutive programs write the same buffers. The sun and moon add to the sky (SRC_ALPHA, ONE) unless blend. says otherwise. Horizon's world pass skips its own sun and moon when the pack draws them; a part without a program stays Horizon's. Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, 2.3, 5.4; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx.rs | 15 +- crates/horizon-client/src/gfx/pack_post.rs | 298 +++++++++++------- crates/horizon-client/src/gfx/pack_sky.rs | 136 ++++++-- crates/horizon-shaders/src/lib.rs | 4 +- crates/horizon-shaders/src/post.rs | 35 +- .../tests/packs/terrain-test/README.md | 3 +- .../shaders/gbuffers_textured.fsh | 15 + .../shaders/gbuffers_textured.vsh | 10 + crates/horizon-shaders/tests/real_packs.rs | 6 +- 9 files changed, 370 insertions(+), 152 deletions(-) create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_textured.fsh create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_textured.vsh diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 7444b136a..bdd8250b8 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -2792,7 +2792,13 @@ impl Renderer { buffers: if flat { &buffers[..1] } else { &buffers }, table: self.flat.as_ref().filter(|_| flat).map(|f| &f.draw_layout), depth_write: true, + atlas: None, }), + celestial: self + .hud + .atlas_texture + .as_ref() + .map(|t| t.create_view(&Default::default())), }; pack_post::PackPost::new( &self.device, @@ -2815,6 +2821,7 @@ impl Renderer { horizon_shaders::GameInputs { terrain: self.pack_terrain_inputs(), sky: Some(pack_sky::inputs()), + celestial: Some(pack_sky::celestial_inputs()), } } @@ -3739,6 +3746,12 @@ impl Renderer { .as_ref() .and_then(|p| p.1.as_ref()) .is_some_and(|p| p.has_sky()); + let pack_draws_celestial = pack_post + && self + .pack_post + .as_ref() + .and_then(|p| p.1.as_ref()) + .is_some_and(|p| p.has_celestial()); let pack_draws_water = pack_draws_terrain && self .pack_post @@ -3814,7 +3827,7 @@ impl Renderer { if frame.end_sky { self.end_sky.draw(&mut pass); } - if celestial_vertices > 0 { + if celestial_vertices > 0 && !pack_draws_celestial { pass.set_vertex_buffer(0, self.celestial_buffer.slice(..)); pass.draw_indexed(0..(celestial_vertices as u32 / 4) * 6, 0, 0..1); } diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 2334b84c9..9002ad963 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -121,12 +121,13 @@ fn share_samplers(module: &mut naga::Module, slot: &HashMap) { } } -/// The game's geometry a gbuffers pass draws. -#[derive(Clone, Copy, PartialEq)] -enum Gbuffers { - Sky, - Terrain, - Water, +/// The sky's geometry and the programs that draw it: `gbuffers_skybasic` (disc, sunset, stars, +/// void) and `gbuffers_skytextured` (sun, moon), each with its fallbacks. A part without a +/// program is Horizon's own. +struct Sky { + geometry: PackSky, + basic: Option, + textured: Option, } struct Pass { @@ -151,6 +152,8 @@ struct Pass { mipmaps: u16, /// Reads the per-draw block (`DrawUniforms`). draw: bool, + /// What `texture` reads instead of the block atlas. + atlas: Option, groups: HashMap, } @@ -181,6 +184,8 @@ pub(super) struct World<'a> { pub lightmap: wgpu::TextureView, /// Terrain's geometry, when the pack draws it. pub terrain: Option>, + /// The atlas of the sun and moon's sprites. + pub celestial: Option, } pub struct PackPost { @@ -191,7 +196,7 @@ pub struct PackPost { /// What draws translucent terrain (`gbuffers_water` or its fallback), after `deferred`. water: Option, /// What draws the sky (`gbuffers_skybasic` or its fallback), before terrain, and its vertices. - sky: Option<(Pass, PackSky)>, + sky: Option, /// A program after translucent terrain reads `depthtex1`: it is copied before water. depth1: bool, /// Buffers with mipmaps (filterable ones a program asks them for). @@ -494,6 +499,8 @@ pub(super) struct Geometry<'a> { pub table: Option<&'a wgpu::BindGroupLayout>, /// Writes depth (the sky does not). pub depth_write: bool, + /// What `texture` reads instead of the block atlas. + pub atlas: Option, } /// The pass of program `p`: a quad, or `geometry` drawn by the game. `None` (with a note) when @@ -737,6 +744,7 @@ fn build_pass( written, mipmaps: p.mipmaps, draw, + atlas: geometry.and_then(|g| g.atlas.clone()), groups: HashMap::new(), }) } @@ -796,15 +804,29 @@ impl PackPost { .water .zip(world.terrain.as_ref()) .and_then(|(p, g)| build(p, Some(g))); - let sky = post.sky.and_then(|p| { - let sky = PackSky::new(device); - let geometry = Geometry { + let sky = (post.sky.is_some() || post.sky_textured.is_some()).then(|| { + let geometry = PackSky::new(device); + let basic = Geometry { buffers: &PackSky::buffers(), - table: Some(&sky.layout), + table: Some(&geometry.layout), depth_write: false, + atlas: None, }; - Some((build(p, Some(&geometry))?, sky)) + let basic = post.sky.and_then(|p| build(p, Some(&basic))); + let textured = Geometry { + buffers: &PackSky::celestial_buffers(), + table: None, + depth_write: false, + atlas: world.celestial.clone(), + }; + let textured = post.sky_textured.and_then(|p| build(p, Some(&textured))); + Sky { + geometry, + basic, + textured, + } }); + let sky = sky.filter(|s| s.basic.is_some() || s.textured.is_some()); // Developers' aid: `HORIZON_SHADER_PACK_SKIP=composite,final` leaves programs out. let skip = std::env::var("HORIZON_SHADER_PACK_SKIP").unwrap_or_default(); let passes: Vec = post @@ -819,8 +841,13 @@ impl PackPost { if let Some(w) = &water { notes.push(format!("translucent terrain draws with {}", w.name)); } - if let Some((s, _)) = &sky { - notes.push(format!("the sky draws with {}", s.name)); + for (part, pass) in sky + .iter() + .flat_map(|s| [("sky", &s.basic), ("sun and moon", &s.textured)]) + { + if let Some(p) = pass { + notes.push(format!("the {part} draw with {}", p.name)); + } } let depth1 = water.is_some() && passes @@ -837,7 +864,7 @@ impl PackPost { // Gbuffers passes cover part of their targets: a clear is needed if they touch it. let first = sky .iter() - .map(|s| &s.0) + .flat_map(|s| s.basic.iter().chain(&s.textured)) .chain(&terrain) .chain(&water) .find(touched) @@ -1083,7 +1110,11 @@ impl PackPost { Source::Noise => &self.noise, Source::Dummy => &self.dummy, Source::Shadow => &self.shadow.0, - Source::Atlas => self.atlas.as_ref().unwrap_or(&self.dummy), + Source::Atlas => pass + .atlas + .as_ref() + .or(self.atlas.as_ref()) + .unwrap_or(&self.dummy), Source::Lightmap => &self.lightmap, }; let sampler = match SamplerKind::of(s.source, s.filtering) { @@ -1113,46 +1144,12 @@ impl PackPost { group.clone() } - /// The sky, terrain or translucent terrain with the pack's program into the buffers it - /// writes, in place, tested against the world's depth; `draw` issues the game's terrain - /// draws (`true`: translucent), the sky draws itself. - #[allow(clippy::too_many_arguments)] - fn draw_gbuffers( + /// A render pass into the buffers `pass` writes, in place, tested against the world's depth. + fn begin<'e>( &mut self, - device: &wgpu::Device, - queue: &wgpu::Queue, - encoder: &mut wgpu::CommandEncoder, - size: [u32; 2], - frame: &super::Frame, - which: Gbuffers, - draw: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), - ) { - let translucent = which == Gbuffers::Water; - let (mut pass, sky) = match which { - Gbuffers::Sky => match self.sky.take() { - Some((pass, sky)) => (pass, Some(sky)), - None => return, - }, - Gbuffers::Terrain => match self.terrain.take() { - Some(pass) => (pass, None), - None => return, - }, - Gbuffers::Water => match self.water.take() { - Some(pass) => (pass, None), - None => return, - }, - }; - if translucent - && let Some(t) = &self.targets - && let Some((copy, _)) = &t.depth1 - { - encoder.copy_texture_to_texture( - t.depth_texture.as_image_copy(), - copy.as_image_copy(), - t.depth_texture.size(), - ); - } - let group = self.bind(device, queue, &mut pass, size); + encoder: &'e mut wgpu::CommandEncoder, + pass: &Pass, + ) -> wgpu::RenderPass<'e> { let targets = self.targets.as_ref().expect("resized"); let load = wgpu::Operations { load: wgpu::LoadOp::Load, @@ -1168,52 +1165,128 @@ impl PackPost { .as_ref() .and_then(|t| attach(&t[usize::from(self.main & 1 << buffer != 0)], load)); } - { - let timestamp_writes = self.timing.as_mut().and_then(|t| t.writes(&pass.name)); - let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some(&pass.name), - color_attachments: &attachments, - timestamp_writes, - depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { - view: &targets.depth, - depth_ops: Some(wgpu::Operations { - load: wgpu::LoadOp::Load, - store: wgpu::StoreOp::Store, - }), - stencil_ops: None, + let timestamp_writes = self.timing.as_mut().and_then(|t| t.writes(&pass.name)); + encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some(&pass.name), + color_attachments: &attachments, + timestamp_writes, + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &targets.depth, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, }), - ..Default::default() - }); + stencil_ops: None, + }), + ..Default::default() + }) + } + + /// Terrain (or, `translucent`, translucent terrain) with the pack's program; `draw` issues + /// the game's draws. + fn draw_terrain( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + encoder: &mut wgpu::CommandEncoder, + size: [u32; 2], + translucent: bool, + draw: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), + ) { + let slot = if translucent { + &mut self.water + } else { + &mut self.terrain + }; + let Some(mut pass) = slot.take() else { + return; + }; + if translucent + && let Some(t) = &self.targets + && let Some((copy, _)) = &t.depth1 + { + encoder.copy_texture_to_texture( + t.depth_texture.as_image_copy(), + copy.as_image_copy(), + t.depth_texture.size(), + ); + } + let group = self.bind(device, queue, &mut pass, size); + { + let mut rp = self.begin(encoder, &pass); rp.set_pipeline(&pass.pipeline); rp.set_bind_group(0, &group, &[]); - let draws = pass.draw.then_some(&self.draws); - match &sky { - Some(sky) => sky.draw(queue, &mut rp, frame, &|rp, stage| { - if let Some(d) = draws { - d.bind(rp, stage); - } - }), - None => { - if let Some(d) = draws { - let stage = if translucent { - "TERRAIN_TRANSLUCENT" - } else { - "TERRAIN_SOLID" - }; - d.bind(&mut rp, stage); - } - draw(&mut rp, translucent); - } + if pass.draw { + let stage = if translucent { + "TERRAIN_TRANSLUCENT" + } else { + "TERRAIN_SOLID" + }; + self.draws.bind(&mut rp, stage); } + draw(&mut rp, translucent); } self.mips_valid &= !pass.written; - match (which, sky) { - (Gbuffers::Sky, Some(sky)) => self.sky = Some((pass, sky)), - (Gbuffers::Water, _) => self.water = Some(pass), - _ => self.terrain = Some(pass), + if translucent { + self.water = Some(pass); + } else { + self.terrain = Some(pass); } } + /// The sky's parts in vanilla's order, each with the program that draws it; one render pass + /// while consecutive programs write the same buffers. + fn draw_sky( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + encoder: &mut wgpu::CommandEncoder, + size: [u32; 2], + frame: &super::Frame, + ) { + let Some(mut sky) = self.sky.take() else { + return; + }; + let parts = sky.geometry.prepare(queue, frame); + let mut groups = [None, None]; + for (i, pass) in [&mut sky.basic, &mut sky.textured].into_iter().enumerate() { + if let Some(pass) = pass { + groups[i] = Some(self.bind(device, queue, pass, size)); + self.mips_valid &= !pass.written; + } + } + let passes = [&sky.basic, &sky.textured]; + let mut open: Option<(wgpu::RenderPass<'_>, usize)> = None; + for part in parts { + let i = usize::from(part.textured); + let (Some(pass), Some(group)) = (passes[i], &groups[i]) else { + continue; + }; + let current = open.as_ref().map(|o| o.1); + if current != Some(i) { + let same = current + .and_then(|c| passes[c].as_ref()) + .is_some_and(|p| p.writes == pass.writes); + if !same { + drop(open.take()); + open = Some((self.begin(encoder, pass).forget_lifetime(), i)); + } + let rp = &mut open.as_mut().expect("open").0; + rp.set_pipeline(&pass.pipeline); + rp.set_bind_group(0, group, &[]); + sky.geometry.bind(rp, part.textured); + open.as_mut().expect("open").1 = i; + } + let rp = &mut open.as_mut().expect("open").0; + if pass.draw { + self.draws.bind(rp, part.stage); + } + rp.draw(part.range, 0..1); + } + drop(open); + self.sky = Some(sky); + } + /// The sky, when it shows, then terrain. fn draw_opaque( &mut self, @@ -1225,17 +1298,9 @@ impl PackPost { terrain: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), ) { if super::sky_visible(frame) { - self.draw_gbuffers(device, queue, encoder, size, frame, Gbuffers::Sky, terrain); + self.draw_sky(device, queue, encoder, size, frame); } - self.draw_gbuffers( - device, - queue, - encoder, - size, - frame, - Gbuffers::Terrain, - terrain, - ); + self.draw_terrain(device, queue, encoder, size, false, terrain); } /// Regenerates the mip chain of buffer `b`'s side that programs read. @@ -1269,9 +1334,14 @@ impl PackPost { self.terrain.is_some() } - /// Whether the pack draws the sky itself. + /// Whether the pack draws the sky (disc, sunset, stars, void) itself. pub fn has_sky(&self) -> bool { - self.sky.is_some() + self.sky.as_ref().is_some_and(|s| s.basic.is_some()) + } + + /// Whether the pack draws the sun and moon itself. + pub fn has_celestial(&self) -> bool { + self.sky.as_ref().is_some_and(|s| s.textured.is_some()) } /// Whether the pack draws translucent terrain itself. @@ -1521,15 +1591,7 @@ impl PackPost { } if !water_drawn && matches!(pass.group, Group::Composite | Group::Final) { water_drawn = true; - self.draw_gbuffers( - device, - queue, - encoder, - size, - frame, - Gbuffers::Water, - terrain, - ); + self.draw_terrain(device, queue, encoder, size, true, terrain); } if pass.group == Group::Deferred && !seen.0 { seen.0 = true; @@ -1584,15 +1646,7 @@ impl PackPost { self.draw_opaque(device, queue, encoder, size, frame, terrain); } if !water_drawn { - self.draw_gbuffers( - device, - queue, - encoder, - size, - frame, - Gbuffers::Water, - terrain, - ); + self.draw_terrain(device, queue, encoder, size, true, terrain); } let targets = self.targets.as_ref().expect("resized"); let view = |m: u16, i: usize, alt: bool| { diff --git a/crates/horizon-client/src/gfx/pack_sky.rs b/crates/horizon-client/src/gfx/pack_sky.rs index cd99d03b9..007fdc034 100644 --- a/crates/horizon-client/src/gfx/pack_sky.rs +++ b/crates/horizon-client/src/gfx/pack_sky.rs @@ -1,7 +1,8 @@ //! The vanilla sky's geometry for a shader pack's `gbuffers_skybasic` (spec 2.3): the sky disc, //! the sunset fan, the stars and the void disc, as Horizon's own sky draws them (terrain.wgsl, //! stars.rs), built once into a vertex buffer. The frame's colours and rotations sit in a small -//! uniform (set 1) that the vertex inputs read. +//! uniform (set 1) that the vertex inputs read. `gbuffers_skytextured` draws the frame's sun and +//! moon quads, textured from the shared atlas. use std::ops::Range; @@ -36,6 +37,21 @@ struct Uniform { sun: [f32; 4], } +/// A sun or moon vertex: position, atlas UV, RGBA8 colour. +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct Celestial { + pos: [f32; 3], + uv: [f32; 2], + color: u32, +} + +/// Quads of the sun and moon (and room to spare). +const CELESTIAL_QUADS: usize = 4; + +const CELESTIAL_ATTRIBUTES: [wgpu::VertexAttribute; 3] = + wgpu::vertex_attr_array![0 => Float32x3, 1 => Float32x2, 2 => Unorm8x4]; + pub(super) const ATTRIBUTES: [wgpu::VertexAttribute; 2] = wgpu::vertex_attr_array![0 => Float32x3, 1 => Uint32]; @@ -83,6 +99,37 @@ pub fn inputs() -> horizon_shaders::GbuffersInputs { } } +/// What `gbuffers_skytextured` reads from the sun and moon's vertices. +pub fn celestial_inputs() -> horizon_shaders::GbuffersInputs { + horizon_shaders::GbuffersInputs { + declarations: + "layout(location = 0) in vec3 hz_v_pos;\nlayout(location = 1) in vec2 hz_v_uv;\n\ + layout(location = 2) in vec4 hz_v_color;\n" + .into(), + body: " hz_Vertex = vec4(hz_v_pos, 1.0);\n \ + hz_MultiTexCoord0 = vec4(hz_v_uv, 0.0, 1.0);\n \ + hz_MultiTexCoord1 = vec4(0.0, 240.0, 0.0, 1.0);\n \ + hz_MultiTexCoord2 = hz_MultiTexCoord1;\n \ + hz_Color = hz_v_color;\n \ + hz_Normal = vec3(0.0, 0.0, 1.0);\n" + .into(), + attributes: vec![ + ("vaPosition".into(), "hz_Vertex.xyz".into()), + ("vaUV0".into(), "hz_v_uv".into()), + ("vaColor".into(), "hz_Color".into()), + ("vaNormal".into(), "hz_Normal".into()), + ], + after: " gl_Position.z = gl_Position.w;\n".into(), + } +} + +/// One draw of the sky: its render stage, whether `gbuffers_skytextured` draws it, its vertices. +pub(super) struct Part { + pub stage: &'static str, + pub textured: bool, + pub range: Range, +} + /// A disc fan at height `y`, radius 512, as triangles (SkyRenderer's sky and void discs). fn disc(y: f32, kind: u32, out: &mut Vec) { let sign = y.signum(); @@ -104,6 +151,7 @@ fn disc(y: f32, kind: u32, out: &mut Vec) { pub(super) struct PackSky { vertices: wgpu::Buffer, + celestial: wgpu::Buffer, uniform: wgpu::Buffer, pub layout: wgpu::BindGroupLayout, group: wgpu::BindGroup, @@ -144,6 +192,12 @@ impl PackSky { contents: bytemuck::cast_slice(&v), usage: wgpu::BufferUsages::VERTEX, }); + let celestial = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("Shader pack sun and moon"), + size: (std::mem::size_of::() * 6 * CELESTIAL_QUADS) as u64, + usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); let uniform = device.create_buffer(&wgpu::BufferDescriptor { label: Some("Shader pack sky"), size: std::mem::size_of::() as u64, @@ -173,6 +227,7 @@ impl PackSky { }); Self { vertices, + celestial, uniform, layout, group, @@ -188,15 +243,18 @@ impl PackSky { })] } - /// Draws the frame's sky, each part with its render stage: the disc, the sunset fan while - /// the sun is near the horizon, the stars and, with the camera low, the void. - pub fn draw( - &self, - queue: &wgpu::Queue, - pass: &mut wgpu::RenderPass<'_>, - frame: &super::Frame, - stage: &dyn Fn(&mut wgpu::RenderPass<'_>, &str), - ) { + pub fn celestial_buffers() -> [Option>; 1] { + [Some(wgpu::VertexBufferLayout { + array_stride: std::mem::size_of::() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &CELESTIAL_ATTRIBUTES, + })] + } + + /// Writes the frame's sky and returns its draws in vanilla's order: the disc, the sunset + /// fan while the sun is near the horizon, the sun and the moon, the stars and, with the + /// camera low, the void. + pub fn prepare(&self, queue: &wgpu::Queue, frame: &super::Frame) -> Vec { let sun_angle = frame.sky.sun_angle.to_radians(); // The fan turns to the other horizon in the afternoon (terrain.wgsl's sunrise_weight). let flip = if sun_angle.sin() < 0.0 { 180f32 } else { 0.0 }; @@ -210,20 +268,52 @@ impl PackSky { sun: [theta.cos(), theta.sin(), brightness.clamp(0.0, 1.0), 0.0], }; queue.write_buffer(&self.uniform, 0, bytemuck::bytes_of(&uniform)); - pass.set_bind_group(1, &self.group, &[]); - pass.set_vertex_buffer(0, self.vertices.slice(..)); + // The sun's quad, then the moon's (Game::celestial_geometry), as triangles. + let quads = frame + .celestial + .as_chunks::<4>() + .0 + .iter() + .take(CELESTIAL_QUADS); + let triangles: Vec = quads + .clone() + .flat_map(|q| [0, 1, 2, 0, 2, 3].map(|i| q[i])) + .map(|v| Celestial { + pos: v.pos, + uv: v.uv, + color: v.color, + }) + .collect(); + queue.write_buffer(&self.celestial, 0, bytemuck::cast_slice(&triangles)); let [sky, sunset, stars, void] = self.ranges.clone(); - let parts = [ - ("SKY", sky, true), - ("SUNSET", sunset, frame.sky.sunrise_color[3] > 0.001), - ("STARS", stars, frame.stars.is_some()), - ("VOID", void, frame.dark_disc), - ]; - for (name, range, shown) in parts { - if shown { - stage(pass, name); - pass.draw(range, 0..1); - } + let part = |stage, textured, range| Part { + stage, + textured, + range, + }; + let mut parts = vec![part("SKY", false, sky)]; + if frame.sky.sunrise_color[3] > 0.001 { + parts.push(part("SUNSET", false, sunset)); + } + for (i, stage) in ["SUN", "MOON"].into_iter().enumerate().take(quads.len()) { + parts.push(part(stage, true, i as u32 * 6..i as u32 * 6 + 6)); + } + if frame.stars.is_some() { + parts.push(part("STARS", false, stars)); + } + if frame.dark_disc { + parts.push(part("VOID", false, void)); + } + parts + } + + /// Binds the vertices of the `textured` (sun and moon) or other parts. + pub fn bind(&self, pass: &mut wgpu::RenderPass<'_>, textured: bool) { + if textured { + pass.set_vertex_buffer(0, self.celestial.slice(..)); + } else { + pass.set_bind_group(1, &self.group, &[]); + pass.set_vertex_buffer(0, self.vertices.slice(..)); } } } diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index 853f2f2ba..f33dfec57 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -34,8 +34,8 @@ pub use macros::{MC_VERSION, Macros, RENDER_STAGES, render_stage}; pub use options::switches_on; pub use pack::{Pack, PackError}; pub use post::{ - Blend, Factor, GameInputs, Group, Post, PostProgram, TRANSLUCENT_BLEND, dimension_folder, - load_post, + Blend, CELESTIAL_BLEND, Factor, GameInputs, Group, Post, PostProgram, TRANSLUCENT_BLEND, + dimension_folder, load_post, }; pub use preprocess::{Diagnostic, Origin, Preprocessed, preprocess}; pub use properties::{Entry, Note, Properties}; diff --git a/crates/horizon-shaders/src/post.rs b/crates/horizon-shaders/src/post.rs index 56303e67b..944cf10b8 100644 --- a/crates/horizon-shaders/src/post.rs +++ b/crates/horizon-shaders/src/post.rs @@ -96,6 +96,21 @@ const TERRAIN: [&str; 4] = [ /// `gbuffers_skybasic` and its fallback (2.3). const SKY: [&str; 2] = ["gbuffers_skybasic", "gbuffers_basic"]; +/// `gbuffers_skytextured` and its fallbacks (2.3). +const SKY_TEXTURED: [&str; 3] = [ + "gbuffers_skytextured", + "gbuffers_textured", + "gbuffers_basic", +]; + +/// The sun and moon's default blend: added to the sky, as vanilla draws them. +pub const CELESTIAL_BLEND: Blend = [ + Factor::SrcAlpha, + Factor::One, + Factor::One, + Factor::OneMinusSrcAlpha, +]; + const GROUPS: [(Group, &str); 4] = [ (Group::Prepare, "prepare"), (Group::Deferred, "deferred"), @@ -132,6 +147,8 @@ pub struct Post { pub water: Option, /// What draws the sky (disc, sunset, stars, void) before terrain. pub sky: Option, + /// What draws the sun and moon, between the sunset and the stars. + pub sky_textured: Option, /// The custom uniforms of `shaders.properties` (3.4). pub custom: Custom, pub config: Config, @@ -156,6 +173,7 @@ pub fn dimension_folder(pack: &Pack, id: i32) -> String { pub struct GameInputs { pub terrain: Option, pub sky: Option, + pub celestial: Option, } /// Loads the post-processing programs of `folder` and, for the geometry `inputs` describes, the @@ -175,6 +193,7 @@ pub fn load_post( terrain: None, water: None, sky: None, + sky_textured: None, custom: Custom::parse(properties), config: Config::default(), notes: Vec::new(), @@ -221,6 +240,12 @@ pub fn load_post( p.blend = blend(properties, &p.name).unwrap_or(Some(TRANSLUCENT_BLEND)); p }); + let celestial = inputs.celestial.as_ref(); + let sky_textured = celestial.and_then(|i| gbuffers(&SKY_TEXTURED, Group::Gbuffers, i)); + let sky_textured = sky_textured.map(|mut p| { + p.blend = blend(properties, &p.name).unwrap_or(Some(CELESTIAL_BLEND)); + p + }); if let Some(inputs) = &inputs.terrain { let terrain = gbuffers(&TERRAIN, Group::Gbuffers, inputs); // gbuffers_water falls back to gbuffers_terrain's chain (2.3). @@ -235,6 +260,7 @@ pub fn load_post( } } loader.post.sky = sky; + loader.post.sky_textured = sky_textured; post.config = std::mem::take(&mut post.config).finish(); post } @@ -511,7 +537,8 @@ mod tests { }; let inputs = GameInputs { terrain: Some(inputs.clone()), - sky: Some(inputs), + sky: Some(inputs.clone()), + celestial: Some(inputs), }; let post = load_post(&pack, "", &Properties::default(), ¯os, &inputs); let (terrain, water) = (post.terrain.unwrap(), post.water.unwrap()); @@ -526,6 +553,12 @@ mod tests { Some(TRANSLUCENT_BLEND) ) ); + // gbuffers_skytextured falls back to gbuffers_textured, then gbuffers_basic. + let sun = post.sky_textured.unwrap(); + assert_eq!( + (sun.name.as_str(), sun.blend), + ("gbuffers_textured", Some(CELESTIAL_BLEND)) + ); assert_eq!( (water.group, water.blend), (Group::Translucent, Some(TRANSLUCENT_BLEND)) diff --git a/crates/horizon-shaders/tests/packs/terrain-test/README.md b/crates/horizon-shaders/tests/packs/terrain-test/README.md index 3fc37bf4d..7d2c168cb 100644 --- a/crates/horizon-shaders/tests/packs/terrain-test/README.md +++ b/crates/horizon-shaders/tests/packs/terrain-test/README.md @@ -5,6 +5,7 @@ Horizon's own shader pack for testing gbuffers terrain (written for Horizon, no the lightmap coordinate's sky light (cooler where the sky is hidden); `mc_midTexCoord` marks each face's centre with a dot, and any stage but solid terrain (translucent terrain falls back to it) is red. `gbuffers_skybasic` draws the sky in its vertex colour, a little greener, with magenta -stars and a dark red void. `composite` puts the depth buffer's view-space distance into the blue +stars and a dark red void; `gbuffers_textured` (the fallback of `gbuffers_skytextured`) the sun, +and the moon tinted green. `composite` puts the depth buffer's view-space distance into the blue channel of the far half. Copy the folder into `shaderpacks/` and run the game with `HORIZON_SHADER_PACK=terrain-test`. diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_textured.fsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_textured.fsh new file mode 100644 index 000000000..563ddeb7c --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_textured.fsh @@ -0,0 +1,15 @@ +#version 120 + +uniform sampler2D texture; +uniform int renderStage; + +varying vec2 texcoord; +varying vec4 glcolor; + +void main() { + vec4 color = texture2D(texture, texcoord) * glcolor; + // The moon tinted green: the sun and the moon are told apart. + if (renderStage == MC_RENDER_STAGE_MOON) color.rb *= 0.3; + /* DRAWBUFFERS:0 */ + gl_FragData[0] = color; +} diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_textured.vsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_textured.vsh new file mode 100644 index 000000000..c29b1bba0 --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_textured.vsh @@ -0,0 +1,10 @@ +#version 120 + +varying vec2 texcoord; +varying vec4 glcolor; + +void main() { + gl_Position = ftransform(); + texcoord = (gl_TextureMatrix[0] * gl_MultiTexCoord0).xy; + glcolor = gl_Color; +} diff --git a/crates/horizon-shaders/tests/real_packs.rs b/crates/horizon-shaders/tests/real_packs.rs index a04a31b15..5ed954898 100644 --- a/crates/horizon-shaders/tests/real_packs.rs +++ b/crates/horizon-shaders/tests/real_packs.rs @@ -171,15 +171,17 @@ fn real_packs_read() { }; let inputs = horizon_shaders::GameInputs { terrain: Some(terrain.clone()), - sky: Some(terrain), + sky: Some(terrain.clone()), + celestial: Some(terrain), }; let post = load_post(&pack, &folder, &shaders, &Macros::standard(), &inputs); println!( - " dimension {id} (folder `{folder}`): {} post programs, terrain {:?}, water {:?}, sky {:?}, {} notes, {:.0} ms", + " dimension {id} (folder `{folder}`): {} post programs, terrain {:?}, water {:?}, sky {:?} {:?}, {} notes, {:.0} ms", post.programs.len(), post.terrain.as_ref().map(|t| &t.name), post.water.as_ref().map(|t| &t.name), post.sky.as_ref().map(|t| &t.name), + post.sky_textured.as_ref().map(|t| &t.name), post.notes.len() + post.config.notes.len(), start.elapsed().as_secs_f64() * 1000.0 ); From b940754555666ba25fa8541e537b7b9a83aa0449 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:23:38 +0100 Subject: [PATCH 12/61] Terrain vertex arena readable as storage where flat draws run The terrain vertex arena's pages also get STORAGE usage on devices that run flat draws (vertex storage and compute), so a shader pack's terrain program can read the other corners of a vertex's quad to give packs a face normal and tangent. Horizon's own pipelines are unchanged; the A/B in the PR shows no frame-time change without a pack. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx.rs | 5 ++++- crates/horizon-client/src/gfx/arena.rs | 15 +++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index bdd8250b8..1883e6077 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -2140,13 +2140,16 @@ impl Renderer { usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, mapped_at_creation: false, }); + let flat_wanted = flat::wanted(&adapter, &device); + // Readable as storage where flat draws run: a shader pack's terrain program reads them. let arena = arena::Arena::new( &device, std::mem::size_of::() as u64, 1 << 20, + flat_wanted, ); // Sections as one index list per frame where multi-draw is emulated. - let flat = flat::wanted(&adapter, &device).then(|| { + let flat = flat_wanted.then(|| { let buffers = [Some(wgpu::VertexBufferLayout { array_stride: std::mem::size_of::() as u64, step_mode: wgpu::VertexStepMode::Vertex, diff --git a/crates/horizon-client/src/gfx/arena.rs b/crates/horizon-client/src/gfx/arena.rs index 8da835aa2..dbbf70140 100644 --- a/crates/horizon-client/src/gfx/arena.rs +++ b/crates/horizon-client/src/gfx/arena.rs @@ -65,8 +65,15 @@ pub struct Arena { } impl Arena { - pub fn new(device: &wgpu::Device, stride: u64, capacity: u64) -> Self { - Self::with_usage(device, stride, capacity, wgpu::BufferUsages::VERTEX) + /// Vertices; `storage` also lets shaders read them as a storage buffer (a shader pack's + /// terrain program reads the other corners of a vertex's quad for its normal). + pub fn new(device: &wgpu::Device, stride: u64, capacity: u64, storage: bool) -> Self { + let usage = if storage { + wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::STORAGE + } else { + wgpu::BufferUsages::VERTEX + }; + Self::with_usage(device, stride, capacity, usage) } /// Sorted translucent indices; `storage` when compute passes read them. pub fn indices(device: &wgpu::Device, capacity: u64, storage: bool) -> Self { @@ -352,7 +359,7 @@ mod tests { eprintln!("no GPU adapter: skipped"); return; }; - let mut arena = Arena::new(&device, 4, GRAIN); + let mut arena = Arena::new(&device, 4, GRAIN, false); arena.page_limit = 2 * GRAIN; assert_eq!(arena.bytes(), 0); let (first, a) = alloc(&mut arena, &device, &queue, GRAIN).unwrap(); @@ -395,7 +402,7 @@ mod tests { return; }; for mut arena in [ - Arena::new(&device, 28, GRAIN), + Arena::new(&device, 28, GRAIN, false), Arena::indices(&device, GRAIN, false), ] { alloc(&mut arena, &device, &queue, GRAIN).unwrap(); From 441f749c25507bfceedc1d618f72e72522e4cf4d Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:59:39 +0100 Subject: [PATCH 13/61] Shader packs, clean room 24: mc_Entity from block.properties block.properties (spec 4.1) is read: `block.=` names with optional namespaces and state filters; the most specific matching entry wins. Terrain vertices carry their sprite layer, not their block, so the game builds a table of the block sprite layers: each takes the id of the block named like its sprite, else the id most blocks drawing it share; y is 1 on the water and lava sprites. Flat terrain reads it from a storage buffer at set 3; other draws and packs without the file read (-1, -1). Building the table takes 14 ms for BSL, 39 ms for Complementary. Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, 4.1, 5.1, 5.3; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- .../horizon-client/src/game/shader_packs.rs | 101 ++++++++++++++- crates/horizon-client/src/gfx.rs | 6 +- crates/horizon-client/src/gfx/pack_atlas.rs | 11 +- crates/horizon-client/src/gfx/pack_post.rs | 71 ++++++++++- crates/horizon-shaders/src/block_ids.rs | 120 ++++++++++++++++++ crates/horizon-shaders/src/lib.rs | 2 + .../tests/packs/terrain-test/README.md | 2 +- .../terrain-test/shaders/block.properties | 3 + .../terrain-test/shaders/gbuffers_terrain.fsh | 3 + .../terrain-test/shaders/gbuffers_terrain.vsh | 3 + 10 files changed, 314 insertions(+), 8 deletions(-) create mode 100644 crates/horizon-shaders/src/block_ids.rs create mode 100644 crates/horizon-shaders/tests/packs/terrain-test/shaders/block.properties diff --git a/crates/horizon-client/src/game/shader_packs.rs b/crates/horizon-client/src/game/shader_packs.rs index 360262640..7fa2ea3f6 100644 --- a/crates/horizon-client/src/game/shader_packs.rs +++ b/crates/horizon-client/src/game/shader_packs.rs @@ -4,7 +4,7 @@ use horizon_client::camera::CameraFluid; use horizon_client::gfx::pack_uniforms::PackInputs; -use horizon_shaders::{Macros, Pack, Properties, dimension_folder, load_post}; +use horizon_shaders::{BlockIds, Macros, Pack, Properties, dimension_folder, load_post}; use super::{Dim, Game}; @@ -44,7 +44,21 @@ impl Game<'_> { let folder = dimension_folder(&pack, dimension); let inputs = renderer.pack_inputs(); let post = load_post(&pack, &folder, &properties, ¯os, &inputs); - if let Some((programs, notes)) = renderer.set_shader_pack(dimension, Some(post)) + let text = pack.read_text("block.properties").unwrap_or_default(); + let ids = BlockIds::parse(&Properties::parse(&text, ¯os)); + for note in &ids.notes { + log::warn!("shader pack {name}: block.properties: {note}"); + } + let started_ids = std::time::Instant::now(); + let table = block_id_table(&self.pack, &ids); + log::info!( + "shader pack {name}: mc_Entity for {} of {} block sprites in {} ms", + table.chunks(2).filter(|p| p[0] != -1).count(), + table.len() / 2, + started_ids.elapsed().as_millis() + ); + if let Some((programs, notes)) = + renderer.set_shader_pack(dimension, Some(post), &table) { for note in notes { log::warn!("shader pack {name}: {note}"); @@ -59,7 +73,7 @@ impl Game<'_> { } Err(e) => { log::warn!("shader pack {name}: {e}"); - renderer.set_shader_pack(dimension, None); + renderer.set_shader_pack(dimension, None, &[]); } } } @@ -83,3 +97,84 @@ impl Game<'_> { } } } + +/// `mc_Entity` (x, y) of each block sprite layer, as (id, fluid) pairs: terrain vertices carry +/// their sprite layer, not their block. A layer takes the id of the block named like its sprite +/// (`block/oak_leaves` for `oak_leaves`), else the id most blocks drawing it share (ties: the +/// lowest); -1 when no block drawing it has one. y is 1 for the fluid sprites, -1 otherwise. +fn block_id_table(pack: &super::PackResources, ids: &BlockIds) -> Vec { + use std::collections::{BTreeMap, BTreeSet}; + let layers = pack + .render + .sprite_layers + .values() + .max() + .map_or(0, |l| l + 1) as usize; + let mut table = vec![-1; layers * 2]; + if ids.is_empty() { + return table; + } + let registry = pack.assets.registry; + let catalog = &pack.render.model_catalog; + let sprite = |name: &str| pack.render.sprite_layers.get(name).copied(); + // (layer, id) -> (a block named like the sprite, the blocks). + let mut votes: BTreeMap<(u32, i32), (bool, BTreeSet<&str>)> = BTreeMap::new(); + for i in 0..registry.len() { + let state_id = horizon_world::registry::BlockStateId(i as u32); + let Some(state) = registry.state(state_id) else { + continue; + }; + let Some(id) = ids.id(&state.name, |k| state.properties.get(k).map(String::as_str)) else { + continue; + }; + let block = state.name.strip_prefix("minecraft:").unwrap_or(&state.name); + let mut vote = |layer: u32| { + let named = pack + .layer_names + .get(&layer) + .and_then(|n| n.strip_prefix("block/")) + == Some(block); + let v = votes.entry((layer, id)).or_default(); + v.0 |= named; + v.1.insert(&state.name); + }; + // Fluids draw from their own sprites. + let fluid = match block { + "water" => ["water_still", "water_flow"], + "lava" => ["lava_still", "lava_flow"], + _ => ["", ""], + }; + for f in fluid.iter().filter(|f| !f.is_empty()) { + if let Some(l) = sprite(&format!("minecraft:block/{f}")) { + vote(l); + } + } + let origin = horizon_core::position::BlockPos::new(0, 0, 0); + for (_, model) in catalog.collect(state_id, origin).unwrap_or_default() { + for face in &model.faces { + vote(face.layer); + } + } + } + let mut best: BTreeMap = BTreeMap::new(); + for ((layer, id), (named, blocks)) in votes { + let candidate = (named, blocks.len(), -id); + let b = best.entry(layer).or_insert(candidate); + if candidate > *b { + *b = candidate; + } + } + for (layer, (_, _, id)) in best { + if let Some(slot) = table.get_mut(layer as usize * 2) { + *slot = -id; + } + } + for f in ["water_still", "water_flow", "lava_still", "lava_flow"] { + if let Some(l) = sprite(&format!("minecraft:block/{f}")) + && let Some(slot) = table.get_mut(l as usize * 2 + 1) + { + *slot = 1; + } + } + table +} diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 1883e6077..9b7fcc303 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -2764,12 +2764,14 @@ impl Renderer { self.world_post.scale = scale.clamp(0.25, 2.0); self.world_post.antialias = antialias; } - /// Runs `post` over the world from now on (`None`: no pack), for the dimension `dimension`. + /// Runs `post` over the world from now on (`None`: no pack), for the dimension `dimension`; + /// `block_ids` gives each block sprite layer's `mc_Entity` (id, fluid) pair. /// Returns the programs that run and the notes of loading them. pub fn set_shader_pack( &mut self, dimension: i32, pack: Option, + block_ids: &[i32], ) -> Option<(Vec, Vec)> { if pack.is_none() { self.pack_atlas = None; @@ -2796,7 +2798,9 @@ impl Renderer { table: self.flat.as_ref().filter(|_| flat).map(|f| &f.draw_layout), depth_write: true, atlas: None, + ids: None, }), + block_ids, celestial: self .hud .atlas_texture diff --git a/crates/horizon-client/src/gfx/pack_atlas.rs b/crates/horizon-client/src/gfx/pack_atlas.rs index 1a14a8da9..ac56985fd 100644 --- a/crates/horizon-client/src/gfx/pack_atlas.rs +++ b/crates/horizon-client/src/gfx/pack_atlas.rs @@ -127,6 +127,10 @@ impl PackAtlas { "layout(set = 1, binding = 0) readonly buffer HzBlockSlots { uint hz_block_slots[]; };\n\ layout(set = 1, binding = 1) readonly buffer HzSlotOrigins { vec4 hz_slot_origins[]; };\n", ); + // `mc_Entity` of each sprite layer: id, then 1 for fluids or -1 (the game's table). + declarations.push_str( + "layout(set = 3, binding = 0) readonly buffer HzBlockIds { int hz_block_ids[]; };\n", + ); "hz_slot_origins[hz_block_slots[uint(gl_VertexIndex) >> 8u]]" } else { declarations.push_str("layout(location = 5) in vec4 hz_v_origin;\n"); @@ -153,8 +157,13 @@ impl PackAtlas { scale[0], scale[1], ); declarations.push_str("vec2 hz_cell; vec2 hz_scale;\n"); + let entity = if flat { + "vec4(float(hz_block_ids[2u * hz_layer]), float(hz_block_ids[2u * hz_layer + 1u]), 0.0, 0.0)" + } else { + "vec4(-1.0, -1.0, 0.0, 0.0)" + }; let attributes = [ - ("mc_Entity", "vec4(-1.0, -1.0, 0.0, 0.0)"), + ("mc_Entity", entity), ( "mc_midTexCoord", "vec4((hz_cell + 0.5) * hz_scale, 0.0, 1.0)", diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 9002ad963..ba8fe62e7 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -154,6 +154,8 @@ struct Pass { draw: bool, /// What `texture` reads instead of the block atlas. atlas: Option, + /// Reads `mc_Entity`'s table (`PackPost::block_ids`). + ids: bool, groups: HashMap, } @@ -184,6 +186,8 @@ pub(super) struct World<'a> { pub lightmap: wgpu::TextureView, /// Terrain's geometry, when the pack draws it. pub terrain: Option>, + /// `mc_Entity` of each block sprite layer: (id, 1 for fluids or -1) pairs. + pub block_ids: &'a [i32], /// The atlas of the sun and moon's sprites. pub celestial: Option, } @@ -207,6 +211,8 @@ pub struct PackPost { down: HashMap, /// The per-draw uniforms of gbuffers programs. draws: DrawUniforms, + /// `mc_Entity`'s table of flat terrain, and its bind group. + block_ids: Option<(wgpu::BindGroupLayout, wgpu::BindGroup)>, /// `HORIZON_SHADER_PACK_TIMING`: GPU time of each pass, in the log. timing: Option, /// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`. @@ -492,6 +498,42 @@ fn attach( }) } +/// `mc_Entity`'s table as a storage buffer (vertex stage) at set 3; -1 pairs when empty. +fn block_ids(device: &wgpu::Device, table: &[i32]) -> (wgpu::BindGroupLayout, wgpu::BindGroup) { + let empty = [-1, -1]; + let table = if table.is_empty() { &empty[..] } else { table }; + let buffer = wgpu::util::DeviceExt::create_buffer_init( + device, + &wgpu::util::BufferInitDescriptor { + label: Some("Shader pack block ids"), + contents: bytemuck::cast_slice(table), + usage: wgpu::BufferUsages::STORAGE, + }, + ); + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Shader pack block ids"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }], + }); + let group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Shader pack block ids"), + layout: &layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: buffer.as_entire_binding(), + }], + }); + (layout, group) +} + /// The game's geometry a gbuffers program draws: its vertex buffers and, for flat draws /// (flat.rs), the layout of their bind group 1. pub(super) struct Geometry<'a> { @@ -501,6 +543,8 @@ pub(super) struct Geometry<'a> { pub depth_write: bool, /// What `texture` reads instead of the block atlas. pub atlas: Option, + /// `mc_Entity`'s table at set 3 (flat terrain). + pub ids: Option<&'a wgpu::BindGroupLayout>, } /// The pass of program `p`: a quad, or `geometry` drawn by the game. `None` (with a note) when @@ -663,6 +707,7 @@ fn build_pass( Some(&layout), geometry.and_then(|g| g.table), draw.then_some(draws), + geometry.and_then(|g| g.ids), ], ..Default::default() }); @@ -745,6 +790,7 @@ fn build_pass( mipmaps: p.mipmaps, draw, atlas: geometry.and_then(|g| g.atlas.clone()), + ids: geometry.is_some_and(|g| g.ids.is_some()), groups: HashMap::new(), }) } @@ -796,13 +842,26 @@ impl PackPost { used |= pass.reads | pass.written | pass.flip; Some(pass) }; + // Flat terrain reads `mc_Entity` from a table of the sprite layers. + let block_ids = world + .terrain + .as_ref() + .filter(|g| g.table.is_some()) + .map(|_| block_ids(device, world.block_ids)); + let terrain_geometry = world.terrain.as_ref().map(|g| Geometry { + buffers: g.buffers, + table: g.table, + depth_write: g.depth_write, + atlas: g.atlas.clone(), + ids: block_ids.as_ref().map(|i| &i.0), + }); let terrain = post .terrain - .zip(world.terrain.as_ref()) + .zip(terrain_geometry.as_ref()) .and_then(|(p, g)| build(p, Some(g))); let water = post .water - .zip(world.terrain.as_ref()) + .zip(terrain_geometry.as_ref()) .and_then(|(p, g)| build(p, Some(g))); let sky = (post.sky.is_some() || post.sky_textured.is_some()).then(|| { let geometry = PackSky::new(device); @@ -811,6 +870,7 @@ impl PackPost { table: Some(&geometry.layout), depth_write: false, atlas: None, + ids: None, }; let basic = post.sky.and_then(|p| build(p, Some(&basic))); let textured = Geometry { @@ -818,6 +878,7 @@ impl PackPost { table: None, depth_write: false, atlas: world.celestial.clone(), + ids: None, }; let textured = post.sky_textured.and_then(|p| build(p, Some(&textured))); Sky { @@ -981,6 +1042,7 @@ impl PackPost { mips_valid: 0, down, draws, + block_ids, timing: super::pack_timing::PassTiming::new(device, queue), custom, atlas: world.atlas.as_ref().map(|a| a.0.clone()), @@ -1216,6 +1278,11 @@ impl PackPost { let mut rp = self.begin(encoder, &pass); rp.set_pipeline(&pass.pipeline); rp.set_bind_group(0, &group, &[]); + if pass.ids + && let Some((_, ids)) = &self.block_ids + { + rp.set_bind_group(3, ids, &[]); + } if pass.draw { let stage = if translucent { "TERRAIN_TRANSLUCENT" diff --git a/crates/horizon-shaders/src/block_ids.rs b/crates/horizon-shaders/src/block_ids.rs new file mode 100644 index 000000000..623473283 --- /dev/null +++ b/crates/horizon-shaders/src/block_ids.rs @@ -0,0 +1,120 @@ +//! `block.properties` (spec 4.1): `block.= ...`, each block +//! `[namespace:]name` with optional state filters `:=,`. Values in one filter +//! are alternatives; separate filters must all match. Names the game does not know are simply +//! never asked for. Block tags (`%tag`, tier E) are not read yet. + +use crate::Properties; + +#[derive(Clone, Debug, PartialEq)] +struct Rule { + id: i32, + /// `minecraft:stone`. + name: String, + filters: Vec<(String, Vec)>, +} + +/// The ids `block.properties` gives block states. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct BlockIds { + rules: Vec, + /// Lines skipped and why. + pub notes: Vec, +} + +impl BlockIds { + pub fn parse(properties: &Properties) -> Self { + let mut out = Self::default(); + for e in &properties.entries { + let Some(id) = e.key.strip_prefix("block.") else { + continue; + }; + let Ok(id) = id.parse::() else { + // `layer.*` (4.2) and other keys are not ids. + continue; + }; + for word in e.value.split_whitespace() { + if word.starts_with('%') { + out.notes.push(format!( + "line {}: block tags are not read yet: {word}", + e.line + )); + continue; + } + let (names, filters): (Vec<&str>, Vec<&str>) = + word.split(':').partition(|p| !p.contains('=')); + let name = match names[..] { + [name] => format!("minecraft:{name}"), + [namespace, name] => format!("{namespace}:{name}"), + _ => { + out.notes + .push(format!("line {}: not a block name: {word}", e.line)); + continue; + } + }; + let filters = filters + .iter() + .filter_map(|f| f.split_once('=')) + .map(|(k, v)| (k.to_owned(), v.split(',').map(str::to_owned).collect())) + .collect(); + out.rules.push(Rule { id, name, filters }); + } + } + out + } + + /// The id of block `name` (`minecraft:stone`) in the state with `properties`. The rule with + /// the most filters that all match wins, then the first in the file (one name belongs to + /// one id; a filtered entry is the more specific one). + pub fn id<'a>(&self, name: &str, properties: impl Fn(&str) -> Option<&'a str>) -> Option { + self.rules + .iter() + .filter(|r| r.name == name) + .filter(|r| { + r.filters + .iter() + .all(|(k, vs)| properties(k).is_some_and(|v| vs.iter().any(|x| x == v))) + }) + .min_by_key(|r| std::cmp::Reverse(r.filters.len())) + .map(|r| r.id) + } + + pub fn is_empty(&self) -> bool { + self.rules.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Macros; + + #[test] + fn names_states_and_precedence() { + let text = "block.10=stone minecraft:dirt mod:thing \\\n grass_block:snowy=false\n\ + block.11=grass_block\nblock.12=oak_leaves:persistent=true:distance=1,2 %leaves\n\ + layer.translucent=glass\n"; + let ids = BlockIds::parse(&Properties::parse(text, &Macros::standard())); + let none = |_: &str| None; + assert_eq!(ids.id("minecraft:stone", none), Some(10)); + assert_eq!(ids.id("mod:thing", none), Some(10)); + assert_eq!(ids.id("minecraft:glass", none), None); + let snowy = |s: &'static str| move |k: &str| (k == "snowy").then_some(s); + // The filtered entry is the more specific one. + assert_eq!(ids.id("minecraft:grass_block", snowy("false")), Some(10)); + assert_eq!(ids.id("minecraft:grass_block", snowy("true")), Some(11)); + let leaves = |p: &'static str, d: &'static str| { + move |k: &str| match k { + "persistent" => Some(p), + "distance" => Some(d), + _ => None, + } + }; + assert_eq!( + ids.id("minecraft:oak_leaves", leaves("true", "2")), + Some(12) + ); + assert_eq!(ids.id("minecraft:oak_leaves", leaves("true", "3")), None); + assert_eq!(ids.id("minecraft:oak_leaves", leaves("false", "1")), None); + assert_eq!(ids.notes.len(), 1, "{:?}", ids.notes); + } +} diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index f33dfec57..fbad9bee0 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -12,6 +12,7 @@ //! - [`load_post`]: a dimension's `prepare`, `deferred`, `composite` and `final` programs, in order (2.1). //! - [`Macros`] and [`eval_condition`]: the macros packs test, and the `#if` arithmetic (1.5). +mod block_ids; mod condition; mod config; mod custom; @@ -25,6 +26,7 @@ mod preprocess; mod properties; mod translate; +pub use block_ids::BlockIds; pub use condition::eval_condition; pub use config::{BUFFERS, Buffer, Config, Format, Kind, buffer_index, draw_buffers}; pub use custom::{Custom, Val}; diff --git a/crates/horizon-shaders/tests/packs/terrain-test/README.md b/crates/horizon-shaders/tests/packs/terrain-test/README.md index 7d2c168cb..13745ed66 100644 --- a/crates/horizon-shaders/tests/packs/terrain-test/README.md +++ b/crates/horizon-shaders/tests/packs/terrain-test/README.md @@ -3,7 +3,7 @@ Horizon's own shader pack for testing gbuffers terrain (written for Horizon, no other pack's code). `gbuffers_terrain` draws blocks from the atlas with their vertex colour and lightmap, tinted by the lightmap coordinate's sky light (cooler where the sky is hidden); `mc_midTexCoord` marks each -face's centre with a dot, and any stage but solid terrain (translucent terrain falls back to it) +face's centre with a dot; leaves (id 1 in `block.properties`) are yellow, and any stage but solid terrain (translucent terrain falls back to it) is red. `gbuffers_skybasic` draws the sky in its vertex colour, a little greener, with magenta stars and a dark red void; `gbuffers_textured` (the fallback of `gbuffers_skytextured`) the sun, and the moon tinted green. `composite` puts the depth buffer's view-space distance into the blue diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/block.properties b/crates/horizon-shaders/tests/packs/terrain-test/shaders/block.properties new file mode 100644 index 000000000..cd08c6a16 --- /dev/null +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/block.properties @@ -0,0 +1,3 @@ +# Leaves draw yellow in gbuffers_terrain. +block.1=oak_leaves spruce_leaves birch_leaves jungle_leaves acacia_leaves dark_oak_leaves \ + mangrove_leaves cherry_leaves azalea_leaves flowering_azalea_leaves pale_oak_leaves diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh index 8be409929..4f6850575 100644 --- a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.fsh @@ -9,6 +9,7 @@ varying vec2 texcoord; varying vec2 lmcoord; varying vec2 mid; varying vec4 glcolor; +varying float blockId; void main() { vec4 color = texture2D(texture, texcoord) * glcolor; @@ -16,6 +17,8 @@ void main() { color.rgb *= texture2D(lightmap, lmcoord).rgb; // Less sky light: bluer. color.rgb = mix(color.rgb * vec3(0.6, 0.7, 1.2), color.rgb, lmcoord.y); + // Leaves (block.properties' id 1) yellow. + if (abs(blockId - 1.0) < 0.5) color.rgb = vec3(color.g, color.g, 0.0); // A dot at the face's centre, one texel wide. if (length((texcoord - mid) * vec2(atlasSize)) < 1.0) color.rgb = vec3(1.0, 0.0, 1.0); // Any other stage than solid terrain (translucent terrain falls back here): red. diff --git a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.vsh b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.vsh index bc16f4aad..1f26fd658 100644 --- a/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.vsh +++ b/crates/horizon-shaders/tests/packs/terrain-test/shaders/gbuffers_terrain.vsh @@ -1,11 +1,13 @@ #version 120 attribute vec4 mc_midTexCoord; +attribute vec4 mc_Entity; varying vec2 texcoord; varying vec2 lmcoord; varying vec2 mid; varying vec4 glcolor; +varying float blockId; void main() { gl_Position = ftransform(); @@ -13,4 +15,5 @@ void main() { lmcoord = (gl_TextureMatrix[1] * gl_MultiTexCoord1).xy; mid = mc_midTexCoord.xy; glcolor = gl_Color; + blockId = mc_Entity.x; } From 732efe69bbe9d05c72f9d9dc9e92de920f37388b Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:16:38 +0100 Subject: [PATCH 14/61] Shader packs, clean room 25: terrain normals and tangents Flat terrain's vertex stage reads its quad's first three corners from the arena page (bound per page at set 3, beside mc_Entity's table) and gives gl_Normal / vaNormal the face normal and at_tangent the tangent along the texture's u, with w the handedness of bitangent = cross(tangent, normal) (spec 5.1). The pack's terrain and water pipelines cull back faces as the game's own terrain does (clockwise front faces, rows being flipped to GL's): water's surface no longer shows its downward copy. Flat draws call a hook per arena page (Flat::draw_with); the arena lists its page buffers. Clean-room: implemented from cleanroom/shader-packs-spec.md revision 1, 5.1; no other loader's code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx.rs | 61 ++++++----- crates/horizon-client/src/gfx/arena.rs | 5 + crates/horizon-client/src/gfx/flat.rs | 12 +++ crates/horizon-client/src/gfx/pack_atlas.rs | 41 +++++++- crates/horizon-client/src/gfx/pack_post.rs | 106 +++++++++++++------- 5 files changed, 162 insertions(+), 63 deletions(-) diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 9b7fcc303..f829b9eb1 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -2799,6 +2799,7 @@ impl Renderer { depth_write: true, atlas: None, ids: None, + cull: true, }), block_ids, celestial: self @@ -4070,31 +4071,42 @@ impl Renderer { { let size = self.world_post.size().unwrap_or([1, 1]); let flat = self.flat.as_ref().filter(|_| flat_frame); - let mut terrain = |rp: &mut wgpu::RenderPass<'_>, translucent: bool| { - if !pack_draws_terrain { - return; - } - match flat { - Some(flat) => { - let calls = if translucent { - &flat.calls[flat.solid_calls..] - } else { - &flat.calls[..flat.solid_calls] - }; - flat.draw(rp, calls, |p| self.arena.buffer(p)); - } - None => { - rp.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32); - rp.set_vertex_buffer(1, self.instances.slice(..)); - let (first, count) = if translucent { - (solid_draws, translucent_draws) - } else { - (0, solid_draws) - }; - self.draw_sections(rp, &self.indirect, 0, first, count); - } - } + let pages: Vec<(u32, wgpu::Buffer)> = match flat { + Some(_) if pack_draws_terrain => self + .arena + .page_buffers() + .map(|(p, b)| (p, b.clone())) + .collect(), + _ => Vec::new(), }; + let mut terrain = + |rp: &mut wgpu::RenderPass<'_>, + translucent: bool, + page: &dyn Fn(&mut wgpu::RenderPass<'_>, u32)| { + if !pack_draws_terrain { + return; + } + match flat { + Some(flat) => { + let calls = if translucent { + &flat.calls[flat.solid_calls..] + } else { + &flat.calls[..flat.solid_calls] + }; + flat.draw_with(rp, calls, |p| self.arena.buffer(p), page); + } + None => { + rp.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32); + rp.set_vertex_buffer(1, self.instances.slice(..)); + let (first, count) = if translucent { + (solid_draws, translucent_draws) + } else { + (0, solid_draws) + }; + self.draw_sections(rp, &self.indirect, 0, first, count); + } + } + }; post.run( &self.device, &self.queue, @@ -4106,6 +4118,7 @@ impl Renderer { &frame.pack, size, [self.config.width, self.config.height], + &pages, &mut terrain, ); self.pack_post = Some((dimension, Some(post))); diff --git a/crates/horizon-client/src/gfx/arena.rs b/crates/horizon-client/src/gfx/arena.rs index dbbf70140..34b4d332d 100644 --- a/crates/horizon-client/src/gfx/arena.rs +++ b/crates/horizon-client/src/gfx/arena.rs @@ -155,6 +155,11 @@ impl Arena { pub fn has_page(&self, page: u32) -> bool { self.pages.get(page as usize).is_some_and(Option::is_some) } + /// Each live page's buffer, by page. + pub fn page_buffers(&self) -> impl Iterator { + let pages = self.pages.iter().enumerate(); + pages.filter_map(|(i, p)| Some((i as u32, &p.as_ref()?.buffer))) + } pub fn buffer(&self, page: u32) -> &wgpu::Buffer { &self.pages[page as usize] .as_ref() diff --git a/crates/horizon-client/src/gfx/flat.rs b/crates/horizon-client/src/gfx/flat.rs index 4c337025a..d167e3f3c 100644 --- a/crates/horizon-client/src/gfx/flat.rs +++ b/crates/horizon-client/src/gfx/flat.rs @@ -649,10 +649,22 @@ impl Flat { pass: &mut wgpu::RenderPass<'_>, calls: &[Call], vertices: impl Fn(u32) -> &'a wgpu::Buffer, + ) { + self.draw_with(pass, calls, vertices, |_, _| {}); + } + /// `draw`, calling `page` before each run of calls from one arena page (a shader pack binds + /// that page's vertices for reading). + pub fn draw_with<'a>( + &self, + pass: &mut wgpu::RenderPass<'_>, + calls: &[Call], + vertices: impl Fn(u32) -> &'a wgpu::Buffer, + page: impl Fn(&mut wgpu::RenderPass<'_>, u32), ) { pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32); for call in calls { self.bind_page(pass, call.page, &vertices); + page(pass, call.page); pass.draw_indexed(call.first..call.first + call.count, 0, 0..1); } } diff --git a/crates/horizon-client/src/gfx/pack_atlas.rs b/crates/horizon-client/src/gfx/pack_atlas.rs index ac56985fd..e1727f1fe 100644 --- a/crates/horizon-client/src/gfx/pack_atlas.rs +++ b/crates/horizon-client/src/gfx/pack_atlas.rs @@ -113,6 +113,30 @@ impl PackAtlas { } } +/// GLSL: the face normal and tangent of a flat-drawn vertex, from its quad's first three corners +/// (quads are four consecutive vertices of the page; position words 0-2, UV word 3). +const FACE: &str = " uint hz_q = (uint(gl_VertexIndex) & ~3u) * 6u;\n \ + vec3 hz_p0 = uintBitsToFloat(uvec3(hz_vertices[hz_q], hz_vertices[hz_q + 1u], hz_vertices[hz_q + 2u]));\n \ + vec3 hz_p1 = uintBitsToFloat(uvec3(hz_vertices[hz_q + 6u], hz_vertices[hz_q + 7u], hz_vertices[hz_q + 8u]));\n \ + vec3 hz_p2 = uintBitsToFloat(uvec3(hz_vertices[hz_q + 12u], hz_vertices[hz_q + 13u], hz_vertices[hz_q + 14u]));\n \ + uvec3 hz_w = uvec3(hz_vertices[hz_q + 3u], hz_vertices[hz_q + 9u], hz_vertices[hz_q + 15u]);\n \ + vec2 hz_t0 = vec2(hz_w.x & 65535u, hz_w.x >> 16u);\n \ + vec2 hz_d1 = vec2(hz_w.y & 65535u, hz_w.y >> 16u) - hz_t0;\n \ + vec2 hz_d2 = vec2(hz_w.z & 65535u, hz_w.z >> 16u) - hz_t0;\n \ + vec3 hz_e1 = hz_p1 - hz_p0;\n \ + vec3 hz_e2 = hz_p2 - hz_p0;\n \ + vec3 hz_n = cross(hz_e1, hz_e2);\n \ + if (dot(hz_n, hz_n) > 1e-12) {\n \ + hz_Normal = normalize(hz_n);\n \ + float hz_r = hz_d1.x * hz_d2.y - hz_d2.x * hz_d1.y;\n \ + vec3 hz_t = (hz_e1 * hz_d2.y - hz_e2 * hz_d1.y) * sign(hz_r);\n \ + vec3 hz_b = (hz_e2 * hz_d1.x - hz_e1 * hz_d2.x) * sign(hz_r);\n \ + if (dot(hz_t, hz_t) > 1e-12) {\n \ + hz_t = normalize(hz_t - hz_Normal * dot(hz_Normal, hz_t));\n \ + hz_tangent = vec4(hz_t, dot(cross(hz_t, hz_Normal), hz_b) < 0.0 ? -1.0 : 1.0);\n \ + }\n \ + }\n"; + impl PackAtlas { /// The vertex inputs of a gbuffers program drawing terrain (`TerrainVertex`, terrain.wgsl's /// `vs_terrain` or, for `flat` draws, `vs_terrain_flat`), in atlas coordinates. @@ -127,9 +151,11 @@ impl PackAtlas { "layout(set = 1, binding = 0) readonly buffer HzBlockSlots { uint hz_block_slots[]; };\n\ layout(set = 1, binding = 1) readonly buffer HzSlotOrigins { vec4 hz_slot_origins[]; };\n", ); - // `mc_Entity` of each sprite layer: id, then 1 for fluids or -1 (the game's table). + // `mc_Entity` of each sprite layer: id, then 1 for fluids or -1 (the game's table), + // and the arena page's vertices (`TerrainVertex`, six words each). declarations.push_str( - "layout(set = 3, binding = 0) readonly buffer HzBlockIds { int hz_block_ids[]; };\n", + "layout(set = 3, binding = 0) readonly buffer HzBlockIds { int hz_block_ids[]; };\n\ + layout(set = 3, binding = 1) readonly buffer HzVertices { uint hz_vertices[]; };\n", ); "hz_slot_origins[hz_block_slots[uint(gl_VertexIndex) >> 8u]]" } else { @@ -153,10 +179,15 @@ impl PackAtlas { hz_MultiTexCoord1 = vec4(float(hz_v_light.y), float(hz_v_light.x), 0.0, 1.0);\n \ hz_MultiTexCoord2 = hz_MultiTexCoord1;\n \ hz_Color = hz_v_color;\n \ - hz_Normal = vec3(0.0, 1.0, 0.0);\n", + hz_Normal = vec3(0.0, 1.0, 0.0);\n \ + hz_tangent = vec4(1.0, 0.0, 0.0, 1.0);\n", scale[0], scale[1], ); - declarations.push_str("vec2 hz_cell; vec2 hz_scale;\n"); + let mut body = body; + if flat { + body.push_str(FACE); + } + declarations.push_str("vec2 hz_cell; vec2 hz_scale; vec4 hz_tangent;\n"); let entity = if flat { "vec4(float(hz_block_ids[2u * hz_layer]), float(hz_block_ids[2u * hz_layer + 1u]), 0.0, 0.0)" } else { @@ -168,7 +199,7 @@ impl PackAtlas { "mc_midTexCoord", "vec4((hz_cell + 0.5) * hz_scale, 0.0, 1.0)", ), - ("at_tangent", "vec4(1.0, 0.0, 0.0, 1.0)"), + ("at_tangent", "hz_tangent"), ("vaPosition", "hz_Vertex.xyz"), ("vaUV0", "hz_MultiTexCoord0.xy"), ("vaUV2", "hz_MultiTexCoord1.xy"), diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index ba8fe62e7..972ac82f3 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -211,8 +211,10 @@ pub struct PackPost { down: HashMap, /// The per-draw uniforms of gbuffers programs. draws: DrawUniforms, - /// `mc_Entity`'s table of flat terrain, and its bind group. - block_ids: Option<(wgpu::BindGroupLayout, wgpu::BindGroup)>, + /// Set 3 of flat terrain (`mc_Entity`'s table) and its bind group for each arena page, + /// with the page buffer it was made for. + block_ids: Option<(wgpu::BindGroupLayout, wgpu::Buffer)>, + page_groups: HashMap, /// `HORIZON_SHADER_PACK_TIMING`: GPU time of each pass, in the log. timing: Option, /// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`. @@ -498,8 +500,14 @@ fn attach( }) } -/// `mc_Entity`'s table as a storage buffer (vertex stage) at set 3; -1 pairs when empty. -fn block_ids(device: &wgpu::Device, table: &[i32]) -> (wgpu::BindGroupLayout, wgpu::BindGroup) { +/// Draws the game's terrain (`true`: translucent) into a pass; calls its third argument with +/// each arena page before drawing from it. +pub(super) type TerrainDraw<'a> = + dyn FnMut(&mut wgpu::RenderPass<'_>, bool, &dyn Fn(&mut wgpu::RenderPass<'_>, u32)) + 'a; + +/// Set 3 of flat terrain, read by the vertex stage: `mc_Entity`'s table (-1 pairs when empty) +/// and, per arena page, the page's vertices (a vertex reads its quad's corners for its normal). +fn block_ids(device: &wgpu::Device, table: &[i32]) -> (wgpu::BindGroupLayout, wgpu::Buffer) { let empty = [-1, -1]; let table = if table.is_empty() { &empty[..] } else { table }; let buffer = wgpu::util::DeviceExt::create_buffer_init( @@ -510,28 +518,21 @@ fn block_ids(device: &wgpu::Device, table: &[i32]) -> (wgpu::BindGroupLayout, wg usage: wgpu::BufferUsages::STORAGE, }, ); + let storage = |binding| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::VERTEX, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }; let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("Shader pack block ids"), - entries: &[wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::VERTEX, - ty: wgpu::BindingType::Buffer { - ty: wgpu::BufferBindingType::Storage { read_only: true }, - has_dynamic_offset: false, - min_binding_size: None, - }, - count: None, - }], + label: Some("Shader pack terrain"), + entries: &[storage(0), storage(1)], }); - let group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("Shader pack block ids"), - layout: &layout, - entries: &[wgpu::BindGroupEntry { - binding: 0, - resource: buffer.as_entire_binding(), - }], - }); - (layout, group) + (layout, buffer) } /// The game's geometry a gbuffers program draws: its vertex buffers and, for flat draws @@ -545,6 +546,8 @@ pub(super) struct Geometry<'a> { pub atlas: Option, /// `mc_Entity`'s table at set 3 (flat terrain). pub ids: Option<&'a wgpu::BindGroupLayout>, + /// Back faces are culled, as for the game's own terrain. + pub cull: bool, } /// The pass of program `p`: a quad, or `geometry` drawn by the game. `None` (with a note) when @@ -728,7 +731,12 @@ fn build_pass( compilation_options: Default::default(), buffers: geometry.map_or(&[], |g| g.buffers), }, - primitive: Default::default(), + // Rows flipped to GL's turn front faces clockwise. + primitive: wgpu::PrimitiveState { + front_face: wgpu::FrontFace::Cw, + cull_mode: geometry.filter(|g| g.cull).map(|_| wgpu::Face::Back), + ..Default::default() + }, depth_stencil: geometry.map(|g| wgpu::DepthStencilState { format: wgpu::TextureFormat::Depth32Float, depth_write_enabled: Some(g.depth_write), @@ -854,6 +862,7 @@ impl PackPost { depth_write: g.depth_write, atlas: g.atlas.clone(), ids: block_ids.as_ref().map(|i| &i.0), + cull: g.cull, }); let terrain = post .terrain @@ -871,6 +880,7 @@ impl PackPost { depth_write: false, atlas: None, ids: None, + cull: false, }; let basic = post.sky.and_then(|p| build(p, Some(&basic))); let textured = Geometry { @@ -879,6 +889,7 @@ impl PackPost { depth_write: false, atlas: world.celestial.clone(), ids: None, + cull: false, }; let textured = post.sky_textured.and_then(|p| build(p, Some(&textured))); Sky { @@ -1043,6 +1054,7 @@ impl PackPost { down, draws, block_ids, + page_groups: HashMap::new(), timing: super::pack_timing::PassTiming::new(device, queue), custom, atlas: world.atlas.as_ref().map(|a| a.0.clone()), @@ -1253,7 +1265,7 @@ impl PackPost { encoder: &mut wgpu::CommandEncoder, size: [u32; 2], translucent: bool, - draw: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), + draw: &mut TerrainDraw<'_>, ) { let slot = if translucent { &mut self.water @@ -1278,11 +1290,6 @@ impl PackPost { let mut rp = self.begin(encoder, &pass); rp.set_pipeline(&pass.pipeline); rp.set_bind_group(0, &group, &[]); - if pass.ids - && let Some((_, ids)) = &self.block_ids - { - rp.set_bind_group(3, ids, &[]); - } if pass.draw { let stage = if translucent { "TERRAIN_TRANSLUCENT" @@ -1291,7 +1298,13 @@ impl PackPost { }; self.draws.bind(&mut rp, stage); } - draw(&mut rp, translucent); + let (groups, ids) = (&self.page_groups, pass.ids); + let page = |rp: &mut wgpu::RenderPass<'_>, page: u32| { + if let Some((_, group)) = groups.get(&page).filter(|_| ids) { + rp.set_bind_group(3, group, &[]); + } + }; + draw(&mut rp, translucent, &page); } self.mips_valid &= !pass.written; if translucent { @@ -1362,7 +1375,7 @@ impl PackPost { encoder: &mut wgpu::CommandEncoder, size: [u32; 2], frame: &super::Frame, - terrain: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), + terrain: &mut TerrainDraw<'_>, ) { if super::sky_visible(frame) { self.draw_sky(device, queue, encoder, size, frame); @@ -1533,9 +1546,34 @@ impl PackPost { inputs: &PackInputs, size: [u32; 2], window: [u32; 2], - terrain: &mut dyn FnMut(&mut wgpu::RenderPass<'_>, bool), + pages: &[(u32, wgpu::Buffer)], + terrain: &mut TerrainDraw<'_>, ) { self.resize(device, size); + if let Some((layout, ids)) = &self.block_ids { + // A page that grew has a new buffer. + self.page_groups + .retain(|p, (b, _)| pages.iter().any(|(q, c)| q == p && c == b)); + for (page, buffer) in pages { + self.page_groups.entry(*page).or_insert_with(|| { + let group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Shader pack terrain"), + layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: ids.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: buffer.as_entire_binding(), + }, + ], + }); + (buffer.clone(), group) + }); + } + } if let Some(t) = &mut self.timing { t.begin(device, queue); } From bf59c0c4f7c532af62056a413db921adfc44885e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 06:11:52 +0100 Subject: [PATCH 15/61] Shader packs, clean room 26: biome uniforms and constants Custom uniform expressions and programs now see the biome at the camera: `biome`, `biome_category`, `biome_precipitation`, `temperature` and `rainfall`, and the `BIOME_*`, `CAT_*` and `PPT_*` constants (spec 3.4). Before, `biome` and every `BIOME_*` read 0, so a test such as `biome == BIOME_PALE_GARDEN` held everywhere: Complementary drew the pale garden's fog over every world, and BSL treated every biome as cold, dry, swampy and so on at once. OQ-11: `biome` is the index in the game's biome registry (vanilla's biomes sorted by identifier); names from before the 1.18 renames map to their successors; a `BIOME_` name that matches no biome reads -1. Co-Authored-By: Claude Opus 5.5 --- .../horizon-client/src/game/shader_packs.rs | 18 +- .../horizon-client/src/gfx/pack_uniforms.rs | 16 ++ crates/horizon-shaders/src/biomes.rs | 159 ++++++++++++++++++ crates/horizon-shaders/src/custom.rs | 55 +++++- crates/horizon-shaders/src/lib.rs | 2 + 5 files changed, 242 insertions(+), 8 deletions(-) create mode 100644 crates/horizon-shaders/src/biomes.rs diff --git a/crates/horizon-client/src/game/shader_packs.rs b/crates/horizon-client/src/game/shader_packs.rs index 7fa2ea3f6..f5f7a5fba 100644 --- a/crates/horizon-client/src/game/shader_packs.rs +++ b/crates/horizon-client/src/game/shader_packs.rs @@ -43,7 +43,10 @@ impl Game<'_> { } let folder = dimension_folder(&pack, dimension); let inputs = renderer.pack_inputs(); - let post = load_post(&pack, &folder, &properties, ¯os, &inputs); + let mut post = load_post(&pack, &folder, &properties, ¯os, &inputs); + let biomes = self.res.biomes(); + let names: Vec<&str> = biomes.list.iter().map(|b| b.name.as_str()).collect(); + post.custom.set_biomes(&names); let text = pack.read_text("block.properties").unwrap_or_default(); let ids = BlockIds::parse(&Properties::parse(&text, ¯os)); for note in &ids.notes { @@ -80,6 +83,12 @@ impl Game<'_> { let lerp = |(old, new): (f32, f32)| old + (new - old) * partial; let at = camera.floor().as_ivec3(); let light = self.world.light(at.x, at.y, at.z); + let biome = self + .world + .chunks + .get(&(at.x >> 4, at.z >> 4)) + .map(|c| c.biome(at.x, at.y, at.z)); + let info = biome.map(|b| self.res.biomes().get(b)); PackInputs { day_time: self.world.day_time, rain: lerp(self.world.weather.rain_level), @@ -94,6 +103,13 @@ impl Game<'_> { eye_light: [i32::from(light & 15), i32::from(light >> 4)], hide_gui: self.ui.hide_gui, brightness: self.settings.gamma, + biome: biome.map_or(-1, i32::from), + biome_category: info.map_or(0, |b| horizon_shaders::biome_category(&b.name)), + biome_precipitation: info.map_or(0, |b| { + horizon_shaders::biome_precipitation(b.precipitation, b.temperature) + }), + temperature: info.map_or(0.0, |b| b.temperature), + rainfall: info.map_or(0.0, |b| b.downfall), } } } diff --git a/crates/horizon-client/src/gfx/pack_uniforms.rs b/crates/horizon-client/src/gfx/pack_uniforms.rs index c7c4cfd8b..ad53b509a 100644 --- a/crates/horizon-client/src/gfx/pack_uniforms.rs +++ b/crates/horizon-client/src/gfx/pack_uniforms.rs @@ -23,6 +23,12 @@ pub struct PackInputs { pub hide_gui: bool, /// The brightness option, 0..1. pub brightness: f32, + /// The biome at the camera (3.4): registry id, `CAT_*`, `PPT_*`, temperature, downfall. + pub biome: i32, + pub biome_category: i32, + pub biome_precipitation: i32, + pub temperature: f32, + pub rainfall: f32, } /// One uniform's value: up to 16 floats or 3 integers. @@ -134,6 +140,11 @@ pub const NAMES: &[&str] = &[ "heldBlockLightValue", "heldBlockLightValue2", "renderStage", + "biome", + "biome_category", + "biome_precipitation", + "temperature", + "rainfall", "gbufferModelView", "gbufferModelViewInverse", "gbufferProjection", @@ -290,6 +301,11 @@ impl FrameUniforms { "screenBrightness" => Value::f(&[inputs.brightness]), "hideGUI" => Value::i(&[inputs.hide_gui as i32]), "eyeBrightness" => Value::i(&eye.map(|v| v as i32)), + "biome" => Value::i(&[inputs.biome]), + "biome_category" => Value::i(&[inputs.biome_category]), + "biome_precipitation" => Value::i(&[inputs.biome_precipitation]), + "temperature" => Value::f(&[inputs.temperature]), + "rainfall" => Value::f(&[inputs.rainfall]), "eyeBrightnessSmooth" => Value::i(&self.eye_smooth.map(|v| v.round() as i32)), "gbufferModelView" => m4(view), "gbufferModelViewInverse" => m4(view.inverse()), diff --git a/crates/horizon-shaders/src/biomes.rs b/crates/horizon-shaders/src/biomes.rs new file mode 100644 index 000000000..c9ede57e0 --- /dev/null +++ b/crates/horizon-shaders/src/biomes.rs @@ -0,0 +1,159 @@ +//! The biome parameters and constants of custom uniform expressions (spec 3.4): `BIOME_`, +//! `CAT_` and `PPT_`. +//! +//! Decisions (impl-plan.md, OQ-11): +//! - `biome` is the biome's index in the game's biome registry, which lists vanilla's biomes +//! sorted by identifier; `BIOME_` is the index of `minecraft:`. Names from before +//! the 1.18 renames map to the biome that replaced them. A `BIOME_` name that matches no biome +//! reads -1, so it never equals the player's biome. +//! - `biome_category` comes from the biome's name (categories left the game in 1.19). + +use std::collections::HashMap; + +/// `CAT_*`, in the order of their values. +pub const CATEGORIES: [&str; 17] = [ + "NONE", + "TAIGA", + "EXTREME_HILLS", + "JUNGLE", + "MESA", + "PLAINS", + "SAVANNA", + "ICY", + "THE_END", + "BEACH", + "FOREST", + "OCEAN", + "DESERT", + "RIVER", + "SWAMP", + "MUSHROOM", + "NETHER", +]; + +/// Old biome names and the biomes that replaced them. +const LEGACY: &[(&str, &str)] = &[ + ("EXTREME_HILLS", "windswept_hills"), + ("MOUNTAINS", "windswept_hills"), + ("GRAVELLY_MOUNTAINS", "windswept_gravelly_hills"), + ("WOODED_MOUNTAINS", "windswept_forest"), + ("SHATTERED_SAVANNA", "windswept_savanna"), + ("GIANT_TREE_TAIGA", "old_growth_pine_taiga"), + ("GIANT_SPRUCE_TAIGA", "old_growth_spruce_taiga"), + ("TALL_BIRCH_FOREST", "old_growth_birch_forest"), + ("SNOWY_TUNDRA", "snowy_plains"), + ("JUNGLE_EDGE", "sparse_jungle"), + ("WOODED_BADLANDS_PLATEAU", "wooded_badlands"), + ("STONE_SHORE", "stony_shore"), + ("NETHER", "nether_wastes"), +]; + +/// The constants every pack sees: `CAT_*` and `PPT_*`, plus `BIOME_*` for `biomes` (the +/// registry's identifiers, in id order). +pub fn constants(biomes: &[&str]) -> HashMap { + let mut out: HashMap = CATEGORIES + .iter() + .enumerate() + .map(|(i, c)| (format!("CAT_{c}"), i as f64)) + .collect(); + for (i, p) in ["NONE", "RAIN", "SNOW"].iter().enumerate() { + out.insert(format!("PPT_{p}"), i as f64); + } + let index = |name: &str| { + biomes + .iter() + .position(|b| b.strip_prefix("minecraft:") == Some(name)) + }; + for (i, b) in biomes.iter().enumerate() { + if let Some(name) = b.strip_prefix("minecraft:") { + out.insert(format!("BIOME_{}", name.to_ascii_uppercase()), i as f64); + } + } + for (old, new) in LEGACY { + if let Some(i) = index(new) { + out.entry(format!("BIOME_{old}")).or_insert(i as f64); + } + } + out +} + +/// `biome_category` of the biome `minecraft:`. +pub fn category(name: &str) -> i32 { + let name = name.strip_prefix("minecraft:").unwrap_or(name); + let has = |words: &[&str]| words.iter().any(|w| name.contains(w)); + let cat = if name == "the_void" { + "NONE" + } else if name == "the_end" || name.starts_with("end_") || name == "small_end_islands" { + "THE_END" + } else if has(&["nether", "crimson", "warped", "basalt", "soul_sand"]) { + "NETHER" + } else if has(&["ocean"]) { + "OCEAN" + } else if has(&["river"]) { + "RIVER" + } else if has(&["beach", "shore"]) { + "BEACH" + } else if has(&["swamp"]) { + "SWAMP" + } else if has(&["mushroom"]) { + "MUSHROOM" + } else if has(&["desert"]) { + "DESERT" + } else if has(&["badlands"]) { + "MESA" + } else if has(&["savanna"]) { + "SAVANNA" + } else if has(&["jungle"]) { + "JUNGLE" + } else if has(&["taiga"]) { + "TAIGA" + } else if has(&["windswept", "peaks", "slopes", "meadow", "grove"]) && name != "cherry_grove" { + "EXTREME_HILLS" + } else if has(&["frozen", "snowy", "ice"]) { + "ICY" + } else if has(&["forest", "garden", "cherry"]) { + "FOREST" + } else if has(&["plains"]) { + "PLAINS" + } else { + "NONE" + }; + CATEGORIES.iter().position(|c| *c == cat).unwrap_or(0) as i32 +} + +/// `biome_precipitation`: none where the biome has none, else rain from temperature 0.15 up, +/// snow below it (spec 3.4). +pub fn precipitation(has_precipitation: bool, temperature: f32) -> i32 { + match (has_precipitation, temperature >= 0.15) { + (false, _) => 0, + (true, true) => 1, + (true, false) => 2, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn constants_categories_and_precipitation() { + let c = constants(&[ + "minecraft:badlands", + "minecraft:pale_garden", + "minecraft:windswept_hills", + ]); + assert_eq!(c["BIOME_PALE_GARDEN"], 1.0); + assert_eq!(c["BIOME_EXTREME_HILLS"], 2.0); + assert_eq!(c["CAT_NETHER"], 16.0); + assert_eq!(c["PPT_SNOW"], 2.0); + assert!(!c.contains_key("BIOME_SULFUR_CAVES")); + assert_eq!(category("minecraft:pale_garden"), 10); + assert_eq!(category("minecraft:deep_frozen_ocean"), 11); + assert_eq!(category("minecraft:snowy_taiga"), 1); + assert_eq!(category("minecraft:crimson_forest"), 16); + assert_eq!(category("minecraft:cherry_grove"), 10); + assert_eq!(precipitation(true, 0.0), 2); + assert_eq!(precipitation(true, 0.8), 1); + assert_eq!(precipitation(false, 2.0), 0); + } +} diff --git a/crates/horizon-shaders/src/custom.rs b/crates/horizon-shaders/src/custom.rs index 34510247e..e27472816 100644 --- a/crates/horizon-shaders/src/custom.rs +++ b/crates/horizon-shaders/src/custom.rs @@ -87,6 +87,8 @@ pub struct Custom { smooth: HashMap, seed: u64, unknown: HashSet, + /// `BIOME_*`, `CAT_*` and `PPT_*` (crate::biomes). + constants: HashMap, /// Expressions that did not parse, and unknown names, as `key: message`. pub notes: Vec, calls: usize, @@ -97,6 +99,7 @@ impl Custom { pub fn parse(properties: &Properties) -> Self { let mut out = Custom { seed: 0x9e37_79b9_7f4a_7c15, + constants: crate::biomes::constants(&[]), ..Default::default() }; for e in &properties.entries { @@ -126,6 +129,11 @@ impl Custom { out } + /// Names the game's biomes (registry identifiers, in id order) for `BIOME_*`. + pub fn set_biomes(&mut self, biomes: &[&str]) { + self.constants = crate::biomes::constants(biomes); + } + /// The uniforms' names and declared types, in file order. pub fn uniforms(&self) -> impl Iterator { self.items @@ -189,8 +197,19 @@ impl Custom { "pi" => Some(Val::scalar(std::f64::consts::PI)), "true" => Some(Val::scalar(1.0)), "false" => Some(Val::scalar(0.0)), - n => vars.get(n).copied().or_else(|| builtin(n)), + n => vars + .get(n) + .copied() + .or_else(|| builtin(n)) + .or_else(|| self.constants.get(n).map(|&x| Val::scalar(x))), }; + if v.is_none() && name.starts_with("BIOME_") { + if self.unknown.insert(name.clone()) { + self.notes + .push(format!("`{name}` names no biome; it reads -1")); + } + return Val::scalar(-1.0); + } let Some(v) = v else { if self.unknown.insert(name.clone()) { self.notes.push(format!("`{name}` is unknown; it reads 0")); @@ -523,16 +542,18 @@ mod tests { uniform.vec3.e = vec3(a, cameraPosition.y, 2) * 2\n\ uniform.float.f = if(a < 0, 1, a < 7, 2, 3) + clamp(fmod(-1, 4), 0, 2.5) + min(4, 2, 3)\n\ uniform.float.g = gbufferModelView.1.0 + frac(sunAngle - 0.25) + pi * 0\n\ - uniform.float.h = unknownThing + in(biome, 3, BIOME_DESERT)\n", + uniform.float.h = unknownThing + in(biome, 3, BIOME_DESERT)\n\ + uniform.int.i = biome == BIOME_PALE_GARDEN || biome == BIOME_OLD\n\ + uniform.int.j = CAT_OCEAN + PPT_RAIN\n", ); + c.set_biomes(&["minecraft:beach", "minecraft:desert"]); assert!(c.notes.is_empty(), "{:?}", c.notes); let names: Vec<_> = c.uniforms().map(|u| u.0).collect(); - assert_eq!(names, ["b", "c", "d", "e", "f", "g", "h"]); + assert_eq!(names, ["b", "c", "d", "e", "f", "g", "h", "i", "j"]); let builtin = |n: &str| match n { "cameraPosition" => Some(Val::vector(&[1.0, 64.0, 2.0])), "sunAngle" => Some(Val::scalar(0.125)), - "biome" => Some(Val::scalar(5.0)), - "BIOME_DESERT" => Some(Val::scalar(5.0)), + "biome" => Some(Val::scalar(1.0)), "gbufferModelView" => { let mut m = Val::scalar(0.0); m.v[1] = 9.0; // column 0, row 1 @@ -543,9 +564,29 @@ mod tests { }; let v = c.eval(0.016, &builtin); let x: Vec = v.iter().map(|v| v.v[0]).collect(); - assert_eq!(x, [3.0, 1.0, -3.0, 12.0, 2.0 + 2.5 + 2.0, 9.0 + 0.875, 1.0]); + assert_eq!( + x, + [ + 3.0, + 1.0, + -3.0, + 12.0, + 2.0 + 2.5 + 2.0, + 9.0 + 0.875, + 1.0, + 0.0, + 12.0 + ] + ); assert_eq!(&v[3].v[..3], &[12.0, 128.0, 4.0]); - assert_eq!(c.notes, ["`unknownThing` is unknown; it reads 0"]); + assert_eq!( + c.notes, + [ + "`unknownThing` is unknown; it reads 0", + "`BIOME_PALE_GARDEN` names no biome; it reads -1", + "`BIOME_OLD` names no biome; it reads -1" + ] + ); } #[test] diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index fbad9bee0..8684be867 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -12,6 +12,7 @@ //! - [`load_post`]: a dimension's `prepare`, `deferred`, `composite` and `final` programs, in order (2.1). //! - [`Macros`] and [`eval_condition`]: the macros packs test, and the `#if` arithmetic (1.5). +mod biomes; mod block_ids; mod condition; mod config; @@ -26,6 +27,7 @@ mod preprocess; mod properties; mod translate; +pub use biomes::{category as biome_category, precipitation as biome_precipitation}; pub use block_ids::BlockIds; pub use condition::eval_condition; pub use config::{BUFFERS, Buffer, Config, Format, Kind, buffer_index, draw_buffers}; From 9a8453c61217599122f7569da3d60217c2996619 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 06:11:52 +0100 Subject: [PATCH 16/61] Shader packs, clean room 27: sample mip chains only while current A program sampled every level of a buffer that has mipmaps, even after a later program had rewritten the buffer: the levels below the first were stale. Complementary's FXAA, which samples its colour buffer with implicit derivatives inside branches, picked those levels at edges and drew black dots along every edge. A buffer now binds its whole chain only while the chain matches its content, and its first level otherwise (6.1.7). Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx/pack_post.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 972ac82f3..c3a67f22e 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -156,7 +156,9 @@ struct Pass { atlas: Option, /// Reads `mc_Entity`'s table (`PackPost::block_ids`). ids: bool, - groups: HashMap, + /// Bind groups by the sides read (low 16 bits) and the buffers whose mip chain is current + /// (high 16 bits). + groups: HashMap, } /// One side of a buffer with mipmaps. @@ -1152,7 +1154,10 @@ impl PackPost { queue.write_buffer(&pass.ubo, 0, &pass.block); } let targets = self.targets.as_ref().expect("resized"); - let key = self.main & pass.reads; + // A chain is sampled only while it is current: a buffer written since its last chain + // reads its first level, never stale levels (6.1.7). + let current = self.mips_valid & self.mipped & pass.reads; + let key = u32::from(self.main & pass.reads) | u32::from(current) << 16; let group = pass.groups.entry(key).or_insert_with(|| { let mut entries = vec![wgpu::BindGroupEntry { binding: 0, @@ -1163,9 +1168,9 @@ impl PackPost { Source::Color(i) => { let side = usize::from(key & 1 << i != 0); match (&targets.mips[i], &targets.color[i]) { - (Some(m), _) => &m[side].all, - (None, Some(t)) => &t[side], - (None, None) => &self.dummy, + (Some(m), _) if current & 1 << i != 0 => &m[side].all, + (_, Some(t)) => &t[side], + (_, None) => &self.dummy, } } Source::Depth => &targets.depth, From 7b35ca27998f246649869056389e57fb429f19aa Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 06:35:43 +0100 Subject: [PATCH 17/61] Shader packs, clean room 28: Sildurs Vibrant renders Three things kept Sildurs Vibrant black or wrong: - Matrix and array varyings (`varying mat3 tbnMatrix;`): wgpu's stage interfaces take only scalars and vectors, so the translator now carries each column or element as its own varying and copies them out of and into a private global around the pack's `main`. - Switch options defined in an included file whose name is not a shader extension (`#include "shaders.settings"`) were not read, so every `program..enabled` that tested them was false and the composites were skipped (OQ-13). - `fogStart`/`fogEnd` mixed the start of one of vanilla's two fogs with the end of the other, so on clear days fog began at the camera and Sildurs' water turned white. They now come as the pair of the fog that is complete nearer (OQ-36). Co-Authored-By: Claude Opus 5.5 --- .../horizon-client/src/gfx/pack_uniforms.rs | 16 ++- crates/horizon-shaders/src/options.rs | 45 +++++-- crates/horizon-shaders/src/translate/mod.rs | 116 +++++++++++++++++- 3 files changed, 155 insertions(+), 22 deletions(-) diff --git a/crates/horizon-client/src/gfx/pack_uniforms.rs b/crates/horizon-client/src/gfx/pack_uniforms.rs index ad53b509a..21cfc2b14 100644 --- a/crates/horizon-client/src/gfx/pack_uniforms.rs +++ b/crates/horizon-client/src/gfx/pack_uniforms.rs @@ -233,12 +233,18 @@ impl FrameUniforms { for (smoothed, target) in self.eye_smooth.iter_mut().zip(eye) { *smoothed = smooth(*smoothed, target, 10.0 / 20.0, dt); } + // Vanilla fades by the stronger of two linear fogs; a pack gets one: the pair (start + // and end together) of the one that is complete nearer (OQ-36). Mixing the nearer start + // of one with the nearer end of the other made fog begin at the camera on clear days. let (fog_color, fog_start, fog_end) = match &frame.fog { - Some(f) => ( - [f.color[0], f.color[1], f.color[2]], - f.environmental_start.min(f.render_distance_start), - f.environmental_end.min(f.render_distance_end), - ), + Some(f) => { + let (start, end) = if f.environmental_end < f.render_distance_end { + (f.environmental_start, f.environmental_end) + } else { + (f.render_distance_start, f.render_distance_end) + }; + ([f.color[0], f.color[1], f.color[2]], start, end) + } None => (frame.sky.fog_color, frame.distance * 0.9, frame.distance), }; // The shadow camera (OQ-6, impl-plan.md): orthographic, `shadowDistance` (160) blocks diff --git a/crates/horizon-shaders/src/options.rs b/crates/horizon-shaders/src/options.rs index 687fd40e0..752dc12a0 100644 --- a/crates/horizon-shaders/src/options.rs +++ b/crates/horizon-shaders/src/options.rs @@ -1,8 +1,9 @@ //! Switch options (spec 8.1), as far as program-enable expressions (3.3) need them: which are on //! by default. A switch is `#define NAME` (on) or `//#define NAME` (off) with nothing after the //! name but a comment, in any stage or included file of the pack. Packs keep their options in -//! included `.glsl` files and test them elsewhere (OQ-13), so the test the spec asks for -//! (`#ifdef NAME` / `#ifndef NAME`) may be in any file of the pack. +//! included `.glsl` files (or files of any name a source includes) and test them elsewhere +//! (OQ-13), so the test the spec asks for (`#ifdef NAME` / `#ifndef NAME`) may be in any file +//! of the pack. use std::collections::BTreeSet; @@ -13,14 +14,27 @@ const SOURCES: [&str; 8] = ["vsh", "fsh", "gsh", "csh", "tcs", "tes", "glsl", "i /// The switch options that are on by default. pub fn switches_on(pack: &Pack) -> BTreeSet { let (mut on, mut off, mut tested) = (BTreeSet::new(), BTreeSet::new(), BTreeSet::new()); - for path in pack.all_files() { - let ext = path.rsplit_once('.').map_or("", |(_, e)| e); - if !SOURCES.contains(&ext) { - continue; - } - let Some(text) = pack.read_text(&path) else { - continue; - }; + let files = pack.all_files(); + let source = |path: &str| SOURCES.contains(&path.rsplit_once('.').map_or("", |(_, e)| e)); + let texts: Vec<(&String, String)> = files + .iter() + .filter(|p| source(p)) + .filter_map(|p| Some((p, pack.read_text(p)?))) + .collect(); + // Files of other names count when a source includes them (`#include "shaders.settings"`). + let included: BTreeSet<&str> = texts + .iter() + .flat_map(|(_, t)| t.lines()) + .filter_map(|l| l.trim().strip_prefix("#include")) + .map(|r| r.trim().trim_matches(|c| c == '"' || c == '<' || c == '>')) + .map(|r| r.rsplit('/').next().unwrap_or(r)) + .collect(); + let extra: Vec<(&String, String)> = files + .iter() + .filter(|p| !source(p) && included.contains(p.rsplit('/').next().unwrap_or(p))) + .filter_map(|p| Some((p, pack.read_text(p)?))) + .collect(); + for (_, text) in texts.iter().chain(&extra) { for line in text.lines() { let line = line.trim(); let (commented, rest) = match line.strip_prefix("//") { @@ -71,10 +85,19 @@ mod tests { "#ifdef AO\n#endif\n#ifndef BLOOM\n#endif\n#ifdef SPLIT\n#endif\n#ifdef LEVEL\n#endif\n", ), ("shaders/world-1/deferred.fsh", "//#define SPLIT\n"), + ( + "shaders/final.fsh", + "#include \"shaders.settings\"\n#ifdef GODRAYS\n#endif\n", + ), + ("shaders/shaders.settings", " #define GODRAYS //Toggle\n"), + ("shaders/notes.settings", "#define NOT_INCLUDED\n#ifdef NOT_INCLUDED\n"), ("shaders/shaders.properties", "#define NOT_SOURCE\n#ifdef NOT_SOURCE\n"), ]), ) .unwrap(); - assert_eq!(switches_on(&pack), BTreeSet::from(["AO".to_owned()])); + assert_eq!( + switches_on(&pack), + BTreeSet::from(["AO".to_owned(), "GODRAYS".to_owned()]) + ); } } diff --git a/crates/horizon-shaders/src/translate/mod.rs b/crates/horizon-shaders/src/translate/mod.rs index 35475ddab..96872da1d 100644 --- a/crates/horizon-shaders/src/translate/mod.rs +++ b/crates/horizon-shaders/src/translate/mod.rs @@ -204,13 +204,30 @@ fn translate( }) .collect(); - // Varyings by name: the vertex outputs, then fragment inputs the vertex stage lacks. + // Varyings by name: the vertex outputs, then fragment inputs the vertex stage lacks. A + // matrix or array travels as one varying per column or element (`varying_parts`). let mut locations: BTreeMap = BTreeMap::new(); + let mut split: BTreeMap)> = BTreeMap::new(); for var in v.outs.iter().chain(&f.ins) { - let next = locations.len() as u32; - locations - .entry(var.name.clone()) - .or_insert((next, var.decl.clone())); + if locations.contains_key(&var.name) || split.contains_key(&var.name) { + continue; + } + match varying_parts(&var.decl) { + Some((global, parts)) => { + let mut copies = Vec::new(); + for (k, (decl, access)) in parts.into_iter().enumerate() { + let part = format!("hz_v_{}_{k}", var.name); + let next = locations.len() as u32; + locations.insert(part.clone(), (next, decl)); + copies.push((part, access)); + } + split.insert(var.name.clone(), (global, copies)); + } + None => { + let next = locations.len() as u32; + locations.insert(var.name.clone(), (next, var.decl.clone())); + } + } } let header = |stage: &Stage| { @@ -255,6 +272,9 @@ fn translate( ) .unwrap(); } + for (name, (global, _)) in &split { + writeln!(h, "{}", global.replace("@", name)).unwrap(); + } if let (Some(inputs), ShaderStage::Vertex) = (gbuffers, stage.kind) { h.push_str(&inputs.declarations); h.push('\n'); @@ -292,9 +312,15 @@ fn translate( for name in &v.inits { writeln!(vertex_main, " {name} = hz_init_{name}();").unwrap(); } + vertex_main.push_str(" hz_main();\n"); + for (name, (_, copies)) in &split { + for (part, access) in copies { + writeln!(vertex_main, " {part} = {name}{access};").unwrap(); + } + } write!( vertex_main, - " hz_main();\n gl_Position = vec4(gl_Position.x, {flip}gl_Position.y, (gl_Position.z + gl_Position.w) * 0.5, gl_Position.w);\n{}}}\n", + " gl_Position = vec4(gl_Position.x, {flip}gl_Position.y, (gl_Position.z + gl_Position.w) * 0.5, gl_Position.w);\n{}}}\n", gbuffers.map_or("", |g| &g.after) ) .unwrap(); @@ -309,6 +335,11 @@ fn translate( Target::Buffers => " hz_FragCoord = gl_FragCoord;\n", }); } + for (name, (_, copies)) in &split { + for (part, access) in copies { + writeln!(fragment_main, " {name}{access} = {part};").unwrap(); + } + } for name in &f.inits { writeln!(fragment_main, " {name} = hz_init_{name}();").unwrap(); } @@ -363,6 +394,52 @@ fn translate( }) } +/// A matrix or array varying (`flat mat3 @[2];`): wgpu's stage interfaces take only scalars +/// and vectors (WGSL), so it becomes a private global (`mat3 @[2];`) and one varying per +/// column or element, each with the qualifiers and how to reach it (`[1][2]`). `None` for +/// scalars and vectors, and for sizes that are not plain numbers. +fn varying_parts(decl: &str) -> Option<(String, Vec<(String, String)>)> { + let (before, after) = decl.split_once('@')?; + let mut words: Vec<&str> = before.split_whitespace().collect(); + let ty = words.pop()?; + let qualifiers: String = words.iter().map(|w| format!("{w} ")).collect(); + let array = after.trim().trim_end_matches(';').trim(); + let mut sizes = Vec::new(); + let mut rest = array; + while let Some(r) = rest.strip_prefix('[') { + let (n, r) = r.split_once(']')?; + sizes.push(n.trim().parse::().ok()?); + rest = r.trim_start(); + } + if !rest.is_empty() { + return None; + } + // matC or matCxR: C columns of R rows. + let column = ty.strip_prefix("mat").and_then(|d| { + let (c, r) = d.split_once('x').unwrap_or((d, d)); + Some((c.parse::().ok()?, r.parse::().ok()?)) + }); + if sizes.is_empty() && column.is_none() { + return None; + } + let part_ty = column.map_or(ty.to_owned(), |(_, r)| format!("vec{r}")); + if let Some((c, _)) = column { + sizes.push(c); + } + let count: usize = sizes.iter().product(); + let parts = (0..count) + .map(|mut k| { + let mut access = String::new(); + for &size in sizes.iter().rev() { + access.insert_str(0, &format!("[{}]", k % size)); + k /= size; + } + (format!("{qualifiers}{part_ty} @;"), access) + }) + .collect(); + Some((format!("{ty} @{array};"), parts)) +} + /// `sampler2DShadow` → (`texture2D`, `samplerShadow`); `isampler3D` → (`itexture3D`, `sampler`). pub(super) fn split_sampler(ty: &str) -> (String, &'static str) { let shadow = ty.ends_with("Shadow"); @@ -643,6 +720,33 @@ mod tests { ); } + #[test] + fn matrix_and_array_varyings_travel_in_parts() { + let vsh = "#version 120\nvarying mat3 tbn;\nflat varying ivec2 ids[2];\nvarying mat2x3 m[2];\n\ + void main() {\n gl_Position = ftransform();\n tbn = mat3(2.0);\n \ + ids[1] = ivec2(3);\n m[1][0] = vec3(4.0);\n}\n"; + let fsh = "#version 120\nvarying mat3 tbn;\nflat varying ivec2 ids[2];\nvarying mat2x3 m[2];\n\ + void main() { gl_FragColor = vec4(tbn[2] + m[1][0], float(ids[1].x)); }\n"; + let p = program( + &[("composite.vsh", vsh), ("composite.fsh", fsh)], + Target::Buffers, + ) + .unwrap_or_else(|e| panic!("{e:?}")); + let inputs = p.fragment.entry_points[0].function.arguments.len(); + // 3 columns + 2 elements + 2 x 2 columns, and the fragment position. + assert!(inputs >= 9, "{inputs} inputs"); + assert_eq!( + varying_parts("flat ivec2 @[2];").unwrap().1, + [ + ("flat ivec2 @;".to_owned(), "[0]".to_owned()), + ("flat ivec2 @;".to_owned(), "[1]".to_owned()) + ] + ); + assert_eq!(varying_parts("mat2x3 @[2];").unwrap().1[3].1, "[1][1]"); + assert!(varying_parts("vec4 @;").is_none()); + assert!(varying_parts("vec4 @[N];").is_none()); + } + #[test] fn locals_may_hide_a_sampler() { let fsh = "#version 120\nuniform sampler2D specular;\nstruct M { float specular; };\n\ From aac2f3a903524bb56685a218a8992b99614a34a5 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:44:21 +0100 Subject: [PATCH 18/61] Shader packs, clean room 29: entity spans While a shader pack is loaded, building the entity geometry records where each entity's vertices begin, with its render stage (entities, block entities, particles, breaking cracks) and its `entityId` or `blockEntityId` from the pack's `entity.properties` and `block.properties` (65535 for a name the file lacks, 0 without the file). The spans reach the renderer in `Frame::entity_spans`, for the pack's per-entity draws that follow; Horizon's own entity draw stays one call and without a pack nothing is recorded. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game.rs | 45 ++++++++++- crates/horizon-client/src/game/creatures.rs | 1 + .../horizon-client/src/game/shader_packs.rs | 80 +++++++++++++++++++ crates/horizon-client/src/gfx.rs | 4 +- crates/horizon-client/src/gfx/pack_draw.rs | 14 ++++ crates/horizon-shaders/src/block_ids.rs | 33 +++++++- 6 files changed, 172 insertions(+), 5 deletions(-) diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index 352e1e30e..f43593360 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -1228,6 +1228,11 @@ struct Game<'a> { luminance: Option, /// Name tags drawn over everything (`RenderPipelines.TEXT_SEE_THROUGH`). see_through_vertices: Vec, + /// Runs of `entity_vertices` by entity, for a shader pack's per-entity draws; recorded only + /// while `pack_ids` is set (game/shader_packs.rs). + entity_spans: std::cell::RefCell>, + /// The shader pack's entity and block entity ids, while one draws entities. + pack_ids: Option, started: Instant, previous: Instant, /// FramerateLimitTracker.latestInputTime. @@ -6668,8 +6673,10 @@ impl Game<'_> { alpha: f64, mut out: Vec, ) -> Vec { + self.entity_spans.borrow_mut().clear(); self.prepare_entity_culling(camera); self.prepare_trims(); + self.mark_entity(out.len(), "minecraft:entity_flame"); let fire_yaw = if self.camera_type == perspective::CameraType::ThirdPersonFront { self.yaw + 180.0 } else { @@ -6761,20 +6768,27 @@ impl Game<'_> { } } // `ItemEntityRenderer`. + self.mark_entity(out.len(), "minecraft:item"); self.dropped_item_geometry(camera, alpha, &mut out); // `VaultRenderer`. + self.mark_entity(out.len(), ""); self.vault_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:ominous_item_spawner"); self.ominous_item_spawner_geometry(camera, alpha, &mut out); - // Mobs and their projectiles. + // Mobs and their projectiles (a span each). self.mob_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:player"); self.player_model_geometry(camera, self.clock.camera_fraction(), &mut out); self.guest_model_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:evoker_fangs"); self.fangs_geometry(camera, alpha, &mut out); let mut see_through = std::mem::take(&mut self.see_through_vertices); see_through.clear(); + self.mark_entity(out.len(), "minecraft:name_tag"); self.name_tag_geometry(camera, alpha, &mut out, &mut see_through); self.see_through_vertices = see_through; // Entity shadows (EntityRenderer.extractShadow). + self.mark_entity(out.len(), "minecraft:entity_shadow"); if let Some(sprite) = self.gui.atlas.sprites.get("misc/shadow").copied() { let ambient = if self.dim == Dim::Nether { 0.1 } else { 0.0 }; let darken = self.sky_darken(); @@ -6868,13 +6882,14 @@ impl Game<'_> { v.intersects(o - glam::Vec3::ONE, o + glam::Vec3::splat(2.0)) }) }); - if self.entity_culler.is_some() { + if self.entity_culler.is_some() || self.pack_ids.is_some() { for block in visible { let start = out.len(); + self.mark_block_entity(start, block.1); self.res .block_entity_render() .render(&scene, std::iter::once(block), &mut out); - if !off_screen(block.1) { + if !off_screen(block.1) && self.entity_culler.is_some() { self.cull_entity_vertices(&mut out, start); } } @@ -6883,6 +6898,7 @@ impl Game<'_> { .block_entity_render() .render(&scene, visible, &mut out); } + self.mark_stage(out.len(), "DESTROY"); for (pos, stage) in self.cracks(camera) { let state = self.world.block(pos[0], pos[1], pos[2]); self.res @@ -6891,6 +6907,7 @@ impl Game<'_> { } } // Experience orbs (ExperienceOrbRenderer): pulsing billboards from a 4x4 icon sheet. + self.mark_entity(out.len(), "minecraft:experience_orb"); if let Some(r) = self .gui .atlas @@ -6954,6 +6971,7 @@ impl Game<'_> { } } // Falling blocks use their baked model, including non-cubic shapes. + self.mark_entity(out.len(), "minecraft:falling_block"); for f in &self.falling { let Some((_, tint)) = self.block_face_sprite(f.state) else { continue; @@ -6984,20 +7002,34 @@ impl Game<'_> { vertex.color = horizon_client::hud_layout::multiply(vertex.color, tint); } } + self.mark_entity(out.len(), ""); self.moving_piston_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:tnt"); self.tnt_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:painting"); self.painting_geometry(camera, &mut out); + self.mark_entity(out.len(), "minecraft:item_frame"); self.frame_geometry(camera, &mut out); + self.mark_entity(out.len(), "minecraft:armor_stand"); self.armor_stand_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:oak_boat"); self.boat_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:minecart"); self.cart_geometry(camera, alpha, &mut out); + self.mark_stage(out.len(), "BLOCK_ENTITIES"); self.beacon_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:fishing_bobber"); self.fishing_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:leash_knot"); self.leash_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:trident"); self.trident_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:firework_rocket"); self.rocket_geometry(camera, alpha, &mut out); + self.mark_entity(out.len(), "minecraft:splash_potion"); self.thrown_potion_geometry(camera, alpha, &mut out); // Particles (ParticleEngine.extract): camera-facing quads. + self.mark_stage(out.len(), "PARTICLES"); let world = &self.world; self.particles.render( camera, @@ -7008,6 +7040,7 @@ impl Game<'_> { &mut out, ); // Breaking progress overlays (destroy_stage_0..9): ours and the other players'. + self.mark_stage(out.len(), "DESTROY"); for (pos, stage) in self.cracks(camera) { let o = (glam::DVec3::new(f64::from(pos[0]), f64::from(pos[1]), f64::from(pos[2])) - camera) @@ -10070,6 +10103,7 @@ impl Game<'_> { } else { self.shader_pack_inputs(camera, alpha as f32) }; + let entity_spans = std::mem::take(self.entity_spans.get_mut()); let frame = Frame { weather: &weather, weather_time: horizon_client::weather::Time::new(self.world.game_time, alpha as f32), @@ -10134,6 +10168,7 @@ impl Game<'_> { color: sky.cloud_color, }), entities: &entities, + entity_spans: &entity_spans, dynamic_lights: &self.dynamic_lights, lightning: horizon_client::gfx::lightning::Input { bolts: &bolts, @@ -10182,6 +10217,7 @@ impl Game<'_> { }; // Restore storage even when rendering fails (for example, a lost surface). self.entity_vertices = entities; + *self.entity_spans.get_mut() = entity_spans; self.see_through_vertices = see_through; result } @@ -11762,6 +11798,8 @@ impl<'a> Game<'a> { dynamic_lights: Vec::new(), luminance: None, see_through_vertices: Vec::new(), + entity_spans: Default::default(), + pack_ids: None, health_hud: hud_layout::HealthHud::default(), started: now, previous: now, @@ -12686,6 +12724,7 @@ fn loading_frame(hud: &HudBatch) -> Frame<'_> { hud, nether_lighting: false, entities: &[], + entity_spans: &[], dynamic_lights: &[], clouds: None, lightning: Default::default(), diff --git a/crates/horizon-client/src/game/creatures.rs b/crates/horizon-client/src/game/creatures.rs index ec9848919..31a1500d7 100644 --- a/crates/horizon-client/src/game/creatures.rs +++ b/crates/horizon-client/src/game/creatures.rs @@ -2996,6 +2996,7 @@ impl Game<'_> { continue; } } + self.mark_entity(entity_start, &mob.kind.id); let light = self.world.light( mob.position.x.floor() as i32, (mob.position.y + mob.eye_height()).floor() as i32, diff --git a/crates/horizon-client/src/game/shader_packs.rs b/crates/horizon-client/src/game/shader_packs.rs index f5f7a5fba..6699a753d 100644 --- a/crates/horizon-client/src/game/shader_packs.rs +++ b/crates/horizon-client/src/game/shader_packs.rs @@ -7,6 +7,21 @@ use horizon_client::gfx::pack_uniforms::PackInputs; use horizon_shaders::{BlockIds, Macros, Pack, Properties, dimension_folder, load_post}; use super::{Dim, Game}; +use horizon_client::gfx::pack_draw::EntitySpan; +use std::cell::RefCell; +use std::collections::HashMap; + +/// The ids a pack gives entities and block entities (4.1, 5.3), for the spans of the entity +/// geometry. +pub(super) struct PackIds { + entities: HashMap, + blocks: BlockIds, + /// `blockEntityId` by block state, filled as states are drawn. + block_states: RefCell>, + /// The id of a name the file lacks: 65535, or 0 without the file. + missing_entity: i32, + missing_block: i32, +} impl Game<'_> { /// Loads the pack for the current dimension when it changed, and returns this frame's @@ -52,6 +67,18 @@ impl Game<'_> { for note in &ids.notes { log::warn!("shader pack {name}: block.properties: {note}"); } + let text = pack.read_text("entity.properties"); + let entities = BlockIds::parse_kind( + &Properties::parse(text.as_deref().unwrap_or_default(), ¯os), + "entity", + ); + self.pack_ids = Some(PackIds { + entities: entities.plain(), + missing_entity: if text.is_some() { 65535 } else { 0 }, + missing_block: if ids.is_empty() { 0 } else { 65535 }, + blocks: ids.clone(), + block_states: RefCell::default(), + }); let started_ids = std::time::Instant::now(); let table = block_id_table(&self.pack, &ids); log::info!( @@ -75,6 +102,7 @@ impl Game<'_> { } } Err(e) => { + self.pack_ids = None; log::warn!("shader pack {name}: {e}"); renderer.set_shader_pack(dimension, None, &[]); } @@ -114,6 +142,58 @@ impl Game<'_> { } } +impl Game<'_> { + /// Starts a span of the entity geometry at vertex `start` for entity type `name` + /// (`minecraft:zombie`; "" for one the pack cannot name). Nothing without a pack. + pub(super) fn mark_entity(&self, start: usize, name: &str) { + if let Some(ids) = &self.pack_ids { + let id = ids.entities.get(name).copied(); + self.push_span(start, "ENTITIES", id.unwrap_or(ids.missing_entity)); + } + } + + /// Starts a span for the block entity of block state `state`. + pub(super) fn mark_block_entity(&self, start: usize, state: u32) { + let Some(ids) = &self.pack_ids else { return }; + let id = *ids + .block_states + .borrow_mut() + .entry(state) + .or_insert_with(|| { + let registry = self.pack.assets.registry; + registry + .state(horizon_world::registry::BlockStateId(state)) + .and_then(|s| { + ids.blocks + .id(&s.name, |k| s.properties.get(k).map(String::as_str)) + }) + .unwrap_or(ids.missing_block) + }); + self.push_span(start, "BLOCK_ENTITIES", id); + } + + /// Starts a span of geometry that is not an entity (`PARTICLES`, `DESTROY`...). + pub(super) fn mark_stage(&self, start: usize, stage: &str) { + if self.pack_ids.is_some() { + self.push_span(start, stage, 0); + } + } + + fn push_span(&self, start: usize, stage: &str, id: i32) { + let mut spans = self.entity_spans.borrow_mut(); + let span = EntitySpan { + start: start as u32, + stage: horizon_shaders::render_stage(stage), + id, + }; + // A span that drew nothing gives way to the next. + match spans.last_mut() { + Some(last) if last.start == span.start => *last = span, + _ => spans.push(span), + } + } +} + /// `mc_Entity` (x, y) of each block sprite layer, as (id, fluid) pairs: terrain vertices carry /// their sprite layer, not their block. A layer takes the id of the block named like its sprite /// (`block/oak_leaves` for `oak_leaves`), else the id most blocks drawing it share (ties: the diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index f829b9eb1..4351fd85d 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -69,7 +69,7 @@ pub mod lightning; mod lod; mod occlusion; mod pack_atlas; -mod pack_draw; +pub mod pack_draw; mod pack_post; mod pack_sky; mod pack_timing; @@ -248,6 +248,8 @@ pub struct Frame<'a> { pub nether_lighting: bool, /// Camera-relative entity geometry textured from the shared atlas. pub entities: &'a [Vertex], + /// Runs of `entities` by entity, while a shader pack draws them (`pack_draw::EntitySpan`). + pub entity_spans: &'a [pack_draw::EntitySpan], pub weather: &'a [crate::weather::Vertex], pub weather_time: crate::weather::Time, /// Horizon Extras' dynamic lights, nearest first (`dynamic_light::select`); empty when off. diff --git a/crates/horizon-client/src/gfx/pack_draw.rs b/crates/horizon-client/src/gfx/pack_draw.rs index 91c77a6bf..f365c484f 100644 --- a/crates/horizon-client/src/gfx/pack_draw.rs +++ b/crates/horizon-client/src/gfx/pack_draw.rs @@ -4,6 +4,20 @@ use horizon_shaders::{RENDER_STAGES, render_stage}; +/// Where a run of `Frame::entities` begins, and what it is (slice 7): the run lasts until the +/// next span. Recorded only while a shader pack draws entities, so Horizon's own frames carry +/// none; Horizon itself still draws the whole list at once. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EntitySpan { + /// The first vertex. + pub start: u32, + /// The render stage (5.4: `ENTITIES`, `BLOCK_ENTITIES`, `PARTICLES`, `DESTROY`), which picks + /// the program. + pub stage: u32, + /// `entityId` for entities, `blockEntityId` for block entities (5.3), else 0. + pub id: i32, +} + /// `HzDraw`'s size: `renderStage`, `entityId`, `blockEntityId`, a pad, `entityColor`. const SIZE: u32 = 32; diff --git a/crates/horizon-shaders/src/block_ids.rs b/crates/horizon-shaders/src/block_ids.rs index 623473283..6ebcf6d99 100644 --- a/crates/horizon-shaders/src/block_ids.rs +++ b/crates/horizon-shaders/src/block_ids.rs @@ -23,9 +23,16 @@ pub struct BlockIds { impl BlockIds { pub fn parse(properties: &Properties) -> Self { + Self::parse_kind(properties, "block") + } + + /// The same for `.=` lines: `entity` in `entity.properties`, `item` in + /// `item.properties` (4.1); names there carry no state filters. + pub fn parse_kind(properties: &Properties, kind: &str) -> Self { let mut out = Self::default(); + let prefix = format!("{kind}."); for e in &properties.entries { - let Some(id) = e.key.strip_prefix("block.") else { + let Some(id) = e.key.strip_prefix(prefix.as_str()) else { continue; }; let Ok(id) = id.parse::() else { @@ -78,6 +85,16 @@ impl BlockIds { .map(|r| r.id) } + /// Every name without state filters and its id (the first line naming it wins): what + /// `entity.properties` and `item.properties` hold. + pub fn plain(&self) -> std::collections::HashMap { + let mut out = std::collections::HashMap::new(); + for r in self.rules.iter().filter(|r| r.filters.is_empty()) { + out.entry(r.name.clone()).or_insert(r.id); + } + out + } + pub fn is_empty(&self) -> bool { self.rules.is_empty() } @@ -117,4 +134,18 @@ mod tests { assert_eq!(ids.id("minecraft:oak_leaves", leaves("false", "1")), None); assert_eq!(ids.notes.len(), 1, "{:?}", ids.notes); } + + #[test] + fn entity_lines_by_name() { + let text = "entity.10=zombie minecraft:husk\nentity.11=zombie mod:thing\nblock.5=stone\n"; + let ids = BlockIds::parse_kind(&Properties::parse(text, &Macros::standard()), "entity"); + let plain = ids.plain(); + assert_eq!( + plain["minecraft:zombie"], 10, + "the first line naming it wins" + ); + assert_eq!(plain["minecraft:husk"], 10); + assert_eq!(plain["mod:thing"], 11); + assert!(!plain.contains_key("minecraft:stone")); + } } From c2c8df4542b857911ec396e0430da5b46fdffdb1 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:47:54 +0100 Subject: [PATCH 19/61] Mods: run a mod's commands on the next tick Run while the mod's output was shown, a command's chat line reached the mods again inside that call: a mod that answers a line with a command that prints one recursed without end, and when a control-socket command set the hook off, the mod's commands ran with the console's level. Queue them for the next tick instead, where the player's own level applies (the server already checks it on a server). Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game/mods.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/horizon-client/src/game/mods.rs b/crates/horizon-client/src/game/mods.rs index c203d0ca2..8364a4d58 100644 --- a/crates/horizon-client/src/game/mods.rs +++ b/crates/horizon-client/src/game/mods.rs @@ -39,6 +39,11 @@ pub(super) struct ModsState { /// The chat's command tree while a running mod owns commands. #[cfg(feature = "mods")] tree: Option, + /// `command.run` lines for this window's own world, run on the next tick: + /// run at once, their chat lines would reach the mods while their output + /// is still being shown, and a mod answering its own lines would recurse. + #[cfg(feature = "mods")] + commands: Vec, } impl ModsState { /// The chat's command tree: vanilla's, plus the running mods' commands. @@ -283,6 +288,11 @@ impl Game<'_> { } /// A game tick. pub(super) fn tick_mods(&mut self) { + // Here, at the player's level: shown inside a console call (a control + // socket command set the hook off), they would run as the console. + for line in std::mem::take(&mut self.mods.commands) { + self.run_command(&line); + } if !self.mods.mods.is_running() { return; } @@ -345,13 +355,13 @@ impl Game<'_> { // Vanilla's commands only: a mod running a mod command could loop. Output::Command { line } => { let name = line.split(' ').next().unwrap_or_default(); - // As handle_chat_input, without the chat history. if !vanilla_command(name) { - // dropped + // Dropped. } else if self.remote.is_some() { + // The server checks the player's permissions. self.send_chat(&format!("/{line}")); } else { - self.run_command(&line); + self.mods.commands.push(line); } continue; } From 5262effa7e4b13cd4737563dabeda3dc03e2f8af Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:48:35 +0100 Subject: [PATCH 20/61] cargo horizon-mod check: list the commands a mod runs Main's check reports every output; the command capability adds one. Co-Authored-By: Claude Opus 5.5 --- crates/cargo-horizon-mod/src/check.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/cargo-horizon-mod/src/check.rs b/crates/cargo-horizon-mod/src/check.rs index 0294bec98..a1cb44c9e 100644 --- a/crates/cargo-horizon-mod/src/check.rs +++ b/crates/cargo-horizon-mod/src/check.rs @@ -214,6 +214,7 @@ fn play(path: &Path, report: &mut Report) -> Result<(), String> { Output::Feedback { text, error } => { format!("{}: {text}", if error { "error" } else { "feedback" }) } + Output::Command { line } => format!("command: /{line}"), Output::Stopped { reason, .. } => return Err(format!("it stopped: {reason}")), }); } From 39904c8051546efdb808f0fa9a8ccb68043f7dd2 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:33:08 +0100 Subject: [PATCH 21/61] docs/mods.md: the contract lists the command interface Co-Authored-By: Claude Opus 5.5 --- docs/mods.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mods.md b/docs/mods.md index 2f7105398..a021d420b 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -12,7 +12,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | Part | Where | State | |---|---|---| -| The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage; tick, frame, chat and command hooks | +| The contract (WIT) | `crates/horizon-mod-sdk/wit/horizon.wit` | 0.1: log, game-read, chat, hud, storage, command; tick, frame, chat and command hooks | | SDK for mod authors | `crates/horizon-mod-sdk` | 0.1 | | Example mods | `crates/horizon-mod-sdk/examples/coords-hud`, `examples/predator-missile` | coordinates, facing and time HUD, `/where`, `/coords`; `/missile`, a guided missile from vanilla commands | | Runtime | `crates/horizon-mods` | wasmtime 36 LTS, component model | From c756a372a3cc5a2a58719187dedfedaef0f39884 Mon Sep 17 00:00:00 2001 From: cristibctr Date: Sun, 25 Oct 2026 15:32:57 +0200 Subject: [PATCH 22/61] Fix stale Windows resource pack overlays Invalidate version 3 overlays that can omit untouched vanilla resources. Cover folder and ZIP pack cache recovery, and use a writable timestamp handle in the Windows cache test. Co-Authored-By: OpenAI Codex --- crates/horizon-client/src/packs/cache.rs | 75 +++++++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/crates/horizon-client/src/packs/cache.rs b/crates/horizon-client/src/packs/cache.rs index efcf01e4a..51b7a9f1d 100644 --- a/crates/horizon-client/src/packs/cache.rs +++ b/crates/horizon-client/src/packs/cache.rs @@ -8,7 +8,8 @@ use std::io::Read; use std::{collections::BTreeSet, fs, io, path::Path, path::PathBuf}; // Bump when overlay/merge semantics change. Markers are disposable cache data. -const VERSION: u32 = 3; +// Version 3 could omit untouched vanilla directories on Windows (#1649, #1688). +const VERSION: u32 = 4; #[derive(Default, Serialize, Deserialize)] pub(super) struct Dependencies { @@ -421,6 +422,73 @@ mod tests { out } + #[test] + #[cfg(not(unix))] + fn texture_only_packs_rebuild_incomplete_windows_v3_overlays() { + let f = Fixture::new(); + let blockstate = "assets/minecraft/blockstates/oak_stairs.json"; + let model = "assets/minecraft/models/block/oak_stairs.json"; + let texture = "assets/minecraft/textures/block/stone.png"; + let font = "assets/minecraft/textures/font/ascii.png"; + f.write(&format!("vanilla/{blockstate}"), br#"{"variants":{}}"#); + f.write(&format!("vanilla/{model}"), br#"{"elements":[]}"#); + f.write(&format!("vanilla/{texture}"), b"vanilla texture"); + f.write(&format!("vanilla/{font}"), b"vanilla font"); + let folder = f.dir_pack("textures"); + f.write(&format!("textures/{texture}"), b"pack texture"); + f.write( + "textures.zip", + &archive( + &[ + ( + "pack.mcmeta", + br#"{"pack":{"description":"test","pack_format":97}}"#.to_vec(), + ), + (texture, b"pack texture".to_vec()), + ], + true, + ), + ); + let zip = packs::pack( + "zip".into(), + Text::empty(), + Source::Zip(f.0.join("textures.zip")), + ) + .unwrap(); + let vanilla = f.0.join("vanilla"); + for pack in [folder, zip] { + let out = f.0.join(format!("cached-{}", pack.id)); + let selected = std::slice::from_ref(&pack); + packs::build_overlay(&vanilla, selected, &out).unwrap(); + let check = || { + for name in [blockstate, model, font] { + assert_eq!( + fs::read(out.join(name)).unwrap(), + fs::read(vanilla.join(name)).unwrap() + ); + } + assert_eq!(fs::read(out.join(texture)).unwrap(), b"pack texture"); + }; + check(); + // Before #1688, Windows silently omitted untouched directories. + // A v3 marker still validates because it checks only source files. + let marker_path = marker_path(&out); + let mut marker: Marker = + serde_json::from_slice(&fs::read(&marker_path).unwrap()).unwrap(); + marker.version = 3; + for dir in ["blockstates", "models", "textures/font"] { + fs::remove_dir_all(out.join("assets/minecraft").join(dir)).unwrap(); + } + fs::write(&marker_path, serde_json::to_vec(&marker).unwrap()).unwrap(); + assert!(marker.dependencies.valid()); + packs::build_overlay(&vanilla, selected, &out).unwrap(); + check(); + // The repaired overlay also remains complete on a cache hit. + packs::build_overlay(&vanilla, selected, &out).unwrap(); + check(); + } + } + #[test] fn directory_inputs_config_and_owned_output_recovery() { let f = Fixture::new(); @@ -456,7 +524,10 @@ mod tests { let path = f.0.join(format!("high/{resource}")); let original_time = fs::metadata(&path).unwrap().modified().unwrap(); fs::write(&path, br#"{"p":"c"}"#).unwrap(); - fs::File::open(path) + // Windows requires a writable handle to restore the timestamp. + fs::OpenOptions::new() + .write(true) + .open(path) .unwrap() .set_times(fs::FileTimes::new().set_modified(original_time)) .unwrap(); From 1506c1808a2e5bd9bee4307090c09d111ad9e446 Mon Sep 17 00:00:00 2001 From: Kacpherek Date: Sun, 25 Oct 2026 13:16:16 +0100 Subject: [PATCH 23/61] Add /pregen: generate, light and save an area ahead of the players A Chunky-like command for single player and servers: /pregen start [square|circle] [ ], status, pause, continue and cancel, with progress (chunks, chunks/s, time left) in the log and to the player who started it. The area is swept strip by strip by a band of rows that streaming treats like a player's area (Streaming::set_band). A row is done once its chunks are lit and post-processed; the rows behind the band then unload and are saved, so memory holds only the rows in flight. The band's chunks are ranked for decoration row by row as they come within reach, so a task decorates in the same order however its jobs are timed. Between ticks the server waits on the workers' results and schedules the next jobs at once instead of once a tick. --- crates/horizon-client/src/command/vanilla.rs | 46 +++- crates/horizon-client/src/game.rs | 1 + crates/horizon-client/src/game/commands.rs | 2 + .../horizon-client/src/game/commands/extra.rs | 2 + .../src/game/commands/pregen.rs | 252 ++++++++++++++++++ .../src/game/commands/terrain_cmds.rs | 2 +- crates/horizon-client/src/game/levels.rs | 13 +- crates/horizon-client/src/lib.rs | 1 + crates/horizon-client/src/pregen.rs | 111 ++++++++ crates/horizon-client/src/streaming.rs | 131 ++++++++- docs/README.md | 1 + docs/hosting.md | 2 + docs/pregeneration.md | 41 +++ 13 files changed, 594 insertions(+), 11 deletions(-) create mode 100644 crates/horizon-client/src/game/commands/pregen.rs create mode 100644 crates/horizon-client/src/pregen.rs create mode 100644 docs/pregeneration.md diff --git a/crates/horizon-client/src/command/vanilla.rs b/crates/horizon-client/src/command/vanilla.rs index cdd4848d7..cba5a5d5b 100644 --- a/crates/horizon-client/src/command/vanilla.rs +++ b/crates/horizon-client/src/command/vanilla.rs @@ -538,12 +538,50 @@ fn apply_vanilla_order(d: &mut Dispatcher) { } /// The tree the game runs: a release integrated server's, or with /// `HORIZON_DEV_COMMANDS` set the development one -/// (SharedConstants.DEBUG_DEV_COMMANDS). +/// (SharedConstants.DEBUG_DEV_COMMANDS), plus Horizon's own commands. pub fn game_dispatcher() -> &'static Dispatcher { - if std::env::var_os("HORIZON_DEV_COMMANDS").is_some() { - dispatcher() + static DEV: std::sync::OnceLock = std::sync::OnceLock::new(); + static RELEASE: std::sync::OnceLock = std::sync::OnceLock::new(); + let (tree, dev) = if std::env::var_os("HORIZON_DEV_COMMANDS").is_some() { + (&DEV, true) } else { - release_dispatcher() + (&RELEASE, false) + }; + tree.get_or_init(|| { + let mut d = build_with(dev); + horizon_commands(&mut d, 0); + d + }) +} +/// Commands vanilla doesn't have, after its own: `/pregen` +/// (docs/pregeneration.md). +fn horizon_commands(d: &mut Dispatcher, root: usize) { + let pregen = d.literal(root, "pregen"); + d.requires(pregen, GAMEMASTERS); + let start = d.literal(pregen, "start"); + let radius = d.argument( + start, + "radius", + ArgType::Integer { + min: 0, + max: 30_000_000, + }, + ); + d.executes(radius, "pregen.start"); + for shape in ["square", "circle"] { + let shape = d.literal(radius, shape); + d.executes(shape, "pregen.start"); + let center = d.argument(shape, "center", ArgType::ColumnPos); + d.executes(center, "pregen.start"); + } + for (word, id) in [ + ("pause", "pregen.pause"), + ("continue", "pregen.continue"), + ("cancel", "pregen.cancel"), + ("status", "pregen.status"), + ] { + let w = d.literal(pregen, word); + d.executes(w, id); } } pub fn dispatcher() -> &'static Dispatcher { diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index 699468651..fa00e3804 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -10590,6 +10590,7 @@ impl Game<'_> { // The loading screen's chunk map fills ring by ring, as vanilla's. self.streaming.loading = self.options.execution_role.is_client() && self.loading_reason().is_some(); + self.tick_pregen(); let (uploads, unloaded) = horizon_client::spans::time("streaming", || self.streaming.update(&mut self.world)); if let Some(lod) = &mut self.lod { diff --git a/crates/horizon-client/src/game/commands.rs b/crates/horizon-client/src/game/commands.rs index 19a16e261..d5f9e6542 100644 --- a/crates/horizon-client/src/game/commands.rs +++ b/crates/horizon-client/src/game/commands.rs @@ -10,6 +10,7 @@ mod items; mod misc; mod place; mod players; +pub(super) mod pregen; mod raid; mod terrain_cmds; mod text_commands; @@ -758,6 +759,7 @@ impl Game<'_> { "ride.mount" | "ride.dismount" => self.cmd_ride(id, ctx), "spreadplayers" => self.cmd_spreadplayers(ctx), id if id.starts_with("forceload.") => self.cmd_forceload(id, ctx), + id if id.starts_with("pregen.") => self.cmd_pregen(id, ctx), "fillbiome" => self.cmd_fillbiome(ctx), id if id.starts_with("worldborder.") => self.cmd_worldborder(id, ctx), id if id.starts_with("loot.") => self.cmd_loot(id, ctx), diff --git a/crates/horizon-client/src/game/commands/extra.rs b/crates/horizon-client/src/game/commands/extra.rs index ef69be6d9..ca431b8e7 100644 --- a/crates/horizon-client/src/game/commands/extra.rs +++ b/crates/horizon-client/src/game/commands/extra.rs @@ -57,6 +57,8 @@ pub(in crate::game) struct CommandExtras { /// The `CommandResultCallback` results of the outermost source, when a /// harness records them. pub results: Option>, + /// The running `/pregen` task. + pub pregen: Option, } use horizon_core::version::{BUILD_TIME, DATA_VERSION, STABLE, VANILLA_VERSION, VERSION_ID}; diff --git a/crates/horizon-client/src/game/commands/pregen.rs b/crates/horizon-client/src/game/commands/pregen.rs new file mode 100644 index 000000000..d61ba63ac --- /dev/null +++ b/crates/horizon-client/src/game/commands/pregen.rs @@ -0,0 +1,252 @@ +//! `/pregen`, Horizon's own command: generate, light and save every chunk of a +//! square or a circle ahead of the players (docs/pregeneration.md). The task +//! lives on the server and hands its band to the level's streaming, whose +//! workers do the work; chunks behind the band unload and are saved as any +//! chunk the players leave. +use super::{CommandError, Context, Dim, Game, R}; +use horizon_client::{ + lan::data::component_nbt, + pregen::{Pregen, Shape}, + text::{self, Text}, +}; +use horizon_protocol::packets::play::{self, Clientbound as C}; +use std::time::{Duration, Instant}; + +/// How often the progress is reported. +const REPORT_EVERY: Duration = Duration::from_secs(5); + +pub(in crate::game) struct PregenTask { + pregen: Pregen, + dim: Dim, + paused: bool, + /// Time spent running, up to the last pause, and when it last resumed. + ran: Duration, + resumed: Instant, + reported: Instant, + /// The player who started it, told the progress too (a guest's connection). + owner: Option, +} + +impl PregenTask { + fn running_for(&self) -> Duration { + if self.paused { + self.ran + } else { + self.ran + self.resumed.elapsed() + } + } + fn progress(&self) -> Text { + let p = &self.pregen; + let seconds = self.running_for().as_secs_f64(); + let rate = p.done as f64 / seconds.max(1e-3); + let left = p.total - p.done; + let eta = if rate > 0.0 { + duration(left as f64 / rate) + } else { + "unknown".into() + }; + Text::literal(format!( + "Pregenerating {}: {} of {} chunks ({:.1}%), {rate:.1} chunks/s, {} left", + self.dim.id(), + p.done, + p.total, + 100.0 * p.done as f64 / p.total.max(1) as f64, + if p.done == p.total { + "nothing".into() + } else { + eta + }, + )) + } +} + +fn failure(message: &str) -> CommandError { + CommandError::new(Text::literal(message)) +} + +/// `1h 02m 03s`. +fn duration(seconds: f64) -> String { + let s = seconds.round() as u64; + match (s / 3600, s / 60 % 60, s % 60) { + (0, 0, s) => format!("{s}s"), + (0, m, s) => format!("{m}m {s:02}s"), + (h, m, s) => format!("{h}h {m:02}m {s:02}s"), + } +} + +impl Game<'_> { + pub(super) fn cmd_pregen(&mut self, id: &str, ctx: &Context) -> R { + match id { + "pregen.start" => { + if self.cmds.pregen.is_some() { + return Err(failure( + "Pregeneration is already running: /pregen cancel stops it", + )); + } + let radius = Self::int(ctx, "radius").unwrap_or(0); + let shape = if ctx.literals.iter().any(|l| l == "circle") { + Shape::Circle + } else { + Shape::Square + }; + let [x, z] = if ctx.get("center").is_some() { + self.column_arg(ctx, "center")? + } else { + let p = self.source_position(); + [p[0].floor() as i32, p[2].floor() as i32] + }; + let pregen = Pregen::new((x >> 4, z >> 4), radius >> 4, shape); + let now = Instant::now(); + let total = pregen.total; + self.feedback(Text::literal(format!( + "Pregenerating {total} chunks of {}: a {} of radius {radius} around {x} {z}", + self.dim.id(), + match shape { + Shape::Square => "square", + Shape::Circle => "circle", + }, + ))); + self.cmds.pregen = Some(PregenTask { + pregen, + dim: self.dim, + paused: false, + ran: Duration::ZERO, + resumed: now, + reported: now, + owner: self.lan_acting, + }); + Ok(total as i32) + } + "pregen.pause" | "pregen.continue" => { + let pause = id == "pregen.pause"; + let task = self + .cmds + .pregen + .as_mut() + .ok_or_else(|| failure("No pregeneration is running"))?; + if task.paused != pause { + task.paused = pause; + if pause { + task.ran += task.resumed.elapsed(); + } else { + task.resumed = Instant::now(); + } + } + let (progress, dim) = (task.progress(), task.dim); + if pause && self.dim == dim { + self.streaming.set_band(None); + } + self.feedback( + Text::literal(if pause { "Paused. " } else { "Continuing. " }).append(progress), + ); + Ok(1) + } + "pregen.cancel" => { + let task = self + .cmds + .pregen + .take() + .ok_or_else(|| failure("No pregeneration is running"))?; + if self.dim == task.dim { + self.streaming.set_band(None); + } + self.feedback(Text::literal("Cancelled. ").append(task.progress())); + Ok(task.pregen.done as i32) + } + _ => { + let task = self + .cmds + .pregen + .as_ref() + .ok_or_else(|| failure("No pregeneration is running"))?; + let progress = task.progress(); + let done = task.pregen.done as i32; + self.feedback(progress); + Ok(done) + } + } + } + + /// Move the task's band past the rows that are lit and hand it to the + /// level's streaming, and report the progress now and then. Runs before + /// each streaming update of the level in place. + pub(in crate::game) fn tick_pregen(&mut self) { + let Some(task) = self.cmds.pregen.as_mut() else { + return; + }; + if task.dim != self.dim || task.paused { + return; + } + // A row is done once its chunks are lit and post-processed + // (`postprocess_chunks`, on the tick), as vanilla's are by the time + // they are FULL: then they may unload and be saved. + let (streaming, chunks) = (&mut self.streaming, &self.world.chunks); + task.pregen.advance(|x, z| { + streaming.is_lit(x, z) + && chunks + .get(&(x, z)) + .is_some_and(|c| c.postprocess.is_empty()) + }); + streaming.set_band(task.pregen.band()); + let finished = task.pregen.finished(); + if finished || task.reported.elapsed() >= REPORT_EVERY { + task.reported = Instant::now(); + let (owner, mut message) = (task.owner, task.progress()); + if finished { + message = Text::literal("Done. ") + .append(message) + .with_color(text::GREEN); + self.cmds.pregen = None; + } + self.pregen_report(owner, message); + } + } + + /// Whether the level in place has a running task, which the server pumps + /// between its ticks (`pump_pregen`). + pub(in crate::game) fn pregen_runs_here(&self) -> bool { + self.cmds + .pregen + .as_ref() + .is_some_and(|t| t.dim == self.dim && !t.paused) + } + + /// The level a running task needs ticked, whoever is in it. + pub(in crate::game) fn pregen_level(&self) -> Option { + self.cmds.pregen.as_ref().map(|t| t.dim) + } + + /// Keep the workers fed until `until`: collect what they finished and + /// hand them the next jobs as soon as each one is done, rather than once + /// a tick. + pub(in crate::game) fn pump_pregen(&mut self, until: Instant) { + while self.pregen_runs_here() { + let now = Instant::now(); + if now >= until { + break; + } + self.streaming + .wait((until - now).min(Duration::from_millis(5))); + self.update_world(); + } + } + + /// The console and the player who started the task get the progress. + fn pregen_report(&mut self, owner: Option, message: Text) { + if self.options.dedicated.is_some() { + super::super::dedicated::log("INFO", &message.plain()); + } else { + log::info!("{}", message.plain()); + } + if let Some(host) = &self.lan { + if let Some(g) = owner.and_then(|c| host.hub.guest(c)) { + g.send(&C::SystemChat(play::SystemChat { + content: component_nbt(&message.component()), + overlay: false, + })); + } + } else if self.options.dedicated.is_none() { + self.system_message(message); + } + } +} diff --git a/crates/horizon-client/src/game/commands/terrain_cmds.rs b/crates/horizon-client/src/game/commands/terrain_cmds.rs index df52f2251..d4b1c125c 100644 --- a/crates/horizon-client/src/game/commands/terrain_cmds.rs +++ b/crates/horizon-client/src/game/commands/terrain_cmds.rs @@ -30,7 +30,7 @@ fn ticks_to_seconds(ticks: i64) -> String { impl Game<'_> { // ---- /forceload ---------------------------------------------------------- - fn column_arg(&self, ctx: &Context, name: &str) -> R<[i32; 2]> { + pub(super) fn column_arg(&self, ctx: &Context, name: &str) -> R<[i32; 2]> { let c = self .coords(ctx, name) .ok_or_else(|| err("argument.pos2d.incomplete", &[]))?; diff --git a/crates/horizon-client/src/game/levels.rs b/crates/horizon-client/src/game/levels.rs index f685200f2..588f425a4 100644 --- a/crates/horizon-client/src/game/levels.rs +++ b/crates/horizon-client/src/game/levels.rs @@ -365,7 +365,12 @@ impl Game<'_> { } *next += self.clock.tick_duration(); if *next > now { - std::thread::sleep(*next - now); + // A `/pregen` task keeps the workers fed until the next tick. + self.pump_pregen(*next); + let now = Instant::now(); + if *next > now { + std::thread::sleep(*next - now); + } } else if now - *next > Duration::from_secs(2) { *next = now; } @@ -373,7 +378,7 @@ impl Game<'_> { /// The levels this server tick runs, in `MinecraftServer.levels` order /// (the Overworld, the Nether, the End): every level with a player in - /// it, or the Overworld while nobody is online. + /// it or a `/pregen` task, or the Overworld while nobody is online. pub(super) fn levels_to_tick(&self) -> Vec { let guests = self .lan @@ -381,7 +386,9 @@ impl Game<'_> { .map_or(&[][..], |h| h.hub.guests.as_slice()); let mut dims: Vec = [Dim::Overworld, Dim::Nether, Dim::End] .into_iter() - .filter(|d| guests.iter().any(|g| g.dimension == d.id())) + .filter(|d| { + guests.iter().any(|g| g.dimension == d.id()) || self.pregen_level() == Some(*d) + }) .collect(); if dims.is_empty() { dims.push(Dim::Overworld); diff --git a/crates/horizon-client/src/lib.rs b/crates/horizon-client/src/lib.rs index ae95e1bdb..5125608d4 100644 --- a/crates/horizon-client/src/lib.rs +++ b/crates/horizon-client/src/lib.rs @@ -145,6 +145,7 @@ pub mod player_sync; pub mod poi_sections; pub mod portal_sections; pub mod pose; +pub mod pregen; pub mod projectiles; pub mod raid; pub mod random_sequences; diff --git a/crates/horizon-client/src/pregen.rs b/crates/horizon-client/src/pregen.rs new file mode 100644 index 000000000..f31a6d712 --- /dev/null +++ b/crates/horizon-client/src/pregen.rs @@ -0,0 +1,111 @@ +//! Pregeneration (`/pregen`): every chunk of a square or a circle generated, +//! lit and saved ahead of the players. The area is cut into strips of columns +//! and each strip is swept north to south by a [`Band`] a few rows deep: +//! streaming generates the band's chunks and keeps them until they are lit, +//! then the band moves on and the rows behind it unload and are saved, so +//! memory stays bounded whatever the radius. +use crate::streaming::Band; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Shape { + Square, + Circle, +} + +/// The widest strip, in chunks. Strips are swept one after another; the few +/// columns along the edge between two are generated with both. +const STRIP_WIDTH: i32 = 256; +/// How many chunks a band holds at least: enough to keep every worker busy +/// while the rows before it finish. +const BAND_CHUNKS: i32 = 1024; + +pub struct Pregen { + /// The centre chunk and the radius in chunks. + pub center: (i32, i32), + pub radius: i32, + pub shape: Shape, + /// The strips' columns, west to east. + strips: Vec<(i32, i32)>, + strip: usize, + /// The band's first row: every row of the strip before it is done. + row: i32, + /// Chunks of the area lit so far, and in all. + pub done: u64, + pub total: u64, +} + +impl Pregen { + pub fn new(center: (i32, i32), radius: i32, shape: Shape) -> Self { + let radius = radius.max(0); + let (x0, width) = (center.0 - radius, 2 * radius + 1); + let count = (width + STRIP_WIDTH - 1) / STRIP_WIDTH; + let strips = (0..count) + .map(|i| (x0 + width * i / count, x0 + width * (i + 1) / count - 1)) + .collect(); + let mut pregen = Self { + center, + radius, + shape, + strips, + strip: 0, + row: center.1 - radius, + done: 0, + total: 0, + }; + pregen.total = (center.1 - radius..=center.1 + radius) + .map(|z| pregen.row_chunks(x0, x0 + width - 1, z)) + .sum(); + pregen + } + + fn contains(&self, x: i32, z: i32) -> bool { + match self.shape { + Shape::Square => true, + Shape::Circle => { + let (dx, dz) = (i64::from(x - self.center.0), i64::from(z - self.center.1)); + dx * dx + dz * dz <= i64::from(self.radius) * i64::from(self.radius) + } + } + } + + fn row_chunks(&self, x0: i32, x1: i32, z: i32) -> u64 { + (x0..=x1).filter(|&x| self.contains(x, z)).count() as u64 + } + + /// The band streaming should hold now; none once every strip is done. + pub fn band(&self) -> Option { + let &(x0, x1) = self.strips.get(self.strip)?; + let rows = (BAND_CHUNKS + (x1 - x0)) / (x1 - x0 + 1); + Some(Band { + x0, + x1, + z0: self.row, + z1: (self.row + rows - 1).min(self.center.1 + self.radius), + circle: (self.shape == Shape::Circle).then_some((self.center, self.radius)), + }) + } + + /// Move past the band's first rows once all their chunks are `lit`. + /// Returns whether the band moved. + pub fn advance(&mut self, lit: impl Fn(i32, i32) -> bool) -> bool { + let mut moved = false; + while let Some(&(x0, x1)) = self.strips.get(self.strip) { + let z = self.row; + if !(x0..=x1).all(|x| !self.contains(x, z) || lit(x, z)) { + break; + } + self.done += self.row_chunks(x0, x1, z); + self.row += 1; + if self.row > self.center.1 + self.radius { + self.strip += 1; + self.row = self.center.1 - self.radius; + } + moved = true; + } + moved + } + + pub fn finished(&self) -> bool { + self.strip >= self.strips.len() + } +} diff --git a/crates/horizon-client/src/streaming.rs b/crates/horizon-client/src/streaming.rs index 55a2f0086..51c4eaea4 100644 --- a/crates/horizon-client/src/streaming.rs +++ b/crates/horizon-client/src/streaming.rs @@ -278,6 +278,11 @@ pub struct Streaming { cover: Option, /// Mesh the sections for drawing (false on a server thread). pub mesh: bool, + /// The pregeneration band (`set_band`). + band: Option, + /// The columns and the next row the band's decoration ranks were given + /// for (`set_band`). + band_ranked: Option<(i32, i32, i32)>, } /// Snapshot only after a scan found no missing candidates. Public range fields /// and `World::chunks` can change without going through Streaming, so compare @@ -289,6 +294,7 @@ struct GenerationScan { radius: i32, extra: Vec<((i32, i32), i32)>, forced: HashSet<(i32, i32)>, + band: Option, chunk_revision: u64, } impl GenerationScan { @@ -298,6 +304,7 @@ impl GenerationScan { && self.radius == ranges.radius && self.extra == ranges.extra && self.forced == *ranges.forced + && self.band.as_ref() == ranges.band && self.chunk_revision == world.chunks.membership_revision() } fn remember(&mut self, ranges: Ranges<'_>, world: &World) { @@ -307,6 +314,7 @@ impl GenerationScan { self.extra.clear(); self.extra.extend_from_slice(ranges.extra); self.forced.clone_from(ranges.forced); + self.band = ranges.band.copied(); self.chunk_revision = world.chunks.membership_revision(); } } @@ -348,6 +356,10 @@ impl GenerationScan { let r = radius + margin.max(reach + 1); runs.extend((center.1 - r..=center.1 + r).map(|z| (z, center.0 - r, center.0 + r))); } + if let Some(band) = input.ranges.band { + let r = margin.max(reach + 1); + runs.extend((band.z0 - r..=band.z1 + r).map(|z| (z, band.x0 - r, band.x1 + r))); + } runs.sort_unstable(); let mut runs = runs.into_iter().peekable(); while let Some((z, x0, mut x1)) = runs.next() { @@ -392,6 +404,7 @@ impl GenerationScan { extra: input.ranges.extra, forced: input.ranges.forced, cover: input.ranges.cover, + band: input.ranges.band, }; self.remember(ranges, input.world); } else { @@ -564,6 +577,8 @@ impl Streaming { extra: Vec::new(), cover: None, mesh: true, + band: None, + band_ranked: None, } } /// Shares generation resources, but gives distant work its own worker state. @@ -916,6 +931,7 @@ impl Streaming { p.is_some() && i64::from(x - center.0).pow(2) + i64::from(z - center.1).pow(2) > r4 * r4 && !ranges.in_extra(x, z, 4) + && !ranges.in_band(x, z, 4) && !ranges .forced .iter() @@ -985,6 +1001,7 @@ impl Streaming { extra: &self.extra, forced: &self.forced, cover: ExtraCover::matching(&self.cover, &self.extra), + band: self.band.as_ref(), }; let (protos, generating) = (&self.protos, &self.generating); self.decoration_order.retain(|x, z| { @@ -1113,6 +1130,7 @@ impl Streaming { extra: &self.extra, forced: &self.forced, cover: ExtraCover::matching(&self.cover, &self.extra), + band: self.band.as_ref(), }; self.waiting.retain(|&(x, sy, z)| { let back = ranges.in_view(x, z); @@ -1189,6 +1207,7 @@ impl Streaming { extra: &self.extra, forced: &self.forced, cover: ExtraCover::matching(&self.cover, &self.extra), + band: self.band.as_ref(), }, world, generating: &self.generating, @@ -1285,6 +1304,7 @@ impl Streaming { extra: &self.extra, forced: &self.forced, cover: ExtraCover::matching(&self.cover, &self.extra), + band: self.band.as_ref(), } } fn near_finished(&self, x: i32, z: i32, reach: i32) -> bool { @@ -1354,6 +1374,45 @@ impl Streaming { fn in_range(&self, x: i32, z: i32, margin: i32) -> bool { self.ranges().in_range(x, z, margin) } + pub fn band(&self) -> Option { + self.band + } + /// Generate, light and keep the chunks of `band` (none: stop), and let + /// go of what earlier bands held as it falls out of range. The chunks a + /// band reaches are ranked for decoration row by row, as they first come + /// within reach, so that a task decorates in the same order however its + /// work is timed and shared among the workers. + pub fn set_band(&mut self, band: Option) { + if self.band == band { + return; + } + self.band = band; + self.generation_scan.exhausted = false; + let Some(band) = band else { + return; + }; + // Decoration reaches two chunks past the band and generation three. + const REACH: i32 = 4; + let (x0, x1) = (band.x0 - REACH, band.x1 + REACH); + let first = match self.band_ranked { + Some((rx0, rx1, next)) if (rx0, rx1) == (x0, x1) => next, + _ => band.z0 - REACH, + }; + let end = band.z1 + REACH; + for z in first..=end { + for x in x0..=x1 { + self.decoration_order.rank_or_append(x, z); + } + } + self.band_ranked = Some((x0, x1, end.max(first - 1) + 1)); + } + /// Wait up to `timeout` for a worker to finish a job; the next `update` + /// takes it. A server pumps generation with this between its ticks. + pub fn wait(&mut self, timeout: std::time::Duration) { + if let Ok(done) = self.results.recv_timeout(timeout) { + self.deferred.push(done); + } + } } /// `ChunkTrackingView.isInViewDistance`: whether the column (x, z) is drawn /// at this render distance around `center`, the camera's column @@ -1373,6 +1432,53 @@ struct Ranges<'a> { /// `extra` as a grid, when one was made for it: `in_extra` without a /// scan of every area. cover: Option<&'a ExtraCover>, + band: Option<&'a Band>, +} +/// The chunks a pregeneration task (`/pregen`) wants lit now: rows `z0..=z1` +/// of the columns `x0..=x1`, those within `circle` (a centre and a radius in +/// chunks) when there is one. Like an extra area, it finishes its chunks and +/// the ring around them, so that its own chunks can be lit, and generation +/// reaches past that. A task moves it down a strip of columns a row at a +/// time, so each chunk is generated once and only the rows in flight stay +/// loaded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Band { + pub x0: i32, + pub x1: i32, + pub z0: i32, + pub z1: i32, + pub circle: Option<((i32, i32), i32)>, +} +impl Band { + /// Whether (x, z) lies within `margin` (Chebyshev) of one of the band's + /// chunks: the square of side `2 * margin + 1` around it meets the + /// rectangle at a point inside the circle. + pub fn within(&self, x: i32, z: i32, margin: i32) -> bool { + let (x0, x1) = ( + x.saturating_sub(margin).max(self.x0), + x.saturating_add(margin).min(self.x1), + ); + let (z0, z1) = ( + z.saturating_sub(margin).max(self.z0), + z.saturating_add(margin).min(self.z1), + ); + if x0 > x1 || z0 > z1 { + return false; + } + self.circle.is_none_or(|((cx, cz), r)| { + let dx = i64::from(cx.clamp(x0, x1)) - i64::from(cx); + let dz = i64::from(cz.clamp(z0, z1)) - i64::from(cz); + dx * dx + dz * dz <= i64::from(r) * i64::from(r) + }) + } + /// The band's chunks go after the players' (lower runs first), row by + /// row, as the task wants them lit. + fn priority(&self, x: i32, z: i32) -> i64 { + const AFTER_PLAYERS: i64 = 1 << 40; + AFTER_PLAYERS + + (i64::from(z - self.z0 + 8).max(0) << 20) + + i64::from(x - self.x0 + 8).max(0) + } } /// For every column near the extra areas, how far it lies outside the /// nearest one's square (Chebyshev distance less its radius; 0 inside). A server has an area per player, and streaming asks of every @@ -1468,10 +1574,14 @@ impl ExtraCover { impl Ranges<'_> { /// `distance`, or `rings` while the level loading screen is up. fn priority(&self, x: i32, z: i32, loading: bool) -> i64 { - if loading { + let own = if loading { self.rings(x, z) } else { self.distance(x, z) + }; + match self.band { + Some(band) if band.within(x, z, 4) => own.min(band.priority(x, z)), + _ => own, } } /// Ring by ring: the Chebyshev distance to the nearest centre first, then @@ -1532,10 +1642,18 @@ impl Ranges<'_> { let (dx, dz) = (gap(x - self.center.0), gap(z - self.center.1)); let r = i64::from(self.radius + 1); // The extra areas are squares: their finished square grown by `reach`. - dx * dx + dz * dz <= r * r || self.in_extra(x, z, 1 + reach) + dx * dx + dz * dz <= r * r + || self.in_extra(x, z, 1 + reach) + || self.in_band(x, z, 1 + reach) } fn in_range(&self, x: i32, z: i32, margin: i32) -> bool { - self.in_main_range(x, z, margin) || self.in_extra(x, z, margin) + self.in_main_range(x, z, margin) + || self.in_extra(x, z, margin) + || self.in_band(x, z, margin) + } + /// Within `margin` of the pregeneration band. + fn in_band(&self, x: i32, z: i32, margin: i32) -> bool { + self.band.is_some_and(|b| b.within(x, z, margin)) } /// Within `margin` of an extra area's square (Chebyshev distance, as /// vanilla's ticket levels spread). @@ -1659,6 +1777,7 @@ mod tests { extra: &self.extra, forced: &self.forced, cover: None, + band: None, }, world: &self.world, generating: &self.generating, @@ -1972,6 +2091,7 @@ mod tests { extra: &[], forced: &forced, cover: None, + band: None, }; let square = |x: i32, z: i32, n: i32| { (-n..=n).flat_map(move |dz| (-n..=n).map(move |dx| (x + dx, z + dz))) @@ -2027,6 +2147,7 @@ mod tests { extra: &[], forced: &forced, cover: None, + band: None, }; let at = |dx: i32, dz: i32| (2 + dx, -1 + dz); let key = |(x, z): (i32, i32), loading| r.priority(x, z, loading); @@ -2053,6 +2174,7 @@ mod tests { extra: &[((radius * 2 + 9, 3), 4)], forced: &forced, cover: None, + band: None, }; let span = radius * 3 + 20; for x in -span..=span { @@ -2093,6 +2215,7 @@ mod tests { extra: &extra, forced: &forced, cover: None, + band: None, }; let level = |x: i32, z: i32| 31 + ((x - 40).abs().max((z + 9).abs()) - view).max(0); for x in 40 - view - 8..=40 + view + 8 { @@ -2161,6 +2284,7 @@ mod cover_tests { extra: &extra, forced: &forced, cover, + band: None, }; let (scan, grid) = (ranges(None), ranges(Some(&cover))); let span = [4, 30, 120][case % 3] + 40; @@ -2218,6 +2342,7 @@ mod cover_tests { extra: &extra, forced: &forced, cover, + band: None, }; // The columns streaming asks about: everything kept, and the // generation scan's ring past it. diff --git a/docs/README.md b/docs/README.md index 1450ba388..48ddfd7b5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,6 +34,7 @@ To play, use [the launcher](../launcher/README.md); the project site is [playhor ## Architecture: world generation - [World generation](world-generation.md): the pipeline from noise to features. +- [Pregeneration](pregeneration.md): `/pregen`, generating, lighting and saving an area ahead of the players. - [Noise generation](noise-generation.md), [compiled density functions](density-functions.md), [structure density](structure-density.md), [terrain filling](terrain-filling.md). - Biomes: [lookup](biome-lookup.md), [temperatures](biome-temperatures.md). - Surfaces: [material surfaces](material-surfaces.md), [material conditions](material-conditions.md), [rule libraries](material-rule-libraries.md), [vertical anchors](vertical-anchors.md), [value providers](value-providers.md). diff --git a/docs/hosting.md b/docs/hosting.md index bf431cada..4c4c954bc 100644 --- a/docs/hosting.md +++ b/docs/hosting.md @@ -218,6 +218,8 @@ horizon-server rollback back to the release before While the server runs, `op`, `whitelist` and `cmd` go to the game's console, so they act exactly as vanilla's console commands do. While it's stopped, they edit vanilla's files directly. Any console command works through `cmd`: `horizon-server cmd ban Griefer`, `horizon-server cmd say Restarting in 5 minutes`. +To generate the world around spawn ahead of the players, Horizon has its own `pregen` command: +`horizon-server cmd pregen start 2500` ([pregeneration](pregeneration.md)). `config set` changes `server.toml` without opening it: `horizon-server config set world.difficulty=hard server.max_players=10`. Each key is a section and a name from the file (`properties.spawn-protection=0` for any diff --git a/docs/pregeneration.md b/docs/pregeneration.md new file mode 100644 index 000000000..a8ca228b3 --- /dev/null +++ b/docs/pregeneration.md @@ -0,0 +1,41 @@ +# Pregeneration: `/pregen` + +`/pregen` generates every chunk of a square or a circle ahead of the players, lights it and saves it, as +Chunky does on a Java server. It is Horizon's own command (vanilla has none), available in single player +with cheats on and to operators (level 2) on a server, from the chat or the server console +(`horizon-server cmd pregen start 2500`). + +```text +/pregen start [square|circle] [ ] radius in blocks; the centre defaults to where you + stand (the world spawn from the console) +/pregen status chunks done, chunks per second, time left +/pregen pause | continue stop scheduling new chunks, and go on +/pregen cancel stop; what is done stays saved +``` + +The area is the chunks within `radius / 16` chunks of the centre chunk: a square of side +`2 * (radius / 16) + 1` (radius 2500 is 313 x 313 = 97,969 chunks), or the chunks whose distance from the +centre chunk is at most `radius / 16`. It runs in the dimension the command was run in; that level keeps +ticking while the task runs, whoever is in it. The progress goes to the server log every five seconds +and to the player who started it, and once more when it is done. A task is not kept across a restart: +start it again and the chunks already saved load instead of generating. + +## How it works + +- `horizon_client::pregen::Pregen` cuts the area into strips at most 256 chunks wide and sweeps each + north to south with a band of rows (`streaming::Band`) holding about a thousand chunks. +- Streaming treats the band like a player's area: it generates its chunks and the rings around them that + decoration and light need, and lights its own chunks. Once every chunk of the band's first row is lit + and post-processed (`LevelChunk.postProcessGeneration`, which vanilla runs before a chunk is FULL), the + band moves down a row; the rows that fall out of its reach unload and are saved like any chunk the + players leave, so memory holds only the rows in flight. +- The chunks a band reaches are ranked for decoration row by row as they come within its reach + (`Streaming::set_band`). Decoration is order dependent where neighbours' writes overlap + (`DecorationOrder`), and this makes a task decorate in the same order however its jobs are timed and + shared among the workers: the same seed gives the same chunks. +- Between server ticks the server waits for the workers' results and hands out the next jobs at once + (`Streaming::wait`, `pump_pregen`), instead of once a tick: a chunk passes through five jobs + (generate, decorate, finish, light, save), each needing its neighbours' earlier ones. + +The chunks of the band's outer ring are finished (FULL) but not lit, as at the edge of what a player has +loaded; they light when their neighbours load later. From 35254c4f0230580e5293146a5203022a3c54d027 Mon Sep 17 00:00:00 2001 From: Kacpherek Date: Sun, 25 Oct 2026 14:04:28 +0100 Subject: [PATCH 24/61] Pregen: keep an empty dedicated server from pausing while it runs pause-when-empty-seconds paused a server with nobody online after a minute, and a paused server skips its tick: post-processing stopped, so the band waited forever on a row of lit chunks. A running /pregen now counts as activity; the countdown starts once it is done. --- crates/horizon-client/src/game/dedicated.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/horizon-client/src/game/dedicated.rs b/crates/horizon-client/src/game/dedicated.rs index 9cc892fe0..edea0e800 100644 --- a/crates/horizon-client/src/game/dedicated.rs +++ b/crates/horizon-client/src/game/dedicated.rs @@ -667,7 +667,9 @@ impl Game<'_> { self.poll_server_control(&mut state); let players = self.lan.as_ref().map_or(0, |h| h.hub.guests.len()); if pause_after > 0 { - if players == 0 { + // A `/pregen` task keeps the server busy as a player would: + // paused, nothing would post-process its chunks. + if players == 0 && self.pregen_level().is_none() { state.empty_ticks += 1; } else { state.empty_ticks = 0; From 4cd1504513ab7c5f9c4bd326fa1e4913adb3147d Mon Sep 17 00:00:00 2001 From: Kacpherek Date: Sun, 25 Oct 2026 14:04:28 +0100 Subject: [PATCH 25/61] Pregen: report what a stalled band waits for, and requeue lost light When the band hasn't moved for 30 seconds the task logs how many chunks of its first row are not loaded, being lit, unlit with no light job or not post-processed, with the streaming stage counts, and queues light again for unlit chunks whose neighbours are all loaded. --- .../src/game/commands/pregen.rs | 28 +++++++++++++++++-- crates/horizon-client/src/pregen.rs | 11 ++++++++ crates/horizon-client/src/streaming.rs | 21 ++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/crates/horizon-client/src/game/commands/pregen.rs b/crates/horizon-client/src/game/commands/pregen.rs index d61ba63ac..d497e2734 100644 --- a/crates/horizon-client/src/game/commands/pregen.rs +++ b/crates/horizon-client/src/game/commands/pregen.rs @@ -14,6 +14,8 @@ use std::time::{Duration, Instant}; /// How often the progress is reported. const REPORT_EVERY: Duration = Duration::from_secs(5); +/// How long the band may stay put before the task looks for what holds it. +const STALL: Duration = Duration::from_secs(30); pub(in crate::game) struct PregenTask { pregen: Pregen, @@ -23,6 +25,8 @@ pub(in crate::game) struct PregenTask { ran: Duration, resumed: Instant, reported: Instant, + /// When the band last moved, to notice a task that stopped advancing. + moved: Instant, /// The player who started it, told the progress too (a guest's connection). owner: Option, } @@ -113,6 +117,7 @@ impl Game<'_> { ran: Duration::ZERO, resumed: now, reported: now, + moved: now, owner: self.lan_acting, }); Ok(total as i32) @@ -181,13 +186,32 @@ impl Game<'_> { // (`postprocess_chunks`, on the tick), as vanilla's are by the time // they are FULL: then they may unload and be saved. let (streaming, chunks) = (&mut self.streaming, &self.world.chunks); - task.pregen.advance(|x, z| { + if task.pregen.advance(|x, z| { streaming.is_lit(x, z) && chunks .get(&(x, z)) .is_some_and(|c| c.postprocess.is_empty()) - }); + }) { + task.moved = Instant::now(); + } streaming.set_band(task.pregen.band()); + if task.moved.elapsed() >= STALL { + // Nothing advanced for a while: say what the first row waits + // for, and queue light again for chunks that lost their job. + task.moved = Instant::now(); + let row = task.pregen.first_row(); + let waiting = row + .iter() + .filter(|k| chunks.get(k).is_some_and(|c| !c.postprocess.is_empty())) + .count(); + let [missing, lighting, unlit] = streaming.unstick_light(&mut self.world, &row); + log::warn!( + "Horizon: pregeneration waits on row {:?}: {missing} not loaded, {lighting} being lit, \ + {unlit} unlit with no light job (queued again), {waiting} not post-processed; {:?}", + row.first(), + streaming.stages() + ); + } let finished = task.pregen.finished(); if finished || task.reported.elapsed() >= REPORT_EVERY { task.reported = Instant::now(); diff --git a/crates/horizon-client/src/pregen.rs b/crates/horizon-client/src/pregen.rs index f31a6d712..e714bcda2 100644 --- a/crates/horizon-client/src/pregen.rs +++ b/crates/horizon-client/src/pregen.rs @@ -105,6 +105,17 @@ impl Pregen { moved } + /// The chunks of the band's first row, which it waits for. + pub fn first_row(&self) -> Vec<(i32, i32)> { + let Some(&(x0, x1)) = self.strips.get(self.strip) else { + return Vec::new(); + }; + (x0..=x1) + .filter(|&x| self.contains(x, self.row)) + .map(|x| (x, self.row)) + .collect() + } + pub fn finished(&self) -> bool { self.strip >= self.strips.len() } diff --git a/crates/horizon-client/src/streaming.rs b/crates/horizon-client/src/streaming.rs index 51c4eaea4..c9e858aea 100644 --- a/crates/horizon-client/src/streaming.rs +++ b/crates/horizon-client/src/streaming.rs @@ -1406,6 +1406,27 @@ impl Streaming { } self.band_ranked = Some((x0, x1, end.max(first - 1) + 1)); } + /// What holds up the columns `keys`, for a task that stopped advancing: + /// how many are not loaded, loaded but unlit with a light job in flight, + /// and unlit with none (those whose neighbours are all loaded are queued + /// for light again). + pub fn unstick_light(&mut self, world: &mut World, keys: &[(i32, i32)]) -> [usize; 3] { + let mut counts = [0; 3]; + for &key in keys { + if !world.chunks.contains_key(&key) { + counts[0] += 1; + } else if self.lit.contains(&key) { + } else if self.lighting.contains_key(&key) { + counts[1] += 1; + } else { + counts[2] += 1; + if Self::region(world, key.0, key.1).is_some() { + world.dirty_light.insert(key); + } + } + } + counts + } /// Wait up to `timeout` for a worker to finish a job; the next `update` /// takes it. A server pumps generation with this between its ticks. pub fn wait(&mut self, timeout: std::time::Duration) { From 749a2dc9bd5d0c0367701c59b814141e24b65b19 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:30:43 +0100 Subject: [PATCH 26/61] Global level events reach a server's players; explosions play their packet's sound ServerLevel.globalLevelEvent (an end portal opening, 1038; the dragon's death roar, 1028) went to the Game's own audio without a position, which is never forwarded: on the server thread, LAN and dedicated servers nobody heard them. They now go to every guest as a global LevelEvent, which a client plays two blocks from its camera towards the event (LevelEventHandler.globalLevelEvent). ClientPacketListener.handleExplosion plays the packet's explosionSound; the client always played entity.generic.explode (a vanilla server's wind charge sounded like TNT). Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game.rs | 3 ++ crates/horizon-client/src/game/dragon.rs | 7 ++-- crates/horizon-client/src/game/end.rs | 7 ++-- crates/horizon-client/src/game/lan.rs | 3 ++ .../horizon-client/src/game/lan_particles.rs | 12 +++---- .../src/game/level_particles.rs | 33 +++++++++++++++++++ crates/horizon-client/src/game/remote.rs | 30 +++++++++++------ crates/horizon-client/src/lan/containers.rs | 11 +++++++ 8 files changed, 84 insertions(+), 22 deletions(-) diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index 699468651..afd8181cc 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -1544,6 +1544,8 @@ struct Game<'a> { /// position) for the guests. lan_block_events: Vec<([i32; 3], i32, i32, u32)>, lan_entity_events: Vec<(i32, i8, [f64; 3])>, + /// Global level events for every guest (`ServerLevel.globalLevelEvent`). + lan_global_events: Vec<(i32, [i32; 3], i32)>, /// `ContainerOpenersCounter`: how many players (the host and the guests) /// have each chest open. lid_openers: HashMap<(i32, i32, i32), i32>, @@ -11990,6 +11992,7 @@ impl<'a> Game<'a> { lan_events: Vec::new(), lan_block_events: Vec::new(), lan_entity_events: Vec::new(), + lan_global_events: Vec::new(), lid_openers: HashMap::new(), lan_acting: None, guest_packets: Vec::new(), diff --git a/crates/horizon-client/src/game/dragon.rs b/crates/horizon-client/src/game/dragon.rs index 54b260c67..fb555c2d8 100644 --- a/crates/horizon-client/src/game/dragon.rs +++ b/crates/horizon-client/src/game/dragon.rs @@ -1963,9 +1963,10 @@ impl Game<'_> { self.spawn_orbs([p.x, p.y, p.z], (xp as f32 * 0.08) as u32); } if t == 1 { - // Level event 1028: the death roar, heard everywhere. - self.audio - .play("minecraft:entity.ender_dragon.death", None, 5.0, 1.0); + // EnderDragon.tickDeath: global level event 1028, the death + // roar, heard everywhere. + let at = [p.x, p.y, p.z].map(|v| v.floor() as i32); + self.global_level_event(1028, at, 0); } if let Some(d) = self.dragon.as_mut() { d.position.y += 0.1; diff --git a/crates/horizon-client/src/game/end.rs b/crates/horizon-client/src/game/end.rs index 599158e3e..d3fb79172 100644 --- a/crates/horizon-client/src/game/end.rs +++ b/crates/horizon-client/src/game/end.rs @@ -311,9 +311,10 @@ impl Game<'_> { )?; } } - // Global LevelEvent 1038: END_PORTAL_SPAWN. - self.audio - .play("minecraft:block.end_portal.spawn", None, 1.0, 1.0); + // EnderEyeItem.useOn: global level event 1038 (END_PORTAL_SPAWN), + // heard by every player (vanilla places it at the frame's corner + // + (1, 0, 1); only its direction matters). + self.global_level_event(1038, c, 0); } Ok(true) } diff --git a/crates/horizon-client/src/game/lan.rs b/crates/horizon-client/src/game/lan.rs index 8460f54e6..442c38a66 100644 --- a/crates/horizon-client/src/game/lan.rs +++ b/crates/horizon-client/src/game/lan.rs @@ -1150,6 +1150,9 @@ impl Game<'_> { for (id, event, at) in std::mem::take(&mut self.lan_entity_events) { host.hub.entity_event(id, event, at); } + for (kind, pos, data) in std::mem::take(&mut self.lan_global_events) { + host.hub.global_level_event(kind, pos, data); + } self.lan_broadcast_effects(host); if host.hub.guests.is_empty() { host.mob_state.clear(); diff --git a/crates/horizon-client/src/game/lan_particles.rs b/crates/horizon-client/src/game/lan_particles.rs index 1efb4ac58..0e35392bf 100644 --- a/crates/horizon-client/src/game/lan_particles.rs +++ b/crates/horizon-client/src/game/lan_particles.rs @@ -161,12 +161,12 @@ impl Game<'_> { + (self.particles.random().next_float() - self.particles.random().next_float()) * 0.2) * 0.7; - self.audio.play( - "minecraft:entity.generic.explode", - Some(p.center), - 4.0, - pitch, - ); + // The packet's explosionSound: a wind charge's burst, a bed's + // or anchor's blast, a modded sound. + let sound = self + .remote_sound_name(&p.sound) + .unwrap_or_else(|| "minecraft:entity.generic.explode".to_owned()); + self.audio.play(&sound, Some(p.center), 4.0, pitch); } if let Some(options) = self.received_particle(&p.particle, p.center) { self.particles.add(&options, p.center, [1.0, 0.0, 0.0]); diff --git a/crates/horizon-client/src/game/level_particles.rs b/crates/horizon-client/src/game/level_particles.rs index d239c69d0..f12aedc5b 100644 --- a/crates/horizon-client/src/game/level_particles.rs +++ b/crates/horizon-client/src/game/level_particles.rs @@ -101,6 +101,39 @@ impl Game<'_> { } } } + /// `ServerLevel.globalLevelEvent`: heard by every player on the server + /// (an end portal opening, the dragon's death roar): here, and sent to + /// every guest whatever their dimension. + pub(super) fn global_level_event(&mut self, id: i32, pos: [i32; 3], data: i32) { + self.global_level_event_sound(id, pos); + if self.audio.capture.is_some() { + self.lan_global_events.push((id, pos, data)); + } + } + /// `LevelEventHandler.globalLevelEvent`: the sound two blocks from the + /// camera towards the event, so it is heard from anywhere. + pub(super) fn global_level_event_sound(&mut self, id: i32, pos: [i32; 3]) { + let (sound, volume) = match id { + 1023 => ("minecraft:entity.wither.spawn", 1.0), + 1028 => ("minecraft:entity.ender_dragon.death", 5.0), + 1038 => ("minecraft:block.end_portal.spawn", 1.0), + _ => return, + }; + let at = self.eye_position().map(|eye| { + let d = [ + f64::from(pos[0]) + 0.5 - eye[0], + f64::from(pos[1]) + 0.5 - eye[1], + f64::from(pos[2]) + 0.5 - eye[2], + ]; + let length = (d[0] * d[0] + d[1] * d[1] + d[2] * d[2]).sqrt(); + if length < 1.0e-4 { + return eye; + } + [0, 1, 2].map(|k| eye[k] + d[k] / length * 2.0) + }); + // Each guest plays it from the global packet. + self.audio.play_unshared(sound, at, volume, 1.0); + } /// Queue and play level event `id` at `pos` (`Level.levelEvent`). pub(super) fn level_event_at(&mut self, id: i32, pos: [i32; 3], data: i32) { self.level_events.push(LevelEvent::LevelEvent { diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index 044e43835..a80f0087a 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -255,6 +255,22 @@ fn number_format(f: &play::NumberFormat) -> horizon_client::scoreboard::NumberFo } impl Game<'_> { + /// A `Holder` off the wire: an inline event's location, or + /// the name of a registered one. + pub(super) fn remote_sound_name( + &self, + sound: &(i32, Option<(String, Option)>), + ) -> Option { + match &sound.1 { + Some((location, _)) => Some(location.clone()), + None => self.remote.as_ref().and_then(|r| { + r.builtin + .get("minecraft:sound_event")? + .name(sound.0) + .map(str::to_owned) + }), + } + } /// Leave the title for a server (ConnectScreen): an empty world fed by the connection. pub(super) fn connect_server(&mut self, address: &str, name: &str) -> Result<()> { if self.defer_world_entry(super::PendingWorldEntry::Server { @@ -1814,20 +1830,14 @@ impl Game<'_> { // ClientLevel.levelEvent: LevelEventHandler, as this client's own // level plays them (particles and sounds: a spawner's burst, bone // meal, waxing, a block broken by someone else). + // ClientPacketListener.handleLevelEvent: a global one is heard + // from anywhere (ClientLevel.globalLevelEvent). + C::LevelEvent(p) if p.global => self.global_level_event_sound(p.kind, p.pos), C::LevelEvent(p) => self.level_event_at(p.kind, p.pos, p.data), C::LevelParticles(p) => self.remote_level_particles(&p), C::Explode(p) => self.remote_explode(&p), C::Sound(p) => { - let name = match &p.sound.1 { - Some((location, _)) => Some(location.clone()), - None => self.remote.as_ref().and_then(|r| { - r.builtin - .get("minecraft:sound_event")? - .name(p.sound.0) - .map(str::to_owned) - }), - }; - if let Some(name) = name { + if let Some(name) = self.remote_sound_name(&p.sound) { let at = [ f64::from(p.x) / 8.0, f64::from(p.y) / 8.0, diff --git a/crates/horizon-client/src/lan/containers.rs b/crates/horizon-client/src/lan/containers.rs index d43cb5c27..f34d5ff58 100644 --- a/crates/horizon-client/src/lan/containers.rs +++ b/crates/horizon-client/src/lan/containers.rs @@ -78,6 +78,17 @@ impl Hub { ); } + /// `ServerLevel.globalLevelEvent`: the packet with its global flag, for + /// every player on the server (`PlayerList.broadcastAll`). + pub fn global_level_event(&self, kind: i32, pos: [i32; 3], data: i32) { + self.broadcast(&C::LevelEvent(play::LevelEvent { + kind, + pos, + data, + global: true, + })); + } + /// ClientboundBlockEventPacket (`PlayerList.broadcast` within 64 blocks). pub fn block_event(&self, pos: [i32; 3], b0: i32, b1: i32, state: u32) { let block = self From 1a39af6b623f3f12fc15ba609a016c2f4e4ea464 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:36:22 +0100 Subject: [PATCH 27/61] /pregen: plan any radius at once, and skip a strip's rows outside the circle Pregen::new counted the area chunk by chunk and split the strips with an i32 product: /pregen start 30000000 overflowed, then hung the server thread for hours. Rows are now counted from the circle's half-width, the strip edges in i64, and each strip starts and ends at the rows the circle reaches, so a band never walks empty corner rows. Unit tests check that every chunk of squares and circles is swept once and that the largest radius plans at once. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/pregen.rs | 99 ++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 15 deletions(-) diff --git a/crates/horizon-client/src/pregen.rs b/crates/horizon-client/src/pregen.rs index e714bcda2..01215928a 100644 --- a/crates/horizon-client/src/pregen.rs +++ b/crates/horizon-client/src/pregen.rs @@ -39,48 +39,69 @@ impl Pregen { let radius = radius.max(0); let (x0, width) = (center.0 - radius, 2 * radius + 1); let count = (width + STRIP_WIDTH - 1) / STRIP_WIDTH; - let strips = (0..count) - .map(|i| (x0 + width * i / count, x0 + width * (i + 1) / count - 1)) - .collect(); + // The columns `i * width / count` on, in i64: the product overflows + // for a radius of a few hundred thousand chunks. + let edge = |i: i32| x0 + (i64::from(width) * i64::from(i) / i64::from(count)) as i32; + let strips = (0..count).map(|i| (edge(i), edge(i + 1) - 1)).collect(); let mut pregen = Self { center, radius, shape, strips, strip: 0, - row: center.1 - radius, + row: 0, done: 0, total: 0, }; pregen.total = (center.1 - radius..=center.1 + radius) .map(|z| pregen.row_chunks(x0, x0 + width - 1, z)) .sum(); + pregen.row = pregen.rows().map_or(0, |r| r.0); pregen } - fn contains(&self, x: i32, z: i32) -> bool { + /// How far from the centre column the area reaches at `d` chunks from + /// the centre row (or, the same, from the centre column), if it does. + fn half_width(&self, d: i32) -> Option { + let (r, d) = (i64::from(self.radius), i64::from(d)); match self.shape { - Shape::Square => true, - Shape::Circle => { - let (dx, dz) = (i64::from(x - self.center.0), i64::from(z - self.center.1)); - dx * dx + dz * dz <= i64::from(self.radius) * i64::from(self.radius) - } + Shape::Square => (d.abs() <= r).then_some(self.radius), + Shape::Circle => (d * d <= r * r).then(|| (r * r - d * d).isqrt() as i32), } } + fn contains(&self, x: i32, z: i32) -> bool { + self.half_width(z - self.center.1) + .is_some_and(|h| (x - self.center.0).abs() <= h) + } + + /// The area's chunks in columns `x0..=x1` of row `z`. fn row_chunks(&self, x0: i32, x1: i32, z: i32) -> u64 { - (x0..=x1).filter(|&x| self.contains(x, z)).count() as u64 + self.half_width(z - self.center.1).map_or(0, |h| { + let (lo, hi) = (x0.max(self.center.0 - h), x1.min(self.center.0 + h)); + (i64::from(hi) - i64::from(lo) + 1).max(0) as u64 + }) + } + + /// The first and last rows of the current strip that hold chunks of the + /// area. + fn rows(&self) -> Option<(i32, i32)> { + let &(x0, x1) = self.strips.get(self.strip)?; + let nearest = (x0 - self.center.0).max(self.center.0 - x1).max(0); + let h = self.half_width(nearest)?; + Some((self.center.1 - h, self.center.1 + h)) } /// The band streaming should hold now; none once every strip is done. pub fn band(&self) -> Option { let &(x0, x1) = self.strips.get(self.strip)?; + let (_, last) = self.rows()?; let rows = (BAND_CHUNKS + (x1 - x0)) / (x1 - x0 + 1); Some(Band { x0, x1, z0: self.row, - z1: (self.row + rows - 1).min(self.center.1 + self.radius), + z1: (self.row + rows - 1).min(last), circle: (self.shape == Shape::Circle).then_some((self.center, self.radius)), }) } @@ -89,16 +110,16 @@ impl Pregen { /// Returns whether the band moved. pub fn advance(&mut self, lit: impl Fn(i32, i32) -> bool) -> bool { let mut moved = false; - while let Some(&(x0, x1)) = self.strips.get(self.strip) { + while let (Some(&(x0, x1)), Some((_, last))) = (self.strips.get(self.strip), self.rows()) { let z = self.row; if !(x0..=x1).all(|x| !self.contains(x, z) || lit(x, z)) { break; } self.done += self.row_chunks(x0, x1, z); self.row += 1; - if self.row > self.center.1 + self.radius { + if self.row > last { self.strip += 1; - self.row = self.center.1 - self.radius; + self.row = self.rows().map_or(self.row, |r| r.0); } moved = true; } @@ -120,3 +141,51 @@ impl Pregen { self.strip >= self.strips.len() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sweeps_every_chunk_of_the_area_once() { + for shape in [Shape::Square, Shape::Circle] { + for (center, radius) in [((0, 0), 0), ((3, -7), 5), ((-40, 12), 130), ((0, 0), 300)] { + let mut p = Pregen::new(center, radius, shape); + let r = i64::from(radius); + let area = (-radius..=radius) + .flat_map(|dz| (-radius..=radius).map(move |dx| (dx, dz))) + .filter(|&(dx, dz)| { + shape == Shape::Square + || i64::from(dx).pow(2) + i64::from(dz).pow(2) <= r * r + }) + .count() as u64; + assert_eq!(p.total, area, "{shape:?} {radius}"); + let mut seen = std::collections::HashSet::new(); + while !p.finished() { + let band = p.band().unwrap(); + assert!(band.z0 <= band.z1); + let row = p.first_row(); + for &(x, z) in &row { + assert!(seen.insert((x, z)), "{x} {z} twice"); + } + assert!( + p.advance(|x, z| z == band.z0 && row.contains(&(x, z))) || row.is_empty() + ); + } + assert_eq!( + (p.done, seen.len() as u64), + (area, area), + "{shape:?} {radius}" + ); + } + } + } + + #[test] + fn a_huge_area_is_planned_at_once() { + let started = std::time::Instant::now(); + let p = Pregen::new((0, 0), 30_000_000 >> 4, Shape::Circle); + assert!(p.total > 11_000_000_000_000); + assert!(started.elapsed() < std::time::Duration::from_secs(5)); + } +} From aa2f60b7bd78179f1e7723c866212fd4e1ec548e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 18:43:03 +0100 Subject: [PATCH 28/61] Copper golem statues turn to their next pose when right-clicked CopperGolemStatueBlock.useItemOn: anything but an axe, an empty hand too, cycles copper_golem_pose with its sound and block_change game event; honeycomb on a weathering statue passes on to waxing. Axe scraping, unwaxing and stripping and honeycomb waxing now count the item as used, and waxing sends its block_change game event, as vanilla's useOn does. The use_item_on harness gains statue cases. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game.rs | 10 ++++++ crates/horizon-world/src/level/interact.rs | 31 ++++++++++++++++++ tools/test_use_item_on_parity.py | 37 ++++++++++++++++++++-- 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index 7fcb3be89..ee6df4c38 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -8332,6 +8332,8 @@ impl Game<'_> { 1.0, 1.0, ); + // ItemStack.useOn: a use that acted counts as one of the item. + self.stats.add("used", &stack.item, 1); self.damage_held(damage); self.post_block_game_event("minecraft:block_change", pos, self.sculk_actor(), Some(new)); Ok(true) @@ -8536,6 +8538,14 @@ impl Game<'_> { 1.0, 1.0, ); + // HoneycombItem.useOn's game event, and ItemStack.useOn's used statistic. + self.post_block_game_event( + "minecraft:block_change", + hit.pos, + self.sculk_actor(), + Some(state), + ); + self.stats.add("used", &stack.item, 1); if !self.creative { self.consume_held(); } diff --git a/crates/horizon-world/src/level/interact.rs b/crates/horizon-world/src/level/interact.rs index b8251ba46..7b2195c34 100644 --- a/crates/horizon-world/src/level/interact.rs +++ b/crates/horizon-world/src/level/interact.rs @@ -592,6 +592,37 @@ impl Behaviours { { return InteractionResult::Pass; } + if self.data(state).chain.contains(&Class::CopperGolemStatue) { + return self.turn_statue(level, ticks, pos, state, item); + } self.use_without_item(level, ticks, pos, face, hit, player) } + + /// `CopperGolemStatueBlock.useItemOn` (and its weathering subclass's): anything but an axe + /// (`#minecraft:axes`, every one named `*_axe`), even an empty hand, turns the statue to its + /// next pose; honeycomb waxes a statue that still weathers instead. + fn turn_statue( + &self, + level: &mut L, + ticks: &mut Ticks, + pos: BlockPos, + state: BlockStateId, + item: Option<&str>, + ) -> InteractionResult { + let item = item.unwrap_or("minecraft:air"); + let waxed = self.table.name(state).starts_with("minecraft:waxed_"); + if item.ends_with("_axe") || (item == "minecraft:honeycomb" && !waxed) { + return InteractionResult::Pass; + } + let mut sim = self.sim(level, ticks); + sim.sound( + pos, + &"minecraft:entity.copper_golem_become_statue".into(), + 1.0, + 1.0, + ); + sim.set_block_and_update(pos, self.table.cycle(state, self.props.copper_golem_pose)); + sim.game_event(pos, "minecraft:block_change"); + InteractionResult::Success + } } diff --git a/tools/test_use_item_on_parity.py b/tools/test_use_item_on_parity.py index 89b463116..eabdd04eb 100644 --- a/tools/test_use_item_on_parity.py +++ b/tools/test_use_item_on_parity.py @@ -2,8 +2,9 @@ """Compare a player's right click on a block with vanilla's, for items that are both food and blocks (sweet and glow berries, carrots, potatoes), for sweet berry bushes, for spawn eggs used on a monster spawner, for shears carving -pumpkins, for blocks placed where mobs and armor stands stand, for respawn -anchors and for beds in the Nether and the End. +pumpkins, for copper golem statues turned to their next pose, for blocks +placed where mobs and armor stands stand, for respawn anchors and for beds in +the Nether and the End. Every case has a block at the origin, something above it, and a player three blocks south, survival or creative, hungry or full, sneaking or not, who @@ -228,6 +229,32 @@ def pumpkin_case(rng, states, case): case['cells'] = cells +STATUES = [f'{wax}{age}copper_golem_statue' for wax in ('', 'waxed_') + for age in ('', 'exposed_', 'weathered_', 'oxidized_')] + + +def statue_case(rng, states, case): + """A copper golem statue right-clicked (`CopperGolemStatueBlock.useItemOn`, the weathering + one's): anything but an axe, an empty hand too, turns it to its next pose; honeycomb waxes a + statue that still weathers instead (`HoneycombItem.useOn`), and an axe scrapes or unwaxes it.""" + statue = rng.choice(STATUES) + case['cells'] = [[0, 0, 0, states.any(rng, statue)]] + items = ['-', '-', 'stick', 'honeycomb', 'honeycomb', 'stone', 'copper_ingot', 'diamond_pickaxe', + 'copper_golem_statue'] + if statue != 'copper_golem_statue': + # An axe scrapes or unwaxes the others; on the unaffected one it brings the golem back + # (`CopperGolemStatueBlockEntity.removeStatue`), which Horizon does not do yet. + items.append('iron_axe') + item = rng.choice(items) + use = {'item': '-' if item == '-' else d_plain(item, rng.choice([1, 16])), 'creative': rng.random() < 0.2, + 'sneaking': rng.random() < 0.15, 'food': 20, + 'face': rng.choice(['up', 'up', 'north', 'south', 'west', 'east'])} + if rng.random() < 0.2: + # The item in the off hand, nothing in the main hand. + use.update(offhand=use['item'], item='-') + case['use_item'] = use + + # Entities a block may be placed into (`Entity.blocksBuilding`), and blocks to place. BLOCKERS = ['skeleton', 'zombie', 'pig', 'chicken', 'cow', 'armor_stand', 'slime', 'bat'] PLACED = ['obsidian', 'stone', 'oak_slab', 'oak_fence', 'white_carpet', 'torch', 'glass', 'oak_trapdoor', 'snow'] @@ -290,6 +317,12 @@ def cases(facts, count, seed): case['mode'] = 'dispense' yield case continue + if rng.random() < 0.08: + statue_case(rng, states, case) + case['items'] = [] + case['mode'] = 'dispense' + yield case + continue if rng.random() < 0.12: pumpkin_case(rng, states, case) case['items'] = [] From 481e75bc33bea120cad48a40294fdaf3df2b19aa Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Sun, 25 Oct 2026 19:16:36 +0000 Subject: [PATCH 29/61] Clear off-hand items in vanilla inventory slot order Co-Authored-By: OpenAI Codex GPT-6.1 --- crates/horizon-client/src/game/commands.rs | 9 ++++++--- tools/command_exec_corpus.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/horizon-client/src/game/commands.rs b/crates/horizon-client/src/game/commands.rs index 19a16e261..1f24c8a4f 100644 --- a/crates/horizon-client/src/game/commands.rs +++ b/crates/horizon-client/src/game/commands.rs @@ -2499,12 +2499,14 @@ impl Game<'_> { }; let max = Self::int(ctx, "maxCount").unwrap_or(-1); let counting = max == 0; - // Inventory.clearOrCountMatchingItems: main, armor, crafting grid, carried. + // Inventory slots 0..40: main, armor feet to head, off hand; + // then clearOrCountMatchingItems visits the crafting grid and carried stack. let mut removed = 0i32; let mut budget = if max < 0 { i32::MAX } else { max }; let all = |g: &Self| -> Vec> { let mut v: Vec> = g.inventory.clone(); - v.extend(g.armor.iter().cloned()); + v.extend(g.armor.iter().rev().cloned()); + v.push(g.offhand.clone()); v.extend(g.craft_grid.iter().cloned()); v.push(g.carried.clone()); v @@ -2518,7 +2520,8 @@ impl Game<'_> { .collect(); let mut slots: Vec<&mut Option> = Vec::new(); slots.extend(self.inventory.iter_mut()); - slots.extend(self.armor.iter_mut()); + slots.extend(self.armor.iter_mut().rev()); + slots.push(&mut self.offhand); slots.extend(self.craft_grid.iter_mut()); slots.push(&mut self.carried); for (slot, matches) in slots.into_iter().zip(matching) { diff --git a/tools/command_exec_corpus.py b/tools/command_exec_corpus.py index 0e376e260..4b24ca80f 100644 --- a/tools/command_exec_corpus.py +++ b/tools/command_exec_corpus.py @@ -182,6 +182,23 @@ def player_state(): cmd('/item modify entity @s weapon.mainhand {type:"minecraft:set_count",count:1}'), cmd('/clear @s'), cmd('/clear @s'), + # Inventory.clearOrCountMatchingItems visits the off hand as well as + # the main inventory, and consumes limited clears in inventory-slot order. + cmd('/item replace entity @s weapon.offhand with minecraft:shield'), + cmd('/clear @s minecraft:shield 0'), + cmd('/clear @s minecraft:shield 1'), + cmd('/clear @s minecraft:shield 0'), + cmd('/item replace entity @s weapon.offhand with minecraft:stone 5'), + cmd('/clear @s minecraft:stone 2'), + cmd('/clear @s minecraft:stone 0'), + cmd('/clear @s'), + cmd('/clear @s minecraft:stone 0'), + cmd('/item replace entity @s armor.head with minecraft:iron_helmet'), + cmd('/item replace entity @s armor.feet with minecraft:iron_boots'), + cmd('/clear @s * 1'), + cmd('/clear @s minecraft:iron_boots 0'), + cmd('/clear @s minecraft:iron_helmet 0'), + cmd('/clear @s'), cmd('/attribute @s minecraft:max_health get'), cmd('/attribute @s minecraft:max_health base get'), cmd('/attribute @s minecraft:max_health base set 30'), From 89d6e1c2a50112cb6d65de4cbfbe8af094ea5952 Mon Sep 17 00:00:00 2001 From: vlesmentie <129341611+vlesmentie@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:34:39 -0300 Subject: [PATCH 30/61] Structure chests keep the enchantments their loot rolled The loot rolls carry enchantments and potions (horizon_world::loot runs enchant_with_levels, enchant_randomly and set_potion), but chests filled when a structure is placed, and mineshaft chest minecarts, turned each LootStack into an item with loot_item, which keeps only the item, count, damage and exploration map. End city gear came out bare. Both now use loot_item_with_components, as vaults, trial spawners and decorated pots already do. Fixes #1437. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/structure_blocks.rs | 53 +++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/crates/horizon-client/src/structure_blocks.rs b/crates/horizon-client/src/structure_blocks.rs index 1e1906f6d..427323c23 100644 --- a/crates/horizon-client/src/structure_blocks.rs +++ b/crates/horizon-client/src/structure_blocks.rs @@ -165,14 +165,10 @@ pub fn block_entities( .iter_mut() .zip(loot.fill(&table, seed, 27, Some(biome))) { - *slot = stack.map(|s| { - let mut item = loot_item(&s); - // An ominous bottle's level (the default, 0, is no component). - if let Some(level) = s.ominous_amplifier.filter(|l| *l > 0) { - item.data_mut().ominous_amplifier = Some(level.clamp(0, 4) as u8); - } - item - }); + // With what the rolls put on them: enchantments + // (enchant_with_levels in End city chests), potions and an + // ominous bottle's level. + *slot = stack.map(|s| loot_item_with_components(&s)); } } out.push(((x, y, z), entity)); @@ -211,7 +207,7 @@ pub fn unpack_entity_loot( let slots: Vec> = loot .fill(&table, seed, 27, Some(biome)) .into_iter() - .map(|s| s.map(|s| loot_item(&s))) + .map(|s| s.map(|s| loot_item_with_components(&s))) .collect(); nbt.remove(&JavaString::from("LootTable")); nbt.remove(&JavaString::from("LootTableSeed")); @@ -224,6 +220,45 @@ mod tests { use super::*; use horizon_world::loot::{ExplorationMap, LootStack}; + #[test] + fn placed_chest_loot_keeps_its_enchantments_and_potions() { + let stack = + |item: &str, enchantments: Vec<(String, i32)>, potion: Option<&str>| LootStack { + item: item.into(), + count: 1, + damage: 0, + enchantments, + potion: potion.map(str::to_owned), + map: false, + exploration: None, + ominous_amplifier: None, + }; + // An End city chest's enchant_with_levels roll. + let boots = loot_item_with_components(&stack( + "minecraft:diamond_boots", + vec![ + ("minecraft:protection".into(), 3), + ("minecraft:unbreaking".into(), 2), + ], + None, + )); + assert_eq!(boots.enchantments().len(), 2); + // A book's go into stored_enchantments. + let book = loot_item_with_components(&stack( + "minecraft:enchanted_book", + vec![("minecraft:mending".into(), 1)], + None, + )); + assert!(book.enchantments().is_empty()); + assert_eq!(book.stored_enchantments().len(), 1); + let arrow = loot_item_with_components(&stack( + "minecraft:tipped_arrow", + Vec::new(), + Some("minecraft:slowness"), + )); + assert_eq!(arrow.potion(), Some("minecraft:slowness")); + } + #[test] fn exploration_maps_wait_in_custom_data_until_searched() { let mut stack = LootStack { From bb9c474c643f45ba551899bfe24ea360daeb2e6e Mon Sep 17 00:00:00 2001 From: levzzz <86072227+levzzz5154@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:48:58 +0500 Subject: [PATCH 31/61] End city banners keep their template patterns A structure template saves a banner's layers in its block entity NBT, and StructureTemplate.placeInWorld hands that NBT to the level, where BannerBlockEntity.loadAdditional reads `patterns`. Horizon built the client block entity from the block alone (BlockEntity::for_block), so the End cities' magenta banners came out bare. Fixes #1438. Co-Authored-By: [CC] --- crates/horizon-client/src/structure_blocks.rs | 89 ++++++++++++++++--- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/crates/horizon-client/src/structure_blocks.rs b/crates/horizon-client/src/structure_blocks.rs index 1e1906f6d..da5b75258 100644 --- a/crates/horizon-client/src/structure_blocks.rs +++ b/crates/horizon-client/src/structure_blocks.rs @@ -128,6 +128,44 @@ fn text(nbt: &Compound, key: &str) -> Option { } } +/// `BannerBlockEntity.loadAdditional`: the `patterns` layers and the custom +/// name a structure template saves with its banner (the End cities' tower +/// tops place black triangles this way). +fn banner_from_nbt(nbt: &Compound) -> crate::containers::Banner { + crate::containers::Banner { + layers: crate::banner::layers_from_nbt(nbt.get(&JavaString::from("patterns"))), + name: text(nbt, "CustomName"), + components: match nbt.get(&JavaString::from("components")) { + Some(Tag::Compound(c)) => c.clone(), + _ => Compound::new(), + }, + } +} + +/// The client block entity for one of a chunk's generated block entities, +/// before a container's loot is unpacked. +fn generated_block_entity(block: &str, nbt: &Compound) -> Option { + let mut entity = BlockEntity::for_block(block)?; + let tag = Tag::Compound(nbt.clone()); + // MonsterRoomFeature and the structures' setEntityId. + if let BlockEntity::Spawner(s) = &mut entity { + **s = crate::mobs::spawner::Spawner::load(&tag); + } + if let BlockEntity::TrialSpawner(s) = &mut entity { + **s = crate::mobs::trial_spawner::TrialSpawner::load(&tag); + } + if let BlockEntity::Vault(v) = &mut entity { + **v = crate::vault::Vault::load(&tag); + } + // StructureTemplate.placeInWorld hands the template's block entity NBT to + // the level, so `BannerBlockEntity.loadAdditional` keeps the banner's + // patterns (the End cities' black triangles) and its name. + if let BlockEntity::Banner(b) = &mut entity { + *b = banner_from_nbt(nbt); + } + Some(entity) +} + /// Client block entities for the generated block entities of a chunk. pub fn block_entities( chunk: &ProtoChunk, @@ -137,19 +175,9 @@ pub fn block_entities( for (pos, nbt) in &chunk.block_entities { let [x, y, z] = *pos; let block = chunk.block_name(x, y, z); - let Some(mut entity) = BlockEntity::for_block(block) else { + let Some(mut entity) = generated_block_entity(block, nbt) else { continue; }; - // MonsterRoomFeature and the structures' setEntityId. - if let BlockEntity::Spawner(s) = &mut entity { - **s = crate::mobs::spawner::Spawner::load(&Tag::Compound(nbt.clone())); - } - if let BlockEntity::TrialSpawner(s) = &mut entity { - **s = crate::mobs::trial_spawner::TrialSpawner::load(&Tag::Compound(nbt.clone())); - } - if let BlockEntity::Vault(v) = &mut entity { - **v = crate::vault::Vault::load(&Tag::Compound(nbt.clone())); - } if let (BlockEntity::Chest(slots), Some(table), Some(loot)) = (&mut entity, text(nbt, "LootTable"), loot) { @@ -278,4 +306,43 @@ mod tests { stack.exploration = None; assert!(!has_pending_map(&loot_item(&stack))); } + + /// The End cities' tower tops save their banners' patterns in the + /// template; the placed banner must keep them (`EndCityPieces`). + #[test] + fn a_generated_banner_keeps_its_template_patterns() { + let patterns = Tag::List(vec![ + Tag::compound([ + ("color", Tag::String("black".into())), + ("pattern", Tag::String("minecraft:triangle_top".into())), + ]), + Tag::compound([ + ("color", Tag::String("black".into())), + ("pattern", Tag::String("minecraft:triangle_bottom".into())), + ]), + ]); + let nbt = Tag::compound([ + ("id", Tag::String("minecraft:banner".into())), + ("patterns", patterns), + ]); + let Tag::Compound(nbt) = nbt else { + unreachable!() + }; + let Some(BlockEntity::Banner(banner)) = + generated_block_entity("minecraft:magenta_banner", &nbt) + else { + panic!("a banner block makes a banner block entity"); + }; + assert_eq!(banner.layers.len(), 2); + assert_eq!(&*banner.layers[0].pattern, "minecraft:triangle_top"); + assert_eq!(&*banner.layers[1].pattern, "minecraft:triangle_bottom"); + assert_eq!(&*banner.layers[0].color, "black"); + // A banner the template saved without patterns stays bare. + let Some(BlockEntity::Banner(bare)) = + generated_block_entity("minecraft:white_banner", &Compound::new()) + else { + panic!("a banner block makes a banner block entity"); + }; + assert!(bare.layers.is_empty()); + } } From 74e3b0adf6937575dd5c5a3d2ea99ed0cb490eec Mon Sep 17 00:00:00 2001 From: socksprox Date: Sun, 25 Oct 2026 21:10:52 +0100 Subject: [PATCH 32/61] ui: recapture the mouse when a mod's screen vanishes, borrow its widgets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review on #1864: when a mod stops while its screen or settings page is up, draw_mod_screen dropped the player back to Playing without capture(true), leaving the mouse free until the next click into the world. The dead surface now leaves through close_mod_screen, which puts back the screen it covered and holds the mouse on a return to Playing; the same goes for a screen dropped while it was up. mod_ui_view also cloned the title and every widget (up to 64, lists to 512 rows) on each frame, click and scroll. It now borrows the mod's stored widgets — the screen's scrolls live in a disjoint field — and the click finds the picked row first so the &mut ui_select runs after. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- crates/horizon-client/src/game/mods.rs | 87 +++++++++++++------------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/crates/horizon-client/src/game/mods.rs b/crates/horizon-client/src/game/mods.rs index 47a3ce07b..d2e9627e6 100644 --- a/crates/horizon-client/src/game/mods.rs +++ b/crates/horizon-client/src/game/mods.rs @@ -425,30 +425,28 @@ impl Game<'_> { } /// A `ui` screen's title and widgets as the mod last pushed them; None - /// when the surface is gone (the mod stopped or closed it). - fn mod_ui_view(&self) -> Option<(String, Vec)> { - let ms = self.mods.screen.as_ref()?; + /// when the surface is gone (the mod stopped or closed it). Borrows + /// `mods` only, so `self.mods.screen` may stay borrowed for the scrolls. + fn mod_ui_view<'a>(mods: &'a Mods, ms: &ModScreen) -> Option<(&'a str, &'a [UiWidget])> { if ms.settings { - let widgets = self.mods.mods.settings_widgets(ms.owner); + let widgets = mods.settings_widgets(ms.owner); if widgets.is_empty() { return None; } - let name = self - .mods - .mods + let name = mods .entries() .get(ms.owner) .and_then(|e| e.manifest.as_ref()) .map_or("Mod", |m| m.name.as_str()); - Some((name.to_string(), widgets.to_vec())) + Some((name, widgets)) } else { - let s: &UiScreen = self.mods.mods.screen(ms.owner)?; - Some((s.title.clone(), s.widgets.clone())) + let s: &UiScreen = mods.screen(ms.owner)?; + Some((s.title.as_str(), s.widgets.as_slice())) } } /// The layout the mod screen's widgets get at `h`: (widget, top, /// height), the column being `COLUMN` wide down the middle. - fn mod_ui_layout(&self, h: f32, widgets: &[UiWidget]) -> Vec<(f32, f32)> { + fn mod_ui_layout(h: f32, widgets: &[UiWidget]) -> Vec<(f32, f32)> { let mut out = Vec::with_capacity(widgets.len()); let mut y = MOD_UI_TOP; for w in widgets { @@ -513,13 +511,19 @@ impl Game<'_> { /// The mod's screen or settings page: its title over widgets stacked /// down the middle, a Done button, Esc to leave (vanilla's menus). pub(super) fn draw_mod_screen(&mut self, batch: &mut HudBatch) { - if self.mods.screen.is_none() { + let Some(ms) = self.mods.screen.as_mut() else { + // The screen was dropped while it was up: back to the world, + // mouse held again like any return to Playing. self.screen = Screen::Playing; + if self.player.is_some() { + self.capture(true); + } return; - } - let Some((title, widgets)) = self.mod_ui_view() else { - self.mods.screen = None; - self.screen = Screen::Playing; + }; + let Some((title, widgets)) = Self::mod_ui_view(&self.mods.mods, ms) else { + // The mod's surface is gone (it stopped or cleared it): leave + // the screen like Done did, without `closed` — the mod ended it. + self.close_mod_screen(false); return; }; // Item icons draw in a second pass: `draw_item` needs `&mut self`, @@ -539,7 +543,7 @@ impl Game<'_> { ); let cx = (batch.width / 2.0).floor(); let x = cx - MOD_UI_COLUMN / 2.0; - let layout = self.mod_ui_layout(batch.height, &widgets); + let layout = Self::mod_ui_layout(batch.height, widgets); let mut list_i = 0usize; for (w, &(wy, wh)) in widgets.iter().zip(&layout) { match w { @@ -580,11 +584,8 @@ impl Game<'_> { ); } UiWidget::List(l) => { - let scroll = { - let ms = self.mods.screen.as_mut().expect("checked"); - ms.scrolls.resize_with(list_i + 1, ui::Scroll::default); - ms.scrolls[list_i] - }; + ms.scrolls.resize_with(list_i + 1, ui::Scroll::default); + let scroll = ms.scrolls[list_i]; list_i += 1; let rows_top = wy + 4.0; let rows_bottom = wy + wh - 4.0; @@ -655,53 +656,53 @@ impl Game<'_> { self.close_mod_screen(true); return; } - let Some((_, widgets)) = self.mod_ui_view() else { + let Some(ms) = self.mods.screen.as_ref() else { return; }; + let Some((_, widgets)) = Self::mod_ui_view(&self.mods.mods, ms) else { + return; + }; + let owner = ms.owner; let x = (w / 2.0).floor() - MOD_UI_COLUMN / 2.0; - let layout = self.mod_ui_layout(h, &widgets); + let layout = Self::mod_ui_layout(h, widgets); let mut list_i = 0usize; + // The `ui_select` call needs `&mut`: find the pick first. + let mut hit = None; for (w, &(wy, wh)) in widgets.iter().zip(&layout) { let UiWidget::List(l) = w else { continue }; - let scroll = self - .mods - .screen - .as_ref() - .and_then(|ms| ms.scrolls.get(list_i).copied()) - .unwrap_or_default(); + let scroll = ms.scrolls.get(list_i).copied().unwrap_or_default(); list_i += 1; let row = ((cursor.1 - (wy + 4.0) + scroll.amount) / 12.0) as i64; if ui::hovered(cursor, x + 1.0, wy + 1.0, MOD_UI_COLUMN - 7.0, wh - 2.0) && row >= 0 && (row as usize) < l.rows.len() { - self.audio.play_button_click(); - self.mods.mods.ui_select( - self.mods.screen.as_ref().map_or(0, |ms| ms.owner), - l.id, - row as u32, - ); - self.show_mod_output(); - return; + hit = Some((l.id, row as u32)); + break; } } + if let Some((id, row)) = hit { + self.audio.play_button_click(); + self.mods.mods.ui_select(owner, id, row); + self.show_mod_output(); + } } /// The wheel over a `list` scrolls it. pub(super) fn mod_screen_scroll(&mut self, delta: f32) { let (w, h) = self.gui_size(); let cursor = self.cursor; - let Some((_, widgets)) = self.mod_ui_view() else { + let Some(ms) = self.mods.screen.as_mut() else { + return; + }; + let Some((_, widgets)) = Self::mod_ui_view(&self.mods.mods, ms) else { return; }; let x = (w / 2.0).floor() - MOD_UI_COLUMN / 2.0; - let layout = self.mod_ui_layout(h, &widgets); + let layout = Self::mod_ui_layout(h, widgets); let mut list_i = 0usize; for (w, &(wy, wh)) in widgets.iter().zip(&layout) { let UiWidget::List(l) = w else { continue }; let over = ui::hovered(cursor, x, wy, MOD_UI_COLUMN, wh); - let Some(ms) = self.mods.screen.as_mut() else { - return; - }; ms.scrolls.resize_with(list_i + 1, ui::Scroll::default); if over { let content = l.rows.len() as f32 * 12.0; From 386912a76d4046a6026e6caa331b5ee5576e217b Mon Sep 17 00:00:00 2001 From: sl4ppleware <66299946+sl4ppleware@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:32:27 +0300 Subject: [PATCH 33/61] Hearts blink again when the server sets health: LocalPlayer.hurtTo's damageCooldownTime --- crates/horizon-client/src/health.rs | 10 ++++++++-- crates/horizon-client/src/motion.rs | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/horizon-client/src/health.rs b/crates/horizon-client/src/health.rs index da40f41fc..25c719bfb 100644 --- a/crates/horizon-client/src/health.rs +++ b/crates/horizon-client/src/health.rs @@ -261,11 +261,17 @@ impl PlayerHealth { self.hurt_animation_time = 10; } /// `LocalPlayer.hurtTo` with `flashOnSetHealth`: a drop below the current - /// health starts the hurt animation. + /// health starts the hurt animation; `damageCooldownTime` (20 on a drop, + /// 10 on a gain) is what lets `Hud.extractPlayerHealth` blink the hearts. pub fn flash_health_change(&mut self, health: f32) { let damage = self.health() - health; - if damage > 0.0 { + if damage <= 0.0 { + if damage < 0.0 { + self.hurt.cooldown = 10; + } + } else { self.hurt.last_hurt = damage; + self.hurt.cooldown = 20; self.animate_hurt(); } } diff --git a/crates/horizon-client/src/motion.rs b/crates/horizon-client/src/motion.rs index e7809055d..a5e3813b1 100644 --- a/crates/horizon-client/src/motion.rs +++ b/crates/horizon-client/src/motion.rs @@ -2934,8 +2934,13 @@ mod tests { if client_side { w.player.health.flash_health_change(20.0); assert_eq!(w.player.health.hurt_animation_time, 0); + assert_eq!(w.player.health.hurt.cooldown, 0); w.player.health.flash_health_change(12.0); assert_eq!(w.player.health.hurt_animation_time, 10); + // LocalPlayer.hurtTo's damageCooldownTime: the HUD's hearts blink. + assert_eq!(w.player.health.hurt.cooldown, 20); + w.player.health.flash_health_change(25.0); + assert_eq!(w.player.health.hurt.cooldown, 10); } } } From 61c44c22efe23f9cbc4ac00ef8c8f20a35b00aa4 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:35:12 +0100 Subject: [PATCH 34/61] Mod API: capability render (experimental): HUD shapes and camera shake Mods draw through the game's own renderer, never their own shaders: - WIT `render`: set-hud(shapes) with fill, quad (any four corners), sprite (the game's GUI sprites), item (scaled) and text shapes, up to 1024, kept until replaced; gui-size(); shake(strength, seconds). - Drawn over the HUD after the extras, under chat and screens, hidden with F1 (game/mod_render.rs). Colours are 0xAARRGGBB. - shake turns the view, not the player: jolts fade out squared, add up to 1, at most 3 degrees of yaw and 2 of pitch, scaled by Distortion Effects (screenEffectScale; 0 turns it off). - Experimental: the manifest needs experimental = ["render"]. - No mod drawing, no cost: an empty iterator and an is_empty check. - cargo horizon-mod check reports shakes and HUD shapes. The ui test guest gains /render_hud, /render_clear and /render_shake, with a sandbox test; HudBatch gets fill_quad. - Docs: mods.md, stability.md, the changelog, the regenerated reference. Co-Authored-By: Claude Opus 5.5 --- crates/cargo-horizon-mod/src/check.rs | 9 + crates/horizon-client/src/game.rs | 3 + crates/horizon-client/src/game/mod_render.rs | 176 ++++++++++++++++++ crates/horizon-client/src/game/mods.rs | 15 +- crates/horizon-client/src/hud.rs | 6 + crates/horizon-mod-sdk/src/lib.rs | 2 +- crates/horizon-mod-sdk/wit/horizon.wit | 82 ++++++++ crates/horizon-mods/src/manifest.rs | 11 +- crates/horizon-mods/src/runtime.rs | 81 ++++++++ crates/horizon-mods/tests/sandbox.rs | 38 +++- crates/horizon-mods/tests/ui_guest/src/lib.rs | 71 ++++++- docs/api/mods-reference.md | 94 +++++++++- docs/api/mods.md | 2 + docs/api/stability.md | 1 + docs/mods.md | 5 +- 15 files changed, 582 insertions(+), 14 deletions(-) create mode 100644 crates/horizon-client/src/game/mod_render.rs diff --git a/crates/cargo-horizon-mod/src/check.rs b/crates/cargo-horizon-mod/src/check.rs index 5d4150ab9..dfb9820a2 100644 --- a/crates/cargo-horizon-mod/src/check.rs +++ b/crates/cargo-horizon-mod/src/check.rs @@ -198,6 +198,8 @@ fn play(path: &Path, report: &mut Report) -> Result<(), String> { game_time: t, day_time: 1000 + t, }), + // The GUI of a 1280 x 720 window at scale 3. + gui_size: (427.0, 240.0), }); mods.tick(t); for _ in 0..FRAMES_PER_TICK { @@ -217,8 +219,15 @@ fn play(path: &Path, report: &mut Report) -> Result<(), String> { Output::Stopped { reason, .. } => return Err(format!("it stopped: {reason}")), Output::OpenScreen { .. } => "opened a screen".to_string(), Output::CloseScreen { .. } => "closed its screen".to_string(), + Output::Shake { strength, seconds } => { + format!("shook the camera: {strength:.2} for {seconds:.1} s") + } }); } + let shapes = mods.hud_shapes().count(); + if shapes > 0 { + report.output.push(format!("drew {shapes} HUD shape(s)")); + } match failed(&mods) { Some(reason) => Err(format!("it stopped: {reason}")), None => Ok(()), diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index eabbc22b8..31bfe74b6 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -96,6 +96,7 @@ mod maps; mod minecarts; mod mob_blocks; mod mob_gear; +mod mod_render; mod mods; mod mount_menu; mod mounts; @@ -4894,6 +4895,7 @@ impl Game<'_> { // Hud: the action bar and titles draw over the hotbar (nextStratum). self.draw_boss_bars(&mut batch); self.draw_extras(&mut batch); + self.draw_mod_shapes(&mut batch); } if show_hud { // Hud.nextContextualInfoState: the jump bar, the experience bar or @@ -9865,6 +9867,7 @@ impl Game<'_> { self.pitch, ), }; + let (view_yaw, view_pitch) = self.shaken_view(view_yaw, view_pitch); self.audio .set_listener([camera.x, camera.y, camera.z], view_yaw, view_pitch); let block = [ diff --git a/crates/horizon-client/src/game/mod_render.rs b/crates/horizon-client/src/game/mod_render.rs new file mode 100644 index 000000000..1c3a20621 --- /dev/null +++ b/crates/horizon-client/src/game/mod_render.rs @@ -0,0 +1,176 @@ +//! Capability `render` (docs/mods.md): the shapes mods keep on the HUD, +//! drawn by the game's own HUD renderer, and their camera shakes. Mods +//! never supply shader code. With no mod drawing, each hook here returns +//! after one check. +use super::Game; +use horizon_client::hud::HudBatch; +#[cfg(feature = "mods")] +use { + horizon_client::{hud::rgba, items::ItemStack, text::Text}, + horizon_mods::Shape, + std::time::Instant, +}; + +/// The camera's jolts from `render.shake`, while any is fading. +#[cfg(feature = "mods")] +#[derive(Default)] +pub(super) struct Shake { + jolts: Vec, +} + +#[cfg(feature = "mods")] +struct Jolt { + start: Instant, + strength: f32, + seconds: f32, +} + +/// The HUD batch's colour (ABGR) from a mod's 0xAARRGGBB. +#[cfg(feature = "mods")] +fn argb(color: u32) -> u32 { + let [a, r, g, b] = color.to_be_bytes(); + rgba(r, g, b, a) +} + +/// The view's turn at full strength, in degrees: yaw, then pitch. +#[cfg(feature = "mods")] +const SHAKE_DEGREES: (f32, f32) = (3.0, 2.0); + +#[cfg(feature = "mods")] +impl Shake { + pub(super) fn add(&mut self, strength: f32, seconds: f32) { + self.jolts.push(Jolt { + start: Instant::now(), + strength, + seconds, + }); + } + /// How far the view turns now (yaw, pitch degrees), `scale` being the + /// player's Distortion Effects; fades jolts out and forgets ended ones. + fn offset(&mut self, now: Instant, scale: f32) -> (f32, f32) { + self.jolts + .retain(|j| now.duration_since(j.start).as_secs_f32() < j.seconds); + let mut strength = 0.0; + for j in &self.jolts { + let left = 1.0 - now.duration_since(j.start).as_secs_f32() / j.seconds; + strength += j.strength * left * left; + } + let strength = strength.min(1.0) * scale.clamp(0.0, 1.0); + if strength <= 0.0 { + return (0.0, 0.0); + } + // Smooth noise: sines at unrelated rates, so it never repeats visibly. + let t = self + .jolts + .first() + .map_or(0.0, |j| now.duration_since(j.start).as_secs_f32()); + let wave = |a: f32, b: f32, c: f32| { + ((t * a).sin() * 0.5 + (t * b).sin() * 0.3 + (t * c).sin() * 0.2).clamp(-1.0, 1.0) + }; + ( + strength * SHAKE_DEGREES.0 * wave(23.0, 37.3, 51.7), + strength * SHAKE_DEGREES.1 * wave(29.1, 43.9, 61.3), + ) + } +} + +#[cfg(not(feature = "mods"))] +impl Game<'_> { + pub(super) fn draw_mod_shapes(&self, _batch: &mut HudBatch) {} + pub(super) fn shaken_view(&mut self, yaw: f32, pitch: f32) -> (f32, f32) { + (yaw, pitch) + } +} + +#[cfg(feature = "mods")] +impl Game<'_> { + /// The running mods' HUD shapes, in order, over the HUD. + pub(super) fn draw_mod_shapes(&self, batch: &mut HudBatch) { + let atlas = &self.gui.atlas; + for shape in self.mods.mods.hud_shapes() { + match shape { + Shape::Fill(f) => { + let r = f.rect; + batch.fill(atlas, r.x, r.y, r.w, r.h, argb(f.color)); + } + Shape::Quad(q) => { + let (a, b, c, d) = q.corners; + let corners = [[a.x, a.y], [b.x, b.y], [c.x, c.y], [d.x, d.y]]; + batch.fill_quad(atlas, corners, argb(q.color)); + } + Shape::Sprite(s) => { + let r = s.rect; + let name = s.sprite.strip_prefix("minecraft:").unwrap_or(&s.sprite); + let name = format!("gui/sprites/{name}"); + batch.sprite_scaled(atlas, &name, r.x, r.y, r.w, r.h, argb(s.color)); + } + Shape::Item(i) => { + if self.res.items().get(&i.item).is_none() { + continue; + } + let start = batch.vertices.len(); + let stack = ItemStack::new(i.item.as_str(), 1); + self.draw_item_model(batch, &stack, i.at.x, i.at.y); + batch.scale_since(start, i.at.x, i.at.y, i.scale, i.scale); + } + Shape::Text(t) => { + let text = Text::literal(t.text.clone()); + let x = if t.centered { + t.at.x - (HudBatch::text_width(atlas, &t.text) / 2.0).floor() + } else { + t.at.x + }; + batch.text_styled( + atlas, + &text, + x, + t.at.y, + argb(t.color | 0xFF00_0000), + t.shadow, + ); + } + } + } + } + /// The view's yaw and pitch after the mods' camera shakes; unchanged + /// while none is fading. + pub(super) fn shaken_view(&mut self, yaw: f32, pitch: f32) -> (f32, f32) { + if self.mods.shake.jolts.is_empty() { + return (yaw, pitch); + } + let scale = self.settings.file.double("screenEffectScale") as f32; + let (dy, dp) = self.mods.shake.offset(Instant::now(), scale); + (yaw + dy, (pitch + dp).clamp(-90.0, 90.0)) + } +} + +#[cfg(all(test, feature = "mods"))] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn a_jolt_fades_out_and_distortion_scales_it() { + let mut shake = Shake::default(); + shake.add(1.0, 2.0); + let start = shake.jolts[0].start; + let at = |s: f32| start + Duration::from_secs_f32(s); + let size = |(y, p): (f32, f32)| y.abs() + p.abs(); + // Off with Distortion Effects at 0, inside the limits otherwise. + assert_eq!(shake.offset(at(0.3), 0.0), (0.0, 0.0)); + let (y, p) = shake.offset(at(0.3), 1.0); + assert!(y.abs() <= SHAKE_DEGREES.0 && p.abs() <= SHAKE_DEGREES.1); + // Fading: the largest turn over a window shrinks. + let mut peak = |from: f32| { + (0..50) + .map(|i| size(shake.offset(at(from + i as f32 * 0.002), 1.0))) + .fold(0.0, f32::max) + }; + let early = peak(0.1); + let late = peak(1.6); + assert!(late < early, "{late} !< {early}"); + // Gone once it ends. + assert_eq!(shake.offset(at(2.1), 1.0), (0.0, 0.0)); + assert!(shake.jolts.is_empty()); + } +} diff --git a/crates/horizon-client/src/game/mods.rs b/crates/horizon-client/src/game/mods.rs index 47a3ce07b..2a07cb940 100644 --- a/crates/horizon-client/src/game/mods.rs +++ b/crates/horizon-client/src/game/mods.rs @@ -33,7 +33,10 @@ use { #[derive(Default)] pub(super) struct ModsState { #[cfg(feature = "mods")] - mods: Mods, + pub(super) mods: Mods, + /// Capability `render`'s camera shakes. + #[cfg(feature = "mods")] + pub(super) shake: super::mod_render::Shake, /// `--dev-mod PATH`: always on, reloaded when rebuilt. pub(super) dev: Vec, /// When dev mods were last checked for a rebuild. @@ -308,7 +311,11 @@ impl Game<'_> { game_time: self.world.game_time.max(0) as u64, day_time: self.world.day_time.max(0) as u64, }); - Snapshot { player, clock } + Snapshot { + player, + clock, + gui_size: self.gui_size(), + } } /// A game tick. pub(super) fn tick_mods(&mut self) { @@ -382,6 +389,10 @@ impl Game<'_> { self.open_mod_screen(owner, false); continue; } + Output::Shake { strength, seconds } => { + self.mods.shake.add(strength, seconds); + continue; + } Output::CloseScreen { owner } => { if self.mods.screen.as_ref().is_some_and(|s| s.owner == owner) { self.close_mod_screen(false); diff --git a/crates/horizon-client/src/hud.rs b/crates/horizon-client/src/hud.rs index 89d1b1c3c..c3a165792 100644 --- a/crates/horizon-client/src/hud.rs +++ b/crates/horizon-client/src/hud.rs @@ -835,6 +835,12 @@ impl HudBatch { r.v1 = cv; self.rect_uv(x, y, w, h, r, color); } + /// Any four-cornered shape in one colour, corners in order round it. + pub fn fill_quad(&mut self, atlas: &HudAtlas, corners: [[f32; 2]; 4], color: u32) { + let w = atlas.white; + let uv = [(w.u0 + w.u1) / 2.0, (w.v0 + w.v1) / 2.0]; + self.quad(corners, [uv; 4], color); + } /// GuiGraphics.fillGradient: `top` to `bottom`, down the rectangle. pub fn fill_gradient( &mut self, diff --git a/crates/horizon-mod-sdk/src/lib.rs b/crates/horizon-mod-sdk/src/lib.rs index aea52339f..8708fb5be 100644 --- a/crates/horizon-mod-sdk/src/lib.rs +++ b/crates/horizon-mod-sdk/src/lib.rs @@ -38,7 +38,7 @@ wit_bindgen::generate!({ pub use exports::horizon::mods::events::{Guest as __Guest, Registration}; pub use exports::horizon::mods::ui_events::Guest as __UiEventsGuest; -pub use horizon::mods::{chat, game, hud, log, storage, ui}; +pub use horizon::mods::{chat, game, hud, log, render, storage, ui}; /// The game's calls into a mod. Only the hooks [`Registration`] asks for /// are called; the others may stay as their defaults. diff --git a/crates/horizon-mod-sdk/wit/horizon.wit b/crates/horizon-mod-sdk/wit/horizon.wit index 1e49919c3..64bbac592 100644 --- a/crates/horizon-mod-sdk/wit/horizon.wit +++ b/crates/horizon-mod-sdk/wit/horizon.wit @@ -163,6 +163,82 @@ interface ui-events { settings-opened: func(); } +/// Capability `render` (experimental: the manifest needs +/// `experimental = ["render"]`): shapes the game draws for the mod with its +/// own renderer, kept until the mod replaces them (no call per frame). A mod +/// never supplies shader code. +interface render { + /// A rectangle in GUI pixels: the top left corner and the size. + record rect { + x: f32, + y: f32, + w: f32, + h: f32, + } + /// A point in GUI pixels from the screen's top left. + record point { + x: f32, + y: f32, + } + record fill-shape { + rect: rect, + /// 0xAARRGGBB. + color: u32, + } + /// Any four-cornered shape (a wheel's segment, an arrow): the corners + /// in order round it. + record quad-shape { + corners: tuple, + /// 0xAARRGGBB. + color: u32, + } + record sprite-shape { + /// One of the game's GUI sprites, as `gui/sprites/` names them + /// (`hud/heart/full`, `widget/button`), the resource packs' versions + /// included. An unknown one draws nothing. + sprite: string, + rect: rect, + /// A tint, 0xAARRGGBB: 0xFFFFFFFF draws it as it is. + color: u32, + } + record item-shape { + /// `minecraft:diamond`, `:`; an unknown one draws nothing. + item: string, + /// The icon's top left. + at: point, + /// 1.0 is the hotbar's 16 × 16. + scale: f32, + } + record text-shape { + /// Up to 256 characters, one line. + text: string, + at: point, + /// 0xRRGGBB, as `hud`'s lines. + color: u32, + /// Centred on `at.x` instead of starting there. + centered: bool, + shadow: bool, + } + variant shape { + fill(fill-shape), + quad(quad-shape), + sprite(sprite-shape), + item(item-shape), + text(text-shape), + } + /// Replaces the mod's HUD shapes: up to 1024, drawn every frame in + /// order over the HUD (after the hotbar, under chat and screens), hidden + /// with the HUD (F1). An empty list removes them. + set-hud: func(shapes: list); + /// The screen's size in GUI pixels (width, height), as of this tick. + gui-size: func() -> tuple; + /// Shakes the camera: a jolt of `strength` (0 to 1, 1 the hardest) that + /// fades out over `seconds` (at most 10). Jolts add up to 1. The view + /// turns, the player doesn't, and the player's Distortion Effects option + /// scales it (0 turns it off). + shake: func(strength: f32, seconds: f32); +} + /// What the game calls. Only the hooks `init` asks for are ever called. interface events { record registration { @@ -203,6 +279,7 @@ world horizon-mod { world horizon-ui-mod { include horizon-mod; import ui; + import render; export ui-events; } @@ -211,3 +288,8 @@ world ui-host { import ui; export ui-events; } + +/// The game's side of capability `render` (imports only). +world render-host { + import render; +} diff --git a/crates/horizon-mods/src/manifest.rs b/crates/horizon-mods/src/manifest.rs index 6070f99bd..1880b9f3e 100644 --- a/crates/horizon-mods/src/manifest.rs +++ b/crates/horizon-mods/src/manifest.rs @@ -84,15 +84,19 @@ pub enum Capability { /// A screen the mod owns and a settings page in Options > Mods, drawn /// with the menus' own widgets (interface `ui`, calls `ui-events`). Ui, + /// Shapes on the HUD and the camera's shake, drawn by the game's own + /// renderer (interface `render`). + Render, } impl Capability { - pub const ALL: [Capability; 6] = [ + pub const ALL: [Capability; 7] = [ Capability::GameRead, Capability::Chat, Capability::Hud, Capability::Storage, Capability::Registries, Capability::Ui, + Capability::Render, ]; /// The WIT interface it grants, in `horizon:mods`. pub fn interface(self) -> &'static str { @@ -103,12 +107,13 @@ impl Capability { Capability::Storage => "storage", Capability::Registries => "registries", Capability::Ui => "ui", + Capability::Render => "render", } } /// May change in any release: a mod uses it only with /// `experimental = [""]` in its manifest. pub fn is_experimental(self) -> bool { - matches!(self, Capability::Ui) + matches!(self, Capability::Ui | Capability::Render) } /// What the player reads before turning the mod on: "May ". pub fn describe(self) -> &'static str { @@ -119,6 +124,7 @@ impl Capability { Capability::Storage => "save its own settings to disk", Capability::Registries => "add blocks and items to your worlds", Capability::Ui => "show its own screens and a settings page", + Capability::Render => "draw on your screen and shake the camera", } } /// The capability a component import needs, `Ok(None)` for one every @@ -150,6 +156,7 @@ impl fmt::Display for Capability { Capability::Storage => "storage", Capability::Registries => "registries", Capability::Ui => "ui", + Capability::Render => "render", }; f.write_str(name) } diff --git a/crates/horizon-mods/src/runtime.rs b/crates/horizon-mods/src/runtime.rs index 1b1795037..e19bfbe3f 100644 --- a/crates/horizon-mods/src/runtime.rs +++ b/crates/horizon-mods/src/runtime.rs @@ -40,6 +40,19 @@ pub use ui_bindings::horizon::mods::ui::{ ItemWidget, ListWidget, Screen as UiScreen, TextWidget, Widget as UiWidget, }; use ui_bindings::{UiHost, horizon::mods::ui}; +/// Capability `render`'s imports (docs/mods.md): the game draws the shapes. +mod render_bindings { + wasmtime::component::bindgen!({ + path: "../horizon-mod-sdk/wit", + world: "render-host", + }); +} +/// What capability `render` asks the game to draw (WIT `interface render`). +pub use render_bindings::horizon::mods::render::{ + FillShape, ItemShape, Point as RenderPoint, QuadShape, Rect as RenderRect, Shape, SpriteShape, + TextShape, +}; +use render_bindings::{RenderHost, horizon::mods::render}; /// A mod's linear memory cap. const MEMORY: usize = 64 << 20; @@ -70,6 +83,8 @@ const STORAGE_WRITE_BUDGET: u64 = 1 << 20; pub struct Snapshot { pub player: Option, pub clock: Option, + /// The screen in GUI pixels (capability `render`'s `gui-size`). + pub gui_size: (f32, f32), } /// What mods asked the game to show since the last [`Mods::drain`]. @@ -86,6 +101,8 @@ pub enum Output { OpenScreen { owner: usize }, /// `ui.close-screen`: the mod closed its own screen (no `closed` back). CloseScreen { owner: usize }, + /// `render.shake`: strength 0 to 1, fading out over `seconds` (≤ 10). + Shake { strength: f32, seconds: f32 }, } /// A mod's text panel, drawn every frame until it changes. @@ -153,6 +170,8 @@ struct State { out: Vec, panel: Option, ui: UiState, + /// Capability `render`'s HUD shapes, drawn until replaced. + shapes: Vec, /// Capability `storage`'s folder; made on first `set`. data_dir: PathBuf, /// Bytes `storage::set` may still write before this hook call ends @@ -231,6 +250,8 @@ impl Mods { .map_err(|e| e.to_string())?; UiHost::add_to_linker::>(&mut linker, |s| s) .map_err(|e| e.to_string())?; + RenderHost::add_to_linker::>(&mut linker, |s| s) + .map_err(|e| e.to_string())?; self.engine = Some((engine, linker)); } Ok(self.engine.as_ref().expect("made above")) @@ -290,6 +311,7 @@ impl Mods { out: Vec::new(), panel: None, ui: UiState::default(), + shapes: Vec::new(), data_dir, storage_budget: STORAGE_WRITE_BUDGET, limits: StoreLimitsBuilder::new() @@ -481,6 +503,14 @@ impl Mods { .iter() .filter_map(|e| e.run.as_ref()?.store.data().panel.as_ref()) } + /// The running mods' HUD shapes (capability `render`), each mod's in + /// its own order, mods in load order. + pub fn hud_shapes(&self) -> impl Iterator { + self.entries + .iter() + .filter_map(|e| e.run.as_ref()) + .flat_map(|r| &r.store.data().shapes) + } /// The screen mod `i` last pushed with `ui.open-screen`, while it is up. pub fn screen(&self, i: usize) -> Option<&UiScreen> { self.entries @@ -668,6 +698,57 @@ impl chat::Host for State { } } } +/// HUD shapes one mod may keep (capability `render`). +const SHAPES: usize = 1024; +/// The longest jolt `render.shake` makes, in seconds. +const SHAKE_SECONDS: f32 = 10.0; + +impl render::Host for State { + fn set_hud(&mut self, shapes: Vec) { + self.shapes = shapes + .into_iter() + .take(SHAPES) + .map(|shape| match shape { + Shape::Sprite(mut s) => { + s.sprite = clip(s.sprite); + Shape::Sprite(s) + } + Shape::Item(mut i) => { + i.item = clip(i.item); + i.scale = if i.scale.is_finite() { + i.scale.clamp(0.0, 16.0) + } else { + 1.0 + }; + Shape::Item(i) + } + Shape::Text(mut t) => { + t.text = clip(t.text); + t.color &= 0xFF_FFFF; + Shape::Text(t) + } + other => other, + }) + .collect(); + } + fn gui_size(&mut self) -> (f32, f32) { + self.snapshot.gui_size + } + fn shake(&mut self, strength: f32, seconds: f32) { + let ok = |v: f32, max: f32| { + if v.is_finite() { + v.clamp(0.0, max) + } else { + 0.0 + } + }; + let (strength, seconds) = (ok(strength, 1.0), ok(seconds, SHAKE_SECONDS)); + if strength > 0.0 && seconds > 0.0 { + self.out.push(Output::Shake { strength, seconds }); + } + } +} + impl hud::Host for State { fn set_panel(&mut self, corner: Corner, lines: Vec) { let lines = lines diff --git a/crates/horizon-mods/tests/sandbox.rs b/crates/horizon-mods/tests/sandbox.rs index 77f0975f3..dac333fe6 100644 --- a/crates/horizon-mods/tests/sandbox.rs +++ b/crates/horizon-mods/tests/sandbox.rs @@ -1,7 +1,7 @@ //! Loads real mods built with the SDK: the coords-hud example and a guest //! that misbehaves on command. Needs `rustup target add wasm32-unknown-unknown`. use horizon_mods::{ - Clock, Corner, Mods, Output, Package, Player, Snapshot, Status, manifest::componentize, + Clock, Corner, Mods, Output, Package, Player, Shape, Snapshot, Status, manifest::componentize, }; use std::{ path::{Path, PathBuf}, @@ -22,8 +22,8 @@ id = "test_ui_guest" name = "Test UI Guest" version = "0.1.0" api = "0.1" -capabilities = ["chat", "ui"] -experimental = ["ui"] +capabilities = ["chat", "ui", "render"] +experimental = ["ui", "render"] "#; /// Builds both mods once and writes their `.hmod` files; returns the folder. @@ -89,6 +89,7 @@ fn player() -> Snapshot { game_time: 100, day_time: 24000 + 6000, }), + gui_size: (427.0, 240.0), } } @@ -416,6 +417,37 @@ fn ui_screens_settings_and_events() { assert!(mods.screen(0).is_none()); } +#[test] +fn render_shapes_and_shakes() { + let mut mods = load("horizon_mods_test_ui_guest.hmod"); + mods.set_snapshot(Snapshot { + gui_size: (427.0, 240.0), + ..Snapshot::default() + }); + assert_eq!(mods.hud_shapes().count(), 0); + assert!(mods.command("render_hud")); + let shapes: Vec<_> = mods.hud_shapes().cloned().collect(); + assert_eq!(shapes.len(), 5); + // The guest read `gui-size`; text is clipped to 256 characters. + let Shape::Text(t) = &shapes[4] else { + panic!("{:?}", shapes[4]) + }; + assert!(t.text.starts_with("gui 427x240 "), "{}", t.text); + assert_eq!(t.text.chars().count(), 256); + assert!(mods.command("render_clear")); + assert_eq!(mods.hud_shapes().count(), 0); + // One jolt, clamped; zero and NaN shake nothing. + mods.drain(); + assert!(mods.command("render_shake")); + assert_eq!( + mods.drain(), + [Output::Shake { + strength: 1.0, + seconds: 10.0 + }] + ); +} + #[test] fn content_only_mods_load_without_an_engine() { let manifest = r#" diff --git a/crates/horizon-mods/tests/ui_guest/src/lib.rs b/crates/horizon-mods/tests/ui_guest/src/lib.rs index 1af3985a1..146ab6f40 100644 --- a/crates/horizon-mods/tests/ui_guest/src/lib.rs +++ b/crates/horizon-mods/tests/ui_guest/src/lib.rs @@ -1,11 +1,61 @@ //! A mod for the runtime's `ui` tests: `/ui_screen` opens a screen with one //! of each widget, `/ui_settings` sets a settings page, `/ui_close` closes //! the screen; the `ui-events` calls answer in chat so the host sees them. +//! `/render_hud`, `/render_clear` and `/render_shake` test `render`. use horizon_mod_sdk::{ Mod, Registration, chat, + render::{self, FillShape, ItemShape, Point, QuadShape, Rect, Shape, SpriteShape, TextShape}, ui::{self, ItemWidget, ListWidget, Screen, TextWidget, Widget}, }; +/// One of each shape, the text too long by 44 characters. +fn shapes() -> Vec { + let (w, h) = render::gui_size(); + let p = |x, y| Point { x, y }; + vec![ + Shape::Fill(FillShape { + rect: Rect { + x: 4.0, + y: 4.0, + w: 60.0, + h: 20.0, + }, + color: 0x80FF0000, + }), + Shape::Quad(QuadShape { + corners: ( + p(w / 2.0, 10.0), + p(w / 2.0 + 20.0, 30.0), + p(w / 2.0, 50.0), + p(w / 2.0 - 20.0, 30.0), + ), + color: 0xC000FF00, + }), + Shape::Sprite(SpriteShape { + sprite: "hud/heart/full".into(), + rect: Rect { + x: 70.0, + y: 4.0, + w: 18.0, + h: 18.0, + }, + color: 0xFFFFFFFF, + }), + Shape::Item(ItemShape { + item: "minecraft:diamond".into(), + at: p(92.0, 4.0), + scale: 2.0, + }), + Shape::Text(TextShape { + text: format!("gui {w}x{h} {}", "x".repeat(300)), + at: p(w / 2.0, h / 2.0 - 30.0), + color: 0xFFFF55, + centered: true, + shadow: true, + }), + ] +} + struct Guest; fn widgets() -> Vec { @@ -30,9 +80,16 @@ fn widgets() -> Vec { impl Mod for Guest { fn init() -> Result<(Self, Registration), String> { - let hooks = ["ui_screen", "ui_settings", "ui_close"] - .into_iter() - .fold(Registration::new(), Registration::command); + let hooks = [ + "ui_screen", + "ui_settings", + "ui_close", + "render_hud", + "render_clear", + "render_shake", + ] + .into_iter() + .fold(Registration::new(), Registration::command); Ok((Guest, hooks)) } fn command(&mut self, name: &str, _args: &str) -> Result, String> { @@ -43,6 +100,14 @@ impl Mod for Guest { }), "ui_settings" => ui::set_settings(&widgets()), "ui_close" => ui::close_screen(), + "render_hud" => render::set_hud(&shapes()), + "render_clear" => render::set_hud(&[]), + // Clamped to (1, 10); the two after it shake nothing. + "render_shake" => { + render::shake(2.0, 99.0); + render::shake(0.0, 1.0); + render::shake(f32::NAN, 1.0); + } _ => {} } Ok(None) diff --git a/docs/api/mods-reference.md b/docs/api/mods-reference.md index 0812cde54..ef889d599 100644 --- a/docs/api/mods-reference.md +++ b/docs/api/mods-reference.md @@ -13,6 +13,7 @@ The contract between Horizon and its mods: the WIT package `horizon:mods@0.1.0`, | `storage` | [`storage`](#storage) | save its own settings to disk | | `registries` | data files (`assets/`, `data/`), no interface | add blocks and items to your worlds | | `ui` | [`ui`](#ui) | show its own screens and a settings page | +| `render` | [`render`](#render) | draw on your screen and shake the camera | ## World `horizon-mod` @@ -23,7 +24,7 @@ The contract between Horizon and its mods: the WIT package `horizon:mods@0.1.0`, A mod with widgets (capability `ui`): the SDK builds every mod against this world. Mods built for `horizon-mod` alone (no `ui-events` export) still load and simply hear nothing of their widgets. -- Imports (what the game offers): `ui`, `log`, `game`, `chat`, `hud`, `storage` +- Imports (what the game offers): `ui`, `render`, `log`, `game`, `chat`, `hud`, `storage` - Exports (what the mod provides): `ui-events`, `events` ## World `ui-host` @@ -33,6 +34,13 @@ The game's side of `horizon-ui-mod` beyond `horizon-mod`. - Imports (what the game offers): `ui` - Exports (what the mod provides): `ui-events` +## World `render-host` + +The game's side of capability `render` (imports only). + +- Imports (what the game offers): `render` +- Exports (what the mod provides): + ## `log` Access: always granted. @@ -205,6 +213,90 @@ What the game calls on a mod that shows widgets (capability `ui`): what the play | `closed` | `func()` | The player left the mod's screen (Esc or Done); not for a screen the mod closed itself or one that went away with the mod. | | `settings-opened` | `func()` | The player opened the mod's settings page in Options > Mods: call `ui::set-settings` now to fill it with fresh widgets. | +## `render` + +Access: capability `render`. + +Capability `render` (experimental: the manifest needs `experimental = ["render"]`): shapes the game draws for the mod with its own renderer, kept until the mod replaces them (no call per frame). A mod never supplies shader code. + +### record `rect` + +A rectangle in GUI pixels: the top left corner and the size. + +| Field | Type | | +| --- | --- | --- | +| `x` | `f32` | | +| `y` | `f32` | | +| `w` | `f32` | | +| `h` | `f32` | | + +### record `point` + +A point in GUI pixels from the screen's top left. + +| Field | Type | | +| --- | --- | --- | +| `x` | `f32` | | +| `y` | `f32` | | + +### record `fill-shape` + +| Field | Type | | +| --- | --- | --- | +| `rect` | `rect` | | +| `color` | `u32` | 0xAARRGGBB. | + +### record `quad-shape` + +Any four-cornered shape (a wheel's segment, an arrow): the corners in order round it. + +| Field | Type | | +| --- | --- | --- | +| `corners` | `tuple` | | +| `color` | `u32` | 0xAARRGGBB. | + +### record `sprite-shape` + +| Field | Type | | +| --- | --- | --- | +| `sprite` | `string` | One of the game's GUI sprites, as `gui/sprites/` names them (`hud/heart/full`, `widget/button`), the resource packs' versions included. An unknown one draws nothing. | +| `rect` | `rect` | | +| `color` | `u32` | A tint, 0xAARRGGBB: 0xFFFFFFFF draws it as it is. | + +### record `item-shape` + +| Field | Type | | +| --- | --- | --- | +| `item` | `string` | `minecraft:diamond`, `:`; an unknown one draws nothing. | +| `at` | `point` | The icon's top left. | +| `scale` | `f32` | 1.0 is the hotbar's 16 × 16. | + +### record `text-shape` + +| Field | Type | | +| --- | --- | --- | +| `text` | `string` | Up to 256 characters, one line. | +| `at` | `point` | | +| `color` | `u32` | 0xRRGGBB, as `hud`'s lines. | +| `centered` | `bool` | Centred on `at.x` instead of starting there. | +| `shadow` | `bool` | | + +### variant `shape` + +| Case | Payload | | +| --- | --- | --- | +| `fill` | `fill-shape` | | +| `quad` | `quad-shape` | | +| `sprite` | `sprite-shape` | | +| `item` | `item-shape` | | +| `text` | `text-shape` | | + +| Function | Signature | What it does | +| --- | --- | --- | +| `set-hud` | `func(shapes: list)` | Replaces the mod's HUD shapes: up to 1024, drawn every frame in order over the HUD (after the hotbar, under chat and screens), hidden with the HUD (F1). An empty list removes them. | +| `gui-size` | `func() -> tuple` | The screen's size in GUI pixels (width, height), as of this tick. | +| `shake` | `func(strength: f32, seconds: f32)` | Shakes the camera: a jolt of `strength` (0 to 1, 1 the hardest) that fades out over `seconds` (at most 10). Jolts add up to 1. The view turns, the player doesn't, and the player's Distortion Effects option scales it (0 turns it off). | + ## `events` Access: exported by the mod: the game calls it. diff --git a/docs/api/mods.md b/docs/api/mods.md index bec03e84e..cda484c31 100644 --- a/docs/api/mods.md +++ b/docs/api/mods.md @@ -95,5 +95,7 @@ What real mods need next, in the order modpacks and servers need it: ## Changelog +- Experimental `render` (`experimental = ["render"]`): HUD shapes (fill, quad, sprite, item, text), `gui-size`, camera + `shake`, drawn by the game's own renderer. - API 0.2 (content): data-driven blocks and items, tags, `horizon:add_to`, terrain adaptation. - API 0.1.0: `log`, `game`, `chat`, `hud`, `storage`; `init`, `tick`, `frame`, `command`, `chat`. diff --git a/docs/api/stability.md b/docs/api/stability.md index ffa9d9b11..76c3c4e51 100644 --- a/docs/api/stability.md +++ b/docs/api/stability.md @@ -88,6 +88,7 @@ Deprecations and breaking changes are in bold. The release notes of the game rel | --- | --- | --- | | Mod API: `log`, `game`, `chat`, `hud`, `storage`, `events` | preview | 0.1.0 | | Mod API: `ui`, `ui-events` (a mod's screens and settings page) | experimental | manifest `experimental = ["ui"]` | +| Mod API: `render` (HUD shapes, camera shake; drawn by the game, no mod shaders) | experimental | manifest `experimental = ["render"]` | | Content mods (`registries`: data-driven blocks and items) | preview | manifest `api = "0.2"` | | Server mods (events, world, server commands) | not started | manifest `api = "0.3"`, experimental first | | Script mods (Lua run by an interpreter mod, [mods.md](mods.md#ways-in)) | not started | | diff --git a/docs/mods.md b/docs/mods.md index 9b3b8aa94..672d6c7fd 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -87,6 +87,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | `hud` | `hud` | `set-panel(corner, lines)`, `clear()`: up to 16 lines of 256 characters | | `storage` | `storage` | `get(key)`, `set(key, value)`, `delete(key)`: a per-mod folder of up to 64 files, 4 MiB total, at most 1 MiB written per call | | `ui` | `ui` | `open-screen`, `close-screen`, `set-settings`: a screen the mod owns and a page in Options > Mods, of text, item and list widgets the game lays out and draws with its own menus' code. **Experimental:** the manifest needs `experimental = ["ui"]` | +| `render` | `render` | `set-hud(shapes)`: up to 1024 fills, four-cornered shapes, GUI sprites, items and text, kept on the HUD (over the hotbar, under chat and screens, hidden with F1) until replaced; `gui-size()`; `shake(strength, seconds)`: the view turns, the player doesn't, scaled by Distortion Effects. The game's own HUD renderer draws them: a mod never supplies shader code. **Experimental:** `experimental = ["render"]` | | Hook (export) | When | |---|---| @@ -114,7 +115,7 @@ loading. Ordered by what the first native ports need: | `inventory-read` | the player's inventory, held item, open container | AppleSkin, recipe viewers | | `recipes` | the recipe registry and lookups | JEI/EMI-style viewers | | `world-read` | blocks and biomes around the player, within the render distance | minimaps, schematics | -| `render` | world-space lines, boxes and ghost blocks in a per-frame budget | Litematica-style placement, waypoints | +| `render` (world) | world-space lines, boxes and billboards, beside the HUD shapes `render` has | Litematica-style placement, waypoints | | `registries` | new blocks, items, entities and recipes, mostly data: JSON models, textures, loot and recipes in the `.hmod`'s `assets/` and `data/` | server side; see [Multiplayer](#multiplayer) | | `server` | world events (block break/place, entity spawn/death, player join), game rules, server commands | runs where the world is simulated | | `net` | a mod's own client↔server channel (custom payloads) | only between the two halves of the same mod | @@ -288,7 +289,7 @@ experimental = [] # experimental capabilities it uses, e.g. ["ui"] ``` Unknown keys are refused, so a typo is an error, not a silently ignored -permission. An experimental capability (today `ui`) may change in any release +permission. An experimental capability (today `ui` and `render`) may change in any release ([stability](api/stability.md)): a mod uses it only by naming it in `experimental` as well, so nobody depends on it by accident. From 09ff66bfeee243b10396a77381095eaa31670327 Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Sun, 25 Oct 2026 20:39:44 +0000 Subject: [PATCH 35/61] Fix stale server digging after instant survival breaks Clear completed guest digging so replanted flowers and crops are not broken again after attack is released. Add a LAN replanting regression for both single-player paths. Co-Authored-By: Codex (gpt-6.1-sol) --- crates/horizon-client/src/game/lan_actions.rs | 11 ++- tools/lan_e2e.py | 68 +++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/crates/horizon-client/src/game/lan_actions.rs b/crates/horizon-client/src/game/lan_actions.rs index 02434d793..ab7803ddd 100644 --- a/crates/horizon-client/src/game/lan_actions.rs +++ b/crates/horizon-client/src/game/lan_actions.rs @@ -1810,14 +1810,13 @@ impl Game<'_> { g.use_delay -= 1; } }); - // ServerPlayerGameMode.handleBlockBreakAction: a creative - // player's START breaks the block once (destroyAndAck) and - // leaves nothing being destroyed; its client sends no STOP. - // Kept, the dig broke whatever was put there next, every - // destroy delay, for as long as the player broke nothing else. + // ServerPlayerGameMode.handleBlockBreakAction: creative and + // instant survival STARTs finish without a STOP from the client. + // tick_mining clears mining on completion; forget the START too, + // so a flower or crop planted here later is not broken again. if digging.is_some() - && host.hub.guests[gi].creative() && let Some(play) = host.play.get_mut(&conn) + && (host.hub.guests[gi].creative() || play.mining.is_none()) { play.digging = None; } diff --git a/tools/lan_e2e.py b/tools/lan_e2e.py index f79e07033..f0333c87b 100755 --- a/tools/lan_e2e.py +++ b/tools/lan_e2e.py @@ -250,6 +250,8 @@ address. The script then checks, from both sides, what a guest's game used to ge - falling: an item and a falling block the host summons high up fall smoothly on the guest, a little further every tick between the server's updates (every 20 ticks), as their own client tick moves them (they used to hang, then jump down once a second, #1568). +- replant: flowers and crops replanted after a short harvest click stay planted for both players; + the server forgets the completed instant break (#1880). Each check is reported on its own (a failing one doesn't stop the others) and the screenshots of both sides go to --out (host-*.png, guest-*.png). @@ -6348,5 +6350,71 @@ CHECKS += ['sprint-fov'] CHECK_FNS['sprint-fov'] = check_sprint_fov +def check_replant(host, guest, out): + """Instant breaks leave no server dig that can destroy a replacement (#1880). + + ServerPlayerGameMode.handleBlockBreakAction's insta-mine branch finishes on + START; MultiPlayerGameMode sends no STOP or ABORT when attack is released. + Exercise actual flower placement and seed planting after a short click. + """ + spots = apart(host, guest) + seen = {} + host('command', command='gamerule random_tick_speed 0') + try: + for label, player in (('host', host), ('guest', guest)): + x, y, z = spots[label] + cell = (x, y, z + 2) + + def observe(block): + # The integrated host is itself a client: wait for both views. + for _ in range(40): + blocks = [g.raw('block', *cell).get('block') for g in (host, guest)] + if blocks == [f'minecraft:{block}'] * 2: + return blocks + host.wait(2) + raise AssertionError(f'{label} expected {block} on both clients: {blocks}') + + for mode, block, item, soil in ( + ('survival', 'dandelion', 'dandelion', 'dirt'), + ('survival', 'wheat', 'wheat_seeds', 'farmland[moisture=7]'), + ('survival', 'carrots', 'carrot', 'farmland[moisture=7]'), + ('creative', 'dandelion', 'dandelion', 'dirt'), + ): + player('command', command=f'gamemode {mode}') + player('command', command='clear @s') + player('command', command=f'item replace entity @s weapon.mainhand with minecraft:{item} 8') + player('command', command=f'setblock {x} {y - 1} {z + 2} minecraft:{soil}') + player('command', command=f'setblock {x} {y} {z + 2} minecraft:{block}') + player.wait(5) + observe(block) + # Flowers' narrow outlines have a position-dependent offset. + for dx in (0.0, -0.2, 0.2): + player.raw('look-at', x + 0.5 + dx, y + (0.3 if block == 'dandelion' else 0.1), z + 2.5) + target = player('looking_at').get('block') or {} + if target.get('pos') == list(cell): + break + assert target.get('pos') == list(cell), f'{label} does not aim at {block}: {target}' + player.raw('mouse', 'left', 'click') + player.wait(5) + observe('air') + # Keep aiming at the supporting soil and use the held flower/seeds. + player.raw('mouse', 'right', 'click') + player.wait(3) + observe(block) + player.wait(20) # past the stale dig's five-tick destroy delay + key = f'{label} {mode} {block}' + seen[key] = [g.raw('block', *cell).get('block') for g in (host, guest)] + player.shot(out, f'replant-{mode}-{block}') + assert seen[key] == [f'minecraft:{block}'] * 2, f'{key}: {seen}' + player('command', command=f'setblock {x} {y} {z + 2} minecraft:air') + finally: + host('command', command='gamerule random_tick_speed 3') + return seen + + +CHECKS += ['replant'] +CHECK_FNS['replant'] = check_replant + + if __name__ == '__main__': main() From 3762ab7cdaa0f67c8cd473a88347098ebc982f92 Mon Sep 17 00:00:00 2001 From: Alexander Krylov Date: Sun, 25 Oct 2026 23:46:43 +0300 Subject: [PATCH 36/61] Enchantments: the name and the level follow the selected language The registry resolved every `description` from the pinned `en_us` map at load and `full_name` took `enchantment.level.` from the same table, so a tooltip showed an English enchantment line whatever language the player picked. Keep the description's translation key and resolve it, and the level, through the current language at display time, with the English text as the fallback for the server thread, the probes and an old resource root. Refs #1774. Co-Authored-By: Hermes Agent (Nous Research) --- crates/horizon-client/src/enchant.rs | 55 +++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/crates/horizon-client/src/enchant.rs b/crates/horizon-client/src/enchant.rs index 2fc9ef2fb..f5ddc84c2 100644 --- a/crates/horizon-client/src/enchant.rs +++ b/crates/horizon-client/src/enchant.rs @@ -33,7 +33,11 @@ impl Cost { #[derive(Clone, Debug)] pub struct Enchantment { pub id: Arc, + /// The name the pinned English map gave at load: the fallback for a + /// language that has no entry for the key below. pub description: String, + /// `description`'s translation key (`enchantment.minecraft.`). + pub description_key: String, pub max_level: u32, pub weight: u32, pub anvil_cost: u32, @@ -224,6 +228,7 @@ impl Enchantments { .get(key) .cloned() .unwrap_or_else(|| id.trim_start_matches("minecraft:").replace('_', " ")), + description_key: key.to_owned(), max_level: v["max_level"].as_u64().unwrap_or(1) as u32, weight: v["weight"].as_u64().unwrap_or(1) as u32, anvil_cost: v["anvil_cost"].as_u64().unwrap_or(0) as u32, @@ -329,16 +334,26 @@ impl Enchantments { } /// `Enchantment.getFullname` as plain text and whether it is a curse (red). pub fn full_name(&self, id: &str, level: u32) -> (String, bool) { + let language = crate::lang::current(); + self.full_name_in(language.as_deref(), id, level) + } + /// `full_name` against `language`; `None` keeps the English names loaded + /// above, which is what the server thread and the probes see. + pub fn full_name_in( + &self, + language: Option<&crate::lang::Language>, + id: &str, + level: u32, + ) -> (String, bool) { + let tr = |key: &str| language.and_then(|l| l.get(key).map(str::to_owned)); let Some(e) = self.get(id) else { return (format!("{id} {}", level_name(level)), false); }; - let mut name = e.description.clone(); + let mut name = tr(&e.description_key).unwrap_or_else(|| e.description.clone()); if level != 1 || e.max_level != 1 { name.push(' '); name.push_str( - self.lang - .get(&format!("enchantment.level.{level}")) - .map_or(&level_name(level), |s| s), + &tr(&format!("enchantment.level.{level}")).unwrap_or_else(|| level_name(level)), ); } (name, self.is_curse(id)) @@ -771,7 +786,7 @@ pub(crate) mod tests { let defs = [ ( "minecraft:sharpness", - json!({"anvil_cost":1,"exclusive_set":"#minecraft:exclusive_set/damage","max_cost":{"base":21,"per_level_above_first":11},"max_level":5,"min_cost":{"base":1,"per_level_above_first":11},"primary_items":"#minecraft:enchantable/melee_weapon","supported_items":"#minecraft:enchantable/sharp_weapon","weight":10,"slots":["mainhand"],"effects":{"minecraft:damage":[{"effect":{"type":"minecraft:add","value":{"type":"minecraft:linear","base":1.0,"per_level_above_first":0.5}}}]}}), + json!({"anvil_cost":1,"description":{"translate":"enchantment.minecraft.sharpness"},"exclusive_set":"#minecraft:exclusive_set/damage","max_cost":{"base":21,"per_level_above_first":11},"max_level":5,"min_cost":{"base":1,"per_level_above_first":11},"primary_items":"#minecraft:enchantable/melee_weapon","supported_items":"#minecraft:enchantable/sharp_weapon","weight":10,"slots":["mainhand"],"effects":{"minecraft:damage":[{"effect":{"type":"minecraft:add","value":{"type":"minecraft:linear","base":1.0,"per_level_above_first":0.5}}}]}}), ), ( "minecraft:smite", @@ -795,6 +810,7 @@ pub(crate) mod tests { .map(|(id, v)| Enchantment { id: Arc::from(*id), description: id.trim_start_matches("minecraft:").into(), + description_key: v["description"]["translate"].as_str().unwrap_or("").into(), max_level: v["max_level"].as_u64().unwrap() as u32, weight: v["weight"].as_u64().unwrap() as u32, anvil_cost: v["anvil_cost"].as_u64().unwrap() as u32, @@ -1088,4 +1104,33 @@ pub(crate) mod tests { assert!(!e.compatible("minecraft:sharpness", "minecraft:smite")); assert!(e.compatible("minecraft:sharpness", "minecraft:unbreaking")); } + + /// The enchantment's name and its level come from the selected language, + /// with the English name loaded above as the fallback. + #[test] + fn names_follow_the_language() { + struct Mem(HashMap>); + impl crate::lang::ResourceSource for Mem { + fn read(&self, path: &str) -> Option> { + self.0.get(path).cloned() + } + } + let ru = Mem(HashMap::from([( + "assets/minecraft/lang/ru_ru.json".to_owned(), + r#"{"enchantment.minecraft.sharpness":"Острота","enchantment.level.5":"V"}"# + .as_bytes() + .to_vec(), + )])); + let ru = crate::lang::Language::load(&[&ru], "ru_ru"); + let e = sample(); + assert_eq!( + e.full_name_in(Some(&ru), "minecraft:sharpness", 5).0, + "Острота V" + ); + // mending carries no key in the fixture: the English name stays. + assert_eq!( + e.full_name_in(Some(&ru), "minecraft:mending", 1).0, + "mending" + ); + } } From 95c336a3870b003b6da5b7baae01d87e8cddee4b Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Sun, 25 Oct 2026 20:50:57 +0000 Subject: [PATCH 37/61] fix(threads): lower helper thread priority on Android Co-Authored-By: Codex (gpt-6.1-sol) --- crates/horizon-threads/Cargo.toml | 2 +- crates/horizon-threads/src/lib.rs | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/horizon-threads/Cargo.toml b/crates/horizon-threads/Cargo.toml index 3d7581fe9..d43b3e4fc 100644 --- a/crates/horizon-threads/Cargo.toml +++ b/crates/horizon-threads/Cargo.toml @@ -6,7 +6,7 @@ license.workspace = true rust-version = "1.85" description = "Scheduling hints for the game's threads: the OS calls the crates that forbid unsafe code can't make" -[target.'cfg(any(target_vendor = "apple", target_os = "linux"))'.dependencies] +[target.'cfg(any(target_vendor = "apple", target_os = "linux", target_os = "android"))'.dependencies] libc = "0.2" [target.'cfg(windows)'.dependencies] diff --git a/crates/horizon-threads/src/lib.rs b/crates/horizon-threads/src/lib.rs index ae4cc753f..7e9146ce7 100644 --- a/crates/horizon-threads/src/lib.rs +++ b/crates/horizon-threads/src/lib.rs @@ -45,8 +45,8 @@ fn favour() -> bool { /// behind the server thread and the render thread (`behind_server`). static HELPERS_BEHIND: AtomicBool = AtomicBool::new(false); -/// How many nice levels a helper thread goes below the process on Linux. -#[cfg(target_os = "linux")] +/// How many nice levels a helper thread goes below the process on Linux and Android. +#[cfg(any(target_os = "linux", target_os = "android"))] const BEHIND: i32 = 10; /// From now on, the helper threads that call `behind_server` as they start @@ -58,7 +58,7 @@ pub fn put_helpers_behind() { /// Called by a helper thread as it starts (world generation, lighting and /// meshing, a connection's packet writer): work no tick or frame waits for. -/// Once `put_helpers_behind` was called, on Linux, where the server and +/// Once `put_helpers_behind` was called, on Linux and Android, where the server and /// render threads can't be raised (`favour_current`), the helper lowers /// itself instead, which an unprivileged thread may: 10 nice levels below /// the process, or, where nice runs out (a game started under nice 10 or @@ -69,10 +69,10 @@ pub fn behind_server() -> bool { HELPERS_BEHIND.load(Ordering::Relaxed) && behind() } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "android"))] fn behind() -> bool { // SAFETY: these read and set the calling thread's own nice value or - // policy (who 0, pid 0 is the calling thread on Linux); `param` lives + // policy (who 0, pid 0 is the calling thread on Linux and Android); `param` lives // across the call. unsafe { let nice = libc::getpriority(libc::PRIO_PROCESS as _, 0); @@ -85,7 +85,7 @@ fn behind() -> bool { } } -#[cfg(not(target_os = "linux"))] +#[cfg(not(any(target_os = "linux", target_os = "android")))] fn behind() -> bool { false } @@ -117,12 +117,12 @@ mod tests { } /// A helper goes behind only once a dedicated server asks for it, and - /// then, on Linux, 10 nice levels below the process or in the idle class. + /// then, on Linux and Android, 10 nice levels below the process or in the idle class. #[test] fn helpers_go_behind_only_when_asked() { assert!(!std::thread::spawn(super::behind_server).join().unwrap()); super::put_helpers_behind(); - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "android"))] std::thread::spawn(|| { // SAFETY: reads the calling thread's own nice value. let before = unsafe { libc::getpriority(libc::PRIO_PROCESS as _, 0) }; @@ -142,7 +142,7 @@ mod tests { }) .join() .unwrap(); - #[cfg(not(target_os = "linux"))] + #[cfg(not(any(target_os = "linux", target_os = "android")))] assert!(!std::thread::spawn(super::behind_server).join().unwrap()); } } From 40b3c1e3d68bf0c96d9eeb5b7c75cd207d0cf51d Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:08:55 +0000 Subject: [PATCH 38/61] Preserve requested GUI scale across temporary small surfaces Keep framebuffer limits in rendering and explicit option cycling rather than overwriting the saved choice during startup or settings application. Co-Authored-By: Codex gpt-6.1-sol --- crates/horizon-client/src/game.rs | 10 ------ .../horizon-client/src/game/startup_menu.rs | 2 -- crates/horizon-client/src/options_ui.rs | 3 -- crates/horizon-client/src/settings.rs | 33 +++++++++++++++---- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index f5803ad08..7b69a2ee3 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -9194,10 +9194,6 @@ impl Game<'_> { ) } fn apply_settings(&mut self) { - if let Some(r) = &self.renderer { - self.settings - .clamp_gui_scale([r.config.width as i32, r.config.height as i32]); - } self.streaming.radius = self.settings.render_distance; // Options.biomeBlendRadius, ambientOcclusion and cutoutLeaves: // LevelExtractor.allChanged when they change. @@ -13094,10 +13090,6 @@ impl Loading { return; } } - if let Some(r) = &self.renderer { - self.settings - .clamp_gui_scale([r.config.width as i32, r.config.height as i32]); - } horizon_client::hud::GUI_SCALE.store(self.settings.gui_scale, Ordering::Relaxed); self.window = Some(window); } @@ -13136,8 +13128,6 @@ impl Loading { self.settings.horizon.api, ) { Ok(r) => { - self.settings - .clamp_gui_scale([r.config.width as i32, r.config.height as i32]); horizon_client::hud::GUI_SCALE.store(self.settings.gui_scale, Ordering::Relaxed); self.renderer = Some(r); } diff --git a/crates/horizon-client/src/game/startup_menu.rs b/crates/horizon-client/src/game/startup_menu.rs index a5d658a4d..f7be6241f 100644 --- a/crates/horizon-client/src/game/startup_menu.rs +++ b/crates/horizon-client/src/game/startup_menu.rs @@ -272,8 +272,6 @@ impl Loading { } pub(super) fn apply_menu_settings(&mut self) { if let Some(renderer) = &mut self.renderer { - self.settings - .clamp_gui_scale([renderer.config.width as i32, renderer.config.height as i32]); renderer.gamma = self.settings.gamma; renderer.set_present_mode(self.settings.vsync, self.settings.horizon.present_mode); } diff --git a/crates/horizon-client/src/options_ui.rs b/crates/horizon-client/src/options_ui.rs index 1d1f099eb..9f91254bd 100644 --- a/crates/horizon-client/src/options_ui.rs +++ b/crates/horizon-client/src/options_ui.rs @@ -3403,9 +3403,6 @@ mod tests { assert_eq!(s.gui_scale, expected); } s.gui_scale = u32::MAX; - s.clamp_gui_scale(framebuffer); - assert_eq!(s.gui_scale, max); - s.gui_scale = u32::MAX; Opt::GuiScale.cycle(&mut s, false, framebuffer); assert_eq!(s.gui_scale, 0); s.gui_scale = u32::MAX; diff --git a/crates/horizon-client/src/settings.rs b/crates/horizon-client/src/settings.rs index 5fc09e35b..2a53e1702 100644 --- a/crates/horizon-client/src/settings.rs +++ b/crates/horizon-client/src/settings.rs @@ -104,7 +104,7 @@ pub struct Settings { pub master_volume: f32, /// Volumes of every other sound source, indexed like SOUND_SOURCES. pub volumes: [f32; 11], - /// 0 = auto. + /// Requested scale (0 = auto); rendering caps it to the current framebuffer. pub gui_scale: u32, pub view_bobbing: bool, /// Brightness 0..1 (gamma). @@ -284,12 +284,6 @@ fn put_s(f: &mut OptionsFile, key: &str, v: &str) { } } impl Settings { - /// `ClampingLazyMaxIntRange.validateValue`: clamp once the framebuffer is known. - pub fn clamp_gui_scale(&mut self, framebuffer: [i32; 2]) { - let max = crate::window::calculate_scale(framebuffer, 0, self.force_unicode) as u32; - self.gui_scale = self.gui_scale.min(max); - } - pub fn load(path: &Path) -> Self { let horizon = path .parent() @@ -597,6 +591,31 @@ impl Settings { #[cfg(test)] mod tests { use super::*; + #[test] + fn gui_scale_survives_temporary_small_framebuffer() { + for requested in [0, 1, 2, 6] { + let settings = Settings::parse(&format!("guiScale:{requested}\n")); + // Hyprland's initial surface can be smaller than the fullscreen one. + for (framebuffer, maximum) in [([715, 860], 2), ([3400, 1913], 7)] { + let effective = crate::window::calculate_scale( + framebuffer, + settings.gui_scale as i32, + settings.force_unicode, + ); + assert_eq!( + effective, + if requested == 0 { + maximum + } else { + (requested as i32).min(maximum) + } + ); + let saved = Settings::parse(&settings.serialize()); + assert_eq!(saved.gui_scale, requested); + } + } + } + #[test] fn round_trips_vanilla_options_txt() { let text = "version:4000\nfov:0.5\nrenderDistance:16\nmaxFps:260\nrenderClouds:\"fast\"\nkey_key.jump:key.keyboard.j\nsoundCategory_music:0.25\nresourcePacks:[\"vanilla\",\"file/x.zip\"]\nlang:fr_fr\nsomeFutureOption:42\n"; From 2cdb6362cae1e4d88894f4fda3a4dc3b49ada9bb Mon Sep 17 00:00:00 2001 From: socksprox Date: Sun, 25 Oct 2026 22:38:40 +0100 Subject: [PATCH 39/61] Particles keep their sprites over a resource reload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The particle engine copied the GUI atlas's sprite rects once at start; a pack or server-pack change rebuilt the atlas but not the engine, so every named and set sprite then drew a stale region — break crumbs showed other blocks and campfire smoke showed random items (#1923). The engine now re-reads the rects and the packs' sprite sets on reload, and a particle whose sprite-set index outlived the rebuild keeps its sprite instead of indexing past the new list. The reload also layers content mods over the root again, which startup does but reload forgot, so the first reload silently dropped their assets and data. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- crates/horizon-client/src/game/packs.rs | 6 ++++ crates/horizon-client/src/particles.rs | 46 +++++++++++++++++++------ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/crates/horizon-client/src/game/packs.rs b/crates/horizon-client/src/game/packs.rs index 7629a5d25..b35eff564 100644 --- a/crates/horizon-client/src/game/packs.rs +++ b/crates/horizon-client/src/game/packs.rs @@ -142,6 +142,8 @@ impl Game<'_> { } let server = &self.server_packs.active; let root = startup_root(&vanilla, &self.settings.resource_packs, server)?; + let root = + super::mods::content_root(&root, &self.settings.mods.enabled, &self.options.dev_mods); self.reload_resources(&root) } fn reload_resources(&mut self, root: &Path) -> horizon_client::Result<()> { @@ -192,6 +194,10 @@ impl Game<'_> { let context = pack.context.clone(); self.pack = pack; self.gui = gui; + // The atlas's sprites moved: the particle engine re-reads every rect + // and the packs' sprite sets, or every particle draws a stale region. + self.particles + .reload(root, &self.gui.atlas.sprites, &self.gui.atlas.translucent); horizon_client::ui_theme::reset_atlas(); // Villager hats come from the pack's texture metadata (shared data, swapped in place). *self diff --git a/crates/horizon-client/src/particles.rs b/crates/horizon-client/src/particles.rs index be7f32551..fa9dc9c6e 100644 --- a/crates/horizon-client/src/particles.rs +++ b/crates/horizon-client/src/particles.rs @@ -1212,10 +1212,7 @@ impl ParticleEngine { requested: None, names: Vec::new(), rects: Vec::new(), - atlas: atlas - .iter() - .map(|(name, &rect)| (name.clone(), (rect, translucent.contains(name)))) - .collect(), + atlas: HashMap::new(), sets: HashMap::new(), set_sprites: Vec::new(), emitters: Vec::new(), @@ -1225,6 +1222,34 @@ impl ParticleEngine { made: Default::default(), item_sprites: HashMap::new(), }; + engine.reload(root, atlas, translucent); + engine + } + /// `onResourceManagerReload`: sprite rects and sets come from the + /// reloaded atlas and pack; a live particle keeps drawing its sprite by + /// name or index. + pub fn reload( + &mut self, + root: &Path, + atlas: &HashMap, + translucent: &HashSet, + ) { + self.atlas = atlas + .iter() + .map(|(name, &rect)| (name.clone(), (rect, translucent.contains(name)))) + .collect(); + self.rects = self + .names + .iter() + .map(|name| self.atlas.get(name).map(|&(rect, _)| rect)) + .collect(); + self.sets.clear(); + self.set_sprites.clear(); + self.item_sprites.clear(); + self.load_sprites(root); + } + /// The sprite sets of every `particles/*.json` under `root`. + fn load_sprites(&mut self, root: &Path) { let dir = root.join("assets/minecraft/particles"); for &ty in ParticleType::ALL { let Ok(bytes) = std::fs::read(dir.join(format!("{}.json", ty.name()))) else { @@ -1239,18 +1264,16 @@ impl ParticleEngine { a.iter() .filter_map(|t| t.as_str()) .map(|t| { - engine - .intern(&format!("particle/{}", t.trim_start_matches("minecraft:"))) + self.intern(&format!("particle/{}", t.trim_start_matches("minecraft:"))) }) .collect() }) .unwrap_or_default(); if !sprites.is_empty() { - engine.sets.insert(ty, engine.set_sprites.len() as u32); - engine.set_sprites.push(sprites); + self.sets.insert(ty, self.set_sprites.len() as u32); + self.set_sprites.push(sprites); } } - engine } fn intern(&mut self, name: &str) -> u32 { if let Some(i) = self.names.iter().position(|n| n == name) { @@ -1601,7 +1624,10 @@ impl ParticleEngine { return; } if let Some(set) = p.set { - let sprites = &self.set_sprites[set as usize]; + // A sprite set's index can outlive the reload that rebuilt them. + let Some(sprites) = self.set_sprites.get(set as usize) else { + return; + }; let lifetime = p.lifetime.max(1); let index = (p.age.max(0) as i64 * (sprites.len() as i64 - 1) / i64::from(lifetime)) .clamp(0, sprites.len() as i64 - 1); From b6efebc87908e113cbf555496e61e61f1c685d16 Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:39:14 +0000 Subject: [PATCH 40/61] Play anvil result sounds once for every player MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deliver anvil level events to the acting player and keep their local sound out of positional sound capture. Extend the anvil LAN check to count sounds for both players using either player’s anvil result. Co-Authored-By: OpenAI Codex --- crates/horizon-client/src/game.rs | 8 +++++++- crates/horizon-client/src/game/lan.rs | 6 +++--- tools/lan_e2e.py | 20 +++++++++++++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index f5803ad08..08abb2d2b 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -2764,7 +2764,13 @@ impl Game<'_> { } _ => 1.0, }; - self.audio.play(name, c, volume, pitch); + if matches!(id, 1029 | 1030) { + // Clients play the anvil sound from its level event; + // capturing it also sends a duplicate sound packet. + self.audio.play_unshared(name, c, volume, pitch); + } else { + self.audio.play(name, c, volume, pitch); + } } } LevelEvent::Sound { diff --git a/crates/horizon-client/src/game/lan.rs b/crates/horizon-client/src/game/lan.rs index ddb959cf3..9b6754661 100644 --- a/crates/horizon-client/src/game/lan.rs +++ b/crates/horizon-client/src/game/lan.rs @@ -1114,9 +1114,9 @@ impl Game<'_> { for e in events { if let horizon_world::level::LevelEvent::LevelEvent { id, pos, data } = e { // The acting guest does not predict the egg's teleport - // particles, nor bone meal's growth (BoneMealItem.useOn's - // levelEvent names no player). - let except = if matches!(*id, 2015 | 1505) { + // particles, bone meal's growth, or anvil result sounds + // (these levelEvent calls name no excluded player). + let except = if matches!(*id, 2015 | 1505 | 1029 | 1030) { None } else { self.lan_acting diff --git a/tools/lan_e2e.py b/tools/lan_e2e.py index 0ec7218d8..8e7985862 100755 --- a/tools/lan_e2e.py +++ b/tools/lan_e2e.py @@ -52,9 +52,9 @@ address. The script then checks, from both sides, what a guest's game used to ge - grindstone: a grindstone opens on the guest's screen; the enchanted sword the guest shift-clicks in shows the server's result, and the guest takes it with only its curse left and the experience orbs the server awards (#868); -- anvil: an anvil opens on the guest's screen; the guest repairs a sword with iron ingots and - renames it (ServerboundRenameItemPacket), takes it for the server's level cost, and the - server has the repaired, renamed sword (#868); +- anvil: each player repairs and renames a sword (ServerboundRenameItemPacket), takes it for + the server's level cost, and the server has the repaired, renamed sword (#868); + both players hear exactly one anvil sound for each result (#1907); - enchanting: an enchanting table opens on the guest's screen with the server's offers for the sword the guest puts in; the guest clicks the first offer and the server enchants the sword for a level and a lapis, and counts it in the guest's enchant_item stat (#868); @@ -1620,6 +1620,12 @@ def check_grindstone(host, guest, out): def check_anvil(host, guest, out): + # AnvilMenu.onTake emits one level event to everyone, including the user. + # Exercise both the in-frame host and a remote user (both remote with --server-thread). + return {actor.name: check_anvil_user(actor, (host, guest), out) for actor in (guest, host)} + + +def check_anvil_user(guest, listeners, out): guest('command', command='gamemode survival') guest('command', command='clear @s') guest('command', command='xp set @s 5 levels') @@ -1638,8 +1644,12 @@ def check_anvil(host, guest, out): menu = guest.raw('menu') offered = (menu['info']['anvil_cost'], station_slots(guest).get(650)) guest.shot(out, 'anvil-rename') + sounds = ('minecraft:block.anvil.use', 'minecraft:block.anvil.destroy') + before = {g.name: sum(heard(g, s) for s in sounds) for g in listeners} guest.raw('click-slot', 650, 'shift') - guest.wait(10) + guest.wait(20) + sound_counts = {g.name: sum(heard(g, s) for s in sounds) - before[g.name] for g in listeners} + assert sound_counts == {g.name: 1 for g in listeners}, f'anvil sounds: {sound_counts}' left = station_slots(guest) close_station(guest, at) sword = server_reply(guest, 'data get entity @s Inventory[{id:"minecraft:iron_sword"}].components', @@ -1653,7 +1663,7 @@ def check_anvil(host, guest, out): want = {'offered': (4, ('minecraft:iron_sword', 1)), 'left': {}, 'name': True, 'damage': True, 'levels': 1, 'ingots': 0} assert seen == want, f'the anvil on the guest: {seen} ({sword!r})' - return seen + return {**seen, 'heard': sound_counts} def check_smithing(host, guest, out): From acb68ed163deb929e1e64b8abf6f39aa9caa99bf Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:44:56 +0100 Subject: [PATCH 41/61] launcher: reuse prepared resources while the preparer's OUTPUT_VERSION is the same A version's resources were reused only when the preparer's sources and the exporters were byte for byte the same, so any change to crates/horizon-prepare (even one that prepares the same files) made every player prepare again. The launcher now compares the number the preparer declares for what it makes (`pub const OUTPUT_VERSION: u32` in crates/horizon-prepare/src/lib.rs, bumped by hand when the output changes and checked by CI in the follow-up). Preparers from before the number are known by a fingerprint of their sources and exporters: v0.2.49 to v0.2.64 prepare version 1's output (measured on the CI runners' preparations). Older versions are compared as before. Co-Authored-By: Claude Opus 5.5 --- launcher/src/backend.rs | 216 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 205 insertions(+), 11 deletions(-) diff --git a/launcher/src/backend.rs b/launcher/src/backend.rs index 7e5c4babb..ce9007868 100644 --- a/launcher/src/backend.rs +++ b/launcher/src/backend.rs @@ -628,17 +628,13 @@ pub fn resources_summary(prepared: bool, sounds: bool, version: &str) -> String } } -/// Whether the resources `from` prepared are the ones preparing `to` would make. The output -/// depends on the Horizon version, not only on Minecraft's: the manifest records the hashes of -/// the vanilla exporters it was made with (mods/horizon/src/test/java/.../parity), and the -/// preparer writes every file and the manifest's `schema`. So both need the same pinned -/// Minecraft jars (research-lock.json), the same preparer and the same exporter sources. The -/// preparer is crates/horizon-prepare (its sources ship in the release folder beside the -/// binary) and tools/prepare_native_client.py (the Python preparer of older versions, a wrapper -/// of the binary since); a file only one of the two versions has is a different preparer. The -/// helpers the Python preparer imported (lab.py and the test_*_parity.py modules) change -/// between releases in how they find and check the player's files, which the output does not -/// depend on; they are not compared. +/// Whether the resources `from` prepared are the ones preparing `to` would make: both pin the same +/// Minecraft jars (research-lock.json) and their preparers make the same output. A preparer +/// declares what it makes with `OUTPUT_VERSION` (crates/horizon-prepare/src/lib.rs, whose sources +/// ship in the release folder), bumped by hand when the output changes and checked by CI +/// (tools/prepare_output.py), so a release that changes the preparer's code but not its output +/// keeps the resources. Preparers from before the number are known by their sources +/// (`LEGACY_OUTPUT_VERSIONS`); older ones still are compared file by file (`same_preparer`). fn resources_compatible(from: &Path, to: &Path, manifest: &serde_json::Value) -> bool { let pins = |dir: &Path| { read_json(&dir.join("research-lock.json")).map(|lock| { @@ -651,6 +647,108 @@ fn resources_compatible(from: &Path, to: &Path, manifest: &serde_json::Value) -> if a != b || a.iter().any(|v| !v.is_string()) { return false; } + match (output_version_at(from), output_version_at(to)) { + (Some(a), Some(b)) => a == b, + (None, None) => same_preparer(from, to, manifest), + _ => false, + } +} + +/// The `OUTPUT_VERSION` of a version folder's preparer: the constant in its sources, or for a +/// preparer from before it, the number `LEGACY_OUTPUT_VERSIONS` gives its fingerprint. None for +/// an unknown older preparer. +fn output_version_at(dir: &Path) -> Option { + output_version_in(dir, &LEGACY_OUTPUT_VERSIONS) +} + +fn output_version_in(dir: &Path, legacy: &[(&str, u32)]) -> Option { + let crate_dir = dir.join(PREPARER_CRATE); + let lib = fs::read_to_string(crate_dir.join("src/lib.rs")).ok()?; + let declared = lib.lines().find_map(|line| { + line.strip_prefix("pub const OUTPUT_VERSION: u32 = ")? + .strip_suffix(';')? + .parse() + .ok() + }); + if declared.is_some() { + return declared; + } + let fingerprint = legacy_fingerprint(dir)?; + legacy + .iter() + .find(|(known, _)| *known == fingerprint) + .map(|&(_, version)| version) +} + +/// Preparers from before `OUTPUT_VERSION`, by `legacy_fingerprint`, whose output +/// tools/prepare_output.py measured: what they prepare is version 1's. +const LEGACY_OUTPUT_VERSIONS: [(&str, u32); 2] = [ + // v0.2.52 to v0.2.64 + ( + "54218ade1c058b1f15ea4094bf8c1a7893adb37b1ed5cc803ddae8e950c31704", + 1, + ), + // v0.2.49 to v0.2.51 + ( + "da9ffe7558129c38b44abf43289e81946fe9f2512e5e07272eeae0cf0b4f1254", + 1, + ), +]; + +/// The vanilla exporters those preparers ran (the `*_exporters_sha256` of their manifests), in +/// mods/horizon/src/test/java/dev/horizonlab/parity. +const LEGACY_EXPORTERS: [&str; 18] = [ + "ParityData.java", + "VanillaBlockBehaviourExporter.java", + "VanillaBlockLandingOracle.java", + "VanillaBlockStateBindingOracle.java", + "VanillaCollisionOracle.java", + "VanillaCommandParseOracle.java", + "VanillaCreativeTabsExporter.java", + "VanillaCuboidModelOracle.java", + "VanillaDamageSourceOracle.java", + "VanillaEntityAttributeOracle.java", + "VanillaEntityModelExporter.java", + "VanillaEntityTypesExporter.java", + "VanillaFaceVisibilityOracle.java", + "VanillaGameplayFactsExporter.java", + "VanillaHeightmapOracle.java", + "VanillaMobDataExporter.java", + "VanillaMovementRegistryOracle.java", + "VanillaPlayerHealthOracle.java", +]; + +/// A version folder's preparer as the launchers before `OUTPUT_VERSION` compared it: the digest +/// of its sources (`horizon_prepare::sources_sha256`) and of the exporters it ran, line endings +/// as checked out ignored. +fn legacy_fingerprint(dir: &Path) -> Option { + use sha2::Digest; + let sources = horizon_prepare::sources_sha256(&dir.join(PREPARER_CRATE))?; + let mut hash = sha2::Sha256::new(); + hash.update(sources.as_bytes()); + for name in LEGACY_EXPORTERS { + let path = format!("mods/horizon/src/test/java/dev/horizonlab/parity/{name}"); + let data: Vec = fs::read(dir.join(&path)) + .ok()? + .into_iter() + .filter(|b| *b != b'\r') + .collect(); + hash.update(format!("{path}\0{}\0", data.len())); + hash.update(&data); + } + Some(hash.finalize().iter().map(|b| format!("{b:02x}")).collect()) +} + +/// How launchers before `OUTPUT_VERSION` compared two preparers, for versions older than the +/// ones `LEGACY_OUTPUT_VERSIONS` knows. The manifest records the hashes of the vanilla exporters +/// the resources were made with (mods/horizon/src/test/java/.../parity), and the preparer writes +/// every file and the manifest's `schema`, so both need the same preparer and the same exporter +/// sources. The preparer is crates/horizon-prepare and tools/prepare_native_client.py (the +/// Python preparer of older versions, a wrapper of the binary since); a file only one of the two +/// versions has is a different preparer. The helpers the Python preparer imported (lab.py and +/// the test_*_parity.py modules) change between releases in how they find and check the +/// player's files, which the output does not depend on; they are not compared. +fn same_preparer(from: &Path, to: &Path, manifest: &serde_json::Value) -> bool { let Some(fields) = manifest.as_object() else { return false; }; @@ -3150,6 +3248,102 @@ mod tests { let _ = fs::remove_dir_all(&versions); } + #[test] + fn prepared_resources_follow_the_preparers_output_version() { + let versions = std::env::temp_dir().join(format!( + "horizon-launcher-output-test-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&versions); + let write = |tag: &str, path: &str, text: &str| { + let path = versions.join(tag).join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + }; + let lib = "crates/horizon-prepare/src/lib.rs"; + // A preparer with its sources (as releases ship them) and the exporters it runs. + let release = |tag: &str, code: &str| { + write(tag, COMPLETE, ""); + let lock = serde_json::json!({"minecraft": {"version": "26.3", "client_sha1": "c", "server_sha1": "s"}}); + write(tag, "research-lock.json", &lock.to_string()); + write(tag, "crates/horizon-prepare/Cargo.toml", "[package]"); + write(tag, "crates/horizon-prepare/build.rs", "fn main() {}"); + write(tag, lib, code); + for name in LEGACY_EXPORTERS { + write( + tag, + &format!("mods/horizon/src/test/java/dev/horizonlab/parity/{name}"), + name, + ); + } + }; + let prepared = |tag: &str| { + write( + tag, + &format!(".local/horizon-client/26.3/{RESOURCES_MANIFEST}"), + r#"{"schema": 7}"#, + ); + write(tag, ".local/horizon-client/26.3/facts.json", tag); + }; + let facts = |tag: &str| { + fs::read_to_string(resources_at(&versions.join(tag)).join("facts.json")).ok() + }; + + // The preparer's code changed, not what it makes: the resources are kept. + release( + "v0.3.0", + "pub const OUTPUT_VERSION: u32 = 1;\nfn prepare() {}", + ); + prepared("v0.3.0"); + release( + "v0.3.1", + "pub const OUTPUT_VERSION: u32 = 1;\nfn prepare() { faster() }", + ); + assert_eq!(reuse_prepared(&versions, "v0.3.1"), Some("v0.3.0".into())); + assert_eq!(facts("v0.3.1").as_deref(), Some("v0.3.0")); + // Its output changed, so the number did: prepare again. + release( + "v0.3.2", + "pub const OUTPUT_VERSION: u32 = 2;\nfn prepare() {}", + ); + assert_eq!(reuse_prepared(&versions, "v0.3.2"), None); + assert_eq!(facts("v0.3.2"), None); + // Another Minecraft jar, same number: prepare again. + release( + "v0.3.3", + "pub const OUTPUT_VERSION: u32 = 1;\nfn prepare() {}", + ); + let lock = serde_json::json!({"minecraft": {"version": "26.3", "client_sha1": "other", "server_sha1": "s"}}); + write("v0.3.3", "research-lock.json", &lock.to_string()); + assert_eq!(reuse_prepared(&versions, "v0.3.3"), None); + + // A preparer from before the number is known by its fingerprint. + release("v0.2.64", "fn prepare() {}"); + let fingerprint = legacy_fingerprint(&versions.join("v0.2.64")).unwrap(); + let legacy = [(fingerprint.as_str(), 1)]; + let at = |tag: &str| output_version_in(&versions.join(tag), &legacy); + assert_eq!(at("v0.2.64"), Some(1)); + assert_eq!(at("v0.3.2"), Some(2)); + // Line endings as checked out don't count; an exporter's or the sources' change does. + write( + "v0.2.64", + "mods/horizon/src/test/java/dev/horizonlab/parity/ParityData.java", + "ParityData.java\r", + ); + assert_eq!(at("v0.2.64"), Some(1)); + write( + "v0.2.64", + "mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java", + "changed", + ); + assert_eq!(at("v0.2.64"), None); + release("v0.2.65", "fn prepare() { changed() }"); + assert_eq!(at("v0.2.65"), None); + // A folder without the preparer's sources has no number. + assert_eq!(output_version_in(&versions.join("missing"), &legacy), None); + let _ = fs::remove_dir_all(&versions); + } + #[test] fn prepared_versions_get_the_run_time_reports_without_preparing_again() { let versions = std::env::temp_dir().join(format!( From 116a7f7dcafc4e36d14b75600f402446ec8056ac Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:48:43 +0100 Subject: [PATCH 42/61] horizon-prepare: OUTPUT_VERSION, the number of what it prepares, checked by CI Launchers copy a player's prepared resources to a new release while its preparer declares the same OUTPUT_VERSION (the launcher side is its own pull request). The number is bumped by hand; CI keeps it honest: the parity job already prepares with every change to the preparer, and tools/prepare_output.py check now fails when that preparation's digest is not the one crates/horizon-prepare/output-versions.txt gives the current number, or when the file changed other than by appending a version. The digest canonicalises what differs between two runs of the same preparer (JSON key order, creative tab tag order, lambda class addresses, logs, the sounds): 10 preparations by 5 preparer versions on the three Mac runners give one digest, and a Linux preparation by an older preparer gives the Mac runners' digest for it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parity.yml | 4 + crates/horizon-prepare/output-versions.txt | 3 + crates/horizon-prepare/src/lib.rs | 32 ++++ docs/release-flow.md | 4 + tests/test_prepare_output.py | 64 ++++++++ tools/prepare_output.py | 176 +++++++++++++++++++++ 6 files changed, 283 insertions(+) create mode 100644 crates/horizon-prepare/output-versions.txt create mode 100644 tests/test_prepare_output.py create mode 100755 tools/prepare_output.py diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index a18fc5019..0588c12a3 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -37,6 +37,10 @@ jobs: run: bash tools/ci/runner-env.sh - name: Vanilla data, and resources prepared by this preparer run: bash tools/ci/vanilla-data.sh + - name: The prepared output is the one the preparer's OUTPUT_VERSION says + env: + BASE_SHA: ${{ inputs.base }} + run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA" - name: The version constants are the pinned jar's run: python3 tools/gen_version_consts.py --check - name: The harnesses this change can affect diff --git a/crates/horizon-prepare/output-versions.txt b/crates/horizon-prepare/output-versions.txt new file mode 100644 index 000000000..71a531fab --- /dev/null +++ b/crates/horizon-prepare/output-versions.txt @@ -0,0 +1,3 @@ +# OUTPUT_VERSION (src/lib.rs) and the digest of what it prepares (python3 tools/prepare_output.py digest), +# one line per version, never edited: a change to the prepared files appends the next version. +1 0a608bf4115a8ed2e2c4f5f2b2609dc10c54e63479af8e56fabaa10af1060389 diff --git a/crates/horizon-prepare/src/lib.rs b/crates/horizon-prepare/src/lib.rs index c7e443c7d..5ea1121df 100644 --- a/crates/horizon-prepare/src/lib.rs +++ b/crates/horizon-prepare/src/lib.rs @@ -31,6 +31,15 @@ pub type Log<'a> = &'a dyn Fn(&str); /// The manifest's `schema`: the prepared files' layout, which the launcher compares. pub const SCHEMA: i64 = 7; +/// What this preparer makes. The launcher copies a prepared version's resources to a newly installed +/// version whose preparer has the same number (and the same pinned jars), so a release that changes +/// this crate without changing its output asks nobody to prepare again. Bump it, and append the new +/// digest to output-versions.txt, when the prepared files change: CI prepares with every change to +/// the preparer and fails when the output is not the one output-versions.txt gives this number +/// (`python3 tools/prepare_output.py check`). The launcher reads this line from the version +/// folder's sources: keep it on one line, as it is. +pub const OUTPUT_VERSION: u32 = 1; + /// Digest of this crate's sources (build.rs): the launcher prepares a version in-process when /// that version ships these same sources, and runs the version's own binary otherwise. pub const SOURCES_SHA256: &str = env!("HORIZON_PREPARE_SOURCES_SHA256"); @@ -1209,6 +1218,29 @@ mod tests { fs::remove_dir_all(dir).unwrap(); } + #[test] + fn output_versions_end_at_the_output_version() { + let text = include_str!("../output-versions.txt"); + let versions: Vec<(u32, &str)> = text + .lines() + .filter(|l| !l.trim().is_empty() && !l.starts_with('#')) + .map(|l| { + let (n, digest) = l.split_once(' ').unwrap(); + (n.parse().unwrap(), digest.trim()) + }) + .collect(); + let numbers: Vec = versions.iter().map(|v| v.0).collect(); + assert_eq!(numbers, (1..=OUTPUT_VERSION).collect::>()); + for (i, (_, digest)) in versions.iter().enumerate() { + assert_eq!(digest.len(), 64); + assert!(!versions[..i].iter().any(|(_, d)| d == digest)); + } + // The launcher finds the number with this line in a version folder's sources. + let lib = include_str!("lib.rs"); + let line = format!("pub const OUTPUT_VERSION: u32 = {OUTPUT_VERSION};"); + assert!(lib.lines().any(|l| l == line)); + } + #[test] fn the_source_digest_is_the_one_build_rs_computed() { assert_eq!( diff --git a/docs/release-flow.md b/docs/release-flow.md index 4fb0209d9..1413cde01 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -39,6 +39,10 @@ copy is: the Mac mini. Its runner runs as the maintainer's account and keeps its depend only on the Minecraft version, are kept once in `shared-local/sounds//` (copied from a folder `horizon-prepare sounds` made from the maintainer's own game; the Mac mini has no official launcher of its own) and cloned into each preparation (not linked: vanilla's packs refuse symbolic links), for the audio harnesses. + `tools/prepare_output.py check` then compares what was prepared with `crates/horizon-prepare/output-versions.txt`: + launchers keep a player's prepared resources across releases whose preparer declares the same `OUTPUT_VERSION` + (`crates/horizon-prepare/src/lib.rs`), so a change to the prepared files fails here until it bumps the number + and appends the new digest (the error gives both lines). 3. `tools/parity.py --changed ` picks the harnesses whose traced dependencies (their script, Python modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to diff --git a/tests/test_prepare_output.py b/tests/test_prepare_output.py new file mode 100644 index 000000000..00c489fbb --- /dev/null +++ b/tests/test_prepare_output.py @@ -0,0 +1,64 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from tools import prepare_output + + +class PrepareOutputTests(unittest.TestCase): + def preparation(self, root: Path, files: dict[str, bytes]) -> Path: + for rel, data in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return root + + def test_what_varies_between_runs_does_not_count(self): + manifest = {'schema': 7, 'data_version': 5023, 'render_files_sha256': {'a.json': 'x'}} + a = { + 'horizon-resources.json': json.dumps(manifest).encode(), + 'render-definitions.json': b'{"a": 1, "b": [1, 2]}', + 'creative-inventory.json': b'{"tags": [["b", "a"]]}', + 'block-behaviours.json': b'{"c": "Foo$$Lambda/0x0000012a"}', + 'export.log': b'run 1', + 'assets/minecraft/sounds/a.ogg': b'one', + } + manifest['render_files_sha256'] = {'a.json': 'y'} + b = { + 'horizon-resources.json': json.dumps(manifest).encode(), + 'render-definitions.json': b'{"b": [1, 2], "a": 1}', + 'creative-inventory.json': b'{"tags": [["a", "b"]]}', + 'block-behaviours.json': b'{"c": "Foo$$Lambda/0x00000ff0"}', + 'export.log': b'run 2', + 'assets/minecraft/sounds/a.ogg': b'two', + } + with tempfile.TemporaryDirectory() as tmp: + first = prepare_output.digest(self.preparation(Path(tmp) / 'a', a)) + second = prepare_output.digest(self.preparation(Path(tmp) / 'b', b)) + self.assertEqual(first, second) + # The data does, and so do the manifest's schema and the files there are. + (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [2, 1], "a": 1}') + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + (Path(tmp) / 'b/render-definitions.json').write_bytes(b'{"b": [1, 2], "a": 1}') + (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps({**manifest, 'schema': 8})) + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + (Path(tmp) / 'b/horizon-resources.json').write_text(json.dumps(manifest)) + (Path(tmp) / 'b/new.nbt').write_bytes(b'') + self.assertNotEqual(first, prepare_output.digest(Path(tmp) / 'b')) + + def test_the_versions_file_ends_at_the_constant(self): + versions = prepare_output.parse_versions('# comment\n1 aa\n2 bb\n') + self.assertEqual(versions, [(1, 'aa'), (2, 'bb')]) + self.assertEqual(prepare_output.problems(versions, 2), []) + self.assertTrue(prepare_output.problems(versions, 1)) + self.assertTrue(prepare_output.problems([(1, 'aa'), (3, 'bb')], 3)) + self.assertTrue(prepare_output.problems([(1, 'aa'), (2, 'aa')], 2)) + root = Path(prepare_output.__file__).resolve().parent.parent + current = prepare_output.parse_versions((root / prepare_output.VERSIONS).read_text()) + constant = prepare_output.CONSTANT.search((root / prepare_output.LIB).read_text()) + self.assertEqual(prepare_output.problems(current, int(constant.group(1))), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/prepare_output.py b/tools/prepare_output.py new file mode 100755 index 000000000..81ed156cb --- /dev/null +++ b/tools/prepare_output.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""The prepared output's version: what keeps `OUTPUT_VERSION` (crates/horizon-prepare/src/lib.rs) honest. + +The launcher copies a prepared version's resources (`.local/horizon-client/`) to a newly installed +one when both pin the same Minecraft jars and their preparers declare the same `OUTPUT_VERSION`, so a release +that changes the preparer's code without changing what it makes asks nobody to prepare again. The number is +bumped by hand; this tool catches a change to the output without a bump. + +crates/horizon-prepare/output-versions.txt lists every `OUTPUT_VERSION` with the digest of what it prepares. +It only grows: the last line is the current version, and a change to the output appends a line. + + python3 tools/prepare_output.py digest [RESOURCES] the digest of a preparation + python3 tools/prepare_output.py check [--base SHA] [--cache] [RESOURCES] + fails when RESOURCES (default: this checkout's .local/horizon-client/, as CI prepares it) is + not what the last line says, or when the file changed other than by appending since SHA. --cache keeps + the digest beside RESOURCES (CI: a preparation never changes once made; it takes ~30 s to compute) + +The digest covers every file of the preparation, canonicalised where the preparer is not deterministic between +runs (verified across the CI runners): JSON is compared as data (key order, the order of each `tags` list of +creative-inventory.json, and the addresses in `$$Lambda/0x...` class names don't count); logs, compiled classes, +the sounds (copied from the player's launcher by another step) and the manifest's file hashes (raw bytes) don't +count. The manifest's own scalar fields (schema, data version, jar hashes, file count) do. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +VERSIONS = 'crates/horizon-prepare/output-versions.txt' +LIB = 'crates/horizon-prepare/src/lib.rs' +MANIFEST = 'horizon-resources.json' +LAMBDA = re.compile(r'\$\$Lambda/0x[0-9a-fA-F]+') +CONSTANT = re.compile(r'^pub const OUTPUT_VERSION: u32 = (\d+);', re.M) + + +def canonical_json(value, key=''): + if isinstance(value, dict): + return {k: canonical_json(v, k) for k, v in value.items()} + if isinstance(value, list): + items = [canonical_json(v, key) for v in value] + if key == 'tags' and all(isinstance(x, list) and all(isinstance(t, str) for t in x) for x in items): + items = [sorted(x) for x in items] + return items + if isinstance(value, str): + return LAMBDA.sub('$$Lambda', value) + return value + + +def canonical(rel: str, data: bytes) -> bytes: + if rel == MANIFEST: + manifest = json.loads(data) + scalars = {k: v for k, v in manifest.items() if not isinstance(v, (dict, list))} + return json.dumps(scalars, sort_keys=True).encode() + if rel.endswith('.json'): + try: + value = json.loads(data) + except ValueError: + return data + return json.dumps(canonical_json(value), sort_keys=True).encode() + return data + + +def skipped(rel: str) -> bool: + parts = rel.split('/') + return rel.endswith(('.log', '.class')) or parts[:3] == ['assets', 'minecraft', 'sounds'] \ + or rel == 'assets/minecraft/sounds.json' + + +def digest(resources: Path) -> tuple[str, int]: + """The digest of a preparation, and how many files it covers.""" + if not (resources / MANIFEST).is_file(): + raise SystemExit(f'{resources} is not a preparation: no {MANIFEST}') + files = [] + for dirpath, dirs, names in os.walk(resources): + dirs.sort() + for name in names: + path = Path(dirpath) / name + rel = path.relative_to(resources).as_posix() + if path.is_symlink() or skipped(rel): + continue + files.append(rel) + total = hashlib.sha256() + for rel in sorted(files): + data = hashlib.sha256(canonical(rel, (resources / rel).read_bytes())).hexdigest() + total.update(f'{rel}\0{data}\n'.encode()) + return total.hexdigest(), len(files) + + +def parse_versions(text: str) -> list[tuple[int, str]]: + out = [] + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + number, value = line.split() + out.append((int(number), value)) + return out + + +def problems(versions: list[tuple[int, str]], constant: int | None) -> list[str]: + found = [] + if not versions: + return [f'{VERSIONS} lists no version'] + if [n for n, _ in versions] != list(range(1, len(versions) + 1)): + found.append(f'{VERSIONS}: the versions must be 1, 2, 3... in order') + if len({d for _, d in versions}) != len(versions): + found.append(f'{VERSIONS}: two versions with the same digest') + if constant != versions[-1][0]: + found.append(f'{LIB} says OUTPUT_VERSION = {constant}, {VERSIONS} ends at {versions[-1][0]}') + return found + + +def default_resources() -> Path: + lock = json.loads((ROOT / 'research-lock.json').read_text()) + return ROOT / '.local/horizon-client' / lock['minecraft']['version'] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + parser.add_argument('command', choices=['digest', 'check']) + parser.add_argument('resources', nargs='?', type=Path) + parser.add_argument('--base', help='the commit the change is based on (check: the file only grew since)') + parser.add_argument('--cache', action='store_true', help='keep the digest beside the preparation') + args = parser.parse_args() + resources = (args.resources or default_resources()).resolve() + tool = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()[:16] + cache = resources.parent / f'.output-digest-{resources.name}-{tool}' + if args.cache and cache.is_file(): + value, count = cache.read_text().split() + else: + value, count = digest(resources) + if args.cache: + cache.write_text(f'{value} {count}\n') + if args.command == 'digest': + print(value) + return 0 + + versions = parse_versions((ROOT / VERSIONS).read_text()) + match = CONSTANT.search((ROOT / LIB).read_text()) + found = problems(versions, int(match.group(1)) if match else None) + if args.base: + shown = subprocess.run(['git', 'show', f'{args.base}:{VERSIONS}'], cwd=ROOT, capture_output=True, text=True) + if shown.returncode == 0: + before = parse_versions(shown.stdout) + if versions[:len(before)] != before: + found.append(f'{VERSIONS} changed lines it already had (since {args.base[:12]}): ' + 'a new output gets a new version, appended') + current = versions[-1] if versions else (0, '') + if value != current[1]: + known = [n for n, d in versions if d == value] + if known: + found.append(f'the preparation is OUTPUT_VERSION {known[0]}\'s output, but OUTPUT_VERSION is {current[0]}') + else: + found.append( + f'the prepared output changed ({count} files, digest {value}) but OUTPUT_VERSION is still ' + f'{current[0]}. Players would keep resources prepared by the old preparer. Bump it:\n' + f' set `pub const OUTPUT_VERSION: u32 = {current[0] + 1};` in {LIB}\n' + f' append `{current[0] + 1} {value}` to {VERSIONS}\n' + ' (if the change is only a nondeterminism this tool should ignore, teach canonical() instead)') + for problem in found: + print(f'::error::{problem}' if os.environ.get('GITHUB_ACTIONS') else f'error: {problem}', file=sys.stderr) + if not found: + print(f'prepared output: OUTPUT_VERSION {current[0]} ({count} files, {value[:16]})') + return 1 if found else 0 + + +if __name__ == '__main__': + sys.exit(main()) From 0b49da246674bef76848ffe3ecb7517db48f0faf Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:07:58 +0000 Subject: [PATCH 43/61] Fix lead drops when leashed mobs die Follow Leashable.tickLeash for dying mobs and cover trader/player-held llama leads through the death animation. Co-Authored-By: Codex --- .../examples/probe/mob_ai_probe.rs | 16 ++++++++++++++++ crates/horizon-client/src/game/leads.rs | 2 ++ crates/horizon-client/src/mobs/leash.rs | 9 +++++++++ .../horizonlab/parity/VanillaMobAiOracle.java | 8 ++++++++ tools/test_mob_ai_parity.py | 15 ++++++++++++++- 5 files changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/horizon-client/examples/probe/mob_ai_probe.rs b/crates/horizon-client/examples/probe/mob_ai_probe.rs index 157baa74c..49f56c250 100644 --- a/crates/horizon-client/examples/probe/mob_ai_probe.rs +++ b/crates/horizon-client/examples/probe/mob_ai_probe.rs @@ -598,6 +598,7 @@ fn print(out: &mut String, t: i32, i: usize, m: &Mob, player_down: bool, now: i6 let _ = writeln!(out, " removed {reason}"); return; } + let _ = write!(out, "\tleashed {}", u8::from(m.extra.leash.is_some())); let p = m.position; let v = m.velocity; let _ = write!( @@ -977,6 +978,7 @@ pub fn main() -> horizon_client::Result<()> { } // Set once the brain exists (below). x if x.starts_with("cooldown=") => {} + "dead" => m.health = 0.0, "leashed" => m.extra.leash = Some(horizon_client::mobs::leash::Holder::Player), x if x.starts_with("leashto=") => { let n: u32 = x[8..].parse().unwrap(); @@ -1097,6 +1099,7 @@ pub fn main() -> horizon_client::Result<()> { } }; } + let mut dropped_leads = 0; for t in 1..=s.ticks { world.game_time += 1; if s.daycycle { @@ -1208,6 +1211,16 @@ pub fn main() -> horizon_client::Result<()> { if !alive { m.removed = true; } + // Apply the same detach/drop event as Game::break_leash. + for event in &events[from..] { + if let MobEvent::LeashBreak { mob, drop, .. } = event + && let Some(m) = mobs.iter_mut().find(|m| m.id == *mob) + && horizon_client::mobs::leash::remove(m).is_some() + && *drop + { + dropped_leads += 1; + } + } horizon_client::mobs::apply_brain_events(&mut mobs, &mut events, from); villager_events( &mut mobs, @@ -1427,6 +1440,9 @@ pub fn main() -> horizon_client::Result<()> { world.game_time, ); } + if s.name.starts_with("leash_death_") { + let _ = writeln!(out, "{t} leads {dropped_leads}"); + } for e in &events { if let MobEvent::Sound(name, _, volume, pitch) = e { let _ = writeln!( diff --git a/crates/horizon-client/src/game/leads.rs b/crates/horizon-client/src/game/leads.rs index 3a5112967..8867157ee 100644 --- a/crates/horizon-client/src/game/leads.rs +++ b/crates/horizon-client/src/game/leads.rs @@ -79,6 +79,8 @@ impl Game<'_> { if horizon_client::mobs::leash::remove(m).is_none() { return; } + // Leashable.tickLeash gates a dying entity's lead drop on entity_drops. + let drop = drop && (m.alive() || self.ui.rules.bool("entity_drops")); let at = [m.position.x, m.position.y, m.position.z]; if sound { self.lead_sound("break", at); diff --git a/crates/horizon-client/src/mobs/leash.rs b/crates/horizon-client/src/mobs/leash.rs index b7c5011ba..3bd0513d7 100644 --- a/crates/horizon-client/src/mobs/leash.rs +++ b/crates/horizon-client/src/mobs/leash.rs @@ -198,6 +198,15 @@ pub fn tick(m: &mut Mob, w: &MobWorld, events: &mut Vec) { m.extra.leash_spin = 0.0; return; }; + // Leashable.tickLeash: a dying entity cannot interact with the level. + if !m.alive() { + events.push(MobEvent::LeashBreak { + mob: m.id, + drop: true, + sound: false, + }); + return; + } let Some(state) = holder_state(w, holder) else { // The holder is gone: the lead drops. if !matches!(holder, Holder::Player) { diff --git a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaMobAiOracle.java b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaMobAiOracle.java index 5808fec0d..eb5235adf 100644 --- a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaMobAiOracle.java +++ b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaMobAiOracle.java @@ -604,6 +604,7 @@ public final class VanillaMobAiOracle { case "tame" -> { if (mob instanceof net.minecraft.world.entity.animal.equine.AbstractHorse horse) horse.setTamed(true); } case "strength" -> method(net.minecraft.world.entity.animal.equine.Llama.class, "setStrength", int.class) .invoke(mob, Integer.parseInt(v)); + case "dead" -> mob.setHealth(0.0F); case "leashed", "leashto" -> { // Leashed once every mob exists (below). } @@ -807,6 +808,12 @@ public final class VanillaMobAiOracle { for (BlockPos h : heartPositions) printHeart(t, level, h); for (String[] b : s.blocks) if (b[0].equals("chest")) printChest(t, s, level, b); for (String[] b : s.blocks) if (isHive(b)) printHive(t, level, b); + if (s.name.startsWith("leash_death_")) { + int leads = level.getEntitiesOfClass(ItemEntity.class, ENTS.get(0).getBoundingBox().inflate(32.0), + item -> item.getItem().is(net.minecraft.world.item.Items.LEAD)) + .stream().mapToInt(item -> item.getItem().getCount()).sum(); + out.println(t + " leads " + leads); + } for (String sound : SOUNDS) out.println(t + " sound " + sound); for (var w : watched.entrySet()) { BlockState now = level.getBlockState(w.getKey()); @@ -1008,6 +1015,7 @@ public final class VanillaMobAiOracle { return; } Mob m = (Mob) e; + sb.append("\tleashed ").append(((net.minecraft.world.entity.Leashable) m).isLeashed() ? 1 : 0); sb.append("\tp ").append(d(m.getX())).append(',').append(d(m.getY())).append(',').append(d(m.getZ())); Vec3 v = m.getDeltaMovement(); sb.append("\tv ").append(d(v.x)).append(',').append(d(v.y)).append(',').append(d(v.z)); diff --git a/tools/test_mob_ai_parity.py b/tools/test_mob_ai_parity.py index 3e6ce74df..87a7a1403 100644 --- a/tools/test_mob_ai_parity.py +++ b/tools/test_mob_ai_parity.py @@ -191,6 +191,16 @@ def scenarios(): 'mob minecraft:wandering_trader 0.5 -60 0.5 0 104 persistent', 'mob minecraft:trader_llama 3.5 -60 1.5 0 105 persistent leashto=0 strength=2', 'mob minecraft:trader_llama 3.5 -60 -1.5 0 106 persistent leashto=0 strength=4']) + # Death must detach immediately and drop exactly one lead throughout the corpse animation. + add('leash_death_trader_llama', 10, [ + 'player 14.5 -60 0.5', + 'mob minecraft:wandering_trader 0.5 -60 0.5 0 104 persistent', + 'mob minecraft:trader_llama 3.5 -60 1.5 0 105 persistent leashto=0 strength=2 dead', + ]) + add('leash_death_player_llama', 10, [ + 'player 0.5 -60 0.5', + 'mob minecraft:llama 3.5 -60 0.5 0 101 persistent leashed strength=3 dead', + ]) # Copper golems sorting chests (TransportItemsBetweenContainers): from a copper chest to a chest, # skipping a chest without the carried item, queueing at a chest another golem has open, a double # chest, a blocked copper chest, a part-full chest, and idling with nothing to carry. @@ -358,6 +368,9 @@ def parse(path): if parts[1] == 'hive': cur.setdefault('hives', {}).setdefault(t, []).append(parts[2]) continue + if parts[1] == 'leads': + cur.setdefault('leads', {}).setdefault(t, []).append(parts[2]) + continue if parts[1] == 'chest': cur.setdefault('chests', {}).setdefault(t, []).append(parts[2]) continue @@ -455,7 +468,7 @@ def compare(vanilla, rust, fields=None): clean = False break # Sounds, watched block changes, chest contents and hives: first differing tick. - for kind in ('sounds', 'blocks', 'chests', 'hives'): + for kind in ('sounds', 'blocks', 'chests', 'hives', 'leads'): va, ra = v.get(kind, {}), r.get(kind, {}) for t in sorted(set(va) | set(ra)): if va.get(t, []) != ra.get(t, []): From 8463325648f90045f83808dd201c07bfad2a264f Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:28:03 +0100 Subject: [PATCH 44/61] CI: Linux runners for the parity job (desktop, ThinkPad), paused while a game runs runner-env.sh and vanilla-data.sh find Python 3.13, the pinned JDK and sccache on NixOS; linux-runner.sh installs a runner as a systemd user service with its own screen; runner-pause.py stops it from taking jobs while a game runs. Co-Authored-By: Claude Opus 5.5 --- tools/ci/linux-runner.sh | 191 +++++++++++++++++++++++++++++++++++++++ tools/ci/runner-env.sh | 32 ++++++- tools/ci/runner-pause.py | 167 ++++++++++++++++++++++++++++++++++ tools/ci/vanilla-data.sh | 8 +- 4 files changed, 392 insertions(+), 6 deletions(-) create mode 100755 tools/ci/linux-runner.sh create mode 100755 tools/ci/runner-pause.py diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh new file mode 100755 index 000000000..0cb677c87 --- /dev/null +++ b/tools/ci/linux-runner.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# A Horizon CI runner (labels self-hosted, Linux, X64, horizon-game and the host's name) on a Linux machine of the +# maintainer's (NixOS: ludodesktop, ludothinkpad), as a systemd user service of the account that holds his +# Minecraft files, like the Mac mini's runners (docs/release-flow.md). Run it on that machine, as that user: +# +# bash tools/ci/linux-runner.sh install # register (gh login), install, start; again to update the scripts +# bash tools/ci/linux-runner.sh status # the services, the runner's state, the pause watcher's last lines +# bash tools/ci/linux-runner.sh uninstall # stop and unregister +# +# Everything lives in ~/.actions-runners/horizon (never in ~/code): the runner, its own rustup and cargo homes, +# the work folder, and copies of this script and runner-pause.py taken at install time (a pull request's checkout +# never runs as the service). The service runs nice, in horizon-ci.slice (low CPU and I/O weight, memory capped), +# on its own virtual screen: vscreen (a headless sway) where it is installed, else Xvfb from nixpkgs with Mesa's +# Vulkan drivers. runner-pause.py pauses it while a game runs (horizon-runner-pause.service). +# HORIZON_SHARED_SOURCE: the imported vanilla data (default ~/horizon-local/shared-local, else +# ~/code/horizon/.local); RUNNER_JOBS: cargo jobs and harnesses at once (default 2/3 of the threads). +set -euo pipefail +repo=ludovic111/horizon +dir=$HOME/.actions-runners/horizon +units=${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user +version=2.338.0 +sha=af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32 +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Nix packages the jobs need, kept alive by GC roots in $dir/nix. +nix_out() { nix-build '' -A "$1" -o "$dir/nix/$2" > /dev/null && readlink -f "$dir/nix/$2"; } + +# The runner is a .NET program: on NixOS its native libraries (libstdc++, ICU, OpenSSL) come from nix-ld. +export LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} +# NixOS has no /bin/bash: the runner's scripts (again after each of its self-updates) get a portable one. +fix_shebangs() { sed -i '1s|^#!/bin/bash|#!/usr/bin/env bash|' "$dir"/*.sh "$dir"/bin/*.sh; } + +display_env() { + # The jobs' screen. Never the maintainer's: the user manager hands every service his DISPLAY and WAYLAND_DISPLAY. + if command -v vscreen > /dev/null; then + VSCREEN=ci-horizon vscreen env + else + echo "unset WAYLAND_DISPLAY; export DISPLAY=:$(cat "$dir/display")" + fi +} + +install() { + local name=${RUNNER_NAME:-$(hostname)-horizon} jobs=${RUNNER_JOBS:-$(( $(nproc) * 2 / 3 ))} + local source=${HORIZON_SHARED_SOURCE:-} + if [ -z "$source" ]; then + for candidate in "$HOME/horizon-local/shared-local" "$HOME/code/horizon/.local"; do + [ -d "$candidate/minecraft" ] && { source=$candidate; break; } + done + fi + [ -n "$source" ] && [ -d "$source/minecraft" ] || { echo "no imported Minecraft (HORIZON_SHARED_SOURCE)"; exit 1; } + mkdir -p "$dir/nix" "$units" + cp "$here/linux-runner.sh" "$here/runner-pause.py" "$dir/" + + if [ ! -f "$dir/.runner" ]; then + local cache=$HOME/.cache/gha-runner/actions-runner-linux-x64-$version.tar.gz + if ! echo "$sha $cache" | sha256sum -c --quiet - > /dev/null 2>&1; then + mkdir -p "$(dirname "$cache")" + curl -fsSL -o "$cache" "https://github.com/actions/runner/releases/download/v$version/actions-runner-linux-x64-$version.tar.gz" + echo "$sha $cache" | sha256sum -c --quiet - + fi + tar xzf "$cache" -C "$dir" + fix_shebangs + gh api -X POST "repos/$repo/actions/runners/registration-token" --jq .token | + (cd "$dir" && bash ./config.sh --unattended --replace --url "https://github.com/$repo" --token "$(cat)" \ + --name "$name" --labels "${RUNNER_LABELS:-horizon-game,$(hostname)}" --work _work > /dev/null) + fi + + # Libraries for building (pkg-config) and running (dlopen) the game, the JDK pinned by research-lock.json, sccache. + local alsa udev java sccache + alsa=$(nix_out alsa-lib.dev alsa-dev) + udev=$(nix_out systemd.dev systemd-dev) + java=$(nix_out "jdk$(sed -n 's/.*"java_major": *\([0-9]*\).*/\1/p' "$here/../../research-lock.json" | head -1)_headless" jdk)/lib/openjdk + sccache=$(nix_out sccache sccache) + local libs=/run/opengl-driver/lib:/run/current-system/sw/share/nix-ld/lib + if ! command -v vscreen > /dev/null; then + local mesa x + mesa=$(nix_out mesa mesa) + x=$(nix_out xvfb-run xvfb-run) + for lib in libx11 libxcursor libxrandr libxi libxext libxinerama libxrender libxxf86vm libxcb libxkbcommon \ + vulkan-loader libGL; do + libs=$libs:$(nix_out "$lib" "$lib")/lib + done + libs=$mesa/lib:$libs + echo "VK_DRIVER_FILES=$mesa/share/vulkan/icd.d/intel_icd.x86_64.json:$mesa/share/vulkan/icd.d/lvp_icd.x86_64.json" \ + > "$dir/.env.display" + echo "$x/bin" > "$dir/.path.display" + fi + cat > "$dir/.env" <> "$dir/.env"; else echo "VSCREEN=ci-horizon" >> "$dir/.env"; fi + echo "$dir/cargo/bin:$(cat "$dir/.path.display" 2>/dev/null || true)${dir}/nix/sccache/bin:$java/bin:$HOME/.local/bin:/run/wrappers/bin:/etc/profiles/per-user/$USER/bin:/run/current-system/sw/bin" \ + | sed 's/::/:/g' > "$dir/.path" + + cat > "$units/horizon-ci.slice" < "$units/horizon-runner.service" < "$units/horizon-runner-pause.service" </dev/null)" != yes ]; then + sudo -n loginctl enable-linger "$USER" || echo "Run once: sudo loginctl enable-linger $USER" + fi + systemctl --user daemon-reload + systemctl --user enable horizon-runner.service horizon-runner-pause.service > /dev/null + systemctl --user restart horizon-runner-pause.service + [ -e "$dir/paused" ] || systemctl --user start horizon-runner.service + status +} + +run() { + cd "$dir" + fix_shebangs + if command -v vscreen > /dev/null; then + eval "$(display_env)" + else + # Xvfb on the first free display from :90, for this service's lifetime. + local n=90 + while [ -e "/tmp/.X$n-lock" ]; do n=$((n + 1)); done + echo "$n" > "$dir/display" + Xvfb ":$n" -screen 0 1280x720x24 -nolisten tcp & + eval "$(display_env)" + fi + exec bash ./bin/runsvc.sh +} + +status() { + systemctl --user --no-pager list-units 'horizon-runner*' 'vscreen-ci-horizon*' + [ -e "$dir/paused" ] && echo "paused: $(cat "$dir/paused")" + tail -5 "$dir/pause.log" 2>/dev/null || true +} + +uninstall() { + systemctl --user disable --now horizon-runner-pause.service horizon-runner.service 2>/dev/null || true + gh api -X POST "repos/$repo/actions/runners/remove-token" --jq .token | + (cd "$dir" && bash ./config.sh remove --token "$(cat)") + rm -f "$units/horizon-runner.service" "$units/horizon-runner-pause.service" "$units/horizon-ci.slice" + systemctl --user daemon-reload +} + +case ${1:-} in + install|run|status|uninstall) "$1" ;; + *) sed -n '2,10p' "$0"; exit 2 ;; +esac diff --git a/tools/ci/runner-env.sh b/tools/ci/runner-env.sh index 49158ac91..39828b7d7 100755 --- a/tools/ci/runner-env.sh +++ b/tools/ci/runner-env.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# The tools the vanilla harnesses need, on the self-hosted Mac mini runner (label horizon-game): its service -# starts with a bare PATH. Python 3.13 and the JDK pinned by research-lock.json come from the machine, never -# from a download. Writes them to $GITHUB_PATH and $GITHUB_ENV for the next steps. +# The tools the vanilla harnesses need, on a self-hosted runner (label horizon-game): the Mac mini's, whose service +# starts with a bare PATH, and the Linux ones (tools/ci/linux-runner.sh, NixOS). Python 3.13 and the JDK pinned by +# research-lock.json come from the machine, never from a download. Writes them to $GITHUB_PATH and $GITHUB_ENV for +# the next steps. set -euo pipefail java_major=$(sed -n 's/.*"java_major": *\([0-9]*\).*/\1/p' research-lock.json | head -1) bin="$RUNNER_TEMP/horizon-bin" @@ -17,5 +18,30 @@ if [ "$(uname)" = Darwin ]; then if [ -x /opt/homebrew/bin/sccache ]; then { echo "RUSTC_WRAPPER=/opt/homebrew/bin/sccache"; echo "CARGO_INCREMENTAL=0"; } >> "$GITHUB_ENV" fi +else + python=$(command -v python3.13 || true) + [ -n "$python" ] || { echo "::error::no python3.13 on this runner"; exit 1; } + ln -sfn "$python" "$bin/python3" + ln -sfn "$python" "$bin/python" + # The JDK: the runner's JAVA_HOME (linux-runner.sh pins one), else the java on PATH, else nixpkgs' jdk. + major_of() { "$1/bin/java" -version 2>&1 | sed -n 's/.*version "\([0-9]*\).*/\1/p' | head -1; } + java_home=${JAVA_HOME:-} + if [ -z "$java_home" ] || [ "$(major_of "$java_home")" != "$java_major" ]; then + java_home="" + if command -v java > /dev/null; then + candidate=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")") + [ "$(major_of "$candidate")" = "$java_major" ] && java_home=$candidate + fi + fi + if [ -z "$java_home" ] && command -v nix-build > /dev/null; then + out=$(nix-build '' --no-out-link -A "jdk${java_major}_headless" 2>/dev/null || true) + [ -n "$out" ] && java_home=$out/lib/openjdk + fi + [ -n "$java_home" ] || { echo "::error::no JDK $java_major on this runner"; exit 1; } + { echo "$bin"; echo "$java_home/bin"; } >> "$GITHUB_PATH" + echo "JAVA_HOME=$java_home" >> "$GITHUB_ENV" + if sccache=$(command -v sccache); then + { echo "RUSTC_WRAPPER=$sccache"; echo "CARGO_INCREMENTAL=0"; } >> "$GITHUB_ENV" + fi fi echo "HORIZON_NO_AUDIO=1" >> "$GITHUB_ENV" diff --git a/tools/ci/runner-pause.py b/tools/ci/runner-pause.py new file mode 100755 index 000000000..4a88c6ae4 --- /dev/null +++ b/tools/ci/runner-pause.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Pauses the Horizon CI runner (tools/ci/linux-runner.sh) while the maintainer plays a game on this machine. + +A game is running when any of these holds: + - fleet-load says so (~/fleet/load: Steam, gamescope, Wine or Proton processes); + - a process of his matches GAME below (the same, plus Minecraft and a Horizon game he started himself); + - a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY % twice in a row); + - ~/.actions-runners/horizon/pause exists (pause by hand: `touch` it; `rm` it to let the watcher decide). + +Paused, the runner takes no new job: while it is idle its service is stopped, so GitHub sends the jobs to the +Mac mini. A job already running is throttled, not stopped: horizon-ci.slice (the job, and the parity workers other machines +send here) drops to CPU weight 1, I/O weight 1 +and a quota of THROTTLE_CPUS threads, and the service stops once the job ends. Freezing it (SIGSTOP or the cgroup +freezer) would stop the runner's heartbeat, so GitHub would drop the job as lost, and the harnesses' wall-clock +timeouts would fail runs that were only waiting. The runner comes back RESUME_AFTER seconds after the last sign +of a game. Every change is written to ~/.actions-runners/horizon/pause.log. +""" +import json +import os +import re +import subprocess +import time +from pathlib import Path + +DIR = Path.home() / ".actions-runners" / "horizon" +SERVICE, SLICE = "horizon-runner.service", "horizon-ci.slice" +MANUAL, STATE, LOG = DIR / "pause", DIR / "paused", DIR / "pause.log" +GAME = re.compile(r"(SteamLaunch|gamescope|wine64-preloader|wineserver|wine-preloader|\.exe(\s|$)|proton\b" + r"|net\.minecraft\.client\.main\.Main|minecraft-launcher|prismlauncher)", re.I) +GPU_BUSY = int(os.environ.get("GPU_BUSY", 30)) +INTERVAL = int(os.environ.get("INTERVAL", 15)) +RESUME_AFTER = int(os.environ.get("RESUME_AFTER", 300)) +THROTTLE_CPUS = int(os.environ.get("THROTTLE_CPUS", 2)) + + +def log(text): + with LOG.open("a") as handle: + handle.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S')} {text}\n") + + +def systemctl(*arguments): + return subprocess.run(["systemctl", "--user", *arguments], capture_output=True, text=True) + + +def ci_process(pid): + """A process of the runner's slice (its jobs, the parity workers other machines start here).""" + try: + return SLICE in Path(f"/proc/{pid}/cgroup").read_text() + except OSError: + return True # gone + + +def fleet_load_gaming(): + status = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "fleet-load" / "status.json" + try: + if time.time() - status.stat().st_mtime > 120: + return None + return "fleet-load: playing a game" if json.loads(status.read_text())["human"]["gaming"] else None + except (OSError, ValueError, KeyError, TypeError): + return None + + +def game_process(): + uid = os.getuid() + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + if entry.stat().st_uid != uid: + continue + command = (entry / "cmdline").read_bytes().replace(b"\0", b" ").decode(errors="replace") + except OSError: + continue + # A Horizon game counts when he started it (its parent is his launcher), not when a harness or agent did. + if GAME.search(command) or (" --offscreen" not in command and "/horizon-launcher/" in command + and "horizon-client" in command.split(" ")[0]): + if not ci_process(int(entry.name)): + return f"process {entry.name}: {command[:120]}" + return None + + +def gpu_users(): + """Processes outside the CI slice using the GPU's shader cores, as {pid: SM %}.""" + try: + output = subprocess.run(["nvidia-smi", "pmon", "-c", "1", "-s", "u"], capture_output=True, text=True, + timeout=20).stdout + except (OSError, subprocess.TimeoutExpired): + return {} + users = {} + for line in output.splitlines(): + fields = line.split() + if line.startswith("#") or len(fields) < 4 or not fields[1].isdigit() or not fields[3].isdigit(): + continue + if not ci_process(int(fields[1])): + users[int(fields[1])] = int(fields[3]) + return users + + +def runner_busy(): + """The runner is in a job: a Runner.Worker runs in its service.""" + group = systemctl("show", "-p", "ControlGroup", "--value", SERVICE).stdout.strip() + try: + pids = Path(f"/sys/fs/cgroup{group}/cgroup.procs").read_text().split() if group else [] + except OSError: + return False + for pid in pids: + try: + if Path(f"/proc/{pid}/comm").read_text().strip() == "Runner.Worker": + return True + except OSError: + pass + return False + + +def throttle(on): + if on: + systemctl("set-property", "--runtime", SLICE, "CPUWeight=1", "IOWeight=1", f"CPUQuota={THROTTLE_CPUS * 100}%") + else: + systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", "CPUQuota=") + + +def main(): + hot = {} # GPU users seen busy at the last sample + last_game = 0.0 + paused = STATE.exists() + while True: + reason = None + if MANUAL.exists(): + reason = "paused by hand (" + str(MANUAL) + ")" + reason = reason or fleet_load_gaming() or game_process() + if not reason: + busy = {pid for pid, sm in gpu_users().items() if sm >= GPU_BUSY} + twice = busy & set(hot) + hot = dict.fromkeys(busy) + if twice: + pid = min(twice) + try: + name = Path(f"/proc/{pid}/comm").read_text().strip() + except OSError: + name = "?" + reason = f"GPU busy: process {pid} ({name})" + now = time.monotonic() + if reason: + last_game = now + if reason and not paused: + paused = True + STATE.write_text(reason + "\n") + throttle(True) # also what other machines' parity workers run here meanwhile + if runner_busy(): + log(f"pause: {reason}; a job is running: throttled to {THROTTLE_CPUS} threads, stops after it") + else: + systemctl("stop", SERVICE) + log(f"pause: {reason}; runner stopped") + elif paused and reason is None and now - last_game >= RESUME_AFTER: + paused = False + STATE.unlink(missing_ok=True) + throttle(False) + systemctl("start", SERVICE) + log(f"resume: no game for {RESUME_AFTER} s; runner started") + elif paused and systemctl("is-active", "--quiet", SERVICE).returncode == 0 and not runner_busy(): + systemctl("stop", SERVICE) + log("the job ended: runner stopped until the game ends") + time.sleep(INTERVAL) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/vanilla-data.sh b/tools/ci/vanilla-data.sh index 5276c54e9..a7bd4a407 100755 --- a/tools/ci/vanilla-data.sh +++ b/tools/ci/vanilla-data.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # The vanilla data for this checkout on the self-hosted runner. Horizon never downloads the game: the runner # holds the maintainer's own copy, imported and decompiled once (HORIZON_SHARED_SOURCE, default -# ~/code/horizon-parity/shared-local). It is cloned once into the runner's work folder ($RUNNER_WORKSPACE/ -# shared-local; APFS clones, so no second copy of the bytes), and .local/ links to it as in a parity worktree. +# ~/code/horizon-parity/shared-local; tools/ci/linux-runner.sh sets it on Linux). It is cloned once into the runner's +# work folder ($RUNNER_WORKSPACE/shared-local; APFS clones, or reflinks where the file system has them, so no second +# copy of the bytes), and .local/ links to it as in a parity worktree. # .local/horizon-client and the exporter outputs beside it, what horizon-prepare makes, are prepared by this # checkout's own preparer once per preparer version (its sources, the exporters and research-lock.json), so a # change to the preparer is checked with what it prepares; the four newest preparations are kept. @@ -31,7 +32,8 @@ for entry in "$shared"/*; do done # HORIZON_PREPARE_KEY: the same key, computed where the Git history is (tools/pr_ready.py on another machine). -key=${HORIZON_PREPARE_KEY:-$(git ls-files -s crates/horizon-prepare mods/horizon/src/test/java research-lock.json | shasum -a 256 | cut -c1-16)} +sha256() { if command -v shasum > /dev/null; then shasum -a 256; else sha256sum; fi; } +key=${HORIZON_PREPARE_KEY:-$(git ls-files -s crates/horizon-prepare mods/horizon/src/test/java research-lock.json | sha256 | cut -c1-16)} prepared="$shared/prepared/$key" # Two runners share the vanilla data (~/.actions-runners/horizon-2/_work/horizon/shared-local links to the first # one's): one preparation at a time. A lock older than 30 minutes belongs to a job that died. From d81aa97e842a753e231ae34b181bc1bd2654c589 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:12:08 +0100 Subject: [PATCH 45/61] CI: the desktop runner works only while nobody is at the screen; desktop as a parity host runner_pause.py pauses on any seat0 session (locked too), games, GPU use or a pause file, and is the only thing that starts the runner; a job under way is throttled, then the runner stops. The desktop host in parity_hosts.json refuses work while paused and its workers stop on a pause. The nightly audit stays on the Mac mini. Docs and a unit test of the watcher. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 3 +- docs/parity-workflow.md | 2 +- docs/release-flow.md | 28 +++- tests/test_runner_pause.py | 82 ++++++++++++ tools/ci/linux-runner.sh | 52 +++++--- tools/ci/{runner-pause.py => runner_pause.py} | 125 +++++++++++------- tools/parity_hosts.json | 9 ++ 7 files changed, 230 insertions(+), 71 deletions(-) create mode 100644 tests/test_runner_pause.py rename tools/ci/{runner-pause.py => runner_pause.py} (51%) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 51f9e89cd..60bbbf057 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -29,7 +29,8 @@ env: jobs: audit: - runs-on: [self-hosted, horizon-game] + # The Mac mini's runners only: the audit compares with the previous one, so it stays on one platform. + runs-on: [self-hosted, horizon-game, macOS] timeout-minutes: 150 steps: - name: 03:00 in Paris diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index f9f222071..80426fa59 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -38,7 +38,7 @@ The game logs into `HORIZON_HOME` and falls back to the player's own game folder ### Other machines -Harnesses can also run on the Mac mini and the ThinkPad (`tools/parity_hosts.json` describes them; its `default` list is empty, so pass `--hosts macmini,thinkpad` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 3 and only those under 800 MB. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. +Harnesses can also run on the Mac mini, the ThinkPad and the desktop (`tools/parity_hosts.json` describes them; `desktop` uses its CI runner's environment and vanilla data, and refuses work, or stops it, while someone is at it: docs/release-flow.md, "The desktop's runner"; its `default` list is empty, so pass `--hosts macmini,thinkpad` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 3 and only those under 800 MB. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. Harnesses a host can't run are listed in the file: `local_only` (`save_roundtrip` runs the game on a virtual screen) and each host's `exclude`. The Mac excludes nothing: vanilla's `Math.sin`, `pow`, `log` and the like don't round the same on an Apple silicon Mac as on Linux (HotSpot uses fdlibm there, Intel's stubs on x86-64), and Rust's float functions round like neither (they call the platform libm), so code that mirrors a vanilla `Math` call uses `horizon_core::jmath`, which computes what HotSpot computes on the host architecture. The ThinkPad helps too, gently: Linux, 2 harnesses at `nice 19`, since it hosts the Claude account pool; it finds ALSA through `~/code/horizon-parity/alsa-dev` like the desktop. diff --git a/docs/release-flow.md b/docs/release-flow.md index 4fb0209d9..e72bebc73 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -13,7 +13,7 @@ reopened; labels never start it): | Job | Runs on | What | Time | | --- | --- | --- | --- | | `verify` | GitHub `ubuntu-latest` | Public-tree check, Python tests, rustfmt, clippy with warnings denied, Rust tests. Dependencies and the workspace build come from a cache saved by main's runs | ~4 min cold, less warm | -| `parity / parity` | The Mac mini's runner (`ludovics-mac-mini`, label `horizon-game`) | The vanilla harnesses the change can affect (`tools/parity.py --changed `) | seconds when nothing is affected; minutes per affected harness | +| `parity / parity` | A runner with the label `horizon-game`: the Mac mini's two, and the desktop's when no game runs there (below) | The vanilla harnesses the change can affect (`tools/parity.py --changed `) | seconds when nothing is affected; minutes per affected harness | Pushes to main run `verify` only; the nightly audit covers main's harnesses. @@ -28,10 +28,13 @@ The job is `.github/workflows/parity.yml`, a reusable workflow: `checks` calls i repository, `run-parity.yml` for a fork's pull request a maintainer labels `run-parity`. Horizon never downloads the game, so harnesses that compare with vanilla run only where the maintainer's own -copy is: the Mac mini. Its runner runs as the maintainer's account and keeps its checkout between runs -(`clean: false`), so the probes build incrementally into `.local/parity-target` and earlier passes stand. +copy is: the Mac mini and his Linux desktop. Their runners run as the maintainer's account and keep their checkout +between runs (`clean: false`), so the probes build incrementally into `.local/parity-target` and earlier passes +stand. -1. `tools/ci/runner-env.sh` puts the machine's Python 3.13 and the JDK `research-lock.json` pins on the PATH. +1. `tools/ci/runner-env.sh` puts the machine's Python 3.13 and the JDK `research-lock.json` pins on the PATH + (Homebrew and `java_home` on the Mac; on NixOS what `linux-runner.sh` pinned, else nixpkgs), and sccache when + the machine has it. 2. `tools/ci/vanilla-data.sh` links `.local/` to the runner's copy of the vanilla data, cloned once from `~/code/horizon-parity/shared-local` into `~/.actions-runners/horizon/_work/horizon/shared-local`, and prepares `.local/horizon-client` with the pull request's own `horizon-prepare` once per preparer version @@ -64,6 +67,23 @@ release's macOS build back. Each keeps its own checkout, `target/` and Rust inst `_work/horizon/shared-local` links to the first's, so they share the vanilla data, the preparations (one at a time, under a lock) and the parity passes and slots. +**The desktop's runner.** `ludodesktop-horizon` (labels `self-hosted`, `Linux`, `X64`, `ludodesktop`, and +`horizon-game` once a full trial has passed there) runs on the maintainer's Linux desktop, only while nobody is +at it. `tools/ci/linux-runner.sh install`, run there as him, registers it and installs two systemd user services: +`horizon-runner` (the runner, in `~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O +weight 20 and memory capped at two thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the +machine froze under ~7 cold cargo builds on 2026-10-14; its own Rust install; the JDK, sccache and the build +libraries from nixpkgs; its own `vscreen` so no game it starts reaches his screen) and `horizon-runner-pause` +(`tools/ci/runner_pause.py`), the only thing that starts the runner. The watcher stops it while a graphical +session is open on the machine, locked or not (`~/fleet/power status` says "IN USE"), while a game runs (Steam, +gamescope, Wine, Proton or Minecraft processes, fleet-load's own detection, or a process outside the CI slice +using the GPU at 50 % or more), or while `~/.actions-runners/horizon/pause` exists, and starts it again 5 minutes +after the last such sign. A job already under way is throttled to 2 threads at the lowest weight and the runner +stops after it: freezing the job would make GitHub drop it as lost and the harnesses time out. Old pull requests' +`runner-env.sh` know only the Mac: the runner's `.env` and `.path` give their jobs Python, the JDK and the +vanilla data (`HORIZON_SHARED_SOURCE`) all the same. The nightly audit stays on the Mac mini's runners, so its +comparison with the previous night is always made on one platform. + ## Merging Pull requests merge with a merge commit. The orchestrator sessions review a pull request, then turn on diff --git a/tests/test_runner_pause.py b/tests/test_runner_pause.py new file mode 100644 index 000000000..c6df08b82 --- /dev/null +++ b/tests/test_runner_pause.py @@ -0,0 +1,82 @@ +"""tools/ci/runner_pause.py: the Linux CI runner works only while nobody is at the screen or playing.""" +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from tools.ci import runner_pause + + +class Machine: + """A fake desktop: a clock, who is there, and the runner service. The watcher samples every 15 s.""" + + def __init__(self, signs): + self.signs = signs # (from, until, reason): a sign of someone at the screen or playing + self.clock = 0.0 + self.active = False + self.job_until = None # the runner is in a job until then + self.throttles = [] # (clock, on) + self.calls = [] + + def reason(self, hot): + return next((reason for start, end, reason in self.signs if start <= self.clock < end), None) + + def systemctl(self, *arguments): + self.calls.append((self.clock, arguments)) + if arguments[1:] == (runner_pause.SERVICE,): + self.active = {"start": True, "stop": False}.get(arguments[0], self.active) + return mock.Mock(returncode=0 if arguments[:2] != ("is-active", "--quiet") or self.active else 3) + + def busy(self): + return self.job_until is not None and self.clock < self.job_until + + def sleep(self, seconds): + self.clock += seconds + if self.clock > 2000: + raise StopIteration + + def run(self): + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.multiple(runner_pause, MANUAL=Path(directory, "pause"), STATE=Path(directory, "paused"), + LOG=Path(directory, "pause.log"), INTERVAL=15, RESUME_AFTER=300, + pause_reason=self.reason, systemctl=self.systemctl, runner_busy=self.busy, + throttle=lambda on: self.throttles.append((self.clock, on))), \ + mock.patch.object(runner_pause.time, "monotonic", lambda: self.clock), \ + mock.patch.object(runner_pause.time, "sleep", self.sleep): + try: + runner_pause.main() + except StopIteration: # the clock ran out + pass + + def first(self, verb, after=0): + return next(clock for clock, arguments in self.calls + if arguments == (verb, runner_pause.SERVICE) and clock >= after) + + +class RunnerPauseTests(unittest.TestCase): + def test_starts_only_after_nobody_was_there_for_resume_after(self): + machine = Machine([(0, 600, "session 3 of ludovic at the screen (seat0)")]) + machine.run() + self.assertEqual(machine.first("start"), 585 + 300) # the last sample that saw him + + def test_a_game_stops_an_idle_runner_at_once(self): + machine = Machine([(1000, 1300, "process 42: wine64-preloader game.exe")]) + machine.run() + self.assertEqual(machine.first("start"), 300) + self.assertEqual(machine.first("stop", after=300), 1005) + self.assertEqual(machine.first("start", after=1005), 1290 + 300) + + def test_a_job_under_way_is_throttled_then_stopped_when_it_ends(self): + machine = Machine([(1000, 1300, "GPU busy: process 7 (game)")]) + machine.job_until = 1200 + machine.run() + self.assertIn((1005, True), machine.throttles) # the game started: the job slows down at once, + self.assertIn((1005, ("stop", "horizon-parity-worker-*.scope")), machine.calls) # other machines' work goes + stop = machine.first("stop", after=300) + self.assertGreaterEqual(stop, 1200) # and the runner stops once it ends, not before + self.assertLess(stop, 1215) + self.assertEqual(machine.throttles[-1], (1290 + 300, False)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index 0cb677c87..adc2b0130 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -8,12 +8,14 @@ # bash tools/ci/linux-runner.sh uninstall # stop and unregister # # Everything lives in ~/.actions-runners/horizon (never in ~/code): the runner, its own rustup and cargo homes, -# the work folder, and copies of this script and runner-pause.py taken at install time (a pull request's checkout -# never runs as the service). The service runs nice, in horizon-ci.slice (low CPU and I/O weight, memory capped), +# the work folder, and copies of this script and runner_pause.py taken at install time (a pull request's checkout +# never runs as the service). horizon-runner-pause.service (runner_pause.py) starts the runner only while nobody is +# at the screen or playing, and stops it otherwise. The runner runs nice, in horizon-ci.slice (low CPU and I/O weight, memory capped), # on its own virtual screen: vscreen (a headless sway) where it is installed, else Xvfb from nixpkgs with Mesa's -# Vulkan drivers. runner-pause.py pauses it while a game runs (horizon-runner-pause.service). +# Vulkan drivers. # HORIZON_SHARED_SOURCE: the imported vanilla data (default ~/horizon-local/shared-local, else -# ~/code/horizon/.local); RUNNER_JOBS: cargo jobs and harnesses at once (default 2/3 of the threads). +# ~/code/horizon/.local); RUNNER_JOBS: cargo jobs and harnesses at once (default a third of the threads, at most 8: +# ludodesktop froze under ~7 cold cargo builds on 2026-10-14, and it is the maintainer's own machine). set -euo pipefail repo=ludovic111/horizon dir=$HOME/.actions-runners/horizon @@ -30,6 +32,18 @@ export LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib${LD_LIBRARY_PATH: # NixOS has no /bin/bash: the runner's scripts (again after each of its self-updates) get a portable one. fix_shebangs() { sed -i '1s|^#!/bin/bash|#!/usr/bin/env bash|' "$dir"/*.sh "$dir"/bin/*.sh; } +# Other machines' parity runs can send harnesses here (tools/parity_hosts.json "desktop": parity.py --worker over +# ssh, in horizon-ci.slice). Their vanilla data is the runner checkout's: parity-host/shared-local links to its +# .local links, so it follows the preparation of the runner's last job. +parity_host_links() { + local checkout=$dir/_work/horizon/horizon/.local shared=$dir/parity-host/shared-local entry + [ -d "$checkout" ] || return 0 + mkdir -p "$shared" + for entry in "$checkout"/*; do + [ -L "$entry" ] && ln -sfn "$entry" "$shared/$(basename "$entry")" + done +} + display_env() { # The jobs' screen. Never the maintainer's: the user manager hands every service his DISPLAY and WAYLAND_DISPLAY. if command -v vscreen > /dev/null; then @@ -40,7 +54,9 @@ display_env() { } install() { - local name=${RUNNER_NAME:-$(hostname)-horizon} jobs=${RUNNER_JOBS:-$(( $(nproc) * 2 / 3 ))} + local name=${RUNNER_NAME:-$(hostname)-horizon} threads jobs + threads=$(nproc) + jobs=${RUNNER_JOBS:-$(( threads / 3 > 8 ? 8 : threads / 3 > 1 ? threads / 3 : 1 ))} local source=${HORIZON_SHARED_SOURCE:-} if [ -z "$source" ]; then for candidate in "$HOME/horizon-local/shared-local" "$HOME/code/horizon/.local"; do @@ -49,7 +65,7 @@ install() { fi [ -n "$source" ] && [ -d "$source/minecraft" ] || { echo "no imported Minecraft (HORIZON_SHARED_SOURCE)"; exit 1; } mkdir -p "$dir/nix" "$units" - cp "$here/linux-runner.sh" "$here/runner-pause.py" "$dir/" + cp "$here/linux-runner.sh" "$here/runner_pause.py" "$dir/" if [ ! -f "$dir/.runner" ]; then local cache=$HOME/.cache/gha-runner/actions-runner-linux-x64-$version.tar.gz @@ -66,11 +82,11 @@ install() { fi # Libraries for building (pkg-config) and running (dlopen) the game, the JDK pinned by research-lock.json, sccache. - local alsa udev java sccache + local alsa udev java alsa=$(nix_out alsa-lib.dev alsa-dev) udev=$(nix_out systemd.dev systemd-dev) java=$(nix_out "jdk$(sed -n 's/.*"java_major": *\([0-9]*\).*/\1/p' "$here/../../research-lock.json" | head -1)_headless" jdk)/lib/openjdk - sccache=$(nix_out sccache sccache) + nix_out sccache sccache > /dev/null # on the PATH below: runner-env.sh hands it to cargo local libs=/run/opengl-driver/lib:/run/current-system/sw/share/nix-ld/lib if ! command -v vscreen > /dev/null; then local mesa x @@ -91,6 +107,7 @@ RUSTUP_HOME=$dir/rustup CARGO_HOME=$dir/cargo CARGO_BUILD_JOBS=$jobs PARITY_MACHINE_JOBS=$jobs +PARITY_LOCAL_JOBS=$jobs JAVA_HOME=$java HORIZON_SHARED_SOURCE=$source HORIZON_NO_AUDIO=1 @@ -118,27 +135,25 @@ MemoryMax=$(( $(awk '/MemTotal/ {print int($2 / 1048576)}' /proc/meminfo) * 2 / EOF cat > "$units/horizon-runner.service" < "$units/horizon-runner-pause.service" < /dev/null + # Only the watcher starts the runner (never the boot itself): it decides whether anyone is at the screen. + systemctl --user enable horizon-runner-pause.service > /dev/null + # The runner reads .env when it starts: stop it, unless it is in a job (that would cancel the job); the watcher + # starts it again when it may run. + if pgrep -f "$dir/bin/Runner.Worker" > /dev/null; then echo "In a job: stop horizon-runner.service after it" + else systemctl --user stop horizon-runner.service; rm -f "$dir/paused"; fi systemctl --user restart horizon-runner-pause.service - [ -e "$dir/paused" ] || systemctl --user start horizon-runner.service status } run() { cd "$dir" fix_shebangs + parity_host_links if command -v vscreen > /dev/null; then eval "$(display_env)" else diff --git a/tools/ci/runner-pause.py b/tools/ci/runner_pause.py similarity index 51% rename from tools/ci/runner-pause.py rename to tools/ci/runner_pause.py index 4a88c6ae4..57f304f1f 100755 --- a/tools/ci/runner-pause.py +++ b/tools/ci/runner_pause.py @@ -1,19 +1,22 @@ #!/usr/bin/env python3 -"""Pauses the Horizon CI runner (tools/ci/linux-runner.sh) while the maintainer plays a game on this machine. +"""Pauses the Horizon CI runner (tools/ci/linux-runner.sh) while the maintainer is at this machine or plays on it. -A game is running when any of these holds: - - fleet-load says so (~/fleet/load: Steam, gamescope, Wine or Proton processes); +The runner works only while nobody is at the screen (rule of 2026-10-25, after ludodesktop was restarted under a +CI load while he used it). It pauses when any of these holds: + - a graphical session is open on seat0, locked or not: what `~/fleet/power status` calls "IN USE"; + - fleet-load says a game runs (~/fleet/load: Steam, gamescope, Wine or Proton processes); - a process of his matches GAME below (the same, plus Minecraft and a Horizon game he started himself); - - a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY % twice in a row); + - a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY %, default 50, in two + samples in a row; the browser, Discord and the compositor stay under 20 %); - ~/.actions-runners/horizon/pause exists (pause by hand: `touch` it; `rm` it to let the watcher decide). Paused, the runner takes no new job: while it is idle its service is stopped, so GitHub sends the jobs to the -Mac mini. A job already running is throttled, not stopped: horizon-ci.slice (the job, and the parity workers other machines -send here) drops to CPU weight 1, I/O weight 1 -and a quota of THROTTLE_CPUS threads, and the service stops once the job ends. Freezing it (SIGSTOP or the cgroup -freezer) would stop the runner's heartbeat, so GitHub would drop the job as lost, and the harnesses' wall-clock -timeouts would fail runs that were only waiting. The runner comes back RESUME_AFTER seconds after the last sign -of a game. Every change is written to ~/.actions-runners/horizon/pause.log. +Mac mini, and the parity workers other machines started here stop (their harnesses go back to those machines). A +job already running is throttled, not stopped: horizon-ci.slice drops to CPU weight 1, I/O weight 1 and a quota of +THROTTLE_CPUS threads, and the service stops once the job ends. Freezing it (SIGSTOP or the cgroup freezer) +would stop the runner's heartbeat, so GitHub would drop the job as lost, and the harnesses' wall-clock timeouts +would fail runs that were only waiting. The runner comes back RESUME_AFTER seconds after the last of these signs. +Every change is written to ~/.actions-runners/horizon/pause.log. """ import json import os @@ -27,7 +30,7 @@ SERVICE, SLICE = "horizon-runner.service", "horizon-ci.slice" MANUAL, STATE, LOG = DIR / "pause", DIR / "paused", DIR / "pause.log" GAME = re.compile(r"(SteamLaunch|gamescope|wine64-preloader|wineserver|wine-preloader|\.exe(\s|$)|proton\b" r"|net\.minecraft\.client\.main\.Main|minecraft-launcher|prismlauncher)", re.I) -GPU_BUSY = int(os.environ.get("GPU_BUSY", 30)) +GPU_BUSY = int(os.environ.get("GPU_BUSY", 50)) INTERVAL = int(os.environ.get("INTERVAL", 15)) RESUME_AFTER = int(os.environ.get("RESUME_AFTER", 300)) THROTTLE_CPUS = int(os.environ.get("THROTTLE_CPUS", 2)) @@ -50,6 +53,16 @@ def ci_process(pid): return True # gone +def at_the_screen(): + """A graphical login on seat0, as ~/fleet/power status counts it (a locked screen counts too).""" + sessions = subprocess.run(["loginctl", "list-sessions", "--no-legend"], capture_output=True, text=True).stdout + for line in sessions.splitlines(): + fields = line.split() + if "seat0" in fields: + return f"session {fields[0]} of {fields[2] if len(fields) > 2 else '?'} at the screen (seat0)" + return None + + def fleet_load_gaming(): status = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "fleet-load" / "status.json" try: @@ -82,7 +95,8 @@ def game_process(): def gpu_users(): """Processes outside the CI slice using the GPU's shader cores, as {pid: SM %}.""" try: - output = subprocess.run(["nvidia-smi", "pmon", "-c", "1", "-s", "u"], capture_output=True, text=True, + # Two samples: the first one nvidia-smi prints has no utilisation yet. + output = subprocess.run(["nvidia-smi", "pmon", "-c", "2", "-s", "u"], capture_output=True, text=True, timeout=20).stdout except (OSError, subprocess.TimeoutExpired): return {} @@ -92,7 +106,7 @@ def gpu_users(): if line.startswith("#") or len(fields) < 4 or not fields[1].isdigit() or not fields[3].isdigit(): continue if not ci_process(int(fields[1])): - users[int(fields[1])] = int(fields[3]) + users[int(fields[1])] = max(users.get(int(fields[1]), 0), int(fields[3])) return users @@ -119,47 +133,60 @@ def throttle(on): systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", "CPUQuota=") +def pause_reason(hot): + """Why the runner may not work now, or None. hot: the GPU users seen busy at the last sample (updated).""" + if MANUAL.exists(): + return f"paused by hand ({MANUAL})" + reason = at_the_screen() or fleet_load_gaming() or game_process() + if reason: + return reason + busy = {pid for pid, sm in gpu_users().items() if sm >= GPU_BUSY} + twice = sorted(busy & set(hot)) + hot.clear() + hot.update(dict.fromkeys(busy)) + if twice: + try: + name = Path(f"/proc/{twice[0]}/comm").read_text().strip() + except OSError: + name = "?" + return f"GPU busy: process {twice[0]} ({name})" + return None + + def main(): - hot = {} # GPU users seen busy at the last sample - last_game = 0.0 - paused = STATE.exists() + hot = {} + # A boot (or a restart of this watcher) counts as a sign: the runner starts RESUME_AFTER seconds later at the + # earliest, which leaves time to log in after switching the machine on. + last_sign = time.monotonic() + paused = None while True: - reason = None - if MANUAL.exists(): - reason = "paused by hand (" + str(MANUAL) + ")" - reason = reason or fleet_load_gaming() or game_process() - if not reason: - busy = {pid for pid, sm in gpu_users().items() if sm >= GPU_BUSY} - twice = busy & set(hot) - hot = dict.fromkeys(busy) - if twice: - pid = min(twice) - try: - name = Path(f"/proc/{pid}/comm").read_text().strip() - except OSError: - name = "?" - reason = f"GPU busy: process {pid} ({name})" + reason = pause_reason(hot) now = time.monotonic() if reason: - last_game = now - if reason and not paused: - paused = True - STATE.write_text(reason + "\n") - throttle(True) # also what other machines' parity workers run here meanwhile - if runner_busy(): - log(f"pause: {reason}; a job is running: throttled to {THROTTLE_CPUS} threads, stops after it") - else: + last_sign = now + active = systemctl("is-active", "--quiet", SERVICE).returncode == 0 + if reason or now - last_sign < RESUME_AFTER: + if paused is not True: + paused = True + STATE.write_text((reason or "starting") + "\n") + throttle(True) + # Parity workers other machines started here (tools/parity_hosts.json): they stop, and those + # machines run the harnesses themselves. + systemctl("stop", "horizon-parity-worker-*.scope") + log(f"pause: {reason or 'waiting ' + str(RESUME_AFTER) + ' s after starting'}" + + (f"; a job is running: throttled to {THROTTLE_CPUS} threads, stops after it" if active else "")) + if active and not runner_busy(): systemctl("stop", SERVICE) - log(f"pause: {reason}; runner stopped") - elif paused and reason is None and now - last_game >= RESUME_AFTER: - paused = False - STATE.unlink(missing_ok=True) - throttle(False) - systemctl("start", SERVICE) - log(f"resume: no game for {RESUME_AFTER} s; runner started") - elif paused and systemctl("is-active", "--quiet", SERVICE).returncode == 0 and not runner_busy(): - systemctl("stop", SERVICE) - log("the job ended: runner stopped until the game ends") + log("runner stopped") + else: + if paused is not False: + paused = False + STATE.unlink(missing_ok=True) + throttle(False) + log(f"resume: nobody at the screen and no game for {RESUME_AFTER} s") + if not active: + systemctl("start", SERVICE) + log("runner started") time.sleep(INTERVAL) diff --git a/tools/parity_hosts.json b/tools/parity_hosts.json index 0aa013a5b..5d561175d 100644 --- a/tools/parity_hosts.json +++ b/tools/parity_hosts.json @@ -11,6 +11,15 @@ "jobs": 14, "exclude": {} }, + "desktop": { + "ssh": "desktop", + "root": ".actions-runners/horizon/parity-host", + "shell": "systemd-run --user --scope --quiet --slice=horizon-ci.slice --unit=horizon-parity-worker-$$ bash -c", + "setup": "test ! -e ~/.actions-runners/horizon/paused && set -a && . ~/.actions-runners/horizon/.env && set +a && export PATH=$(cat ~/.actions-runners/horizon/.path) && eval \"$(vscreen env)\"", + "jobs": 8, + "nice": 19, + "note": "ludodesktop, the maintainer's own machine: only while nobody is at the screen (tools/ci/runner_pause.py: a paused desktop refuses workers and stops the running ones, whose harnesses then run here); 8 jobs at most, in the CI runner's environment and slice" + }, "thinkpad": { "ssh": "thinkpad", "root": "code/horizon-parity", From 401ac1322ffdd9aa72f2fffd927f135176a0ec98 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:43:03 +0100 Subject: [PATCH 46/61] Banners: one loader for saved and generated banners The generated banner reuses save.rs's BannerBlockEntity.loadAdditional (now decode_banner) instead of a copy of it, and the generated block entity clones its NBT only for the kinds that read it. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/save.rs | 23 +++++----- crates/horizon-client/src/structure_blocks.rs | 45 ++++++------------- 2 files changed, 26 insertions(+), 42 deletions(-) diff --git a/crates/horizon-client/src/save.rs b/crates/horizon-client/src/save.rs index da5735c17..69b6a44f5 100644 --- a/crates/horizon-client/src/save.rs +++ b/crates/horizon-client/src/save.rs @@ -567,6 +567,17 @@ fn merge_block_entity(original: &Tag, loaded: &Tag, current: &Tag) -> Tag { } merged } +/// `BannerBlockEntity.loadAdditional` (and the `components` a block entity keeps). +pub fn decode_banner(tag: &Tag) -> crate::containers::Banner { + crate::containers::Banner { + layers: crate::banner::layers_from_nbt(tag.get("patterns")), + name: text(tag.get("CustomName")), + components: match tag.get("components") { + Some(Tag::Compound(c)) => c.clone(), + _ => Compound::new(), + }, + } +} pub fn decode_block_entity(tag: &Tag) -> Option<((i32, i32, i32), crate::containers::BlockEntity)> { use crate::containers::BlockEntity; let int = |k: &str| number(tag.get(k)).map(|v| v as i32); @@ -600,17 +611,7 @@ pub fn decode_block_entity(tag: &Tag) -> Option<((i32, i32, i32), crate::contain return Some((pos, BlockEntity::Lectern(lectern))); } if id == "minecraft:banner" { - return Some(( - pos, - BlockEntity::Banner(crate::containers::Banner { - layers: crate::banner::layers_from_nbt(tag.get("patterns")), - name: text(tag.get("CustomName")), - components: match tag.get("components") { - Some(Tag::Compound(c)) => c.clone(), - _ => Compound::new(), - }, - }), - )); + return Some((pos, BlockEntity::Banner(decode_banner(tag)))); } if id == "minecraft:beacon" { let beacon = crate::containers::Beacon { diff --git a/crates/horizon-client/src/structure_blocks.rs b/crates/horizon-client/src/structure_blocks.rs index da5b75258..ff51116bc 100644 --- a/crates/horizon-client/src/structure_blocks.rs +++ b/crates/horizon-client/src/structure_blocks.rs @@ -128,40 +128,23 @@ fn text(nbt: &Compound, key: &str) -> Option { } } -/// `BannerBlockEntity.loadAdditional`: the `patterns` layers and the custom -/// name a structure template saves with its banner (the End cities' tower -/// tops place black triangles this way). -fn banner_from_nbt(nbt: &Compound) -> crate::containers::Banner { - crate::containers::Banner { - layers: crate::banner::layers_from_nbt(nbt.get(&JavaString::from("patterns"))), - name: text(nbt, "CustomName"), - components: match nbt.get(&JavaString::from("components")) { - Some(Tag::Compound(c)) => c.clone(), - _ => Compound::new(), - }, - } -} - /// The client block entity for one of a chunk's generated block entities, -/// before a container's loot is unpacked. +/// before a container's loot is unpacked: `StructureTemplate.placeInWorld` +/// hands the template's block entity NBT to the level's block entity. fn generated_block_entity(block: &str, nbt: &Compound) -> Option { let mut entity = BlockEntity::for_block(block)?; - let tag = Tag::Compound(nbt.clone()); - // MonsterRoomFeature and the structures' setEntityId. - if let BlockEntity::Spawner(s) = &mut entity { - **s = crate::mobs::spawner::Spawner::load(&tag); - } - if let BlockEntity::TrialSpawner(s) = &mut entity { - **s = crate::mobs::trial_spawner::TrialSpawner::load(&tag); - } - if let BlockEntity::Vault(v) = &mut entity { - **v = crate::vault::Vault::load(&tag); - } - // StructureTemplate.placeInWorld hands the template's block entity NBT to - // the level, so `BannerBlockEntity.loadAdditional` keeps the banner's - // patterns (the End cities' black triangles) and its name. - if let BlockEntity::Banner(b) = &mut entity { - *b = banner_from_nbt(nbt); + match &mut entity { + // MonsterRoomFeature and the structures' setEntityId. + BlockEntity::Spawner(s) => { + **s = crate::mobs::spawner::Spawner::load(&Tag::Compound(nbt.clone())); + } + BlockEntity::TrialSpawner(s) => { + **s = crate::mobs::trial_spawner::TrialSpawner::load(&Tag::Compound(nbt.clone())); + } + BlockEntity::Vault(v) => **v = crate::vault::Vault::load(&Tag::Compound(nbt.clone())), + // BannerBlockEntity.loadAdditional: the End cities' black triangles. + BlockEntity::Banner(b) => *b = crate::save::decode_banner(&Tag::Compound(nbt.clone())), + _ => {} } Some(entity) } From 072f5ce2ee412f735e9b2ec70c29126695dc15b0 Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:43:07 +0000 Subject: [PATCH 47/61] Fix custom item names above targeted item frames Show the framed item's custom name through the existing world name-label renderer, matching vanilla's HUD and crosshair conditions. Extend the LAN frame check for normal and glow frames. Co-Authored-By: Codex --- crates/horizon-client/src/game/names.rs | 36 +++++++++++++++++++++++++ tools/lan_e2e.py | 17 ++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/crates/horizon-client/src/game/names.rs b/crates/horizon-client/src/game/names.rs index 7655fb489..d5e8446c3 100644 --- a/crates/horizon-client/src/game/names.rs +++ b/crates/horizon-client/src/game/names.rs @@ -240,6 +240,42 @@ impl Game<'_> { discrete: false, }); } + // ItemFrameRenderer.shouldShowName/getNameTag: only the targeted + // frame's custom-named item, with the HUD visible (also glow frames). + if !self.ui.hide_gui + && let Some((i, distance)) = self.frame_hit() + && self.before_block(distance) + && self.mob_hit().is_none_or(|(_, d)| d > distance) + && let Some(name) = self.item_frames[i] + .item + .as_ref() + .and_then(|item| item.custom_name()) + { + let frame = &self.item_frames[i]; + let (lo, hi) = frame.bounds(); + let center = glam::DVec3::from_array(std::array::from_fn(|a| (lo[a] + hi[a]) / 2.0)); + if center.distance_squared(camera) < 64.0 * 64.0 { + // NAME_TAG is at the entity height (0.5), then the name + // submitter lifts it 0.5. ItemFrameRenderer offsets rendering + // 0.3 toward its facing and -0.25 vertically. + let step = frame.step(); + tags.push(NameTag { + at: center + + glam::DVec3::new( + f64::from(step[0] as f32 * 0.3), + 0.75, + f64::from(step[2] as f32 * 0.3), + ), + lines: vec![(Text::literal(name).resolve_legacy(), 0.0)], + light: self.world.light( + center.x.floor() as i32, + center.y.floor() as i32, + center.z.floor() as i32, + ), + discrete: false, + }); + } + } tags } /// `SubmitNodeCollection.submitNameTag`: a discrete name is drawn once, diff --git a/tools/lan_e2e.py b/tools/lan_e2e.py index 0ec7218d8..df78489f9 100755 --- a/tools/lan_e2e.py +++ b/tools/lan_e2e.py @@ -3762,17 +3762,21 @@ def check_frames(host, guest, out): # Within a survival player's 3 blocks of reach: a frame on a wall, an armor stand beside it. player('command', command=f'fill {x - 2} {y} {z + 1} {x + 2} {y + 2} {z + 2} minecraft:air') player('command', command=f'fill {x - 2} {y} {z + 3} {x + 2} {y + 2} {z + 3} minecraft:stone') - player('command', command=f'summon minecraft:item_frame {x + 0.5} {y + 1.5} {z + 2.97} ' + frame_kind = 'item_frame' if label == 'host' else 'glow_item_frame' + player('command', command=f'summon minecraft:{frame_kind} {x + 0.5} {y + 1.5} {z + 2.97} ' f'{{block_pos:[I;{x},{y + 1},{z + 2}],Facing:2b}}') player('command', command=f'summon minecraft:armor_stand {x - 1.5} {y} {z + 1.5}') player('command', command='clear @s') - player('command', command='item replace entity @s weapon.mainhand with minecraft:diamond') + frame_name = f'{label} framed diamond' + player('command', command='item replace entity @s weapon.mainhand with ' + f"minecraft:diamond[minecraft:custom_name='{frame_name}']") player.wait(30) try: frame = [o for o in player('status')['objects'] if o['kind'].endswith('item_frame') and o['position'] == [x, y + 1, z + 2]] assert frame, f'the {label} sees no item frame at {[x, y + 1, z + 2]}' player.raw('look-at', f'{x + 0.5:.2f}', f'{y + 1.5:.2f}', f'{z + 2.95:.2f}') + assert frame_name not in (player('status').get('name_tags') or []), 'empty frame name visible' player.raw('mouse', 'right', 'click') player.wait(10) player.raw('mouse', 'right', 'click') @@ -3780,6 +3784,15 @@ def check_frames(host, guest, out): frame = [o for o in player('status')['objects'] if o['id'] == frame[0]['id']][0] assert frame['item'] == 'minecraft:diamond', f'the {label}\'s frame holds {frame["item"]}' assert frame['rotation'] == 1, f'the {label}\'s frame turned to {frame["rotation"]}' + assert frame_name in (player('status').get('name_tags') or []), \ + f"the {label}'s targeted frame has no item name" + player.shot(out, f'{label}-framed-item-name') + player.raw('key', 'f1', 'tap') + player.wait(2) + assert frame_name not in (player('status').get('name_tags') or []), 'F1 left the frame name visible' + player.raw('key', 'f1', 'tap') + player.raw('look-at', f'{x + 2:.2f}', f'{y + 1.5:.2f}', f'{z + 2.95:.2f}') + assert frame_name not in (player('status').get('name_tags') or []), 'untargeted frame name visible' theirs = [o for o in other('status')['objects'] if o['kind'].endswith('item_frame') and o['position'] == [x, y + 1, z + 2]] assert theirs and theirs[0]['item'] == 'minecraft:diamond' and theirs[0]['rotation'] == 1, \ From b7629cf6560ac31557d751638910acdc8bf7e0f8 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:46:36 +0100 Subject: [PATCH 48/61] Enchantments: say which callers have no language full_name reads the process's language, which the server thread shares with its window; only a process with no language loaded (a dedicated server, the probes) keeps the English names. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/enchant.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/horizon-client/src/enchant.rs b/crates/horizon-client/src/enchant.rs index f5ddc84c2..2f418ae82 100644 --- a/crates/horizon-client/src/enchant.rs +++ b/crates/horizon-client/src/enchant.rs @@ -337,8 +337,8 @@ impl Enchantments { let language = crate::lang::current(); self.full_name_in(language.as_deref(), id, level) } - /// `full_name` against `language`; `None` keeps the English names loaded - /// above, which is what the server thread and the probes see. + /// `full_name` against `language`; `None` (no language loaded: a + /// dedicated server, the probes) keeps the English names loaded above. pub fn full_name_in( &self, language: Option<&crate::lang::Language>, From 4be53ee28a64f126a1ab08b929eae354cba138c4 Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:58:00 +0000 Subject: [PATCH 49/61] Reduce resource preparation filesystem work Share the asset index and verification state, stage checked imports while hashing, preserve indexed replacements, and clone independent resource copies where supported. Co-Authored-By: Codex (GPT-6.1) --- Cargo.lock | 2 + crates/horizon-prepare/Cargo.toml | 4 + crates/horizon-prepare/src/artifacts.rs | 78 +++-- crates/horizon-prepare/src/files.rs | 180 ++++++++++++ crates/horizon-prepare/src/lib.rs | 105 +++++-- crates/horizon-prepare/src/sounds.rs | 362 +++++++++++++++++++----- docs/prepare-performance.md | 44 +++ launcher/Cargo.lock | 2 + launcher/src/backend.rs | 2 +- 9 files changed, 637 insertions(+), 142 deletions(-) create mode 100644 crates/horizon-prepare/src/files.rs create mode 100644 docs/prepare-performance.md diff --git a/Cargo.lock b/Cargo.lock index 4d749add6..bb4e72446 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1408,6 +1408,8 @@ dependencies = [ name = "horizon-prepare" version = "0.1.0" dependencies = [ + "crc32fast", + "libc", "regex", "sha1", "sha2", diff --git a/crates/horizon-prepare/Cargo.toml b/crates/horizon-prepare/Cargo.toml index 3d62202bd..9bcf5464d 100644 --- a/crates/horizon-prepare/Cargo.toml +++ b/crates/horizon-prepare/Cargo.toml @@ -6,11 +6,15 @@ license = "MIT" description = "Prepares Horizon's resources from the player's own Minecraft: imports and checks the pinned jars, runs the vanilla exporters, writes the resource folder" [dependencies] +crc32fast = "1" regex = { version = "1", default-features = false, features = ["std", "unicode-perl"] } sha1 = "0.10" sha2 = "0.10" zip = { version = "2", default-features = false, features = ["deflate"] } +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +libc = "0.2" + [build-dependencies] sha2 = "0.10" diff --git a/crates/horizon-prepare/src/artifacts.rs b/crates/horizon-prepare/src/artifacts.rs index 805dd80d4..a3bf945d5 100644 --- a/crates/horizon-prepare/src/artifacts.rs +++ b/crates/horizon-prepare/src/artifacts.rs @@ -143,31 +143,17 @@ pub(crate) fn import_copy( algorithm: &str, hint: &str, ) -> Result { - fs::create_dir_all(destination.parent().unwrap()).map_err(|e| e.to_string())?; - if destination.exists() && digest(destination, algorithm)? == checksum { - return Ok(destination.to_path_buf()); - } - if !source.is_file() { - return Err(format!("Missing {}. {hint}", source.display()) - .trim() - .to_string()); - } - let actual = digest(source, algorithm)?; - if actual != checksum { - return Err(format!( - "{} is not the pinned file: expected {algorithm} {checksum}, got {actual}", - source.display() - )); - } - let temporary = destination.with_file_name(format!( - ".{}.partial-{}", - destination.file_name().unwrap().to_string_lossy(), - std::process::id() - )); - let copied = fs::copy(source, &temporary).and_then(|_| fs::rename(&temporary, destination)); - let _ = fs::remove_file(&temporary); - copied.map_err(|e| format!("{}: {e}", destination.display()))?; - Ok(destination.to_path_buf()) + crate::files::Import::stage(source, destination, checksum, algorithm, false) + .map_err(|error| { + if !source.is_file() { + format!("Missing {}. {hint}", source.display()) + .trim() + .to_string() + } else { + error + } + })? + .commit() } /// The pinned version JSON from the launcher's versions/ directory. The launcher's copy wins @@ -653,9 +639,17 @@ pub struct Artifacts { /// Reuses checked imports from this root or another installed version beside it. pub fn reuse_imported(root: &Path, log: Log) -> Result { + Ok(reuse_verified(root, log)?.is_some()) +} + +/// Keep this preparation's verified artifacts and client libraries for the exporters. +pub(crate) fn reuse_verified( + root: &Path, + log: Log, +) -> Result)>, String> { let root = fs::canonicalize(root).map_err(|e| format!("{}: {e}", root.display()))?; let lock = Lock::read(&root)?; - let checked = |directory: PathBuf| -> Result, String> { + let checked = |directory: PathBuf| -> Result<(Artifacts, Vec), String> { let candidate = Lock { json: lock.json.clone(), version: lock.version.clone(), @@ -664,17 +658,9 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { }; let found = verify(&candidate)?; let client = client_libraries(&candidate, &found.manifest)?; - let mut files = vec![ - candidate.directory.join("manifest.json"), - candidate.directory.join("jars/client.jar"), - candidate.directory.join("jars/server.jar"), - found.jars["server"].clone(), - ]; - files.extend(found.libraries); - files.extend(client); - Ok(files) + Ok((found, client)) }; - if checked(lock.directory.clone()).is_ok() { + if let Ok((found, client_libraries)) = checked(lock.directory.clone()) { if [ "prepared.json", "reports/client-inventory.json", @@ -683,8 +669,6 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { .iter() .any(|path| !lock.directory.join(path).is_file()) { - let found = verify(&lock)?; - let client_libraries = client_libraries(&lock, &found.manifest)?; write_import_metadata( &root, &lock, @@ -696,10 +680,10 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { )?; } log(&format!("Using imported Minecraft {}", lock.version)); - return Ok(true); + return Ok(Some((found, client_libraries))); } let Some(versions) = root.parent() else { - return Ok(false); + return Ok(None); }; // Sibling discovery is intended for the launcher's versions/ folder. let mut siblings: Vec<_> = fs::read_dir(versions) @@ -711,9 +695,17 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { siblings.sort(); for sibling in siblings { let source = sibling.join(".local/minecraft").join(&lock.version); - let Ok(files) = checked(source.clone()) else { + let Ok((found, client)) = checked(source.clone()) else { continue; }; + let mut files = vec![ + source.join("manifest.json"), + source.join("jars/client.jar"), + source.join("jars/server.jar"), + found.jars["server"].clone(), + ]; + files.extend(found.libraries); + files.extend(client); for file in files { let relative = file.strip_prefix(&source).map_err(|e| e.to_string())?; import_copy( @@ -740,9 +732,9 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { lock.version, sibling.display() )); - return Ok(true); + return Ok(Some((found, client_libraries))); } - Ok(false) + Ok(None) } pub fn verify(lock: &Lock) -> Result { diff --git a/crates/horizon-prepare/src/files.rs b/crates/horizon-prepare/src/files.rs new file mode 100644 index 000000000..484c7bf7b --- /dev/null +++ b/crates/horizon-prepare/src/files.rs @@ -0,0 +1,180 @@ +//! Independent copies, using filesystem cloning when available. Never hard links: exporters +//! rewrite prepared resources in place. Checked imports publish only after verifying the copy. + +use crate::{digest, hex}; +use sha2::Digest; +use std::fs; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +fn clone_file(from: &Path, to: &Path) -> io::Result<()> { + #[cfg(target_os = "linux")] + { + use std::os::fd::AsRawFd; + let source = fs::File::open(from)?; + let target = fs::File::create(to)?; + // SAFETY: FICLONE takes a source fd, both descriptors stay open for the call. + if unsafe { libc::ioctl(target.as_raw_fd(), libc::FICLONE, source.as_raw_fd()) } == 0 { + return Ok(()); + } + Err(io::Error::last_os_error()) + } + #[cfg(target_os = "macos")] + { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + let path = |p: &Path| { + CString::new(p.as_os_str().as_bytes()) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) + }; + let (source, target) = (path(from)?, path(to)?); + // SAFETY: both paths are valid NUL-terminated strings for the duration of the call. + if unsafe { libc::clonefile(source.as_ptr(), target.as_ptr(), 0) } == 0 { + return Ok(()); + } + Err(io::Error::last_os_error()) + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = (from, to); + Err(io::ErrorKind::Unsupported.into()) + } +} + +/// Copies an independent file, attempting a reflink first. The destination may be replaced. +pub fn clone_or_copy(from: &Path, to: &Path) -> io::Result<()> { + clone_file(from, to).or_else(|_| fs::copy(from, to).map(|_| ())) +} + +fn copy_hash(from: &Path, to: &Path) -> io::Result { + let mut source = fs::File::open(from)?; + let mut target = fs::File::create(to)?; + let mut hash = D::new(); + let mut buffer = vec![0; 64 * 1024]; + loop { + let n = source.read(&mut buffer)?; + if n == 0 { + return Ok(hex(&hash.finalize())); + } + target.write_all(&buffer[..n])?; + hash.update(&buffer[..n]); + } +} + +/// A verified replacement staged beside its destination. Dropping an unfinished set cleans +/// every temporary file, so a corrupt late panorama face cannot replace earlier placeholders. +pub(crate) struct Import { + temporary: Option, + destination: PathBuf, +} + +impl Import { + pub(crate) fn stage( + source: &Path, + destination: &Path, + checksum: &str, + algorithm: &str, + verified: bool, + ) -> Result { + let mut import = Self { + temporary: None, + destination: destination.into(), + }; + if verified || (destination.is_file() && digest(destination, algorithm)? == checksum) { + return Ok(import); + } + fs::create_dir_all(destination.parent().unwrap()).map_err(|e| e.to_string())?; + static COPIES: AtomicUsize = AtomicUsize::new(0); + let temporary = destination.with_file_name(format!( + ".{}.partial-{}-{}", + destination.file_name().unwrap().to_string_lossy(), + std::process::id(), + COPIES.fetch_add(1, Ordering::Relaxed) + )); + import.temporary = Some(temporary.clone()); + let actual = if clone_file(source, &temporary).is_ok() { + digest(&temporary, algorithm)? + } else { + match algorithm { + "sha1" => copy_hash::(source, &temporary), + _ => copy_hash::(source, &temporary), + } + .map_err(|e| format!("{}: {e}", source.display()))? + }; + if actual != checksum { + return Err(format!( + "{} is not the pinned file: expected {algorithm} {checksum}, got {actual}", + source.display() + )); + } + Ok(import) + } + + pub(crate) fn commit(mut self) -> Result { + if let Some(temporary) = &self.temporary { + fs::rename(temporary, &self.destination) + .map_err(|e| format!("{}: {e}", self.destination.display()))?; + } + self.temporary = None; + Ok(self.destination.clone()) + } +} + +impl Drop for Import { + fn drop(&mut self) { + if let Some(temporary) = &self.temporary { + let _ = fs::remove_file(temporary); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn copies_are_independent_and_checked_before_publication() { + let dir = std::env::temp_dir().join(format!("horizon-file-copy-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let source = dir.join("source"); + let target = dir.join("target"); + let data = vec![42; 200_000]; // More than one streaming buffer. + fs::write(&source, &data).unwrap(); + for algorithm in ["sha1", "sha256"] { + let checksum = digest(&source, algorithm).unwrap(); + Import::stage(&source, &target, &checksum, algorithm, false) + .unwrap() + .commit() + .unwrap(); + assert_eq!(digest(&target, algorithm).unwrap(), checksum); + fs::write(&target, b"old destination").unwrap(); + assert_eq!(fs::read(&source).unwrap(), data); + assert!(Import::stage(&source, &target, "bad hash", algorithm, false).is_err()); + assert_eq!(fs::read(&target).unwrap(), b"old destination"); + // Explicitly exercise ordinary streaming copies even on a reflink filesystem. + let streamed = if algorithm == "sha1" { + copy_hash::(&source, &target).unwrap() + } else { + copy_hash::(&source, &target).unwrap() + }; + assert_eq!(streamed, checksum); + assert_eq!(fs::read(&target).unwrap(), data); + } + clone_or_copy(&source, &target).unwrap(); + fs::write(&target, b"rewrite").unwrap(); + assert_eq!(fs::read(&source).unwrap(), data); + let staged = Import::stage( + &source, + &target, + &digest(&source, "sha1").unwrap(), + "sha1", + false, + ) + .unwrap(); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 3); + drop(staged); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 2); + fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/horizon-prepare/src/lib.rs b/crates/horizon-prepare/src/lib.rs index c7e443c7d..daee30444 100644 --- a/crates/horizon-prepare/src/lib.rs +++ b/crates/horizon-prepare/src/lib.rs @@ -11,12 +11,14 @@ //! run, keyed by the sha256 of their inputs and outputs recorded in the manifest, as before. pub mod artifacts; +mod files; mod inventory; pub mod java; pub mod pyjson; mod sounds; mod survival; +pub use files::clone_or_copy; pub use sounds::{fonts, languages, packs, panorama, sounds, BUILT_IN_PACKS}; use pyjson::Value; @@ -410,16 +412,28 @@ pub fn prepare(options: &Options, log: Log) -> Result { &artifacts::minecraft_dir(options.minecraft_dir.as_deref()).join("runtime"), )); let jdk = java::find(lock.java_major, &homes)?; - if options.import && !artifacts::reuse_imported(root, log)? { - artifacts::import( - root, - &lock, - options.minecraft_dir.as_deref(), - options.server_jar.as_deref(), - log, - )?; - } - let found = artifacts::verify(&lock)?; + let reused = if options.import { + artifacts::reuse_verified(root, log)? + } else { + None + }; + let (found, client_libraries) = match reused { + Some(checked) => checked, + None => { + if options.import { + artifacts::import( + root, + &lock, + options.minecraft_dir.as_deref(), + options.server_jar.as_deref(), + log, + )?; + } + let found = artifacts::verify(&lock)?; + let libraries = artifacts::client_libraries(&lock, &found.manifest)?; + (found, libraries) + } + }; let local = root.join(".local"); let resources = local.join("horizon-client").join(&lock.version); mkdir(&resources)?; @@ -445,7 +459,6 @@ pub fn prepare(options: &Options, log: Log) -> Result { } else { Value::Object(Vec::new()) }; - let client_libraries = artifacts::client_libraries(&lock, &found.manifest)?; let run = Run { root: root.clone(), local, @@ -679,25 +692,9 @@ fn steps( // and every data-pack resource (worldgen, recipes, loot tables, tags), and // the vanilla pack's icon (`pack.png`) for the pack screen. (run.log)("Copying the client's assets and data"); - let mut count = 0usize; - let mut client = open_zip(&run.client)?; - let version = pyjson::loads(&zip_text(&mut client, "version.json")?)?; - for name in zip_names(&client) { - let wanted = name.starts_with("assets/") || name.starts_with("data/") || name == "pack.png"; - if !wanted || name.ends_with('/') { - continue; - } - let relative = member_path(&name, "Invalid resource path")?; - let data = zip_bytes(&mut client, &name)?; - write_if_changed(&root.join(relative), &data)?; - count += 1; - } - - // The JAR holds only gray 1x1 panorama placeholders and an empty Unifont - // include, and none of the built-in packs. Import the indexed ones afterwards, including on reruns that just - // extracted the JAR again. They are optional when preparing from - // already-imported JARs. - sounds::optional_assets(repo, minecraft, run.log); + let mut indexed = sounds::Assets::load(repo, minecraft, run.log); + let (version, count) = extract_client_resources(&run.client, root, indexed.as_mut().ok())?; + sounds::optional_imports(indexed.as_mut().map_err(|e| e.as_str()), run.log); // One file each, exported again when the exporter or the jar it reads changed. let single = |what: &str, @@ -931,6 +928,54 @@ fn steps( ) } +/// Check existing JAR assets by their ZIP size/CRC without inflating every member again. +/// Indexed panorama/font/language replacements have stronger SHA-1 checks and take precedence. +fn extract_client_resources( + jar: &Path, + root: &Path, + mut indexed: Option<&mut sounds::Assets>, +) -> Result<(Value, usize), String> { + let mut client = open_zip(jar)?; + let version = pyjson::loads(&zip_text(&mut client, "version.json")?)?; + let mut count = 0; + for name in zip_names(&client) { + if !(name.starts_with("assets/") || name.starts_with("data/") || name == "pack.png") + || name.ends_with('/') + { + continue; + } + let relative = member_path(&name, "Invalid resource path")?; + count += 1; + if let Some(assets) = &mut indexed { + if assets.imported(&name)? { + continue; + } + } + let target = root.join(relative); + let member = client.by_name(&name).map_err(|e| e.to_string())?; + let (size, crc) = (member.size(), member.crc32()); + drop(member); + if target.is_file() && fs::metadata(&target).map_err(|e| e.to_string())?.len() == size { + let mut file = fs::File::open(&target).map_err(|e| e.to_string())?; + let mut hash = crc32fast::Hasher::new(); + let mut buffer = [0; 16 * 1024]; + loop { + let n = file.read(&mut buffer).map_err(|e| e.to_string())?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + if hash.finalize() == crc { + continue; + } + } + mkdir(target.parent().unwrap())?; + write(&target, &zip_bytes(&mut client, &name)?)?; + } + Ok((version, count)) +} + fn datagen_reports_missing(root: &Path) -> bool { ["items.json", "registries.json", "commands.json"] .iter() diff --git a/crates/horizon-prepare/src/sounds.rs b/crates/horizon-prepare/src/sounds.rs index 15d4a7a08..fd171ec12 100644 --- a/crates/horizon-prepare/src/sounds.rs +++ b/crates/horizon-prepare/src/sounds.rs @@ -5,7 +5,8 @@ use crate::artifacts::{self, Lock}; use crate::pyjson::Value; -use crate::{digest, read_json, Log}; +use crate::{digest, Log}; +use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; @@ -18,8 +19,9 @@ fn launcher_assets( lock: &Lock, manifest: &Value, minecraft: &Path, + cached_index: &Path, log: Log, -) -> Result<(PathBuf, PathBuf), String> { +) -> Result<(PathBuf, PathBuf, Vec), String> { let assets = minecraft.join("assets"); let info = manifest .get("assetIndex") @@ -33,7 +35,10 @@ fn launcher_assets( if artifacts::safe_relative(id).is_none_or(|p| p.components().count() != 1) { return Err(format!("Unsafe asset index id {id:?}")); } - let index = assets.join("indexes").join(format!("{id}.json")); + let mut index = assets.join("indexes").join(format!("{id}.json")); + if !index.is_file() && cached_index.is_file() { + index = cached_index.to_path_buf(); + } if !index.is_file() { let version = manifest.get("id").and_then(Value::as_str).unwrap_or(""); return Err(format!( @@ -42,7 +47,8 @@ fn launcher_assets( artifacts::launcher_hint(version) )); } - let checksum = digest(&index, "sha1")?; + let bytes = fs::read(&index).map_err(|e| format!("{}: {e}", index.display()))?; + let checksum = crate::hex(&::digest(&bytes)); if checksum != sha1 && !lock.asset_index_accepted(&checksum) { log(&format!( "{} is an asset index Horizon does not know yet (sha1 {checksum}); Mojang may have \ @@ -50,7 +56,7 @@ fn launcher_assets( index.display() )); } - Ok((assets, index)) + Ok((assets, index, bytes)) } /// Where the launcher's asset store keeps the object with this SHA-1. @@ -61,24 +67,6 @@ fn object_path(assets: &Path, checksum: &str) -> Result { Ok(assets.join("objects").join(&checksum[..2]).join(checksum)) } -/// One verified object from the launcher's asset store (tools/lab.py `launcher_object`). -fn launcher_object(assets: &Path, checksum: &str) -> Result { - let path = object_path(assets, checksum)?; - if !path.is_file() { - return Err(format!( - "Missing asset object {}. Play the pinned version once in your launcher.", - path.display() - )); - } - if digest(&path, "sha1")? != checksum { - return Err(format!( - "Corrupt asset object {}; let the launcher repair it", - path.display() - )); - } - Ok(path) -} - /// Whether the game plays this asset: sounds.json and everything under sounds/. fn is_sound(name: &str) -> bool { name == "minecraft/sounds.json" || name.starts_with("minecraft/sounds/") @@ -127,26 +115,56 @@ fn is_language(name: &str) -> bool { } /// Full preparation can run from cached JARs without an official launcher install. -pub(crate) fn optional_assets(root: &Path, minecraft: Option<&Path>, log: Log) { - if let Err(error) = panorama(root, minecraft, log) { - log(&format!( - "Title panorama not imported: {error}; run horizon-prepare panorama later" - )); - } - if let Err(error) = fonts(root, minecraft, log) { - log(&format!( - "Unifont glyphs not imported: {error}; run horizon-prepare fonts later" - )); - } - if let Err(error) = packs(root, minecraft, log) { - log(&format!( - "Programmer Art and High Contrast not imported: {error}; run horizon-prepare packs later" - )); - } - if let Err(error) = languages(root, minecraft, log) { - log(&format!( - "Languages not imported: {error}; run horizon-prepare languages later" - )); +#[cfg(test)] +fn optional_assets(root: &Path, minecraft: Option<&Path>, log: Log) { + optional_imports( + Assets::load(root, minecraft, log) + .as_mut() + .map_err(|e| e.as_str()), + log, + ); +} + +pub(crate) fn optional_imports(mut assets: Result<&mut Assets, &str>, log: Log) { + for (wanted, label, required, failure, command) in [ + ( + is_panorama as fn(&str) -> bool, + "Panorama", + PANORAMA.as_slice(), + "Title panorama", + "panorama", + ), + ( + is_font, + "Font", + UNIFONT.as_slice(), + "Unifont glyphs", + "fonts", + ), + ( + is_built_in_pack, + "Built-in pack", + BUILT_IN_PACKS.as_slice(), + "Programmer Art and High Contrast", + "packs", + ), + ( + is_language, + "Language", + LANGUAGES.as_slice(), + "Languages", + "languages", + ), + ] { + let result = match &mut assets { + Ok(assets) => assets.copy(wanted, label, required, log).map(|_| ()), + Err(error) => Err(error.to_string()), + }; + if let Err(error) = result { + log(&format!( + "{failure} not imported: {error}; run horizon-prepare {command} later" + )); + } } } @@ -207,29 +225,98 @@ fn copy_assets( label: &str, required: &[&str], ) -> Result { - let lock = Lock::read(root)?; - let minecraft = artifacts::minecraft_dir(minecraft); - let manifest = artifacts::import_manifest(&lock, &minecraft)?; - let (store, index) = launcher_assets(&lock, &manifest, &minecraft, log)?; - let id = manifest - .get("id") - .and_then(Value::as_str) - .ok_or("The version JSON has no id")?; - let target = root.join(".local/horizon-client").join(id); - fs::create_dir_all(&target).map_err(|e| format!("{}: {e}", target.display()))?; - fs::copy(&index, target.join("asset-index.json")) - .map_err(|e| format!("{}: {e}", target.join("asset-index.json").display()))?; - let objects = read_json(&index)?; - let objects = objects - .get("objects") - .and_then(Value::as_object) - .ok_or_else(|| format!("{} has no objects", index.display()))?; + Assets::load(root, minecraft, log)?.copy(wanted_asset, label, required, log) +} + +/// A preparation's index and verified destinations. Never persisted: a later invocation +/// checks the bytes again, including when repairing a corrupt prepared file. +pub(crate) struct Assets { + store: PathBuf, + target: PathBuf, + objects: Vec<(String, Value)>, + verified: HashSet, +} + +impl Assets { + pub(crate) fn load(root: &Path, minecraft: Option<&Path>, log: Log) -> Result { + let lock = Lock::read(root)?; + let minecraft = artifacts::minecraft_dir(minecraft); + let manifest = artifacts::import_manifest(&lock, &minecraft)?; + let target = root.join(".local/horizon-client").join(&lock.version); + let (store, index, bytes) = launcher_assets( + &lock, + &manifest, + &minecraft, + &target.join("asset-index.json"), + log, + )?; + let text = std::str::from_utf8(&bytes).map_err(|e| e.to_string())?; + let objects = crate::pyjson::loads(text)?; + let objects = objects + .get("objects") + .and_then(Value::as_object) + .ok_or_else(|| format!("{} has no objects", index.display()))? + .to_vec(); + fs::create_dir_all(&target).map_err(|e| e.to_string())?; + crate::write_if_changed(&target.join("asset-index.json"), &bytes)?; + Ok(Self { + store, + target, + objects, + verified: HashSet::new(), + }) + } + + /// Preserve indexed replacements instead of overwriting them with JAR placeholders. + pub(crate) fn imported(&mut self, name: &str) -> Result { + let Some(name) = name.strip_prefix("assets/") else { + return Ok(false); + }; + if !(is_panorama(name) || is_font(name) || is_language(name)) { + return Ok(false); + } + let Some((_, info)) = self.objects.iter().rev().find(|(n, _)| n == name) else { + return Ok(false); + }; + let Some(hash) = info.get("hash").and_then(Value::as_str) else { + return Ok(false); + }; + let destination = self.target.join("assets").join(name); + if destination.is_file() && digest(&destination, "sha1")? == hash { + self.verified.insert(destination); + return Ok(true); + } + Ok(false) + } + + fn copy( + &mut self, + wanted_asset: fn(&str) -> bool, + label: &str, + required: &[&str], + log: Log, + ) -> Result { + copy_indexed(self, wanted_asset, label, required, log) + } +} + +fn copy_indexed( + assets: &mut Assets, + wanted_asset: fn(&str) -> bool, + label: &str, + required: &[&str], + log: Log, +) -> Result { + let objects = &assets.objects; // A repeated name keeps its first place and its last value, as a Python dict does. let mut wanted: Vec<(&str, &Value)> = Vec::new(); + let mut positions = HashMap::new(); for (name, info) in objects.iter().filter(|(name, _)| wanted_asset(name)) { - match wanted.iter_mut().find(|(n, _)| n == name) { - Some(entry) => entry.1 = info, - None => wanted.push((name, info)), + if let Some(&position) = positions.get(name.as_str()) { + wanted[position] = (name.as_str(), info); + } else { + positions.insert(name.as_str(), wanted.len()); + wanted.push((name, info)); } } for name in required { @@ -251,21 +338,45 @@ fn copy_assets( .ok_or_else(|| format!("{name} has no size in the asset index"))?; let relative = artifacts::safe_relative(name) .ok_or_else(|| format!("Unsafe asset name {name:?} in the asset index"))?; - let source = launcher_object(&store, hash)?; - copies.push((source, target.join("assets").join(relative), hash)); + let source = object_path(&assets.store, hash)?; + let destination = assets.target.join("assets").join(relative); + let import = crate::files::Import::stage( + &source, + &destination, + hash, + "sha1", + assets.verified.contains(&destination), + ) + .map_err(|error| { + if !source.is_file() { + format!( + "Missing asset object {}. Play the pinned version once in your launcher.", + source.display() + ) + } else if error.contains(" is not the pinned file:") { + format!( + "Corrupt asset object {}; let the launcher repair it: {error}", + source.display() + ) + } else { + error + } + })?; + copies.push((import, destination)); total += size; } // Check the entire set before replacing any placeholders: a missing late face // must not leave a mixture of real textures and gray faces in either entry point. - for (source, destination, hash) in copies { - artifacts::import_copy(&source, &destination, hash, "sha1", "")?; + for (import, destination) in copies { + import.commit()?; + assets.verified.insert(destination); } log(&format!( "{label} assets ready: {} files, {:.1} MB", wanted.len(), total as f64 / 1e6 )); - Ok(target) + Ok(assets.target.clone()) } #[cfg(test)] @@ -680,6 +791,7 @@ mod tests { "minecraft/sounds.json is missing from the asset index" ); fs::remove_file(&index).unwrap(); + fs::remove_file(f.root.join(".local/horizon-client/9.9/asset-index.json")).unwrap(); let error = f.run().unwrap_err(); assert!( error.starts_with("Missing ") && error.contains("play 9.9 once"), @@ -687,6 +799,120 @@ mod tests { ); } + #[test] + fn late_corruption_discards_all_staged_replacements() { + let f = Fixture::new("late-corruption"); + let target = f.root.join(".local/horizon-client/9.9/assets"); + for name in PANORAMA { + let path = target.join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, b"placeholder").unwrap(); + } + fs::write(f.object(FIRST_FACE + 6), b"bad overlay").unwrap(); + assert!(panorama(&f.root, Some(&f.minecraft), &|_| {}).is_err()); + for name in PANORAMA { + assert_eq!(fs::read(target.join(name)).unwrap(), b"placeholder"); + } + assert_eq!( + fs::read_dir(target.join("minecraft/textures/gui/title/background")) + .unwrap() + .count(), + 7 + ); + } + + #[test] + fn one_index_snapshot_serves_every_import_and_reruns_repair_corruption() { + let f = Fixture::with_extras("snapshot", &PANORAMA, &[("pack.mcmeta", b"{}")]); + let mut assets = Assets::load(&f.root, Some(&f.minecraft), &|_| {}).unwrap(); + fs::remove_file(f.minecraft.join("assets/indexes/7.json")).unwrap(); + let lines = std::cell::RefCell::new(Vec::new()); + optional_imports(Ok(&mut assets), &|line| { + lines.borrow_mut().push(line.to_string()) + }); + assert_eq!(lines.borrow().len(), 4); + assert!(lines + .borrow() + .iter() + .all(|line| line.contains("assets ready"))); + assets + .copy(is_sound, "Sound", &["minecraft/sounds.json"], &|_| {}) + .unwrap(); + let sound = assets + .target + .join("assets/minecraft/sounds/ambient/cave1.ogg"); + let time = fs::metadata(&sound).unwrap().modified().unwrap(); + fs::remove_file(f.object(1)).unwrap(); + f.run().unwrap(); // A verified destination works even without the source object/index. + assert_eq!(fs::metadata(&sound).unwrap().modified().unwrap(), time); + fs::write(&sound, b"corrupt").unwrap(); + assert!(f.run().is_err()); // Verification never carries over to another invocation. + fs::write(f.object(1), &f.objects[1].1).unwrap(); + f.run().unwrap(); + assert_eq!(fs::read(sound).unwrap(), f.objects[1].1); + } + + #[test] + fn jar_reruns_preserve_indexed_assets_and_repair_missing_or_corrupt_members() { + use std::io::Write; + let f = Fixture::with_extras( + "extract", + &PANORAMA, + &[ + ("pack.mcmeta", b"{}"), + ("minecraft/lang/en_us.json", b"{\"indexed\":true}"), + ], + ); + optional_assets(&f.root, Some(&f.minecraft), &|_| {}); + let target = f.root.join(".local/horizon-client/9.9"); + let jar = f.root.join("client.jar"); + let mut zip = zip::ZipWriter::new(fs::File::create(&jar).unwrap()); + for (name, data) in [ + ("version.json", "{\"world_version\": 1}"), + ( + "assets/minecraft/textures/gui/title/background/panorama_0.png", + "gray", + ), + ("assets/minecraft/font/include/unifont.json", "{}"), + ("assets/minecraft/lang/en_us.json", "{}"), + ("data/minecraft/test.json", "data"), + ("pack.png", "icon"), + ] { + zip.start_file(name, zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(data.as_bytes()).unwrap(); + } + zip.finish().unwrap(); + let face = target.join("assets").join(PANORAMA[0]); + let time = fs::metadata(&face).unwrap().modified().unwrap(); + // Preparing from cached JARs must also preserve full imports without the launcher. + fs::remove_dir_all(f.minecraft.join("assets")).unwrap(); + for _ in 0..2 { + let mut assets = Assets::load(&f.root, Some(&f.minecraft), &|_| {}).unwrap(); + let (_, count) = + crate::extract_client_resources(&jar, &target, Some(&mut assets)).unwrap(); + assert_eq!(count, 5); + optional_imports(Ok(&mut assets), &|_| {}); + assert_eq!(fs::read(&face).unwrap(), PANORAMA[0].as_bytes()); + assert_eq!(fs::metadata(&face).unwrap().modified().unwrap(), time); + assert_eq!( + fs::read(target.join("assets").join(UNIFONT[0])).unwrap(), + f.objects[4].1 + ); + assert_eq!( + fs::read(target.join("assets/minecraft/lang/en_us.json")).unwrap(), + b"{\"indexed\":true}" + ); + assert_eq!( + fs::read(target.join("data/minecraft/test.json")).unwrap(), + b"data" + ); + assert_eq!(fs::read(target.join("pack.png")).unwrap(), b"icon"); + fs::write(target.join("data/minecraft/test.json"), b"oops").unwrap(); + fs::remove_file(target.join("pack.png")).unwrap(); + } + } + #[test] fn object_paths_come_from_real_hashes() { let store = Path::new("assets"); diff --git a/docs/prepare-performance.md b/docs/prepare-performance.md new file mode 100644 index 000000000..3a3587111 --- /dev/null +++ b/docs/prepare-performance.md @@ -0,0 +1,44 @@ +# Resource preparation and filesystem work + +The reported preparation timings put sounds at 20–29 seconds with little CPU use, versus +0.9–1.4 seconds for fresh client asset extraction. Prefer fewer file opens, reads and writes +over parallelizing decompression. + +Preparation now shares one parsed asset index between panorama, font, pack and language imports. +Its destination verification results live only for that preparation. A subsequent invocation +checks bytes again, so it can repair corrupt resources. Existing valid destination files do not +require the original asset objects to remain installed. A saved asset index also lets preparation +preserve full imports when only cached Minecraft JARs remain. + +New imports are staged beside their destination. On Linux, FICLONE is attempted first; other +filesystems fall back to copying while hashing. Cloned copies are hashed before publication. +macOS attempts clonefile with the same checked streaming fallback. A whole asset group is staged and +verified before any replacement is published; corrupt late panorama faces leave placeholders +alone. Temporary files are cleaned up on errors. This is not a transaction for rename failures +or process termination during publication: a later run repairs incomplete groups. + +Unchanged JAR resources are checked by the ZIP member's size and CRC, avoiding repeated +decompression. Valid indexed replacements take precedence over JAR placeholders. Reusing checked +Minecraft imports also passes the verified artifact/library results directly to the exporters. + +## An immutable store across releases + +The official launcher's SHA-1 object store already provides an immutable source for indexed +assets. Independent filesystem clones let installed Horizon releases share physical blocks +where supported, without sharing writable inodes. Ordinary copies remain the fallback. + +A separate Horizon store could additionally share complete preparations on filesystems without +cloning. Its key would need the pinned client/server hashes, the preparer's output version (or +the legacy source/exporter fingerprint), and the actual asset-index digest: Mojang republishes +translations without changing the Minecraft release. Two preparations with different optional +asset availability must not claim the same complete snapshot. + +Before adopting that store, separate immutable runtime resources from mutable exporter scratch, +classes and logs. Publish snapshots through a temporary directory and atomic rename, give each +release an explicit reference, and define locking, corruption repair, migration and garbage +collection. Exporters must never receive a shared writable directory or hard-linked files. +The existing launcher compatibility check should choose reuse; the store should only supply its +bytes. Keep this separate from the open work on output-version compatibility (#1946/#1947). + +For now, retain independent release directories and clone their resource files. This reduces +copying where the filesystem supports it without introducing a second ownership/cleanup scheme. diff --git a/launcher/Cargo.lock b/launcher/Cargo.lock index 1e8a5aee2..ee0cb882a 100644 --- a/launcher/Cargo.lock +++ b/launcher/Cargo.lock @@ -2726,6 +2726,8 @@ dependencies = [ name = "horizon-prepare" version = "0.1.0" dependencies = [ + "crc32fast", + "libc", "regex", "sha1", "sha2 0.10.9", diff --git a/launcher/src/backend.rs b/launcher/src/backend.rs index 7e5c4babb..dadef73d3 100644 --- a/launcher/src/backend.rs +++ b/launcher/src/backend.rs @@ -719,7 +719,7 @@ fn copy_dir(from: &Path, to: &Path) -> std::io::Result<()> { if entry.file_type()?.is_dir() { copy_dir(&entry.path(), &target)?; } else { - fs::copy(entry.path(), target)?; + horizon_prepare::clone_or_copy(&entry.path(), &target)?; } } Ok(()) From 5e9d45bc9afe0bf91feb22e49a5232cb00f38287 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:11:39 +0100 Subject: [PATCH 50/61] /pregen: off unless the server turns it on Vanilla has no /pregen, so with it always registered /help and the command tree differed from vanilla's (the command_exec harness failed on /help). Horizon's own commands are now added to the game's tree only when the owner asks: `horizon-pregen=true` in server.properties (`horizon-pregen = true` under [properties] in server.toml) for a dedicated server, `--pregen` for the game (single player, LAN). Off, the trees are vanilla's own, unchanged. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/command/tests.rs | 18 ++++++++ crates/horizon-client/src/command/vanilla.rs | 45 ++++++++++++++------ crates/horizon-client/src/game.rs | 4 +- crates/horizon-client/src/game/dedicated.rs | 8 ++++ docs/hosting.md | 5 ++- docs/pregeneration.md | 6 +++ 6 files changed, 70 insertions(+), 16 deletions(-) diff --git a/crates/horizon-client/src/command/tests.rs b/crates/horizon-client/src/command/tests.rs index 2bdea1aba..d61fd8ae1 100644 --- a/crates/horizon-client/src/command/tests.rs +++ b/crates/horizon-client/src/command/tests.rs @@ -592,3 +592,21 @@ fn mod_commands_join_the_tree_for_every_player() { // The game's own tree is left as it was. assert!(vanilla::game_dispatcher().find("where").is_none()); } + +#[test] +fn horizon_commands_are_off_unless_the_server_turns_them_on() { + // By default the game runs vanilla's tree: /help lists no /pregen. + assert!(vanilla::game_dispatcher().find("pregen").is_none()); + for dev in [false, true] { + let vanilla_tree = vanilla::game_tree(dev, false); + assert!(vanilla_tree.find("pregen").is_none()); + let d = vanilla::game_tree(dev, true); + let parse = d.parse("pregen start 100 circle", ®()); + assert_eq!(parse.executable().unwrap().1, "pregen.start"); + // Vanilla's commands are the same with or without it. + assert_eq!( + d.nodes[0].children.len(), + vanilla_tree.nodes[0].children.len() + 1 + ); + } +} diff --git a/crates/horizon-client/src/command/vanilla.rs b/crates/horizon-client/src/command/vanilla.rs index cba5a5d5b..2ce892077 100644 --- a/crates/horizon-client/src/command/vanilla.rs +++ b/crates/horizon-client/src/command/vanilla.rs @@ -538,20 +538,39 @@ fn apply_vanilla_order(d: &mut Dispatcher) { } /// The tree the game runs: a release integrated server's, or with /// `HORIZON_DEV_COMMANDS` set the development one -/// (SharedConstants.DEBUG_DEV_COMMANDS), plus Horizon's own commands. +/// (SharedConstants.DEBUG_DEV_COMMANDS), plus Horizon's own commands when +/// the server turned them on (`set_horizon_commands`). pub fn game_dispatcher() -> &'static Dispatcher { - static DEV: std::sync::OnceLock = std::sync::OnceLock::new(); - static RELEASE: std::sync::OnceLock = std::sync::OnceLock::new(); - let (tree, dev) = if std::env::var_os("HORIZON_DEV_COMMANDS").is_some() { - (&DEV, true) - } else { - (&RELEASE, false) - }; - tree.get_or_init(|| { - let mut d = build_with(dev); - horizon_commands(&mut d, 0); - d - }) + game_tree( + std::env::var_os("HORIZON_DEV_COMMANDS").is_some(), + HORIZON_COMMANDS.load(std::sync::atomic::Ordering::Relaxed), + ) +} +/// Whether this process's server has Horizon's own commands (`/pregen`). Off +/// by default, so `/help` and the command tree are vanilla's; a dedicated +/// server turns them on with `horizon-pregen=true` in server.properties, the +/// game with `--pregen` (docs/pregeneration.md). +static HORIZON_COMMANDS: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); +/// Turns Horizon's own commands on or off for the trees `game_dispatcher` +/// returns from now on. +pub fn set_horizon_commands(on: bool) { + HORIZON_COMMANDS.store(on, std::sync::atomic::Ordering::Relaxed); +} +/// The server.properties key that turns `/pregen` on (Horizon's own; vanilla +/// ignores keys it doesn't know). +pub const PREGEN_PROPERTY: &str = "horizon-pregen"; +pub(crate) fn game_tree(dev: bool, horizon: bool) -> &'static Dispatcher { + static HORIZON: [std::sync::OnceLock; 2] = + [const { std::sync::OnceLock::new() }; 2]; + match (dev, horizon) { + (true, false) => dispatcher(), + (false, false) => release_dispatcher(), + (dev, true) => HORIZON[usize::from(dev)].get_or_init(|| { + let mut d = build_with(dev); + horizon_commands(&mut d, 0); + d + }), + } } /// Commands vanilla doesn't have, after its own: `/pregen` /// (docs/pregeneration.md). diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index fa00e3804..08e1ff68b 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -360,7 +360,7 @@ impl Options { match arg.as_str() { "--help" | "-h" => { println!( - "Horizon — Minecraft Java Edition {} in Rust\n\n --resources PATH Prepared resources (horizon-prepare)\n --render-distance N Chunks, 2..64 (default 12)\n --world NAME World folder under saves/ (default \"New World\")\n --seed N World seed for new worlds (default 0)\n --width N, --height N Window size (default 1280 x 720)\n --creative Creative mode (default survival)\n --test-terrain Simple test terrain instead of vanilla generation\n --frames N Exit after N frames (testing)\n --world-frames N Exit after N frames drawn in the world (testing)\n --smoke-report PATH Write a JSON report on exit\n --benchmark SCENARIO Run a benchmark (standard, village, forest, ocean, mountain), write its report to benchmarks/ and quit\n --control Accept horizon-ctl commands on a local socket\n --control-socket PATH Control socket path (default $XDG_RUNTIME_DIR/horizon-control.sock)\n --server HOST[:PORT] Join a multiplayer server (offline mode) at start\n --name NAME Player name for multiplayer (default: generated once, kept in horizon-player.txt)\n --session-file PATH Signed-in account from the launcher (read once, then deleted)\n --dev-mod PATH A Horizon mod (.hmod or folder) to develop: always on, reloaded when rebuilt\n --in-frame Run single player's world in the frame, not on its server thread\n\nControls: mouse look, WASD move, Space jump, Left Shift sneak, Left Ctrl sprint,\nleft click break, right click place/use, 1-9/scroll hotbar, E inventory,\nQ drop, F3 debug, Esc pause.", + "Horizon — Minecraft Java Edition {} in Rust\n\n --resources PATH Prepared resources (horizon-prepare)\n --render-distance N Chunks, 2..64 (default 12)\n --world NAME World folder under saves/ (default \"New World\")\n --seed N World seed for new worlds (default 0)\n --width N, --height N Window size (default 1280 x 720)\n --creative Creative mode (default survival)\n --test-terrain Simple test terrain instead of vanilla generation\n --frames N Exit after N frames (testing)\n --world-frames N Exit after N frames drawn in the world (testing)\n --smoke-report PATH Write a JSON report on exit\n --benchmark SCENARIO Run a benchmark (standard, village, forest, ocean, mountain), write its report to benchmarks/ and quit\n --control Accept horizon-ctl commands on a local socket\n --control-socket PATH Control socket path (default $XDG_RUNTIME_DIR/horizon-control.sock)\n --server HOST[:PORT] Join a multiplayer server (offline mode) at start\n --name NAME Player name for multiplayer (default: generated once, kept in horizon-player.txt)\n --session-file PATH Signed-in account from the launcher (read once, then deleted)\n --dev-mod PATH A Horizon mod (.hmod or folder) to develop: always on, reloaded when rebuilt\n --in-frame Run single player's world in the frame, not on its server thread\n --pregen Add Horizon's /pregen command (docs/pregeneration.md)\n\nControls: mouse look, WASD move, Space jump, Left Shift sneak, Left Ctrl sprint,\nleft click break, right click place/use, 1-9/scroll hotbar, E inventory,\nQ drop, F3 debug, Esc pause.", horizon_core::VANILLA_VERSION ); return Ok(None); @@ -377,6 +377,8 @@ impl Options { "--control" => o.control = Some(horizon_client::control::default_path()), "--server-thread" => o.server_thread = true, "--in-frame" | "--no-server-thread" => o.server_thread = false, + // Horizon's own commands (/pregen), off by default as on vanilla. + "--pregen" => horizon_client::command::vanilla::set_horizon_commands(true), _ => { let value = args .next() diff --git a/crates/horizon-client/src/game/dedicated.rs b/crates/horizon-client/src/game/dedicated.rs index edea0e800..fa5365195 100644 --- a/crates/horizon-client/src/game/dedicated.rs +++ b/crates/horizon-client/src/game/dedicated.rs @@ -155,6 +155,14 @@ pub(super) fn preflight(options: &mut Options) -> Result { for w in &props.warnings { log("WARN", w); } + // Horizon's own commands are the owner's choice, off as on vanilla. + if props + .raw + .get(horizon_client::command::vanilla::PREGEN_PROPERTY) + .is_some_and(|v| v.trim().eq_ignore_ascii_case("true")) + { + horizon_client::command::vanilla::set_horizon_commands(true); + } props.store(&settings)?; let eula_file = PathBuf::from("eula.txt"); let agreed = match eula::agreed(&eula_file) { diff --git a/docs/hosting.md b/docs/hosting.md index 4c4c954bc..a9624dc2c 100644 --- a/docs/hosting.md +++ b/docs/hosting.md @@ -218,8 +218,9 @@ horizon-server rollback back to the release before While the server runs, `op`, `whitelist` and `cmd` go to the game's console, so they act exactly as vanilla's console commands do. While it's stopped, they edit vanilla's files directly. Any console command works through `cmd`: `horizon-server cmd ban Griefer`, `horizon-server cmd say Restarting in 5 minutes`. -To generate the world around spawn ahead of the players, Horizon has its own `pregen` command: -`horizon-server cmd pregen start 2500` ([pregeneration](pregeneration.md)). +To generate the world around spawn ahead of the players, Horizon has its own `pregen` command, off until +`server.toml` has `horizon-pregen = true` under `[properties]`: `horizon-server cmd pregen start 2500` +([pregeneration](pregeneration.md)). `config set` changes `server.toml` without opening it: `horizon-server config set world.difficulty=hard server.max_players=10`. Each key is a section and a name from the file (`properties.spawn-protection=0` for any diff --git a/docs/pregeneration.md b/docs/pregeneration.md index a8ca228b3..706527661 100644 --- a/docs/pregeneration.md +++ b/docs/pregeneration.md @@ -5,6 +5,12 @@ Chunky does on a Java server. It is Horizon's own command (vanilla has none), av with cheats on and to operators (level 2) on a server, from the chat or the server console (`horizon-server cmd pregen start 2500`). +It is off until you turn it on, so `/help` and the command tree stay vanilla's: + +- a server: `horizon-pregen = true` under `[properties]` in `server.toml` (or `horizon-pregen=true` in a + vanilla folder's `server.properties`), then restart it; +- single player and LAN: start the game with `--pregen`. + ```text /pregen start [square|circle] [ ] radius in blocks; the centre defaults to where you stand (the world spawn from the console) From 0baf6cbd62e5dc053732354b5ebd356aae3ffd83 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:27:22 +0100 Subject: [PATCH 51/61] CI: the desktop runner works while its screen is locked or untouched for 15 min MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ludovic's rule (2026-10-26): runner_pause.py pauses at once on keyboard, mouse or gamepad input (read from /dev/input, only whether bytes came), on an unlocked session touched in the last 15 min, at 90 °C, on a game or the GPU. Fixes the service start: the link loop's last test failed the script under set -e. Co-Authored-By: Claude Opus 5.5 --- docs/parity-workflow.md | 2 +- docs/release-flow.md | 32 +++++++------- tests/test_runner_pause.py | 13 +++++- tools/ci/linux-runner.sh | 12 +++--- tools/ci/runner_pause.py | 88 ++++++++++++++++++++++++++++++++------ tools/parity_hosts.json | 2 +- 6 files changed, 113 insertions(+), 36 deletions(-) diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index 80426fa59..7eecdd8bb 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -38,7 +38,7 @@ The game logs into `HORIZON_HOME` and falls back to the player's own game folder ### Other machines -Harnesses can also run on the Mac mini, the ThinkPad and the desktop (`tools/parity_hosts.json` describes them; `desktop` uses its CI runner's environment and vanilla data, and refuses work, or stops it, while someone is at it: docs/release-flow.md, "The desktop's runner"; its `default` list is empty, so pass `--hosts macmini,thinkpad` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 3 and only those under 800 MB. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. +Harnesses can also run on the Mac mini, the ThinkPad and the desktop (`tools/parity_hosts.json` describes them; `desktop` uses its CI runner's environment and vanilla data, and refuses work, or stops it, while someone uses it: docs/release-flow.md, "The desktop's runner"; its `default` list is empty, so pass `--hosts macmini,thinkpad` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 8. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. Harnesses a host can't run are listed in the file: `local_only` (`save_roundtrip` runs the game on a virtual screen) and each host's `exclude`. The Mac excludes nothing: vanilla's `Math.sin`, `pow`, `log` and the like don't round the same on an Apple silicon Mac as on Linux (HotSpot uses fdlibm there, Intel's stubs on x86-64), and Rust's float functions round like neither (they call the platform libm), so code that mirrors a vanilla `Math` call uses `horizon_core::jmath`, which computes what HotSpot computes on the host architecture. The ThinkPad helps too, gently: Linux, 2 harnesses at `nice 19`, since it hosts the Claude account pool; it finds ALSA through `~/code/horizon-parity/alsa-dev` like the desktop. diff --git a/docs/release-flow.md b/docs/release-flow.md index e72bebc73..d9d1801aa 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -68,21 +68,23 @@ release's macOS build back. Each keeps its own checkout, `target/` and Rust inst time, under a lock) and the parity passes and slots. **The desktop's runner.** `ludodesktop-horizon` (labels `self-hosted`, `Linux`, `X64`, `ludodesktop`, and -`horizon-game` once a full trial has passed there) runs on the maintainer's Linux desktop, only while nobody is -at it. `tools/ci/linux-runner.sh install`, run there as him, registers it and installs two systemd user services: -`horizon-runner` (the runner, in `~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O -weight 20 and memory capped at two thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the -machine froze under ~7 cold cargo builds on 2026-10-14; its own Rust install; the JDK, sccache and the build -libraries from nixpkgs; its own `vscreen` so no game it starts reaches his screen) and `horizon-runner-pause` -(`tools/ci/runner_pause.py`), the only thing that starts the runner. The watcher stops it while a graphical -session is open on the machine, locked or not (`~/fleet/power status` says "IN USE"), while a game runs (Steam, -gamescope, Wine, Proton or Minecraft processes, fleet-load's own detection, or a process outside the CI slice -using the GPU at 50 % or more), or while `~/.actions-runners/horizon/pause` exists, and starts it again 5 minutes -after the last such sign. A job already under way is throttled to 2 threads at the lowest weight and the runner -stops after it: freezing the job would make GitHub drop it as lost and the harnesses time out. Old pull requests' -`runner-env.sh` know only the Mac: the runner's `.env` and `.path` give their jobs Python, the JDK and the -vanilla data (`HORIZON_SHARED_SOURCE`) all the same. The nightly audit stays on the Mac mini's runners, so its -comparison with the previous night is always made on one platform. +`horizon-game` once a full trial has passed there) runs on the maintainer's Linux desktop while nobody uses it: its +screen locked, or untouched for 15 minutes (his rule, 2026-10-26). `tools/ci/linux-runner.sh install`, run there as +him, registers it and installs two systemd user services: `horizon-runner` (the runner, in +`~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O weight 20 and memory capped at two +thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the machine froze under ~7 cold cargo +builds on 2026-10-14; its own Rust install; the JDK, sccache and the build libraries from nixpkgs; its own +`vscreen` so no game it starts reaches his screen) and `horizon-runner-pause` (`tools/ci/runner_pause.py`), the +only thing that starts the runner. The watcher stops it at once on keyboard, mouse or gamepad input (read from +`/dev/input`: only whether bytes came, never what), while an unlocked session has had input in the last 15 minutes, +while the CPU package is at 90 °C or more, while a game runs (Steam, gamescope, Wine, Proton or Minecraft +processes, fleet-load's own detection, or a process outside the CI slice using the GPU at 50 % or more), or while +`~/.actions-runners/horizon/pause` exists, and starts it again 5 minutes after the last such sign. A job already +under way is throttled to 2 threads at the lowest weight and the runner stops after it: freezing the job would make +GitHub drop it as lost and the harnesses time out. Old pull requests' `runner-env.sh` know only the Mac: the +runner's `.env` and `.path` give their jobs Python, the JDK and the vanilla data (`HORIZON_SHARED_SOURCE`) all the +same. The nightly audit stays on the Mac mini's runners, so its comparison with the previous night is always made +on one platform. ## Merging diff --git a/tests/test_runner_pause.py b/tests/test_runner_pause.py index c6df08b82..38c4affd8 100644 --- a/tests/test_runner_pause.py +++ b/tests/test_runner_pause.py @@ -18,7 +18,7 @@ class Machine: self.throttles = [] # (clock, on) self.calls = [] - def reason(self, hot): + def reason(self, hot, keys): return next((reason for start, end, reason in self.signs if start <= self.clock < end), None) def systemctl(self, *arguments): @@ -77,6 +77,17 @@ class RunnerPauseTests(unittest.TestCase): self.assertLess(stop, 1215) self.assertEqual(machine.throttles[-1], (1290 + 300, False)) + def test_the_desktop_is_free_when_locked_or_untouched_for_15_minutes(self): + cases = [ # (session, seconds since the last input, someone there) + (("3", False), 30, True), (("3", True), 30, True), # any input pauses at once, even on the lock screen + (("3", False), 120, True), (("3", False), 1000, False), # unlocked: free after 15 min untouched + (("3", True), 120, False), (None, 120, False), # locked, or nobody logged in + ] + for session, idle, there in cases: + with self.subTest(session=session, idle=idle), \ + mock.patch.object(runner_pause, "seat_session", lambda: session): + self.assertEqual(runner_pause.at_the_screen(idle) is not None, there) + if __name__ == "__main__": unittest.main() diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index adc2b0130..1bd587e79 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -9,10 +9,10 @@ # # Everything lives in ~/.actions-runners/horizon (never in ~/code): the runner, its own rustup and cargo homes, # the work folder, and copies of this script and runner_pause.py taken at install time (a pull request's checkout -# never runs as the service). horizon-runner-pause.service (runner_pause.py) starts the runner only while nobody is -# at the screen or playing, and stops it otherwise. The runner runs nice, in horizon-ci.slice (low CPU and I/O weight, memory capped), -# on its own virtual screen: vscreen (a headless sway) where it is installed, else Xvfb from nixpkgs with Mesa's -# Vulkan drivers. +# never runs as the service). horizon-runner-pause.service (runner_pause.py) starts the runner only while nobody +# uses the machine (screen locked or untouched for 15 min, no game), and stops it otherwise. The runner runs nice, +# in horizon-ci.slice (low CPU and I/O weight, memory capped), on its own virtual screen: vscreen (a headless sway) +# where it is installed, else Xvfb from nixpkgs with Mesa's Vulkan drivers. # HORIZON_SHARED_SOURCE: the imported vanilla data (default ~/horizon-local/shared-local, else # ~/code/horizon/.local); RUNNER_JOBS: cargo jobs and harnesses at once (default a third of the threads, at most 8: # ludodesktop froze under ~7 cold cargo builds on 2026-10-14, and it is the maintainer's own machine). @@ -40,7 +40,7 @@ parity_host_links() { [ -d "$checkout" ] || return 0 mkdir -p "$shared" for entry in "$checkout"/*; do - [ -L "$entry" ] && ln -sfn "$entry" "$shared/$(basename "$entry")" + if [ -L "$entry" ]; then ln -sfn "$entry" "$shared/$(basename "$entry")"; fi done } @@ -151,7 +151,7 @@ RestartSec=30 EOF cat > "$units/horizon-runner-pause.service" <= GPU_BUSY %, default 50, in two @@ -34,6 +36,9 @@ GPU_BUSY = int(os.environ.get("GPU_BUSY", 50)) INTERVAL = int(os.environ.get("INTERVAL", 15)) RESUME_AFTER = int(os.environ.get("RESUME_AFTER", 300)) THROTTLE_CPUS = int(os.environ.get("THROTTLE_CPUS", 2)) +INPUT_PAUSE = int(os.environ.get("INPUT_PAUSE", 60)) +IDLE_AFTER = int(os.environ.get("IDLE_AFTER", 900)) +TEMP_LIMIT = int(os.environ.get("TEMP_LIMIT", 90)) def log(text): @@ -53,13 +58,71 @@ def ci_process(pid): return True # gone -def at_the_screen(): - """A graphical login on seat0, as ~/fleet/power status counts it (a locked screen counts too).""" +class Input: + """Keyboard, mouse and gamepad activity, from /dev/input (the input group): only whether bytes came, never what.""" + + def __init__(self): + self.devices = {} # path -> file descriptor + self.last = time.monotonic() # a start counts as input: nothing is known yet + + def poll(self): + try: + text = Path("/proc/bus/input/devices").read_text() + except OSError: + text = "" + wanted = {f"/dev/input/{handler}" for block in text.split("\n\n") for line in block.splitlines() + if line.startswith("H: Handlers=") and re.search(r"\b(kbd|mouse\d*|js\d*)\b", line) + for handler in line.split("=", 1)[1].split() if handler.startswith("event")} + for path in set(self.devices) - wanted: + os.close(self.devices.pop(path)) + for path in wanted - set(self.devices): + try: + self.devices[path] = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + except OSError: + pass + for path, descriptor in list(self.devices.items()): + try: + while os.read(descriptor, 4096): + self.last = time.monotonic() + except BlockingIOError: + pass + except OSError: # unplugged + os.close(self.devices.pop(path)) + return time.monotonic() - self.last + + +def seat_session(): + """The graphical session on seat0 and whether its screen is locked, or None.""" sessions = subprocess.run(["loginctl", "list-sessions", "--no-legend"], capture_output=True, text=True).stdout for line in sessions.splitlines(): fields = line.split() if "seat0" in fields: - return f"session {fields[0]} of {fields[2] if len(fields) > 2 else '?'} at the screen (seat0)" + locked = subprocess.run(["loginctl", "show-session", fields[0], "-p", "LockedHint", "--value"], + capture_output=True, text=True).stdout.strip() == "yes" + return fields[0], locked + return None + + +def at_the_screen(input_idle): + """Someone is at the machine: input in the last INPUT_PAUSE seconds, or an unlocked session with input in the last + IDLE_AFTER seconds. A locked screen, or one nobody touched for IDLE_AFTER, leaves the machine to CI.""" + if input_idle < INPUT_PAUSE: + return f"input {input_idle:.0f} s ago" + session = seat_session() + if session and not session[1] and input_idle < IDLE_AFTER: + return f"session {session[0]} unlocked, input {input_idle:.0f} s ago" + return None + + +def too_hot(): + """The CPU package above TEMP_LIMIT °C (coretemp or k10temp).""" + for monitor in Path("/sys/class/hwmon").glob("hwmon*"): + try: + if (monitor / "name").read_text().strip() in ("coretemp", "k10temp"): + celsius = int((monitor / "temp1_input").read_text()) / 1000 + return f"CPU at {celsius:.0f} °C" if celsius >= TEMP_LIMIT else None + except (OSError, ValueError): + pass return None @@ -133,11 +196,12 @@ def throttle(on): systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", "CPUQuota=") -def pause_reason(hot): - """Why the runner may not work now, or None. hot: the GPU users seen busy at the last sample (updated).""" +def pause_reason(hot, keys): + """Why the runner may not work now, or None. hot: the GPU users seen busy at the last sample (updated); + keys: the Input watcher.""" if MANUAL.exists(): return f"paused by hand ({MANUAL})" - reason = at_the_screen() or fleet_load_gaming() or game_process() + reason = at_the_screen(keys.poll()) or too_hot() or fleet_load_gaming() or game_process() if reason: return reason busy = {pid for pid, sm in gpu_users().items() if sm >= GPU_BUSY} @@ -154,13 +218,13 @@ def pause_reason(hot): def main(): - hot = {} + hot, keys = {}, Input() # A boot (or a restart of this watcher) counts as a sign: the runner starts RESUME_AFTER seconds later at the # earliest, which leaves time to log in after switching the machine on. last_sign = time.monotonic() paused = None while True: - reason = pause_reason(hot) + reason = pause_reason(hot, keys) now = time.monotonic() if reason: last_sign = now diff --git a/tools/parity_hosts.json b/tools/parity_hosts.json index 5d561175d..f2ba65c68 100644 --- a/tools/parity_hosts.json +++ b/tools/parity_hosts.json @@ -18,7 +18,7 @@ "setup": "test ! -e ~/.actions-runners/horizon/paused && set -a && . ~/.actions-runners/horizon/.env && set +a && export PATH=$(cat ~/.actions-runners/horizon/.path) && eval \"$(vscreen env)\"", "jobs": 8, "nice": 19, - "note": "ludodesktop, the maintainer's own machine: only while nobody is at the screen (tools/ci/runner_pause.py: a paused desktop refuses workers and stops the running ones, whose harnesses then run here); 8 jobs at most, in the CI runner's environment and slice" + "note": "ludodesktop, the maintainer's own machine: only while nobody uses it (tools/ci/runner_pause.py: screen locked or untouched for 15 min, no game; a paused desktop refuses workers and stops the running ones, whose harnesses then run here); 8 jobs at most, in the CI runner's environment and slice" }, "thinkpad": { "ssh": "thinkpad", From 836c1ee3b999823ab5d0db767ffbb6b71db4ab1e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:27:32 +0100 Subject: [PATCH 52/61] CI parity: pick harnesses with main's selection code, run them at the head The parity job checks out the PR head, so every branch made before #1964 still selected with the old per-crate rule. The step now runs main's tools/parity.py and tools/parity_reach.py with --changed --list, restores the head's files, and runs the named harnesses with the head's parity.py. A branch that edits either file, or a failure of main's selection, falls back to the head's own `--changed` run. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parity.yml | 32 +++++++++++++++++++++++++++++++- docs/release-flow.md | 3 +++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index a18fc5019..be87bc965 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -43,7 +43,37 @@ jobs: env: PARITY_HOSTS: '' BASE_SHA: ${{ inputs.base }} - run: python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400 # world_spawn takes ~1480 s on a busy runner + run: | + # main's selection (tools/parity.py --changed --list with main's parity.py and parity_reach.py), so a + # branch made before a better selection landed gets it too; the head's own runner then runs those + # harnesses. A branch that changes the selection itself selects with its own code. + set -euo pipefail + own() { python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400; } # world_spawn takes ~1480 s on a busy runner + fork_point=$(git merge-base "$BASE_SHA" HEAD) + if ! git rev-parse --verify -q origin/main >/dev/null || ! git diff --quiet "$fork_point" HEAD -- tools/parity.py tools/parity_reach.py; then + own + exit + fi + had_reach=$(git ls-files tools/parity_reach.py) + git show origin/main:tools/parity.py > tools/parity.py + git show origin/main:tools/parity_reach.py > tools/parity_reach.py + status=0 + python3 tools/parity.py --changed "$BASE_SHA" --list > .local/parity-selection.txt || status=$? + git checkout -- tools/parity.py + if [ -n "$had_reach" ]; then git checkout -- tools/parity_reach.py; else rm -f tools/parity_reach.py; fi + cat .local/parity-selection.txt + if [ "$status" != 0 ]; then + echo "main's selection failed: the head's own selection runs" + own + exit + fi + names=$(sed '$d' .local/parity-selection.txt | grep -E '^[A-Za-z0-9_#]+$' || true) + if [ -z "$names" ]; then + echo "Nothing to run." + exit 0 + fi + # shellcheck disable=SC2086 + python3 tools/parity.py $names --timeout 2400 - name: Logs of the failed harnesses if: failure() uses: actions/upload-artifact@v4 diff --git a/docs/release-flow.md b/docs/release-flow.md index 4fb0209d9..3024e0f53 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -43,6 +43,9 @@ copy is: the Mac mini. Its runner runs as the maintainer's account and keeps its modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to `Cargo.lock` or `research-lock.json` affects every harness. Each harness has 25 minutes, the job 45. + The selection runs with main's `tools/parity.py` and `tools/parity_reach.py` (`--changed --list`), so a + branch made before a better selection landed gets it too, and the head's own `parity.py` then runs the named + harnesses. A branch that changes either file selects with its own code. Failed harnesses' logs are uploaded as the `parity-logs` artifact. From 0b970c0c9a0eb259652f5bee756073563e471faf Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:35:22 +0000 Subject: [PATCH 53/61] Cover instant-break sounds alongside replanting Move the instant_digging regression from redundant PR #1966 into the existing stale-dig fix for #1880 and #1908. Co-Authored-By: Codex (gpt-6.1-sol) --- tools/lan_e2e.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tools/lan_e2e.py b/tools/lan_e2e.py index f0333c87b..11354b970 100755 --- a/tools/lan_e2e.py +++ b/tools/lan_e2e.py @@ -250,6 +250,7 @@ address. The script then checks, from both sides, what a guest's game used to ge - falling: an item and a falling block the host summons high up fall smoothly on the guest, a little further every tick between the server's updates (every 20 ticks), as their own client tick moves them (they used to hang, then jump down once a second, #1568). +- instant_digging: an instant survival break leaves no repeated stone hit sounds or stale server dig (#1908). - replant: flowers and crops replanted after a short harvest click stay planted for both players; the server forgets the completed instant break (#1880). @@ -5398,6 +5399,42 @@ CHECKS += ['digging'] CHECK_FNS.update(digging=check_digging) +def check_instant_digging(host, guest, out): + """#1908: an instant survival START has no STOP. ServerPlayerGameMode.tick + forgets a dig once its block is air; it must neither hit air nor break a replacement. + Exercise the in-frame host and network guest (both are clients with --server-thread). + """ + result = {} + for label, player in (('host', host), ('guest', guest)): + x, y, z = (int(v // 1) for v in pos_of(player)) + cell = (x, y + 1, z + 2) + player('command', command='gamemode survival') + player('command', command='clear @s') + host('command', command=f'fill {x - 1} {y} {z + 1} {x + 1} {y + 3} {z + 4} minecraft:air') + host('command', command=f'setblock {x} {y} {z + 2} minecraft:sand') + host('command', command=f'setblock {cell[0]} {cell[1]} {cell[2]} minecraft:dead_bush') + player.wait(10) + before = heard(player, 'minecraft:block.stone.hit') + player.raw('look-at', x + 0.5, y + 1.4, z + 2.5) + player.raw('mouse', 'left', 'click') + player.wait(20) + broken = host.raw('block', *cell).get('block') + hits = heard(player, 'minecraft:block.stone.hit') - before + host('command', command=f'setblock {cell[0]} {cell[1]} {cell[2]} minecraft:dead_bush') + player.wait(20) + replacement = host.raw('block', *cell).get('block') + result[label] = {'broken': broken, 'stone_hits': hits, 'replacement': replacement} + assert broken == 'minecraft:air', f'{label}: the bush did not break: {result}' + assert hits == 0, f'{label}: hitting air after an instant break: {result}' + assert replacement == 'minecraft:dead_bush', f'{label}: the server kept digging: {result}' + host('command', command=f'setblock {cell[0]} {cell[1]} {cell[2]} minecraft:air') + return result + + +CHECKS += ['instant_digging'] +CHECK_FNS.update(instant_digging=check_instant_digging) + + def check_vehicles(host, guest, out): """The host and the guest each (#1551): - in creative, break a block and have it put back: the server's dig ends with the break From 73da2f04ddd56cb2ca1f480f54ab0aad31478223 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:35:28 +0100 Subject: [PATCH 54/61] CI: parity on main, a batch of merges at a time; releases wait for it parity-main runs the vanilla harnesses on every push to main, one run at a time: merges that land during a run wait in one pending run, which checks every merge since main's last green parity-main (tools/parity_main.py base, then parity.py --changed). A failure is bisected over the batch's first-parent merges, running only the failing harnesses, and the breaking merge's pull request gets a comment. Game releases ship only a commit both checks and parity-main passed on (release_train.py; release.yml also starts when parity-main finishes; a forced release doesn't wait). The per-PR parity job now runs only for a PR that changes a harness, an oracle, a probe or the runner, or carries run-parity. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 34 +++++++- .github/workflows/parity-main.yml | 70 ++++++++++++++++ .github/workflows/release.yml | 10 ++- docs/parity-workflow.md | 2 + docs/release-flow.md | 30 ++++++- tests/test_parity_main.py | 43 ++++++++++ tests/test_release_train.py | 21 +++++ tools/parity_main.py | 129 ++++++++++++++++++++++++++++++ tools/release_train.py | 34 ++++++-- 9 files changed, 356 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/parity-main.yml create mode 100644 tests/test_parity_main.py create mode 100644 tools/parity_main.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98d5b9093..eff82fa99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,8 @@ name: checks # check main requires, so it never has a condition: a skipped job counts as passed, and every run of this workflow # must report a real verify (a pull request that changes no Rust finishes it in seconds instead). # parity: the vanilla harnesses a pull request's changes can affect, on the Mac mini's runner, which holds the -# maintainer's own Minecraft files (Horizon never downloads the game); .github/workflows/parity.yml. A fork's pull +# maintainer's own Minecraft files (Horizon never downloads the game); .github/workflows/parity.yml. Only for a pull +# request that changes a harness, oracle, probe or the runner: the rest are checked on main by parity-main. A fork's pull # request runs it only when a maintainer adds the run-parity label (.github/workflows/run-parity.yml), so labels # never start this workflow. See docs/release-flow.md. on: @@ -72,8 +73,37 @@ jobs: run: | cargo +stable test --workspace --lib --bins --tests --features horizon-client/mods,horizon-mods/pack cargo +stable test --workspace --doc --features horizon-client/mods,horizon-mods/pack - parity: + # Most pull requests leave the harnesses to parity-main, which runs them on main a batch of merges at a time + # (.github/workflows/parity-main.yml; releases wait for it). A pull request still runs them itself when it changes + # a harness, an oracle, a probe or the runner, or carries the run-parity label (run-parity.yml, when it is added). + parity-scope: if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + outputs: + run: ${{ steps.scope.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + - id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + LABELS: ${{ join(github.event.pull_request.labels.*.name, ' ') }} + run: | + changed=$(git diff --name-only "$(git merge-base "$BASE_SHA" HEAD)" HEAD) + harness=$(grep -E '^(tools/test_[a-z0-9_]+_parity\.py|tools/(parity[a-z_]*|lab)\.py|tools/parity_hosts\.json|mods/horizon/src/test/|crates/[^/]+/examples/|crates/horizon-client/src/game/[a-z_]*(fixture|harness)\.rs|\.github/workflows/parity[a-z-]*\.yml)' <<< "$changed" || true) + if [ -n "$harness" ] || [[ " $LABELS " == *" run-parity "* ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + echo "The pull request runs its harnesses: ${harness:-label run-parity}" + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "No harness, oracle, probe or runner change: parity-main checks this on main" + fi + parity: + needs: parity-scope + if: needs.parity-scope.outputs.run == 'true' uses: ./.github/workflows/parity.yml with: base: ${{ github.event.pull_request.base.sha }} diff --git a/.github/workflows/parity-main.yml b/.github/workflows/parity-main.yml new file mode 100644 index 000000000..1ddfb24bb --- /dev/null +++ b/.github/workflows/parity-main.yml @@ -0,0 +1,70 @@ +name: parity-main +# The vanilla harnesses on main, a batch of merges at a time (docs/release-flow.md#parity-on-main), on the Mac mini's +# runner. One run at a time: merges that land while a run goes on wait in one pending run (GitHub keeps only the +# newest), which checks every merge since main's last green parity-main (tools/parity_main.py base). On a failure it +# bisects the failing harnesses over those merges and comments on the pull request that broke them. Releases ship +# only a commit whose parity-main passed (tools/release_train.py), so nothing a harness catches reaches players. +on: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read + # The comment on the merge that broke a harness. + pull-requests: write + # tools/parity_main.py base reads earlier runs. + actions: read +concurrency: + group: parity-main + cancel-in-progress: false +env: + # Test runs never reach the telemetry server: no live-count ping, no uploads (docs/telemetry.md). + HORIZON_TELEMETRY_URL: "off" + +jobs: + parity-main: + runs-on: [self-hosted, horizon-game] + timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + # Keeps .local/ (parity state, probe builds, oracle results) and target/ between runs. + clean: false + - uses: dtolnay/rust-toolchain@stable + - name: Tools from the runner + run: bash tools/ci/runner-env.sh + - name: Vanilla data, and resources prepared by this preparer + run: bash tools/ci/vanilla-data.sh + - name: Main's last green parity-main + id: base + env: + GH_TOKEN: ${{ github.token }} + run: echo "sha=$(python3 tools/parity_main.py base)" >> "$GITHUB_OUTPUT" + - name: The harnesses the merges since then can affect + env: + PARITY_HOSTS: '' + BASE_SHA: ${{ steps.base.outputs.sha }} + run: | + echo "Merges since ${BASE_SHA}:" + git log --first-parent --format=' %h %s' "$BASE_SHA..HEAD" + python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400 + - name: The merge that broke them + if: failure() + env: + GH_TOKEN: ${{ github.token }} + PARITY_HOSTS: '' + BASE_SHA: ${{ steps.base.outputs.sha }} + run: | + rm -rf .local/parity-main-logs && cp -R .local/parity-runner/logs .local/parity-main-logs 2>/dev/null || true + python3 tools/parity_main.py bisect "$BASE_SHA" --comment + - name: Logs of the failed harnesses + if: failure() + uses: actions/upload-artifact@v4 + with: + name: parity-main-logs + path: .local/parity-main-logs/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4cba851fa..05f590096 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,7 @@ name: release # Automatic game releases (docs/release-flow.md). After every green `checks` run on main, every 10 minutes (so a -# release waiting for merges to settle goes out without another merge) and by hand, tools/release_train.py decides: +# release waiting for merges to settle goes out without another merge) and by hand, tools/release_train.py decides, +# for a commit whose checks and parity-main (the vanilla harnesses on main) both passed: # a release when 5 pull requests merged since the last one changed what players run and the newest merged 5 minutes # ago, or when the oldest has waited 6 hours, or at once for a pull request labelled release-now. # RELEASES_PAUSED=1 (repository variable) stops them; so does an open issue labelled release-blocker, forced runs too. @@ -18,7 +19,8 @@ name: release # gh workflow run release -f dry_run=false -f force=true # on main: release now, whatever the rules say on: workflow_run: - workflows: [checks] + # A game release ships a commit both passed on (parity-main runs on batches: the newest commit it passed on). + workflows: [checks, parity-main] types: [completed] branches: [main] schedule: @@ -64,8 +66,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASES_PAUSED: ${{ vars.RELEASES_PAUSED }} - # The commit whose checks just passed; a dry run builds the branch it was started on; otherwise - # main's newest commit with green checks. + # The commit whose checks or parity-main just passed; a dry run builds the branch it was started on; + # otherwise main's newest commit with green checks and parity-main. HEAD_SHA: ${{ github.event.workflow_run.head_sha || (steps.mode.outputs.dry_run == 'true' && github.sha) || '' }} FORCE: ${{ inputs.force && 'yes' || '' }} run: | diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index d794b9c71..d64ec3087 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -20,6 +20,8 @@ python3 tools/parity.py --changed origin/main # those a branch's changes ca python3 tools/parity.py --changed-files paths.txt # the same for listed paths, in a copy without Git (tools/pr_ready.py) ``` +On CI, a pull request runs the parity job only when it changes a harness, an oracle, a probe or the runner; everything else is checked on main by `parity-main`, a batch of merges at a time, which bisects a failure to the merge that caused it (`tools/parity_main.py`; [release-flow.md](release-flow.md#parity-on-main)). Game releases wait for it. + Each harness runs unmodified under a tracer that records the files it depended on: its Python modules, its Java oracle and the oracle sources javac pulled in, other repository files it read, and the Rust sources of the probes it ran (from Cargo's `.d` dep-info files). A pass stands until one of those files, the pinned vanilla artifacts, `Cargo.lock` or the Rust/Java toolchains change. State and logs are in `.local/parity-runner/`; each harness's own outputs stay in its `.local//` directory. Vanilla oracle runs are reused as well. `tools/parity_shim.py` stands in for `java` and `javac`: it records each oracle run's outputs and the content of every file under the paths its arguments name, and replays the run while those are unchanged. After a Rust-only change a harness pays only for its Rust side (`block_update`: 53 s to 5 s). Files an oracle writes outside its named paths aren't tracked, so a harness that fails while reusing oracle results is rerun fresh, and stops reusing them if it then passes. `--all` and `--fresh` never reuse them. diff --git a/docs/release-flow.md b/docs/release-flow.md index 4fb0209d9..b551bad21 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -13,7 +13,7 @@ reopened; labels never start it): | Job | Runs on | What | Time | | --- | --- | --- | --- | | `verify` | GitHub `ubuntu-latest` | Public-tree check, Python tests, rustfmt, clippy with warnings denied, Rust tests. Dependencies and the workspace build come from a cache saved by main's runs | ~4 min cold, less warm | -| `parity / parity` | The Mac mini's runner (`ludovics-mac-mini`, label `horizon-game`) | The vanilla harnesses the change can affect (`tools/parity.py --changed `) | seconds when nothing is affected; minutes per affected harness | +| `parity / parity` | The Mac mini's runner (`ludovics-mac-mini`, label `horizon-game`) | Only for a change to a harness, an oracle, a probe or the runner (or the `run-parity` label): the vanilla harnesses the change can affect (`tools/parity.py --changed `). Other pull requests skip it: [parity-main](#parity-on-main) checks them on main | seconds when nothing is affected; minutes per affected harness | Pushes to main run `verify` only; the nightly audit covers main's harnesses. @@ -80,10 +80,34 @@ gh api -X POST repos/ludovic111/horizon/branches/main/protection/required_status A skipped `parity` (a fork's pull request without the label) counts as passed, so requiring it never blocks a fork's pull request; the maintainer decides whether to run it. +## Parity on main + +Most pull requests merge on `verify` alone and leave the vanilla harnesses to `.github/workflows/parity-main.yml`, +which runs them on main a batch of merges at a time, on the Mac mini's runner: + +- It starts on every push to main, one run at a time (`concurrency: parity-main`). Merges that land while a run + goes on wait in one pending run: GitHub keeps only the newest and cancels the others, so the next run takes them + all. +- `tools/parity_main.py base` gives the newest commit main contains whose parity-main passed (else the last + merge's parent), and the run is `tools/parity.py --changed `: the union of the harnesses every merge + since then can affect, once, with every earlier pass reused. +- On a failure, `tools/parity_main.py bisect` checks out the batch's first-parent merges by halves and runs only + the failing harnesses, names the first merge they fail on in the run's summary and comments on its pull request + (`🤖 Agent: parity-main found that this merge breaks ...`). Fix it with a pull request as usual; the next + run checks from the last green commit again. +- Game releases ship only a commit both `checks` and `parity-main` passed on (`tools/release_train.py`, see + below), so a broken merge holds releases instead of reaching players. `release.yml` also starts when a + parity-main run finishes. A forced release (`force=true`) doesn't wait for parity-main. + +A pull request still runs its own parity job when it changes a harness script, an oracle +(`mods/horizon/src/test/`), a probe (`crates/*/examples/`, `game/*_fixture.rs`, `game/*_harness.rs`), the runner +(`tools/parity*.py`, `tools/lab.py`, `tools/parity_hosts.json`, `parity*.yml`), or carries the `run-parity` label +(`run-parity.yml` runs it when the label is added). + ## Game releases -`.github/workflows/release.yml` releases the game by itself. It starts after every green `checks` run of a push -to main, every 10 minutes (so a release waiting for merges to settle goes out without another merge), and by hand. +`.github/workflows/release.yml` releases the game by itself. It starts after every green `checks` or `parity-main` +run of a push to main (it releases a commit only once both passed on it), every 10 minutes (so a release waiting for merges to settle goes out without another merge), and by hand. ```mermaid flowchart LR diff --git a/tests/test_parity_main.py b/tests/test_parity_main.py new file mode 100644 index 000000000..7a72ecae5 --- /dev/null +++ b/tests/test_parity_main.py @@ -0,0 +1,43 @@ +"""tools/parity_main.py: the base of a parity-main run, and the bisect after a failure.""" +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from tools import parity_main + + +class ParityMainTests(unittest.TestCase): + def test_first_bad_finds_the_first_failing_merge(self): + commits = list("abcdefgh") # h failed; the base before a passed + for culprit in commits: + tried = [] + fails = lambda commit: tried.append(commit) or commit >= culprit + self.assertEqual(parity_main.first_bad(commits, fails), culprit) + self.assertLessEqual(len(tried), 3, "log2 of 8 merges") + self.assertEqual(parity_main.first_bad(["a"], lambda commit: self.fail("nothing to try")), "a") + + def test_base_is_the_newest_green_run_head_contains(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.object(parity_main, "ROOT", Path(tmp)): + git = lambda *arguments: parity_main.git(*arguments).stdout.strip() + git("init", "-q") + for n in range(3): + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", str(n)) + head, parent, grandparent = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^1"), git("rev-parse", "HEAD~2") + git("checkout", "-q", "-b", "side", grandparent) + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "side") + side = git("rev-parse", "HEAD") + git("checkout", "-q", head) + self.assertEqual(parity_main.base(runs=[side, grandparent, parent]), grandparent, + "a run on a commit HEAD doesn't contain is skipped") + self.assertEqual(parity_main.base(runs=[]), parent, "no green run yet: the last merge") + self.assertEqual(parity_main.base(runs=[head]), head) + + def test_a_harness_a_commit_does_not_have_cannot_fail_there(self): + with mock.patch.object(parity_main.subprocess, "run") as run: + self.assertFalse(parity_main.run_harnesses(["not_a_harness_yet"], 60)) + run.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_train.py b/tests/test_release_train.py index 56064fa42..ee1fbcae5 100644 --- a/tests/test_release_train.py +++ b/tests/test_release_train.py @@ -2,6 +2,7 @@ import datetime as dt from pathlib import Path import tempfile import unittest +from unittest import mock from tools import release_train as train @@ -147,6 +148,26 @@ class ReleaseTrainTests(unittest.TestCase): self.assertFalse(train.decision("game", [], NOW, new_line="0.3", blockers=[7])[0]) self.assertTrue(train.decision("game", [pr(1)], NOW, blockers=[])[0]) + def test_game_releases_ship_only_what_parity_main_passed(self): + def runs(parity, checks): + def gh_json(*arguments): + table = parity if "parity-main" in arguments else checks + if "--commit" in arguments: + sha = arguments[arguments.index("--commit") + 1] + return [{"status": s, "conclusion": c} for h, s, c in table if h == sha] + return [{"headSha": h, "status": s, "conclusion": c} for h, s, c in table] + return mock.patch.object(train, "gh_json", gh_json) + checks = [("c", "in_progress", ""), ("b", "completed", "success"), ("a", "completed", "success")] + with runs([("c", "queued", ""), ("b", "completed", "cancelled"), ("a", "completed", "success")], checks): + self.assertEqual(train.green_main_head(), "b", "checks alone: the newest green checks") + self.assertEqual(train.green_main_head(parity=True), "a", "a superseded parity-main run counts for nothing") + self.assertTrue(train.passed_on("parity-main", "a")) + self.assertFalse(train.passed_on("parity-main", "b")) + with runs([("b", "completed", "failure"), ("a", "completed", "success")], checks): + self.assertIsNone(train.green_main_head(parity=True), "a failed batch holds releases until fixed") + with runs([("c", "completed", "success")], checks): + self.assertIsNone(train.green_main_head(parity=True), "its checks must pass too") + def test_a_release_is_published_only_after_its_smoke_test_and_with_no_blocker(self): drew = {"error": None, "frames": 12710, "chunks": 81, "meshed_sections": 550, "player": [160.5, 71.0, 156.5]} self.assertIsNone(train.gate(drew, [])) diff --git a/tools/parity_main.py b/tools/parity_main.py new file mode 100644 index 000000000..ff319aed0 --- /dev/null +++ b/tools/parity_main.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""parity-main (.github/workflows/parity-main.yml): the vanilla harnesses run on main, a batch of merges at a time. + + python3 tools/parity_main.py base # the commit this run checks from: main's last green parity-main + python3 tools/parity_main.py bisect BASE # after a failure: the first merge since BASE that broke a harness + +`base` prints the newest commit on main whose parity-main run passed and that HEAD contains, else HEAD's first parent, +so a run tests every merge since the last green one (`tools/parity.py --changed `), however many landed while +the previous run went on. `bisect` reads the harnesses that failed (.local/parity-runner/last-run.json), checks out +the first-parent merges between BASE and HEAD by halves and runs only those harnesses, then names the first merge they +fail on (written to $GITHUB_STEP_SUMMARY and, with --comment, on its pull request). HEAD is checked out again at the end. +""" + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +REPO = os.environ.get("GITHUB_REPOSITORY", "ludovic111/horizon") +LAST_RUN = ROOT / ".local" / "parity-runner" / "last-run.json" + + +def git(*arguments, check=True): + result = subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, text=True) + if check and result.returncode: + sys.exit(f"git {' '.join(arguments)}: {result.stderr.strip()}") + return result + + +def green_runs(): + """Head commits of main's passing parity-main runs, newest first.""" + result = subprocess.run(["gh", "run", "list", "--repo", REPO, "--workflow", "parity-main", "--branch", "main", + "--status", "success", "--limit", "30", "--json", "headSha"], capture_output=True, text=True) + if result.returncode: + print(f"note: no parity-main history ({result.stderr.strip()})", file=sys.stderr) + return [] + return [run["headSha"] for run in json.loads(result.stdout or "[]")] + + +def base(head="HEAD", runs=None): + for sha in green_runs() if runs is None else runs: + if git("merge-base", "--is-ancestor", sha, head, check=False).returncode == 0: + return git("rev-parse", sha).stdout.strip() + return git("rev-parse", f"{head}^1").stdout.strip() + + +def first_bad(commits, fails): + """The first of commits (oldest first, the last known to fail; the one before them known to pass) where fails().""" + good, bad = -1, len(commits) - 1 + while bad - good > 1: + middle = (good + bad) // 2 + if fails(commits[middle]): + bad = middle + else: + good = middle + return commits[bad] + + +def harness_exists(name): + return (ROOT / "tools" / f"test_{name.split('#')[0]}_parity.py").exists() + + +def run_harnesses(names, timeout): + """Whether any of the harnesses fails at the checked-out commit (those it doesn't have yet can't).""" + present = [name for name in names if harness_exists(name)] + if not present: + return False + return subprocess.run([sys.executable, "tools/parity.py", *present, "--timeout", str(timeout)], cwd=ROOT).returncode != 0 + + +def bisect(since, timeout, comment): + failed = [] + if LAST_RUN.exists(): + failed = json.loads(LAST_RUN.read_text(encoding="utf-8")).get("failed") or [] + if not failed: + print("No failed harness recorded: nothing to bisect.") + return 0 + head = git("rev-parse", "HEAD").stdout.strip() + commits = git("rev-list", "--first-parent", "--reverse", f"{since}..{head}").stdout.split() + if not commits: + print("No merge since the base.") + return 0 + + def fails(commit): + print(f"--- {commit[:9]} {git('log', '-1', '--format=%s', commit).stdout.strip()}", flush=True) + git("checkout", "-q", "--detach", commit) + return run_harnesses(failed, timeout) + + try: + culprit = first_bad(commits, fails) if len(commits) > 1 else commits[0] + finally: + git("checkout", "-q", "--detach", head) + subject = git("log", "-1", "--format=%s", culprit).stdout.strip() + number = re.search(r"Merge pull request #(\d+)", subject) + line = (f"parity-main: {', '.join(failed)} first fail at {culprit[:9]} ({subject}), " + f"of {len(commits)} merges since {since[:9]}.") + print(line) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write(f"### {line}\n") + if comment and number: + url = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{REPO}/actions/runs/{os.environ.get('GITHUB_RUN_ID', '')}" + body = (f"🤖 Agent: parity-main found that this merge breaks {', '.join(f'`{name}`' for name in failed)} on main " + f"(bisected over {len(commits)} merges: {url}). Releases wait until parity-main passes again.") + subprocess.run(["gh", "pr", "comment", number.group(1), "--repo", REPO, "--body", body]) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("base") + bisecting = commands.add_parser("bisect") + bisecting.add_argument("since", help="the last commit known to pass (the run's base)") + bisecting.add_argument("--timeout", type=int, default=2400, help="seconds per harness") + bisecting.add_argument("--comment", action="store_true", help="comment on the breaking merge's pull request") + args = parser.parse_args() + if args.command == "base": + print(base()) + return 0 + return bisect(args.since, args.timeout, args.comment) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/release_train.py b/tools/release_train.py index 48d4d46cf..72d81ce57 100644 --- a/tools/release_train.py +++ b/tools/release_train.py @@ -414,13 +414,26 @@ def gather(kind, head, releases=None): return last, prs -def green_main_head(): - """The newest commit on main whose checks passed: what an automatic release builds.""" - runs = gh_json("run", "list", "--repo", REPO, "--workflow", "checks", "--branch", "main", "--event", "push", +def passed_on(workflow, sha): + """Whether the newest finished push run of workflow on main for commit sha passed.""" + runs = gh_json("run", "list", "--repo", REPO, "--workflow", workflow, "--branch", "main", "--event", "push", + "--commit", sha, "--limit", "5", "--json", "status,conclusion") + done = [entry for entry in runs if entry["status"] == "completed" and entry["conclusion"] != "cancelled"] + return bool(done) and done[0]["conclusion"] == "success" + + +def green_main_head(parity=False): + """The newest commit on main whose checks passed: what an automatic release builds. With parity (game + releases), the newest commit parity-main passed on, if its checks passed too: parity-main runs on batches of + merges, so the commits a run skipped are covered by the next one, and a superseded run counts for nothing.""" + workflow = "parity-main" if parity else "checks" + runs = gh_json("run", "list", "--repo", REPO, "--workflow", workflow, "--branch", "main", "--event", "push", "--limit", "10", "--json", "headSha,status,conclusion") for entry in runs: - if entry["status"] == "completed": - return entry["headSha"] if entry["conclusion"] == "success" else None + if entry["status"] == "completed" and entry["conclusion"] != "cancelled": + if entry["conclusion"] != "success": + return None + return entry["headSha"] if not parity or passed_on("checks", entry["headSha"]) else None return None @@ -437,7 +450,7 @@ def main(): parser.add_argument("--kind", choices=["game", "launcher"], required=True) parser.add_argument("--head", help="the commit to release (default: the newest commit on main whose checks passed)") parser.add_argument("--tag", help="notes: the release's tag") - parser.add_argument("--force", action="store_true", help="release even without player-facing changes, before merges settle or while paused") + parser.add_argument("--force", action="store_true", help="release even without player-facing changes, before merges settle, while paused or before parity-main passed") parser.add_argument("--out", type=Path, help="notes: write title.txt and body.md here") parser.add_argument("--now", help="the time to decide at (ISO 8601; default now)") parser.add_argument("--smoke", type=Path, help="gate: the release smoke test's report (smoke-test.json)") @@ -456,9 +469,12 @@ def main(): summary.write(f"### {line}\n") return 1 if problem else 0 - head = args.head or green_main_head() + # Game releases ship only a commit the vanilla harnesses passed on (parity-main, docs/release-flow.md). + parity = args.kind == "game" and args.command == "decide" and not args.force + head = args.head or green_main_head(parity) if not head: - result = {"release": False, "reason": "main's latest checks did not pass"} + result = {"release": False, "reason": "main's latest checks did not pass" + ( + " or parity-main did not pass on them" if parity else "")} print(json.dumps(result, indent=1)) write_outputs({"release": "false", "reason": result["reason"]}) return 0 @@ -482,6 +498,8 @@ def main(): blockers = open_blockers() if args.kind == "game" else [] release, reason = decision(args.kind, prs, now, os.environ.get("RELEASES_PAUSED") == "1", args.force, new_line, blockers) + if release and args.head and parity and not (passed_on("parity-main", head) and passed_on("checks", head)): + release, reason = False, f"waiting for checks and parity-main to pass on {head[:9]}" minor = any("feature" in pr["labels"] for pr in prs if pr["paths"]) tag = next_game_tag(tags, now.date(), line) if args.kind == "game" else next_launcher_tag(tags, minor, line) result = { From 40c08c2d12ebfb8d0a56e9bc2f71b72460cfff0c Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:38:47 +0100 Subject: [PATCH 55/61] streaming: check every fourth cell of the cover test in debug builds the_grid_answers_as_the_scan compared the grid with the scan on every cell of 60 cases and took 20 of the ~30 s of `cargo test -p horizon-client` on the ThinkPad. A debug build now checks every fourth cell of each row, shifted by row and case, so the cases together still meet every cell position relative to the areas; a release build checks every cell. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/streaming.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/horizon-client/src/streaming.rs b/crates/horizon-client/src/streaming.rs index 55a2f0086..13b19fefc 100644 --- a/crates/horizon-client/src/streaming.rs +++ b/crates/horizon-client/src/streaming.rs @@ -2148,8 +2148,13 @@ mod cover_tests { } /// Every question streaming asks gets the same answer with the grid. + /// + /// A debug build checks every fourth cell of each row, shifted by row and case so the cases still cover every + /// cell's position relative to the areas: the full check took 20 of the ~30 s of the crate's tests. + /// `cargo test --release -p horizon-client --lib the_grid_answers_as_the_scan` checks every cell. #[test] fn the_grid_answers_as_the_scan() { + let stride = if cfg!(debug_assertions) { 4 } else { 1 }; let mut seed = 0x9E37_79B9_7F4A_7C15u64; let forced: HashSet<(i32, i32)> = [(40, -40)].into(); for case in 0..60 { @@ -2165,7 +2170,8 @@ mod cover_tests { let (scan, grid) = (ranges(None), ranges(Some(&cover))); let span = [4, 30, 120][case % 3] + 40; for z in -span..=span { - for x in -span..=span { + let first = -span + (z + case as i32).rem_euclid(stride); + for x in (first..=span).step_by(stride as usize) { for m in 0..=ExtraCover::REACH + 2 { assert_eq!( scan.in_extra(x, z, m), From 5034e4557dba8c54ebd406a50ef4449b8d7ec2ea Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:44:22 +0100 Subject: [PATCH 56/61] parity: clear rust--parity outputs after a pass, reuse oracles on CI Disk: clear_large_outputs only looked at .local directories named after the harness, so the 174 harnesses writing to rust--parity kept every dump (86 GB in one runner checkout, 13 GB for state_provider alone). It now clears those too, keeps vanilla outputs only up to 256 MB (larger ones cost more disk than rerunning their oracle), and handles shard names. collision, world and geometry dropped one mkdtemp run directory per run (11 GB): each run now removes the earlier ones. Time: harnesses inherited the runner's stdin, a pipe on GitHub's runner, and parity_shim never reuses an oracle run whose stdin is a pipe, so oracle reuse never worked on CI. Harnesses now get /dev/null. CI: parity.yml falls back to the head's own selection when the names main's `--list` printed don't match its summary count. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parity.yml | 11 ++++++++++- tests/test_parity_runner.py | 14 ++++++++++++++ tools/parity.py | 20 ++++++++++++++------ tools/test_collision_parity.py | 3 +++ tools/test_geometry_parity.py | 3 +++ tools/test_world_parity.py | 3 +++ 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index be87bc965..9dcb21775 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -68,7 +68,16 @@ jobs: exit fi names=$(sed '$d' .local/parity-selection.txt | grep -E '^[A-Za-z0-9_#]+$' || true) - if [ -z "$names" ]; then + # The summary line says how many would run: a count that doesn't match the names read (a format change) + # must not let the job pass untested. + expected=$(tail -1 .local/parity-selection.txt | sed -nE 's/^([0-9]+) of [0-9]+ harnesses would run$/\1/p') + count=$(printf '%s' "$names" | grep -c . || true) + if [ -z "$expected" ] || [ "$expected" != "$count" ]; then + echo "main's selection listed $count names, its summary says '${expected:-?}': the head's own selection runs" + own + exit + fi + if [ "$count" = 0 ]; then echo "Nothing to run." exit 0 fi diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py index d10fde854..2d16e7179 100644 --- a/tests/test_parity_runner.py +++ b/tests/test_parity_runner.py @@ -300,3 +300,17 @@ class LargeOutputTests(unittest.TestCase): self.assertFalse((own / "dump.txt").exists()) # chunk_generation may still be writing there. self.assertTrue((shared / "dump.txt").exists()) + + def test_rust_named_directories_and_big_oracle_outputs_go_too(self): + with tempfile.TemporaryDirectory() as tmp: + local, state = Path(tmp) / ".local", Path(tmp) / "state" + own = local / "rust-state-provider-parity" + own.mkdir(parents=True) + for name, size in (("rust.txt", 16), ("server.txt", 16), ("client.txt", 64), ("report.json", 4)): + (own / name).write_bytes(b"x" * size) + trace = {"commands": [["java", "Oracle", str(own)], ["probe", "fixture", str(own)]]} + with mock.patch.multiple(parity, LOCAL=local, STATE_DIR=state, LARGE_OUTPUT=8, KEPT_ORACLE_OUTPUT=32): + listing = parity.outputs_listing("state_provider#2", fresh=True) + listing.write_text(f"{own / 'server.txt'}\n{own / 'client.txt'}\n") + parity.clear_large_outputs("state_provider#2", trace) + self.assertEqual(sorted(path.name for path in own.iterdir()), ["report.json", "server.txt"]) diff --git a/tools/parity.py b/tools/parity.py index dc5fdb1c9..3f18b922d 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -657,6 +657,9 @@ def prebuild(artifacts, jobs): LARGE_OUTPUT = 64 << 20 +# Vanilla outputs above this are deleted after a pass too: rerunning those few oracles (12 harnesses on the Mac +# mini's runner) costs less than the 26 GB they held in every checkout; smaller ones stay for oracle reuse. +KEPT_ORACLE_OUTPUT = 256 << 20 OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"} @@ -672,11 +675,12 @@ def clear_large_outputs(name, trace): """After a pass, delete the large files a harness regenerates on every run. Harness outputs reach gigabytes and used to stay in every worktree. Vanilla - oracle outputs are kept, since reusing an oracle run needs them in place; - everything else large under the harness's .local directories is rebuilt on - its next run.""" + oracle outputs under KEPT_ORACLE_OUTPUT are kept, since reusing an oracle + run needs them in place; everything else large under the harness's .local + directories (`...` or `rust--parity`) is rebuilt on its next run.""" local = LOCAL.resolve() keep = {os.path.realpath(line) for line in outputs_listing(name).read_text(encoding="utf-8").splitlines() if line} + own = name.split("#")[0].replace("_", "-") directories = set() for command in trace["commands"]: for argument in command[1:]: @@ -688,12 +692,13 @@ def clear_large_outputs(name, trace): # Only the harness's own directory: another harness running beside it may be writing # large files in a directory this one merely read (decorated_region compiles its oracle # into chunk-generation-parity while chunk_generation writes its dumps there). - if top not in OWN_DIRECTORIES and top.startswith(name.replace("_", "-")): + if top not in OWN_DIRECTORIES and (top.startswith(own) or top == f"rust-{own}-parity"): directories.add(local / top) for directory in directories: for file in (directory.rglob("*") if directory.is_dir() else []): try: - if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep: + size = file.stat().st_size if file.is_file() and not file.is_symlink() else 0 + if size >= LARGE_OUTPUT and (str(file) not in keep or size > KEPT_ORACLE_OUTPUT): file.unlink() except OSError: pass @@ -828,8 +833,11 @@ class LocalExecutor: environment.pop("PARITY_SHARD", None) if shard: environment["PARITY_SHARD"] = shard + # stdin from /dev/null: tools/parity_shim.py never reuses an oracle run that reads a pipe, and CI runners + # (and --worker's ssh) give the runner one, which kept oracle reuse off on CI. process = subprocess.Popen([sys.executable, str(Path(__file__)), "--trace", str(trace), str(script)], - cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + cwd=ROOT, env=environment, stdin=subprocess.DEVNULL, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) self.running[name] = {"process": process, "started": time.monotonic(), "log": log, "expected": expected, "slot": slot, "script": script} return True diff --git a/tools/test_collision_parity.py b/tools/test_collision_parity.py index ad99ab960..26f7f25df 100644 --- a/tools/test_collision_parity.py +++ b/tools/test_collision_parity.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path import platform +import shutil import subprocess import tempfile @@ -37,6 +38,8 @@ def main(): if args.prepare: lab.prepare() lock, jars, libraries = test_rust_parity.verify_artifacts() + for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks + shutil.rmtree(earlier, ignore_errors=True) base = Path(tempfile.mkdtemp(prefix="run-", dir=directory)) oracle = lab.ROOT / "mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java" sources = [oracle, Path(__file__).resolve(), lab.ROOT / "tools/test_rust_parity.py", lab.ROOT / "Cargo.toml", lab.ROOT / "Cargo.lock", lab.ROOT / "crates/horizon-core/Cargo.toml", lab.ROOT / "crates/horizon-world/Cargo.toml", *sorted((lab.ROOT / "crates/horizon-core/src").rglob("*.rs")), *sorted((lab.ROOT / "crates/horizon-world/src").rglob("*.rs")), lab.ROOT / "crates/horizon-world/examples/probe/collision_fixture.rs"] diff --git a/tools/test_geometry_parity.py b/tools/test_geometry_parity.py index 54ebc21ae..fa58bf66f 100644 --- a/tools/test_geometry_parity.py +++ b/tools/test_geometry_parity.py @@ -9,6 +9,7 @@ from pathlib import Path import platform import random import struct +import shutil import subprocess import tempfile @@ -273,6 +274,8 @@ def main(): if args.prepare: lab.prepare() lock, jars, libraries = test_rust_parity.verify_artifacts() + for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks + shutil.rmtree(earlier, ignore_errors=True) base = Path(tempfile.mkdtemp(prefix="run-", dir=directory)) commands = list(corpus()) inputs = base / "inputs.txt" diff --git a/tools/test_world_parity.py b/tools/test_world_parity.py index 0b4a3deb7..9344a47b0 100644 --- a/tools/test_world_parity.py +++ b/tools/test_world_parity.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path import platform +import shutil import subprocess import tempfile @@ -35,6 +36,8 @@ def main(): if args.prepare: lab.prepare() lock, jars, libraries = test_rust_parity.verify_artifacts() + for earlier in directory.glob("run-*"): # one run per run used to stay behind: 5 GB here after a few weeks + shutil.rmtree(earlier, ignore_errors=True) base = Path(tempfile.mkdtemp(prefix="run-", dir=directory)) classes = base / "classes" classes.mkdir() From 9db6a0ee48a6862df8e27fe0e5cb587a61c67e0f Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:23:27 +0100 Subject: [PATCH 57/61] CI: parity-main stays on the Mac; heat check on a median, boot as the first sign; nix roots fixed nix-build names a dev output's GC root -dev, so the runner's PKG_CONFIG_PATH pointed at links that didn't exist and every client build failed on libudev; it now uses the store path nix-build prints. The package temperature spikes for a second under load, so the heat check takes the median of 5 readings. Restarting the watcher no longer throttles a running job. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parity-main.yml | 4 +++- docs/release-flow.md | 5 +++-- tools/ci/linux-runner.sh | 5 +++-- tools/ci/runner_pause.py | 26 ++++++++++++++++---------- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/parity-main.yml b/.github/workflows/parity-main.yml index 1ddfb24bb..1debcc694 100644 --- a/.github/workflows/parity-main.yml +++ b/.github/workflows/parity-main.yml @@ -23,7 +23,9 @@ env: jobs: parity-main: - runs-on: [self-hosted, horizon-game] + # The Mac mini's runners only: a desktop job slowed down because the maintainer came back could time out, and a + # failure here bisects and comments on merges. + runs-on: [self-hosted, horizon-game, macOS] timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures steps: - uses: actions/checkout@v4 diff --git a/docs/release-flow.md b/docs/release-flow.md index 28d1feef9..dc96498a0 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -90,8 +90,9 @@ processes, fleet-load's own detection, or a process outside the CI slice using t under way is throttled to 2 threads at the lowest weight and the runner stops after it: freezing the job would make GitHub drop it as lost and the harnesses time out. Old pull requests' `runner-env.sh` know only the Mac: the runner's `.env` and `.path` give their jobs Python, the JDK and the vanilla data (`HORIZON_SHARED_SOURCE`) all the -same. The nightly audit stays on the Mac mini's runners, so its comparison with the previous night is always made -on one platform. +same. The nightly audit and parity-main stay on the Mac mini's runners: the audit compares with the previous night, +so it stays on one platform, and a parity-main job slowed down on the desktop because he came back could time out, +and its failures bisect and comment on merges. ## Merging diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index 1bd587e79..c7235f5c1 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -24,8 +24,9 @@ version=2.338.0 sha=af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32 here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# Nix packages the jobs need, kept alive by GC roots in $dir/nix. -nix_out() { nix-build '' -A "$1" -o "$dir/nix/$2" > /dev/null && readlink -f "$dir/nix/$2"; } +# Nix packages the jobs need, kept alive by GC roots in $dir/nix. nix-build prints the store path; the root's name +# gets the output's suffix (alsa-dev-dev for alsa-lib.dev), so it is not read back from the link. +nix_out() { nix-build '' -A "$1" -o "$dir/nix/$2" | tail -1; } # The runner is a .NET program: on NixOS its native libraries (libstdc++, ICU, OpenSSL) come from nix-ld. export LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} diff --git a/tools/ci/runner_pause.py b/tools/ci/runner_pause.py index 35476a0b2..8fa0a4c5b 100755 --- a/tools/ci/runner_pause.py +++ b/tools/ci/runner_pause.py @@ -5,7 +5,7 @@ The runner works while nobody uses the machine: its screen locked, or untouched It pauses when any of these holds: - keyboard, mouse or gamepad input in the last INPUT_PAUSE seconds (60), or an unlocked session on seat0 with input in the last IDLE_AFTER seconds; - - the CPU package at TEMP_LIMIT °C (90) or more; + - the CPU package at TEMP_LIMIT °C (90) or more (the median of 5 readings); - fleet-load says a game runs (~/fleet/load: Steam, gamescope, Wine or Proton processes); - a process of his matches GAME below (the same, plus Minecraft and a Horizon game he started himself); - a process outside horizon-ci.slice keeps the GPU busy (nvidia-smi: SM use >= GPU_BUSY %, default 50, in two @@ -63,7 +63,7 @@ class Input: def __init__(self): self.devices = {} # path -> file descriptor - self.last = time.monotonic() # a start counts as input: nothing is known yet + self.last = 0.0 # the boot (the monotonic clock's zero on Linux) counts as input def poll(self): try: @@ -115,14 +115,20 @@ def at_the_screen(input_idle): def too_hot(): - """The CPU package above TEMP_LIMIT °C (coretemp or k10temp).""" + """The CPU package at TEMP_LIMIT °C or more (coretemp or k10temp), as the median of 5 readings over 2 s: under + load it jumps for a second at a time (to 96 °C on ludodesktop at a median of 68 °C), which is no reason to stop.""" for monitor in Path("/sys/class/hwmon").glob("hwmon*"): try: - if (monitor / "name").read_text().strip() in ("coretemp", "k10temp"): - celsius = int((monitor / "temp1_input").read_text()) / 1000 - return f"CPU at {celsius:.0f} °C" if celsius >= TEMP_LIMIT else None + if (monitor / "name").read_text().strip() not in ("coretemp", "k10temp"): + continue + readings = [] + for _ in range(5): + readings.append(int((monitor / "temp1_input").read_text()) / 1000) + time.sleep(0.5) except (OSError, ValueError): - pass + continue + celsius = sorted(readings)[2] + return f"CPU at {celsius:.0f} °C" if celsius >= TEMP_LIMIT else None return None @@ -219,9 +225,9 @@ def pause_reason(hot, keys): def main(): hot, keys = {}, Input() - # A boot (or a restart of this watcher) counts as a sign: the runner starts RESUME_AFTER seconds later at the - # earliest, which leaves time to log in after switching the machine on. - last_sign = time.monotonic() + # The boot counts as a sign: the runner starts RESUME_AFTER seconds after it at the earliest, which leaves time + # to log in after switching the machine on. Restarting this watcher later changes nothing for a running job. + last_sign = 0.0 paused = None while True: reason = pause_reason(hot, keys) From bdbf67ef9399a05e741c64c85e46e98fec69964e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:31:38 +0100 Subject: [PATCH 58/61] pr_ready: work out the affected harnesses once, not once per harness affected_here called parity.affected_harnesses inside the filter, once for each of the 343 harnesses. Since the selection reads the Rust items each probe reaches (#1964) one call takes about a minute and a half, so pr_ready sat for hours before its first step. Co-Authored-By: Claude Opus 5.5 --- tools/pr_ready.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/pr_ready.py b/tools/pr_ready.py index 505de28c1..4a3da6bae 100755 --- a/tools/pr_ready.py +++ b/tools/pr_ready.py @@ -164,7 +164,8 @@ def affected_here(changed): (parity.SHARED / "dependencies" / f"{name}.json").exists() for name in found) if not known and not changed & parity.SHARED_INPUTS: return None - return harness_names(name for name in found if name in parity.affected_harnesses(found, records, changed)) + affected = parity.affected_harnesses(found, records, changed) + return harness_names(name for name in found if name in affected) def short_list(names, limit=12): From 49bbdf8954bdc8958bc6f8a2e7917bb8035f710c Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:38:23 +0100 Subject: [PATCH 59/61] CI: the desktop's CI slice gets half the threads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On all 24, a cargo build held ludodesktop's CPU at 92 °C and the watcher kept pausing; at half, harnesses run at a median of 80 °C. Co-Authored-By: Claude Opus 5.5 --- tools/ci/linux-runner.sh | 1 + tools/ci/runner_pause.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index c7235f5c1..699742bf1 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -131,6 +131,7 @@ Description=Horizon CI: the runner's jobs and parity workers sent from other mac [Slice] CPUWeight=20 IOWeight=20 +CPUQuota=$(( threads * 50 ))% MemoryHigh=$(( $(awk '/MemTotal/ {print int($2 / 1048576)}' /proc/meminfo) / 2 ))G MemoryMax=$(( $(awk '/MemTotal/ {print int($2 / 1048576)}' /proc/meminfo) * 2 / 3 ))G EOF diff --git a/tools/ci/runner_pause.py b/tools/ci/runner_pause.py index 8fa0a4c5b..845627243 100755 --- a/tools/ci/runner_pause.py +++ b/tools/ci/runner_pause.py @@ -36,6 +36,8 @@ GPU_BUSY = int(os.environ.get("GPU_BUSY", 50)) INTERVAL = int(os.environ.get("INTERVAL", 15)) RESUME_AFTER = int(os.environ.get("RESUME_AFTER", 300)) THROTTLE_CPUS = int(os.environ.get("THROTTLE_CPUS", 2)) +# Working, the slice gets half the threads: on all of them ludodesktop's CPU sits at 92 °C through a cargo build. +NORMAL_CPUS = int(os.environ.get("NORMAL_CPUS", max(1, (os.cpu_count() or 2) // 2))) INPUT_PAUSE = int(os.environ.get("INPUT_PAUSE", 60)) IDLE_AFTER = int(os.environ.get("IDLE_AFTER", 900)) TEMP_LIMIT = int(os.environ.get("TEMP_LIMIT", 90)) @@ -199,7 +201,7 @@ def throttle(on): if on: systemctl("set-property", "--runtime", SLICE, "CPUWeight=1", "IOWeight=1", f"CPUQuota={THROTTLE_CPUS * 100}%") else: - systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", "CPUQuota=") + systemctl("set-property", "--runtime", SLICE, "CPUWeight=20", "IOWeight=20", f"CPUQuota={NORMAL_CPUS * 100}%") def pause_reason(hot, keys): From 160c1e8b7a3bebcd7157feff5c4cf17ada0198c2 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:54:02 +0100 Subject: [PATCH 60/61] CI: a failed preparation leaves nothing in .local When horizon-prepare failed, vanilla-data.sh deleted the preparation folder but kept what it had already written to .local/ (horizon-client, horizon-client-motion, ...). The next job then saw those real folders as already there: its preparation listed only horizon-client-structures in .entries, and the checkout kept using the stale resources outside the per-key cache. Now a failed preparation removes the .local/ entries it created; a job first removes real folders named like preparation outputs, and makes again a preparation whose .entries lacks horizon-client. Co-Authored-By: Claude Opus 5.5 --- tools/ci/vanilla-data.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tools/ci/vanilla-data.sh b/tools/ci/vanilla-data.sh index 5276c54e9..f548e1f14 100755 --- a/tools/ci/vanilla-data.sh +++ b/tools/ci/vanilla-data.sh @@ -42,7 +42,21 @@ until mkdir "$lock" 2>/dev/null; do done trap 'rmdir "$lock" 2>/dev/null || true' EXIT # horizon-prepare writes .local/horizon-client and its exporters' outputs beside it: all of them live in -# $prepared, listed in $prepared/.entries, and .local/ links to them. +# $prepared, listed in $prepared/.entries, and .local/ links to them. A real folder of that kind in .local/ is +# what an older failed preparation left: it would hide the preparations (and a new one would not count it as its +# own output), so it goes. +outputs=$({ cat "$shared"/prepared/*/.entries; cd .local && ls -d horizon-client*; } 2>/dev/null || true) +for name in $(echo "$outputs" | LC_ALL=C sort -u); do + if [ -e ".local/$name" ] && [ ! -L ".local/$name" ]; then + rm -rf ".local/${name:?}" + echo "Removed .local/$name, left by a failed preparation" + fi +done +# A preparation made while such a folder was there did not list it (horizon-client above all), so it is made again. +if [ -f "$prepared/.entries" ] && ! grep -qx horizon-client "$prepared/.entries"; then + echo "Preparation $key lacks horizon-client: preparing again" + rm -rf "$prepared" +fi if [ ! -f "$prepared/.entries" ]; then echo "Preparing resources with this checkout's horizon-prepare ($key)" rm -rf "$prepared" && mkdir -p "$prepared" @@ -55,6 +69,9 @@ if [ ! -f "$prepared/.entries" ]; then cargo build --release --locked -p horizon-prepare if ! target/release/horizon-prepare --root "$PWD" > "$RUNNER_TEMP/prepare.log" 2>&1; then tail -40 "$RUNNER_TEMP/prepare.log" + # Nothing of a failed preparation stays in .local/: the next job prepares again. + LC_ALL=C comm -13 <(echo "$before") <(ls -1A .local | LC_ALL=C sort) | + while read -r name; do rm -rf ".local/${name:?}"; done rm -rf "$prepared" exit 1 fi From d782e455a41f21e79675347c4314c9e8e2bdb535 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:55:57 +0100 Subject: [PATCH 61/61] docs: the desktop's CI slice gets half the threads Co-Authored-By: Claude Opus 5.5 --- docs/release-flow.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release-flow.md b/docs/release-flow.md index dc96498a0..7bb64d21e 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -78,8 +78,8 @@ time, under a lock) and the parity passes and slots. `horizon-game` once a full trial has passed there) runs on the maintainer's Linux desktop while nobody uses it: its screen locked, or untouched for 15 minutes (his rule, 2026-10-26). `tools/ci/linux-runner.sh install`, run there as him, registers it and installs two systemd user services: `horizon-runner` (the runner, in -`~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O weight 20 and memory capped at two -thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the machine froze under ~7 cold cargo +`~/.actions-runners/horizon`, nice 19, in `horizon-ci.slice` with CPU and I/O weight 20, half the threads (on all of +them the CPU package reached 92–96 °C) and memory capped at two thirds of the RAM; `CARGO_BUILD_JOBS` and the harness slots at 8, after the machine froze under ~7 cold cargo builds on 2026-10-14; its own Rust install; the JDK, sccache and the build libraries from nixpkgs; its own `vscreen` so no game it starts reaches his screen) and `horizon-runner-pause` (`tools/ci/runner_pause.py`), the only thing that starts the runner. The watcher stops it at once on keyboard, mouse or gamepad input (read from