From 7a1638a1908f5183102b7f649a79c4d95db8fe7c Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:12:55 +0200 Subject: [PATCH 01/51] Renderer: remove the section face groups modelled on another renderer #503 grouped each section's solid quads by the axis side they face and skipped the groups facing away from the eye. Its code and PR named an internal class of another project as the model. Horizon doesn't build on another project's code, so the grouping goes and every section draws its solid quads in one draw, as vanilla does; the GPU's back-face culling discards the back faces, so the picture is unchanged. This costs frame time, measured on the Mac mini in the PR. A clean-room version can come back later, designed from vanilla and from first principles only. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx.rs | 32 +-- crates/horizon-client/src/lod.rs | 4 - crates/horizon-client/src/lod/mesh.rs | 4 +- crates/horizon-client/src/mesher.rs | 11 - crates/horizon-client/src/mesher/facing.rs | 267 ------------------ .../src/mesher/leaf_culling_tests.rs | 2 +- 6 files changed, 15 insertions(+), 305 deletions(-) delete mode 100644 crates/horizon-client/src/mesher/facing.rs diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 59a3b5d03..b1143be6c 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -177,8 +177,6 @@ struct SectionGpu { start: u64, solid: u32, translucent: u32, - /// The solid quads by facing: back-facing groups are skipped. - faces: crate::mesher::FaceGroups, sort: translucency::SortState, sorted_indices: Option<(u32, u64)>, /// Its id for flat draws (`flat::Flat::add`), when they are on. @@ -2751,7 +2749,6 @@ impl Renderer { start, solid, translucent, - faces: mesh.faces, sort: translucency::SortState::new( mesh.translucent .as_chunks::<4>() @@ -3621,10 +3618,6 @@ impl Renderer { // as one list of indirect draws over the section arena. self.draws.clear(); self.draw_pages.clear(); - // The eye, camera-relative: view bobbing moves it off the camera. - let eye = (frame.bob * crate::camera::view_matrix(frame.yaw, frame.pitch)) - .inverse() - .transform_point3(Vec3::ZERO); // A Java pack that draws the terrain (`gfx/java_terrain.rs`): the // world pass leaves it out, and occlusion culling tests against the // pack's depth (`Cull::test_gl`). @@ -3655,14 +3648,11 @@ impl Renderer { }; let start = draws.len(); let instances = u32::from(first_phase); - let mut indices = 0; - // Only the solid groups that can face the eye. - s.faces.ranges(eye - Vec3::from(v.origin), |first, quads| { - let first = s.start + u64::from(first) * 4; - draws.push([quads * 6, instances, 0, first as u32, i as u32]); + let indices = s.solid * 6; + if indices > 0 { + draws.push([indices, instances, 0, s.start as u32, i as u32]); pages.push((s.page, None)); - indices += quads * 6; - }); + } let runs = if first_phase { &mut self.solid_runs } else { @@ -3699,12 +3689,16 @@ impl Renderer { for (i, v) in visible_lods.iter().enumerate() { let s = &self.lods[&v.key].mesh; let start = self.draws.len(); - s.faces.ranges(eye - Vec3::from(v.origin), |first, quads| { - let first = s.start + u64::from(first) * 4; - self.draws - .push([quads * 6, 1, 0, first as u32, (visible.len() + i) as u32]); + if s.solid > 0 { + self.draws.push([ + s.solid * 6, + 1, + 0, + s.start as u32, + (visible.len() + i) as u32, + ]); self.draw_pages.push((s.page, None)); - }); + } self.solid_runs .push(v.distance, s.page, start, self.draws.len() - start, [0; 3]); } diff --git a/crates/horizon-client/src/lod.rs b/crates/horizon-client/src/lod.rs index f8adb1f16..ee17acee1 100644 --- a/crates/horizon-client/src/lod.rs +++ b/crates/horizon-client/src/lod.rs @@ -1727,10 +1727,6 @@ mod tests { let expected = (key.size() * key.size() - 16 * 16) as f32; assert_eq!(top_area(&mesh.solid, 70.0), expected); assert_eq!(top_area(&mesh.translucent, 80.0 - 1.0 / 9.0), expected); - assert_eq!( - mesh.faces.quads.iter().sum::() as usize, - mesh.solid.len() / 4 - ); assert!(mesh.translucent.iter().all(|v| v.light == 240)); let side = |axis: usize, plane: f32, low: f32, high: f32| { let other = if axis == 0 { 2 } else { 0 }; diff --git a/crates/horizon-client/src/lod/mesh.rs b/crates/horizon-client/src/lod/mesh.rs index 73287af50..852f45e5b 100644 --- a/crates/horizon-client/src/lod/mesh.rs +++ b/crates/horizon-client/src/lod/mesh.rs @@ -2,7 +2,7 @@ use super::{Column, Span, TileKey}; use crate::{ fast_hash::{FxHashMap, FxHashSet}, - mesher::{FaceGroups, OPAQUE_LAYER, SectionMesh, TerrainVertex}, + mesher::{OPAQUE_LAYER, SectionMesh, TerrainVertex}, streaming::WorkerContext, }; use horizon_assets::model::Direction; @@ -558,7 +558,6 @@ fn mesh_columns_with_edges( emit(&mut mesh, key, rect) } } - mesh.faces = FaceGroups::group(&mut mesh.solid); mesh } @@ -1003,7 +1002,6 @@ mod tests { } let mesh = mesh(&columns); assert_eq!(mesh.solid.len(), 6 * 4); - assert_eq!(mesh.faces.quads.iter().sum::(), 6); assert!(mesh.translucent.is_empty()); for q in mesh.solid.as_chunks::<4>().0.iter() { let center = q diff --git a/crates/horizon-client/src/mesher.rs b/crates/horizon-client/src/mesher.rs index a9e9f6823..ec6f2ae73 100644 --- a/crates/horizon-client/src/mesher.rs +++ b/crates/horizon-client/src/mesher.rs @@ -3,7 +3,6 @@ //! `ModelBlockRenderer`/`BlockModelLighter` (smooth light, ambient occlusion, //! cardinal shading, tint sources, block offsets; see `lighting`). Fluids //! follow `FluidRenderer` (see `fluid`). -mod facing; mod fluid; mod lighting; use crate::{ @@ -13,7 +12,6 @@ use crate::{ chunk::{Chunk, MIN_Y}, models::catalog::BakedFace, }; -pub use facing::{FaceGroups, GROUPS as FACE_GROUPS}; use horizon_core::{ biome::BiomeZoom, geometry::Direction, position::BlockPos, random::position_seed, }; @@ -116,8 +114,6 @@ mod vertex_tests { } pub struct SectionMesh { pub solid: Vec, - /// `solid`'s quads by the side they face (`facing`). - pub faces: FaceGroups, pub translucent: Vec, /// VisGraph: bit `a * 6 + b` is set when faces `a` and `b` (Direction /// order) are connected through non-opaque blocks. @@ -131,7 +127,6 @@ impl Default for SectionMesh { fn default() -> Self { Self { solid: Vec::new(), - faces: FaceGroups::default(), translucent: Vec::new(), visibility: ALL_VISIBLE, block_entities: Vec::new(), @@ -698,12 +693,6 @@ pub struct Mesher<'a> { impl Mesher<'_> { /// `region[dz][dx]` are the chunks around the meshed column (centre 1,1). pub fn mesh(&mut self, region: &[[Option<&Chunk>; 3]; 3], sy: i32) -> SectionMesh { - let mut out = self.tesselate(region, sy); - out.faces = facing::group(&mut out.solid); - out - } - /// The section's quads in model order, before `facing::group`. - pub(crate) fn tesselate(&mut self, region: &[[Option<&Chunk>; 3]; 3], sy: i32) -> SectionMesh { let mut out = SectionMesh::default(); let center = region[1][1].expect("centre chunk"); let section = ¢er.sections[sy as usize]; diff --git a/crates/horizon-client/src/mesher/facing.rs b/crates/horizon-client/src/mesher/facing.rs deleted file mode 100644 index 7bafe9e82..000000000 --- a/crates/horizon-client/src/mesher/facing.rs +++ /dev/null @@ -1,267 +0,0 @@ -//! Solid quads grouped by the side they face, as Sodium's `ModelQuadFacing` -//! does. A quad in the plane `x = p` facing +X is front-facing only while the -//! eye is at `x > p`, and terrain culls back faces, so when the eye is on the -//! back side of every quad of a group the renderer skips the group's draw: -//! the GPU would have discarded all of it after shading its vertices. The -//! picture is unchanged; the vertex work for those quads is gone. -use super::TerrainVertex; -use glam::Vec3; - -/// Groups in buffer order: the negative sides, quads not in an axis plane -/// (rotated model elements, sloped fluid tops), then the positive sides. -/// With the eye outside a section on every axis, one side per axis is -/// visible, so the drawn groups form one to three runs. -pub const GROUPS: usize = 7; -const UNALIGNED: usize = 3; -/// The group of a quad facing `-axis` (0 x, 1 y, 2 z) is `NEGATIVE[axis]`, of -/// one facing `+axis` `GROUPS - 1 - NEGATIVE[axis]`. -const NEGATIVE: [usize; 3] = [0, 2, 1]; -/// How far (blocks) the eye must be behind every plane of a group before -/// the group is skipped: rounding on the GPU never makes such a quad -/// front-facing. -const MARGIN: f32 = 1.0 / 16.0; - -/// The solid quads' groups, filled by `group`. -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct FaceGroups { - /// Quads per group, in buffer order. - pub quads: [u32; GROUPS], - /// Per axis group, the plane nearest the side the group faces: the - /// lowest plane of a positive group and the highest of a negative one. - planes: [f32; GROUPS], -} - -impl Default for FaceGroups { - /// Everything unaligned: always drawn. - fn default() -> Self { - Self { - quads: [0; GROUPS], - planes: [0.0; GROUPS], - } - } -} - -impl FaceGroups { - /// Group arbitrary-sized terrain quads, including distant LOD tiles. - pub fn group(solid: &mut Vec) -> Self { - group(solid) - } - - /// Whether group `g` can hold a front-facing quad for an eye at `eye`, - /// relative to the section's minimum corner. - pub fn visible(&self, g: usize, eye: Vec3) -> bool { - if g == UNALIGNED { - return true; - } - let (axis, positive) = axis(g); - if positive { - eye[axis] > self.planes[g] - MARGIN - } else { - eye[axis] < self.planes[g] + MARGIN - } - } - /// The quad ranges (first quad, quad count) to draw for an eye at `eye` - /// relative to the section's minimum corner, adjacent groups merged. - pub fn ranges(&self, eye: Vec3, mut out: impl FnMut(u32, u32)) { - let mut first = 0; - let mut run: Option<(u32, u32)> = None; - for g in 0..GROUPS { - let quads = self.quads[g]; - if quads > 0 { - if self.visible(g, eye) { - run = Some(match run { - Some((start, len)) => (start, len + quads), - None => (first, quads), - }); - } else if let Some((start, len)) = run.take() { - out(start, len); - } - } - first += quads; - } - if let Some((start, len)) = run { - out(start, len); - } - } -} - -/// The axis and sign of each axis group (`UNALIGNED`'s entry is unused). -const AXES: [(usize, bool); GROUPS] = { - let mut axes = [(0, false); GROUPS]; - let mut axis = 0; - while axis < 3 { - axes[NEGATIVE[axis]] = (axis, false); - axes[GROUPS - 1 - NEGATIVE[axis]] = (axis, true); - axis += 1; - } - axes -}; - -/// The axis and sign of axis group `g`. -fn axis(g: usize) -> (usize, bool) { - debug_assert_ne!(g, UNALIGNED); - AXES[g] -} - -/// The group of one quad, and its plane: an axis group only when all four -/// corners lie exactly in one axis plane. -fn facing(quad: &[TerrainVertex]) -> (usize, f32) { - let p = |i: usize| Vec3::from(quad[i].pos); - let normal = (p(1) - p(0)).cross(p(2) - p(0)); - for axis in 0..3 { - let [a, b] = [(axis + 1) % 3, (axis + 2) % 3]; - let plane = quad[0].pos[axis]; - if normal[axis] != 0.0 - && normal[a] == 0.0 - && normal[b] == 0.0 - && quad.iter().all(|v| v.pos[axis] == plane) - { - let negative = NEGATIVE[axis]; - let g = if normal[axis] > 0.0 { - GROUPS - 1 - negative - } else { - negative - }; - return (g, plane); - } - } - (UNALIGNED, 0.0) -} - -/// Reorder `solid` by group, keeping the order of quads within a group (so -/// coplanar quads such as a grass block's overlay still draw after their -/// base), and return the groups. -pub fn group(solid: &mut Vec) -> FaceGroups { - let mut groups = FaceGroups::default(); - for (g, plane) in groups.planes.iter_mut().enumerate() { - if g != UNALIGNED { - *plane = if axis(g).1 { f32::MAX } else { f32::MIN }; - } - } - let faces: Vec = solid - .as_chunks::<4>() - .0 - .iter() - .map(|quad| { - let (g, plane) = facing(quad); - groups.quads[g] += 1; - if g != UNALIGNED { - let nearest = &mut groups.planes[g]; - *nearest = if axis(g).1 { - nearest.min(plane) - } else { - nearest.max(plane) - }; - } - g as u8 - }) - .collect(); - if faces.windows(2).all(|w| w[0] <= w[1]) { - return groups; - } - let mut next = [0usize; GROUPS]; - let mut start = 0; - for g in 0..GROUPS { - next[g] = start; - start += groups.quads[g] as usize; - } - let mut sorted = vec![solid[0]; solid.len()]; - for (quad, &g) in solid.as_chunks::<4>().0.iter().zip(&faces) { - let at = next[g as usize] * 4; - sorted[at..at + 4].copy_from_slice(quad); - next[g as usize] += 1; - } - *solid = sorted; - groups -} - -#[cfg(test)] -mod tests { - use super::*; - - fn vertex(pos: [f32; 3], layer: u16) -> TerrainVertex { - TerrainVertex::new(pos, [0.0; 2], layer, 0, 0) - } - /// A unit quad in the plane `axis = plane`, wound to face `sign`, tagged - /// with `layer` so tests can follow it. - fn quad(axis: usize, plane: f32, sign: f32, layer: u16) -> Vec { - let corner = |u: f32, v: f32| { - let mut p = [0.0; 3]; - p[axis] = plane; - p[(axis + 1) % 3] = u; - p[(axis + 2) % 3] = v; - vertex(p, layer) - }; - let mut q = vec![ - corner(0.0, 0.0), - corner(1.0, 0.0), - corner(1.0, 1.0), - corner(0.0, 1.0), - ]; - if sign < 0.0 { - q.reverse(); - } - q - } - - #[test] - fn quads_group_by_the_side_their_winding_faces() { - for a in 0..3 { - for sign in [-1.0, 1.0] { - let q = quad(a, 4.0, sign, 0); - let (g, plane) = facing(&q); - assert_eq!(axis(g), (a, sign > 0.0), "axis {a} sign {sign}"); - assert_eq!(plane, 4.0); - let normal = (Vec3::from(q[1].pos) - Vec3::from(q[0].pos)) - .cross(Vec3::from(q[2].pos) - Vec3::from(q[0].pos)); - assert_eq!(normal[a].signum(), sign); - } - } - // A cross plant's diagonal quad faces no axis. - let diagonal = [ - [0.0, 0.0, 0.0], - [1.0, 0.0, 1.0], - [1.0, 1.0, 1.0], - [0.0, 1.0, 0.0], - ] - .map(|p| vertex(p, 0)); - assert_eq!(facing(&diagonal).0, UNALIGNED); - } - - #[test] - fn grouping_is_stable_and_skips_only_back_facing_groups() { - // +X at x = 3 and x = 7 (layers 1, 2), -X at x = 5 (3), an unaligned - // quad (4), a coplanar overlay on the x = 3 quad (5). - let diagonal = [ - [0.0, 0.0, 0.0], - [1.0, 0.0, 1.0], - [1.0, 1.0, 1.0], - [0.0, 1.0, 0.0], - ] - .map(|p| vertex(p, 4)); - let mut solid = [ - quad(0, 3.0, 1.0, 1), - quad(0, 5.0, -1.0, 3), - diagonal.to_vec(), - quad(0, 7.0, 1.0, 2), - quad(0, 3.0, 1.0, 5), - ] - .concat(); - let groups = group(&mut solid); - let layers: Vec = solid.chunks(4).map(|q| q[0].layer).collect(); - assert_eq!(layers, [3, 4, 1, 2, 5]); - let ranges = |eye: Vec3| { - let mut out = Vec::new(); - groups.ranges(eye, |first, count| out.push((first, count))); - out - }; - // Between the planes: every group can face the eye. - assert_eq!(ranges(Vec3::new(4.0, 8.0, 8.0)), [(0, 5)]); - // Beyond x = 7: the -X quad at x = 5 faces away. - assert_eq!(ranges(Vec3::new(20.0, 8.0, 8.0)), [(1, 4)]); - // Before x = 3: every +X quad faces away. - assert_eq!(ranges(Vec3::new(-10.0, 8.0, 8.0)), [(0, 2)]); - // Within the margin of the nearest plane: still drawn. - assert_eq!(ranges(Vec3::new(3.0, 8.0, 8.0)), [(0, 5)]); - } -} diff --git a/crates/horizon-client/src/mesher/leaf_culling_tests.rs b/crates/horizon-client/src/mesher/leaf_culling_tests.rs index 679b19090..1273c806d 100644 --- a/crates/horizon-client/src/mesher/leaf_culling_tests.rs +++ b/crates/horizon-client/src/mesher/leaf_culling_tests.rs @@ -42,7 +42,7 @@ fn leaf_culling_removes_only_internal_leaf_and_root_faces() { cutout_leaves, cull_leaves, } - .tesselate(®ion, (0 - MIN_Y) / 16) + .mesh(®ion, (0 - MIN_Y) / 16) .solid }; From d271176f2342e1c2033f4ca38005892c60801f3e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:31:02 +0200 Subject: [PATCH 02/51] Tools and harnesses start the game in scratch folders, not a player's The game logs into HORIZON_HOME and falls back to the player's own game folder when it is unset. Harnesses that set only HORIZON_NO_AUDIO (and test_native_client, which set nothing) wrote logs, and on a world run advancements, into ~/Library/Application Support/horizon. lab.game_env gives every game a fresh HORIZON_HOME, HORIZON_LAUNCHER_HOME, XDG_DATA_HOME and XDG_CONFIG_HOME, no audio and no launcher self-update; a folder the caller passes is kept. Every launcher of the game uses it, parity.py gives each harness its own HORIZON_HOME, and tests/test_game_home.py fails for a script in tools/ that starts the game without it. Co-Authored-By: Claude Sonnet 5.5 --- docs/parity-workflow.md | 4 + tests/test_game_home.py | 102 ++++++++++++++++++++++++++ tools/agent_control_e2e.py | 15 ++-- tools/graphics_bench.py | 7 +- tools/lab.py | 33 +++++++++ tools/lan_e2e.py | 3 +- tools/launcher_mcp_e2e.py | 3 +- tools/mod_browser_harness.py | 8 +- tools/parity.py | 7 ++ tools/perf_bench.py | 2 +- tools/server_bench.py | 2 +- tools/test_block_removal_parity.py | 2 +- tools/test_command_exec_parity.py | 2 +- tools/test_composter_parity.py | 2 +- tools/test_crafter_parity.py | 2 +- tools/test_dedicated_server_parity.py | 4 +- tools/test_dispenser_parity.py | 2 +- tools/test_friends.py | 2 +- tools/test_mace_parity.py | 2 +- tools/test_native_client.py | 2 +- tools/test_online_mode_parity.py | 4 +- tools/test_save_roundtrip_parity.py | 4 +- tools/test_sculk_parity.py | 2 +- tools/test_secure_chat_parity.py | 2 +- tools/test_use_item_on_parity.py | 2 +- tools/test_world_reopen_parity.py | 5 +- tools/test_world_spawn_parity.py | 2 +- 27 files changed, 190 insertions(+), 37 deletions(-) create mode 100644 tests/test_game_home.py diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index 1b00080e7..f9f222071 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -32,6 +32,10 @@ Passes are shared between worktrees. A harness's key depends only on the content Every runner on the machine shares `PARITY_MACHINE_JOBS` slots (default: half the CPU threads), so several agents' runs together can't overload it. Parallel harnesses are also admitted by memory: each harness's recorded peak is reserved, within three quarters of the memory available at the start, and a harness starts only if the memory free at that moment still leaves 2 GB. `-j` caps the count here (default: `PARITY_LOCAL_JOBS`, else 3 while other machines help or a third of the CPU threads alone, within `$FLEET_LOAD_JOBS`). +### Harnesses that start the game + +The game logs into `HORIZON_HOME` and falls back to the player's own game folder when it is unset, so a harness that starts the game, `horizon-ctl`, the launcher or the dedicated server builds the environment with `lab.game_env(**extra)`: no audio, and fresh scratch folders for `HORIZON_HOME`, `HORIZON_LAUNCHER_HOME`, `XDG_DATA_HOME` and `XDG_CONFIG_HOME`, removed when the harness exits (a folder you pass, or that the environment already has, is kept). The runner also gives each harness its own `HORIZON_HOME` under `.local/parity-runner/game-home/`. `tests/test_game_home.py` fails for a script in `tools/` that starts the game without it. + ### Other machines Harnesses can also run on the Mac mini and the ThinkPad (`tools/parity_hosts.json` describes them; its `default` list is empty, so pass `--hosts macmini,thinkpad` or set `PARITY_HOSTS`, after copying `shared-local/` to `~/code/horizon-parity/shared-local` on each host). The runner mirrors the worktree there with rsync (not `.git`, `.local`, `target` or ignored files) into `~/code/horizon-parity/remote/`, whose `.local` links to the vanilla data in `~/code/horizon-parity/shared-local`, and starts `tools/parity.py --worker` over ssh. The worker builds the probes (about 80 s the first time) and runs what it is sent under the same tracer, admitting harnesses against its own free memory. One queue, longest first, feeds every machine, remote ones first: the Mac takes up to 14 harnesses, the desktop 3 and only those under 800 MB. Each pass comes back with the content hash of every file it depended on, so it is keyed on exactly what ran, and logs come back to `.local/parity-runner/logs/`; state, sharing and `gate.py` work as before. If a host can't be reached, or drops out, its harnesses run here. diff --git a/tests/test_game_home.py b/tests/test_game_home.py new file mode 100644 index 000000000..b4d9e017f --- /dev/null +++ b/tests/test_game_home.py @@ -0,0 +1,102 @@ +"""A tool or harness that starts the game must not write into a real player's folders. + +The game logs into HORIZON_HOME and falls back to the player's own game folder when it is unset, so every +spawn goes through lab.game_env, which sets fresh scratch folders. This test fails for a script in tools/ +that starts the game (or horizon-ctl, the launcher or the dedicated server) without it.""" + +import ast +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from tools import lab + +TOOLS = Path(lab.ROOT) / "tools" + +# Strings that mean "this script starts the game": a binary a harness runs, or the audio switch every game run sets. +SPAWN_MARKERS = { + "HORIZON_NO_AUDIO", "horizon-ctl", "horizon-ctl.exe", "horizon-server", "horizon-server.exe", + "horizon-client.exe", "release/horizon-client", "fast/horizon-client", "debug/horizon-client", +} + +# Scripts that mention those strings without starting a game themselves, and why. +EXEMPT = { + "lab.py": "defines game_env", + "parity.py": "runs the harnesses and gives each its own HORIZON_HOME", + "package_native_client.py": "packs the binaries into an archive", + "build_release.py": "builds and packs the release", + "release_train.py": "release bookkeeping", + "pr_ready.py": "runs cargo checks and parity.py", + "port_harness.py": "ports Java code, starts no game", +} + + +def constants(tree): + """Every string in a script that is a whole literal or a keyword name (docstrings are longer text).""" + for node in ast.walk(tree): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + yield node.value + elif isinstance(node, ast.keyword) and node.arg: + yield node.arg + + +def uses_game_env(tree): + """The script calls lab.game_env, or starts its servers through test_dedicated_server_parity.Server, which does.""" + return any(isinstance(node, ast.Attribute) and node.attr == "game_env" + or isinstance(node, ast.Name) and node.id == "game_env" + or isinstance(node, ast.ImportFrom) and (node.module or "").endswith("test_dedicated_server_parity") + for node in ast.walk(tree)) + + +def sets_audio_by_hand(tree): + return "HORIZON_NO_AUDIO" in set(constants(tree)) + + +class GameEnvTests(unittest.TestCase): + def test_gives_every_game_scratch_folders(self): + with mock.patch.dict(os.environ, {}, clear=False): + for key in ("HORIZON_HOME", "HORIZON_LAUNCHER_HOME", "XDG_DATA_HOME", "XDG_CONFIG_HOME"): + os.environ.pop(key, None) + first, second = lab.game_env(), lab.game_env() + self.assertEqual(first["HORIZON_NO_AUDIO"], "1") + self.assertEqual(first["HORIZON_LAUNCHER_NO_SELF_UPDATE"], "1") + for key in ("HORIZON_HOME", "HORIZON_LAUNCHER_HOME", "XDG_DATA_HOME", "XDG_CONFIG_HOME"): + self.assertTrue(Path(first[key]).is_dir(), key) + self.assertNotEqual(first[key], second[key], key) + + def test_keeps_the_callers_folders(self): + with tempfile.TemporaryDirectory() as temp: + env = lab.game_env({"PATH": "x", "HORIZON_HOME": "/from/base"}, HORIZON_LAUNCHER_HOME=Path(temp), A=1) + self.assertEqual(env["HORIZON_HOME"], "/from/base") + self.assertEqual(env["HORIZON_LAUNCHER_HOME"], temp) + self.assertEqual((env["PATH"], env["A"]), ("x", "1")) + + def test_does_not_touch_the_process_environment(self): + before = dict(os.environ) + lab.game_env() + self.assertEqual(dict(os.environ), before) + + +class SpawnerTests(unittest.TestCase): + def test_every_script_that_starts_the_game_uses_game_env(self): + offenders = [] + for path in sorted(TOOLS.rglob("*.py")): + if path.name in EXEMPT: + continue + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + if SPAWN_MARKERS & set(constants(tree)) and not uses_game_env(tree): + offenders.append(str(path.relative_to(lab.ROOT))) + self.assertEqual(offenders, [], "these start the game without lab.game_env (HORIZON_HOME falls back to a " + "real player's folder); use it, or list the script in EXEMPT") + + def test_nobody_sets_the_audio_switch_by_hand(self): + offenders = [str(path.relative_to(lab.ROOT)) for path in sorted(TOOLS.rglob("*.py")) + if path.name not in EXEMPT + and sets_audio_by_hand(ast.parse(path.read_text(encoding="utf-8"), filename=str(path)))] + self.assertEqual(offenders, [], "set it through lab.game_env, which also gives the game scratch folders") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/agent_control_e2e.py b/tools/agent_control_e2e.py index 3fcf39553..7f0c52bce 100644 --- a/tools/agent_control_e2e.py +++ b/tools/agent_control_e2e.py @@ -67,17 +67,20 @@ class Failed(Exception): pass -def game_env(screen): +def game_env(screen, home=None): + """The environment of a game: lab.game_env with HORIZON_HOME `home` (a scratch folder when None), no live-count + ping, and on Linux the virtual screen `screen`.""" + extra = {'HORIZON_TELEMETRY_URL': 'off'} # no live-count ping from test games + if home is not None: + extra['HORIZON_HOME'] = home if sys.platform == 'darwin': - return dict(os.environ, HORIZON_NO_AUDIO='1', HORIZON_TELEMETRY_URL='off') - env = dict(os.environ, VSCREEN=screen) + return lab.game_env(**extra) + env = lab.game_env(VSCREEN=screen, **extra) subprocess.run(['vscreen', 'up'], env=env, check=False, capture_output=True) text = subprocess.run(['vscreen', 'env'], env=env, capture_output=True, text=True, check=True).stdout env.pop('DISPLAY', None) for match in re.finditer(r'(\w+)=(\S+)', text.split('unset')[0]): env[match.group(1)] = match.group(2) - env['HORIZON_NO_AUDIO'] = '1' - env['HORIZON_TELEMETRY_URL'] = 'off' # no live-count ping from test games if LAVAPIPE.exists(): env['VK_ICD_FILENAMES'] = str(LAVAPIPE) return env @@ -632,7 +635,7 @@ def scratch(mode, resources): (work / '.local/horizon-client').mkdir() os.symlink(resources, work / '.local/horizon-client' / lab.VERSION) # horizon-ctl reads No AI in the game home: this one, not the player's. - env = dict(game_env(os.environ.get('VSCREEN', 'agent-control')), HORIZON_HOME=str(work)) + env = game_env(os.environ.get('VSCREEN', 'agent-control'), home=work) return work, env diff --git a/tools/graphics_bench.py b/tools/graphics_bench.py index 0e42e94d5..f19165f62 100644 --- a/tools/graphics_bench.py +++ b/tools/graphics_bench.py @@ -53,15 +53,14 @@ guiScale:0 def vscreen_env(): out = subprocess.run(["vscreen", "env"], cwd=ROOT, capture_output=True, text=True, check=True) - env = dict(os.environ) + env = lab.game_env() for part in out.stdout.replace(";", " ").split(): if "=" in part and not part.startswith("export"): k, v = part.split("=", 1) env[k] = v env.pop("DISPLAY", None) - # Benchmark games never play sound on the machine's speakers. - env["HORIZON_NO_AUDIO"] = "1" - # Nor reach the telemetry server (docs/telemetry.md). + # lab.game_env: no sound on the machine's speakers, and scratch folders instead of the player's. + # Benchmark games do not reach the telemetry server (docs/telemetry.md). env["HORIZON_TELEMETRY_URL"] = "off" return env diff --git a/tools/lab.py b/tools/lab.py index a62a38f59..b7cc2f795 100644 --- a/tools/lab.py +++ b/tools/lab.py @@ -5,6 +5,7 @@ generated files stay in .local/. Also the small helpers the harnesses share (free ports, offline UUIDs, VarInts, float bits).""" import argparse +import atexit from collections import Counter import hashlib import json @@ -54,6 +55,38 @@ def cargo_target(): return ROOT / os.environ.get("CARGO_TARGET_DIR", "target") +_SCRATCH_DIRS = [] + + +def _scratch_dir(prefix): + path = tempfile.mkdtemp(prefix=prefix) + _SCRATCH_DIRS.append(path) + return path + + +@atexit.register +def _remove_scratch_dirs(): + for path in _SCRATCH_DIRS: + shutil.rmtree(path, ignore_errors=True) + + +def game_env(base=None, **extra): + """The environment for every run of the game, the launcher or horizon-ctl from a tool or a harness: no audio, + and fresh scratch folders for the game's home (logs, saves, options), the launcher's home and the XDG folders, + so a run never writes into a real player's folders (the game falls back to them when HORIZON_HOME is unset). + A folder the caller already has in `base` (default os.environ) or passes in `extra` is kept. The scratch folders + are removed when this process exits.""" + env = dict(os.environ if base is None else base) + env.update({key: str(value) for key, value in extra.items()}) + env["HORIZON_NO_AUDIO"] = "1" + env["HORIZON_LAUNCHER_NO_SELF_UPDATE"] = "1" + for key, prefix in (("HORIZON_HOME", "horizon-home-"), ("HORIZON_LAUNCHER_HOME", "horizon-launcher-home-"), + ("XDG_DATA_HOME", "horizon-xdg-data-"), ("XDG_CONFIG_HOME", "horizon-xdg-config-")): + if not env.get(key): + env[key] = _scratch_dir(prefix) + return env + + def write_json(path, data): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") diff --git a/tools/lan_e2e.py b/tools/lan_e2e.py index 3ece4b448..9d0586a2c 100755 --- a/tools/lan_e2e.py +++ b/tools/lan_e2e.py @@ -275,8 +275,7 @@ class Game: (self.work / '.local/horizon-client').mkdir(parents=True) os.symlink(resources, self.work / '.local/horizon-client' / lab.VERSION) self.socket = self.work / 'c.sock' - env = game_env(os.environ.get('VSCREEN', 'lan-e2e')) - env['HORIZON_HOME'] = str(self.work / 'home') + env = game_env(os.environ.get('VSCREEN', 'lan-e2e'), home=self.work / 'home') self.d = Cli(ctl, self.socket, self.work, env) self.d.name = name diff --git a/tools/launcher_mcp_e2e.py b/tools/launcher_mcp_e2e.py index 0b827d288..748acafc2 100644 --- a/tools/launcher_mcp_e2e.py +++ b/tools/launcher_mcp_e2e.py @@ -167,8 +167,7 @@ def main(): os.symlink(bins / name, version / name) os.symlink(resources, version / '.local/horizon-client' / lab.VERSION) (version / '.installed').write_text('') - env = dict(os.environ, HORIZON_LAUNCHER_HOME=str(work / 'launcher'), HORIZON_HOME=str(work / 'home'), - HORIZON_NO_AUDIO='1', HORIZON_TELEMETRY_URL='off') + env = lab.game_env(HORIZON_LAUNCHER_HOME=work / 'launcher', HORIZON_HOME=work / 'home', HORIZON_TELEMETRY_URL='off') m = Mcp(Path(args.launcher).resolve(), env) try: summary = scenario(m, out) diff --git a/tools/mod_browser_harness.py b/tools/mod_browser_harness.py index 4531570bc..cf33d85de 100755 --- a/tools/mod_browser_harness.py +++ b/tools/mod_browser_harness.py @@ -33,6 +33,11 @@ import tempfile import time from pathlib import Path +try: + from . import lab +except ImportError: + import lab + ROOT = Path(__file__).resolve().parents[1] @@ -154,8 +159,7 @@ def main(): args = ap.parse_args() home = Path(tempfile.mkdtemp(prefix="horizon-mod-browser-")) - env = dict(os.environ, HORIZON_HOME=str(home), HORIZON_NO_AUDIO="1", HORIZON_NO_HEARTBEAT="1", - HORIZON_CONTROL=str(home / "ctl.sock")) + env = lab.game_env(HORIZON_HOME=home, HORIZON_NO_HEARTBEAT="1", HORIZON_CONTROL=home / "ctl.sock") launcher = [args.launcher] game = Game(str(Path(args.bin) / "horizon-ctl"), str(Path(args.bin) / "horizon-client"), args.resources, home, env) diff --git a/tools/parity.py b/tools/parity.py index 4ed75f5ef..38a2fcde6 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -39,6 +39,7 @@ import json import os from pathlib import Path, PurePosixPath import re +import shutil import socket import subprocess import sys @@ -809,6 +810,12 @@ class LocalExecutor: trace.unlink(missing_ok=True) log = (STATE_DIR / "logs" / f"{name}.log").open("w") environment = run_environment(reuse) + # A game the harness starts logs into HORIZON_HOME, which is a real player's folder when it is unset + # (lab.game_env keeps this one). + game_home = STATE_DIR / "game-home" / name + shutil.rmtree(game_home, ignore_errors=True) + game_home.mkdir(parents=True) + environment.setdefault("HORIZON_HOME", str(game_home)) environment["PARITY_SHIM_OUTPUTS"] = str(outputs_listing(name, fresh=True)) environment.pop("PARITY_SHARD", None) if shard: diff --git a/tools/perf_bench.py b/tools/perf_bench.py index 31024737c..5f5f57305 100644 --- a/tools/perf_bench.py +++ b/tools/perf_bench.py @@ -400,7 +400,7 @@ def run_horizon(args, distance): for carry in dev_mods(args): command += ["--dev-mod", str(carry)] # Benchmark runs stay out of the live count and send nothing (docs/telemetry.md). - env = dict(os.environ, HORIZON_NO_AUDIO="1", HORIZON_TELEMETRY_URL="off") + env = lab.game_env(HORIZON_TELEMETRY_URL="off") if args.warm: # A first start to the title screen builds what later starts reuse (the content-mod # root, the block behaviour snapshot). It is not measured: players see it once. diff --git a/tools/server_bench.py b/tools/server_bench.py index a9a494efd..bf1cdabeb 100755 --- a/tools/server_bench.py +++ b/tools/server_bench.py @@ -120,7 +120,7 @@ class Server: self.process = subprocess.Popen([str(c) for c in command], cwd=directory, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace', - env=dict(os.environ, HORIZON_NO_AUDIO='1')) + env=lab.game_env()) threading.Thread(target=self._read, daemon=True).start() def _read(self): diff --git a/tools/test_block_removal_parity.py b/tools/test_block_removal_parity.py index 6ebbbf1d3..3185bd4bb 100755 --- a/tools/test_block_removal_parity.py +++ b/tools/test_block_removal_parity.py @@ -183,7 +183,7 @@ def main(): d.run(['cargo', 'build', '-q', '--release', '-p', 'horizon-client', '--bin', 'horizon-client'], cwd=lab.ROOT, env={**os.environ, 'CARGO_BUILD_JOBS': os.environ.get('CARGO_BUILD_JOBS', '3')}) d.run([str(lab.cargo_target() / 'release/horizon-client'), '--resources', str(resources)], cwd=work, - env={**os.environ, 'HORIZON_DISPENSE_FIXTURE': f'{work / "cases.jsonl"},{work / "rust.txt"}', 'HORIZON_NO_AUDIO': '1'}) + env=lab.game_env(HORIZON_DISPENSE_FIXTURE=f'{work / "cases.jsonl"},{work / "rust.txt"}')) # Loot table names (D) are vanilla's bookkeeping; the drops themselves are compared. ignore = tuple(prefix + ' ' for prefix in args.ignore.split(',') if prefix) + ('D ',) native = h.split_cases((work / 'native.txt').read_text(), ignore) diff --git a/tools/test_command_exec_parity.py b/tools/test_command_exec_parity.py index 9d57d3f69..877bd0b38 100644 --- a/tools/test_command_exec_parity.py +++ b/tools/test_command_exec_parity.py @@ -82,7 +82,7 @@ def main(): rust_dir = build / 'rust' shutil.rmtree(rust_dir, ignore_errors=True) rust_dir.mkdir() - env = dict(os.environ, HORIZON_NO_AUDIO='1', HORIZON_COMMAND_SCRIPT=str(script_path), HORIZON_COMMAND_OUTPUT=str(build / 'rust.jsonl')) + env = lab.game_env(HORIZON_COMMAND_SCRIPT=script_path, HORIZON_COMMAND_OUTPUT=build / 'rust.jsonl') run([binary, '--resources', lab.RESOURCES, '--world', 'harness', '--seed', SEED, '--radius', 4], 'rust', cwd=rust_dir, env=env) vanilla = [json.loads(l) for l in (build / 'vanilla.jsonl').open(encoding='utf-8')] diff --git a/tools/test_composter_parity.py b/tools/test_composter_parity.py index e632527ce..074b227a8 100644 --- a/tools/test_composter_parity.py +++ b/tools/test_composter_parity.py @@ -125,7 +125,7 @@ def main(): d.run(['cargo', 'build', '-q', '--release', '-p', 'horizon-client', '--bin', 'horizon-client'], cwd=lab.ROOT, env={**os.environ, 'CARGO_BUILD_JOBS': os.environ.get('CARGO_BUILD_JOBS', '3')}) d.run([str(lab.cargo_target() / 'release/horizon-client'), '--resources', str(resources)], cwd=work, - env={**os.environ, 'HORIZON_DISPENSE_FIXTURE': f'{work / "cases.jsonl"},{work / "rust.txt"}', 'HORIZON_NO_AUDIO': '1'}) + env=lab.game_env(HORIZON_DISPENSE_FIXTURE=f'{work / "cases.jsonl"},{work / "rust.txt"}')) ignore = tuple(prefix + ' ' for prefix in args.ignore.split(',') if prefix) + ('D ',) native = h.split_cases((work / 'native.txt').read_text(), ignore) rust = h.split_cases((work / 'rust.txt').read_text(), ignore) diff --git a/tools/test_crafter_parity.py b/tools/test_crafter_parity.py index 1b9564f41..b9c343d4d 100644 --- a/tools/test_crafter_parity.py +++ b/tools/test_crafter_parity.py @@ -245,7 +245,7 @@ def main(): d.run(['cargo', 'build', '-q', '--release', '-p', 'horizon-client', '--bin', 'horizon-client'], cwd=lab.ROOT, env={**os.environ, 'CARGO_BUILD_JOBS': os.environ.get('CARGO_BUILD_JOBS', '3')}) d.run([str(lab.cargo_target() / 'release/horizon-client'), '--resources', str(resources)], cwd=work, - env={**os.environ, 'HORIZON_DISPENSE_FIXTURE': f'{work / "cases.jsonl"},{work / "rust.txt"}', 'HORIZON_NO_AUDIO': '1'}) + env=lab.game_env(HORIZON_DISPENSE_FIXTURE=f'{work / "cases.jsonl"},{work / "rust.txt"}')) ignore = tuple(prefix + ' ' for prefix in args.ignore.split(',') if prefix) + ('D ',) native = h.split_cases((work / 'native.txt').read_text(), ignore) rust = h.split_cases((work / 'rust.txt').read_text(), ignore) diff --git a/tools/test_dedicated_server_parity.py b/tools/test_dedicated_server_parity.py index f45df9822..6a6d258d4 100644 --- a/tools/test_dedicated_server_parity.py +++ b/tools/test_dedicated_server_parity.py @@ -126,7 +126,7 @@ class Server: self.lines = [] self.process = subprocess.Popen(list(map(str, command)), cwd=directory, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, - encoding='utf-8', errors='replace') + encoding='utf-8', errors='replace', env=lab.game_env()) self.reader = threading.Thread(target=self._read, daemon=True) self.reader.start() @@ -352,7 +352,7 @@ def horizon_client_joins(vanilla_server, binary, world, build, reference): server = Server(vanilla_server, directory, label) try: server.wait_for(r'Done \(') - env = dict(os.environ, HORIZON_NO_AUDIO='1', HORIZON_NET_REPORT=str(report)) + env = lab.game_env(HORIZON_NET_REPORT=report) with (build / 'horizon-client-run.log').open('w') as log: subprocess.run([str(c) for c in [binary, '--resources', lab.RESOURCES, '--render-distance', 2, '--server', f'127.0.0.1:{port}', '--name', PLAYER]], diff --git a/tools/test_dispenser_parity.py b/tools/test_dispenser_parity.py index 844322aeb..a6d1f429a 100644 --- a/tools/test_dispenser_parity.py +++ b/tools/test_dispenser_parity.py @@ -417,7 +417,7 @@ def main(): run(['cargo', 'build', '-q', '--release', '-p', 'horizon-client', '--bin', 'horizon-client'], cwd=lab.ROOT, env={**os.environ, 'CARGO_BUILD_JOBS': os.environ.get('CARGO_BUILD_JOBS', '3')}) run([str(lab.cargo_target() / 'release/horizon-client'), '--resources', str(resources)], cwd=work, - env={**os.environ, 'HORIZON_DISPENSE_FIXTURE': f'{work / "cases.jsonl"},{work / "rust.txt"}', 'HORIZON_NO_AUDIO': '1'}) + env=lab.game_env(HORIZON_DISPENSE_FIXTURE=f'{work / "cases.jsonl"},{work / "rust.txt"}')) # Loot table names (D) are vanilla's bookkeeping; the drops themselves are compared. ignore = tuple(prefix + ' ' for prefix in args.ignore.split(',') if prefix) + ('D ',) native = h.split_cases((work / 'native.txt').read_text(), ignore) diff --git a/tools/test_friends.py b/tools/test_friends.py index 8fa1a4fee..cc5d15b92 100644 --- a/tools/test_friends.py +++ b/tools/test_friends.py @@ -96,7 +96,7 @@ class Game: self.process = subprocess.Popen(Game.prefix + [ str(client), '--resources', str(resources), '--render-distance', '2', '--test-terrain', '--control-socket', str(self.endpoint), '--timeout-seconds', '600', - ] + Game.extra, cwd=self.home, env=dict(os.environ, HORIZON_HOME=str(self.home), HORIZON_NO_AUDIO='1', **env), + ] + Game.extra, cwd=self.home, env=lab.game_env(HORIZON_HOME=self.home, **env), stdout=self.log, stderr=subprocess.STDOUT) wait_for(self.endpoint, 'screen', lambda v: v['screen'] == 'title', 120, f'{name} title') diff --git a/tools/test_mace_parity.py b/tools/test_mace_parity.py index 123f19693..578e9e6ee 100644 --- a/tools/test_mace_parity.py +++ b/tools/test_mace_parity.py @@ -170,7 +170,7 @@ def main(): d.run(['cargo', 'build', '-q', '--release', '-p', 'horizon-client', '--bin', 'horizon-client'], cwd=lab.ROOT, env={**os.environ, 'CARGO_BUILD_JOBS': os.environ.get('CARGO_BUILD_JOBS', '3')}) d.run([str(lab.cargo_target() / 'release/horizon-client'), '--resources', str(resources)], cwd=work, - env={**os.environ, 'HORIZON_DISPENSE_FIXTURE': f'{work / "cases.jsonl"},{work / "rust.txt"}', 'HORIZON_NO_AUDIO': '1'}) + env=lab.game_env(HORIZON_DISPENSE_FIXTURE=f'{work / "cases.jsonl"},{work / "rust.txt"}')) ignore = tuple(prefix + ' ' for prefix in args.ignore.split(',') if prefix) + ('D ',) native = h.split_cases((work / 'native.txt').read_text(), ignore) rust = h.split_cases((work / 'rust.txt').read_text(), ignore) diff --git a/tools/test_native_client.py b/tools/test_native_client.py index 73e6f3223..552ea8c42 100644 --- a/tools/test_native_client.py +++ b/tools/test_native_client.py @@ -71,7 +71,7 @@ def main(): str(executable), '--radius', '1', '--frames', '30', '--smoke-report', str(report), '--timeout-seconds', '60'] with (root / 'startup-smoke.log').open('w') as log: - subprocess.run(command, cwd=lab.ROOT, env=environment, check=True, + subprocess.run(command, cwd=lab.ROOT, env=lab.game_env(environment), check=True, stdout=log, stderr=subprocess.STDOUT, timeout=120) result = json.loads(report.read_text()) if (result.get('error') is not None or result.get('presented_frames') != 30 diff --git a/tools/test_online_mode_parity.py b/tools/test_online_mode_parity.py index d430c2a68..d03838bf3 100644 --- a/tools/test_online_mode_parity.py +++ b/tools/test_online_mode_parity.py @@ -179,7 +179,7 @@ def run_client(binary, build, port, label, token, discovery): shutil.rmtree(directory, ignore_errors=True) directory.mkdir() report = directory / 'report.json' - env = dict(os.environ, HORIZON_NO_AUDIO='1', HORIZON_NET_REPORT=str(report), HORIZON_API_DISCOVERY=discovery) + env = lab.game_env(HORIZON_NET_REPORT=report, HORIZON_API_DISCOVERY=discovery) command = [binary, '--resources', lab.RESOURCES, '--render-distance', 2, '--server', f'127.0.0.1:{port}', '--name', PLAYER] if token: @@ -246,7 +246,7 @@ def run_side(label, command, binary, build, services, discovery): props = dict(PROPERTIES, **{'server-port': str(port)}) (directory / 'server.properties').write_text(''.join(f'{k}={v}\n' for k, v in props.items())) (directory / 'eula.txt').write_text('eula=true\n') - os.environ.update(HORIZON_NO_AUDIO='1', HORIZON_API_DISCOVERY=discovery) + os.environ.update(lab.game_env(HORIZON_API_DISCOVERY=discovery)) server = Server(command, directory, f'{label}-online') checks = [] starts = [] diff --git a/tools/test_save_roundtrip_parity.py b/tools/test_save_roundtrip_parity.py index bd8b07a29..732429881 100644 --- a/tools/test_save_roundtrip_parity.py +++ b/tools/test_save_roundtrip_parity.py @@ -330,7 +330,7 @@ def entity_uuid_text(i): def vscreen_env(): text = subprocess.run(['vscreen', 'env'], capture_output=True, text=True, check=True).stdout - env = dict(os.environ) + env = lab.game_env() env.pop('DISPLAY', None) for match in re.finditer(r'(\w+)=(\S+)', text.split('unset')[0]): env[match.group(1)] = match.group(2) @@ -346,7 +346,7 @@ def horizon_resave(source, destination, bin_dir): ctl = [str(bin_dir / 'horizon-ctl'), '--socket', str(socket)] extra = [] if sys.platform == 'darwin': - env = dict(os.environ, HORIZON_NO_AUDIO='1') + env = lab.game_env() extra = ['--offscreen'] else: alsa = destination / 'null-asound.conf' diff --git a/tools/test_sculk_parity.py b/tools/test_sculk_parity.py index b9c7417a5..ae2c1c6aa 100644 --- a/tools/test_sculk_parity.py +++ b/tools/test_sculk_parity.py @@ -339,7 +339,7 @@ def main(): oracle_harness.run_logged(['cargo', 'build', '--locked', '--release', '-p', 'horizon-client', '--bin', 'horizon-client'], build, 'compile-rust') oracle_harness.run_logged([lab.cargo_target() / 'release/horizon-client', '--resources', resources], build, 'horizon', - env={**os.environ, 'HORIZON_SCULK_FIXTURE': f'{inputs},{build / "horizon.txt"}', 'HORIZON_NO_AUDIO': '1'}) + env=lab.game_env(HORIZON_SCULK_FIXTURE=f'{inputs},{build / "horizon.txt"}')) if args.java_only: return names = {s[0] for _, s in chosen} diff --git a/tools/test_secure_chat_parity.py b/tools/test_secure_chat_parity.py index c6c76a892..b9062568a 100644 --- a/tools/test_secure_chat_parity.py +++ b/tools/test_secure_chat_parity.py @@ -264,7 +264,7 @@ def main(): oh.run_logged([*java, 'keys', build], build, 'keys') services = Services(json.loads((build / 'publickeys.json').read_text())) discovery = services.url + '/minecraft/client' - os.environ.update(HORIZON_NO_AUDIO='1', HORIZON_API_DISCOVERY=discovery) + os.environ.update(lab.game_env(HORIZON_API_DISCOVERY=discovery)) servers = { 'vanilla': ['java', '-Xmx2g', '-Duser.timezone=UTC', f'-Dminecraft.api.discovery.host={discovery}', '-cp', dependencies, 'net.minecraft.server.Main', 'nogui'], diff --git a/tools/test_use_item_on_parity.py b/tools/test_use_item_on_parity.py index d6e5f899f..89b463116 100644 --- a/tools/test_use_item_on_parity.py +++ b/tools/test_use_item_on_parity.py @@ -375,7 +375,7 @@ def main(): d.run(['cargo', 'build', '-q', '--release', '-p', 'horizon-client', '--bin', 'horizon-client'], cwd=lab.ROOT, env={**os.environ, 'CARGO_BUILD_JOBS': os.environ.get('CARGO_BUILD_JOBS', '3')}) d.run([str(lab.cargo_target() / 'release/horizon-client'), '--resources', str(resources)], cwd=work, - env={**os.environ, 'HORIZON_DISPENSE_FIXTURE': f'{work / "cases.jsonl"},{work / "rust.txt"}', 'HORIZON_NO_AUDIO': '1'}) + env=lab.game_env(HORIZON_DISPENSE_FIXTURE=f'{work / "cases.jsonl"},{work / "rust.txt"}')) ignore = tuple(prefix + ' ' for prefix in args.ignore.split(',') if prefix) + ('D ',) native = h.split_cases((work / 'native.txt').read_text(), ignore) rust = h.split_cases((work / 'rust.txt').read_text(), ignore) diff --git a/tools/test_world_reopen_parity.py b/tools/test_world_reopen_parity.py index 381927013..4ef9a5e87 100644 --- a/tools/test_world_reopen_parity.py +++ b/tools/test_world_reopen_parity.py @@ -24,6 +24,9 @@ import sys import tempfile from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import lab # noqa: E402 + ROOT = Path(__file__).resolve().parent.parent RESOURCES = ROOT / '.local' / 'horizon-client' / '26.3' RUN = ROOT / '.local' / 'world-reopen' @@ -32,7 +35,7 @@ CYCLES = 3 def game_env(): """macOS renders offscreen; Linux uses the virtual screen (as test_save_roundtrip_parity).""" - env = dict(os.environ, HORIZON_NO_AUDIO='1') + env = lab.game_env() if sys.platform == 'darwin': return env, ['--offscreen'] subprocess.run(['vscreen', 'up'], check=False, capture_output=True) diff --git a/tools/test_world_spawn_parity.py b/tools/test_world_spawn_parity.py index 5daa38a45..00c390005 100644 --- a/tools/test_world_spawn_parity.py +++ b/tools/test_world_spawn_parity.py @@ -140,7 +140,7 @@ def main(): # One worker thread: the chunks decorated in a fixed order (see above). vanilla_server = ['java', '-Xmx2g', '-Dmax.bg.threads=1', '-cp', dependencies, 'net.minecraft.server.Main', 'nogui'] horizon_server = [binary, '--resources', RESOURCES, '--dedicated', '--nogui'] - os.environ['HORIZON_NO_AUDIO'] = '1' + os.environ.update(lab.game_env()) def one(seed): t0 = time.time() From 9223b401e7bb8864711e1f05c891b9d3187bbadb Mon Sep 17 00:00:00 2001 From: some-du6e <103129948+some-du6e@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:20:45 -0600 Subject: [PATCH 03/51] Decode every item data component a server sends Horizon decoded 25 of 26.3's 122 data component types. Item data has no length prefix, so a stack with any other component (an armour trim, banner patterns, pot sherds, a stew's effects, a plugin's item) made the whole packet undecodable and it was dropped: on a Paper server the inventory arrived empty until each slot was clicked, the advancements screen said "There doesn't seem to be anything here..." and the recipe book was empty. Every other component is now read to its end with its vanilla StreamCodec (item/skip.rs) and kept as its bytes, written back as they came; components without a network codec go as their NBT. The client leaves them out of the item's components, so the item keeps its prototype's values for them. tools/test_item_wire_parity.py checks it against vanilla: the oracle encodes 374 stacks with ItemStack.OPTIONAL_STREAM_CODEC (every default component value of every item, SNBT fixtures for the rest, direct holders and removals built in Java), covering all 122 component types, and the Rust probe must consume every byte, find the same component ids and encode the read-past ones back to the same bytes. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game/remote.rs | 2 + crates/horizon-protocol/Cargo.toml | 4 + .../examples/item_wire_probe.rs | 64 +++ crates/horizon-protocol/src/item.rs | 45 +- crates/horizon-protocol/src/item/skip.rs | 427 ++++++++++++++++++ .../parity/VanillaItemWireOracle.java | 155 +++++++ tools/test_item_wire_parity.py | 177 ++++++++ 7 files changed, 871 insertions(+), 3 deletions(-) create mode 100644 crates/horizon-protocol/examples/item_wire_probe.rs create mode 100644 crates/horizon-protocol/src/item/skip.rs create mode 100644 mods/horizon/src/test/java/dev/horizonlab/parity/VanillaItemWireOracle.java create mode 100755 tools/test_item_wire_parity.py diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index da0def0fb..0e6127178 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -596,6 +596,8 @@ impl Game<'_> { continue; }; let value = match c { + // Read past, not decoded: the item keeps its prototype's value. + Component::Raw(_) => continue, Component::Int(v) if *kind == component::RARITY => { let rarity = ["common", "uncommon", "rare", "epic"]; Tag::String(rarity.get(*v as usize).copied().unwrap_or("common").into()) diff --git a/crates/horizon-protocol/Cargo.toml b/crates/horizon-protocol/Cargo.toml index 71b93a1fe..6467ed849 100644 --- a/crates/horizon-protocol/Cargo.toml +++ b/crates/horizon-protocol/Cargo.toml @@ -22,5 +22,9 @@ sha2 = "0.10" name = "protocol_probe" path = "examples/probe.rs" +[[example]] +name = "item_wire_probe" +path = "examples/item_wire_probe.rs" + [lints] workspace = true diff --git a/crates/horizon-protocol/examples/item_wire_probe.rs b/crates/horizon-protocol/examples/item_wire_probe.rs new file mode 100644 index 000000000..19760ca1a --- /dev/null +++ b/crates/horizon-protocol/examples/item_wire_probe.rs @@ -0,0 +1,64 @@ +//! The item wire harness's Rust side (tools/test_item_wire_parity.py): reads +//! the oracle's stacks (JSON lines with `label` and `hex`), decodes each with +//! `item::Slot` and prints, per stack, whether every byte was consumed, the +//! patch's component ids (added, then removed), which of them Horizon +//! decoded into its own types (the rest it reads past, keeping their bytes) +//! and whether encoding the decoded stack gives the same bytes back. +//! +//! cargo run -p horizon-protocol --example item_wire_probe -- vanilla.jsonl rust.jsonl +use horizon_protocol::{ + buf::{Codec, Reader, Writer}, + item::{Component, Slot}, +}; +use serde_json::{Value, json}; +use std::io::Write; + +fn hex(s: &str) -> Option> { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(s.get(i..i + 2)?, 16).ok()) + .collect() +} + +fn main() -> Result<(), Box> { + let args: Vec = std::env::args().collect(); + let input = std::fs::read_to_string(args.get(1).ok_or("usage: item_wire_probe IN OUT")?)?; + let mut out = std::io::BufWriter::new(std::fs::File::create( + args.get(2).ok_or("usage: item_wire_probe IN OUT")?, + )?); + for line in input.lines().filter(|l| !l.trim().is_empty()) { + let v: Value = serde_json::from_str(line)?; + let label = v["label"].clone(); + let Some(bytes) = v["hex"].as_str().and_then(hex) else { + continue; + }; + let mut r = Reader::new(&bytes); + let result = match Slot::read(&mut r) { + Err(e) => json!({"label": label, "error": e.to_string()}), + Ok(stack) => { + let mut w = Writer::new(); + Slot::write(&stack, &mut w); + let (added, removed): (Vec, Vec) = + stack.as_ref().map_or_else(Default::default, |s| { + (s.added.iter().map(|(k, _)| *k).collect(), s.removed.clone()) + }); + let typed: Vec = stack + .iter() + .flat_map(|s| &s.added) + .filter(|(_, c)| !matches!(c, Component::Raw(_))) + .map(|(k, _)| *k) + .collect(); + json!({ + "label": label, + "left": r.remaining(), + "added": added, + "removed": removed, + "typed": typed, + "same_bytes": w.data == bytes, + }) + } + }; + writeln!(out, "{result}")?; + } + Ok(()) +} diff --git a/crates/horizon-protocol/src/item.rs b/crates/horizon-protocol/src/item.rs index 650fa4baa..0aa940a38 100644 --- a/crates/horizon-protocol/src/item.rs +++ b/crates/horizon-protocol/src/item.rs @@ -2,11 +2,13 @@ //! item's registry id and a data component patch, plus the hashed form the //! client sends back in container clicks (HashedStack). use crate::{ - Error, Result, + Result, buf::{Codec, Reader, Writer}, }; use horizon_world::nbt::Tag; +mod skip; + /// minecraft:data_component_type registry ids (26.3). pub mod component { pub const CUSTOM_DATA: i32 = 0; @@ -78,6 +80,9 @@ pub enum Component { WrittenBook(WrittenBook), /// `ChargedProjectiles`: a loaded crossbow's ammunition, as templates. Stacks(Vec), + /// A component Horizon reads past without interpreting it: its encoded + /// bytes, written back as they came. + Raw(Vec), } /// `Filterable.streamCodec`: the raw value, then an optional filtered one @@ -224,11 +229,16 @@ impl Component { hidden: (0..n).map(|_| r.varint()).collect::>()?, } } - other => return Err(Error::new(format!("unsupported item component {other}"))), + other => { + let start = r.position(); + skip::component(other, r)?; + Self::Raw(r.since(start).to_vec()) + } }) } fn write(&self, kind: i32, w: &mut Writer) { match self { + Self::Raw(bytes) => w.bytes(bytes), Self::Int(v) if kind == component::DYED_COLOR => w.i32(*v), Self::Int(v) => w.varint(*v), Self::Unit => {} @@ -582,7 +592,8 @@ mod tests { #[test] fn unknown_components_are_errors() { - let bytes = [1, 5, 1, 0, 99]; + // 122 is past DataComponents' last id (cushion_color, 121). + let bytes = [1, 5, 1, 0, 122]; assert!(Slot::read(&mut Reader::new(&bytes)).is_err()); } @@ -631,4 +642,32 @@ mod tests { [("textures".to_owned(), "x".to_owned(), None)] ); } + + #[test] + fn components_read_past_keep_their_bytes() { + // Vanilla 26.3's encoding (tools/test_item_wire_parity.py) of a chestplate's + // armour trim, a banner's patterns (one direct), a pot's sherds and a stew's + // effects: one server's inventory, recipe book and advancements had them. + for (hex, kind) in [ + ("01bd0801003a0502", 58), + ("01890b01004a02200e000b6d696e6563726166743a7801790b", 74), + ("01880301004c01f60801000001be0c0100000001d30c010000", 76), + ("01a408010037020f6412a001", 55), + ] { + let bytes: Vec = (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()) + .collect(); + let mut r = Reader::new(&bytes); + let stack = Slot::read(&mut r).unwrap(); + assert_eq!(r.remaining(), 0, "component {kind}"); + assert!(matches!( + stack.as_ref().and_then(|s| s.component(kind)), + Some(Component::Raw(_)) + )); + let mut w = Writer::new(); + Slot::write(&stack, &mut w); + assert_eq!(w.data, bytes, "component {kind}"); + } + } } diff --git a/crates/horizon-protocol/src/item/skip.rs b/crates/horizon-protocol/src/item/skip.rs new file mode 100644 index 000000000..152d0f0f1 --- /dev/null +++ b/crates/horizon-protocol/src/item/skip.rs @@ -0,0 +1,427 @@ +//! Reading past the data components Horizon doesn't interpret: each +//! function consumes exactly what its vanilla StreamCodec writes +//! (DataComponents' `networkSynchronized` codecs; a component without one +//! goes as its persistent Codec's NBT, `ByteBufCodecs.fromCodecWithRegistries`). +//! Item data has no length prefix, so a component read short or long makes +//! everything after it unreadable. +use super::{Component, Template}; +use crate::{ + Error, Result, + buf::{Codec, Reader}, +}; + +type Read = fn(&mut Reader) -> Result<()>; + +/// DataComponents' registry order (26.3) for the ids `Component::read` +/// doesn't decode itself. +pub(super) fn component(kind: i32, r: &mut Reader) -> Result<()> { + match kind { + // Unit.STREAM_CODEC: glider, waxed. + 34 | 120 => Ok(()), + // Components without a network codec go as their NBT: intangible_projectile, + // map_decorations, debug_stick_state, recipes, lock, container_loot. + 22 | 49 | 59 | 68 | 81 | 82 => nbt(r), + // A VarInt (VAR_INT, an idMapper, a registry or holderRegistry id): damage_type, + // enchantable, additional_trade_cost, block_transformer, villager_food, dye, + // ominous_bottle_amplifier, base_color, the mob variants and colours, the pottery + // pattern and cushion_color. + 8 | 31 | 42 | 43 | 44 | 46 | 65 | 75 | 88..=108 | 110..=117 | 121 => varint(r), + // FLOAT: minimum_attack_charge, potion_duration_scale. + 7 | 54 => f32(r), + // Identifier.STREAM_CODEC: tooltip_style, note_block_sound. + 35 | 73 => ident(r), + // UseEffects: canSprint, interactVibrations, speedMultiplier. + 5 => { + r.bool()?; + r.bool()?; + f32(r) + } + // AdventureModePredicate: can_place_on, can_break. + 14 | 15 => list(r, block_predicate), + 16 => list(r, attribute_modifier), + // FoodProperties.DIRECT_STREAM_CODEC: nutrition, saturation, canAlwaysEat. + 23 => { + varint(r)?; + f32(r)?; + boolean(r) + } + // Consumable: consumeSeconds, animation, sound, hasConsumeParticles, onConsumeEffects. + 24 => { + f32(r)?; + varint(r)?; + sound(r)?; + boolean(r)?; + list(r, consume_effect) + } + // UseRemainder, SulfurCubeContent: one template. + 25 | 80 => template(r), + // UseCooldown: seconds, cooldownGroup. + 26 => { + f32(r)?; + optional(r, ident) + } + // DamageResistant, Repairable, provides_banner_patterns: a holder set. + 27 | 33 | 67 => holder_set(r), + 28 => tool(r), + // Weapon: itemDamagePerAttack, disableBlockingForSeconds. + 29 => { + varint(r)?; + f32(r) + } + 32 => equippable(r), + // DeathProtection: deathEffects. + 36 => list(r, consume_effect), + 37 => blocks_attacks(r), + // PiercingWeapon: dealsKnockback, dismounts, sound, hitSound. + 38 => { + boolean(r)?; + boolean(r)?; + optional(r, sound)?; + optional(r, sound) + } + 39 => kinetic_weapon(r), + // SwingAnimation: type, duration. + 40 | 41 => { + varint(r)?; + varint(r) + } + // BundleContents: templates. + 52 => list(r, template), + 53 => potion_contents(r), + // SuspiciousStewEffects: effect, duration. + 55 => list(r, |r| { + varint(r)?; + varint(r) + }), + // ArmorTrim: material, then Holder (assetId, description, decal). + 58 => { + trim_material(r)?; + holder(r, |r| { + ident(r)?; + nbt(r)?; + boolean(r) + }) + } + // TypedEntityData (entity_data, block_entity_data): the type id, then a compound. + 60 | 62 => { + varint(r)?; + nbt(r) + } + // CustomData: bucket_entity_data. + 61 => nbt(r), + // InstrumentComponent: Holder (soundEvent, useDuration, range, + // durabilityDamage, description). + 63 => holder(r, |r| { + sound(r)?; + f32(r)?; + f32(r)?; + varint(r)?; + nbt(r) + }), + 64 => trim_material(r), + // JukeboxPlayable: Holder (soundEvent, description, lengthInSeconds, + // comparatorOutput). + 66 => holder(r, |r| { + sound(r)?; + nbt(r)?; + f32(r)?; + varint(r) + }), + // LodestoneTracker: target (GlobalPos: dimension, BlockPos), tracked. + 69 => { + optional(r, |r| { + ident(r)?; + r.i64().map(drop) + })?; + boolean(r) + } + 70 => firework_explosion(r), + // Fireworks: flightDuration, explosions. + 71 => { + varint(r)?; + list(r, firework_explosion) + } + // BannerPatternLayers: Holder (assetId, translationKey), colour. + 74 => list(r, |r| { + holder(r, |r| { + ident(r)?; + string(r) + })?; + varint(r) + }), + // PotDecorations: back, left, right, front. + 76 => (0..4).try_for_each(|_| optional(r, template)), + // ItemContainerContents: optional templates. + 77 => list(r, |r| optional(r, template)), + // BlockItemStateProperties: a string to string map. + 78 => list(r, |r| { + string(r)?; + string(r) + }), + // Bees: Occupant (entity type and data, ticksInHive, minTicksInHive). + 79 => list(r, |r| { + varint(r)?; + nbt(r)?; + varint(r)?; + varint(r) + }), + 83 => sound(r), + // Compostable: layers. + 84 => resolvable(r, i32), + // CookingFuel (burnTime, speedMultiplier), BrewingFuel (uses, speedMultiplier). + 85 | 86 => { + resolvable(r, i32)?; + resolvable(r, f32) + } + // MobVisibility: targetingEntityTypes, visibility. + 87 => { + holder_set(r)?; + f32(r) + } + // Holder: width, height, assetId, title, author. + 109 => holder(r, |r| { + varint(r)?; + varint(r)?; + ident(r)?; + optional(r, nbt)?; + optional(r, nbt) + }), + // SignText: four messages, four optional filtered ones, colour, hasGlowingText. + 118 | 119 => { + (0..4).try_for_each(|_| nbt(r))?; + optional(r, |r| (0..4).try_for_each(|_| nbt(r)))?; + varint(r)?; + boolean(r) + } + other => Err(Error::new(format!("unsupported item component {other}"))), + } +} + +fn boolean(r: &mut Reader) -> Result<()> { + r.bool().map(drop) +} +fn varint(r: &mut Reader) -> Result<()> { + r.varint().map(drop) +} +fn f32(r: &mut Reader) -> Result<()> { + r.f32().map(drop) +} +fn i32(r: &mut Reader) -> Result<()> { + r.i32().map(drop) +} +/// ByteBufCodecs.STRING_UTF8. +fn string(r: &mut Reader) -> Result<()> { + r.string(32767).map(drop) +} +/// Identifier.STREAM_CODEC, ResourceKey.streamCodec. +fn ident(r: &mut Reader) -> Result<()> { + string(r) +} +/// A text component or compound (fromCodecWithRegistries, COMPOUND_TAG). +fn nbt(r: &mut Reader) -> Result<()> { + r.nbt().map(drop) +} +fn template(r: &mut Reader) -> Result<()> { + Template::read(r).map(drop) +} +/// ByteBufCodecs.optional: a flag, then the value. +fn optional(r: &mut Reader, value: Read) -> Result<()> { + if r.bool()? { value(r) } else { Ok(()) } +} +/// ByteBufCodecs.list and map: a VarInt count, then the elements (a map's +/// key and value each). Every element is at least a byte, so the count +/// can't pass what is left. +fn list(r: &mut Reader, element: Read) -> Result<()> { + let n = r.count(r.remaining())?; + (0..n).try_for_each(|_| element(r)) +} +/// ByteBufCodecs.holder: 0 and the value itself, or the registry id plus one. +fn holder(r: &mut Reader, direct: Read) -> Result<()> { + if r.varint()? == 0 { direct(r) } else { Ok(()) } +} +/// ByteBufCodecs.holderSet: 0 and a tag id, or the size plus one and the ids. +fn holder_set(r: &mut Reader) -> Result<()> { + match r.varint()? { + 0 => ident(r), + n => { + let n = usize::try_from(n - 1).map_err(|_| Error::new("negative holder set size"))?; + if n > r.remaining() { + return Err(Error::new("holder set past the end")); + } + (0..n).try_for_each(|_| varint(r)) + } + } +} +/// SoundEvent.STREAM_CODEC: a holder of a location and an optional fixed range. +fn sound(r: &mut Reader) -> Result<()> { + holder(r, |r| { + ident(r)?; + optional(r, f32) + }) +} +/// ResolvableInt, ResolvableFloat: Either a constant (true) or a context provider's key. +fn resolvable(r: &mut Reader, constant: Read) -> Result<()> { + if r.bool()? { constant(r) } else { ident(r) } +} +/// TrimMaterial.STREAM_CODEC: a holder of a palette id and a description. +fn trim_material(r: &mut Reader) -> Result<()> { + holder(r, |r| { + ident(r)?; + nbt(r) + }) +} +/// MobEffectInstance: the effect id, then its Details. +fn mob_effect(r: &mut Reader) -> Result<()> { + varint(r)?; + effect_details(r) +} +/// MobEffectInstance.Details: amplifier, duration, ambient, showParticles, showIcon, then the +/// hidden effect's Details, recursively. +fn effect_details(r: &mut Reader) -> Result<()> { + varint(r)?; + varint(r)?; + boolean(r)?; + boolean(r)?; + boolean(r)?; + optional(r, effect_details) +} +/// ConsumeEffect.STREAM_CODEC: the consume_effect_type id, then that type's fields. +fn consume_effect(r: &mut Reader) -> Result<()> { + match r.varint()? { + // apply_effects: effects, probability. + 0 => { + list(r, mob_effect)?; + f32(r) + } + // remove_effects: a mob effect holder set. + 1 => holder_set(r), + // clear_all_effects. + 2 => Ok(()), + // teleport_randomly: diameter, directionalParticles. + 3 => { + f32(r)?; + boolean(r) + } + // play_sound. + 4 => sound(r), + other => Err(Error::new(format!("unknown consume effect type {other}"))), + } +} +/// BlockPredicate: blocks, properties, nbt, then DataComponentMatchers (exact, partial). +fn block_predicate(r: &mut Reader) -> Result<()> { + optional(r, holder_set)?; + // StatePropertiesPredicate: PropertyMatchers, each a name and an + // Either. + optional(r, |r| { + list(r, |r| { + string(r)?; + if r.bool()? { + string(r) + } else { + optional(r, string)?; + optional(r, string) + } + }) + })?; + optional(r, nbt)?; + // DataComponentExactPredicate: TypedDataComponents, a component type id and its value. + list(r, |r| { + let kind = r.varint()?; + Component::read(kind, r).map(drop) + })?; + // DataComponentPredicate.STREAM_CODEC: an Either of a predicate type id or a component + // type id, then the predicate as NBT (fromCodecWithRegistries). + list(r, |r| { + boolean(r)?; + varint(r)?; + nbt(r) + }) +} +/// ItemAttributeModifiers.Entry: attribute, modifier (id, amount, operation), slot, display. +fn attribute_modifier(r: &mut Reader) -> Result<()> { + varint(r)?; + ident(r)?; + r.f64()?; + varint(r)?; + varint(r)?; + // Display.Type: default, hidden, or override and its text. + match r.varint()? { + 0 | 1 => Ok(()), + 2 => nbt(r), + other => Err(Error::new(format!( + "unknown attribute modifier display {other}" + ))), + } +} +/// Tool: rules (blocks, speed, correctForDrops), defaultMiningSpeed, damagePerBlock, +/// canDestroyBlocksInCreative. +fn tool(r: &mut Reader) -> Result<()> { + list(r, |r| { + holder_set(r)?; + optional(r, f32)?; + optional(r, boolean) + })?; + f32(r)?; + varint(r)?; + boolean(r) +} +/// Equippable: slot, equipSound, assetId, cameraOverlay, allowedEntities, dispensable, +/// swappable, damageOnHurt, equipOnInteract, canBeSheared, shearingSound. +fn equippable(r: &mut Reader) -> Result<()> { + varint(r)?; + sound(r)?; + optional(r, ident)?; + optional(r, ident)?; + optional(r, holder_set)?; + (0..5).try_for_each(|_| boolean(r))?; + sound(r) +} +/// BlocksAttacks: blockDelaySeconds, disableCooldownScale, damageReductions (angle, type, +/// base, factor), itemDamage (threshold, base, factor), bypassedBy, blockSound, disableSound. +fn blocks_attacks(r: &mut Reader) -> Result<()> { + f32(r)?; + f32(r)?; + list(r, |r| { + f32(r)?; + optional(r, holder_set)?; + f32(r)?; + f32(r) + })?; + (0..3).try_for_each(|_| f32(r))?; + optional(r, holder_set)?; + optional(r, sound)?; + optional(r, sound) +} +/// KineticWeapon: contactCooldownTicks, delayTicks, the dismount, knockback and damage +/// conditions (maxDurationTicks, minSpeed, minRelativeSpeed), forwardMovement, +/// damageMultiplier, sound, hitSound. +fn kinetic_weapon(r: &mut Reader) -> Result<()> { + varint(r)?; + varint(r)?; + (0..3).try_for_each(|_| { + optional(r, |r| { + varint(r)?; + f32(r)?; + f32(r) + }) + })?; + f32(r)?; + f32(r)?; + optional(r, sound)?; + optional(r, sound) +} +/// PotionContents: potion, customColor, customEffects, customName. +fn potion_contents(r: &mut Reader) -> Result<()> { + optional(r, varint)?; + optional(r, i32)?; + list(r, mob_effect)?; + optional(r, string) +} +/// FireworkExplosion: shape, colors, fadeColors, hasTrail, hasTwinkle. +fn firework_explosion(r: &mut Reader) -> Result<()> { + varint(r)?; + list(r, i32)?; + list(r, i32)?; + boolean(r)?; + boolean(r) +} diff --git a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaItemWireOracle.java b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaItemWireOracle.java new file mode 100644 index 000000000..cc95bb311 --- /dev/null +++ b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaItemWireOracle.java @@ -0,0 +1,155 @@ +package dev.horizonlab.parity; + +import io.netty.buffer.Unpooled; +import it.unimi.dsi.fastutil.objects.Reference2ObjectMap; +import it.unimi.dsi.fastutil.objects.Reference2ObjectMaps; +import java.io.PrintWriter; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Optional; +import net.minecraft.core.Holder; +import net.minecraft.core.component.DataComponentPatch; +import net.minecraft.core.component.DataComponentType; +import net.minecraft.core.component.DataComponents; +import net.minecraft.core.component.TypedDataComponent; +import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.network.RegistryFriendlyByteBuf; +import net.minecraft.network.chat.Component; +import net.minecraft.resources.Identifier; +import net.minecraft.sounds.SoundEvents; +import net.minecraft.world.entity.decoration.painting.PaintingVariant; +import net.minecraft.world.item.JukeboxPlayable; +import net.minecraft.world.item.JukeboxSong; +import net.minecraft.world.item.component.MapPostProcessing; +import net.minecraft.util.Unit; +import net.minecraft.world.item.ItemStack; +import net.minecraft.world.item.Items; + +/** + * Item stacks as a server sends them (ItemStack.OPTIONAL_STREAM_CODEC), for + * every data component: each default component value of every vanilla item + * set on a stick (so it is in the patch), the SNBT fixtures given in a file + * (one `labelstack SNBT` per line), and a few patches built directly. + * Prints one JSON line per stack: its label, the encoded bytes in hex and the + * patch's component type ids in encoding order (added, then removed), or + * the error. + */ +public final class VanillaItemWireOracle { + private static final int PER_TYPE = 12; + + public static void main(String[] args) throws Exception { + ParityData.init(); + List fixtures = new ArrayList<>(); + for (String line : Files.readAllLines(Path.of(args[0]))) { + if (line.isBlank() || line.startsWith("#")) continue; + fixtures.add(line.split("\t", 2)); + } + try (PrintWriter out = new PrintWriter(Files.newBufferedWriter(Path.of(args[1])))) { + // Each distinct default value, once per component type and + // encoding, and at most PER_TYPE of them for one type. + Set seen = new HashSet<>(); + Map, Integer> perType = new HashMap<>(); + for (var item : BuiltInRegistries.ITEM) { + for (TypedDataComponent c : item.components()) { + ItemStack stack = new ItemStack(Items.STICK); + if (!set(stack, c)) continue; + String hex = encode(stack); + if (hex == null || perType.getOrDefault(c.type(), 0) >= PER_TYPE) continue; + if (!seen.add(BuiltInRegistries.DATA_COMPONENT_TYPE.getKey(c.type()) + hex)) continue; + perType.merge(c.type(), 1, Integer::sum); + print(out, "default " + BuiltInRegistries.ITEM.getKey(item) + " " + BuiltInRegistries.DATA_COMPONENT_TYPE.getKey(c.type()), stack); + } + } + for (String[] f : fixtures) { + try { + print(out, f[0], ParityData.stack(f[1])); + } catch (Exception e) { + out.println("{\"label\":" + json(f[0]) + ",\"error\":" + json(String.valueOf(e)) + "}"); + } + } + // Components without a persistent codec, direct holders item SNBT + // can't name, and removals. + ItemStack direct = new ItemStack(Items.STICK); + direct.set(DataComponents.ADDITIONAL_TRADE_COST, 4); + direct.set(DataComponents.MAP_POST_PROCESSING, MapPostProcessing.SCALE); + direct.set(DataComponents.JUKEBOX_PLAYABLE, new JukeboxPlayable(Holder.direct( + new JukeboxSong(SoundEvents.ARMOR_EQUIP_CHAIN, Component.literal("S"), 10f, 3)))); + direct.set(DataComponents.PAINTING_VARIANT, Holder.direct(new PaintingVariant( + 2, 1, Identifier.withDefaultNamespace("kebab"), Optional.of(Component.literal("T")), Optional.empty()))); + print(out, "non-persistent and direct", direct); + ItemStack locked = new ItemStack(Items.STONE); + locked.set(DataComponents.CREATIVE_SLOT_LOCK, Unit.INSTANCE); + print(out, "creative_slot_lock", locked); + ItemStack removed = new ItemStack(Items.APPLE, 3); + removed.remove(DataComponents.FOOD); + removed.remove(DataComponents.CONSUMABLE); + removed.set(DataComponents.DAMAGE, 4); + print(out, "removed food and consumable", removed); + print(out, "empty", ItemStack.EMPTY); + } + } + + @SuppressWarnings("unchecked") + private static boolean set(ItemStack stack, TypedDataComponent c) { + if (stack.getPrototype().has(c.type()) && java.util.Objects.equals(stack.getPrototype().get(c.type()), c.value())) return false; + stack.set((DataComponentType) c.type(), c.value()); + return true; + } + + private static String encode(ItemStack stack) { + var buf = new RegistryFriendlyByteBuf(Unpooled.buffer(), ParityData.access); + try { + ItemStack.OPTIONAL_STREAM_CODEC.encode(buf, stack); + } catch (RuntimeException e) { + return null; + } + byte[] bytes = new byte[buf.readableBytes()]; + buf.readBytes(bytes); + return HexFormat.of().formatHex(bytes); + } + + private static void print(PrintWriter out, String label, ItemStack stack) { + String hex = encode(stack); + if (hex == null) { + out.println("{\"label\":" + json(label) + ",\"error\":\"encode failed\"}"); + return; + } + // DataComponentPatch's encoding order: its map's fast iteration, added + // components, then removals. + StringBuilder kinds = new StringBuilder(); + StringBuilder removed = new StringBuilder(); + for (var e : Reference2ObjectMaps.fastIterable(patchMap(stack.getComponentsPatch()))) { + // Removed (package-private) marks a removal. + StringBuilder to = e.getValue().getClass().getName().equals("net.minecraft.core.component.Removed") ? removed : kinds; + to.append(to.isEmpty() ? "" : ",").append(id(e.getKey())); + } + out.println("{\"label\":" + json(label) + ",\"hex\":\"" + hex + "\",\"added\":[" + kinds + "],\"removed\":[" + removed + "]}"); + } + + @SuppressWarnings("unchecked") + private static Reference2ObjectMap, Object> patchMap(DataComponentPatch patch) { + try { + Field map = DataComponentPatch.class.getDeclaredField("map"); + map.setAccessible(true); + return (Reference2ObjectMap, Object>) map.get(patch); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + } + + private static int id(DataComponentType type) { + return BuiltInRegistries.DATA_COMPONENT_TYPE.getId(type); + } + + private static String json(String s) { + return "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n") + "\""; + } +} diff --git a/tools/test_item_wire_parity.py b/tools/test_item_wire_parity.py new file mode 100755 index 000000000..1be6042ec --- /dev/null +++ b/tools/test_item_wire_parity.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Item stacks on the wire: every data component, as vanilla 26.3 encodes it. + +VanillaItemWireOracle encodes stacks with ItemStack.OPTIONAL_STREAM_CODEC +(what a server sends in inventories, recipe books and advancements): each +default component value of every vanilla item, the SNBT fixtures below +(direct holders, tags and lists, optional fields, every consume effect, +block predicates with component matchers, the mob variants) and patches +built in Java (components without a persistent codec, removals). The Rust +probe decodes each with horizon_protocol::item::Slot and must consume every +byte and find the same component ids in the same order; a stack whose +components Horizon only reads past must also encode back to the same bytes. +Every id in DataComponents must be covered.""" +import argparse +import json +import os +import platform +import subprocess +from pathlib import Path +try: + from . import lab, test_rust_parity as base +except ImportError: + import lab + import test_rust_parity as base + +COMPONENT_TYPES = 122 + +FIXTURES = [ + ('custom_data', '{custom_data:{a:1,b:"x",c:[1,2]}}'), + ('unbreakable', '{unbreakable:{}}'), + ('custom_name', '{custom_name:{text:"Hi",color:"red",bold:1b}}'), + ('lore', '{lore:["a",{text:"b",italic:0b}]}'), + ('enchantments', '{enchantments:{"minecraft:sharpness":5,"minecraft:unbreaking":3}}'), + ('can_break tag', '{can_break:{blocks:"#minecraft:logs"}}'), + ('can_break list state nbt', '{can_break:[{blocks:["minecraft:stone","minecraft:dirt"],state:{snowy:"true",age:{min:"1",max:"3"},level:{min:"2"}},nbt:"{a:1b}"}]}'), + ('can_place_on components', '{can_place_on:{blocks:"minecraft:chest",components:{"minecraft:custom_name":"x","minecraft:damage":3},predicates:{"minecraft:custom_data":{a:1},"minecraft:damage":{durability:{min:1}}}}}'), + ('can_place_on empty', '{can_place_on:[{}]}'), + ('custom_model_data', '{custom_model_data:{floats:[1.0f,2.5f],flags:[1b,0b],strings:["a"],colors:[16711680]}}'), + ('tooltip_display', '{tooltip_display:{hide_tooltip:1b,hidden_components:["minecraft:enchantments","minecraft:damage"]}}'), + ('repair_cost', '{repair_cost:7}'), + ('intangible_projectile', '{intangible_projectile:{}}'), + ('tooltip_style', '{tooltip_style:"minecraft:fancy"}'), + ('interact_animation', '{interact_animation:{type:"stab",duration:9}}'), + ('dyed_color', '{dyed_color:16711935}'), + ('map_id', '{map_id:42}'), + ('written_book_content', '{written_book_content:{title:{raw:"T",filtered:"F"},author:"me",generation:1,pages:[{raw:"p1",filtered:"q1"},"p2"],resolved:1b}}'), + ('trim registry', '{trim:{material:"minecraft:gold",pattern:"minecraft:coast"}}'), + ('trim direct', '{trim:{material:{palette_id:"minecraft:x",description:"X"},pattern:{asset_id:"minecraft:y",description:"Y",decal:1b}}}'), + ('block_entity_data', '{block_entity_data:{id:"minecraft:chest",Items:[]}}'), + ('lodestone_tracker target', '{lodestone_tracker:{target:{dimension:"minecraft:the_nether",pos:[1,-64,3]},tracked:0b}}'), + ('lodestone_tracker empty', '{lodestone_tracker:{}}'), + ('firework_explosion', '{firework_explosion:{shape:"star",colors:[1,2],fade_colors:[3],has_trail:1b,has_twinkle:0b}}'), + ('profile name', '{profile:{name:"Notch"}}'), + ('profile full', '{profile:{name:"jeb_",id:[I;1,2,3,4],properties:[{name:"textures",value:"abc",signature:"sig"}]}}'), + ('note_block_sound', '{note_block_sound:"minecraft:block.note_block.bell"}'), + ('base_color', '{base_color:"lime"}'), + ('sulfur_cube_content', '{sulfur_cube_content:{id:"minecraft:stone",count:1}}'), + ('lock', '{lock:{items:"minecraft:diamond"}}'), + ('container_loot', '{container_loot:{loot_table:"minecraft:chests/simple_dungeon",seed:5L}}'), + ('waxed', '{waxed:{}}'), + ('banner_patterns', '{banner_patterns:[{pattern:"minecraft:stripe_bottom",color:"red"},{pattern:{asset_id:"minecraft:x",translation_key:"y"},color:"blue"}]}'), + ('pot_decorations', '{pot_decorations:{back:{id:"minecraft:brick"},left:{id:"minecraft:angler_pottery_sherd"},front:{id:"minecraft:skull_pottery_sherd",count:1}}}'), + ('suspicious_stew_effects', '{suspicious_stew_effects:[{id:"minecraft:night_vision",duration:100},{id:"minecraft:poison"}]}'), + ('potion_contents full', '{potion_contents:{potion:"minecraft:swiftness",custom_color:123,custom_effects:[{id:"minecraft:speed",amplifier:1b,duration:200,ambient:1b,show_particles:0b,show_icon:1b,hidden_effect:{id:"minecraft:speed",duration:100,hidden_effect:{id:"minecraft:speed",duration:50}}}],custom_name:"x"}}'), + ('potion_contents bare', '{potion_contents:{}}'), + ('container', '{container:[{slot:0,item:{id:"minecraft:stone",count:3}},{slot:5,item:{id:"minecraft:diamond_sword",components:{"minecraft:damage":5,"minecraft:trim":{material:"minecraft:gold",pattern:"minecraft:coast"}}}}]}'), + ('bundle_contents', '{bundle_contents:[{id:"minecraft:stone",count:2},{id:"minecraft:potion",components:{"minecraft:potion_contents":{potion:"minecraft:water"}}}]}'), + ('charged_projectiles', '{charged_projectiles:[{id:"minecraft:arrow"},{id:"minecraft:firework_rocket",components:{"minecraft:fireworks":{flight_duration:2b,explosions:[{shape:"burst",colors:[5]}]}}}]}'), + ('block_state', '{block_state:{facing:"north",half:"top"}}'), + ('bees', '{bees:[{entity_data:{id:"minecraft:bee",Health:5f},ticks_in_hive:5,min_ticks_in_hive:10}]}'), + ('entity_data', '{entity_data:{id:"minecraft:pig",NoAI:1b}}'), + ('bucket_entity_data', '{bucket_entity_data:{Health:5f}}'), + ('instrument direct', '{instrument:{sound_event:{sound_id:"minecraft:x",range:8f},use_duration:3f,range:64f,description:"Horn"}}'), + ('painting variant', '{"minecraft:painting/variant":"minecraft:kebab"}'), + ('sign_text_front', '{sign_text_front:{messages:["a","b","c","d"],filtered_messages:["e","f","g","h"],color:"red",has_glowing_text:1b}}'), + ('sign_text_back', '{sign_text_back:{messages:["a","","",""]}}'), + ('map_decorations', '{map_decorations:{a:{type:"minecraft:player",x:1.0d,z:2.0d,rotation:0f}}}'), + ('debug_stick_state', '{debug_stick_state:{"minecraft:oak_stairs":"facing"}}'), + ('recipes', '{recipes:["minecraft:stone_bricks"]}'), + ('attribute_modifiers', '{attribute_modifiers:[{type:"minecraft:attack_damage",id:"minecraft:x",amount:1.5d,operation:"add_value",slot:"mainhand",display:{type:"override",value:"hi"}},{type:"minecraft:armor",id:"minecraft:y",amount:2d,operation:"add_multiplied_total",display:{type:"hidden"}}]}'), + ('consumable all effects', '{consumable:{consume_seconds:2f,animation:"drink",sound:{sound_id:"minecraft:x",range:4f},has_consume_particles:0b,on_consume_effects:[{type:"apply_effects",effects:[{id:"minecraft:speed",duration:20}],probability:0.5f},{type:"remove_effects",effects:"minecraft:poison"},{type:"remove_effects",effects:["minecraft:poison","minecraft:wither"]},{type:"clear_all_effects"},{type:"teleport_randomly",diameter:8f},{type:"play_sound",sound:"minecraft:entity.generic.eat"}]}}'), + ('death_protection', '{death_protection:{death_effects:[{type:"clear_all_effects"}]}}'), + ('equippable full', '{equippable:{slot:"head",equip_sound:"minecraft:item.armor.equip_gold",asset_id:"minecraft:gold",camera_overlay:"minecraft:misc/pumpkinblur",allowed_entities:"#minecraft:skeletons",dispensable:0b,swappable:0b,damage_on_hurt:0b,equip_on_interact:1b,can_be_sheared:1b,shearing_sound:{sound_id:"minecraft:y"}}}'), + ('equippable bare', '{equippable:{slot:"chest"}}'), + ('tool rules', '{tool:{rules:[{blocks:"#minecraft:mineable/pickaxe",speed:4f,correct_for_drops:1b},{blocks:["minecraft:stone"]}],default_mining_speed:2f,damage_per_block:2,can_destroy_blocks_in_creative:0b}}'), + ('blocks_attacks full', '{blocks_attacks:{block_delay_seconds:0.5f,disable_cooldown_scale:2f,damage_reductions:[{horizontal_blocking_angle:45f,type:"#minecraft:is_fire",base:1f,factor:0.5f},{base:0f,factor:1f}],item_damage:{threshold:3f,base:1f,factor:2f},bypassed_by:"#minecraft:bypasses_shield",block_sound:"minecraft:item.shield.block",disable_sound:{sound_id:"minecraft:z"}}}'), + ('damage_resistant', '{damage_resistant:{types:"#minecraft:is_fire"}}'), + ('repairable list', '{repairable:{items:["minecraft:diamond","minecraft:emerald"]}}'), + ('use_cooldown group', '{use_cooldown:{seconds:1.5f,cooldown_group:"minecraft:x"}}'), + ('use_remainder', '{use_remainder:{id:"minecraft:bowl",count:2}}'), + ('fireworks', '{fireworks:{flight_duration:3b,explosions:[{shape:"creeper",colors:[1],fade_colors:[2,3],has_trail:1b},{shape:"small_ball"}]}}'), + ('provides_banner_patterns', '{provides_banner_patterns:"#minecraft:pattern_item/globe"}'), + ('wolf variant', '{"minecraft:wolf/variant":"minecraft:ashen","minecraft:wolf/sound_variant":"minecraft:angry","minecraft:wolf/collar":"red"}'), + ('fox salmon parrot', '{"minecraft:fox/variant":"snow","minecraft:salmon/size":"large","minecraft:parrot/variant":"blue"}'), + ('tropical fish', '{"minecraft:tropical_fish/pattern":"flopper","minecraft:tropical_fish/base_color":"white","minecraft:tropical_fish/pattern_color":"black"}'), + ('mooshroom rabbit', '{"minecraft:mooshroom/variant":"brown","minecraft:rabbit/variant":"gold"}'), + ('pig cow chicken', '{"minecraft:pig/variant":"minecraft:cold","minecraft:pig/sound_variant":"minecraft:big","minecraft:cow/variant":"minecraft:warm","minecraft:cow/sound_variant":"minecraft:classic","minecraft:chicken/variant":"minecraft:cold","minecraft:chicken/sound_variant":"minecraft:classic"}'), + ('nautilus frog horse', '{"minecraft:zombie_nautilus/variant":"minecraft:warm","minecraft:frog/variant":"minecraft:cold","minecraft:horse/variant":"chestnut"}'), + ('llama axolotl cat', '{"minecraft:llama/variant":"brown","minecraft:axolotl/variant":"blue","minecraft:cat/variant":"minecraft:tabby","minecraft:cat/sound_variant":"minecraft:classic","minecraft:cat/collar":"blue"}'), + ('sheep shulker villager', '{"minecraft:sheep/color":"pink","minecraft:shulker/color":"purple","minecraft:villager/variant":"minecraft:taiga"}'), + ('trim material direct provider', '{provides_trim_material:{palette_id:"minecraft:x",description:"X"}}'), + ('everything at once', '{damage:3,unbreakable:{},custom_name:"n",enchantments:{"minecraft:mending":1},trim:{material:"minecraft:gold",pattern:"minecraft:coast"},banner_patterns:[{pattern:"minecraft:globe",color:"red"}],pot_decorations:{right:{id:"minecraft:brick"}},lore:["x"],rarity:"epic"}'), +] +# Fixtures on another item than a stick (the components need it, or read better there). +ITEMS = {'profile name': 'player_head', 'profile full': 'player_head', 'trim registry': 'diamond_chestplate', + 'trim direct': 'diamond_chestplate', 'banner_patterns': 'white_banner', 'pot_decorations': 'decorated_pot', + 'everything at once': 'diamond_chestplate'} + + +def compare(vanilla, rust): + """Failures, stacks checked, and the component ids covered.""" + results = {r['label']: r for r in rust} + failures, covered = [], set() + for v in vanilla: + if 'error' in v: + failures.append(f"oracle {v['label']}: {v['error']}") + continue + covered.update(v['added'] + v['removed']) + r = results.get(v['label']) + if r is None or 'error' in r: + failures.append(f"{v['label']}: {r and r['error']}") + elif r['left'] or r['added'] != v['added'] or r['removed'] != v['removed']: + failures.append(f"{v['label']}: {r['left']} bytes left, ids {r['added']}/{r['removed']}, vanilla {v['added']}/{v['removed']}") + elif not r['same_bytes'] and not r['typed']: + failures.append(f"{v['label']}: read past, but encoded back differently") + missing = sorted(set(range(COMPONENT_TYPES)) - covered) + if missing: + failures.append(f'component ids no stack covers: {missing}') + return failures, len(vanilla), covered + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--prepare', action='store_true') + args = parser.parse_args() + build = lab.LOCAL / 'item-wire-parity' + build.mkdir(parents=True, exist_ok=True) + report_path = build / 'report.json' + report_path.unlink(missing_ok=True) + if args.prepare: + lab.prepare() + lock, jars, libraries = base.verify_artifacts() + oracle = lab.ROOT / 'mods/horizon/src/test/java/dev/horizonlab/parity/VanillaItemWireOracle.java' + classes = build / 'classes' + classes.mkdir(exist_ok=True) + + def run(command, name): + with (build / (name + '.log')).open('w', encoding='utf-8') as log: + base.run(list(map(str, command)), stdout=log, stderr=subprocess.STDOUT) + dependency_path = os.pathsep.join(map(str, libraries)) + run(['javac', '--release', lock['minecraft']['java_major'], '-proc:none', '-sourcepath', + lab.ROOT / 'mods/horizon/src/test/java', '-cp', str(jars['server']) + os.pathsep + dependency_path, + '-d', classes, oracle], 'compile-java') + fixtures = build / 'fixtures.tsv' + fixtures.write_text(''.join(f'{label}\t{{id:"minecraft:{ITEMS.get(label, "stick")}",count:1,components:{snbt}}}\n' + for label, snbt in FIXTURES), encoding='utf-8') + vanilla_path, rust_path = build / 'vanilla.jsonl', build / 'rust.jsonl' + run(['java', '-Xmx1G', '-cp', os.pathsep.join([str(classes), str(jars['server']), dependency_path]), + 'dev.horizonlab.parity.VanillaItemWireOracle', fixtures, vanilla_path], 'oracle') + run(['cargo', 'build', '--release', '-p', 'horizon-protocol', '--example', 'item_wire_probe'], 'compile-rust') + binary = lab.cargo_target() / 'release/examples' / ('item_wire_probe.exe' if os.name == 'nt' else 'item_wire_probe') + run([binary, vanilla_path, rust_path], 'rust') + read = lambda path: [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines() if line.strip()] + failures, stacks, covered = compare(read(vanilla_path), read(rust_path)) + for failure in failures[:40]: + print(failure) + if failures: + raise SystemExit(f'{len(failures)} item wire mismatches (of {stacks} stacks)') + lab.write_json(report_path, { + 'schema': 1, 'minecraft': lock['minecraft']['version'], 'platform': platform.platform(), + 'stacks': stacks, 'component_types_covered': len(covered), + 'oracle_sha256': lab.digest(vanilla_path, 'sha256'), 'rust_sha256': lab.digest(rust_path, 'sha256')}) + print(f'Item wire parity passed: {stacks} stacks, all {len(covered)} component types: {report_path}') + + +if __name__ == '__main__': + main() From 7c1dd4981062ce9e0cb0db0b7d919109243b2470 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:13:00 +0200 Subject: [PATCH 04/51] Renderer: leave out solid quads that face away from the eye Meshing sorts each section's and distant tile's solid quads into seven buckets: one per axis direction (all four corners in one axis plane, both triangles wound the same way) and one for everything else, and keeps each axis bucket's extreme plane. A quad facing +X at x = p is a back face from an eye at x <= p, which the solid pipeline culls anyway, so the frame draws only the buckets that can face the eye, merged into runs of adjacent buckets: one to three indirect draws per section on the existing page-run, occlusion-culling and flat paths. The eye comes from the frame's view-projection, so view bobbing is exact. The picture does not change; `horizon-ctl section-sides on|off` switches it for A/B checks. Written from scratch from the general back-face technique; no other renderer's code or Horizon's removed code read. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game/control.rs | 10 + .../horizon-client/src/game/render_control.rs | 1 + crates/horizon-client/src/gfx.rs | 94 ++++- crates/horizon-client/src/lod/mesh.rs | 6 +- crates/horizon-client/src/mesher.rs | 14 + crates/horizon-client/src/mesher/sides.rs | 346 ++++++++++++++++++ 6 files changed, 455 insertions(+), 16 deletions(-) create mode 100644 crates/horizon-client/src/mesher/sides.rs diff --git a/crates/horizon-client/src/game/control.rs b/crates/horizon-client/src/game/control.rs index 205a66074..6fbd984fd 100644 --- a/crates/horizon-client/src/game/control.rs +++ b/crates/horizon-client/src/game/control.rs @@ -226,6 +226,16 @@ impl Game<'_> { } Ok(Some(json!({ "ok": true, "cull": r.section_cull() }))) } + "section-sides" => { + let r = self.renderer.as_mut().ok_or("no renderer")?; + match words.get(1).copied() { + None => {} + Some("on") => r.set_section_sides(true), + Some("off") => r.set_section_sides(false), + Some(other) => return Err(format!("unknown section sides mode {other}").into()), + } + Ok(Some(json!({ "ok": true, "sides": r.section_sides() }))) + } "sculk" => Ok(Some(self.sculk_command(&words)?)), "status" => Ok(Some(self.control_status())), "pad" => Ok(Some(self.pad_command(&words)?)), diff --git a/crates/horizon-client/src/game/render_control.rs b/crates/horizon-client/src/game/render_control.rs index 31a7a33a0..0a2f9b715 100644 --- a/crates/horizon-client/src/game/render_control.rs +++ b/crates/horizon-client/src/game/render_control.rs @@ -13,6 +13,7 @@ pub const HELP: &[&str] = &[ "signedit X Y Z [front|back] open the sign editor for the sign at X Y Z", "section-draws [flat|indirect] how terrain sections are drawn (A/B checks); replies the mode", "section-cull [on|off] occlusion culling of solid sections (A/B checks); replies the mode", + "section-sides [on|off] leave out solid quads facing away from the eye (A/B checks); replies the mode", ]; impl Game<'_> { diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 9ed0bd8a5..e4c3367f9 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -6,7 +6,7 @@ use crate::{ environment::Sky, graphics::GraphicsApi, hud::{HudBatch, HudRenderer}, - mesher::{SectionMesh, TerrainVertex, Vertex}, + mesher::{SectionMesh, Sides, TerrainVertex, Vertex}, sprites::SpriteAnimation, }; use glam::{DVec3, Mat4, Vec3, Vec4}; @@ -156,6 +156,8 @@ struct SectionGpu { page: u32, start: u64, solid: u32, + /// Where the solid quads' buckets by facing side end (`mesher::Sides`). + sides: Sides, translucent: u32, sort: translucency::SortState, sorted_indices: Option<(u32, u64)>, @@ -323,6 +325,8 @@ pub struct Renderer { /// counts reach a multi-draw's entries (Vulkan, Metal). cull: Option, cull_on: bool, + /// Solid quads facing away from the eye are left out (`mesher::Sides`). + sides_on: bool, sky: wgpu::RenderPipeline, dark_disc: wgpu::RenderPipeline, lines: wgpu::RenderPipeline, @@ -2202,6 +2206,7 @@ impl Renderer { translucent, flat_on: flat.is_some(), cull_on: cull.is_some(), + sides_on: true, cull, flat, sky, @@ -2451,6 +2456,14 @@ impl Renderer { self.flat_on = on; Ok(()) } + /// Whether solid quads facing away from the eye are left out. + pub fn section_sides(&self) -> bool { + self.sides_on + } + /// Switches leaving out solid quads facing away, for A/B checks. + pub fn set_section_sides(&mut self, on: bool) { + self.sides_on = on; + } /// Whether solid sections are occlusion culled (`cull`). pub fn section_cull(&self) -> bool { self.cull_on @@ -2573,6 +2586,11 @@ impl Renderer { page, start, solid, + sides: if mesh.sides.quads() == solid { + mesh.sides + } else { + Sides::unsorted(solid) + }, translucent, sort: translucency::SortState::new( mesh.translucent @@ -3039,6 +3057,9 @@ impl Renderer { }; let camera = frame.camera; let camera_section = camera_section(camera); + // Solid quads facing away from the eye are left out: the bob moves + // the eye off `camera`, so it is found from the projection. + let eye = projection_center(view_proj).filter(|_| self.sides_on); let culling = !frame.title && frame.cave_culling; if culling { crate::spans::time("occlusion", || { @@ -3127,11 +3148,13 @@ impl Renderer { }; let start = draws.len(); let instances = u32::from(first_phase); - let indices = s.solid * 6; - if indices > 0 { - draws.push([indices, instances, 0, s.start as u32, i as u32]); + let mut indices = 0; + facing_runs(&s.sides, eye, v.origin, |first, quads| { + indices += quads * 6; + let base = s.start as u32 + first * 4; + draws.push([quads * 6, instances, 0, base, i as u32]); pages.push((s.page, None)); - } + }); let runs = if first_phase { &mut self.solid_runs } else { @@ -3168,16 +3191,12 @@ impl Renderer { for (i, v) in visible_lods.iter().enumerate() { let s = &self.lods[&v.key].mesh; let start = self.draws.len(); - if s.solid > 0 { - self.draws.push([ - s.solid * 6, - 1, - 0, - s.start as u32, - (visible.len() + i) as u32, - ]); + let instance = (visible.len() + i) as u32; + facing_runs(&s.sides, eye, v.origin, |first, quads| { + let base = s.start as u32 + first * 4; + self.draws.push([quads * 6, 1, 0, base, instance]); self.draw_pages.push((s.page, None)); - } + }); self.solid_runs .push(v.distance, s.page, start, self.draws.len() - start, [0; 3]); } @@ -4196,6 +4215,34 @@ fn world_projection(fov_degrees: f32, aspect: f32, far: f32) -> Mat4 { Mat4::perspective_rh(fov_degrees.to_radians(), aspect, far, 0.05) } +/// Where a perspective `view_proj` sees from, in its own (camera-relative) +/// space: the point whose clip x, y and w are all 0. `None` for a +/// projection without one (orthographic). +fn projection_center(view_proj: Mat4) -> Option { + let m = view_proj.as_dmat4(); + let rows = [m.row(0), m.row(1), m.row(3)]; + let a = glam::DMat3::from_cols( + DVec3::new(rows[0].x, rows[1].x, rows[2].x), + DVec3::new(rows[0].y, rows[1].y, rows[2].y), + DVec3::new(rows[0].z, rows[1].z, rows[2].z), + ); + let det = a.determinant(); + if !det.is_normal() { + return None; + } + let b = DVec3::new(rows[0].w, rows[1].w, rows[2].w); + Some((a.inverse() * -b).as_vec3()) +} +/// Calls `draw(first, quads)` for each run of a mesh's solid quads that can +/// face `eye` (`projection_center`), the mesh at the camera-relative +/// `origin`; all of them when there is no eye. +fn facing_runs(sides: &Sides, eye: Option, origin: [f32; 3], mut draw: impl FnMut(u32, u32)) { + match eye { + Some(eye) => sides.visible((eye - Vec3::from(origin)).to_array(), draw), + None if sides.quads() > 0 => draw(0, sides.quads()), + None => {} + } +} fn frustum(m: Mat4) -> [Vec4; 6] { let r = |i| m.row(i); let planes = [ @@ -4423,6 +4470,25 @@ mod tests { assert_eq!(super::outline_width(3840), 5.0); } #[test] + fn the_projection_center_follows_the_view_bob() { + use glam::{Mat4, Vec3}; + let projection = super::world_projection(70.0, 16.0 / 9.0, 512.0); + let view = crate::camera::view_matrix(30.0, 20.0); + let eye = super::projection_center(projection * view).unwrap(); + assert!(eye.length() < 1e-5, "{eye}"); + // A pose translated by the bob: the eye is where the pose's inverse + // takes the view's origin. + let bob = Mat4::from_rotation_z(0.02) * Mat4::from_translation(Vec3::new(0.03, -0.08, 0.0)); + let expected = (bob * view).inverse().transform_point3(Vec3::ZERO); + let eye = super::projection_center(projection * bob * view).unwrap(); + assert!(eye.distance(expected) < 1e-5, "{eye} {expected}"); + assert!(expected.length() > 0.05); + assert!( + super::projection_center(Mat4::orthographic_rh(-1.0, 1.0, -1.0, 1.0, 0.0, 1.0)) + .is_none() + ); + } + #[test] fn indirect_draws_are_validated_only_in_debug_builds_and_on_directx_12() { let validated = |backends| { super::instance_descriptor(backends) diff --git a/crates/horizon-client/src/lod/mesh.rs b/crates/horizon-client/src/lod/mesh.rs index 852f45e5b..bdd77fbea 100644 --- a/crates/horizon-client/src/lod/mesh.rs +++ b/crates/horizon-client/src/lod/mesh.rs @@ -67,7 +67,7 @@ impl Mesher { neighbors: &[Vec; 4], finer: &[TileKey], ) -> SectionMesh { - mesh_columns_with_edges( + let mut mesh = mesh_columns_with_edges( key, columns, covered, @@ -75,7 +75,9 @@ impl Mesher { neighbors, finer, |span, direction| self.material(span, direction, context), - ) + ); + mesh.sort_solid(); + mesh } fn material(&mut self, span: &Span, direction: Direction, context: &WorkerContext) -> Material { let styles = *self diff --git a/crates/horizon-client/src/mesher.rs b/crates/horizon-client/src/mesher.rs index ec6f2ae73..d1b27bec9 100644 --- a/crates/horizon-client/src/mesher.rs +++ b/crates/horizon-client/src/mesher.rs @@ -5,6 +5,7 @@ //! follow `FluidRenderer` (see `fluid`). mod fluid; mod lighting; +mod sides; use crate::{ assets::{Assets, RenderAssets}, biome::Biomes, @@ -20,6 +21,7 @@ use horizon_world::{ fluid::{FluidKind, FluidState}, registry::BlockStateId, }; +pub use sides::Sides; use std::sync::Arc; /// An entity, item and effect vertex (the entity pipelines), with a face @@ -114,6 +116,9 @@ mod vertex_tests { } pub struct SectionMesh { pub solid: Vec, + /// The buckets `solid` is sorted in by the side its quads face + /// (`sort_solid`; empty until then). + pub sides: Sides, pub translucent: Vec, /// VisGraph: bit `a * 6 + b` is set when faces `a` and `b` (Direction /// order) are connected through non-opaque blocks. @@ -127,12 +132,20 @@ impl Default for SectionMesh { fn default() -> Self { Self { solid: Vec::new(), + sides: Sides::default(), translucent: Vec::new(), visibility: ALL_VISIBLE, block_entities: Vec::new(), } } } +impl SectionMesh { + /// Sorts the solid quads by the side they face, for the renderer to + /// leave out those facing away from the eye (`sides`). + pub fn sort_solid(&mut self) { + self.sides = sides::sort(&mut self.solid); + } +} pub const ALL_VISIBLE: u64 = (1 << 36) - 1; /// VisGraph.resolve: flood fill non-opaque cells and connect the faces each region touches. fn visibility(pad: &Padded, info: &[Info]) -> u64 { @@ -798,6 +811,7 @@ impl Mesher<'_> { } } } + out.sort_solid(); out } /// `BlockColors` tint source `index`, or `None` for an untinted (-1) layer. diff --git a/crates/horizon-client/src/mesher/sides.rs b/crates/horizon-client/src/mesher/sides.rs new file mode 100644 index 000000000..797d60127 --- /dev/null +++ b/crates/horizon-client/src/mesher/sides.rs @@ -0,0 +1,346 @@ +//! A mesh's solid quads sorted by the side they face, so that the renderer +//! can leave out the quads turned away from the eye before the GPU shades +//! their vertices. +//! +//! A quad in the plane `x = p` whose corners go counter-clockwise seen from +//! +X faces +X. From an eye at `x < p` it is seen from behind, and at +//! `x = p` edge on; the solid pipeline culls back faces, so it draws nothing +//! from there. A bucket of such quads draws nothing at all while the eye is +//! at or below the lowest of their planes, and likewise for the other five +//! axis directions. The seventh bucket holds every other quad (sloped, +//! rotated, degenerate, or whose two triangles wind opposite ways): it is +//! drawn from everywhere. +//! +//! The buckets lie one after another in the mesh, in an order chosen so that +//! the buckets an eye can see are usually one or two runs of quads: one draw +//! each. Inside a bucket quads keep their model order, so coplanar quads +//! facing one way (an overlay on its base) still draw in that order. +use super::TerrainVertex; + +/// The buckets in mesh order: +X, +Z, other, +Y, -Z, -X, -Y. Seen from +/// above (+Y, the commonest case), each pair of horizontal sides with a +/// common corner is then one run, or two. +const BUCKETS: [Bucket; 7] = [ + Bucket::Side(0, true), + Bucket::Side(2, true), + Bucket::Other, + Bucket::Side(1, true), + Bucket::Side(2, false), + Bucket::Side(0, false), + Bucket::Side(1, false), +]; +const OTHER: usize = 2; +/// How far, in blocks, the eye must be behind a bucket's planes before the +/// bucket is left out. The renderer computes both the eye and the section +/// origins in `f32`, with errors far below this. +const MARGIN: f32 = 1.0 / 16.0; + +#[derive(Clone, Copy)] +enum Bucket { + /// Quads in a plane across `axis` facing its positive (`true`) or + /// negative direction. + Side(usize, bool), + Other, +} + +/// Where each bucket of a mesh's solid quads ends, and the planes that tell +/// whether it can face the eye. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Sides { + /// The solid quads up to the end of each bucket, in `BUCKETS` order. + ends: [u32; 7], + /// Per bucket facing a positive direction, its lowest plane; facing a + /// negative one, its highest (unused for the other bucket). + planes: [f32; 7], +} + +impl Sides { + /// `quads` solid quads in one bucket drawn from everywhere: for meshes + /// that were not sorted. + pub fn unsorted(quads: u32) -> Self { + let mut ends = [0; 7]; + ends[OTHER..].fill(quads); + Self { + ends, + planes: [0.0; 7], + } + } + + /// The number of solid quads the buckets hold. + pub fn quads(&self) -> u32 { + self.ends[6] + } + + /// Calls `draw(first, count)` for each run of quads an eye at `eye` + /// (relative to the mesh's origin) can see the front of, nearest the + /// start first. Empty buckets do not break a run. + pub fn visible(&self, eye: [f32; 3], mut draw: impl FnMut(u32, u32)) { + let mut start = 0; + let mut run: Option = None; + for (i, bucket) in BUCKETS.iter().enumerate() { + let end = self.ends[i]; + if end == start { + continue; + } + let seen = match *bucket { + Bucket::Side(axis, true) => eye[axis] > self.planes[i] - MARGIN, + Bucket::Side(axis, false) => eye[axis] < self.planes[i] + MARGIN, + Bucket::Other => true, + }; + match (seen, run) { + (true, None) => run = Some(start), + (false, Some(first)) => { + draw(first, start - first); + run = None; + } + _ => {} + } + start = end; + } + if let Some(first) = run { + draw(first, start - first); + } + } +} + +/// The bucket of a quad: the axis plane it lies in and the side its +/// triangles (corners 0, 1, 2 and 0, 2, 3, as the renderer draws them) face. +fn bucket(quad: &[TerrainVertex; 4]) -> usize { + let p = quad.map(|v| v.pos); + for axis in 0..3 { + if p.iter().any(|v| v[axis] != p[0][axis]) { + continue; + } + // The normal's component along `axis`, twice each triangle's area. + let (u, w) = ((axis + 1) % 3, (axis + 2) % 3); + let area = |a: usize, b: usize, c: usize| { + (p[b][u] - p[a][u]) * (p[c][w] - p[a][w]) - (p[b][w] - p[a][w]) * (p[c][u] - p[a][u]) + }; + let (first, second) = (area(0, 1, 2), area(0, 2, 3)); + let positive = first >= 0.0 && second >= 0.0; + let negative = first <= 0.0 && second <= 0.0; + if positive == negative { + // Opposite windings, or no area at all. + return OTHER; + } + return BUCKETS + .iter() + .position(|b| matches!(*b, Bucket::Side(a, s) if a == axis && s == positive)) + .expect("every side has a bucket"); + } + OTHER +} + +/// Reorders `solid`'s quads by bucket, keeping their order inside each. +pub fn sort(solid: &mut Vec) -> Sides { + let (quads, rest) = solid.as_chunks::<4>(); + debug_assert!(rest.is_empty(), "solid vertices come in quads"); + let buckets: Vec = quads.iter().map(|q| bucket(q) as u8).collect(); + let mut counts = [0u32; 7]; + let mut planes = [0f32; 7]; + for (quad, &b) in quads.iter().zip(&buckets) { + let b = usize::from(b); + if let Bucket::Side(axis, positive) = BUCKETS[b] { + let p = quad[0].pos[axis]; + planes[b] = match (counts[b], positive) { + (0, _) => p, + (_, true) => planes[b].min(p), + (_, false) => planes[b].max(p), + }; + } + counts[b] += 1; + } + let mut ends = [0u32; 7]; + let mut total = 0; + for (end, count) in ends.iter_mut().zip(counts) { + total += count; + *end = total; + } + if counts.iter().filter(|&&c| c > 0).count() > 1 { + let mut next: [u32; 7] = std::array::from_fn(|b| ends[b] - counts[b]); + let mut sorted = vec![quads[0]; quads.len()]; + for (quad, &b) in quads.iter().zip(&buckets) { + let slot = &mut next[usize::from(b)]; + sorted[*slot as usize] = *quad; + *slot += 1; + } + *solid = sorted.into_flattened(); + } + Sides { ends, planes } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn quad(corners: [[f32; 3]; 4], layer: u16) -> [TerrainVertex; 4] { + corners.map(|p| TerrainVertex::new(p, [0.0; 2], layer, 0, 0)) + } + + /// A unit block's six faces at `origin`, wound counter-clockwise seen + /// from outside, as block models are. + fn cube(origin: [f32; 3], layer: u16) -> Vec { + let [x, y, z] = origin; + let (a, b, c) = (x + 1.0, y + 1.0, z + 1.0); + [ + [[a, y, z], [a, b, z], [a, b, c], [a, y, c]], // +X + [[x, y, z], [x, y, c], [x, b, c], [x, b, z]], // -X + [[x, b, z], [x, b, c], [a, b, c], [a, b, z]], // +Y + [[x, y, z], [a, y, z], [a, y, c], [x, y, c]], // -Y + [[x, y, c], [a, y, c], [a, b, c], [x, b, c]], // +Z + [[x, y, z], [x, b, z], [a, b, z], [a, y, z]], // -Z + ] + .into_iter() + .flat_map(|q| quad(q, layer)) + .collect() + } + + fn runs(sides: &Sides, eye: [f32; 3]) -> Vec<(u32, u32)> { + let mut out = Vec::new(); + sides.visible(eye, |first, count| out.push((first, count))); + out + } + + /// The quads of `solid` whose triangles face `eye`, by their layer. + fn front_facing(solid: &[TerrainVertex], eye: [f32; 3]) -> Vec { + let sub = |a: [f32; 3], b: [f32; 3]| [a[0] - b[0], a[1] - b[1], a[2] - b[2]]; + solid + .as_chunks::<4>() + .0 + .iter() + .filter(|q| { + [[0, 1, 2], [0, 2, 3]].iter().any(|t| { + let (e1, e2) = (sub(q[t[1]].pos, q[t[0]].pos), sub(q[t[2]].pos, q[t[0]].pos)); + let n = [ + e1[1] * e2[2] - e1[2] * e2[1], + e1[2] * e2[0] - e1[0] * e2[2], + e1[0] * e2[1] - e1[1] * e2[0], + ]; + let to_eye = sub(eye, q[t[0]].pos); + n[0] * to_eye[0] + n[1] * to_eye[1] + n[2] * to_eye[2] > 0.0 + }) + }) + .map(|q| q[0].layer) + .collect() + } + + #[test] + fn cube_faces_go_to_their_buckets_in_order() { + let mut solid = cube([3.0, 4.0, 5.0], 0); + for (i, v) in solid.iter_mut().enumerate() { + v.layer = (i / 4) as u16; + } + let sides = sort(&mut solid); + assert_eq!(sides.ends, [1, 2, 2, 3, 4, 5, 6]); + let layers: Vec = solid.iter().step_by(4).map(|v| v.layer).collect(); + // +X, +Z, +Y, -Z, -X, -Y. + assert_eq!(layers, [0, 4, 2, 5, 1, 3]); + assert_eq!(sides.planes[0], 4.0); // +X at x = 4 + assert_eq!(sides.planes[5], 3.0); // -X at x = 3 + assert_eq!(sides.planes[3], 5.0); // +Y at y = 5 + } + + #[test] + fn the_drawn_runs_hold_every_quad_that_faces_the_eye() { + // Blocks across a section, with a sloped quad and a bowtie. + let mut solid = Vec::new(); + let mut layer = 0; + for origin in [ + [0.0, 0.0, 0.0], + [7.0, 3.0, 12.0], + [15.0, 15.0, 15.0], + [2.0, 9.0, 4.0], + ] { + for q in cube(origin, 0).chunks(4) { + solid.extend(q.iter().map(|v| TerrainVertex { layer, ..*v })); + layer += 1; + } + } + let slope = [ + [1.0, 1.0, 1.0], + [2.0, 1.0, 1.0], + [2.0, 2.0, 2.0], + [1.0, 2.0, 2.0], + ]; + let bowtie = [ + [5.0, 1.0, 1.0], + [5.0, 2.0, 2.0], + [5.0, 1.0, 2.0], + [5.0, 2.0, 1.0], + ]; + for corners in [slope, bowtie] { + solid.extend(quad(corners, layer)); + layer += 1; + } + let sides = sort(&mut solid); + assert_eq!(sides.quads(), u32::from(layer)); + let eyes = [-20.0f32, -0.5, 0.0, 3.5, 8.0, 15.99, 16.0, 30.0]; + for &x in &eyes { + for &y in &eyes { + for &z in &eyes { + let eye = [x, y, z]; + let mut drawn = Vec::new(); + for (first, count) in runs(&sides, eye) { + drawn.extend( + solid[first as usize * 4..(first + count) as usize * 4] + .iter() + .step_by(4) + .map(|v| v.layer), + ); + } + for layer in front_facing(&solid, eye) { + assert!(drawn.contains(&layer), "{eye:?} misses quad {layer}"); + } + } + } + } + } + + #[test] + fn runs_merge_across_hidden_and_empty_buckets() { + let mut solid = cube([0.0; 3], 0); + let sides = sort(&mut solid); + // From above, +X and +Z: +X, +Z, (no other), +Y is one run. + assert_eq!(runs(&sides, [8.0, 8.0, 8.0]), [(0, 3)]); + // From below, -X and -Z: -Z, -X, -Y. + assert_eq!(runs(&sides, [-8.0, -8.0, -8.0]), [(3, 3)]); + // From above, -X and +Z: +Z, +Y, then -X. + assert_eq!(runs(&sides, [-8.0, 8.0, 8.0]), [(1, 2), (4, 1)]); + // Inside the block every face is seen from behind. + assert!(runs(&sides, [0.5; 3]).is_empty()); + // Level with its top, the eye may see it: the margin keeps it. + assert_eq!(runs(&sides, [0.5, 1.0, 0.5]), [(2, 1)]); + } + + #[test] + fn unsorted_meshes_draw_everything() { + let sides = Sides::unsorted(9); + assert_eq!(sides.quads(), 9); + assert_eq!(runs(&sides, [-100.0; 3]), [(0, 9)]); + assert!(runs(&Sides::unsorted(0), [0.0; 3]).is_empty()); + } + + #[test] + fn one_bucket_keeps_the_mesh_as_it_is() { + let mut solid: Vec<_> = (0..3) + .flat_map(|i| { + quad( + [ + [1.0, 0.0, 0.0], + [1.0, 1.0, 0.0], + [1.0, 1.0, 1.0], + [1.0, 0.0, 1.0], + ], + i, + ) + }) + .collect(); + let before = solid.clone(); + let sides = sort(&mut solid); + assert_eq!(sides.ends, [3; 7]); + assert_eq!( + solid.iter().map(|v| v.layer).collect::>(), + before.iter().map(|v| v.layer).collect::>() + ); + } +} From 8c611e677b6e84f27116cd0c486f2f64703e1568 Mon Sep 17 00:00:00 2001 From: some-du6e <103129948+some-du6e@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:44:17 -0600 Subject: [PATCH 05/51] Advancement titles and descriptions show a whole text component A server's custom advancements often name themselves with an empty text and the words in its siblings ({"text":"","extra":[...]}); only the top-level text or translate key was read, so their tooltips were blank. Read them as the component they are (ComponentSerialization), with siblings, styles, translation arguments and fallbacks. Co-Authored-By: Claude Opus 5.5 --- .../horizon-client/src/game/advancements.rs | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/crates/horizon-client/src/game/advancements.rs b/crates/horizon-client/src/game/advancements.rs index b0f948c5d..20d54d290 100644 --- a/crates/horizon-client/src/game/advancements.rs +++ b/crates/horizon-client/src/game/advancements.rs @@ -1051,17 +1051,10 @@ impl Game<'_> { } } } -/// A text component from advancement JSON (`translate` or `text`). +/// An advancement's title or description: a whole component (ComponentSerialization.CODEC), +/// with its siblings and styles, as a plugin's `{"text":"","extra":[...]}` needs. pub(super) fn text_of(v: &Value) -> Text { - if let Some(key) = v["translate"].as_str() { - Text::translate(key, &[]) - } else if let Some(s) = v["text"].as_str() { - Text::literal(s) - } else if let Some(s) = v.as_str() { - Text::literal(s) - } else { - Text::empty() - } + horizon_client::scoreboard::comp_text(v) } #[cfg(test)] @@ -1095,6 +1088,20 @@ mod tests { (seen, None) } + #[test] + fn titles_keep_their_siblings_and_styles() { + let title = text_of(&serde_json::json!({ + "text": "", + "extra": [{"text": "Quest ", "color": "gold"}, "Board"], + })); + assert_eq!(title.plain(), "Quest Board"); + assert_eq!(title.spans[0].style.color, Some(horizon_client::text::GOLD)); + assert_eq!( + text_of(&serde_json::json!({"translate": "advancements.story.root.title"})), + Text::translate("advancements.story.root.title", &[]) + ); + } + #[test] fn toasts_slide_in_squared_and_out_after_five_seconds() { let mut toast = recipes(1); From 4ad05aa94a4a1a7c4c9b4b39717a238af5d6584c Mon Sep 17 00:00:00 2001 From: some-du6e <103129948+some-du6e@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:44:17 -0600 Subject: [PATCH 06/51] Show armour trims on items from a server Decode the trim component as its material and pattern registry ids when both are references (a direct holder stays read past) and name them through the server's synced trim_material and trim_pattern registries, as enchantments are, so worn and held trimmed armour draws its trim in multiplayer. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game/remote.rs | 13 ++++++ .../examples/item_wire_probe.rs | 6 +-- crates/horizon-protocol/src/item.rs | 45 ++++++++++++++++--- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index 0e6127178..8cf3d52a0 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -598,6 +598,19 @@ impl Game<'_> { let value = match c { // Read past, not decoded: the item keeps its prototype's value. Component::Raw(_) => continue, + // Trim materials and patterns are synced registries, as enchantments are. + Component::Trim { material, pattern } => { + let (Some(material), Some(pattern)) = ( + r.registries.name("minecraft:trim_material", *material), + r.registries.name("minecraft:trim_pattern", *pattern), + ) else { + continue; + }; + Tag::compound([ + ("material", Tag::String(material.into())), + ("pattern", Tag::String(pattern.into())), + ]) + } Component::Int(v) if *kind == component::RARITY => { let rarity = ["common", "uncommon", "rare", "epic"]; Tag::String(rarity.get(*v as usize).copied().unwrap_or("common").into()) diff --git a/crates/horizon-protocol/examples/item_wire_probe.rs b/crates/horizon-protocol/examples/item_wire_probe.rs index 19760ca1a..097a9f40d 100644 --- a/crates/horizon-protocol/examples/item_wire_probe.rs +++ b/crates/horizon-protocol/examples/item_wire_probe.rs @@ -2,8 +2,8 @@ //! the oracle's stacks (JSON lines with `label` and `hex`), decodes each with //! `item::Slot` and prints, per stack, whether every byte was consumed, the //! patch's component ids (added, then removed), which of them Horizon -//! decoded into its own types (the rest it reads past, keeping their bytes) -//! and whether encoding the decoded stack gives the same bytes back. +//! decoded into its own lossy types (the rest it reads past keeping their +//! bytes, or decodes into ids it writes back unchanged, like trims) and whether encoding the decoded stack gives the same bytes back. //! //! cargo run -p horizon-protocol --example item_wire_probe -- vanilla.jsonl rust.jsonl use horizon_protocol::{ @@ -45,7 +45,7 @@ fn main() -> Result<(), Box> { let typed: Vec = stack .iter() .flat_map(|s| &s.added) - .filter(|(_, c)| !matches!(c, Component::Raw(_))) + .filter(|(_, c)| !matches!(c, Component::Raw(_) | Component::Trim { .. })) .map(|(k, _)| *k) .collect(); json!({ diff --git a/crates/horizon-protocol/src/item.rs b/crates/horizon-protocol/src/item.rs index 0aa940a38..2872c0228 100644 --- a/crates/horizon-protocol/src/item.rs +++ b/crates/horizon-protocol/src/item.rs @@ -37,6 +37,7 @@ pub mod component { pub const CHARGED_PROJECTILES: i32 = 51; pub const WRITABLE_BOOK_CONTENT: i32 = 56; pub const WRITTEN_BOOK_CONTENT: i32 = 57; + pub const TRIM: i32 = 58; } /// A written book's content (`WrittenBookContent.STREAM_CODEC`): the @@ -80,6 +81,12 @@ pub enum Component { WrittenBook(WrittenBook), /// `ChargedProjectiles`: a loaded crossbow's ammunition, as templates. Stacks(Vec), + /// `ArmorTrim` naming its material and pattern by registry id (`trim_material`, + /// `trim_pattern`); a trim with a direct holder stays [`Component::Raw`]. + Trim { + material: i32, + pattern: i32, + }, /// A component Horizon reads past without interpreting it: its encoded /// bytes, written back as they came. Raw(Vec), @@ -232,13 +239,28 @@ impl Component { other => { let start = r.position(); skip::component(other, r)?; - Self::Raw(r.since(start).to_vec()) + let bytes = r.since(start); + // Two references: each holder is just its id plus one. + let mut ids = Reader::new(bytes); + match (other, ids.varint(), ids.varint()) { + (TRIM, Ok(material @ 1..), Ok(pattern @ 1..)) if ids.remaining() == 0 => { + Self::Trim { + material: material - 1, + pattern: pattern - 1, + } + } + _ => Self::Raw(bytes.to_vec()), + } } }) } fn write(&self, kind: i32, w: &mut Writer) { match self { Self::Raw(bytes) => w.bytes(bytes), + Self::Trim { material, pattern } => { + w.varint(material + 1); + w.varint(pattern + 1); + } Self::Int(v) if kind == component::DYED_COLOR => w.i32(*v), Self::Int(v) => w.varint(*v), Self::Unit => {} @@ -646,7 +668,7 @@ mod tests { #[test] fn components_read_past_keep_their_bytes() { // Vanilla 26.3's encoding (tools/test_item_wire_parity.py) of a chestplate's - // armour trim, a banner's patterns (one direct), a pot's sherds and a stew's + // armour trim (decoded: both holders are references), a banner's patterns (one direct), a pot's sherds and a stew's // effects: one server's inventory, recipe book and advancements had them. for (hex, kind) in [ ("01bd0801003a0502", 58), @@ -661,10 +683,21 @@ mod tests { let mut r = Reader::new(&bytes); let stack = Slot::read(&mut r).unwrap(); assert_eq!(r.remaining(), 0, "component {kind}"); - assert!(matches!( - stack.as_ref().and_then(|s| s.component(kind)), - Some(Component::Raw(_)) - )); + let expected = |c: &Component| match kind { + 58 => { + *c == Component::Trim { + material: 4, + pattern: 1, + } + } + _ => matches!(c, Component::Raw(_)), + }; + assert!( + stack + .as_ref() + .and_then(|s| s.component(kind)) + .is_some_and(expected) + ); let mut w = Writer::new(); Slot::write(&stack, &mut w); assert_eq!(w.data, bytes, "component {kind}"); From bffb68b0c59168f4354f9664023e18f4bed0b9f9 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:03:00 +0200 Subject: [PATCH 07/51] Mod registry index: schema 1, .hpack and the checks before install horizon_mods::index (feature index): index.json's types and rules (unique ids, https URLs, plain file names, hex SHA-256, minisign keys and signatures, a modpack's mods present in the index), open-ended kinds (mod, modpack, resourcepack, shader; unknown kinds kept, not offered), revocations, and check_file: size, hash, revocation, signature and the file's own manifest against its entry. horizon_mods::modpack reads .hpack files: horizon-modpack.toml and an allow-list of overrides. docs/api/mod-index.md documents both, with a sample index the tests read. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 10 +- crates/horizon-mods/Cargo.toml | 5 + crates/horizon-mods/src/index.rs | 538 ++++++++++++++++++ crates/horizon-mods/src/index/tests.rs | 162 ++++++ crates/horizon-mods/src/lib.rs | 3 + crates/horizon-mods/src/manifest.rs | 9 +- crates/horizon-mods/src/modpack.rs | 299 ++++++++++ crates/horizon-mods/src/runtime.rs | 5 +- .../tests/index/coords_hud-0.1.0.hmod | Bin 0 -> 438 bytes .../tests/index/coords_hud-0.1.0.hmod.minisig | 4 + .../horizon-mods/tests/index/cozy-1.0.0.hpack | Bin 0 -> 471 bytes crates/horizon-mods/tests/index/index-v1.json | 109 ++++ crates/horizon-mods/tests/index/test.pub | 2 + docs/api/mod-index.md | 95 ++++ docs/mods.md | 3 +- 15 files changed, 1235 insertions(+), 9 deletions(-) create mode 100644 crates/horizon-mods/src/index.rs create mode 100644 crates/horizon-mods/src/index/tests.rs create mode 100644 crates/horizon-mods/src/modpack.rs create mode 100644 crates/horizon-mods/tests/index/coords_hud-0.1.0.hmod create mode 100644 crates/horizon-mods/tests/index/coords_hud-0.1.0.hmod.minisig create mode 100644 crates/horizon-mods/tests/index/cozy-1.0.0.hpack create mode 100644 crates/horizon-mods/tests/index/index-v1.json create mode 100644 crates/horizon-mods/tests/index/test.pub create mode 100644 docs/api/mod-index.md diff --git a/Cargo.lock b/Cargo.lock index 27eaad0c4..57006aa82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -155,7 +155,7 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "windows-sys 0.59.0", + "windows-sys 0.60.2", "wl-clipboard-rs", "x11rb", ] @@ -1331,8 +1331,10 @@ name = "horizon-mods" version = "0.1.0" dependencies = [ "log", + "minisign-verify", "serde", "serde_json", + "sha2", "toml", "wasmtime", "wit-component 0.239.0", @@ -1898,6 +1900,12 @@ dependencies = [ "libc", ] +[[package]] +name = "minisign-verify" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871285dc19d8d0ebe0eef3d0e99a205f2a71363b122632cbbfa4a6c370a960ce" + [[package]] name = "miniz_oxide" version = "0.8.9" diff --git a/crates/horizon-mods/Cargo.toml b/crates/horizon-mods/Cargo.toml index 3dfc02331..840fc28c5 100644 --- a/crates/horizon-mods/Cargo.toml +++ b/crates/horizon-mods/Cargo.toml @@ -17,6 +17,9 @@ wasmtime = { version = "36.0.18", default-features = false, features = ["runtime # Turns a mod's core module into a component (cargo horizon-mod build, the tests). wit-component = { version = "0.239", optional = true } wit-parser = { version = "0.239", optional = true } +# The registry index's checks (feature `index`): SHA-256, and minisign signatures (no dependencies of its own). +sha2 = { version = "0.10", optional = true } +minisign-verify = { version = "0.3", optional = true } [features] # Loads and runs mods (the game's `mods` feature); without it only manifests, @@ -24,6 +27,8 @@ wit-parser = { version = "0.239", optional = true } # game's tests and probes) skip wasmtime and cranelift. runtime = ["dep:wasmtime"] pack = ["dep:wit-component", "dep:wit-parser"] +# The mod registry's index (docs/api/mod-index.md): the launcher, the registry server and its CI. +index = ["dep:sha2", "dep:minisign-verify"] # Runs real mods: `cargo test -p horizon-mods --features runtime,pack`. [[test]] diff --git a/crates/horizon-mods/src/index.rs b/crates/horizon-mods/src/index.rs new file mode 100644 index 000000000..73af7b46d --- /dev/null +++ b/crates/horizon-mods/src/index.rs @@ -0,0 +1,538 @@ +//! The mod registry's index, `index.json` (docs/api/mod-index.md): every +//! reviewed mod and modpack, its versions, the file of each with its size, +//! SHA-256 and minisign signature, and the files taken down. The launcher +//! reads it to browse and install, the website to show it, and the +//! registry's CI writes it. +//! +//! [`check_file`] is what a file must pass before it is installed: its size +//! and hash are the ones the index lists, its signature verifies against the +//! project's pinned key, and what its own manifest says matches the entry. +use crate::{ + Capability, Package, Side, + modpack::{self, Modpack}, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{io::Read, path::Path}; + +/// The index format this build reads and writes. A reader refuses a newer +/// one rather than misread it. +pub const SCHEMA: u32 = 1; +/// The biggest file the registry takes (a `.hmod` with content, a `.hpack`). +pub const MAX_FILE: u64 = 64 << 20; +/// Where files may be downloaded from: the registry's release assets. +pub const FILE_HOST: &str = "https://github.com/ludovic111/horizon-mods/releases/download/"; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Index { + pub schema: u32, + /// When CI built it, RFC 3339. + #[serde(default)] + pub generated: String, + #[serde(default)] + pub projects: Vec, + #[serde(default)] + pub revoked: Vec, +} + +/// What a project is. New kinds (data packs, worlds, themes) can be added +/// without a new schema: a reader keeps a kind it doesn't know as +/// [`Kind::Other`] and doesn't offer it. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(from = "String", into = "String")] +pub enum Kind { + /// A `.hmod`. + Mod, + /// A `.hpack`: registry mods played together as a profile. + Modpack, + /// A resource pack (`.zip` with `pack.mcmeta`) made or tested for Horizon. + ResourcePack, + /// A Horizon shader pack (the native WGSL format, not Iris or OptiFine + /// packs). Listed by the schema; submissions open with its API. + Shader, + Other(String), +} + +impl Kind { + pub fn name(&self) -> &str { + match self { + Kind::Mod => "mod", + Kind::Modpack => "modpack", + Kind::ResourcePack => "resourcepack", + Kind::Shader => "shader", + Kind::Other(name) => name, + } + } + + /// The extension of its files; None for a kind this build doesn't know. + pub fn extension(&self) -> Option<&'static str> { + match self { + Kind::Mod => Some("hmod"), + Kind::Modpack => Some("hpack"), + Kind::ResourcePack | Kind::Shader => Some("zip"), + Kind::Other(_) => None, + } + } +} + +impl From for Kind { + fn from(name: String) -> Kind { + match name.as_str() { + "mod" => Kind::Mod, + "modpack" => Kind::Modpack, + "resourcepack" => Kind::ResourcePack, + "shader" => Kind::Shader, + _ => Kind::Other(name), + } + } +} + +impl From for String { + fn from(kind: Kind) -> String { + kind.name().to_owned() + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Project { + /// The manifest's id, `[a-z0-9_]{1,64}`: unique across the index. + pub id: String, + pub kind: Kind, + pub name: String, + /// One line for lists. + pub summary: String, + /// Markdown, for the project's page. + #[serde(default)] + pub description: String, + #[serde(default)] + pub authors: Vec, + /// The GitHub logins allowed to publish its updates. + #[serde(default)] + pub owners: Vec, + /// An SPDX expression. + pub license: String, + /// Where its source code is, `https://`. + pub source: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub icon: Option, + /// The author's minisign public key: every version must be signed with + /// it. None: unsigned (the SHA-256 still pins the reviewed file). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + /// Newest first. + pub versions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Version { + pub version: String, + /// The day it was published, `YYYY-MM-DD`. + #[serde(default)] + pub published: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub changelog: String, + /// A mod's API line (`0.1`), from its manifest; a modpack's format (`1`). + pub api: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub side: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub capabilities: Vec, + /// The Minecraft version it is for. + pub game: String, + /// A modpack's mods, as its manifest lists them. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub requires: Vec, + pub file: File, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Requirement { + pub id: String, + pub version: String, + pub sha256: String, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct File { + pub name: String, + pub url: String, + pub size: u64, + /// Lowercase hex. + pub sha256: String, + /// The `.minisig` file's text. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signature: Option, +} + +/// A file taken down: by hash, or every version of `id` when `sha256` is +/// None. The launcher refuses to install it and turns installed copies off. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Revoked { + pub id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sha256: Option, + pub reason: String, + #[serde(default)] + pub date: String, +} + +/// How far a checked file is trusted. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Trust { + /// Signed with the project's pinned key. + Verified, + /// The project has no key: only the hash pins it. + Unsigned, +} + +impl Index { + /// Parses and checks an index (see [`Index::check`]). + pub fn parse(text: &str) -> Result { + Self::parse_with(text, false) + } + + /// [`Index::parse`] for a local test registry, whose files may be + /// `file://` paths (trying a registry out before it is published). + pub fn parse_local(text: &str) -> Result { + Self::parse_with(text, true) + } + + fn parse_with(text: &str, local_files: bool) -> Result { + #[derive(Deserialize)] + struct Head { + schema: u32, + } + let head: Head = serde_json::from_str(text).map_err(|e| format!("not a mod index: {e}"))?; + if head.schema > SCHEMA { + return Err(format!( + "the mod index is schema {}; this build reads {SCHEMA}: update Horizon", + head.schema + )); + } + let index: Index = serde_json::from_str(text).map_err(|e| format!("mod index: {e}"))?; + index.check_with(local_files)?; + Ok(index) + } + + /// Every rule a reader relies on: unique valid ids, at least one + /// version each, plain file names with the kind's extension, `https` + /// URLs, hex hashes, keys and signatures that decode, and each + /// modpack's mods present in the index with the same hash. + pub fn check(&self) -> Result<(), String> { + self.check_with(false) + } + + fn check_with(&self, local_files: bool) -> Result<(), String> { + for (i, p) in self.projects.iter().enumerate() { + let at = |e: String| format!("{}: {e}", p.id); + if !valid_id(&p.id) { + return Err(format!("project id {:?} is not [a-z0-9_]{{1,64}}", p.id)); + } + if self.projects[..i].iter().any(|q| q.id == p.id) { + return Err(at("listed twice".into())); + } + if p.name.trim().is_empty() || p.license.trim().is_empty() { + return Err(at("needs a name and a licence".into())); + } + https(&p.source).map_err(at)?; + if let Some(icon) = &p.icon { + https(icon).map_err(at)?; + } + if let Some(key) = &p.key { + minisign_verify::PublicKey::from_base64(key.trim()) + .map_err(|e| at(format!("key: {e}")))?; + } + if p.versions.is_empty() { + return Err(at("has no version".into())); + } + for (j, v) in p.versions.iter().enumerate() { + let at = |e: String| format!("{} {}: {e}", p.id, v.version); + if v.version.trim().is_empty() + || p.versions[..j].iter().any(|w| w.version == v.version) + { + return Err(at("empty or repeated version".into())); + } + let f = &v.file; + let extension_ok = p + .kind + .extension() + .is_none_or(|e| f.name.ends_with(&format!(".{e}"))); + if !plain_name(&f.name) || !extension_ok { + return Err(at(format!( + "{:?} is not a plain file name for a {}", + f.name, + p.kind.name() + ))); + } + if !(local_files && f.url.starts_with("file:///")) { + https(&f.url).map_err(at)?; + } + hex64(&f.sha256).map_err(at)?; + if f.size == 0 || f.size > MAX_FILE { + return Err(at(format!( + "size {} is not 1 byte to {} MiB", + f.size, + MAX_FILE >> 20 + ))); + } + match (&p.key, &f.signature) { + (Some(_), None) => { + return Err(at("unsigned, but the project has a key".into())); + } + (None, Some(_)) => return Err(at("signed, but the project has no key".into())), + (Some(_), Some(sig)) => { + minisign_verify::Signature::decode(sig) + .map_err(|e| at(format!("signature: {e}")))?; + } + (None, None) => {} + } + for r in &v.requires { + let found = self + .project(&r.id) + .and_then(|q| q.versions.iter().find(|w| w.version == r.version)); + match found { + Some(w) if w.file.sha256 == r.sha256 => {} + Some(_) => { + return Err(at(format!( + "{} {}: another file than the index's", + r.id, r.version + ))); + } + None => { + return Err(at(format!( + "needs {} {}, which is not in the index", + r.id, r.version + ))); + } + } + } + } + } + for r in &self.revoked { + if let Some(h) = &r.sha256 { + hex64(h).map_err(|e| format!("revoked {}: {e}", r.id))?; + } + } + Ok(()) + } + + pub fn project(&self, id: &str) -> Option<&Project> { + self.projects.iter().find(|p| p.id == id) + } + + /// Why the file of `id` with this hash was taken down, if it was. + pub fn revoked(&self, id: &str, sha256: &str) -> Option<&Revoked> { + self.revoked.iter().find(|r| { + r.id == id + && r.sha256 + .as_ref() + .is_none_or(|h| h.eq_ignore_ascii_case(sha256)) + }) + } +} + +/// `[a-z0-9_]{1,64}`, the manifests' id rule. +pub fn valid_id(id: &str) -> bool { + (1..=64).contains(&id.len()) + && id + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') +} + +fn https(url: &str) -> Result<(), String> { + match url.strip_prefix("https://") { + Some(rest) if !rest.is_empty() && !url.chars().any(char::is_whitespace) => Ok(()), + _ => Err(format!("{url:?} is not an https:// address")), + } +} + +fn hex64(h: &str) -> Result<(), String> { + if h.len() == 64 + && h.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + Ok(()) + } else { + Err(format!("{h:?} is not a lowercase hex SHA-256")) + } +} + +/// One plain file name: no folder, no hidden file, nothing a file system +/// reads as something else. +fn plain_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 128 + && !name.starts_with('.') + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-+".contains(&b)) +} + +/// The SHA-256 of a file, lowercase hex. +pub fn sha256_file(path: &Path) -> Result { + let mut file = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; 1 << 16]; + loop { + let n = file.read(&mut buf).map_err(|e| e.to_string())?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Ok(hex(&hasher.finalize())) +} + +pub fn sha256(bytes: &[u8]) -> String { + hex(&Sha256::digest(bytes)) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Checks a minisign signature (`.minisig` text) of `path` against `key`. +pub fn verify_signature(key: &str, signature: &str, path: &Path) -> Result<(), String> { + let key = + minisign_verify::PublicKey::from_base64(key.trim()).map_err(|e| format!("key: {e}"))?; + let sig = + minisign_verify::Signature::decode(signature).map_err(|e| format!("signature: {e}"))?; + let file = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut verifier = key + .verify_stream(&sig) + .map_err(|e| format!("signature: {e}"))?; + let mut reader = std::io::BufReader::new(file); + let mut buf = vec![0u8; 1 << 16]; + loop { + let n = reader.read(&mut buf).map_err(|e| e.to_string())?; + if n == 0 { + break; + } + verifier.update(&buf[..n]); + } + verifier + .finalize() + .map_err(|_| "the signature doesn't match the project's key".to_string()) +} + +/// Checks the file at `path` against its entry before it is installed: +/// size, SHA-256, revocation, signature, and that its manifest says what the +/// index says (id, version, API, side, capabilities; a modpack's mods). +pub fn check_file( + index: &Index, + project: &Project, + version: &Version, + path: &Path, +) -> Result { + let size = std::fs::metadata(path) + .map_err(|e| format!("{}: {e}", path.display()))? + .len(); + if size != version.file.size { + return Err(format!( + "{} bytes, the index lists {}", + size, version.file.size + )); + } + let hash = sha256_file(path)?; + if hash != version.file.sha256 { + return Err("its SHA-256 isn't the one the index lists: nothing was installed".into()); + } + if let Some(r) = index.revoked(&project.id, &hash) { + return Err(format!("{} was taken down: {}", project.name, r.reason)); + } + let trust = match (&project.key, &version.file.signature) { + (Some(key), Some(sig)) => { + verify_signature(key, sig, path)?; + Trust::Verified + } + (None, None) => Trust::Unsigned, + _ => return Err("its signature and the project's key don't go together".into()), + }; + match project.kind { + Kind::Mod => { + let m = Package::read(path)?.manifest; + crate::Content::read(path, &m)?; + let mut caps = m.capabilities.clone(); + caps.sort(); + let mut listed = version.capabilities.clone(); + listed.sort(); + let differs = m.id != project.id + || m.version != version.version + || m.api != version.api + || Some(m.side) != version.side + || caps != listed; + if differs { + return Err(format!( + "its horizon-mod.toml ({} {}, API {}, {:?}, {}) isn't what the index lists", + m.id, + m.version, + m.api, + m.side, + names(&caps) + )); + } + } + Kind::ResourcePack => check_resource_pack(path)?, + Kind::Shader | Kind::Other(_) => { + return Err(format!( + "this Horizon can't install a {} yet", + project.kind.name() + )); + } + Kind::Modpack => { + let pack = Modpack::read(path)?; + let m = &pack.manifest; + let mods: Vec = m.mods.iter().map(modpack::PackMod::requirement).collect(); + if m.id != project.id + || m.version != version.version + || m.format.to_string() != version.api + || m.game != version.game + || mods != version.requires + { + return Err(format!( + "its {} isn't what the index lists", + modpack::MANIFEST + )); + } + } + } + Ok(trust) +} + +/// A resource pack: a zip with `pack.mcmeta` at its root holding a `pack` +/// object, and only plain paths. +fn check_resource_pack(path: &Path) -> Result<(), String> { + let file = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut zip = zip::ZipArchive::new(file).map_err(|e| e.to_string())?; + for i in 0..zip.len() { + let name = zip + .by_index_raw(i) + .map_err(|e| e.to_string())? + .name() + .to_owned(); + if name.starts_with('/') || name.split('/').any(|part| part == "..") || name.contains('\\') + { + return Err(format!("{name:?} is not a plain path")); + } + } + let mut meta = String::new(); + zip.by_name("pack.mcmeta") + .map_err(|_| "pack.mcmeta: missing".to_string())? + .take(1 << 20) + .read_to_string(&mut meta) + .map_err(|e| format!("pack.mcmeta: {e}"))?; + let json: serde_json::Value = + serde_json::from_str(&meta).map_err(|e| format!("pack.mcmeta: {e}"))?; + if !json["pack"].is_object() { + return Err("pack.mcmeta: no \"pack\" object".into()); + } + Ok(()) +} + +fn names(caps: &[Capability]) -> String { + let names: Vec = caps.iter().map(ToString::to_string).collect(); + format!("[{}]", names.join(", ")) +} + +#[cfg(test)] +mod tests; diff --git a/crates/horizon-mods/src/index/tests.rs b/crates/horizon-mods/src/index/tests.rs new file mode 100644 index 000000000..b6d7f5bfb --- /dev/null +++ b/crates/horizon-mods/src/index/tests.rs @@ -0,0 +1,162 @@ +//! The sample index (`tests/index/index-v1.json`, the one docs/api/mod-index.md +//! shows) and its files: `coords_hud-0.1.0.hmod`, signed with the throwaway +//! key `test.pub` by minisign 0.12 (its secret key was deleted), and +//! `cozy-1.0.0.hpack`, unsigned. +use super::*; +use std::path::PathBuf; + +const OTHER_KEY: &str = "RWSe8ZD+cTOAqNd/jnXTmDtoKp0c0HxkGMrBBWcWQeEkPmcHB+XkaE9S"; + +fn fixtures() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/index") +} + +fn sample() -> Index { + Index::parse(&std::fs::read_to_string(fixtures().join("index-v1.json")).unwrap()).unwrap() +} + +fn check(index: &Index, id: &str, file: &Path) -> Result { + let p = index.project(id).unwrap(); + check_file(index, p, &p.versions[0], file) +} + +#[test] +fn reads_the_sample_and_keeps_unknown_kinds() { + let index = sample(); + assert_eq!(index.schema, SCHEMA); + let kinds: Vec<&str> = index.projects.iter().map(|p| p.kind.name()).collect(); + assert_eq!(kinds, ["mod", "modpack", "world"]); + assert_eq!(index.projects[2].kind, Kind::Other("world".into())); + let hud = index.project("coords_hud").unwrap(); + assert_eq!( + hud.versions[0].capabilities, + [Capability::GameRead, Capability::Hud, Capability::Chat] + ); + // What CI writes reads back the same. + let again = Index::parse(&serde_json::to_string(&index).unwrap()).unwrap(); + assert_eq!(again, index); +} + +#[test] +fn files_pass_against_their_entries() { + let index = sample(); + let hmod = fixtures().join("coords_hud-0.1.0.hmod"); + assert_eq!(check(&index, "coords_hud", &hmod), Ok(Trust::Verified)); + let hpack = fixtures().join("cozy-1.0.0.hpack"); + assert_eq!(check(&index, "cozy", &hpack), Ok(Trust::Unsigned)); + let world = index.project("skyblock_start").unwrap(); + let e = check_file(&index, world, &world.versions[0], &hpack).unwrap_err(); + assert!(e.contains("bytes"), "{e}"); +} + +#[test] +fn a_changed_file_key_or_manifest_is_refused() { + let index = sample(); + let dir = std::env::temp_dir().join(format!("horizon-index-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let hmod = fixtures().join("coords_hud-0.1.0.hmod"); + let mut bytes = std::fs::read(&hmod).unwrap(); + let last = bytes.len() - 1; + bytes[last] ^= 1; + let tampered = dir.join("coords_hud-0.1.0.hmod"); + std::fs::write(&tampered, &bytes).unwrap(); + let e = check(&index, "coords_hud", &tampered).unwrap_err(); + assert!(e.contains("SHA-256"), "{e}"); + + let mut other_key = index.clone(); + other_key.projects[0].key = Some(OTHER_KEY.into()); + let e = check(&other_key, "coords_hud", &hmod).unwrap_err(); + assert!(e.contains("key"), "{e}"); + + let mut more_caps = index.clone(); + more_caps.projects[0].versions[0].capabilities.pop(); + let e = check(&more_caps, "coords_hud", &hmod).unwrap_err(); + assert!(e.contains("isn't what the index lists"), "{e}"); + + let mut revoked = index.clone(); + revoked.revoked.push(Revoked { + id: "coords_hud".into(), + sha256: None, + reason: "Test.".into(), + date: String::new(), + }); + let e = check(&revoked, "coords_hud", &hmod).unwrap_err(); + assert!(e.contains("taken down: Test."), "{e}"); + std::fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn revocations_match_by_hash_or_by_id() { + let index = sample(); + let ones = "1".repeat(64); + assert!(index.revoked("bad_mod", &ones).is_some()); + assert!(index.revoked("bad_mod", &"2".repeat(64)).is_none()); + assert!(index.revoked("coords_hud", &ones).is_none()); +} + +#[test] +fn refuses_broken_indexes() { + let text = std::fs::read_to_string(fixtures().join("index-v1.json")).unwrap(); + let newer = text.replacen("\"schema\": 1", "\"schema\": 2", 1); + assert!(Index::parse(&newer).unwrap_err().contains("schema 2")); + let broken = |edit: &dyn Fn(&mut Index)| { + let mut index = sample(); + edit(&mut index); + index.check().unwrap_err() + }; + let e = broken(&|i| { + let twin = i.projects[0].clone(); + i.projects.push(twin) + }); + assert!(e.contains("listed twice"), "{e}"); + let e = broken(&|i| i.projects[0].versions[0].file.url = "http://example.org/x.hmod".into()); + assert!(e.contains("https"), "{e}"); + // A local test registry may list files on disk; the published one may not. + let local = text.replace( + "https://github.com/ludovic111/horizon-mods/releases/download/coords_hud-0.1.0/", + "file:///tmp/", + ); + assert!(Index::parse(&local).unwrap_err().contains("https")); + assert!(Index::parse_local(&local).is_ok()); + let e = broken(&|i| i.projects[0].versions[0].file.name = "../coords_hud.hmod".into()); + assert!(e.contains("plain file name"), "{e}"); + let e = broken(&|i| i.projects[0].versions[0].file.name = "coords_hud.zip".into()); + assert!(e.contains("plain file name"), "{e}"); + let e = broken(&|i| i.projects[0].versions[0].file.signature = None); + assert!(e.contains("unsigned"), "{e}"); + let e = broken(&|i| i.projects[0].key = Some("not a key".into())); + assert!(e.contains("key"), "{e}"); + let e = broken(&|i| i.projects[1].versions[0].requires[0].version = "9.9".into()); + assert!(e.contains("not in the index"), "{e}"); + let e = broken(&|i| i.projects[1].versions[0].requires[0].sha256 = "2".repeat(64)); + assert!(e.contains("another file"), "{e}"); + let e = broken(&|i| i.projects[0].id = "Coords".into()); + assert!(e.contains("is not"), "{e}"); +} + +#[test] +fn resource_packs_need_a_pack_mcmeta() { + use std::io::Write; + let dir = std::env::temp_dir().join(format!("horizon-index-rp-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let pack = |files: &[(&str, &str)]| { + let path = dir.join("look.zip"); + let mut zip = zip::ZipWriter::new(std::fs::File::create(&path).unwrap()); + for (name, text) in files { + zip.start_file(*name, zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(text.as_bytes()).unwrap(); + } + zip.finish().unwrap(); + path + }; + let meta = r#"{"pack": {"description": "Look", "min_format": 75, "max_format": 75}}"#; + assert_eq!(check_resource_pack(&pack(&[("pack.mcmeta", meta)])), Ok(())); + let e = check_resource_pack(&pack(&[("assets/x.png", "")])).unwrap_err(); + assert!(e.contains("missing"), "{e}"); + let e = check_resource_pack(&pack(&[("pack.mcmeta", "{}")])).unwrap_err(); + assert!(e.contains("\"pack\""), "{e}"); + let e = check_resource_pack(&pack(&[("pack.mcmeta", meta), ("../x", "")])).unwrap_err(); + assert!(e.contains("plain path"), "{e}"); + std::fs::remove_dir_all(&dir).unwrap(); +} diff --git a/crates/horizon-mods/src/lib.rs b/crates/horizon-mods/src/lib.rs index 5060bf747..17a6c5aa0 100644 --- a/crates/horizon-mods/src/lib.rs +++ b/crates/horizon-mods/src/lib.rs @@ -6,7 +6,10 @@ //! Nothing here runs unless a mod is installed and turned on: an empty //! [`Mods`] makes no engine and every hook returns at once. pub mod content; +#[cfg(feature = "index")] +pub mod index; pub mod manifest; +pub mod modpack; #[cfg(feature = "runtime")] mod runtime; diff --git a/crates/horizon-mods/src/manifest.rs b/crates/horizon-mods/src/manifest.rs index a71210883..3a93b2512 100644 --- a/crates/horizon-mods/src/manifest.rs +++ b/crates/horizon-mods/src/manifest.rs @@ -1,5 +1,5 @@ //! `horizon-mod.toml`, a mod's manifest, and the `.hmod` file around it. -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::{ fmt, io::Read, @@ -20,6 +20,9 @@ pub const APIS: [&str; 2] = [API, "0.2"]; /// Namespaces no mod may claim: vanilla's, the convention tags' (mods add /// to those, docs/mods.md) and the game's own. const RESERVED_NAMESPACES: [&str; 3] = ["minecraft", "c", "horizon"]; +/// A mod's data (capability `storage`), in `horizon-mods/` beside the mods +/// (`find` never mistakes it for a mod: it has no manifest). +pub const STORAGE_DIR: &str = ".horizon-mod-data"; /// The biggest `mod.wasm` a mod may ship. const MAX_WASM: u64 = 32 << 20; @@ -48,7 +51,7 @@ pub struct Manifest { pub namespaces: Vec, } -#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "lowercase")] pub enum Side { /// Runs in the player's game only (HUD, keys, chat commands). @@ -62,7 +65,7 @@ pub enum Side { /// What a mod may do beyond writing its log. Each is a WIT interface the /// component imports; the player sees them before turning the mod on. -#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] #[serde(rename_all = "kebab-case")] pub enum Capability { GameRead, diff --git a/crates/horizon-mods/src/modpack.rs b/crates/horizon-mods/src/modpack.rs new file mode 100644 index 000000000..83c9eef5c --- /dev/null +++ b/crates/horizon-mods/src/modpack.rs @@ -0,0 +1,299 @@ +//! `.hpack`, a Horizon modpack (docs/api/mod-index.md): registry mods +//! played together as a profile. A zip holding `horizon-modpack.toml`, which +//! names each mod by id, version and SHA-256 (the files themselves come from +//! the registry), and optionally `overrides/`: settings and resource packs +//! copied into the profile's game folder. +use crate::manifest::STORAGE_DIR; +use serde::Deserialize; +use std::{ + io::Read, + path::{Path, PathBuf}, +}; + +/// The manifest's file name in a `.hpack`. +pub const MANIFEST: &str = "horizon-modpack.toml"; +/// The manifest formats this build reads. +pub const FORMAT: u32 = 1; +const OVERRIDES: &str = "overrides/"; +/// Bytes per override file and in all, unpacked. +const MAX_OVERRIDE: u64 = 64 << 20; +const MAX_OVERRIDES: u64 = 256 << 20; +/// The settings files an override may replace. +const SETTINGS: [&str; 3] = ["options.txt", "horizon-graphics.txt", "horizon-extras.txt"]; + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct Manifest { + pub format: u32, + /// `[a-z0-9_]{1,64}`: its profile's folder. + pub id: String, + pub name: String, + pub version: String, + #[serde(default)] + pub description: String, + #[serde(default)] + pub authors: Vec, + /// The Minecraft version it is for. + pub game: String, + #[serde(default)] + pub mods: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct PackMod { + pub id: String, + pub version: String, + pub sha256: String, + /// Turned on in the profile (`horizon-mods.txt`). + #[serde(default = "on")] + pub on: bool, +} + +fn on() -> bool { + true +} + +#[cfg(feature = "index")] +impl PackMod { + pub fn requirement(&self) -> crate::index::Requirement { + crate::index::Requirement { + id: self.id.clone(), + version: self.version.clone(), + sha256: self.sha256.clone(), + } + } +} + +impl Manifest { + pub fn parse(text: &str) -> Result { + let m: Manifest = toml::from_str(text).map_err(|e| format!("{MANIFEST}: {e}"))?; + if m.format != FORMAT { + return Err(format!( + "{MANIFEST}: format {}; this Horizon reads {FORMAT}", + m.format + )); + } + let id_ok = |id: &str| { + (1..=64).contains(&id.len()) + && id + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') + }; + if !id_ok(&m.id) { + return Err(format!( + "{MANIFEST}: id {:?} is not [a-z0-9_]{{1,64}}", + m.id + )); + } + for (i, pm) in m.mods.iter().enumerate() { + let hex = pm.sha256.len() == 64 + && pm + .sha256 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)); + if !id_ok(&pm.id) || !hex || pm.version.is_empty() { + return Err(format!( + "{MANIFEST}: mod {:?} needs an id, a version and a lowercase hex sha256", + pm.id + )); + } + if m.mods[..i].iter().any(|other| other.id == pm.id) { + return Err(format!("{MANIFEST}: mod {} is listed twice", pm.id)); + } + } + Ok(m) + } +} + +/// A `.hpack` as read from disk: its manifest and its override files, by +/// path under `overrides/`. +#[derive(Debug)] +pub struct Modpack { + pub manifest: Manifest, + pub overrides: Vec, +} + +impl Modpack { + pub fn read(path: &Path) -> Result { + let file = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut zip = zip::ZipArchive::new(file).map_err(|e| e.to_string())?; + let mut text = String::new(); + zip.by_name(MANIFEST) + .map_err(|_| format!("{MANIFEST}: missing"))? + .take(1 << 20) + .read_to_string(&mut text) + .map_err(|e| format!("{MANIFEST}: {e}"))?; + let manifest = Manifest::parse(&text)?; + let (mut overrides, mut total) = (Vec::new(), 0u64); + for i in 0..zip.len() { + let entry = zip.by_index_raw(i).map_err(|e| e.to_string())?; + let name = entry.name(); + if name == MANIFEST || entry.is_dir() { + continue; + } + let Some(rest) = name.strip_prefix(OVERRIDES) else { + return Err(format!( + "{name}: a modpack holds only {MANIFEST} and overrides/" + )); + }; + check_override(rest, &manifest)?; + if entry.size() > MAX_OVERRIDE { + return Err(format!("{name} is over {} MiB", MAX_OVERRIDE >> 20)); + } + total += entry.size(); + overrides.push(rest.to_owned()); + } + if total > MAX_OVERRIDES { + return Err(format!("overrides are over {} MiB", MAX_OVERRIDES >> 20)); + } + overrides.sort(); + Ok(Modpack { + manifest, + overrides, + }) + } + + /// Writes the overrides of the pack at `path` into the game folder + /// `home`, each checked again. + pub fn extract_overrides(path: &Path, home: &Path) -> Result, String> { + let pack = Modpack::read(path)?; + let file = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut zip = zip::ZipArchive::new(file).map_err(|e| e.to_string())?; + let mut written = Vec::new(); + for name in &pack.overrides { + let mut entry = zip + .by_name(&format!("{OVERRIDES}{name}")) + .map_err(|e| format!("{name}: {e}"))?; + let target = home.join(name); + if let Some(dir) = target.parent() { + std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + } + let mut out = + std::fs::File::create(&target).map_err(|e| format!("{}: {e}", target.display()))?; + std::io::copy(&mut (&mut entry).take(MAX_OVERRIDE), &mut out) + .map_err(|e| format!("{}: {e}", target.display()))?; + written.push(target); + } + Ok(written) + } +} + +/// An override's path under `overrides/`: a settings file, a resource pack +/// zip, or the stored settings of one of the pack's mods. Plain parts only. +fn check_override(path: &str, manifest: &Manifest) -> Result<(), String> { + let bad = || { + format!( + "overrides/{path}: only {}, resourcepacks/.zip and horizon-mods/{STORAGE_DIR}// may be overridden", + SETTINGS.join(", ") + ) + }; + let parts: Vec<&str> = path.split('/').collect(); + let plain = |p: &&str| { + !p.is_empty() + && !p.starts_with('.') + && p.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-+ ".contains(&b)) + }; + let ok = match parts.as_slice() { + [file] => SETTINGS.contains(file), + ["resourcepacks", name] => plain(name) && name.ends_with(".zip"), + ["horizon-mods", dir, id, rest @ ..] => { + *dir == STORAGE_DIR + && manifest.mods.iter().any(|m| m.id == *id) + && !rest.is_empty() + && rest.iter().all(plain) + } + _ => false, + }; + if ok { Ok(()) } else { Err(bad()) } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + const PACK: &str = r#" +format = 1 +id = "cozy" +name = "Cozy" +version = "1.0.0" +game = "26.3" +[[mods]] +id = "coords_hud" +version = "0.1.0" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +"#; + + fn write_pack(dir: &Path, manifest: &str, files: &[(&str, &[u8])]) -> PathBuf { + let path = dir.join("cozy.hpack"); + let mut zip = zip::ZipWriter::new(std::fs::File::create(&path).unwrap()); + let options = zip::write::SimpleFileOptions::default(); + zip.start_file(MANIFEST, options).unwrap(); + zip.write_all(manifest.as_bytes()).unwrap(); + for (name, bytes) in files { + zip.start_file(*name, options).unwrap(); + zip.write_all(bytes).unwrap(); + } + zip.finish().unwrap(); + path + } + + fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("horizon-hpack-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn reads_a_pack_and_writes_its_overrides() { + let dir = scratch("read"); + let storage = format!("overrides/horizon-mods/{STORAGE_DIR}/coords_hud/corner.txt"); + let path = write_pack( + &dir, + PACK, + &[ + ("overrides/options.txt", b"renderDistance:8\n"), + ("overrides/resourcepacks/Cozy Look.zip", b"zip"), + (&storage, b"top-left"), + ], + ); + let pack = Modpack::read(&path).unwrap(); + assert_eq!(pack.manifest.mods[0].id, "coords_hud"); + assert!(pack.manifest.mods[0].on); + assert_eq!(pack.overrides.len(), 3); + let home = dir.join("home"); + Modpack::extract_overrides(&path, &home).unwrap(); + let options = std::fs::read_to_string(home.join("options.txt")).unwrap(); + assert_eq!(options, "renderDistance:8\n"); + let corner = home.join(format!("horizon-mods/{STORAGE_DIR}/coords_hud/corner.txt")); + assert!(corner.is_file()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn refuses_what_a_pack_may_not_hold() { + let dir = scratch("refuse"); + for name in [ + "mods/evil.jar", + "overrides/../../.bashrc", + "overrides/horizon-mods/evil.hmod", + "overrides/resourcepacks/../options.txt", + "overrides/servers.dat", + "overrides/horizon-mods/.horizon-mod-data/other_mod/x", + "overrides/resourcepacks/.hidden.zip", + ] { + let path = write_pack(&dir, PACK, &[(name, b"x")]); + assert!(Modpack::read(&path).is_err(), "{name} was accepted"); + } + let twice = format!("{PACK}{}", &PACK[PACK.find("[[mods]]").unwrap()..]); + assert!(Manifest::parse(&twice).unwrap_err().contains("twice")); + let newer = PACK.replace("format = 1", "format = 2"); + assert!(Manifest::parse(&newer).unwrap_err().contains("format 2")); + let short = PACK.replace("0000000000000000", "00"); + assert!(Manifest::parse(&short).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/horizon-mods/src/runtime.rs b/crates/horizon-mods/src/runtime.rs index 9dd2eb54e..52da641b4 100644 --- a/crates/horizon-mods/src/runtime.rs +++ b/crates/horizon-mods/src/runtime.rs @@ -49,9 +49,6 @@ const STORAGE_MAX_KEYS: usize = 64; /// a cheap-in-fuel loop replacing one key with large payloads needs its own /// cap, separate from the on-disk quota it would otherwise stay under. const STORAGE_WRITE_BUDGET: u64 = 1 << 20; -/// A mod's data, beside its `horizon-mods` folder (`find`'s `path.parent()` -/// never has a manifest there, so `find` never mistakes it for a mod). -const STORAGE_DIR: &str = ".horizon-mod-data"; /// What mods may read of the game, copied once per tick by the game. #[derive(Clone, Debug, Default)] @@ -237,7 +234,7 @@ impl Mods { let data_dir = path .parent() .unwrap_or(Path::new(".")) - .join(STORAGE_DIR) + .join(crate::manifest::STORAGE_DIR) .join(&m.id); let state = State { name: m.name.clone(), diff --git a/crates/horizon-mods/tests/index/coords_hud-0.1.0.hmod b/crates/horizon-mods/tests/index/coords_hud-0.1.0.hmod new file mode 100644 index 0000000000000000000000000000000000000000..1a4a5ba35e922e3837f43a4c85ae532885f95956 GIT binary patch literal 438 zcmWIWW@Zs#U|`??Vnqg{g-3vzl6r(a7JuqU{R$1LbxwMuKUL{jDkW%b}I^Y(EnZ0!2-=l}P9)q{Knvnp5Juk@9e zqGxlX&LA)==HZ8bZ4cY`cDlZZ|~@Tc;v&WexPvjzn;Ig z&*cM9{C>{ZTZRi75}?ok`MEr?I5*it=Ui$^LIMLrfHxzP2s7@`0%`(-24Lu6h8?FzzT36_V0ctN$u+$EayiFBiS(Qnx9%$Lz5emRdaZeBKlskns86f<A=Xu84Z|kN9Oh8i@ ynM9azhaS)KywhREqEtx6+ literal 0 HcmV?d00001 diff --git a/crates/horizon-mods/tests/index/index-v1.json b/crates/horizon-mods/tests/index/index-v1.json new file mode 100644 index 000000000..215c94667 --- /dev/null +++ b/crates/horizon-mods/tests/index/index-v1.json @@ -0,0 +1,109 @@ +{ + "schema": 1, + "generated": "2026-10-24T18:00:00Z", + "projects": [ + { + "id": "coords_hud", + "kind": "mod", + "name": "Coordinates HUD", + "summary": "Your coordinates, facing and the time of day in a corner of the screen.", + "description": "`/where` puts them in chat, `/coords` hides or shows the panel.", + "authors": [ + "Horizon" + ], + "owners": [ + "ludovic111" + ], + "license": "MIT", + "source": "https://github.com/ludovic111/horizon/tree/main/crates/horizon-mod-sdk/examples/coords-hud", + "icon": "https://ludovic111.github.io/horizon-mods/icons/coords_hud.png", + "key": "RWTVtuGb/GRXcBf3Re5hzM07EzoStr2sLTJjghJia0xX15ULoZRFFZDb", + "versions": [ + { + "version": "0.1.0", + "published": "2026-10-24", + "api": "0.1", + "side": "client", + "capabilities": [ + "game-read", + "hud", + "chat" + ], + "game": "26.3", + "file": { + "name": "coords_hud-0.1.0.hmod", + "url": "https://github.com/ludovic111/horizon-mods/releases/download/coords_hud-0.1.0/coords_hud-0.1.0.hmod", + "size": 438, + "sha256": "e1135345692f6dd8fd4c51920151944b82333e17049c9bb7d756f27b8deefc53", + "signature": "untrusted comment: signature from minisign secret key\nRUTVtuGb/GRXcIa5yCyCHFGFt55tj3T/FLUDv3ecNLVNNiQqQu7xyBrfc3wsvMHmudsEq6yr+QF21XN3+hnr/Ged3ULr5rv3zgE=\ntrusted comment: coords_hud 0.1.0\nIESQkJdTzLOvWegcGFxOMrQbnwJp2OKSSxoCKAxdfwyRb+9atdF+Q5/OpbdYurMnrNN1X1ybb43AszPVecdiCw==\n" + } + } + ] + }, + { + "id": "cozy", + "kind": "modpack", + "name": "Cozy", + "summary": "Coordinates and a calm render distance.", + "authors": [ + "Horizon" + ], + "owners": [ + "ludovic111" + ], + "license": "MIT", + "source": "https://github.com/ludovic111/horizon-mods", + "versions": [ + { + "version": "1.0.0", + "published": "2026-10-24", + "api": "1", + "game": "26.3", + "requires": [ + { + "id": "coords_hud", + "version": "0.1.0", + "sha256": "e1135345692f6dd8fd4c51920151944b82333e17049c9bb7d756f27b8deefc53" + } + ], + "file": { + "name": "cozy-1.0.0.hpack", + "url": "https://github.com/ludovic111/horizon-mods/releases/download/cozy-1.0.0/cozy-1.0.0.hpack", + "size": 471, + "sha256": "241ac840f9c5ea3f6252d849da10c90d36cb1da4d6cd8d1d268c2760a353cd65" + } + } + ] + }, + { + "id": "skyblock_start", + "kind": "world", + "name": "Skyblock start", + "summary": "A kind this reader doesn't know yet: kept, not offered.", + "license": "CC0-1.0", + "source": "https://example.org/skyblock", + "future_field": true, + "versions": [ + { + "version": "1", + "api": "1", + "game": "26.3", + "file": { + "name": "skyblock.hworld", + "url": "https://example.org/skyblock.hworld", + "size": 1, + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } + } + ] + } + ], + "revoked": [ + { + "id": "bad_mod", + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "reason": "Sent chat messages it didn't declare.", + "date": "2026-10-24" + } + ] +} diff --git a/crates/horizon-mods/tests/index/test.pub b/crates/horizon-mods/tests/index/test.pub new file mode 100644 index 000000000..5cb41bd62 --- /dev/null +++ b/crates/horizon-mods/tests/index/test.pub @@ -0,0 +1,2 @@ +untrusted comment: minisign public key 705764FC9BE1B6D5 +RWTVtuGb/GRXcBf3Re5hzM07EzoStr2sLTJjghJia0xX15ULoZRFFZDb diff --git a/docs/api/mod-index.md b/docs/api/mod-index.md new file mode 100644 index 000000000..959b969fb --- /dev/null +++ b/docs/api/mod-index.md @@ -0,0 +1,95 @@ +# The registry index and `.hpack` + +Horizon's registry lists what is made for Horizon and reviewed: mods (`.hmod`), modpacks (`.hpack`), resource packs +and, once their format exists, Horizon shader packs. Submissions are pull requests in +[ludovic111/horizon-mods](https://github.com/ludovic111/horizon-mods); merging one publishes it. The launcher and the +website read one file, `index.json`, which that repository's CI builds. The Rust types and every check below are +`horizon_mods::index` and `horizon_mods::modpack` (feature `index`); the sample is +`crates/horizon-mods/tests/index/index-v1.json`, read by the tests with the files it lists. + +Level: preview ([stability.md](stability.md)). Schema 1. + +## `index.json` + +```json +{ + "schema": 1, + "generated": "2026-10-24T18:00:00Z", + "projects": [{ + "id": "coords_hud", "kind": "mod", "name": "Coordinates HUD", "summary": "One line.", + "description": "Markdown.", "authors": ["Horizon"], "owners": ["ludovic111"], + "license": "MIT", "source": "https://github.com/...", "icon": "https://...png", "key": "RWT...", + "versions": [{ + "version": "0.1.0", "published": "2026-10-24", "changelog": "", + "api": "0.1", "side": "client", "capabilities": ["game-read", "hud", "chat"], "game": "26.3", + "file": { "name": "coords_hud-0.1.0.hmod", "url": "https://github.com/ludovic111/horizon-mods/releases/download/...", + "size": 438, "sha256": "e113...", "signature": "untrusted comment: ...\n...\ntrusted comment: ...\n...\n" } + }] + }], + "revoked": [{ "id": "bad_mod", "sha256": "1111...", "reason": "Sent chat messages it didn't declare.", "date": "2026-10-24" }] +} +``` + +| Field | Meaning | +| --- | --- | +| `schema` | 1. A reader refuses a higher number with "update Horizon" rather than misread it. | +| `projects[].id` | `[a-z0-9_]{1,64}`, unique; a mod's is its manifest's `id`. | +| `kind` | `mod`, `modpack`, `resourcepack`, `shader` (Horizon's own shader packs, not Iris or OptiFine packs). Other kinds (data packs, worlds, themes) may come without a new schema: a reader keeps a kind it doesn't know and doesn't offer it. | +| `summary`, `description` | One line for lists; Markdown for the project's page. | +| `owners` | GitHub logins allowed to publish updates. | +| `license`, `source` | An SPDX expression and an `https://` address of the source code: both required. | +| `icon` | `https://`, optional. | +| `key` | The author's [minisign](https://jedisct1.github.io/minisign/) public key. When set, every version must be signed with it; an update signed with another key is refused. Absent: the project is unsigned and the launcher says so. | +| `versions` | Newest first, at least one. | +| `api` | A mod's API line (its manifest's `api`); a modpack's manifest format (`1`). | +| `side`, `capabilities` | A mod's, read from its manifest by CI, never typed by hand. | +| `game` | The Minecraft version it is for. | +| `requires` | A modpack's mods: `{ id, version, sha256 }`, each a version in this index with that hash. | +| `file` | A plain name with the kind's extension (`.hmod`, `.hpack`, `.zip`), an `https://` URL, the size (at most 64 MiB), the lowercase hex SHA-256, and the `.minisig` text when the project has a key. | +| `revoked` | Files taken down: one file by `sha256`, or every version of `id` without it. The launcher refuses to install them and turns installed copies off with the reason. | + +Readers ignore fields they don't know, so an optional field can be added to schema 1. Renaming or removing a field, or +changing what one means, makes schema 2. + +## Before a file is installed + +`index::check_file` runs on the downloaded file, before it reaches the game folder: + +1. Its size and SHA-256 are the ones the entry lists, and it isn't revoked. +2. With a project key: its signature verifies with that key (minisign's prehashed and legacy signatures both). +3. What the file says matches the entry: a mod's `horizon-mod.toml` (id, version, API, side, capabilities) and its + content's fence ([mods.md](../mods.md#content-mods-api-02)); a modpack's manifest (id, version, format, game, + mods); a resource pack's `pack.mcmeta`. +4. A kind this build can't install is refused with "this Horizon can't install a … yet". + +The game still checks a mod's component imports against its capabilities when it starts it, as for any mod. + +## `.hpack` + +A zip holding `horizon-modpack.toml` and, optionally, `overrides/`. The mods are not inside: the launcher downloads +each from the index and checks it as above. + +```toml +format = 1 +id = "cozy" # [a-z0-9_]{1,64}: the profile's folder +name = "Cozy" +version = "1.0.0" +description = "Coordinates and a calm render distance." +authors = ["Horizon"] +game = "26.3" + +[[mods]] +id = "coords_hud" +version = "0.1.0" +sha256 = "e1135345692f6dd8fd4c51920151944b82333e17049c9bb7d756f27b8deefc53" +on = true # optional, default true: turned on in the profile's horizon-mods.txt +``` + +Unknown keys are refused. `overrides/` may hold only `options.txt`, `horizon-graphics.txt`, `horizon-extras.txt`, +`resourcepacks/.zip` and `horizon-mods/.horizon-mod-data//...` (a mod's own saved settings, +capability `storage`), each at most 64 MiB and 256 MiB in all; anything else refuses the pack. The launcher copies +them into the profile's game folder. + +## Changelog + +- Schema 1 (2026-10-24): kinds `mod`, `modpack`, `resourcepack`, `shader`; `.hpack` format 1. diff --git a/docs/mods.md b/docs/mods.md index 94abfd636..6f3216d38 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -19,7 +19,8 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | Authoring tool | `crates/cargo-horizon-mod` | `cargo horizon-mod new / build / run` | | In the game | `crates/horizon-client/src/game/mods.rs`, Options > Mods | client mods | | Launcher mod browser (Modrinth, CurseForge with a key) | [mod-browser.md](mod-browser.md) | resource packs, modpacks | -| `.hmod` index, signing, server mods, registries | this page, [Roadmap](#roadmap) | planned | +| Registry index (`index.json`, `.hpack`), signatures | `horizon_mods::index`, [api/mod-index.md](api/mod-index.md) | schema 1; launcher and website next | +| Server mods, registries | this page, [Roadmap](#roadmap) | planned | ## Design goals From be34f4cd5f1db4dc36eb44d92d0fccfdfd9afd54 Mon Sep 17 00:00:00 2001 From: some-du6e <103129948+some-du6e@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:20:35 -0600 Subject: [PATCH 08/51] Connect to a server's IPv4 address first, as vanilla does Vanilla resolves a server with InetAddress.getByName, and Java's default (java.net.preferIPv6Addresses=false) puts IPv4 addresses first. Horizon tried addresses in the system resolver's order, which on macOS puts IPv6 first: a server whose IPv6 address resets the connection showed "Connection reset by peer (os error 54)" where vanilla joined. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-protocol/src/connection.rs | 39 ++++++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/crates/horizon-protocol/src/connection.rs b/crates/horizon-protocol/src/connection.rs index 8f7ae28cc..e7d465288 100644 --- a/crates/horizon-protocol/src/connection.rs +++ b/crates/horizon-protocol/src/connection.rs @@ -6,7 +6,7 @@ use crate::{Error, Result, buf::Reader, crypto::Cfb8, frame}; use std::{ io::{Read, Write}, - net::{Shutdown, TcpStream, ToSocketAddrs}, + net::{Shutdown, SocketAddr, TcpStream, ToSocketAddrs}, sync::{ Arc, Condvar, Mutex, atomic::{AtomicI64, Ordering}, @@ -180,6 +180,13 @@ pub struct ConnWriter { pub threshold: Option, counters: Arc, } +/// InetAddress.getByName with Java's default `java.net.preferIPv6Addresses=false`: IPv4 addresses +/// first, whatever order the system resolver gives (macOS puts IPv6 first). A server whose IPv6 +/// address doesn't answer the game still lets vanilla in over IPv4. +fn ipv4_first(mut addrs: Vec) -> Vec { + addrs.sort_by_key(SocketAddr::is_ipv6); + addrs +} /// Both halves before the split. pub struct Connection { pub reader: ConnReader, @@ -187,10 +194,12 @@ pub struct Connection { } impl Connection { pub fn connect(host: &str, port: u16, timeout: Duration) -> Result { - let addrs: Vec<_> = (host, port) - .to_socket_addrs() - .map_err(|e| Error::new(format!("Unknown host {host}: {e}")))? - .collect(); + let addrs = ipv4_first( + (host, port) + .to_socket_addrs() + .map_err(|e| Error::new(format!("Unknown host {host}: {e}")))? + .collect(), + ); let mut last = Error::new(format!("Unknown host {host}")); for addr in addrs { match TcpStream::connect_timeout(&addr, timeout) { @@ -315,6 +324,26 @@ mod tests { use super::*; use std::net::TcpListener; + #[test] + fn ipv4_addresses_come_first_in_resolver_order() { + let a = |s: &str| s.parse::().unwrap(); + let resolved = vec![ + a("[2001:db8::1]:25565"), + a("192.0.2.1:25565"), + a("[2001:db8::2]:25565"), + a("10.0.0.1:25565"), + ]; + assert_eq!( + ipv4_first(resolved), + [ + a("192.0.2.1:25565"), + a("10.0.0.1:25565"), + a("[2001:db8::1]:25565"), + a("[2001:db8::2]:25565"), + ] + ); + } + #[test] fn loopback_with_compression_and_encryption() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); From 2716ad41bed4da8f7ac71e7a5bfe482cfc93881e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:16:05 +0200 Subject: [PATCH 09/51] Renderer: Metal's flat draws and occlusion culling switch on by sections in view Both switched on past 4,096 section draws, measured when a section had 2.2 to 2.5 draws. A section's draw count now depends on the sides it shows the eye, so the bars count sections in view: 1,600, where 4,096 draws fell. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/gfx.rs | 8 +++++--- crates/horizon-client/src/gfx/cull.rs | 22 ++++++++++++---------- crates/horizon-client/src/gfx/flat.rs | 22 ++++++++++++---------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index e4c3367f9..a848d36a5 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -2159,7 +2159,7 @@ impl Renderer { cull::Cull::new(&device, config.format.remove_srgb_suffix(), depth_test); cull.always = cull_env.as_deref() == Ok("all"); if !cull.always && adapter.get_info().backend == wgpu::Backend::Metal { - cull.min_draws = cull::TILED_MIN_DRAWS; + cull.min_sections = cull::TILED_MIN_SECTIONS; } cull }); @@ -3131,7 +3131,7 @@ impl Renderer { .cull .as_mut() .filter(|c| { - self.cull_on && !frame.title && self.multi_draw && self.section_draws >= c.min_draws + self.cull_on && !frame.title && self.multi_draw && visible.len() >= c.min_sections }) .and_then(|cull| cull.begin(&self.device).then_some(cull)); self.phase2_draws.clear(); @@ -3533,7 +3533,9 @@ impl Renderer { } let flat_frame = match &mut self.flat { Some(flat) - if self.flat_on && !self.draws.is_empty() && self.draws.len() >= flat.min_draws => + if self.flat_on + && !self.draws.is_empty() + && self.drawn_sections >= flat.min_sections => { let sections = &self.sections; let lods = &self.lods; diff --git a/crates/horizon-client/src/gfx/cull.rs b/crates/horizon-client/src/gfx/cull.rs index 856bd86dd..449401b96 100644 --- a/crates/horizon-client/src/gfx/cull.rs +++ b/crates/horizon-client/src/gfx/cull.rs @@ -46,13 +46,15 @@ const RETEST: u32 = 8; /// more than they save. Else one frame in `PROBE` tests every section. const MIN_SHARE: f32 = 0.1; const PROBE: u32 = 64; -/// On Metal, section draws from which a frame culls. Apple's GPUs store and -/// reload the whole frame at each of the passes the test adds, a fixed cost -/// that the sections it hides repay only at high render distances: on an M4 -/// Pro at 1080p and render distance 12 (1,000–1,600 draws) culling added -/// about 0.05 ms of GPU time while hiding 15–30% of the solid indices; at -/// 32 (drawn flat, over this many draws) it saved 0.25–1.6 ms. -pub const TILED_MIN_DRAWS: usize = 4096; +/// On Metal, sections in view from which a frame culls. Apple's GPUs store +/// and reload the whole frame at each of the passes the test adds, a fixed +/// cost that the sections it hides repay only at high render distances: on +/// an M4 Pro at 1080p and render distance 12 (1,000–1,600 draws) culling +/// added about 0.05 ms of GPU time while hiding 15–30% of the solid +/// indices; at 32 (over 4,096 draws) it saved 0.25–1.6 ms. Those draws were +/// 2.2 to 2.5 per section; a section's draw count now depends on the sides +/// it shows the eye (`mesher::Sides`), so the bar counts sections. +pub const TILED_MIN_SECTIONS: usize = 1600; /// Boxes over more than this share of the screen count as seen untested. const BIG: f32 = 0.25; /// Results read back at once, some frames late so the CPU never waits. @@ -148,8 +150,8 @@ pub struct Cull { pub hidden: ([usize; 2], f32), /// Cull every frame, whatever the hidden share (A/B checks). pub always: bool, - /// Frames with fewer section draws than this do not cull. - pub min_draws: usize, + /// Frames with fewer sections in view than this do not cull. + pub min_sections: usize, } fn buffer( @@ -348,7 +350,7 @@ impl Cull { stamp: 0, hidden: ([0; 2], 1.0), always: false, - min_draws: 0, + min_sections: 0, } } diff --git a/crates/horizon-client/src/gfx/flat.rs b/crates/horizon-client/src/gfx/flat.rs index f6823c5d8..ea6df0d5b 100644 --- a/crates/horizon-client/src/gfx/flat.rs +++ b/crates/horizon-client/src/gfx/flat.rs @@ -96,17 +96,19 @@ pub struct Flat { pub calls: Vec, pub phase1_calls: usize, pub solid_calls: usize, - /// Frames with fewer section draws keep the per-section draws. - pub min_draws: usize, + /// Frames with fewer sections in view keep the per-section draws. + pub min_sections: usize, range_align: u64, } -/// Section draws from which a frame is drawn flat. Below it the index list -/// costs the GPU more than the draw calls cost the CPU: on an M4 Pro at -/// render distance 16 (2,900 draws) the copy and the list add about 0.4 ms -/// of GPU time for 0.5–0.8 ms less CPU time; at 32 (9,300 draws) the CPU -/// saves 1.3 ms. -const MIN_DRAWS: usize = 4096; +/// Sections in view (and distant tiles) from which a frame is drawn flat. +/// Below it the index list costs the GPU more than the draw calls cost the +/// CPU: on an M4 Pro at render distance 16 (2,900 draws) the copy and the +/// list add about 0.4 ms of GPU time for 0.5–0.8 ms less CPU time; at 32 +/// (9,300 draws) the CPU saves 1.3 ms. The bar was 4,096 draws when they +/// were 2.2 to 2.5 per section; a section's draw count now depends on the +/// sides it shows the eye (`mesher::Sides`), so it counts sections. +const MIN_SECTIONS: usize = 1600; /// Whether flat draws apply: the backend emulates multi-draw, and vertex /// shaders can read storage buffers large enough for an index page. @@ -337,10 +339,10 @@ impl Flat { calls: Vec::new(), phase1_calls: 0, solid_calls: 0, - min_draws: if std::env::var("HORIZON_FLAT_DRAWS").as_deref() == Ok("1") { + min_sections: if std::env::var("HORIZON_FLAT_DRAWS").as_deref() == Ok("1") { 0 } else { - MIN_DRAWS + MIN_SECTIONS }, range_align: (u64::from(device.limits().min_storage_buffer_offset_alignment) / 16) .max(1), From d604409be256d03a861b94080cba657714d734e7 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:44:53 +0200 Subject: [PATCH 10/51] A client's own fall no longer hurts it: the server's hurt starts the tilt As a client (multiplayer, a LAN guest, single player on its server thread) Horizon ran its own fall damage, fire, freezing, berry bushes and starvation through the local hurt, so the camera's hurt tilt started even when the server cancelled the damage (lobbies that turn fall damage off). Vanilla's client never hurts itself: Entity.hurt does nothing outside a ServerLevel and FoodData.tick is ServerPlayer's. Its hurt animation comes from the server: LocalPlayer.hurtTo on a health drop, and ClientboundDamageEventPacket / ClientboundHurtAnimationPacket for itself. - PlayerMotion.client_side: hurt_blocking and food_tick do nothing; landing sounds and fall-distance tracking still run. - remote: SetHealth flashes on a drop after the first health, and the player's own damage events and hurt animations restart its hurt time. - horizon-ctl state reports the player's hurt_time. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game/agent.rs | 2 + crates/horizon-client/src/game/remote.rs | 28 ++++++++++-- crates/horizon-client/src/health.rs | 19 ++++++++ crates/horizon-client/src/hunger.rs | 3 ++ crates/horizon-client/src/motion.rs | 57 ++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 4 deletions(-) diff --git a/crates/horizon-client/src/game/agent.rs b/crates/horizon-client/src/game/agent.rs index 0eb215aa9..f6b68aa74 100644 --- a/crates/horizon-client/src/game/agent.rs +++ b/crates/horizon-client/src/game/agent.rs @@ -472,6 +472,8 @@ impl Game<'_> { "sprinting": self.keys.contains(&KeyCode::ControlLeft), "health": p.health.health(), "absorption": p.health.absorption(), + // The hurt animation's ticks left (the camera's hurt tilt). + "hurt_time": p.health.hurt_animation_time, "air": p.air, "food": p.health.food.food, "saturation": p.health.food.saturation, diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index 32cffa306..4efbedd9f 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -143,6 +143,9 @@ pub(super) struct Remote { /// Item entities waiting for their item data. pending_items: HashMap, health: (f32, i32, f32), + /// `LocalPlayer.flashOnSetHealth`: set by the first health the server + /// sends; from then on a drop starts the hurt animation. + flash_on_set_health: bool, game_mode: i32, pub chat: Vec<(String, Instant)>, /// Lines received since joining, for the control socket's events. @@ -411,6 +414,7 @@ impl Game<'_> { head_lerps: HashMap::new(), pending_items: HashMap::new(), health: (20.0, 20, 5.0), + flash_on_set_health: false, game_mode: 0, chat: Vec::new(), chat_received: 0, @@ -2128,14 +2132,24 @@ impl Game<'_> { // ClientPacketListener.handleDamageEvent: LivingEntity.handleDamageEvent // jolts the walk animation and restarts the hurt flash. Its hurt // sound comes as a sound from Horizon's host, which plays it itself. + // This player's own hurt tilt and flash come from the server too + // (sent to the player itself): its own moves never hurt it. C::DamageEvent(p) => { - if let Some(m) = self.mobs.iter_mut().find(|m| m.id == p.id as u32) { + if self.remote.as_ref().is_some_and(|r| r.entity_id == p.id) { + if let Some(pl) = self.player.as_mut() { + pl.health.animate_hurt(); + } + } else if let Some(m) = self.mobs.iter_mut().find(|m| m.id == p.id as u32) { m.limb_speed = 1.5; m.hurt_time = 10; } } C::HurtAnimation(p) => { - if let Some(m) = self.mobs.iter_mut().find(|m| m.id == p.id as u32) { + if self.remote.as_ref().is_some_and(|r| r.entity_id == p.id) { + if let Some(pl) = self.player.as_mut() { + pl.health.animate_hurt(); + } + } else if let Some(m) = self.mobs.iter_mut().find(|m| m.id == p.id as u32) { m.hurt_time = 10; } } @@ -2201,9 +2215,14 @@ impl Game<'_> { self.remote_world_options_difficulty(self.ui.difficulty, p.locked); } C::GameRuleValues(p) => self.remote_world_options_rules(p.entries), + // ClientPacketListener.handleSetHealth: LocalPlayer.hurtTo. C::SetHealth(p) => { - if let Some(r) = self.remote.as_mut() { + let flash = self.remote.as_mut().is_some_and(|r| { r.health = (p.health, p.food, p.saturation); + std::mem::replace(&mut r.flash_on_set_health, true) + }); + if flash && let Some(pl) = self.player.as_mut() { + pl.health.flash_health_change(p.health); } self.apply_remote_health(); if p.health <= 0.0 && self.screen != Screen::Dead && self.player.is_some() { @@ -3176,6 +3195,7 @@ impl Game<'_> { ); player.velocity = Vec3::ZERO; } else if !dead && !riding { + player.client_side = true; player.velocity = player.velocity.add_vector(pushed); MotionWorld { entities: &entity_collisions, @@ -3219,7 +3239,7 @@ impl Game<'_> { _ => {} } } - // The server owns health and hunger: undo local food/fall changes. + // The server owns health and hunger. self.apply_remote_health(); self.tick_avatar(before, false); // The player's own model (third person, the inventory's preview). diff --git a/crates/horizon-client/src/health.rs b/crates/horizon-client/src/health.rs index 9d53869ec..da40f41fc 100644 --- a/crates/horizon-client/src/health.rs +++ b/crates/horizon-client/src/health.rs @@ -253,6 +253,22 @@ impl PlayerHealth { self.hurt_animation_time -= 1; } } + /// `LivingEntity.animateHurt` / `handleDamageEvent` on a client: the hurt + /// tilt and red flash restart. + pub fn animate_hurt(&mut self) { + self.hurt.hurt_duration = 10; + self.hurt.hurt_time = 10; + self.hurt_animation_time = 10; + } + /// `LocalPlayer.hurtTo` with `flashOnSetHealth`: a drop below the current + /// health starts the hurt animation. + pub fn flash_health_change(&mut self, health: f32) { + let damage = self.health() - health; + if damage > 0.0 { + self.hurt.last_hurt = damage; + self.animate_hurt(); + } + } } impl PlayerMotion { pub fn hurt( @@ -274,6 +290,9 @@ impl PlayerMotion { amount: f32, blocking: Option, ) -> Result { + if self.client_side { + return Ok(BlockOutcome::default()); + } let source = resources .damage .source::<()>(source, None, None, None) diff --git a/crates/horizon-client/src/hunger.rs b/crates/horizon-client/src/hunger.rs index 0d9fb02e0..15eac144f 100644 --- a/crates/horizon-client/src/hunger.rs +++ b/crates/horizon-client/src/hunger.rs @@ -55,6 +55,9 @@ impl PlayerMotion { Ok(()) } pub fn food_tick(&mut self, resources: &HealthResources, rules: HealthRules) -> Result<()> { + if self.client_side { + return Ok(()); + } let h = &mut self.health; let mut world = TickWorld { resources, diff --git a/crates/horizon-client/src/motion.rs b/crates/horizon-client/src/motion.rs index 0a626b1e7..5da5d8e79 100644 --- a/crates/horizon-client/src/motion.rs +++ b/crates/horizon-client/src/motion.rs @@ -300,6 +300,10 @@ pub struct PlayerMotion { /// it travels on the server: `isLocalInstanceAuthoritative` is false, so /// its moves check no fall damage. pub server_copy: bool, + /// A client's copy of its own player (`Level.isClientSide`): only the + /// server hurts it (`Entity.hurt` does nothing on a client) and ticks its + /// food (`ServerPlayer.tick`). Its hurt animation comes from the server. + pub client_side: bool, /// `Entity.applyEffectsFromBlocks` runs after travel (always in the game; /// differential fixtures turn it off where their reference does). pub block_effects: bool, @@ -465,6 +469,7 @@ impl PlayerMotion { freeze_immune: false, old_position: position, server_copy: false, + client_side: false, block_effects: true, powder_snow_entered: Vec::new(), hurt_log: Vec::new(), @@ -2860,6 +2865,58 @@ mod tests { } } } + /// A client's own fall (multiplayer, a LAN guest, single player on its + /// server thread) plays its landing but never hurts it: only the server + /// does (`Entity.hurt` on a client), so a server that cancels fall damage + /// leaves the screen still. + #[test] + fn a_clients_own_fall_does_not_hurt_it() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../.local/horizon-client") + .join(horizon_core::VANILLA_VERSION); + if !root.join("movement-properties.nbt").exists() { + return; + } + let assets = Assets::load(&root).unwrap(); + let render = assets.reload(&root).unwrap(); + let binding = crate::collisions::load(&root, assets.registry).unwrap(); + let resources = MotionResources::load(&root, &assets).unwrap(); + for client_side in [false, true] { + let mut scene = World::test_scene(&assets, &render, 2).unwrap(); + let mut player = PlayerMotion::new(&resources, Vec3::new(20.5, 20.0, 20.5)).unwrap(); + player.client_side = client_side; + player.fall_distance = 30.0; + let mut w = MotionWorld { + entities: &[], + player: &mut player, + scene: &mut scene, + assets: &assets, + collision_binding: &binding, + resources: &resources, + }; + w.move_self(Vec3::new(0.0, -30.0, 0.0)).unwrap(); + assert!(w.player.ground.on_ground); + assert_eq!(w.player.fall_distance, 0.0); + // LivingEntity.causeFallDamage's sounds play on both sides. + assert!(w.player.events.iter().any(|e| matches!( + e, + MotionEvent::Fall(crate::landing::FallEvent::Sound { + sound: crate::landing::SceneSound::BigFall, + .. + }) + ))); + let hurt = w.player.health.health() < 20.0; + assert_eq!(hurt, !client_side); + assert_eq!(w.player.health.hurt_animation_time > 0, !client_side); + // The server's health then starts the client's hurt animation. + if client_side { + w.player.health.flash_health_change(20.0); + assert_eq!(w.player.health.hurt_animation_time, 0); + w.player.health.flash_health_change(12.0); + assert_eq!(w.player.health.hurt_animation_time, 10); + } + } + } #[test] fn a_riptide_spin_lies_flat_and_ends_on_the_first_entity_it_touches() { let root = Path::new(env!("CARGO_MANIFEST_DIR")) From 59281c5c7357c8e8659c491f3312b3db229607c6 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:42:17 +0200 Subject: [PATCH 11/51] Mod registry: cargo horizon-mod check, each kind's own checks - `horizon_mods::index::inspect`: each kind's own check (a mod's manifest and content, a modpack's manifest and overrides, a resource pack's pack.mcmeta), returning what the file says; `check_file` now compares those facts with the index. Shader packs and unknown kinds are refused, and `index::open` says which kinds the registry takes: shader submissions stay closed until Horizon's own format exists. - `cargo horizon-mod check `: those checks plus a trial run of a mod in the sandbox (ticks, frames, a chat line, each command), text or JSON. Co-Authored-By: Claude Opus 5.5 --- crates/cargo-horizon-mod/Cargo.toml | 4 +- crates/cargo-horizon-mod/src/check.rs | 268 +++++++++++++++++++++++++ crates/cargo-horizon-mod/src/main.rs | 38 +++- crates/horizon-mods/src/index.rs | 176 +++++++++++----- crates/horizon-mods/src/index/tests.rs | 5 +- docs/mods.md | 14 +- 6 files changed, 449 insertions(+), 56 deletions(-) create mode 100644 crates/cargo-horizon-mod/src/check.rs diff --git a/crates/cargo-horizon-mod/Cargo.toml b/crates/cargo-horizon-mod/Cargo.toml index 6d0a231ec..34eefe227 100644 --- a/crates/cargo-horizon-mod/Cargo.toml +++ b/crates/cargo-horizon-mod/Cargo.toml @@ -4,10 +4,10 @@ version = "0.1.0" edition.workspace = true license.workspace = true rust-version = "1.86" -description = "cargo horizon-mod new/build/run: make Horizon mods in Rust (docs/mods.md)" +description = "cargo horizon-mod new/build/run/check: make Horizon mods in Rust (docs/mods.md)" [dependencies] -horizon-mods = { path = "../horizon-mods", default-features = false, features = ["pack"] } +horizon-mods = { path = "../horizon-mods", default-features = false, features = ["pack", "runtime", "index"] } serde_json = "1.0.151" [lints] diff --git a/crates/cargo-horizon-mod/src/check.rs b/crates/cargo-horizon-mod/src/check.rs new file mode 100644 index 000000000..0294bec98 --- /dev/null +++ b/crates/cargo-horizon-mod/src/check.rs @@ -0,0 +1,268 @@ +//! `cargo horizon-mod check `: what the registry checks of a submission, +//! run anywhere: its kind's own checks (docs/api/mod-index.md), its signature +//! when given a key, and for a mod a trial run in the sandbox: it must start, +//! survive ticks, frames, a chat line and each of its commands, and import +//! no more than its capabilities. +use horizon_mods::{ + Clock, Mods, Output, Package, Player, Snapshot, Status, + index::{self, Facts, Kind}, + manifest::component_imports, +}; +use serde_json::{Value, json}; +use std::path::Path; + +/// How long the trial run plays: two seconds of game. +const TICKS: u64 = 40; +const FRAMES_PER_TICK: u64 = 3; + +pub struct Report { + pub kind: Kind, + pub facts: Facts, + pub sha256: String, + pub size: u64, + pub signed: bool, + /// A mod's component imports. + pub imports: Vec, + /// A mod's commands, each run once. + pub commands: Vec, + /// What the mod showed during the trial run. + pub output: Vec, +} + +impl Report { + pub fn json(&self) -> Value { + let requires: Vec = self + .facts + .requires + .iter() + .map(|r| json!({"id": r.id, "version": r.version, "sha256": r.sha256})) + .collect(); + json!({ + "kind": self.kind.name(), + "id": self.facts.id, + "version": self.facts.version, + "api": self.facts.api, + "side": self.facts.side, + "capabilities": self.facts.capabilities, + "game": self.facts.game, + "requires": requires, + "sha256": self.sha256, + "size": self.size, + "signed": self.signed, + "imports": self.imports, + "commands": self.commands, + "output": self.output, + }) + } + + pub fn text(&self) -> String { + let f = &self.facts; + let mut out = format!( + "{} {} ({}", + f.id.as_deref().unwrap_or("resource pack"), + f.version.as_deref().unwrap_or(""), + self.kind.name() + ); + if let Some(side) = f.side { + let side = serde_json::to_value(side).unwrap_or_default(); + out += &format!(", API {}, {}", f.api, side.as_str().unwrap_or("?")); + } else if !f.api.is_empty() { + out += &format!(", format {}", f.api); + } + if let Some(game) = &f.game { + out += &format!(", for {game}"); + } + let signed = if self.signed { + "signature verified" + } else { + "unsigned" + }; + out += &format!( + ")\n {} bytes, SHA-256 {}, {signed}\n", + self.size, self.sha256 + ); + for c in &f.capabilities { + out += &format!(" may {}\n", c.describe()); + } + for r in &f.requires { + out += &format!(" needs {} {} ({})\n", r.id, r.version, &r.sha256[..12]); + } + if self.kind == Kind::Mod { + let commands = if self.commands.is_empty() { + String::new() + } else { + format!(", /{}", self.commands.join(", /")) + }; + out += &format!( + " trial run: started, {TICKS} ticks, {} frames, a chat line{commands}: ok\n", + TICKS * FRAMES_PER_TICK + ); + } + out + } +} + +/// The kind a file's extension says. +pub fn kind_of(path: &Path) -> Option { + match path.extension()?.to_str()? { + "hmod" => Some(Kind::Mod), + "hpack" => Some(Kind::Modpack), + "zip" => Some(Kind::ResourcePack), + _ => None, + } +} + +/// Checks the file at `path` as `kind`; `key` and `signature` (the +/// `.minisig` text) check its signature. +pub fn check( + path: &Path, + kind: Kind, + key: Option<&str>, + signature: Option<&str>, +) -> Result { + let size = std::fs::metadata(path) + .map_err(|e| format!("{}: {e}", path.display()))? + .len(); + if size > index::MAX_FILE { + return Err(format!("over {} MiB", index::MAX_FILE >> 20)); + } + index::open(&kind)?; + let facts = index::inspect(&kind, path)?; + let signed = match (key, signature) { + (Some(key), Some(sig)) => { + index::verify_signature(key, sig, path)?; + true + } + (None, None) => false, + _ => return Err("--key and --signature go together".into()), + }; + let mut report = Report { + kind, + facts, + sha256: index::sha256_file(path)?, + size, + signed, + imports: Vec::new(), + commands: Vec::new(), + output: Vec::new(), + }; + if report.kind == Kind::Mod { + trial_run(path, &mut report)?; + } + Ok(report) +} + +/// Runs a mod in the sandbox, as the game does, from a scratch copy so its +/// storage lands in a folder removed afterwards. +fn trial_run(path: &Path, report: &mut Report) -> Result<(), String> { + let package = Package::read(path)?; + let wasm = package.wasm.as_deref().ok_or("the .hmod has no mod.wasm")?; + report.imports = component_imports(wasm)?; + package + .manifest + .check_imports(report.imports.iter().map(String::as_str))?; + let dir = std::env::temp_dir().join(format!("horizon-mod-check-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + let copy = dir.join("mod.hmod"); + let result = std::fs::copy(path, ©) + .map_err(|e| e.to_string()) + .and_then(|_| play(©, report)); + let _ = std::fs::remove_dir_all(&dir); + result +} + +fn play(path: &Path, report: &mut Report) -> Result<(), String> { + let mut mods = Mods::load(&[path.to_path_buf()], |_| true, |_| false); + let failed = |mods: &Mods| match &mods.entries()[0].status { + Status::Failed(reason) => Some(reason.clone()), + _ => None, + }; + if let Some(reason) = failed(&mods) { + return Err(format!("it didn't start: {reason}")); + } + report.commands = mods.commands().map(str::to_owned).collect(); + for t in 0..TICKS { + mods.set_snapshot(Snapshot { + player: Some(Player { + x: 0.5 + t as f64 * 0.1, + y: 64.0, + z: 0.5, + yaw: 90.0, + pitch: 0.0, + health: 20.0, + food: 20, + dimension: "minecraft:overworld".into(), + }), + clock: Some(Clock { + game_time: t, + day_time: 1000 + t, + }), + }); + mods.tick(t); + for _ in 0..FRAMES_PER_TICK { + mods.frame(1.0 / 60.0); + } + } + mods.chat(" hello"); + for name in report.commands.clone() { + mods.command(&name); + } + for out in mods.drain() { + report.output.push(match out { + Output::Chat { text, .. } => format!("chat: {text}"), + Output::Feedback { text, error } => { + format!("{}: {text}", if error { "error" } else { "feedback" }) + } + Output::Stopped { reason, .. } => return Err(format!("it stopped: {reason}")), + }); + } + match failed(&mods) { + Some(reason) => Err(format!("it stopped: {reason}")), + None => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture(name: &str) -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../horizon-mods/tests/index") + .join(name) + } + + #[test] + fn checks_each_kind_its_own_way() { + let pack = fixture("cozy-1.0.0.hpack"); + let report = check(&pack, kind_of(&pack).unwrap(), None, None).unwrap(); + assert_eq!(report.facts.requires[0].id, "coords_hud"); + assert!( + report + .text() + .contains("cozy 1.0.0 (modpack, format 1, for 26.3)") + ); + // The fixture mod's code is a stand-in: its manifest reads, its run fails. + let hmod = fixture("coords_hud-0.1.0.hmod"); + let key = std::fs::read_to_string(fixture("test.pub")).unwrap(); + let sig = std::fs::read_to_string(fixture("coords_hud-0.1.0.hmod.minisig")).unwrap(); + let key = key.lines().last().unwrap(); + let e = check(&hmod, Kind::Mod, Some(key), Some(&sig)) + .err() + .unwrap(); + assert!(e.contains("didn't start"), "{e}"); + let e = check( + &hmod, + Kind::Mod, + Some(key), + Some("untrusted comment: x\nRUQ=\n"), + ) + .err() + .unwrap(); + assert!(e.contains("signature"), "{e}"); + let e = check(&pack, Kind::Shader, None, None).err().unwrap(); + assert!(e.contains("shader packs open"), "{e}"); + assert!(check(&pack, Kind::Mod, None, None).is_err()); + } +} diff --git a/crates/cargo-horizon-mod/src/main.rs b/crates/cargo-horizon-mod/src/main.rs index 6b088d278..9e0076083 100644 --- a/crates/cargo-horizon-mod/src/main.rs +++ b/crates/cargo-horizon-mod/src/main.rs @@ -4,10 +4,14 @@ //! - `build [--debug]`: the `.hmod` in `target/horizon-mod/`. //! - `run [--game ] [-- ]`: build, then play with //! the mod; run `build` again and the game reloads it. +//! - `check [--json]`: what the registry checks of a submission +//! (`.hmod`, `.hpack`, resource pack `.zip`), with a trial run for a mod. use horizon_mods::{ Manifest, Package, manifest::{MANIFEST, component_imports, componentize}, }; +mod check; + use std::{ path::{Path, PathBuf}, process::{Command, ExitCode, Stdio}, @@ -28,7 +32,8 @@ fn main() -> ExitCode { println!("Built {}", p.display()); }), Some("run") => run(&args[1..]), - _ => Err("usage: cargo horizon-mod new | build [--debug] | run [--game ] [-- ]".into()), + Some("check") => check(&args[1..]), + _ => Err("usage: cargo horizon-mod new | build [--debug] | run [--game ] [-- ] | check [--kind ] [--key --signature ] [--json]".into()), }; match result { Ok(()) => ExitCode::SUCCESS, @@ -150,6 +155,37 @@ fn build(release: bool) -> Result { Ok(path) } +/// `check `: the registry's checks of a submission, printed; fails +/// when one does. +fn check(args: &[String]) -> Result<(), String> { + let file = args + .first() + .filter(|a| !a.starts_with("--")) + .ok_or("cargo horizon-mod check ")?; + let path = Path::new(file); + let kind = match flag(args, "--kind") { + Some(k) => horizon_mods::index::Kind::from(k.to_string()), + None => check::kind_of(path) + .ok_or("pass --kind mod|modpack|resourcepack: the extension doesn't say")?, + }; + let signature = flag(args, "--signature") + .map(|p| std::fs::read_to_string(p).map_err(|e| format!("{p}: {e}"))) + .transpose()?; + let report = check::check(path, kind, flag(args, "--key"), signature.as_deref()); + if args.iter().any(|a| a == "--json") { + let json = match &report { + Ok(r) => serde_json::json!({"ok": true, "report": r.json()}), + Err(e) => serde_json::json!({"ok": false, "error": e}), + }; + println!("{json}"); + } + let report = report?; + if !args.iter().any(|a| a == "--json") { + print!("{}", report.text()); + } + Ok(()) +} + fn run(args: &[String]) -> Result<(), String> { let path = build(true)?; let game = flag(args, "--game") diff --git a/crates/horizon-mods/src/index.rs b/crates/horizon-mods/src/index.rs index 73af7b46d..692b1ae73 100644 --- a/crates/horizon-mods/src/index.rs +++ b/crates/horizon-mods/src/index.rs @@ -448,60 +448,109 @@ pub fn check_file( (None, None) => Trust::Unsigned, _ => return Err("its signature and the project's key don't go together".into()), }; - match project.kind { - Kind::Mod => { - let m = Package::read(path)?.manifest; - crate::Content::read(path, &m)?; - let mut caps = m.capabilities.clone(); - caps.sort(); - let mut listed = version.capabilities.clone(); - listed.sort(); - let differs = m.id != project.id - || m.version != version.version - || m.api != version.api - || Some(m.side) != version.side - || caps != listed; - if differs { - return Err(format!( - "its horizon-mod.toml ({} {}, API {}, {:?}, {}) isn't what the index lists", - m.id, - m.version, - m.api, - m.side, - names(&caps) - )); - } - } - Kind::ResourcePack => check_resource_pack(path)?, - Kind::Shader | Kind::Other(_) => { - return Err(format!( - "this Horizon can't install a {} yet", - project.kind.name() - )); - } - Kind::Modpack => { - let pack = Modpack::read(path)?; - let m = &pack.manifest; - let mods: Vec = m.mods.iter().map(modpack::PackMod::requirement).collect(); - if m.id != project.id - || m.version != version.version - || m.format.to_string() != version.api - || m.game != version.game - || mods != version.requires - { - return Err(format!( - "its {} isn't what the index lists", - modpack::MANIFEST - )); - } - } + if matches!(project.kind, Kind::Shader | Kind::Other(_)) { + return Err(format!( + "this Horizon can't install a {} yet", + project.kind.name() + )); } + inspect(&project.kind, path)?.matches(project, version)?; Ok(trust) } +/// What a file says about itself, read by its kind's own check +/// ([`inspect`]). The registry's CI writes these into the index; the +/// launcher compares them with the index before it installs. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Facts { + /// A mod's or a modpack's manifest id; None for a resource pack. + pub id: Option, + pub version: Option, + /// A mod's API line, a modpack's format, a resource pack's pack format. + pub api: String, + pub side: Option, + /// In the manifest's order. + pub capabilities: Vec, + /// The Minecraft version, when the file names one (a modpack does). + pub game: Option, + pub requires: Vec, +} + +impl Facts { + /// Whether the file is the one `version` of `project` describes. + pub fn matches(&self, project: &Project, version: &Version) -> Result<(), String> { + let (mut caps, mut listed) = (self.capabilities.clone(), version.capabilities.clone()); + caps.sort(); + listed.sort(); + let differs = self.id.as_ref().is_some_and(|id| *id != project.id) + || self.version.as_ref().is_some_and(|v| *v != version.version) + || self.api != version.api + || self.side != version.side + || caps != listed + || self.game.as_ref().is_some_and(|g| *g != version.game) + || self.requires != version.requires; + if !differs { + return Ok(()); + } + Err(match project.kind { + Kind::Mod => format!( + "its horizon-mod.toml ({} {}, API {}, {:?}, {}) isn't what the index lists", + self.id.as_deref().unwrap_or("?"), + self.version.as_deref().unwrap_or("?"), + self.api, + self.side, + names(&self.capabilities) + ), + Kind::Modpack => format!("its {} isn't what the index lists", modpack::MANIFEST), + _ => format!("its pack format {} isn't what the index lists", self.api), + }) + } +} + +/// Reads and checks a file as its kind: a mod's manifest and content, a +/// modpack's manifest and overrides, a resource pack's `pack.mcmeta` and +/// paths. Shader packs and kinds this build doesn't know are refused: their +/// checks come with their formats. +pub fn inspect(kind: &Kind, path: &Path) -> Result { + match kind { + Kind::Mod => { + let m = Package::read(path)?.manifest; + crate::Content::read(path, &m)?; + Ok(Facts { + capabilities: m.capabilities, + id: Some(m.id), + version: Some(m.version), + api: m.api, + side: Some(m.side), + ..Facts::default() + }) + } + Kind::Modpack => { + let m = Modpack::read(path)?.manifest; + Ok(Facts { + requires: m.mods.iter().map(modpack::PackMod::requirement).collect(), + id: Some(m.id), + version: Some(m.version), + api: m.format.to_string(), + game: Some(m.game), + ..Facts::default() + }) + } + Kind::ResourcePack => Ok(Facts { + api: check_resource_pack(path)?, + ..Facts::default() + }), + Kind::Shader => Err( + "Horizon's own shader pack format isn't out yet: shader packs open when it is".into(), + ), + Kind::Other(name) => Err(format!("this Horizon can't read a {name} yet")), + } +} + /// A resource pack: a zip with `pack.mcmeta` at its root holding a `pack` -/// object, and only plain paths. -fn check_resource_pack(path: &Path) -> Result<(), String> { +/// object, and only plain paths. Returns its pack format (`pack_format`, or +/// `min_format` in newer packs), empty when it names none. +fn check_resource_pack(path: &Path) -> Result { let file = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; let mut zip = zip::ZipArchive::new(file).map_err(|e| e.to_string())?; for i in 0..zip.len() { @@ -523,10 +572,35 @@ fn check_resource_pack(path: &Path) -> Result<(), String> { .map_err(|e| format!("pack.mcmeta: {e}"))?; let json: serde_json::Value = serde_json::from_str(&meta).map_err(|e| format!("pack.mcmeta: {e}"))?; - if !json["pack"].is_object() { + let pack = &json["pack"]; + if !pack.is_object() { return Err("pack.mcmeta: no \"pack\" object".into()); } - Ok(()) + let format = [&pack["pack_format"], &pack["min_format"]] + .into_iter() + .find(|f| !f.is_null()) + .map(|f| match f { + serde_json::Value::Array(parts) => parts + .iter() + .map(ToString::to_string) + .collect::>() + .join("."), + other => other.to_string(), + }) + .unwrap_or_default(); + Ok(format) +} + +/// The kinds the registry takes submissions for. Shader packs wait for +/// Horizon's own format; other kinds for theirs. +pub fn open(kind: &Kind) -> Result<(), String> { + match kind { + Kind::Mod | Kind::Modpack | Kind::ResourcePack => Ok(()), + Kind::Shader => Err( + "shader packs open when Horizon's own shader pack format is out (Iris and OptiFine packs stay on Modrinth)".into(), + ), + Kind::Other(name) => Err(format!("the registry doesn't take {name} yet")), + } } fn names(caps: &[Capability]) -> String { diff --git a/crates/horizon-mods/src/index/tests.rs b/crates/horizon-mods/src/index/tests.rs index b6d7f5bfb..e3883f79e 100644 --- a/crates/horizon-mods/src/index/tests.rs +++ b/crates/horizon-mods/src/index/tests.rs @@ -151,7 +151,10 @@ fn resource_packs_need_a_pack_mcmeta() { path }; let meta = r#"{"pack": {"description": "Look", "min_format": 75, "max_format": 75}}"#; - assert_eq!(check_resource_pack(&pack(&[("pack.mcmeta", meta)])), Ok(())); + assert_eq!( + check_resource_pack(&pack(&[("pack.mcmeta", meta)])), + Ok("75".into()) + ); let e = check_resource_pack(&pack(&[("assets/x.png", "")])).unwrap_err(); assert!(e.contains("missing"), "{e}"); let e = check_resource_pack(&pack(&[("pack.mcmeta", "{}")])).unwrap_err(); diff --git a/docs/mods.md b/docs/mods.md index 6f3216d38..ac8afda94 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -16,7 +16,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | SDK for mod authors | `crates/horizon-mod-sdk` | 0.1 | | Example mod | `crates/horizon-mod-sdk/examples/coords-hud` | coordinates, facing and time HUD, `/where`, `/coords` | | Runtime | `crates/horizon-mods` | wasmtime 36 LTS, component model | -| Authoring tool | `crates/cargo-horizon-mod` | `cargo horizon-mod new / build / run` | +| Authoring tool | `crates/cargo-horizon-mod` | `cargo horizon-mod new / build / run / check` | | In the game | `crates/horizon-client/src/game/mods.rs`, Options > Mods | client mods | | Launcher mod browser (Modrinth, CurseForge with a key) | [mod-browser.md](mod-browser.md) | resource packs, modpacks | | Registry index (`index.json`, `.hpack`), signatures | `horizon_mods::index`, [api/mod-index.md](api/mod-index.md) | schema 1; launcher and website next | @@ -226,6 +226,18 @@ always on, and the game checks its file once a second: rebuild and it restarts with "Mods reloaded" in chat. Its in-memory state starts over; whatever it saved with capability `storage` survives the reload. +Before you submit it to the registry, run what its CI runs: + +```sh +cargo horizon-mod check target/horizon-mod/my_mod.hmod # --json for the report as JSON +``` + +It reads the manifest and content, refuses imports the capabilities don't +cover, then plays the mod in the sandbox for 40 ticks, 120 frames, a chat line +and each of its commands: it must start and never trap or overrun. It checks a +`.hpack` and a resource pack `.zip` too, and a signature with `--key --signature .minisig`. + ### Template layout ``` From 7d76923a04fbbae783e56577a93465dceeab8bec Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:42:57 +0200 Subject: [PATCH 12/51] Mod registry tools: the registry's entries, build and review - `horizon_mods::index::registry`: the registry repository's `mods/.toml` and `revoked.toml`, `build` (every entry and file checked, every problem listed, the index written) and `review` (a published file never changes or disappears, a kind never changes; non-owner, owner and key changes pointed out). - `cargo horizon-mod registry build|review|check|list` for the registry's CI. Co-Authored-By: Claude Opus 5.5 --- crates/cargo-horizon-mod/src/main.rs | 5 +- crates/cargo-horizon-mod/src/registry.rs | 159 +++++++ crates/horizon-mods/src/index.rs | 1 + crates/horizon-mods/src/index/registry.rs | 415 ++++++++++++++++++ .../horizon-mods/src/index/registry/tests.rs | 237 ++++++++++ docs/api/mod-index.md | 53 +++ docs/mods.md | 2 +- 7 files changed, 870 insertions(+), 2 deletions(-) create mode 100644 crates/cargo-horizon-mod/src/registry.rs create mode 100644 crates/horizon-mods/src/index/registry.rs create mode 100644 crates/horizon-mods/src/index/registry/tests.rs diff --git a/crates/cargo-horizon-mod/src/main.rs b/crates/cargo-horizon-mod/src/main.rs index 9e0076083..925633930 100644 --- a/crates/cargo-horizon-mod/src/main.rs +++ b/crates/cargo-horizon-mod/src/main.rs @@ -6,11 +6,13 @@ //! the mod; run `build` again and the game reloads it. //! - `check [--json]`: what the registry checks of a submission //! (`.hmod`, `.hpack`, resource pack `.zip`), with a trial run for a mod. +//! - `registry build|review|check|list`: the registry repository's CI. use horizon_mods::{ Manifest, Package, manifest::{MANIFEST, component_imports, componentize}, }; mod check; +mod registry; use std::{ path::{Path, PathBuf}, @@ -33,7 +35,8 @@ fn main() -> ExitCode { }), Some("run") => run(&args[1..]), Some("check") => check(&args[1..]), - _ => Err("usage: cargo horizon-mod new | build [--debug] | run [--game ] [-- ] | check [--kind ] [--key --signature ] [--json]".into()), + Some("registry") => registry::main(&args[1..]), + _ => Err("usage: cargo horizon-mod new | build [--debug] | run [--game ] [-- ] | check [--kind ] [--key --signature ] [--json] | registry build|review|check|list".into()), }; match result { Ok(()) => ExitCode::SUCCESS, diff --git a/crates/cargo-horizon-mod/src/registry.rs b/crates/cargo-horizon-mod/src/registry.rs new file mode 100644 index 000000000..ac5e77ea4 --- /dev/null +++ b/crates/cargo-horizon-mod/src/registry.rs @@ -0,0 +1,159 @@ +//! `cargo horizon-mod registry build|review|check|list`: the registry repository's CI +//! (`ludovic111/horizon-mods`, docs/api/mod-index.md). Network-free: the +//! workflow downloads the files and publishes what this writes. +use horizon_mods::index::{ + FILE_HOST, + registry::{self, Sources}, +}; +use std::path::Path; + +/// Where the registry's site is published (its icons). +const SITE: &str = "https://ludovic111.github.io/horizon-mods/"; + +const USAGE: &str = "usage: cargo horizon-mod registry build --files --out [--release ] [--site ] [--generated