From 07e46daa59e2c6238a443c7992beb10af1307266 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:41:16 +0200 Subject: [PATCH] Shader packs, clean room 1: read packs and their lang files A new crate, horizon-shaders, the first piece of shader pack support written again from a specification of the format. It opens a pack (a folder or a zip whose root holds a shaders folder; decoy folders and files beside it are ignored), lists the packs in a shaderpacks folder with the reason a broken one can't be used, and reads a pack's lang files: English under the player's locale, file names matched ignoring case (packs ship en_US.lang). Nothing uses it yet: players see no change. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 1 + Cargo.lock | 7 + Cargo.toml | 2 +- crates/horizon-shaders/Cargo.toml | 13 + crates/horizon-shaders/src/discover.rs | 82 +++++++ crates/horizon-shaders/src/lang.rs | 74 ++++++ crates/horizon-shaders/src/lib.rs | 15 ++ crates/horizon-shaders/src/pack.rs | 262 +++++++++++++++++++++ crates/horizon-shaders/tests/real_packs.rs | 43 ++++ 9 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 crates/horizon-shaders/Cargo.toml create mode 100644 crates/horizon-shaders/src/discover.rs create mode 100644 crates/horizon-shaders/src/lang.rs create mode 100644 crates/horizon-shaders/src/lib.rs create mode 100644 crates/horizon-shaders/src/pack.rs create mode 100644 crates/horizon-shaders/tests/real_packs.rs diff --git a/AGENTS.md b/AGENTS.md index f6f6b463b..0656e40fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,7 @@ Game workspace (root `Cargo.toml`); `launcher/` and `crates/horizon-telemetry-se | `horizon-friends`, `horizon-relay` | Play with Friends: invite codes, encrypted tunnel, the relay ([docs/play-with-friends.md](docs/play-with-friends.md)) | | `horizon-mods`, `horizon-mod-sdk`, `cargo-horizon-mod` | Sandboxed WebAssembly mods: runtime, SDK + WIT, `cargo horizon-mod` ([docs/mods.md](docs/mods.md)) | | `horizon-port-pack` | Porting Java modpacks ([docs/modpacks.md](docs/modpacks.md)) | +| `horizon-shaders` | Shader packs (the `shaders/` folder format), rebuilt clean-room from a written spec of the format; no wgpu | | `horizon-theme` | Design tokens and fonts for the launcher, the game's own menus and the website | | `horizon-telemetry`, `horizon-telemetry-server`, `horizon-bench` | Opt-in data collection, its ingest server, the benchmark report format | | `horizon-dualsense`, `horizon-threads`, `horizon-release-sign`, `horizon-bots`, `xtask` | DualSense output, thread priorities, signed release checksums, load-test bots, repo tasks | diff --git a/Cargo.lock b/Cargo.lock index 6da6a64b1..6c7279aa7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1430,6 +1430,13 @@ dependencies = [ "ureq", ] +[[package]] +name = "horizon-shaders" +version = "0.1.0" +dependencies = [ + "zip", +] + [[package]] name = "horizon-telemetry" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f197c34fe..b8e6b044e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] exclude = ["launcher", "crates/horizon-telemetry-server", "third_party/wgpu-hal"] -members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"] +members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-shaders", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"] resolver = "2" [workspace.package] diff --git a/crates/horizon-shaders/Cargo.toml b/crates/horizon-shaders/Cargo.toml new file mode 100644 index 000000000..ed3a24266 --- /dev/null +++ b/crates/horizon-shaders/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "horizon-shaders" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version = "1.85" +description = "Shader packs (the shaders/ folder format): reading packs, their .properties and .lang files, and later their GLSL" + +[dependencies] +zip = { version = "2", default-features = false, features = ["deflate"] } + +[lints] +workspace = true diff --git a/crates/horizon-shaders/src/discover.rs b/crates/horizon-shaders/src/discover.rs new file mode 100644 index 000000000..7d01d41b3 --- /dev/null +++ b/crates/horizon-shaders/src/discover.rs @@ -0,0 +1,82 @@ +//! The packs in the game's `shaderpacks` folder: every folder and `.zip` in it (spec 1.1). + +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::Pack; + +/// One entry of the `shaderpacks` folder that is or looks like a pack. +#[derive(Debug, PartialEq)] +pub struct Found { + /// The folder or file name, which is how players and settings name the pack. + pub name: String, + pub path: PathBuf, + /// Why the entry can't be used, said to the player (a zip that doesn't open, a folder with + /// no `shaders` folder at its root). `None` for a usable pack. + pub problem: Option, +} + +/// The packs in `dir`, sorted by name ignoring case. Hidden entries and files that are not +/// `.zip` (the game's own option files live beside the packs) are skipped. +pub fn discover(dir: &Path) -> Vec { + let mut found: Vec = fs::read_dir(dir) + .into_iter() + .flatten() + .flatten() + .filter_map(|entry| { + let name = entry.file_name().to_string_lossy().into_owned(); + let path = entry.path(); + let is_zip = path + .extension() + .is_some_and(|e| e.eq_ignore_ascii_case("zip")); + if name.starts_with('.') || !(path.is_dir() || is_zip) { + return None; + } + let problem = Pack::open(&path).err().map(|e| e.to_string()); + Some(Found { + name, + path, + problem, + }) + }) + .collect(); + found.sort_by_cached_key(|f| (f.name.to_lowercase(), f.name.clone())); + found +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pack::tests::{temp_dir, zip}; + + #[test] + fn lists_folders_and_zips_with_their_problems() { + let dir = temp_dir("discover"); + fs::create_dir_all(dir.join("Zeta/shaders")).unwrap(); + fs::create_dir_all(dir.join("Extracted wrong/Inner/shaders")).unwrap(); + fs::create_dir_all(dir.join(".hidden/shaders")).unwrap(); + fs::write(dir.join("alpha.ZIP"), zip(&[("shaders/final.fsh", "")])).unwrap(); + fs::write(dir.join("broken.zip"), "not a zip").unwrap(); + fs::write(dir.join("alpha.ZIP.txt"), "OPTION=1").unwrap(); + let found = discover(&dir); + let summary: Vec<(&str, bool)> = found + .iter() + .map(|f| (f.name.as_str(), f.problem.is_none())) + .collect(); + assert_eq!( + summary, + [ + ("alpha.ZIP", true), + ("broken.zip", false), + ("Extracted wrong", false), + ("Zeta", true) + ] + ); + assert_eq!( + found[2].problem.as_deref(), + Some("no shaders folder at the root of the pack") + ); + assert!(discover(&dir.join("missing")).is_empty()); + fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/horizon-shaders/src/lang.rs b/crates/horizon-shaders/src/lang.rs new file mode 100644 index 000000000..483de67d2 --- /dev/null +++ b/crates/horizon-shaders/src/lang.rs @@ -0,0 +1,74 @@ +//! A pack's `lang/.lang` files (spec 8.4): `key=value` lines naming options, values, +//! screens and profiles. + +use std::collections::HashMap; + +use crate::Pack; + +#[derive(Debug, Default, PartialEq)] +pub struct Lang(HashMap); + +impl Lang { + /// English with the player's locale over it, so a key the locale lacks still has a name. + /// File names match ignoring case: packs ship `en_US.lang` for Minecraft's `en_us`. + /// Strings keep their `§` formatting codes for the game's text renderer. + pub fn load(pack: &Pack, locale: &str) -> Self { + let files = pack.list("lang"); + let mut lang = Self::default(); + for wanted in ["en_us", locale] { + let file = files.iter().find(|f| { + f.strip_suffix(".lang") + .is_some_and(|stem| stem.eq_ignore_ascii_case(wanted)) + }); + if let Some(text) = file.and_then(|f| pack.read_text(&format!("lang/{f}"))) { + lang.read(&text); + } + } + lang + } + + fn read(&mut self, text: &str) { + for line in text.lines().map(str::trim) { + if line.starts_with('#') { + continue; + } + if let Some((key, value)) = line.split_once('=') { + self.0 + .insert(key.trim().to_owned(), value.trim().to_owned()); + } + } + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.0.get(key).map(String::as_str) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pack::tests::zip; + + #[test] + fn lang_falls_back_by_case_then_english() { + let pack = Pack::from_zip_bytes( + "P.zip", + zip(&[ + ( + "shaders/lang/en_US.lang", + "# names\noption.A=Alpha\noption.B = §6Beta§r\n", + ), + ("shaders/lang/fr_FR.lang", "option.A=Alpha (fr)\n"), + ]), + ) + .unwrap(); + let fr = Lang::load(&pack, "fr_fr"); + assert_eq!(fr.get("option.A"), Some("Alpha (fr)")); + assert_eq!(fr.get("option.B"), Some("§6Beta§r")); + assert_eq!(fr.get("# names"), None); + let de = Lang::load(&pack, "de_de"); + assert_eq!(de.get("option.A"), Some("Alpha")); + let none = Pack::from_zip_bytes("Q.zip", zip(&[("shaders/final.fsh", "")])).unwrap(); + assert_eq!(Lang::load(&none, "en_us"), Lang::default()); + } +} diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs new file mode 100644 index 000000000..1c9b46dd1 --- /dev/null +++ b/crates/horizon-shaders/src/lib.rs @@ -0,0 +1,15 @@ +//! Shader packs: the `shaderpacks/*.zip` format, written from Horizon's clean-room specification +//! (`shader-packs-spec.md`, revision 1). This crate reads the format and knows nothing of wgpu; +//! the game's renderer runs what it produces. +//! +//! - [`Pack`]: a pack as a folder or a zip, with paths relative to its `shaders` folder (spec 1.1). +//! - [`discover`]: the packs in a `shaderpacks` folder. +//! - [`Lang`]: the pack's translated names (8.4). + +mod discover; +mod lang; +mod pack; + +pub use discover::{Found, discover}; +pub use lang::Lang; +pub use pack::{Pack, PackError}; diff --git a/crates/horizon-shaders/src/pack.rs b/crates/horizon-shaders/src/pack.rs new file mode 100644 index 000000000..113db9ac9 --- /dev/null +++ b/crates/horizon-shaders/src/pack.rs @@ -0,0 +1,262 @@ +//! A pack is a folder or a `.zip` whose root holds a folder named `shaders`; everything else at +//! the root (licences, readmes, decoy folders) is ignored (spec 1.1). Every path a pack uses is +//! relative to that `shaders` folder, written with `/`. + +use std::collections::BTreeMap; +use std::fmt; +use std::fs; +use std::io::{self, Cursor, Read}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use zip::ZipArchive; + +const ROOT: &str = "shaders"; + +pub struct Pack { + name: String, + files: Files, +} + +enum Files { + /// The pack's `shaders` folder. + Folder(PathBuf), + /// The archive and its entries under `shaders/`, by their path inside that folder. + Zip { + archive: ZipArchive>>, + entries: BTreeMap, + }, +} + +#[derive(Debug)] +pub enum PackError { + Io(io::Error), + Zip(String), + /// A readable folder or archive with no `shaders` folder at its root. + NoShadersFolder, +} + +impl fmt::Display for PackError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Io(e) => write!(f, "{e}"), + Self::Zip(e) => write!(f, "not a readable zip: {e}"), + Self::NoShadersFolder => f.write_str("no shaders folder at the root of the pack"), + } + } +} + +impl std::error::Error for PackError {} + +impl From for PackError { + fn from(e: io::Error) -> Self { + Self::Io(e) + } +} + +impl Pack { + /// Opens a pack folder or a `.zip` archive. + pub fn open(path: &Path) -> Result { + let name = path + .file_name() + .map_or_else(String::new, |n| n.to_string_lossy().into_owned()); + let files = if path.is_dir() { + let root = path.join(ROOT); + if !root.is_dir() { + return Err(PackError::NoShadersFolder); + } + Files::Folder(root) + } else { + Self::zip_files(fs::read(path)?)? + }; + Ok(Self { name, files }) + } + + /// An archive held in memory; `name` is what [`Pack::name`] returns. + pub fn from_zip_bytes(name: &str, bytes: Vec) -> Result { + let files = Self::zip_files(bytes)?; + Ok(Self { + name: name.to_owned(), + files, + }) + } + + fn zip_files(bytes: Vec) -> Result { + let archive = ZipArchive::new(Cursor::new(Arc::from(bytes))) + .map_err(|e| PackError::Zip(e.to_string()))?; + let entries: BTreeMap = (0..archive.len()) + .filter_map(|i| { + let name = archive.name_for_index(i)?.replace('\\', "/"); + let inner = name.strip_prefix(ROOT)?.strip_prefix('/')?; + (!inner.is_empty() && !inner.ends_with('/')).then(|| (inner.to_owned(), i)) + }) + .collect(); + if entries.is_empty() { + return Err(PackError::NoShadersFolder); + } + Ok(Files::Zip { archive, entries }) + } + + /// The folder or file name the pack was opened from (`BSL.zip`). + pub fn name(&self) -> &str { + &self.name + } + + /// The bytes of a file, by its path inside `shaders` (`lang/en_us.lang`). `None` when it + /// doesn't exist or the path leaves the folder. + pub fn read(&self, path: &str) -> Option> { + let path = normalize(path)?; + match &self.files { + Files::Folder(root) => fs::read(root.join(&path)).ok(), + Files::Zip { archive, entries } => { + // A clone shares the bytes and the central directory; reading needs `&mut`. + let mut archive = archive.clone(); + let mut entry = archive.by_index(*entries.get(&path)?).ok()?; + let mut out = Vec::with_capacity(entry.size() as usize); + entry.read_to_end(&mut out).ok()?; + Some(out) + } + } + } + + /// A text file, decoded as UTF-8 (invalid bytes replaced) without its byte-order mark. + pub fn read_text(&self, path: &str) -> Option { + let bytes = self.read(path)?; + let text = String::from_utf8_lossy(&bytes); + Some(text.strip_prefix('\u{feff}').unwrap_or(&text).to_owned()) + } + + pub fn exists(&self, path: &str) -> bool { + let Some(path) = normalize(path) else { + return false; + }; + match &self.files { + Files::Folder(root) => root.join(path).is_file(), + Files::Zip { entries, .. } => entries.contains_key(&path), + } + } + + /// The names of the files directly inside a folder of the pack (`""` for `shaders` itself), + /// sorted. + pub fn list(&self, dir: &str) -> Vec { + let Some(dir) = normalize(dir) else { + return Vec::new(); + }; + let mut names: Vec = match &self.files { + Files::Folder(root) => fs::read_dir(root.join(&dir)) + .into_iter() + .flatten() + .flatten() + .filter(|e| e.file_type().is_ok_and(|t| t.is_file())) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect(), + Files::Zip { entries, .. } => { + let prefix = if dir.is_empty() { + String::new() + } else { + format!("{dir}/") + }; + entries + .keys() + .filter_map(|k| k.strip_prefix(&prefix)) + .filter(|rest| !rest.contains('/')) + .map(str::to_owned) + .collect() + } + }; + names.sort(); + names + } +} + +/// `a/./b/../c` → `a/c`; `None` for a path that climbs out of `shaders`. +fn normalize(path: &str) -> Option { + let mut parts: Vec<&str> = Vec::new(); + for part in path.split(['/', '\\']) { + match part { + "" | "." => {} + ".." => { + parts.pop()?; + } + _ => parts.push(part), + } + } + Some(parts.join("/")) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::io::Write; + + /// A zip of `(path, contents)` pairs, paths from the archive root. + pub(crate) fn zip(files: &[(&str, &str)]) -> Vec { + let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated); + for (path, contents) in files { + w.start_file(*path, options).unwrap(); + w.write_all(contents.as_bytes()).unwrap(); + } + w.finish().unwrap().into_inner() + } + + /// A fresh empty folder under the system's temp folder. + pub(crate) fn temp_dir(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("horizon-shaders-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn zip_packs_read_only_the_shaders_folder() { + let bytes = zip(&[ + ("License.txt", "x"), + ("shaderFile_A.placebo/shaders.properties", "decoy"), + ("shaders/shaders.properties", "sun=false"), + ("shaders/lang/en_US.lang", "a=b"), + ("shaders/lang/fr_FR.lang", "a=c"), + ("shaders/lib/common.glsl", ""), + ]); + let pack = Pack::from_zip_bytes("P.zip", bytes).unwrap(); + assert_eq!(pack.read_text("shaders.properties").unwrap(), "sun=false"); + assert!(pack.exists("lib/common.glsl")); + assert!(pack.exists("lib/../shaders.properties")); + assert!(!pack.exists("../License.txt")); + assert!(!pack.exists("License.txt")); + assert_eq!(pack.list("lang"), ["en_US.lang", "fr_FR.lang"]); + assert_eq!(pack.list(""), ["shaders.properties"]); + } + + #[test] + fn archives_without_a_shaders_folder_are_not_packs() { + let bytes = zip(&[("Pack/shaders/final.fsh", ""), ("readme.txt", "")]); + assert!(matches!( + Pack::from_zip_bytes("P.zip", bytes), + Err(PackError::NoShadersFolder) + )); + assert!(matches!( + Pack::from_zip_bytes("P.zip", b"not a zip".to_vec()), + Err(PackError::Zip(_)) + )); + } + + #[test] + fn folder_packs_read_the_same_way() { + let dir = temp_dir("folder"); + fs::create_dir_all(dir.join("shaders/lang")).unwrap(); + fs::write( + dir.join("shaders/lang/en_us.lang"), + "\u{feff}option.A=Alpha", + ) + .unwrap(); + fs::write(dir.join("secret.txt"), "outside").unwrap(); + let pack = Pack::open(&dir).unwrap(); + assert_eq!(pack.read_text("lang/en_us.lang").unwrap(), "option.A=Alpha"); + assert_eq!(pack.list("lang"), ["en_us.lang"]); + assert!(pack.read("../secret.txt").is_none()); + fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/horizon-shaders/tests/real_packs.rs b/crates/horizon-shaders/tests/real_packs.rs new file mode 100644 index 000000000..b35aef519 --- /dev/null +++ b/crates/horizon-shaders/tests/real_packs.rs @@ -0,0 +1,43 @@ +//! Reads real packs as black boxes: `HORIZON_TEST_SHADER_PACKS= cargo test -p +//! horizon-shaders --release -- --ignored --nocapture`. The folder (and its subfolders) holds +//! pack zips or folders, which are never committed; this prints what was read and how long it took. + +use std::path::Path; +use std::time::Instant; + +use horizon_shaders::{Lang, Pack, discover}; + +#[test] +#[ignore = "needs HORIZON_TEST_SHADER_PACKS"] +fn real_packs_read() { + let dir = std::env::var_os("HORIZON_TEST_SHADER_PACKS").expect("HORIZON_TEST_SHADER_PACKS"); + let dir = Path::new(&dir); + let mut found = discover(dir); + for sub in std::fs::read_dir(dir).unwrap().flatten() { + if sub.path().is_dir() { + found.extend(discover(&sub.path())); + } + } + let mut read = 0; + for f in found.iter().filter(|f| f.problem.is_none()) { + let start = Instant::now(); + let pack = Pack::open(&f.path).unwrap(); + let properties = pack.read_text("shaders.properties"); + let lang = Lang::load(&pack, "en_us"); + println!( + "{}: {} files at the root, shaders.properties {} bytes, lang {}, {:.2} ms", + f.name, + pack.list("").len(), + properties.as_ref().map_or(0, String::len), + if lang == Lang::default() { + "none" + } else { + "read" + }, + start.elapsed().as_secs_f64() * 1000.0 + ); + assert!(properties.is_some(), "{}: no shaders.properties", f.name); + read += 1; + } + assert!(read > 0, "no packs under {}", dir.display()); +}