From 8a03758c3a41c9d113864ddab3271bf523c6e8a6 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:47:03 +0200 Subject: [PATCH] Load time: build a pack's content root and load its sprites and models in parallel A content pack's first start rewrote 19 per-state tables one after another and reopened each zip mod's archive for every file it copied (its whole index each time). Every start then decoded the atlases' mod textures and parsed the mods' block models one file at a time on the main thread: with ATM10's 42 data mods that was about 70 % of the sampled startup. - mod_content: the tables are rewritten on up to three threads, largest first; a zip mod's archive is opened once for all its files (horizon-mods' Content::reader). - hud::atlas_sprites and ParsedModels::load list their files first, then decode and parse them on every core (par::par_map, results in the input order). The sprites come out in the order the one-by-one loop gave, and the content root is byte-identical. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/hud.rs | 59 ++++++++++----- crates/horizon-client/src/lib.rs | 1 + crates/horizon-client/src/mod_content.rs | 86 ++++++++++++++++------ crates/horizon-client/src/model_library.rs | 29 ++++++-- crates/horizon-client/src/par.rs | 54 ++++++++++++++ crates/horizon-mods/src/content.rs | 60 +++++++++------ 6 files changed, 216 insertions(+), 73 deletions(-) create mode 100644 crates/horizon-client/src/par.rs diff --git a/crates/horizon-client/src/hud.rs b/crates/horizon-client/src/hud.rs index 0cc99b07b..b0c57f1d7 100644 --- a/crates/horizon-client/src/hud.rs +++ b/crates/horizon-client/src/hud.rs @@ -82,11 +82,13 @@ fn atlas_sprites(root: &Path, packed: &[PackedImage]) -> Vec { use crate::atlas_sources::{self, SpriteOrigin}; let have: std::collections::HashSet<&str> = packed.iter().map(|i| i.0.as_str()).collect(); let textures = atlas_sources::list_textures(root); - let load = |texture: &str, directory: &str| { - decode(&atlas_sources::texture_path(root, texture, directory)).ok() - }; - let mut out: Vec = Vec::new(); - let mut added = std::collections::HashSet::new(); + // Each new name with its origins in the atlases' order: the first that + // loads is the sprite. They load at once (a content pack's atlases list + // tens of thousands of files), and come out in the order a load of one + // after another would give. + let mut names: Vec<(String, Vec<(usize, SpriteOrigin)>)> = Vec::new(); + let mut slots = std::collections::HashMap::new(); + let mut order = 0; for (atlas, sources) in atlas_sources::atlases(root) { // The other atlases' renamed files (GUI sprites, particles...) are // looked up by their texture paths here. @@ -96,18 +98,33 @@ fn atlas_sprites(root: &Path, packed: &[PackedImage]) -> Vec { continue; } let name = id.strip_prefix("minecraft:").unwrap_or(&id).to_owned(); - if have.contains(name.as_str()) || added.contains(&name) { + if have.contains(name.as_str()) { continue; } - let image = match &origin { + // A vanilla file is packed under its own name already: only a + // renamed one is new. Other namespaces' files (content mods) are + // never in the walked directories. + if let SpriteOrigin::File(texture) = &origin + && texture.strip_prefix("minecraft:") == Some(name.as_str()) + { + continue; + } + let slot = *slots.entry(name.clone()).or_insert_with(|| { + names.push((name, Vec::new())); + names.len() - 1 + }); + names[slot].1.push((order, origin)); + order += 1; + } + } + let load = |texture: &str, directory: &str| { + decode(&atlas_sources::texture_path(root, texture, directory)).ok() + }; + let first = |(name, origins): &(String, Vec<(usize, SpriteOrigin)>)| { + origins.iter().find_map(|(order, origin)| { + let image = match origin { SpriteOrigin::File(texture) => { let path = texture.strip_prefix("minecraft:").unwrap_or(texture); - // A vanilla file is packed under its own name already: - // only a renamed one is new. Other namespaces' files - // (content mods) are never in the walked directories. - if texture.starts_with("minecraft:") && path == name { - continue; - } load(texture, "textures").map(|(w, h, p)| { // Animated strips keep their first frame, like the packed files. if h > w && (path.starts_with("block/") || path.starts_with("item/")) { @@ -119,13 +136,15 @@ fn atlas_sprites(root: &Path, packed: &[PackedImage]) -> Vec { } generated => atlas_sources::generate(generated, &load), }; - if let Some((w, h, pixels)) = image { - added.insert(name.clone()); - out.push((name, w, h, pixels)); - } - } - } - out + image.map(|(w, h, pixels)| (*order, (name.clone(), w, h, pixels))) + }) + }; + let mut out: Vec<_> = crate::par::par_map(&names, first) + .into_iter() + .flatten() + .collect(); + out.sort_unstable_by_key(|(order, _)| *order); + out.into_iter().map(|(_, image)| image).collect() } impl HudAtlas { /// A font glyph's atlas cell (8x8 GUI pixels) and advance. diff --git a/crates/horizon-client/src/lib.rs b/crates/horizon-client/src/lib.rs index 7a69de4aa..f5c19bdc6 100644 --- a/crates/horizon-client/src/lib.rs +++ b/crates/horizon-client/src/lib.rs @@ -131,6 +131,7 @@ pub mod osk; pub mod overlay; pub mod pack_screen; pub mod packs; +pub mod par; pub mod particles; pub mod patrol; pub mod perf_overlay; diff --git a/crates/horizon-client/src/mod_content.rs b/crates/horizon-client/src/mod_content.rs index 0497f06b4..52e6d30ff 100644 --- a/crates/horizon-client/src/mod_content.rs +++ b/crates/horizon-client/src/mod_content.rs @@ -996,6 +996,66 @@ impl TagMerge { } } +/// Threads rewriting the tables at once. Each holds a whole table as a tree +/// (gameplay-facts.json's is over a gigabyte with ATM10's mods), so more +/// would mostly add memory. +const TABLE_THREADS: usize = 3; + +/// Rewrites the base's per-state tables for the mods' states into the +/// overlay. They are independent: [`TABLE_THREADS`] threads take them, +/// the largest first. +fn tables(base: &Path, plan: &Plan, overlay: &Overlay<'_>) -> Result<()> { + let mut jobs: Vec<(&str, u64)> = JSON_TABLES + .iter() + .chain(&NBT_TABLES) + .filter_map(|name| Some((*name, fs::metadata(base.join(name)).ok()?.len()))) + .collect(); + jobs.sort_by_key(|(_, size)| std::cmp::Reverse(*size)); + // Their directories become real here, so the threads only write files. + for (name, _) in &jobs { + if let Some(parent) = Path::new(name).parent() { + overlay.dir(parent)?; + } + } + let table = |name: &str| -> Result<()> { + let path = base.join(name); + let bytes = if JSON_TABLES.contains(&name) { + let mut doc = read_json(&path)?; + plan.json(name, &mut doc) + .map_err(|e| format!("{name}: {e}"))?; + serde_json::to_vec(&doc)? + } else { + let (mut named, _) = nbt::read_named(&fs::read(&path)?, Limits::tables()) + .map_err(|e| format!("{name}: {e}"))?; + plan.nbt(&mut named.tag); + nbt::write_named(&named, Limits::tables()).map_err(|e| format!("{name}: {e}"))? + }; + overlay.write(name, &bytes) + }; + let next = std::sync::atomic::AtomicUsize::new(0); + let threads = std::thread::available_parallelism().map_or(1, |n| n.get()); + std::thread::scope(|scope| { + let workers: Vec<_> = (0..threads.min(TABLE_THREADS)) + .map(|_| { + scope.spawn(|| -> std::result::Result<(), String> { + loop { + let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let Some((name, _)) = jobs.get(i) else { + return Ok(()); + }; + table(name).map_err(|e| e.to_string())?; + } + }) + }) + .collect(); + workers.into_iter().try_for_each(|w| { + w.join() + .unwrap_or_else(|_| Err("a table thread panicked".into())) + }) + })?; + Ok(()) +} + /// Builds the content root for `mods` over `base` in `out`. Returns the /// warnings for the log: the shared tags a mod replaced. pub fn build(base: &Path, mods: &[ContentMod], out: &Path) -> Result> { @@ -1015,28 +1075,7 @@ pub fn build(base: &Path, mods: &[ContentMod], out: &Path) -> Result let base = &std::path::absolute(base)?; let plan = Plan::new(base, mods)?; let overlay = Overlay::new(base, out)?; - for name in JSON_TABLES { - let path = base.join(name); - if !path.exists() { - continue; - } - let mut doc = read_json(&path)?; - plan.json(name, &mut doc) - .map_err(|e| format!("{name}: {e}"))?; - overlay.write(name, &serde_json::to_vec(&doc)?)?; - } - for name in NBT_TABLES { - let path = base.join(name); - if !path.exists() { - continue; - } - let (mut named, _) = nbt::read_named(&fs::read(&path)?, Limits::tables()) - .map_err(|e| format!("{name}: {e}"))?; - plan.nbt(&mut named.tag); - let bytes = - nbt::write_named(&named, Limits::tables()).map_err(|e| format!("{name}: {e}"))?; - overlay.write(name, &bytes)?; - } + tables(base, &plan, &overlay)?; // Mod files, and every tag addition gathered per tag file. let mut tags: BTreeMap = BTreeMap::new(); let mut warnings = Vec::new(); @@ -1044,8 +1083,9 @@ pub fn build(base: &Path, mods: &[ContentMod], out: &Path) -> Result let mut additions: BTreeMap<&str, Vec> = BTreeMap::new(); for m in mods { let declarations = format!("data/{}/horizon/", m.id); + let mut reader = m.content.reader()?; for file in &m.content.files { - let bytes = m.content.file(file)?; + let bytes = reader.read(file)?; let is_tag = file.starts_with("data/") && file.contains("/tags/"); if is_tag && file.ends_with(".json") { let doc: J = serde_json::from_slice(&bytes).map_err(|e| format!("{file}: {e}"))?; diff --git a/crates/horizon-client/src/model_library.rs b/crates/horizon-client/src/model_library.rs index c6a1a9d87..018bc6af2 100644 --- a/crates/horizon-client/src/model_library.rs +++ b/crates/horizon-client/src/model_library.rs @@ -10,10 +10,12 @@ pub(crate) struct ParsedModels { impl ParsedModels { /// Every `models/**.json` of a namespace directory, by identifier. pub(crate) fn load(root: &Path) -> Self { - let mut out = BTreeMap::new(); let Ok(namespaces) = fs::read_dir(root.join("assets")) else { - return Self { models: out }; + return Self::default(); }; + // Listed first, then read and parsed at once: a content pack has tens + // of thousands of models. + let mut files = Vec::new(); for ns in namespaces.flatten() { let namespace = ns.file_name().to_string_lossy().into_owned(); let base = ns.path().join("models"); @@ -24,20 +26,33 @@ impl ParsedModels { }; for e in entries.flatten() { let path = e.path(); - if path.is_dir() { + // The entry's own type saves a stat per file; a link + // (content roots link into the base) is followed. + let is_dir = match e.file_type() { + Ok(t) if !t.is_symlink() => t.is_dir(), + _ => path.is_dir(), + }; + if is_dir { stack.push(path); } else if path.extension().is_some_and(|x| x == "json") && let Ok(rel) = path.strip_prefix(&base) - && let Ok(text) = fs::read_to_string(&path) - && let Ok(model) = CuboidModel::from_json(&text) { let rel = rel.with_extension("").to_string_lossy().replace('\\', "/"); - out.insert(format!("{namespace}:{rel}"), Arc::new(model)); + files.push((format!("{namespace}:{rel}"), path)); } } } } - Self { models: out } + let parsed = crate::par::par_map(&files, |(_, path)| { + let text = fs::read_to_string(path).ok()?; + CuboidModel::from_json(&text).ok().map(Arc::new) + }); + let models = files + .into_iter() + .zip(parsed) + .filter_map(|((id, _), model)| Some((id, model?))) + .collect(); + Self { models } } /// Blocks require every referenced resource, including parents. Items instead diff --git a/crates/horizon-client/src/par.rs b/crates/horizon-client/src/par.rs new file mode 100644 index 000000000..b4861b7a6 --- /dev/null +++ b/crates/horizon-client/src/par.rs @@ -0,0 +1,54 @@ +//! Loading many small files at once: startup reads tens of thousands of +//! textures and models with a content pack, and one at a time it waits on +//! each open and read in turn. +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// `f` over `items` on the machine's threads, results in `items`' order. +/// Threads take the next item as they finish one, so a run of slow items +/// doesn't hold back the others. +pub fn par_map(items: &[T], f: impl Fn(&T) -> R + Sync) -> Vec { + let threads = std::thread::available_parallelism() + .map_or(1, |n| n.get()) + .min(items.len()); + if threads <= 1 { + return items.iter().map(f).collect(); + } + let next = AtomicUsize::new(0); + let mut out: Vec<(usize, R)> = std::thread::scope(|scope| { + let workers: Vec<_> = (0..threads) + .map(|_| { + scope.spawn(|| { + let mut done = Vec::new(); + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(item) = items.get(i) else { + return done; + }; + done.push((i, f(item))); + } + }) + }) + .collect(); + workers + .into_iter() + .flat_map(|w| w.join().unwrap_or_else(|e| std::panic::resume_unwind(e))) + .collect() + }); + out.sort_unstable_by_key(|(i, _)| *i); + out.into_iter().map(|(_, r)| r).collect() +} + +#[cfg(test)] +mod tests { + use super::par_map; + + #[test] + fn results_keep_the_items_order() { + let items: Vec = (0..1000).collect(); + assert_eq!( + par_map(&items, |x| x * 2), + items.iter().map(|x| x * 2).collect::>() + ); + assert!(par_map(&[] as &[u32], |x| *x).is_empty()); + } +} diff --git a/crates/horizon-mods/src/content.rs b/crates/horizon-mods/src/content.rs index f17c2bb8a..494e734ec 100644 --- a/crates/horizon-mods/src/content.rs +++ b/crates/horizon-mods/src/content.rs @@ -147,6 +147,33 @@ pub struct Content { pub namespaces: Vec, } +/// Reads a mod's content files ([`Content::reader`]). +pub enum FileReader<'a> { + Dir(&'a Path), + Zip(Box>), +} +impl FileReader<'_> { + /// One content file's bytes. + pub fn read(&mut self, name: &str) -> Result, String> { + check_path(name)?; + let reader: Box = match self { + Self::Dir(root) => { + Box::new(std::fs::File::open(root.join(name)).map_err(|e| format!("{name}: {e}"))?) + } + Self::Zip(zip) => Box::new(zip.by_name(name).map_err(|e| format!("{name}: {e}"))?), + }; + let mut out = Vec::new(); + reader + .take(MAX_FILE + 1) + .read_to_end(&mut out) + .map_err(|e| format!("{name}: {e}"))?; + if out.len() as u64 > MAX_FILE { + return Err(format!("{name} is over {} MiB", MAX_FILE >> 20)); + } + Ok(out) + } +} + /// Why two content mods cannot be on together: a namespace both own (one's /// id or listed namespace is the other's), which would mix their files. /// `a` and `b` are (mod id, owned namespaces). @@ -222,29 +249,16 @@ impl Content { /// One content file's bytes. pub fn file(&self, name: &str) -> Result, String> { - check_path(name)?; - let mut out = Vec::new(); - match &self.source { - Source::Dir(root) => { - let file = - std::fs::File::open(root.join(name)).map_err(|e| format!("{name}: {e}"))?; - file.take(MAX_FILE + 1) - .read_to_end(&mut out) - .map_err(|e| format!("{name}: {e}"))?; - } - Source::Zip(path) => { - let mut zip = open_zip(path)?; - let entry = zip.by_name(name).map_err(|e| format!("{name}: {e}"))?; - entry - .take(MAX_FILE + 1) - .read_to_end(&mut out) - .map_err(|e| format!("{name}: {e}"))?; - } - } - if out.len() as u64 > MAX_FILE { - return Err(format!("{name} is over {} MiB", MAX_FILE >> 20)); - } - Ok(out) + self.reader()?.read(name) + } + + /// A reader of its content files. A packaged mod's archive is opened + /// once for all of them (opening it reads its whole index). + pub fn reader(&self) -> Result, String> { + Ok(match &self.source { + Source::Dir(root) => FileReader::Dir(root), + Source::Zip(path) => FileReader::Zip(Box::new(open_zip(path)?)), + }) } /// Where content may go, and the declarations.