diff --git a/crates/horizon-client/src/hud.rs b/crates/horizon-client/src/hud.rs index 0cc99b07b..b0c57f1d7 100644 --- a/crates/horizon-client/src/hud.rs +++ b/crates/horizon-client/src/hud.rs @@ -82,11 +82,13 @@ fn atlas_sprites(root: &Path, packed: &[PackedImage]) -> Vec { use crate::atlas_sources::{self, SpriteOrigin}; let have: std::collections::HashSet<&str> = packed.iter().map(|i| i.0.as_str()).collect(); let textures = atlas_sources::list_textures(root); - let load = |texture: &str, directory: &str| { - decode(&atlas_sources::texture_path(root, texture, directory)).ok() - }; - let mut out: Vec = Vec::new(); - let mut added = std::collections::HashSet::new(); + // Each new name with its origins in the atlases' order: the first that + // loads is the sprite. They load at once (a content pack's atlases list + // tens of thousands of files), and come out in the order a load of one + // after another would give. + let mut names: Vec<(String, Vec<(usize, SpriteOrigin)>)> = Vec::new(); + let mut slots = std::collections::HashMap::new(); + let mut order = 0; for (atlas, sources) in atlas_sources::atlases(root) { // The other atlases' renamed files (GUI sprites, particles...) are // looked up by their texture paths here. @@ -96,18 +98,33 @@ fn atlas_sprites(root: &Path, packed: &[PackedImage]) -> Vec { continue; } let name = id.strip_prefix("minecraft:").unwrap_or(&id).to_owned(); - if have.contains(name.as_str()) || added.contains(&name) { + if have.contains(name.as_str()) { continue; } - let image = match &origin { + // A vanilla file is packed under its own name already: only a + // renamed one is new. Other namespaces' files (content mods) are + // never in the walked directories. + if let SpriteOrigin::File(texture) = &origin + && texture.strip_prefix("minecraft:") == Some(name.as_str()) + { + continue; + } + let slot = *slots.entry(name.clone()).or_insert_with(|| { + names.push((name, Vec::new())); + names.len() - 1 + }); + names[slot].1.push((order, origin)); + order += 1; + } + } + let load = |texture: &str, directory: &str| { + decode(&atlas_sources::texture_path(root, texture, directory)).ok() + }; + let first = |(name, origins): &(String, Vec<(usize, SpriteOrigin)>)| { + origins.iter().find_map(|(order, origin)| { + let image = match origin { SpriteOrigin::File(texture) => { let path = texture.strip_prefix("minecraft:").unwrap_or(texture); - // A vanilla file is packed under its own name already: - // only a renamed one is new. Other namespaces' files - // (content mods) are never in the walked directories. - if texture.starts_with("minecraft:") && path == name { - continue; - } load(texture, "textures").map(|(w, h, p)| { // Animated strips keep their first frame, like the packed files. if h > w && (path.starts_with("block/") || path.starts_with("item/")) { @@ -119,13 +136,15 @@ fn atlas_sprites(root: &Path, packed: &[PackedImage]) -> Vec { } generated => atlas_sources::generate(generated, &load), }; - if let Some((w, h, pixels)) = image { - added.insert(name.clone()); - out.push((name, w, h, pixels)); - } - } - } - out + image.map(|(w, h, pixels)| (*order, (name.clone(), w, h, pixels))) + }) + }; + let mut out: Vec<_> = crate::par::par_map(&names, first) + .into_iter() + .flatten() + .collect(); + out.sort_unstable_by_key(|(order, _)| *order); + out.into_iter().map(|(_, image)| image).collect() } impl HudAtlas { /// A font glyph's atlas cell (8x8 GUI pixels) and advance. diff --git a/crates/horizon-client/src/lib.rs b/crates/horizon-client/src/lib.rs index 7a69de4aa..f5c19bdc6 100644 --- a/crates/horizon-client/src/lib.rs +++ b/crates/horizon-client/src/lib.rs @@ -131,6 +131,7 @@ pub mod osk; pub mod overlay; pub mod pack_screen; pub mod packs; +pub mod par; pub mod particles; pub mod patrol; pub mod perf_overlay; diff --git a/crates/horizon-client/src/mod_content.rs b/crates/horizon-client/src/mod_content.rs index 0497f06b4..52e6d30ff 100644 --- a/crates/horizon-client/src/mod_content.rs +++ b/crates/horizon-client/src/mod_content.rs @@ -996,6 +996,66 @@ impl TagMerge { } } +/// Threads rewriting the tables at once. Each holds a whole table as a tree +/// (gameplay-facts.json's is over a gigabyte with ATM10's mods), so more +/// would mostly add memory. +const TABLE_THREADS: usize = 3; + +/// Rewrites the base's per-state tables for the mods' states into the +/// overlay. They are independent: [`TABLE_THREADS`] threads take them, +/// the largest first. +fn tables(base: &Path, plan: &Plan, overlay: &Overlay<'_>) -> Result<()> { + let mut jobs: Vec<(&str, u64)> = JSON_TABLES + .iter() + .chain(&NBT_TABLES) + .filter_map(|name| Some((*name, fs::metadata(base.join(name)).ok()?.len()))) + .collect(); + jobs.sort_by_key(|(_, size)| std::cmp::Reverse(*size)); + // Their directories become real here, so the threads only write files. + for (name, _) in &jobs { + if let Some(parent) = Path::new(name).parent() { + overlay.dir(parent)?; + } + } + let table = |name: &str| -> Result<()> { + let path = base.join(name); + let bytes = if JSON_TABLES.contains(&name) { + let mut doc = read_json(&path)?; + plan.json(name, &mut doc) + .map_err(|e| format!("{name}: {e}"))?; + serde_json::to_vec(&doc)? + } else { + let (mut named, _) = nbt::read_named(&fs::read(&path)?, Limits::tables()) + .map_err(|e| format!("{name}: {e}"))?; + plan.nbt(&mut named.tag); + nbt::write_named(&named, Limits::tables()).map_err(|e| format!("{name}: {e}"))? + }; + overlay.write(name, &bytes) + }; + let next = std::sync::atomic::AtomicUsize::new(0); + let threads = std::thread::available_parallelism().map_or(1, |n| n.get()); + std::thread::scope(|scope| { + let workers: Vec<_> = (0..threads.min(TABLE_THREADS)) + .map(|_| { + scope.spawn(|| -> std::result::Result<(), String> { + loop { + let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let Some((name, _)) = jobs.get(i) else { + return Ok(()); + }; + table(name).map_err(|e| e.to_string())?; + } + }) + }) + .collect(); + workers.into_iter().try_for_each(|w| { + w.join() + .unwrap_or_else(|_| Err("a table thread panicked".into())) + }) + })?; + Ok(()) +} + /// Builds the content root for `mods` over `base` in `out`. Returns the /// warnings for the log: the shared tags a mod replaced. pub fn build(base: &Path, mods: &[ContentMod], out: &Path) -> Result> { @@ -1015,28 +1075,7 @@ pub fn build(base: &Path, mods: &[ContentMod], out: &Path) -> Result let base = &std::path::absolute(base)?; let plan = Plan::new(base, mods)?; let overlay = Overlay::new(base, out)?; - for name in JSON_TABLES { - let path = base.join(name); - if !path.exists() { - continue; - } - let mut doc = read_json(&path)?; - plan.json(name, &mut doc) - .map_err(|e| format!("{name}: {e}"))?; - overlay.write(name, &serde_json::to_vec(&doc)?)?; - } - for name in NBT_TABLES { - let path = base.join(name); - if !path.exists() { - continue; - } - let (mut named, _) = nbt::read_named(&fs::read(&path)?, Limits::tables()) - .map_err(|e| format!("{name}: {e}"))?; - plan.nbt(&mut named.tag); - let bytes = - nbt::write_named(&named, Limits::tables()).map_err(|e| format!("{name}: {e}"))?; - overlay.write(name, &bytes)?; - } + tables(base, &plan, &overlay)?; // Mod files, and every tag addition gathered per tag file. let mut tags: BTreeMap = BTreeMap::new(); let mut warnings = Vec::new(); @@ -1044,8 +1083,9 @@ pub fn build(base: &Path, mods: &[ContentMod], out: &Path) -> Result let mut additions: BTreeMap<&str, Vec> = BTreeMap::new(); for m in mods { let declarations = format!("data/{}/horizon/", m.id); + let mut reader = m.content.reader()?; for file in &m.content.files { - let bytes = m.content.file(file)?; + let bytes = reader.read(file)?; let is_tag = file.starts_with("data/") && file.contains("/tags/"); if is_tag && file.ends_with(".json") { let doc: J = serde_json::from_slice(&bytes).map_err(|e| format!("{file}: {e}"))?; diff --git a/crates/horizon-client/src/model_library.rs b/crates/horizon-client/src/model_library.rs index c6a1a9d87..018bc6af2 100644 --- a/crates/horizon-client/src/model_library.rs +++ b/crates/horizon-client/src/model_library.rs @@ -10,10 +10,12 @@ pub(crate) struct ParsedModels { impl ParsedModels { /// Every `models/**.json` of a namespace directory, by identifier. pub(crate) fn load(root: &Path) -> Self { - let mut out = BTreeMap::new(); let Ok(namespaces) = fs::read_dir(root.join("assets")) else { - return Self { models: out }; + return Self::default(); }; + // Listed first, then read and parsed at once: a content pack has tens + // of thousands of models. + let mut files = Vec::new(); for ns in namespaces.flatten() { let namespace = ns.file_name().to_string_lossy().into_owned(); let base = ns.path().join("models"); @@ -24,20 +26,33 @@ impl ParsedModels { }; for e in entries.flatten() { let path = e.path(); - if path.is_dir() { + // The entry's own type saves a stat per file; a link + // (content roots link into the base) is followed. + let is_dir = match e.file_type() { + Ok(t) if !t.is_symlink() => t.is_dir(), + _ => path.is_dir(), + }; + if is_dir { stack.push(path); } else if path.extension().is_some_and(|x| x == "json") && let Ok(rel) = path.strip_prefix(&base) - && let Ok(text) = fs::read_to_string(&path) - && let Ok(model) = CuboidModel::from_json(&text) { let rel = rel.with_extension("").to_string_lossy().replace('\\', "/"); - out.insert(format!("{namespace}:{rel}"), Arc::new(model)); + files.push((format!("{namespace}:{rel}"), path)); } } } } - Self { models: out } + let parsed = crate::par::par_map(&files, |(_, path)| { + let text = fs::read_to_string(path).ok()?; + CuboidModel::from_json(&text).ok().map(Arc::new) + }); + let models = files + .into_iter() + .zip(parsed) + .filter_map(|((id, _), model)| Some((id, model?))) + .collect(); + Self { models } } /// Blocks require every referenced resource, including parents. Items instead diff --git a/crates/horizon-client/src/par.rs b/crates/horizon-client/src/par.rs new file mode 100644 index 000000000..b4861b7a6 --- /dev/null +++ b/crates/horizon-client/src/par.rs @@ -0,0 +1,54 @@ +//! Loading many small files at once: startup reads tens of thousands of +//! textures and models with a content pack, and one at a time it waits on +//! each open and read in turn. +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// `f` over `items` on the machine's threads, results in `items`' order. +/// Threads take the next item as they finish one, so a run of slow items +/// doesn't hold back the others. +pub fn par_map(items: &[T], f: impl Fn(&T) -> R + Sync) -> Vec { + let threads = std::thread::available_parallelism() + .map_or(1, |n| n.get()) + .min(items.len()); + if threads <= 1 { + return items.iter().map(f).collect(); + } + let next = AtomicUsize::new(0); + let mut out: Vec<(usize, R)> = std::thread::scope(|scope| { + let workers: Vec<_> = (0..threads) + .map(|_| { + scope.spawn(|| { + let mut done = Vec::new(); + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(item) = items.get(i) else { + return done; + }; + done.push((i, f(item))); + } + }) + }) + .collect(); + workers + .into_iter() + .flat_map(|w| w.join().unwrap_or_else(|e| std::panic::resume_unwind(e))) + .collect() + }); + out.sort_unstable_by_key(|(i, _)| *i); + out.into_iter().map(|(_, r)| r).collect() +} + +#[cfg(test)] +mod tests { + use super::par_map; + + #[test] + fn results_keep_the_items_order() { + let items: Vec = (0..1000).collect(); + assert_eq!( + par_map(&items, |x| x * 2), + items.iter().map(|x| x * 2).collect::>() + ); + assert!(par_map(&[] as &[u32], |x| *x).is_empty()); + } +} diff --git a/crates/horizon-mods/src/content.rs b/crates/horizon-mods/src/content.rs index f17c2bb8a..494e734ec 100644 --- a/crates/horizon-mods/src/content.rs +++ b/crates/horizon-mods/src/content.rs @@ -147,6 +147,33 @@ pub struct Content { pub namespaces: Vec, } +/// Reads a mod's content files ([`Content::reader`]). +pub enum FileReader<'a> { + Dir(&'a Path), + Zip(Box>), +} +impl FileReader<'_> { + /// One content file's bytes. + pub fn read(&mut self, name: &str) -> Result, String> { + check_path(name)?; + let reader: Box = match self { + Self::Dir(root) => { + Box::new(std::fs::File::open(root.join(name)).map_err(|e| format!("{name}: {e}"))?) + } + Self::Zip(zip) => Box::new(zip.by_name(name).map_err(|e| format!("{name}: {e}"))?), + }; + let mut out = Vec::new(); + reader + .take(MAX_FILE + 1) + .read_to_end(&mut out) + .map_err(|e| format!("{name}: {e}"))?; + if out.len() as u64 > MAX_FILE { + return Err(format!("{name} is over {} MiB", MAX_FILE >> 20)); + } + Ok(out) + } +} + /// Why two content mods cannot be on together: a namespace both own (one's /// id or listed namespace is the other's), which would mix their files. /// `a` and `b` are (mod id, owned namespaces). @@ -222,29 +249,16 @@ impl Content { /// One content file's bytes. pub fn file(&self, name: &str) -> Result, String> { - check_path(name)?; - let mut out = Vec::new(); - match &self.source { - Source::Dir(root) => { - let file = - std::fs::File::open(root.join(name)).map_err(|e| format!("{name}: {e}"))?; - file.take(MAX_FILE + 1) - .read_to_end(&mut out) - .map_err(|e| format!("{name}: {e}"))?; - } - Source::Zip(path) => { - let mut zip = open_zip(path)?; - let entry = zip.by_name(name).map_err(|e| format!("{name}: {e}"))?; - entry - .take(MAX_FILE + 1) - .read_to_end(&mut out) - .map_err(|e| format!("{name}: {e}"))?; - } - } - if out.len() as u64 > MAX_FILE { - return Err(format!("{name} is over {} MiB", MAX_FILE >> 20)); - } - Ok(out) + self.reader()?.read(name) + } + + /// A reader of its content files. A packaged mod's archive is opened + /// once for all of them (opening it reads its whole index). + pub fn reader(&self) -> Result, String> { + Ok(match &self.source { + Source::Dir(root) => FileReader::Dir(root), + Source::Zip(path) => FileReader::Zip(Box::new(open_zip(path)?)), + }) } /// Where content may go, and the declarations.