diff --git a/Cargo.lock b/Cargo.lock index 4d749add6..bb4e72446 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1408,6 +1408,8 @@ dependencies = [ name = "horizon-prepare" version = "0.1.0" dependencies = [ + "crc32fast", + "libc", "regex", "sha1", "sha2", diff --git a/crates/horizon-prepare/Cargo.toml b/crates/horizon-prepare/Cargo.toml index 3d62202bd..9bcf5464d 100644 --- a/crates/horizon-prepare/Cargo.toml +++ b/crates/horizon-prepare/Cargo.toml @@ -6,11 +6,15 @@ license = "MIT" description = "Prepares Horizon's resources from the player's own Minecraft: imports and checks the pinned jars, runs the vanilla exporters, writes the resource folder" [dependencies] +crc32fast = "1" regex = { version = "1", default-features = false, features = ["std", "unicode-perl"] } sha1 = "0.10" sha2 = "0.10" zip = { version = "2", default-features = false, features = ["deflate"] } +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +libc = "0.2" + [build-dependencies] sha2 = "0.10" diff --git a/crates/horizon-prepare/src/artifacts.rs b/crates/horizon-prepare/src/artifacts.rs index 805dd80d4..a3bf945d5 100644 --- a/crates/horizon-prepare/src/artifacts.rs +++ b/crates/horizon-prepare/src/artifacts.rs @@ -143,31 +143,17 @@ pub(crate) fn import_copy( algorithm: &str, hint: &str, ) -> Result { - fs::create_dir_all(destination.parent().unwrap()).map_err(|e| e.to_string())?; - if destination.exists() && digest(destination, algorithm)? == checksum { - return Ok(destination.to_path_buf()); - } - if !source.is_file() { - return Err(format!("Missing {}. {hint}", source.display()) - .trim() - .to_string()); - } - let actual = digest(source, algorithm)?; - if actual != checksum { - return Err(format!( - "{} is not the pinned file: expected {algorithm} {checksum}, got {actual}", - source.display() - )); - } - let temporary = destination.with_file_name(format!( - ".{}.partial-{}", - destination.file_name().unwrap().to_string_lossy(), - std::process::id() - )); - let copied = fs::copy(source, &temporary).and_then(|_| fs::rename(&temporary, destination)); - let _ = fs::remove_file(&temporary); - copied.map_err(|e| format!("{}: {e}", destination.display()))?; - Ok(destination.to_path_buf()) + crate::files::Import::stage(source, destination, checksum, algorithm, false) + .map_err(|error| { + if !source.is_file() { + format!("Missing {}. {hint}", source.display()) + .trim() + .to_string() + } else { + error + } + })? + .commit() } /// The pinned version JSON from the launcher's versions/ directory. The launcher's copy wins @@ -653,9 +639,17 @@ pub struct Artifacts { /// Reuses checked imports from this root or another installed version beside it. pub fn reuse_imported(root: &Path, log: Log) -> Result { + Ok(reuse_verified(root, log)?.is_some()) +} + +/// Keep this preparation's verified artifacts and client libraries for the exporters. +pub(crate) fn reuse_verified( + root: &Path, + log: Log, +) -> Result)>, String> { let root = fs::canonicalize(root).map_err(|e| format!("{}: {e}", root.display()))?; let lock = Lock::read(&root)?; - let checked = |directory: PathBuf| -> Result, String> { + let checked = |directory: PathBuf| -> Result<(Artifacts, Vec), String> { let candidate = Lock { json: lock.json.clone(), version: lock.version.clone(), @@ -664,17 +658,9 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { }; let found = verify(&candidate)?; let client = client_libraries(&candidate, &found.manifest)?; - let mut files = vec![ - candidate.directory.join("manifest.json"), - candidate.directory.join("jars/client.jar"), - candidate.directory.join("jars/server.jar"), - found.jars["server"].clone(), - ]; - files.extend(found.libraries); - files.extend(client); - Ok(files) + Ok((found, client)) }; - if checked(lock.directory.clone()).is_ok() { + if let Ok((found, client_libraries)) = checked(lock.directory.clone()) { if [ "prepared.json", "reports/client-inventory.json", @@ -683,8 +669,6 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { .iter() .any(|path| !lock.directory.join(path).is_file()) { - let found = verify(&lock)?; - let client_libraries = client_libraries(&lock, &found.manifest)?; write_import_metadata( &root, &lock, @@ -696,10 +680,10 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { )?; } log(&format!("Using imported Minecraft {}", lock.version)); - return Ok(true); + return Ok(Some((found, client_libraries))); } let Some(versions) = root.parent() else { - return Ok(false); + return Ok(None); }; // Sibling discovery is intended for the launcher's versions/ folder. let mut siblings: Vec<_> = fs::read_dir(versions) @@ -711,9 +695,17 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { siblings.sort(); for sibling in siblings { let source = sibling.join(".local/minecraft").join(&lock.version); - let Ok(files) = checked(source.clone()) else { + let Ok((found, client)) = checked(source.clone()) else { continue; }; + let mut files = vec![ + source.join("manifest.json"), + source.join("jars/client.jar"), + source.join("jars/server.jar"), + found.jars["server"].clone(), + ]; + files.extend(found.libraries); + files.extend(client); for file in files { let relative = file.strip_prefix(&source).map_err(|e| e.to_string())?; import_copy( @@ -740,9 +732,9 @@ pub fn reuse_imported(root: &Path, log: Log) -> Result { lock.version, sibling.display() )); - return Ok(true); + return Ok(Some((found, client_libraries))); } - Ok(false) + Ok(None) } pub fn verify(lock: &Lock) -> Result { diff --git a/crates/horizon-prepare/src/files.rs b/crates/horizon-prepare/src/files.rs new file mode 100644 index 000000000..484c7bf7b --- /dev/null +++ b/crates/horizon-prepare/src/files.rs @@ -0,0 +1,180 @@ +//! Independent copies, using filesystem cloning when available. Never hard links: exporters +//! rewrite prepared resources in place. Checked imports publish only after verifying the copy. + +use crate::{digest, hex}; +use sha2::Digest; +use std::fs; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +fn clone_file(from: &Path, to: &Path) -> io::Result<()> { + #[cfg(target_os = "linux")] + { + use std::os::fd::AsRawFd; + let source = fs::File::open(from)?; + let target = fs::File::create(to)?; + // SAFETY: FICLONE takes a source fd, both descriptors stay open for the call. + if unsafe { libc::ioctl(target.as_raw_fd(), libc::FICLONE, source.as_raw_fd()) } == 0 { + return Ok(()); + } + Err(io::Error::last_os_error()) + } + #[cfg(target_os = "macos")] + { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + let path = |p: &Path| { + CString::new(p.as_os_str().as_bytes()) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) + }; + let (source, target) = (path(from)?, path(to)?); + // SAFETY: both paths are valid NUL-terminated strings for the duration of the call. + if unsafe { libc::clonefile(source.as_ptr(), target.as_ptr(), 0) } == 0 { + return Ok(()); + } + Err(io::Error::last_os_error()) + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = (from, to); + Err(io::ErrorKind::Unsupported.into()) + } +} + +/// Copies an independent file, attempting a reflink first. The destination may be replaced. +pub fn clone_or_copy(from: &Path, to: &Path) -> io::Result<()> { + clone_file(from, to).or_else(|_| fs::copy(from, to).map(|_| ())) +} + +fn copy_hash(from: &Path, to: &Path) -> io::Result { + let mut source = fs::File::open(from)?; + let mut target = fs::File::create(to)?; + let mut hash = D::new(); + let mut buffer = vec![0; 64 * 1024]; + loop { + let n = source.read(&mut buffer)?; + if n == 0 { + return Ok(hex(&hash.finalize())); + } + target.write_all(&buffer[..n])?; + hash.update(&buffer[..n]); + } +} + +/// A verified replacement staged beside its destination. Dropping an unfinished set cleans +/// every temporary file, so a corrupt late panorama face cannot replace earlier placeholders. +pub(crate) struct Import { + temporary: Option, + destination: PathBuf, +} + +impl Import { + pub(crate) fn stage( + source: &Path, + destination: &Path, + checksum: &str, + algorithm: &str, + verified: bool, + ) -> Result { + let mut import = Self { + temporary: None, + destination: destination.into(), + }; + if verified || (destination.is_file() && digest(destination, algorithm)? == checksum) { + return Ok(import); + } + fs::create_dir_all(destination.parent().unwrap()).map_err(|e| e.to_string())?; + static COPIES: AtomicUsize = AtomicUsize::new(0); + let temporary = destination.with_file_name(format!( + ".{}.partial-{}-{}", + destination.file_name().unwrap().to_string_lossy(), + std::process::id(), + COPIES.fetch_add(1, Ordering::Relaxed) + )); + import.temporary = Some(temporary.clone()); + let actual = if clone_file(source, &temporary).is_ok() { + digest(&temporary, algorithm)? + } else { + match algorithm { + "sha1" => copy_hash::(source, &temporary), + _ => copy_hash::(source, &temporary), + } + .map_err(|e| format!("{}: {e}", source.display()))? + }; + if actual != checksum { + return Err(format!( + "{} is not the pinned file: expected {algorithm} {checksum}, got {actual}", + source.display() + )); + } + Ok(import) + } + + pub(crate) fn commit(mut self) -> Result { + if let Some(temporary) = &self.temporary { + fs::rename(temporary, &self.destination) + .map_err(|e| format!("{}: {e}", self.destination.display()))?; + } + self.temporary = None; + Ok(self.destination.clone()) + } +} + +impl Drop for Import { + fn drop(&mut self) { + if let Some(temporary) = &self.temporary { + let _ = fs::remove_file(temporary); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn copies_are_independent_and_checked_before_publication() { + let dir = std::env::temp_dir().join(format!("horizon-file-copy-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let source = dir.join("source"); + let target = dir.join("target"); + let data = vec![42; 200_000]; // More than one streaming buffer. + fs::write(&source, &data).unwrap(); + for algorithm in ["sha1", "sha256"] { + let checksum = digest(&source, algorithm).unwrap(); + Import::stage(&source, &target, &checksum, algorithm, false) + .unwrap() + .commit() + .unwrap(); + assert_eq!(digest(&target, algorithm).unwrap(), checksum); + fs::write(&target, b"old destination").unwrap(); + assert_eq!(fs::read(&source).unwrap(), data); + assert!(Import::stage(&source, &target, "bad hash", algorithm, false).is_err()); + assert_eq!(fs::read(&target).unwrap(), b"old destination"); + // Explicitly exercise ordinary streaming copies even on a reflink filesystem. + let streamed = if algorithm == "sha1" { + copy_hash::(&source, &target).unwrap() + } else { + copy_hash::(&source, &target).unwrap() + }; + assert_eq!(streamed, checksum); + assert_eq!(fs::read(&target).unwrap(), data); + } + clone_or_copy(&source, &target).unwrap(); + fs::write(&target, b"rewrite").unwrap(); + assert_eq!(fs::read(&source).unwrap(), data); + let staged = Import::stage( + &source, + &target, + &digest(&source, "sha1").unwrap(), + "sha1", + false, + ) + .unwrap(); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 3); + drop(staged); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 2); + fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/horizon-prepare/src/lib.rs b/crates/horizon-prepare/src/lib.rs index c7e443c7d..daee30444 100644 --- a/crates/horizon-prepare/src/lib.rs +++ b/crates/horizon-prepare/src/lib.rs @@ -11,12 +11,14 @@ //! run, keyed by the sha256 of their inputs and outputs recorded in the manifest, as before. pub mod artifacts; +mod files; mod inventory; pub mod java; pub mod pyjson; mod sounds; mod survival; +pub use files::clone_or_copy; pub use sounds::{fonts, languages, packs, panorama, sounds, BUILT_IN_PACKS}; use pyjson::Value; @@ -410,16 +412,28 @@ pub fn prepare(options: &Options, log: Log) -> Result { &artifacts::minecraft_dir(options.minecraft_dir.as_deref()).join("runtime"), )); let jdk = java::find(lock.java_major, &homes)?; - if options.import && !artifacts::reuse_imported(root, log)? { - artifacts::import( - root, - &lock, - options.minecraft_dir.as_deref(), - options.server_jar.as_deref(), - log, - )?; - } - let found = artifacts::verify(&lock)?; + let reused = if options.import { + artifacts::reuse_verified(root, log)? + } else { + None + }; + let (found, client_libraries) = match reused { + Some(checked) => checked, + None => { + if options.import { + artifacts::import( + root, + &lock, + options.minecraft_dir.as_deref(), + options.server_jar.as_deref(), + log, + )?; + } + let found = artifacts::verify(&lock)?; + let libraries = artifacts::client_libraries(&lock, &found.manifest)?; + (found, libraries) + } + }; let local = root.join(".local"); let resources = local.join("horizon-client").join(&lock.version); mkdir(&resources)?; @@ -445,7 +459,6 @@ pub fn prepare(options: &Options, log: Log) -> Result { } else { Value::Object(Vec::new()) }; - let client_libraries = artifacts::client_libraries(&lock, &found.manifest)?; let run = Run { root: root.clone(), local, @@ -679,25 +692,9 @@ fn steps( // and every data-pack resource (worldgen, recipes, loot tables, tags), and // the vanilla pack's icon (`pack.png`) for the pack screen. (run.log)("Copying the client's assets and data"); - let mut count = 0usize; - let mut client = open_zip(&run.client)?; - let version = pyjson::loads(&zip_text(&mut client, "version.json")?)?; - for name in zip_names(&client) { - let wanted = name.starts_with("assets/") || name.starts_with("data/") || name == "pack.png"; - if !wanted || name.ends_with('/') { - continue; - } - let relative = member_path(&name, "Invalid resource path")?; - let data = zip_bytes(&mut client, &name)?; - write_if_changed(&root.join(relative), &data)?; - count += 1; - } - - // The JAR holds only gray 1x1 panorama placeholders and an empty Unifont - // include, and none of the built-in packs. Import the indexed ones afterwards, including on reruns that just - // extracted the JAR again. They are optional when preparing from - // already-imported JARs. - sounds::optional_assets(repo, minecraft, run.log); + let mut indexed = sounds::Assets::load(repo, minecraft, run.log); + let (version, count) = extract_client_resources(&run.client, root, indexed.as_mut().ok())?; + sounds::optional_imports(indexed.as_mut().map_err(|e| e.as_str()), run.log); // One file each, exported again when the exporter or the jar it reads changed. let single = |what: &str, @@ -931,6 +928,54 @@ fn steps( ) } +/// Check existing JAR assets by their ZIP size/CRC without inflating every member again. +/// Indexed panorama/font/language replacements have stronger SHA-1 checks and take precedence. +fn extract_client_resources( + jar: &Path, + root: &Path, + mut indexed: Option<&mut sounds::Assets>, +) -> Result<(Value, usize), String> { + let mut client = open_zip(jar)?; + let version = pyjson::loads(&zip_text(&mut client, "version.json")?)?; + let mut count = 0; + for name in zip_names(&client) { + if !(name.starts_with("assets/") || name.starts_with("data/") || name == "pack.png") + || name.ends_with('/') + { + continue; + } + let relative = member_path(&name, "Invalid resource path")?; + count += 1; + if let Some(assets) = &mut indexed { + if assets.imported(&name)? { + continue; + } + } + let target = root.join(relative); + let member = client.by_name(&name).map_err(|e| e.to_string())?; + let (size, crc) = (member.size(), member.crc32()); + drop(member); + if target.is_file() && fs::metadata(&target).map_err(|e| e.to_string())?.len() == size { + let mut file = fs::File::open(&target).map_err(|e| e.to_string())?; + let mut hash = crc32fast::Hasher::new(); + let mut buffer = [0; 16 * 1024]; + loop { + let n = file.read(&mut buffer).map_err(|e| e.to_string())?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + if hash.finalize() == crc { + continue; + } + } + mkdir(target.parent().unwrap())?; + write(&target, &zip_bytes(&mut client, &name)?)?; + } + Ok((version, count)) +} + fn datagen_reports_missing(root: &Path) -> bool { ["items.json", "registries.json", "commands.json"] .iter() diff --git a/crates/horizon-prepare/src/sounds.rs b/crates/horizon-prepare/src/sounds.rs index 15d4a7a08..fd171ec12 100644 --- a/crates/horizon-prepare/src/sounds.rs +++ b/crates/horizon-prepare/src/sounds.rs @@ -5,7 +5,8 @@ use crate::artifacts::{self, Lock}; use crate::pyjson::Value; -use crate::{digest, read_json, Log}; +use crate::{digest, Log}; +use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; @@ -18,8 +19,9 @@ fn launcher_assets( lock: &Lock, manifest: &Value, minecraft: &Path, + cached_index: &Path, log: Log, -) -> Result<(PathBuf, PathBuf), String> { +) -> Result<(PathBuf, PathBuf, Vec), String> { let assets = minecraft.join("assets"); let info = manifest .get("assetIndex") @@ -33,7 +35,10 @@ fn launcher_assets( if artifacts::safe_relative(id).is_none_or(|p| p.components().count() != 1) { return Err(format!("Unsafe asset index id {id:?}")); } - let index = assets.join("indexes").join(format!("{id}.json")); + let mut index = assets.join("indexes").join(format!("{id}.json")); + if !index.is_file() && cached_index.is_file() { + index = cached_index.to_path_buf(); + } if !index.is_file() { let version = manifest.get("id").and_then(Value::as_str).unwrap_or(""); return Err(format!( @@ -42,7 +47,8 @@ fn launcher_assets( artifacts::launcher_hint(version) )); } - let checksum = digest(&index, "sha1")?; + let bytes = fs::read(&index).map_err(|e| format!("{}: {e}", index.display()))?; + let checksum = crate::hex(&::digest(&bytes)); if checksum != sha1 && !lock.asset_index_accepted(&checksum) { log(&format!( "{} is an asset index Horizon does not know yet (sha1 {checksum}); Mojang may have \ @@ -50,7 +56,7 @@ fn launcher_assets( index.display() )); } - Ok((assets, index)) + Ok((assets, index, bytes)) } /// Where the launcher's asset store keeps the object with this SHA-1. @@ -61,24 +67,6 @@ fn object_path(assets: &Path, checksum: &str) -> Result { Ok(assets.join("objects").join(&checksum[..2]).join(checksum)) } -/// One verified object from the launcher's asset store (tools/lab.py `launcher_object`). -fn launcher_object(assets: &Path, checksum: &str) -> Result { - let path = object_path(assets, checksum)?; - if !path.is_file() { - return Err(format!( - "Missing asset object {}. Play the pinned version once in your launcher.", - path.display() - )); - } - if digest(&path, "sha1")? != checksum { - return Err(format!( - "Corrupt asset object {}; let the launcher repair it", - path.display() - )); - } - Ok(path) -} - /// Whether the game plays this asset: sounds.json and everything under sounds/. fn is_sound(name: &str) -> bool { name == "minecraft/sounds.json" || name.starts_with("minecraft/sounds/") @@ -127,26 +115,56 @@ fn is_language(name: &str) -> bool { } /// Full preparation can run from cached JARs without an official launcher install. -pub(crate) fn optional_assets(root: &Path, minecraft: Option<&Path>, log: Log) { - if let Err(error) = panorama(root, minecraft, log) { - log(&format!( - "Title panorama not imported: {error}; run horizon-prepare panorama later" - )); - } - if let Err(error) = fonts(root, minecraft, log) { - log(&format!( - "Unifont glyphs not imported: {error}; run horizon-prepare fonts later" - )); - } - if let Err(error) = packs(root, minecraft, log) { - log(&format!( - "Programmer Art and High Contrast not imported: {error}; run horizon-prepare packs later" - )); - } - if let Err(error) = languages(root, minecraft, log) { - log(&format!( - "Languages not imported: {error}; run horizon-prepare languages later" - )); +#[cfg(test)] +fn optional_assets(root: &Path, minecraft: Option<&Path>, log: Log) { + optional_imports( + Assets::load(root, minecraft, log) + .as_mut() + .map_err(|e| e.as_str()), + log, + ); +} + +pub(crate) fn optional_imports(mut assets: Result<&mut Assets, &str>, log: Log) { + for (wanted, label, required, failure, command) in [ + ( + is_panorama as fn(&str) -> bool, + "Panorama", + PANORAMA.as_slice(), + "Title panorama", + "panorama", + ), + ( + is_font, + "Font", + UNIFONT.as_slice(), + "Unifont glyphs", + "fonts", + ), + ( + is_built_in_pack, + "Built-in pack", + BUILT_IN_PACKS.as_slice(), + "Programmer Art and High Contrast", + "packs", + ), + ( + is_language, + "Language", + LANGUAGES.as_slice(), + "Languages", + "languages", + ), + ] { + let result = match &mut assets { + Ok(assets) => assets.copy(wanted, label, required, log).map(|_| ()), + Err(error) => Err(error.to_string()), + }; + if let Err(error) = result { + log(&format!( + "{failure} not imported: {error}; run horizon-prepare {command} later" + )); + } } } @@ -207,29 +225,98 @@ fn copy_assets( label: &str, required: &[&str], ) -> Result { - let lock = Lock::read(root)?; - let minecraft = artifacts::minecraft_dir(minecraft); - let manifest = artifacts::import_manifest(&lock, &minecraft)?; - let (store, index) = launcher_assets(&lock, &manifest, &minecraft, log)?; - let id = manifest - .get("id") - .and_then(Value::as_str) - .ok_or("The version JSON has no id")?; - let target = root.join(".local/horizon-client").join(id); - fs::create_dir_all(&target).map_err(|e| format!("{}: {e}", target.display()))?; - fs::copy(&index, target.join("asset-index.json")) - .map_err(|e| format!("{}: {e}", target.join("asset-index.json").display()))?; - let objects = read_json(&index)?; - let objects = objects - .get("objects") - .and_then(Value::as_object) - .ok_or_else(|| format!("{} has no objects", index.display()))?; + Assets::load(root, minecraft, log)?.copy(wanted_asset, label, required, log) +} + +/// A preparation's index and verified destinations. Never persisted: a later invocation +/// checks the bytes again, including when repairing a corrupt prepared file. +pub(crate) struct Assets { + store: PathBuf, + target: PathBuf, + objects: Vec<(String, Value)>, + verified: HashSet, +} + +impl Assets { + pub(crate) fn load(root: &Path, minecraft: Option<&Path>, log: Log) -> Result { + let lock = Lock::read(root)?; + let minecraft = artifacts::minecraft_dir(minecraft); + let manifest = artifacts::import_manifest(&lock, &minecraft)?; + let target = root.join(".local/horizon-client").join(&lock.version); + let (store, index, bytes) = launcher_assets( + &lock, + &manifest, + &minecraft, + &target.join("asset-index.json"), + log, + )?; + let text = std::str::from_utf8(&bytes).map_err(|e| e.to_string())?; + let objects = crate::pyjson::loads(text)?; + let objects = objects + .get("objects") + .and_then(Value::as_object) + .ok_or_else(|| format!("{} has no objects", index.display()))? + .to_vec(); + fs::create_dir_all(&target).map_err(|e| e.to_string())?; + crate::write_if_changed(&target.join("asset-index.json"), &bytes)?; + Ok(Self { + store, + target, + objects, + verified: HashSet::new(), + }) + } + + /// Preserve indexed replacements instead of overwriting them with JAR placeholders. + pub(crate) fn imported(&mut self, name: &str) -> Result { + let Some(name) = name.strip_prefix("assets/") else { + return Ok(false); + }; + if !(is_panorama(name) || is_font(name) || is_language(name)) { + return Ok(false); + } + let Some((_, info)) = self.objects.iter().rev().find(|(n, _)| n == name) else { + return Ok(false); + }; + let Some(hash) = info.get("hash").and_then(Value::as_str) else { + return Ok(false); + }; + let destination = self.target.join("assets").join(name); + if destination.is_file() && digest(&destination, "sha1")? == hash { + self.verified.insert(destination); + return Ok(true); + } + Ok(false) + } + + fn copy( + &mut self, + wanted_asset: fn(&str) -> bool, + label: &str, + required: &[&str], + log: Log, + ) -> Result { + copy_indexed(self, wanted_asset, label, required, log) + } +} + +fn copy_indexed( + assets: &mut Assets, + wanted_asset: fn(&str) -> bool, + label: &str, + required: &[&str], + log: Log, +) -> Result { + let objects = &assets.objects; // A repeated name keeps its first place and its last value, as a Python dict does. let mut wanted: Vec<(&str, &Value)> = Vec::new(); + let mut positions = HashMap::new(); for (name, info) in objects.iter().filter(|(name, _)| wanted_asset(name)) { - match wanted.iter_mut().find(|(n, _)| n == name) { - Some(entry) => entry.1 = info, - None => wanted.push((name, info)), + if let Some(&position) = positions.get(name.as_str()) { + wanted[position] = (name.as_str(), info); + } else { + positions.insert(name.as_str(), wanted.len()); + wanted.push((name, info)); } } for name in required { @@ -251,21 +338,45 @@ fn copy_assets( .ok_or_else(|| format!("{name} has no size in the asset index"))?; let relative = artifacts::safe_relative(name) .ok_or_else(|| format!("Unsafe asset name {name:?} in the asset index"))?; - let source = launcher_object(&store, hash)?; - copies.push((source, target.join("assets").join(relative), hash)); + let source = object_path(&assets.store, hash)?; + let destination = assets.target.join("assets").join(relative); + let import = crate::files::Import::stage( + &source, + &destination, + hash, + "sha1", + assets.verified.contains(&destination), + ) + .map_err(|error| { + if !source.is_file() { + format!( + "Missing asset object {}. Play the pinned version once in your launcher.", + source.display() + ) + } else if error.contains(" is not the pinned file:") { + format!( + "Corrupt asset object {}; let the launcher repair it: {error}", + source.display() + ) + } else { + error + } + })?; + copies.push((import, destination)); total += size; } // Check the entire set before replacing any placeholders: a missing late face // must not leave a mixture of real textures and gray faces in either entry point. - for (source, destination, hash) in copies { - artifacts::import_copy(&source, &destination, hash, "sha1", "")?; + for (import, destination) in copies { + import.commit()?; + assets.verified.insert(destination); } log(&format!( "{label} assets ready: {} files, {:.1} MB", wanted.len(), total as f64 / 1e6 )); - Ok(target) + Ok(assets.target.clone()) } #[cfg(test)] @@ -680,6 +791,7 @@ mod tests { "minecraft/sounds.json is missing from the asset index" ); fs::remove_file(&index).unwrap(); + fs::remove_file(f.root.join(".local/horizon-client/9.9/asset-index.json")).unwrap(); let error = f.run().unwrap_err(); assert!( error.starts_with("Missing ") && error.contains("play 9.9 once"), @@ -687,6 +799,120 @@ mod tests { ); } + #[test] + fn late_corruption_discards_all_staged_replacements() { + let f = Fixture::new("late-corruption"); + let target = f.root.join(".local/horizon-client/9.9/assets"); + for name in PANORAMA { + let path = target.join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, b"placeholder").unwrap(); + } + fs::write(f.object(FIRST_FACE + 6), b"bad overlay").unwrap(); + assert!(panorama(&f.root, Some(&f.minecraft), &|_| {}).is_err()); + for name in PANORAMA { + assert_eq!(fs::read(target.join(name)).unwrap(), b"placeholder"); + } + assert_eq!( + fs::read_dir(target.join("minecraft/textures/gui/title/background")) + .unwrap() + .count(), + 7 + ); + } + + #[test] + fn one_index_snapshot_serves_every_import_and_reruns_repair_corruption() { + let f = Fixture::with_extras("snapshot", &PANORAMA, &[("pack.mcmeta", b"{}")]); + let mut assets = Assets::load(&f.root, Some(&f.minecraft), &|_| {}).unwrap(); + fs::remove_file(f.minecraft.join("assets/indexes/7.json")).unwrap(); + let lines = std::cell::RefCell::new(Vec::new()); + optional_imports(Ok(&mut assets), &|line| { + lines.borrow_mut().push(line.to_string()) + }); + assert_eq!(lines.borrow().len(), 4); + assert!(lines + .borrow() + .iter() + .all(|line| line.contains("assets ready"))); + assets + .copy(is_sound, "Sound", &["minecraft/sounds.json"], &|_| {}) + .unwrap(); + let sound = assets + .target + .join("assets/minecraft/sounds/ambient/cave1.ogg"); + let time = fs::metadata(&sound).unwrap().modified().unwrap(); + fs::remove_file(f.object(1)).unwrap(); + f.run().unwrap(); // A verified destination works even without the source object/index. + assert_eq!(fs::metadata(&sound).unwrap().modified().unwrap(), time); + fs::write(&sound, b"corrupt").unwrap(); + assert!(f.run().is_err()); // Verification never carries over to another invocation. + fs::write(f.object(1), &f.objects[1].1).unwrap(); + f.run().unwrap(); + assert_eq!(fs::read(sound).unwrap(), f.objects[1].1); + } + + #[test] + fn jar_reruns_preserve_indexed_assets_and_repair_missing_or_corrupt_members() { + use std::io::Write; + let f = Fixture::with_extras( + "extract", + &PANORAMA, + &[ + ("pack.mcmeta", b"{}"), + ("minecraft/lang/en_us.json", b"{\"indexed\":true}"), + ], + ); + optional_assets(&f.root, Some(&f.minecraft), &|_| {}); + let target = f.root.join(".local/horizon-client/9.9"); + let jar = f.root.join("client.jar"); + let mut zip = zip::ZipWriter::new(fs::File::create(&jar).unwrap()); + for (name, data) in [ + ("version.json", "{\"world_version\": 1}"), + ( + "assets/minecraft/textures/gui/title/background/panorama_0.png", + "gray", + ), + ("assets/minecraft/font/include/unifont.json", "{}"), + ("assets/minecraft/lang/en_us.json", "{}"), + ("data/minecraft/test.json", "data"), + ("pack.png", "icon"), + ] { + zip.start_file(name, zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(data.as_bytes()).unwrap(); + } + zip.finish().unwrap(); + let face = target.join("assets").join(PANORAMA[0]); + let time = fs::metadata(&face).unwrap().modified().unwrap(); + // Preparing from cached JARs must also preserve full imports without the launcher. + fs::remove_dir_all(f.minecraft.join("assets")).unwrap(); + for _ in 0..2 { + let mut assets = Assets::load(&f.root, Some(&f.minecraft), &|_| {}).unwrap(); + let (_, count) = + crate::extract_client_resources(&jar, &target, Some(&mut assets)).unwrap(); + assert_eq!(count, 5); + optional_imports(Ok(&mut assets), &|_| {}); + assert_eq!(fs::read(&face).unwrap(), PANORAMA[0].as_bytes()); + assert_eq!(fs::metadata(&face).unwrap().modified().unwrap(), time); + assert_eq!( + fs::read(target.join("assets").join(UNIFONT[0])).unwrap(), + f.objects[4].1 + ); + assert_eq!( + fs::read(target.join("assets/minecraft/lang/en_us.json")).unwrap(), + b"{\"indexed\":true}" + ); + assert_eq!( + fs::read(target.join("data/minecraft/test.json")).unwrap(), + b"data" + ); + assert_eq!(fs::read(target.join("pack.png")).unwrap(), b"icon"); + fs::write(target.join("data/minecraft/test.json"), b"oops").unwrap(); + fs::remove_file(target.join("pack.png")).unwrap(); + } + } + #[test] fn object_paths_come_from_real_hashes() { let store = Path::new("assets"); diff --git a/docs/prepare-performance.md b/docs/prepare-performance.md new file mode 100644 index 000000000..3a3587111 --- /dev/null +++ b/docs/prepare-performance.md @@ -0,0 +1,44 @@ +# Resource preparation and filesystem work + +The reported preparation timings put sounds at 20–29 seconds with little CPU use, versus +0.9–1.4 seconds for fresh client asset extraction. Prefer fewer file opens, reads and writes +over parallelizing decompression. + +Preparation now shares one parsed asset index between panorama, font, pack and language imports. +Its destination verification results live only for that preparation. A subsequent invocation +checks bytes again, so it can repair corrupt resources. Existing valid destination files do not +require the original asset objects to remain installed. A saved asset index also lets preparation +preserve full imports when only cached Minecraft JARs remain. + +New imports are staged beside their destination. On Linux, FICLONE is attempted first; other +filesystems fall back to copying while hashing. Cloned copies are hashed before publication. +macOS attempts clonefile with the same checked streaming fallback. A whole asset group is staged and +verified before any replacement is published; corrupt late panorama faces leave placeholders +alone. Temporary files are cleaned up on errors. This is not a transaction for rename failures +or process termination during publication: a later run repairs incomplete groups. + +Unchanged JAR resources are checked by the ZIP member's size and CRC, avoiding repeated +decompression. Valid indexed replacements take precedence over JAR placeholders. Reusing checked +Minecraft imports also passes the verified artifact/library results directly to the exporters. + +## An immutable store across releases + +The official launcher's SHA-1 object store already provides an immutable source for indexed +assets. Independent filesystem clones let installed Horizon releases share physical blocks +where supported, without sharing writable inodes. Ordinary copies remain the fallback. + +A separate Horizon store could additionally share complete preparations on filesystems without +cloning. Its key would need the pinned client/server hashes, the preparer's output version (or +the legacy source/exporter fingerprint), and the actual asset-index digest: Mojang republishes +translations without changing the Minecraft release. Two preparations with different optional +asset availability must not claim the same complete snapshot. + +Before adopting that store, separate immutable runtime resources from mutable exporter scratch, +classes and logs. Publish snapshots through a temporary directory and atomic rename, give each +release an explicit reference, and define locking, corruption repair, migration and garbage +collection. Exporters must never receive a shared writable directory or hard-linked files. +The existing launcher compatibility check should choose reuse; the store should only supply its +bytes. Keep this separate from the open work on output-version compatibility (#1946/#1947). + +For now, retain independent release directories and clone their resource files. This reduces +copying where the filesystem supports it without introducing a second ownership/cleanup scheme. diff --git a/launcher/Cargo.lock b/launcher/Cargo.lock index 1e8a5aee2..ee0cb882a 100644 --- a/launcher/Cargo.lock +++ b/launcher/Cargo.lock @@ -2726,6 +2726,8 @@ dependencies = [ name = "horizon-prepare" version = "0.1.0" dependencies = [ + "crc32fast", + "libc", "regex", "sha1", "sha2 0.10.9", diff --git a/launcher/src/backend.rs b/launcher/src/backend.rs index 7e5c4babb..dadef73d3 100644 --- a/launcher/src/backend.rs +++ b/launcher/src/backend.rs @@ -719,7 +719,7 @@ fn copy_dir(from: &Path, to: &Path) -> std::io::Result<()> { if entry.file_type()?.is_dir() { copy_dir(&entry.path(), &target)?; } else { - fs::copy(entry.path(), target)?; + horizon_prepare::clone_or_copy(&entry.path(), &target)?; } } Ok(())