diff --git a/crates/horizon-client/examples/load_bench.rs b/crates/horizon-client/examples/load_bench.rs new file mode 100644 index 000000000..5c30a71ff --- /dev/null +++ b/crates/horizon-client/examples/load_bench.rs @@ -0,0 +1,153 @@ +//! Read-only saved chunk benchmark. All timings exclude resource/corpus setup. +//! +//! load_bench RESOURCES DIMENSION_DIR [--rounds N] [--limit N] [--checksum] +//! Add --profile-worker when a sampling collector cannot sample the main thread. +//! Reports warm filesystem SaveDir::load (read, inflate, NBT, decode and light), +//! plus isolated inflate/NBT and native decode. No files in the save are changed. +use horizon_client::{ + assets::Assets, + biome::Biomes, + blocks::BlockFacts, + save::{SaveDir, SaveTables, decode_with}, +}; +use horizon_world::{nbt::Limits, region::RegionFile}; +use sha2::{Digest, Sha256}; +use std::{hint::black_box, path::PathBuf, sync::Arc, time::Instant}; + +type Result = std::result::Result>; + +fn main() -> Result<()> { + if std::env::args().any(|a| a == "--profile-worker") { + std::thread::spawn(|| run().map_err(|e| e.to_string())) + .join() + .expect("benchmark worker") + .map_err(Into::into) + } else { + run() + } +} + +fn run() -> Result<()> { + let args: Vec = std::env::args().skip(1).collect(); + let usage = "load_bench RESOURCES DIMENSION_DIR [--rounds N] [--limit N] [--checksum]"; + let root = PathBuf::from(args.first().ok_or(usage)?); + let dimension = PathBuf::from(args.get(1).ok_or(usage)?); + let flag = |name: &str, default: usize| -> Result { + Ok(match args.iter().position(|a| a == name) { + Some(i) => args.get(i + 1).ok_or(usage)?.parse()?, + None => default, + }) + }; + let rounds = flag("--rounds", 5)?; + let limit = flag("--limit", 256)?; + if rounds == 0 || limit == 0 { + return Err("rounds and limit must be positive".into()); + } + let assets = Assets::load(&root)?; + let facts = BlockFacts::load(&root, assets.registry)?; + let biomes = Biomes::load(&root)?; + let tables = Arc::new(SaveTables::new(&assets).with_block_entities(&facts)); + let source = SaveDir::new(dimension.clone()); + let _ = source.tables.set(tables.clone()); + let mut paths: Vec<_> = std::fs::read_dir(dimension.join("region"))? + .map(|e| e.map(|e| e.path())) + .collect::>()?; + paths.retain(|p| p.extension().is_some_and(|e| e == "mca")); + paths.sort(); + let mut corpus = Vec::new(); + 'regions: for path in paths { + let mut region = RegionFile::open(path)?; + let mut positions: Vec<_> = region.chunks().map(|(pos, _)| pos).collect(); + positions.sort_by_key(|p| (p.x, p.z)); + for pos in positions { + let record = region.read(pos)?.ok_or("missing record")?; + let tag = record.nbt(Limits::default())?; + if decode_with(&tag, assets.registry, &biomes, Some(&tables)).is_none() { + continue; + } + corpus.push((pos, record, tag)); + if corpus.len() == limit { + break 'regions; + } + } + } + if corpus.is_empty() { + return Err("no full chunks in corpus".into()); + } + let mut records_hash = Sha256::new(); + for (pos, record, _) in &corpus { + records_hash.update(pos.x.to_le_bytes()); + records_hash.update(pos.z.to_le_bytes()); + records_hash.update((record.compressed.len() as u64).to_le_bytes()); + records_hash.update(&record.compressed); + } + println!( + "{}", + serde_json::json!({"corpus": dimension, "chunks": corpus.len(), "rounds": rounds, + "compressed_records_sha256": format!("{:x}", records_hash.finalize()), + "compressed_bytes": corpus.iter().map(|(_, r, _)| r.compressed.len()).sum::()}) + ); + // Prime SaveDir's region handles and the filesystem cache before measurement. + for (pos, _, _) in &corpus { + black_box( + source + .load(pos.x, pos.z, assets.registry, &biomes) + .ok_or("load failed")?, + ); + } + for round in 0..rounds { + let started = Instant::now(); + for (pos, _, _) in &corpus { + black_box( + source + .load(pos.x, pos.z, assets.registry, &biomes) + .ok_or("load failed")?, + ); + } + let load_s = started.elapsed().as_secs_f64(); + let started = Instant::now(); + for (_, record, _) in &corpus { + black_box(record.nbt(Limits::default())?); + } + let nbt_s = started.elapsed().as_secs_f64(); + let started = Instant::now(); + for (_, _, tag) in &corpus { + black_box( + decode_with(tag, assets.registry, &biomes, Some(&tables)).ok_or("decode failed")?, + ); + } + let decode_s = started.elapsed().as_secs_f64(); + println!( + "{}", + serde_json::json!({"round": round, "load_s": load_s, + "inflate_nbt_s": nbt_s, "decode_s": decode_s}) + ); + } + if args.iter().any(|a| a == "--checksum") { + let mut hash = Sha256::new(); + let mut light_loaded = 0; + for (pos, _, _) in &corpus { + let chunk = source + .load(pos.x, pos.z, assets.registry, &biomes) + .ok_or("load failed")?; + hash.update(pos.x.to_le_bytes()); + hash.update(pos.z.to_le_bytes()); + light_loaded += usize::from(chunk.light_loaded); + for section in &chunk.sections { + for i in 0..4096 { + hash.update(section.get(i).to_le_bytes()); + hash.update([section.light(i)]); + } + for i in 0..64 { + hash.update(section.biomes.get(i).to_le_bytes()); + } + } + } + println!( + "{}", + serde_json::json!({"blocks_biomes_light_sha256": format!("{:x}", hash.finalize()), + "light_loaded": light_loaded}) + ); + } + Ok(()) +} diff --git a/crates/horizon-core/src/paletted.rs b/crates/horizon-core/src/paletted.rs index 3d56bb987..0730cf5e4 100644 --- a/crates/horizon-core/src/paletted.rs +++ b/crates/horizon-core/src/paletted.rs @@ -577,14 +577,21 @@ impl PalettedContainer { if list.len() == 1 << self.bits { return Ok(()); } - for i in 0..self.strategy.entries() { - let index = read(&self.words, self.bits, i); - if index as usize >= list.len() { - return Err(PaletteError::BadIndex { - index, - palette: list.len(), - }); + let per = 64 / usize::from(self.bits); + let mut remaining = self.strategy.entries(); + let mask = mask(self.bits); + for &word in &self.words { + let count = remaining.min(per); + for slot in 0..count { + let index = ((word >> (slot * usize::from(self.bits))) & mask) as u32; + if index as usize >= list.len() { + return Err(PaletteError::BadIndex { + index, + palette: list.len(), + }); + } } + remaining -= count; } Ok(()) } @@ -644,6 +651,18 @@ impl PalettedContainer { actual: data.len(), }); } + if let Config::List(bits) = config { + // List palettes already use the saved indices as resident storage. + // Validate them without allocating an unpacked copy that is unused. + let container = Self { + strategy, + bits, + words: data.to_vec(), + palette: Palette::List(palette), + }; + container.check_indices()?; + return Ok(container); + } let mut ids = Vec::with_capacity(entries); for i in 0..entries { let index = read(data, disc_bits, i); @@ -655,15 +674,7 @@ impl PalettedContainer { } ids.push(index); } - Ok(match config { - Config::List(bits) => Self { - strategy, - bits, - words: data.to_vec(), - palette: Palette::List(palette), - }, - _ => Self::build(strategy, palette, &ids), - }) + Ok(Self::build(strategy, palette, &ids)) } /// Whether any palette entry satisfies `predicate` (`maybeHas`): entries /// no longer used by any position still count. @@ -796,6 +807,113 @@ mod tests { assert!(PalettedContainer::from_wire(BLOCKS, 0, vec![9], vec![]).is_ok()); } #[test] + fn saved_validation_matches_entrywise_reference() { + // Keep the previous loader as an independent entry-at-a-time reference: + // errors, resident palette order and unused storage bits must all agree. + fn reference( + strategy: Strategy, + palette: Vec, + data: Option<&[u64]>, + ) -> Result { + if palette.is_empty() { + return Err(PaletteError::BadPalette { len: 0 }); + } + let config = strategy.config(ceil_log2(palette.len())); + let entries = strategy.entries(); + let bits = match config { + Config::Single => return Ok(PalettedContainer::new(strategy, palette[0])), + Config::List(bits) => bits, + Config::Global => ceil_log2(palette.len()), + }; + let expected = storage_len(bits, entries); + let data = data.ok_or(PaletteError::BadStorage { + expected, + actual: 0, + })?; + if data.len() != expected { + return Err(PaletteError::BadStorage { + expected, + actual: data.len(), + }); + } + let mut ids = Vec::with_capacity(entries); + for i in 0..entries { + let index = read(data, bits, i); + if index as usize >= palette.len() { + return Err(PaletteError::BadIndex { + index, + palette: palette.len(), + }); + } + ids.push(index); + } + Ok(match config { + Config::List(bits) => PalettedContainer { + strategy, + bits, + palette: Palette::List(palette), + words: data.to_vec(), + }, + _ => PalettedContainer::build(strategy, palette, &ids), + }) + } + fn compare(strategy: Strategy, palette: &[u32], data: Option<&[u64]>) { + let expected = reference(strategy, palette.to_vec(), data); + let actual = PalettedContainer::from_packed(strategy, palette.to_vec(), data); + match (actual, expected) { + (Ok(a), Ok(b)) => { + assert_eq!(a.strategy, b.strategy); + assert_eq!(a.bits, b.bits); + assert_eq!(a.palette, b.palette); + assert_eq!(a.words, b.words); + } + (Err(a), Err(b)) => assert_eq!(a, b), + (a, b) => panic!("loader disagreement: {a:?} vs {b:?}"), + } + } + for strategy in [BLOCKS, BIOMES] { + for len in [ + 0, 1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, + 256, 257, 511, 512, 513, 1023, 1024, 1025, 2047, 2048, 2049, 4096, 4097, + ] { + let palette: Vec = (0..len).map(|i| (i as u32 * 7) % 127).collect(); + compare(strategy, &palette, None); + compare(strategy, &palette, Some(&[])); + if len <= 1 { + compare(strategy, &palette, Some(&[u64::MAX])); + continue; + } + let bits = match strategy.config(ceil_log2(len)) { + Config::List(bits) => bits, + _ => ceil_log2(len), + }; + let entries = strategy.entries(); + let mut words = pack_words(bits, entries, (0..entries).map(|i| (i % len) as u32)); + compare(strategy, &palette, Some(&words)); + compare(strategy, &palette, Some(&words[..words.len() - 1])); + // Nonzero padding in every word and in the final partial word + // is legal and must be preserved by resident list palettes. + let per = 64 / usize::from(bits); + for (word, positions) in words.iter_mut().zip((0..entries).step_by(per)) { + let used = (entries - positions).min(per) * usize::from(bits); + if used < 64 { + *word |= u64::MAX << used; + } + } + compare(strategy, &palette, Some(&words)); + if len < 1usize << bits { + for i in [0, per - 1, per, entries / 2, entries - 1] { + let mut invalid = words.clone(); + write(&mut invalid, bits, i, len as u32); + // A second invalid entry verifies first-error ordering. + write(&mut invalid, bits, entries - 1, mask(bits) as u32); + compare(strategy, &palette, Some(&invalid)); + } + } + } + } + } + #[test] fn division_magic_is_exact() { for bits in 1..=32u8 { let per = 64 / usize::from(bits);