Merge pull request #1223 from rdbtCVS/perf/chunk-palette-loads
🤖 Agent: merged after verify passed and the integration run on the desktop (main 34c3c1cf + the 14 contributor PRs: workspace tests 1,869/0, launcher 113, parity --changed 320/0).
This commit is contained in:
@@ -0,0 +1,153 @@
|
||||
//! Read-only saved chunk benchmark. All timings exclude resource/corpus setup.
|
||||
//!
|
||||
//! load_bench RESOURCES DIMENSION_DIR [--rounds N] [--limit N] [--checksum]
|
||||
//! Add --profile-worker when a sampling collector cannot sample the main thread.
|
||||
//! Reports warm filesystem SaveDir::load (read, inflate, NBT, decode and light),
|
||||
//! plus isolated inflate/NBT and native decode. No files in the save are changed.
|
||||
use horizon_client::{
|
||||
assets::Assets,
|
||||
biome::Biomes,
|
||||
blocks::BlockFacts,
|
||||
save::{SaveDir, SaveTables, decode_with},
|
||||
};
|
||||
use horizon_world::{nbt::Limits, region::RegionFile};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{hint::black_box, path::PathBuf, sync::Arc, time::Instant};
|
||||
|
||||
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
if std::env::args().any(|a| a == "--profile-worker") {
|
||||
std::thread::spawn(|| run().map_err(|e| e.to_string()))
|
||||
.join()
|
||||
.expect("benchmark worker")
|
||||
.map_err(Into::into)
|
||||
} else {
|
||||
run()
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<()> {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
let usage = "load_bench RESOURCES DIMENSION_DIR [--rounds N] [--limit N] [--checksum]";
|
||||
let root = PathBuf::from(args.first().ok_or(usage)?);
|
||||
let dimension = PathBuf::from(args.get(1).ok_or(usage)?);
|
||||
let flag = |name: &str, default: usize| -> Result<usize> {
|
||||
Ok(match args.iter().position(|a| a == name) {
|
||||
Some(i) => args.get(i + 1).ok_or(usage)?.parse()?,
|
||||
None => default,
|
||||
})
|
||||
};
|
||||
let rounds = flag("--rounds", 5)?;
|
||||
let limit = flag("--limit", 256)?;
|
||||
if rounds == 0 || limit == 0 {
|
||||
return Err("rounds and limit must be positive".into());
|
||||
}
|
||||
let assets = Assets::load(&root)?;
|
||||
let facts = BlockFacts::load(&root, assets.registry)?;
|
||||
let biomes = Biomes::load(&root)?;
|
||||
let tables = Arc::new(SaveTables::new(&assets).with_block_entities(&facts));
|
||||
let source = SaveDir::new(dimension.clone());
|
||||
let _ = source.tables.set(tables.clone());
|
||||
let mut paths: Vec<_> = std::fs::read_dir(dimension.join("region"))?
|
||||
.map(|e| e.map(|e| e.path()))
|
||||
.collect::<std::io::Result<_>>()?;
|
||||
paths.retain(|p| p.extension().is_some_and(|e| e == "mca"));
|
||||
paths.sort();
|
||||
let mut corpus = Vec::new();
|
||||
'regions: for path in paths {
|
||||
let mut region = RegionFile::open(path)?;
|
||||
let mut positions: Vec<_> = region.chunks().map(|(pos, _)| pos).collect();
|
||||
positions.sort_by_key(|p| (p.x, p.z));
|
||||
for pos in positions {
|
||||
let record = region.read(pos)?.ok_or("missing record")?;
|
||||
let tag = record.nbt(Limits::default())?;
|
||||
if decode_with(&tag, assets.registry, &biomes, Some(&tables)).is_none() {
|
||||
continue;
|
||||
}
|
||||
corpus.push((pos, record, tag));
|
||||
if corpus.len() == limit {
|
||||
break 'regions;
|
||||
}
|
||||
}
|
||||
}
|
||||
if corpus.is_empty() {
|
||||
return Err("no full chunks in corpus".into());
|
||||
}
|
||||
let mut records_hash = Sha256::new();
|
||||
for (pos, record, _) in &corpus {
|
||||
records_hash.update(pos.x.to_le_bytes());
|
||||
records_hash.update(pos.z.to_le_bytes());
|
||||
records_hash.update((record.compressed.len() as u64).to_le_bytes());
|
||||
records_hash.update(&record.compressed);
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::json!({"corpus": dimension, "chunks": corpus.len(), "rounds": rounds,
|
||||
"compressed_records_sha256": format!("{:x}", records_hash.finalize()),
|
||||
"compressed_bytes": corpus.iter().map(|(_, r, _)| r.compressed.len()).sum::<usize>()})
|
||||
);
|
||||
// Prime SaveDir's region handles and the filesystem cache before measurement.
|
||||
for (pos, _, _) in &corpus {
|
||||
black_box(
|
||||
source
|
||||
.load(pos.x, pos.z, assets.registry, &biomes)
|
||||
.ok_or("load failed")?,
|
||||
);
|
||||
}
|
||||
for round in 0..rounds {
|
||||
let started = Instant::now();
|
||||
for (pos, _, _) in &corpus {
|
||||
black_box(
|
||||
source
|
||||
.load(pos.x, pos.z, assets.registry, &biomes)
|
||||
.ok_or("load failed")?,
|
||||
);
|
||||
}
|
||||
let load_s = started.elapsed().as_secs_f64();
|
||||
let started = Instant::now();
|
||||
for (_, record, _) in &corpus {
|
||||
black_box(record.nbt(Limits::default())?);
|
||||
}
|
||||
let nbt_s = started.elapsed().as_secs_f64();
|
||||
let started = Instant::now();
|
||||
for (_, _, tag) in &corpus {
|
||||
black_box(
|
||||
decode_with(tag, assets.registry, &biomes, Some(&tables)).ok_or("decode failed")?,
|
||||
);
|
||||
}
|
||||
let decode_s = started.elapsed().as_secs_f64();
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::json!({"round": round, "load_s": load_s,
|
||||
"inflate_nbt_s": nbt_s, "decode_s": decode_s})
|
||||
);
|
||||
}
|
||||
if args.iter().any(|a| a == "--checksum") {
|
||||
let mut hash = Sha256::new();
|
||||
let mut light_loaded = 0;
|
||||
for (pos, _, _) in &corpus {
|
||||
let chunk = source
|
||||
.load(pos.x, pos.z, assets.registry, &biomes)
|
||||
.ok_or("load failed")?;
|
||||
hash.update(pos.x.to_le_bytes());
|
||||
hash.update(pos.z.to_le_bytes());
|
||||
light_loaded += usize::from(chunk.light_loaded);
|
||||
for section in &chunk.sections {
|
||||
for i in 0..4096 {
|
||||
hash.update(section.get(i).to_le_bytes());
|
||||
hash.update([section.light(i)]);
|
||||
}
|
||||
for i in 0..64 {
|
||||
hash.update(section.biomes.get(i).to_le_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::json!({"blocks_biomes_light_sha256": format!("{:x}", hash.finalize()),
|
||||
"light_loaded": light_loaded})
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -577,14 +577,21 @@ impl PalettedContainer {
|
||||
if list.len() == 1 << self.bits {
|
||||
return Ok(());
|
||||
}
|
||||
for i in 0..self.strategy.entries() {
|
||||
let index = read(&self.words, self.bits, i);
|
||||
if index as usize >= list.len() {
|
||||
return Err(PaletteError::BadIndex {
|
||||
index,
|
||||
palette: list.len(),
|
||||
});
|
||||
let per = 64 / usize::from(self.bits);
|
||||
let mut remaining = self.strategy.entries();
|
||||
let mask = mask(self.bits);
|
||||
for &word in &self.words {
|
||||
let count = remaining.min(per);
|
||||
for slot in 0..count {
|
||||
let index = ((word >> (slot * usize::from(self.bits))) & mask) as u32;
|
||||
if index as usize >= list.len() {
|
||||
return Err(PaletteError::BadIndex {
|
||||
index,
|
||||
palette: list.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
remaining -= count;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -644,6 +651,18 @@ impl PalettedContainer {
|
||||
actual: data.len(),
|
||||
});
|
||||
}
|
||||
if let Config::List(bits) = config {
|
||||
// List palettes already use the saved indices as resident storage.
|
||||
// Validate them without allocating an unpacked copy that is unused.
|
||||
let container = Self {
|
||||
strategy,
|
||||
bits,
|
||||
words: data.to_vec(),
|
||||
palette: Palette::List(palette),
|
||||
};
|
||||
container.check_indices()?;
|
||||
return Ok(container);
|
||||
}
|
||||
let mut ids = Vec::with_capacity(entries);
|
||||
for i in 0..entries {
|
||||
let index = read(data, disc_bits, i);
|
||||
@@ -655,15 +674,7 @@ impl PalettedContainer {
|
||||
}
|
||||
ids.push(index);
|
||||
}
|
||||
Ok(match config {
|
||||
Config::List(bits) => Self {
|
||||
strategy,
|
||||
bits,
|
||||
words: data.to_vec(),
|
||||
palette: Palette::List(palette),
|
||||
},
|
||||
_ => Self::build(strategy, palette, &ids),
|
||||
})
|
||||
Ok(Self::build(strategy, palette, &ids))
|
||||
}
|
||||
/// Whether any palette entry satisfies `predicate` (`maybeHas`): entries
|
||||
/// no longer used by any position still count.
|
||||
@@ -796,6 +807,113 @@ mod tests {
|
||||
assert!(PalettedContainer::from_wire(BLOCKS, 0, vec![9], vec![]).is_ok());
|
||||
}
|
||||
#[test]
|
||||
fn saved_validation_matches_entrywise_reference() {
|
||||
// Keep the previous loader as an independent entry-at-a-time reference:
|
||||
// errors, resident palette order and unused storage bits must all agree.
|
||||
fn reference(
|
||||
strategy: Strategy,
|
||||
palette: Vec<u32>,
|
||||
data: Option<&[u64]>,
|
||||
) -> Result<PalettedContainer, PaletteError> {
|
||||
if palette.is_empty() {
|
||||
return Err(PaletteError::BadPalette { len: 0 });
|
||||
}
|
||||
let config = strategy.config(ceil_log2(palette.len()));
|
||||
let entries = strategy.entries();
|
||||
let bits = match config {
|
||||
Config::Single => return Ok(PalettedContainer::new(strategy, palette[0])),
|
||||
Config::List(bits) => bits,
|
||||
Config::Global => ceil_log2(palette.len()),
|
||||
};
|
||||
let expected = storage_len(bits, entries);
|
||||
let data = data.ok_or(PaletteError::BadStorage {
|
||||
expected,
|
||||
actual: 0,
|
||||
})?;
|
||||
if data.len() != expected {
|
||||
return Err(PaletteError::BadStorage {
|
||||
expected,
|
||||
actual: data.len(),
|
||||
});
|
||||
}
|
||||
let mut ids = Vec::with_capacity(entries);
|
||||
for i in 0..entries {
|
||||
let index = read(data, bits, i);
|
||||
if index as usize >= palette.len() {
|
||||
return Err(PaletteError::BadIndex {
|
||||
index,
|
||||
palette: palette.len(),
|
||||
});
|
||||
}
|
||||
ids.push(index);
|
||||
}
|
||||
Ok(match config {
|
||||
Config::List(bits) => PalettedContainer {
|
||||
strategy,
|
||||
bits,
|
||||
palette: Palette::List(palette),
|
||||
words: data.to_vec(),
|
||||
},
|
||||
_ => PalettedContainer::build(strategy, palette, &ids),
|
||||
})
|
||||
}
|
||||
fn compare(strategy: Strategy, palette: &[u32], data: Option<&[u64]>) {
|
||||
let expected = reference(strategy, palette.to_vec(), data);
|
||||
let actual = PalettedContainer::from_packed(strategy, palette.to_vec(), data);
|
||||
match (actual, expected) {
|
||||
(Ok(a), Ok(b)) => {
|
||||
assert_eq!(a.strategy, b.strategy);
|
||||
assert_eq!(a.bits, b.bits);
|
||||
assert_eq!(a.palette, b.palette);
|
||||
assert_eq!(a.words, b.words);
|
||||
}
|
||||
(Err(a), Err(b)) => assert_eq!(a, b),
|
||||
(a, b) => panic!("loader disagreement: {a:?} vs {b:?}"),
|
||||
}
|
||||
}
|
||||
for strategy in [BLOCKS, BIOMES] {
|
||||
for len in [
|
||||
0, 1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255,
|
||||
256, 257, 511, 512, 513, 1023, 1024, 1025, 2047, 2048, 2049, 4096, 4097,
|
||||
] {
|
||||
let palette: Vec<u32> = (0..len).map(|i| (i as u32 * 7) % 127).collect();
|
||||
compare(strategy, &palette, None);
|
||||
compare(strategy, &palette, Some(&[]));
|
||||
if len <= 1 {
|
||||
compare(strategy, &palette, Some(&[u64::MAX]));
|
||||
continue;
|
||||
}
|
||||
let bits = match strategy.config(ceil_log2(len)) {
|
||||
Config::List(bits) => bits,
|
||||
_ => ceil_log2(len),
|
||||
};
|
||||
let entries = strategy.entries();
|
||||
let mut words = pack_words(bits, entries, (0..entries).map(|i| (i % len) as u32));
|
||||
compare(strategy, &palette, Some(&words));
|
||||
compare(strategy, &palette, Some(&words[..words.len() - 1]));
|
||||
// Nonzero padding in every word and in the final partial word
|
||||
// is legal and must be preserved by resident list palettes.
|
||||
let per = 64 / usize::from(bits);
|
||||
for (word, positions) in words.iter_mut().zip((0..entries).step_by(per)) {
|
||||
let used = (entries - positions).min(per) * usize::from(bits);
|
||||
if used < 64 {
|
||||
*word |= u64::MAX << used;
|
||||
}
|
||||
}
|
||||
compare(strategy, &palette, Some(&words));
|
||||
if len < 1usize << bits {
|
||||
for i in [0, per - 1, per, entries / 2, entries - 1] {
|
||||
let mut invalid = words.clone();
|
||||
write(&mut invalid, bits, i, len as u32);
|
||||
// A second invalid entry verifies first-error ordering.
|
||||
write(&mut invalid, bits, entries - 1, mask(bits) as u32);
|
||||
compare(strategy, &palette, Some(&invalid));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn division_magic_is_exact() {
|
||||
for bits in 1..=32u8 {
|
||||
let per = 64 / usize::from(bits);
|
||||
|
||||
Reference in New Issue
Block a user