diff --git a/crates/horizon-client/examples/chunk_bench.rs b/crates/horizon-client/examples/chunk_bench.rs new file mode 100644 index 000000000..b99f49a2a --- /dev/null +++ b/crates/horizon-client/examples/chunk_bench.rs @@ -0,0 +1,196 @@ +//! Resident chunk storage benchmark: generates a block of vanilla terrain, then +//! reports the resident block/biome memory, the cost of lookups, meshing and +//! lighting on it. +//! +//! chunk_bench [--seed S] [--size N] +#![forbid(unsafe_code)] +use horizon_client::{ + assets::Assets, + biome::Biomes, + blocks::BlockFacts, + chunk::{Chunk, MIN_Y, SECTIONS}, + light::{LightTables, light_chunk}, + mesher::{MeshTables, Mesher}, + models, + streaming::{ChunkGenerator, Proto}, + vanillagen::VanillaTerrain, +}; +use horizon_core::biome::BiomeZoom; +use std::{path::PathBuf, sync::Arc, time::Instant}; +type Result = std::result::Result>; + +fn main() -> Result<()> { + let args: Vec = std::env::args().skip(1).collect(); + let root = PathBuf::from(args.first().ok_or("usage: chunk_bench RESOURCES")?); + let flag = |name: &str, default: i64| -> i64 { + args.iter() + .position(|a| a == name) + .and_then(|i| args.get(i + 1)) + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + }; + let (seed, size) = (flag("--seed", 12345), flag("--size", 8) as i32); + let assets = Assets::load(&root)?; + let facts = BlockFacts::load(&root, assets.registry)?; + let biomes = Biomes::load(&root)?; + let tables = MeshTables::new(&assets, &facts, models::load_visibility(&root, assets.registry)?)?; + let light = LightTables::load(&root, assets.registry, &facts)?; + let terrain = VanillaTerrain::new(&root.join("worldgen"), assets.registry, &biomes, seed)?; + let mut worker = terrain.worker(); + + // Protos for size+4 columns, decorated for the inner size+2, finished as size+2. + let started = Instant::now(); + let outer = size + 4; + let mut protos: Vec> = (0..outer * outer) + .map(|i| Some(terrain.proto(&mut worker, i % outer, i / outer))) + .collect(); + eprintln!("proto: {:.2}s", started.elapsed().as_secs_f64()); + let started = Instant::now(); + let inner = size + 2; + for z in 1..=inner { + for x in 1..=inner { + let mut region: [Proto; 9] = std::array::from_fn(|k| { + let (dx, dz) = (k as i32 % 3 - 1, k as i32 / 3 - 1); + protos[((z + dz) * outer + x + dx) as usize].take().expect("proto") + }); + terrain.decorate(&mut worker, &mut region); + for (k, p) in region.into_iter().enumerate() { + let (dx, dz) = (k as i32 % 3 - 1, k as i32 / 3 - 1); + protos[((z + dz) * outer + x + dx) as usize] = Some(p); + } + } + } + eprintln!("decorate: {:.2}s", started.elapsed().as_secs_f64()); + let started = Instant::now(); + let mut chunks: Vec = Vec::new(); + for z in 1..=inner { + for x in 1..=inner { + let proto = protos[(z * outer + x) as usize].take().expect("proto"); + let mut chunk = terrain.finish(proto); + chunk.x = x; + chunk.z = z; + chunks.push(chunk); + } + } + let finish = started.elapsed().as_secs_f64(); + println!("finish (proto -> resident chunk): {:.1} ms per chunk", finish * 1e3 / chunks.len() as f64); + + let chunks: Vec> = chunks.into_iter().map(Arc::new).collect(); + let at = |x: i32, z: i32| chunks[((z - 1) * inner + x - 1) as usize].clone(); + + // Light all inner-inner columns. + let started = Instant::now(); + let mut lit = Vec::new(); + for z in 2..inner { + for x in 2..inner { + let region: [[Option>; 3]; 3] = std::array::from_fn(|dz| { + std::array::from_fn(|dx| Some(at(x + dx as i32 - 1, z + dz as i32 - 1))) + }); + let borrowed = horizon_client::mesher::borrow(®ion); + lit.push(((x, z), light_chunk(&borrowed, &light, true))); + } + } + println!("light: {:.2} ms per column", started.elapsed().as_secs_f64() * 1e3 / lit.len() as f64); + let mut chunks: Vec = chunks.iter().map(|c| (**c).clone()).collect(); + for ((x, z), sections) in lit { + chunks[((z - 1) * inner + x - 1) as usize].sections = sections; + } + let chunks: Vec> = chunks.into_iter().map(Arc::new).collect(); + let at = |x: i32, z: i32| chunks[((z - 1) * inner + x - 1) as usize].clone(); + + // Memory. + let (mut bytes, mut sections, mut uniform) = (0usize, 0usize, 0usize); + for chunk in &chunks { + bytes += chunk.heap_bytes(); + for s in &chunk.sections { + sections += 1; + uniform += usize::from(s.is_uniform(s.get(0))); + } + } + println!( + "resident: {} chunks, {} sections ({} uniform), {:.1} KiB per chunk ({:.2} MiB total)", + chunks.len(), + sections, + uniform, + bytes as f64 / 1024.0 / chunks.len() as f64, + bytes as f64 / 1048576.0 + ); + + // Meshing. + let mut mesher = Mesher { + nether_shading: false, + assets: &assets, + tables: &tables, + biomes: &biomes, + visibility: tables.visibility.clone(), + zoom: BiomeZoom::new(7), + blend_radius: 2, + smooth_lighting: true, + cutout_leaves: true, + }; + let mut best = f64::MAX; + let mut quads = 0usize; + for _ in 0..3 { + let started = Instant::now(); + let mut jobs = 0; + quads = 0; + for z in 2..inner { + for x in 2..inner { + let region: [[Option>; 3]; 3] = std::array::from_fn(|dz| { + std::array::from_fn(|dx| Some(at(x + dx as i32 - 1, z + dz as i32 - 1))) + }); + let borrowed = horizon_client::mesher::borrow(®ion); + for sy in 0..SECTIONS as i32 { + let mesh = mesher.mesh(&borrowed, sy); + quads += (mesh.solid.len() + mesh.translucent.len()) / 4; + jobs += 1; + } + } + } + best = best.min(started.elapsed().as_secs_f64() / f64::from(jobs)); + } + println!("mesh: {:.1} us per section ({} quads total)", best * 1e6, quads); + + // Random lookups, as collision and entity queries do. + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let started = Instant::now(); + let mut sum = 0u64; + let lookups = 20_000_000u64; + for _ in 0..lookups { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + let cx = 2 + (state % u64::from((inner - 2) as u32)) as i32; + let cz = 2 + ((state >> 8) % u64::from((inner - 2) as u32)) as i32; + let chunk = &chunks[((cz - 1) * inner + cx - 1) as usize]; + let (x, y, z) = ( + (state >> 16) as i32 & 15, + MIN_Y + ((state >> 24) % 384) as i32, + (state >> 40) as i32 & 15, + ); + sum += u64::from(chunk.get(x, y, z)); + } + println!( + "random get: {:.2} ns per lookup (checksum {sum})", + started.elapsed().as_secs_f64() * 1e9 / lookups as f64 + ); + // Coherent scan, as a column or area query does. + let started = Instant::now(); + let mut sum = 0u64; + for _ in 0..4 { + for chunk in &chunks { + for y in MIN_Y..MIN_Y + 384 { + for z in 0..16 { + for x in 0..16 { + sum += u64::from(chunk.get(x, y, z)); + } + } + } + } + } + println!( + "scan get: {:.2} ns per lookup (checksum {sum})", + started.elapsed().as_secs_f64() * 1e9 / (4.0 * chunks.len() as f64 * 98304.0) + ); + Ok(()) +} diff --git a/crates/horizon-client/examples/section_mesh_fixture.rs b/crates/horizon-client/examples/section_mesh_fixture.rs index d4c710fbc..d17c33466 100644 --- a/crates/horizon-client/examples/section_mesh_fixture.rs +++ b/crates/horizon-client/examples/section_mesh_fixture.rs @@ -104,14 +104,21 @@ fn scene( blend_radius = int(&field["radius"])?; let default = biomes.id(field["default"].as_str().ok_or("missing default biome")?); for chunk in &mut chunks { - Arc::make_mut(&mut chunk.biomes).fill(default); + for qy in 0..96 { + for cell in 0..16 { + chunk.set_biome_cell(cell & 3, qy, cell >> 2, default); + } + } } for cell in field["cells"].as_array().ok_or("missing biome cells")? { let (qx, qy, qz) = (int(&cell[0])?, int(&cell[1])?, int(&cell[2])?); let i = chunk_index(qx * 4, qz * 4)?; - let slot = ((qy - MIN_Y / 4) * 16 + (qz & 3) * 4 + (qx & 3)) as usize; - Arc::make_mut(&mut chunks[i].biomes)[slot] = - biomes.id(cell[3].as_str().ok_or("biome name")?); + chunks[i].set_biome_cell( + qx, + qy - MIN_Y / 4, + qz, + biomes.id(cell[3].as_str().ok_or("biome name")?), + ); } } let region: [[Option<&Chunk>; 3]; 3] = diff --git a/crates/horizon-client/src/chunk.rs b/crates/horizon-client/src/chunk.rs index d5e65655c..207e8df52 100644 --- a/crates/horizon-client/src/chunk.rs +++ b/crates/horizon-client/src/chunk.rs @@ -2,17 +2,18 @@ //! threads can mesh and light immutable snapshots while the game thread edits //! (copy-on-write through `Arc::make_mut`). use horizon_world::registry::BlockStateId; +use horizon_core::paletted::{PalettedContainer, Strategy}; use std::sync::Arc; pub const MIN_Y: i32 = -64; pub const HEIGHT: i32 = 384; pub const SECTIONS: usize = (HEIGHT / 16) as usize; -#[derive(Clone)] -pub enum Blocks { - Uniform(u16), - Full(Box<[u16; 4096]>), -} +/// Block states are `PalettedContainer`s over the pinned registry (35,723 +/// states: a 16-bit global palette); biomes are 4x4x4 containers over the +/// biome table (67 entries: a 7-bit global palette). +pub const BLOCK_STRATEGY: Strategy = Strategy::blocks(16); +pub const BIOME_STRATEGY: Strategy = Strategy::biomes(7); /// Packed light per block: sky light in the high nibble, block light in the low one. #[derive(Clone)] pub enum Light { @@ -25,13 +26,16 @@ pub fn index(x: i32, y: i32, z: i32) -> usize { } #[derive(Clone)] pub struct Section { - pub blocks: Blocks, + pub blocks: PalettedContainer, + /// Biome table index per 4x4x4 cell, `(y << 2 | z) << 2 | x`. + pub biomes: PalettedContainer, pub light: Light, } impl Default for Section { fn default() -> Self { Self { - blocks: Blocks::Uniform(0), + blocks: PalettedContainer::new(BLOCK_STRATEGY, 0), + biomes: PalettedContainer::new(BIOME_STRATEGY, 0), light: Light::Uniform(0xF0), } } @@ -39,23 +43,19 @@ impl Default for Section { impl Section { #[inline] pub fn get(&self, i: usize) -> u16 { - match &self.blocks { - Blocks::Uniform(v) => *v, - Blocks::Full(a) => a[i], - } + self.blocks.get(i) as u16 } pub fn set(&mut self, i: usize, v: u16) -> u16 { - match &mut self.blocks { - Blocks::Uniform(u) if *u == v => v, - Blocks::Uniform(u) => { - let old = *u; - let mut a = Box::new([old; 4096]); - a[i] = v; - self.blocks = Blocks::Full(a); - old - } - Blocks::Full(a) => std::mem::replace(&mut a[i], v), - } + self.blocks.set(i, u32::from(v)) as u16 + } + /// All 4,096 block states in index order. + pub fn unpack(&self, out: &mut [u16; 4096]) { + self.blocks.unpack_u16(out); + } + /// The block states of a whole section at once. + pub fn from_blocks(values: &[u16; 4096]) -> PalettedContainer { + let wide: Vec = values.iter().map(|&v| u32::from(v)).collect(); + PalettedContainer::from_values(BLOCK_STRATEGY, &wide) } #[inline] pub fn light(&self, i: usize) -> u8 { @@ -65,17 +65,26 @@ impl Section { } } pub fn is_uniform(&self, v: u16) -> bool { - match &self.blocks { - Blocks::Uniform(u) => *u == v, - Blocks::Full(a) => a.iter().all(|&b| b == v), - } + self.blocks.is_uniform(u32::from(v)) } - /// Collapse storage that became uniform. + /// Heap bytes held by the block, biome and light storage (benchmark accounting). + pub fn heap_bytes(&self) -> usize { + self.blocks.heap_bytes() + + self.biomes.heap_bytes() + + match &self.light { + Light::Uniform(_) => 0, + Light::Full(_) => 4096, + } + } + /// Collapse storage that became uniform, and drop palette entries no + /// block uses any more. pub fn compact(&mut self) { - if let Blocks::Full(a) = &self.blocks - && a.iter().all(|&b| b == a[0]) - { - self.blocks = Blocks::Uniform(a[0]); + for container in [&mut self.blocks, &mut self.biomes] { + if container.bits() != 0 { + if let Some(v) = container.uniform() { + *container = PalettedContainer::new(container.strategy(), v); + } + } } if let Light::Full(a) = &self.light && a.iter().all(|&b| b == a[0]) @@ -89,8 +98,6 @@ pub struct Chunk { pub x: i32, pub z: i32, pub sections: Vec>, - /// Biome index per 4x4x4 cell (x,z in 0..4, y in 0..96) into `Biomes`. - pub biomes: Arc>, /// Highest non-air, light-blocking-or-fluid Y per column (MOTION_BLOCKING-like), for rain/sky. pub heights: Arc<[i16; 256]>, pub lit: bool, @@ -119,7 +126,6 @@ impl Chunk { sections: (0..SECTIONS) .map(|_| Arc::new(Section::default())) .collect(), - biomes: Arc::new(vec![0; 16 * (HEIGHT as usize / 4)]), heights: Arc::new([MIN_Y as i16; 256]), lit: false, light_loaded: false, @@ -161,9 +167,24 @@ impl Chunk { } self.sections[s as usize].light(index(x, y, z)) } + /// Heap bytes held by the sections (benchmark accounting). + pub fn heap_bytes(&self) -> usize { + self.sections.iter().map(|s| s.heap_bytes()).sum::() + } + /// Biome table index at a block position (`ChunkAccess.getNoiseBiome` with + /// the quart y clamped to the column). #[inline] pub fn biome(&self, x: i32, y: i32, z: i32) -> u16 { let qy = ((y - MIN_Y) >> 2).clamp(0, HEIGHT / 4 - 1); - self.biomes[(qy * 16 + ((z & 15) >> 2) * 4 + ((x & 15) >> 2)) as usize] + self.sections[(qy >> 2) as usize] + .biomes + .get(((qy & 3) << 4 | ((z & 15) >> 2) << 2 | (x & 15) >> 2) as usize) as u16 + } + /// Sets the biome of one quart cell (`qy` counts from the column's bottom). + pub fn set_biome_cell(&mut self, qx: i32, qy: i32, qz: i32, biome: u16) { + let section = Arc::make_mut(&mut self.sections[(qy >> 2) as usize]); + section + .biomes + .set(((qy & 3) << 4 | (qz & 3) << 2 | (qx & 3)) as usize, u32::from(biome)); } } diff --git a/crates/horizon-client/src/game/end.rs b/crates/horizon-client/src/game/end.rs index 3f88a5ed9..080525a43 100644 --- a/crates/horizon-client/src/game/end.rs +++ b/crates/horizon-client/src/game/end.rs @@ -933,9 +933,14 @@ impl Game<'_> { }; let mut found = Vec::new(); for (s, section) in chunk.sections.iter().enumerate() { - let horizon_client::chunk::Blocks::Full(blocks) = §ion.blocks else { + if !section + .blocks + .maybe_has(|v| v == u32::from(portal) || v == u32::from(gateway)) + { continue; - }; + } + let mut blocks = Box::new([0u16; 4096]); + section.unpack(&mut blocks); for (i, &b) in blocks.iter().enumerate() { if b == portal || b == gateway { let y = horizon_client::chunk::MIN_Y + s as i32 * 16 + (i >> 8) as i32; diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index 1e9f59167..ab9585758 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -9,7 +9,7 @@ use super::{Dim, Game, Mining, Screen}; use horizon_client::{ Result, - chunk::{Blocks, Chunk, Light, MIN_Y, SECTIONS, Section}, + chunk::{BIOME_STRATEGY, BLOCK_STRATEGY, Chunk, Light, MIN_Y, SECTIONS, Section}, containers::{BlockEntity, Furnace, FurnaceKind}, entities::{Experience, ItemEntity}, hud::{HudBatch, rgba}, @@ -22,7 +22,7 @@ use horizon_client::{ streaming::{ChunkGenerator, Proto, Streaming}, world::World, }; -use horizon_core::{geometry::Vec3, random::RandomSource}; +use horizon_core::{geometry::Vec3, paletted::PalettedContainer, random::RandomSource}; use horizon_protocol::{ entity_data::{ITEM_ENTITY_ITEM, Value}, item::{Hashed, Stack}, @@ -434,54 +434,55 @@ impl Game<'_> { .sections(r.dimension.sections(), 16, r.biome_bits) .map_err(|e| e.to_string())?; let mut chunk = Chunk::empty(p.x, p.z); - let mut biomes = vec![0u16; chunk.biomes.len()]; let offset = (r.dimension.min_y - MIN_Y).div_euclid(16); + // Biome table index per quart cell of the dimension's own sections. + let mut biome_cells: Vec<[u32; 64]> = Vec::with_capacity(sections.len()); for (i, s) in sections.iter().enumerate() { let target = offset + i as i32; + let mut cells = [0u32; 64]; + for (j, cell) in cells.iter_mut().enumerate() { + *cell = u32::from( + r.biome_map + .get(s.biomes.get(j) as usize) + .copied() + .unwrap_or(0), + ); + } + biome_cells.push(cells); if !(0..SECTIONS as i32).contains(&target) { continue; } - let blocks = if s.states.bits == 0 { - Blocks::Uniform(s.states.get(0) as u16) - } else { - let mut a = Box::new([0u16; 4096]); - for (j, v) in a.iter_mut().enumerate() { - *v = s.states.get(j) as u16; - } - Blocks::Full(a) - }; + let strategy = BLOCK_STRATEGY; + let palette = s.states.palette.iter().map(|&p| p as u32).collect(); + let words = s.states.data.iter().map(|&w| w as u64).collect(); + let blocks = PalettedContainer::from_wire(strategy, s.states.bits, palette, words) + .map_err(|e| e.to_string())?; let mut section = Section { blocks, + biomes: PalettedContainer::from_values(BIOME_STRATEGY, &cells), light: Light::Uniform(0xF0), }; section.compact(); chunk.sections[target as usize] = Arc::new(section); - for j in 0..64 { - let biome = r - .biome_map - .get(s.biomes.get(j) as usize) - .copied() - .unwrap_or(0); - if let Some(cell) = biomes.get_mut(target as usize * 64 + j) { - *cell = biome; + } + // Sections outside the dimension's range take its nearest biomes + // (the column's top or bottom cell layer repeated). + if !biome_cells.is_empty() { + let last = biome_cells.len() - 1; + for si in 0..SECTIONS as i32 { + let i = si - offset; + if (0..sections.len() as i32).contains(&i) { + continue; } + let (edge, layer) = if i < 0 { (0, 0) } else { (last, 3) }; + let mut cells = [0u32; 64]; + for (j, cell) in cells.iter_mut().enumerate() { + *cell = biome_cells[edge][layer * 16 + j % 16]; + } + Arc::make_mut(&mut chunk.sections[si as usize]).biomes = + PalettedContainer::from_values(BIOME_STRATEGY, &cells); } } - // Columns outside the dimension's range take its nearest biome. - let (lo, hi) = ( - offset.max(0) as usize * 64, - ((offset + sections.len() as i32).min(SECTIONS as i32)) as usize * 64, - ); - if lo < hi { - for j in 0..biomes.len() { - if j < lo { - biomes[j] = biomes[lo + j % 16]; - } else if j >= hi { - biomes[j] = biomes[hi - 16 + j % 16]; - } - } - } - chunk.biomes = Arc::new(biomes); Ok(chunk) } /// Keep the renderable block entities (chests, furnaces) in step with blocks. diff --git a/crates/horizon-client/src/light.rs b/crates/horizon-client/src/light.rs index 930a26610..93ce0f653 100644 --- a/crates/horizon-client/src/light.rs +++ b/crates/horizon-client/src/light.rs @@ -216,6 +216,7 @@ pub fn light_chunk( let mut block = vec![0u8; W * W * h]; let mut queue = VecDeque::new(); let mut sky_queue = VecDeque::new(); + let mut ids = Box::new([0u16; 4096]); for (dz, row) in region.iter().enumerate() { for (dx, chunk) in row.iter().enumerate() { let Some(chunk) = chunk else { continue }; @@ -223,8 +224,8 @@ pub fn light_chunk( if section.is_uniform(0) { continue; } - for i in 0..4096 { - let id = section.get(i); + section.unpack(&mut ids); + for (i, &id) in ids.iter().enumerate() { let (lx, lz, ly) = (i & 15, (i >> 4) & 15, i >> 8); let p = at(dx * 16 + lx, si * 16 + ly, dz * 16 + lz); states[p] = id; diff --git a/crates/horizon-client/src/mesher.rs b/crates/horizon-client/src/mesher.rs index 63a929074..52c8f22ff 100644 --- a/crates/horizon-client/src/mesher.rs +++ b/crates/horizon-client/src/mesher.rs @@ -383,6 +383,13 @@ impl Padded { let mut blocks = vec![0u16; P * P * P]; let mut light = vec![0u8; P * P * P]; let base_y = MIN_Y + sy * 16; + // The section itself is unpacked in one pass; the border cells of the + // neighbouring sections are read one by one. + let mut own = Box::new([0u16; 4096]); + let center = region[1][1].map(|c| &c.sections[sy as usize]); + if let Some(section) = center { + section.unpack(&mut own); + } for y in -B..16 + B { let wy = base_y + y; for z in -B..16 + B { @@ -390,7 +397,14 @@ impl Padded { let chunk = region[((z + 16) >> 4) as usize][((x + 16) >> 4) as usize]; let i = pi(x, y, z); if let Some(c) = chunk { - blocks[i] = c.get(x & 15, wy, z & 15); + blocks[i] = if (0..16).contains(&x) + && (0..16).contains(&y) + && (0..16).contains(&z) + { + own[(y << 8 | z << 4 | x) as usize] + } else { + c.get(x & 15, wy, z & 15) + }; light[i] = c.light(x & 15, wy, z & 15); } else { light[i] = 0xF0; diff --git a/crates/horizon-client/src/save.rs b/crates/horizon-client/src/save.rs index 1b532c59b..29a1b1dc5 100644 --- a/crates/horizon-client/src/save.rs +++ b/crates/horizon-client/src/save.rs @@ -16,10 +16,11 @@ use crate::{ Result, biome::Biomes, - chunk::{Blocks, Chunk, Light, MIN_Y, SECTIONS}, + chunk::{BIOME_STRATEGY, BLOCK_STRATEGY, Chunk, Light, MIN_Y, SECTIONS}, streaming::ChunkGenerator, }; use horizon_core::{ + paletted::PalettedContainer, position::{BlockPos, ChunkPos}, ticks::{SavedTick, TickPriority, TickTypeId}, }; @@ -42,29 +43,6 @@ use std::{ pub const VERSION_NAME: &str = "26.3"; -fn pack(indices: &[u16], bits: u32) -> Vec { - let per_long = (64 / bits) as usize; - let mut longs = vec![0i64; indices.len().div_ceil(per_long)]; - for (i, &v) in indices.iter().enumerate() { - longs[i / per_long] |= - (i64::from(v) & ((1i64 << bits) - 1)) << ((i % per_long) as u32 * bits); - } - longs -} -fn unpack(longs: &[i64], bits: u32, count: usize) -> Vec { - let per_long = (64 / bits) as usize; - let mask = (1u64 << bits) - 1; - (0..count) - .map(|i| { - longs.get(i / per_long).map_or(0, |&l| { - ((l as u64 >> ((i % per_long) as u32 * bits)) & mask) as u16 - }) - }) - .collect() -} -fn bits_for(len: usize, min: u32) -> u32 { - (usize::BITS - (len.max(1) - 1).leading_zeros()).max(min) -} fn string(s: &str) -> Tag { Tag::String(s.into()) } @@ -584,33 +562,22 @@ pub fn encode_with( }); for (si, section) in chunk.sections.iter().enumerate() { let y = (MIN_Y >> 4) + si as i32; - let ids: Vec = match §ion.blocks { - Blocks::Uniform(v) => vec![*v], - Blocks::Full(a) => a.to_vec(), - }; - let mut palette: Vec = Vec::new(); - let mut lookup: HashMap = HashMap::new(); - let indices: Vec = ids - .iter() - .map(|id| { - *lookup.entry(*id).or_insert_with(|| { - palette.push(*id); - (palette.len() - 1) as u16 - }) - }) - .collect(); + // PalettedContainer.pack: distinct states in index order, indices at + // the saved width. + let packed = section.blocks.pack(); let mut states = vec![( "palette", Tag::List( - palette + packed + .palette .iter() - .map(|id| { - let state = registry.state(BlockStateId(u32::from(*id))); + .map(|&id| { + let state = registry.state(BlockStateId(id)); match state { // BlockState.CODEC: the block name for default states. Some(s) if tables.is_some_and(|t| { - t.default_state.get(*id as usize) == Some(&true) + t.default_state.get(id as usize) == Some(&true) }) => { string(&s.name) @@ -622,37 +589,27 @@ pub fn encode_with( .collect(), ), )]; - if palette.len() > 1 { + if let Some(data) = &packed.data { states.push(( "data", - Tag::LongArray(pack(&indices, bits_for(palette.len(), 4))), + Tag::LongArray(data.iter().map(|&w| w as i64).collect()), )); } - let quart: Vec = (0..64).map(|i| chunk.biomes[si * 64 + i]).collect(); - let mut biome_palette: Vec = Vec::new(); - let biome_indices: Vec = quart - .iter() - .map(|b| match biome_palette.iter().position(|p| p == b) { - Some(i) => i as u16, - None => { - biome_palette.push(*b); - (biome_palette.len() - 1) as u16 - } - }) - .collect(); + let biome_packed = section.biomes.pack(); let mut biome_tag = vec![( "palette", Tag::List( - biome_palette + biome_packed + .palette .iter() - .map(|b| string(&biomes.get(*b).name)) + .map(|&b| string(&biomes.get(b as u16).name)) .collect(), ), )]; - if biome_palette.len() > 1 { + if let Some(data) = &biome_packed.data { biome_tag.push(( "data", - Tag::LongArray(pack(&biome_indices, bits_for(biome_palette.len(), 0))), + Tag::LongArray(data.iter().map(|&w| w as i64).collect()), )); } let mut entries = vec![ @@ -811,7 +768,6 @@ pub fn decode_with( } let air = registry.default_state("minecraft:air")?.0 as u16; let mut cache: HashMap = HashMap::new(); - let mut biome_data = (*chunk.biomes).clone(); let Some(Tag::List(sections)) = tag.get("sections") else { return Some(chunk); }; @@ -854,15 +810,14 @@ pub fn decode_with( }) .collect(); let target = Arc::make_mut(&mut chunk.sections[si as usize]); - if ids.len() <= 1 { - target.blocks = Blocks::Uniform(ids.first().copied().unwrap_or(air)); - } else if let Some(Tag::LongArray(data)) = states.get("data") { - let indices = unpack(data, bits_for(ids.len(), 4), 4096); - let mut full = Box::new([air; 4096]); - for (i, idx) in indices.iter().enumerate() { - full[i] = ids.get(*idx as usize).copied().unwrap_or(air); - } - target.blocks = Blocks::Full(full); + let data = match states.get("data") { + Some(Tag::LongArray(data)) => Some(data.iter().map(|&w| w as u64).collect::>()), + _ => None, + }; + let palette: Vec = ids.iter().map(|&id| u32::from(id)).collect(); + if let Ok(blocks) = PalettedContainer::from_packed(BLOCK_STRATEGY, palette, data.as_deref()) + { + target.blocks = blocks; } } if let Some(b) = section.get("biomes") @@ -875,18 +830,21 @@ pub fn decode_with( _ => 0, }) .collect(); - let indices = match b.get("data") { - Some(Tag::LongArray(data)) if ids.len() > 1 => { - unpack(data, bits_for(ids.len(), 0), 64) - } - _ => vec![0; 64], + let data = match b.get("data") { + Some(Tag::LongArray(data)) => Some(data.iter().map(|&w| w as u64).collect::>()), + _ => None, }; - for (i, idx) in indices.iter().enumerate() { - biome_data[si as usize * 64 + i] = ids.get(*idx as usize).copied().unwrap_or(0); + let palette: Vec = ids.iter().map(|&id| u32::from(id)).collect(); + let target = Arc::make_mut(&mut chunk.sections[si as usize]); + if let Ok(container) = + PalettedContainer::from_packed(BIOME_STRATEGY, palette.clone(), data.as_deref()) + { + target.biomes = container; + } else if let Some(&first) = palette.first() { + target.biomes = PalettedContainer::new(BIOME_STRATEGY, first); } } } - chunk.biomes = Arc::new(biome_data); let mut extras = ChunkExtras { last_update: number(tag.get("LastUpdate")).unwrap_or(0), ..ChunkExtras::default() diff --git a/crates/horizon-client/src/testgen.rs b/crates/horizon-client/src/testgen.rs index 3ca7fff7d..2581d9229 100644 --- a/crates/horizon-client/src/testgen.rs +++ b/crates/horizon-client/src/testgen.rs @@ -76,7 +76,13 @@ impl ChunkGenerator for TestTerrain { impl TestTerrain { fn generate(&self, _: &mut (), cx: i32, cz: i32) -> Chunk { let mut chunk = Chunk::empty(cx, cz); - chunk.biomes = std::sync::Arc::new(vec![self.plains; chunk.biomes.len()]); + for s in &mut chunk.sections { + std::sync::Arc::make_mut(s).biomes = + horizon_core::paletted::PalettedContainer::new( + crate::chunk::BIOME_STRATEGY, + u32::from(self.plains), + ); + } for z in 0..16 { for x in 0..16 { let (wx, wz) = (cx * 16 + x, cz * 16 + z); diff --git a/crates/horizon-client/src/vanillagen.rs b/crates/horizon-client/src/vanillagen.rs index 19dc4ec2b..b0d56a66e 100644 --- a/crates/horizon-client/src/vanillagen.rs +++ b/crates/horizon-client/src/vanillagen.rs @@ -2,10 +2,10 @@ //! aquifers, surface rules, carvers and placed features) to chunk streaming. use crate::{ biome::Biomes, - chunk::{Chunk, HEIGHT, MIN_Y}, + chunk::{BIOME_STRATEGY, Chunk, HEIGHT, MIN_Y, SECTIONS, Section}, streaming::{ChunkGenerator, Proto}, }; -use horizon_core::position::ChunkPos; +use horizon_core::{paletted::PalettedContainer, position::ChunkPos}; use horizon_world::{ registry::BlockRegistry, worldgen::generator::{GeneratorData, GeneratorWorker, OverworldGenerator, ProtoChunk}, @@ -177,15 +177,26 @@ impl ChunkGenerator for VanillaTerrain { chunk.terrain.min_y().max(MIN_Y), (chunk.terrain.min_y() + chunk.terrain.height()).min(MIN_Y + HEIGHT), ); - for y in lo..hi { - for z in 0..16 { - for x in 0..16 { - let id = chunk.block(bx + x, y, bz + z).0 as u16; - if id != 0 { - out.set(x, y, z, id); + let mut values = Box::new([0u16; 4096]); + for si in 0..SECTIONS as i32 { + let base = MIN_Y + si * 16; + if base + 16 <= lo || base >= hi { + continue; + } + values.fill(0); + let mut any = false; + for y in base.max(lo)..(base + 16).min(hi) { + for z in 0..16 { + for x in 0..16 { + let id = chunk.block(bx + x, y, bz + z).0 as u16; + values[((y & 15) << 8 | z << 4 | x) as usize] = id; + any |= id != 0; } } } + if any { + Arc::make_mut(&mut out.sections[si as usize]).blocks = Section::from_blocks(&values); + } } let base_section = chunk.terrain.min_y() >> 4; for (i, marks) in chunk.terrain.postprocessing().iter().enumerate() { @@ -199,19 +210,17 @@ impl ChunkGenerator for VanillaTerrain { } } } - let mut biomes = vec![0u16; 16 * (HEIGHT as usize / 4)]; - for qy in 0..HEIGHT / 4 { - for qz in 0..4 { - for qx in 0..4 { - let y = (MIN_Y + qy * 4).clamp(lo, hi - 1); - let id = chunk.biome_id_at(bx + qx * 4, y, bz + qz * 4); - let index = id.0 as usize; - biomes[(qy * 16 + qz * 4 + qx) as usize] = - self.biome_map.get(index).copied().unwrap_or(0); - } + for si in 0..SECTIONS as i32 { + let mut cells = [0u32; 64]; + for (j, cell) in cells.iter_mut().enumerate() { + let (qx, qz, qy) = (j as i32 & 3, (j as i32 >> 2) & 3, si * 4 + (j as i32 >> 4)); + let y = (MIN_Y + qy * 4).clamp(lo, hi - 1); + let id = chunk.biome_id_at(bx + qx * 4, y, bz + qz * 4); + *cell = u32::from(self.biome_map.get(id.0 as usize).copied().unwrap_or(0)); } + Arc::make_mut(&mut out.sections[si as usize]).biomes = + PalettedContainer::from_values(BIOME_STRATEGY, &cells); } - out.biomes = Arc::new(biomes); out.block_entities = crate::structure_blocks::block_entities(&chunk, self.loot.as_deref()); out.structure_refs = chunk .structures() diff --git a/crates/horizon-core/examples/paletted_fixture.rs b/crates/horizon-core/examples/paletted_fixture.rs new file mode 100644 index 000000000..4008e573f --- /dev/null +++ b/crates/horizon-core/examples/paletted_fixture.rs @@ -0,0 +1,135 @@ +//! Text transport for the PalettedContainer comparison: replays scripted edits +//! and prints the same hashes as `VanillaPalettedContainerOracle`. +use horizon_core::paletted::{PalettedContainer, Strategy}; +use std::{ + fs, + io::{self, Write}, +}; +const BASIS: u64 = 0xcbf2_9ce4_8422_2325; +fn fnv(h: u64, b: u8) -> u64 { + (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3) +} +fn hash_bytes(bytes: &[u8]) -> u64 { + bytes.iter().fold(BASIS, |h, &b| fnv(h, b)) +} +fn varint(out: &mut Vec, mut v: u32) { + while v >= 0x80 { + out.push((v & 0x7F) as u8 | 0x80); + v >>= 7; + } + out.push(v as u8); +} +/// `PalettedContainer.write`. +fn write(c: &PalettedContainer) -> Vec { + let (bits, palette, words) = c.to_wire(); + let mut out = vec![bits]; + if bits == 0 { + varint(&mut out, palette[0]); + } else if !palette.is_empty() { + varint(&mut out, palette.len() as u32); + for p in palette { + varint(&mut out, p); + } + } + for w in words { + out.extend(w.to_be_bytes()); + } + out +} +/// `PalettedContainer.read` for the leading byte, palette and longs. +fn read(strategy: Strategy, bytes: &[u8]) -> PalettedContainer { + let mut at = 1; + let mut varint = || { + let (mut v, mut shift) = (0u32, 0); + loop { + let b = bytes[at]; + at += 1; + v |= u32::from(b & 0x7F) << shift; + if b & 0x80 == 0 { + return v; + } + shift += 7; + } + }; + let bits = bytes[0]; + let palette = if bits == 0 { + vec![varint()] + } else if strategy.wire_has_palette(bits) { + let n = varint(); + (0..n).map(|_| varint()).collect() + } else { + Vec::new() + }; + let words = bytes[at..] + .chunks_exact(8) + .map(|c| u64::from_be_bytes(c.try_into().unwrap())) + .collect(); + PalettedContainer::from_wire(strategy, bits, palette, words).unwrap() +} +fn wire(c: &PalettedContainer) -> String { + let bytes = write(c); + format!("{}:{}", bytes.len(), hash_bytes(&bytes) as i64) +} +fn values(c: &PalettedContainer) -> i64 { + let mut h = BASIS; + for i in 0..c.strategy().entries() { + for b in c.get(i).to_le_bytes() { + h = fnv(h, b); + } + } + h as i64 +} +fn main() -> Result<(), Box> { + let args: Vec = std::env::args().collect(); + let mut out = io::BufWriter::new(fs::File::create(&args[2])?); + for line in fs::read_to_string(&args[1])?.lines() { + let t: Vec<&str> = line.split_whitespace().collect(); + let strategy = if t[1] == "blocks" { + Strategy::blocks(16) + } else { + Strategy::biomes(7) + }; + let mut c = PalettedContainer::new(strategy, t[2].parse()?); + let every: usize = t[3].parse()?; + let (mut olds, mut n) = (BASIS, 0usize); + let mut checkpoints = Vec::new(); + for pair in t[4..].chunks(2) { + let old = c.set(pair[0].parse()?, pair[1].parse()?); + for b in old.to_le_bytes() { + olds = fnv(olds, b); + } + n += 1; + if n % every == 0 { + checkpoints.push(wire(&c)); + } + } + let reread = read(strategy, &write(&c)); + let packed = c.pack(); + let storage = packed.data.as_ref().map_or(BASIS, |data| { + data.iter() + .flat_map(|w| w.to_le_bytes()) + .fold(BASIS, |h, b| fnv(h, b)) + }); + let palette: Vec = packed.palette.iter().map(u32::to_string).collect(); + let loaded = + PalettedContainer::from_packed(strategy, packed.palette.clone(), packed.data.as_deref())?; + writeln!( + out, + "{} {} cp={} bits={} final={} values={} read={}/{} packed={}/{}/{} loaded={}/{}", + t[0], + olds as i64, + checkpoints.join(","), + c.bits(), + wire(&c), + values(&c), + values(&reread), + wire(&reread), + packed.bits, + storage as i64, + palette.join(","), + values(&loaded), + wire(&loaded), + )?; + } + Ok(()) +} diff --git a/crates/horizon-core/src/lib.rs b/crates/horizon-core/src/lib.rs index 01af3f932..79045910a 100644 --- a/crates/horizon-core/src/lib.rs +++ b/crates/horizon-core/src/lib.rs @@ -9,6 +9,7 @@ pub mod carving; pub mod density; pub mod geometry; pub mod noise; +pub mod paletted; pub mod position; pub mod random; pub mod storage; diff --git a/crates/horizon-core/src/paletted.rs b/crates/horizon-core/src/paletted.rs new file mode 100644 index 000000000..8f39cebb7 --- /dev/null +++ b/crates/horizon-core/src/paletted.rs @@ -0,0 +1,739 @@ +//! Resident `PalettedContainer` of Java Edition 26.3: single-value, linear and +//! hash-map palettes with vanilla's bit-width thresholds, and the global +//! palette beyond them, over `SimpleBitStorage` words (values never span +//! longs). One container type serves blocks (16^3 entries) and biomes (4^3). +//! +//! Growth follows `PalettedContainer.onResize`: a full palette asks for one +//! more bit, the new configuration is built by re-adding the distinct values +//! in index order, and the value that did not fit is added last. Palettes +//! never shrink while resident; `pack` writes the canonical saved form. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + /// `Strategy.createForBlockStates`: 4096 entries, linear palettes up to 4 + /// bits and hash-map palettes from 5 to 8. + Blocks, + /// `Strategy.createForBiomes`: 64 entries, linear palettes up to 3 bits. + Biomes, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Strategy { + pub kind: Kind, + /// `ceillog2` of the registry size: the width of the global palette. + pub global_bits: u8, +} +/// `getConfigurationForBitCount`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Config { + Single, + /// Palette of up to `1 << bits` entries, stored in `bits` bits. + List(u8), + Global, +} +impl Strategy { + pub const fn blocks(global_bits: u8) -> Self { + Self { + kind: Kind::Blocks, + global_bits, + } + } + pub const fn biomes(global_bits: u8) -> Self { + Self { + kind: Kind::Biomes, + global_bits, + } + } + pub const fn entries(self) -> usize { + match self.kind { + Kind::Blocks => 4096, + Kind::Biomes => 64, + } + } + fn config(self, bits: u8) -> Config { + match (self.kind, bits) { + (_, 0) => Config::Single, + (Kind::Blocks, 1..=4) => Config::List(4), + (Kind::Blocks, 5..=8) => Config::List(bits), + (Kind::Biomes, 1..=3) => Config::List(bits), + _ => Config::Global, + } + } + /// Whether a container written with leading byte `bits` lists its palette + /// (all but the global one). + pub fn wire_has_palette(self, bits: u8) -> bool { + self.config(bits) != Config::Global + } + /// Bits of the storage a configuration keeps in memory. + fn storage_bits(self, config: Config) -> u8 { + match config { + Config::Single => 0, + Config::List(bits) => bits, + Config::Global => self.global_bits, + } + } +} +/// `Mth.ceillog2`. +pub fn ceil_log2(n: usize) -> u8 { + if n <= 1 { + 0 + } else { + (usize::BITS - (n - 1).leading_zeros()) as u8 + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PaletteError { + /// A palette bigger than its configuration allows, or empty. + BadPalette { len: usize }, + /// Storage with the wrong number of longs. + BadStorage { expected: usize, actual: usize }, + /// A stored index past the palette. + BadIndex { index: u32, palette: usize }, + /// The declared bit count does not match the palette. + BadBits { declared: u8, expected: u8 }, +} +impl std::fmt::Display for PaletteError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} +impl std::error::Error for PaletteError {} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Palette { + Single(u32), + /// Linear and hash-map palettes alike: ids in insertion order. + List(Vec), + Global, +} +/// `n / per_word` for `n < 4096` by multiplication: `MAGIC[bits] = ceil(2^20 / +/// (64 / bits))` is exact while `n * (64 / bits) < 2^20`. +const MAGIC: [u32; 33] = { + let mut table = [0; 33]; + let mut bits = 1; + while bits <= 32 { + let per = 64 / bits as u32; + table[bits] = (1u32 << 20).div_ceil(per); + bits += 1; + } + table +}; +/// Words of `bits`-wide values: `entries.div_ceil(64 / bits)`. +pub fn storage_len(bits: u8, entries: usize) -> usize { + if bits == 0 { + 0 + } else { + entries.div_ceil(64 / usize::from(bits)) + } +} +#[inline] +fn locate(bits: u8, i: usize) -> (usize, u32) { + if bits.is_power_of_two() { + // 64 / bits is a power of two: shifts instead of the multiplication. + let log = bits.trailing_zeros(); + let word = i >> (6 - log); + (word, ((i & ((64 >> log) - 1)) as u32) << log) + } else { + let per = 64 / u32::from(bits); + let word = (i as u32 * MAGIC[usize::from(bits)]) >> 20; + (word as usize, (i as u32 - word * per) * u32::from(bits)) + } +} +#[inline] +fn mask(bits: u8) -> u64 { + (1u64 << bits) - 1 +} +fn read(words: &[u64], bits: u8, i: usize) -> u32 { + let (word, shift) = locate(bits, i); + ((words[word] >> shift) & mask(bits)) as u32 +} +fn write(words: &mut [u64], bits: u8, i: usize, value: u32) { + let (word, shift) = locate(bits, i); + words[word] = (words[word] & !(mask(bits) << shift)) | (u64::from(value) << shift); +} +/// Words holding `ids` in `bits`-wide slots. +fn pack_words(bits: u8, entries: usize, ids: impl Iterator) -> Vec { + let mut words = vec![0u64; storage_len(bits, entries)]; + for (i, id) in ids.enumerate() { + write(&mut words, bits, i, id); + } + words +} +/// The saved form: distinct values in index order, and the packed indices +/// when there is more than one value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Packed { + pub palette: Vec, + /// Bits per index in `data` (0 when `data` is absent). + pub bits: u8, + pub data: Option>, +} + +#[derive(Clone, Debug)] +pub struct PalettedContainer { + strategy: Strategy, + bits: u8, + palette: Palette, + words: Vec, +} +impl PalettedContainer { + /// `new PalettedContainer(initialValue, strategy)`. + pub fn new(strategy: Strategy, value: u32) -> Self { + Self { + strategy, + bits: 0, + palette: Palette::Single(value), + words: Vec::new(), + } + } + pub fn strategy(&self) -> Strategy { + self.strategy + } + /// Bits per stored index (`BitStorage.getBits`). + pub fn bits(&self) -> u8 { + self.bits + } + /// Entries in the palette (the global palette reports its bit capacity). + pub fn palette_len(&self) -> usize { + match &self.palette { + Palette::Single(_) => 1, + Palette::List(list) => list.len(), + Palette::Global => 1 << self.bits.min(31), + } + } + pub fn is_global(&self) -> bool { + matches!(self.palette, Palette::Global) + } + #[inline] + fn raw(&self, i: usize) -> usize { + let (word, shift) = locate(self.bits, i); + ((self.words[word] >> shift) & mask(self.bits)) as usize + } + #[inline] + pub fn get(&self, i: usize) -> u32 { + match &self.palette { + Palette::Single(value) => *value, + Palette::List(list) => list[self.raw(i)], + Palette::Global => self.raw(i) as u32, + } + } + /// The value when every entry is equal, without the palette having to be + /// minimal. + pub fn uniform(&self) -> Option { + match &self.palette { + Palette::Single(value) => Some(*value), + Palette::List(list) if list.len() == 1 => Some(list[0]), + Palette::Global => self.all_equal(self.get(0)).then(|| self.get(0)), + Palette::List(list) => { + let first = list[self.raw(0)]; + self.all_equal(first).then_some(first) + } + } + } + /// Whether all entries equal `value`. + pub fn is_uniform(&self, value: u32) -> bool { + match &self.palette { + Palette::Single(v) => *v == value, + Palette::List(list) => match list.iter().position(|&v| v == value) { + None => false, + Some(_) if list.len() == 1 => true, + Some(_) => self.all_equal(value), + }, + Palette::Global => self.all_equal(value), + } + } + fn all_equal(&self, value: u32) -> bool { + let id = match &self.palette { + Palette::Single(v) => return *v == value, + Palette::List(list) => match list.iter().position(|&v| v == value) { + Some(id) => id as u64, + None => return false, + }, + Palette::Global => u64::from(value), + }; + let bits = usize::from(self.bits); + let per = 64 / bits; + let pattern = (0..per).fold(0u64, |word, slot| word | id << (slot * bits)); + let entries = self.strategy.entries(); + let full = entries / per; + if self.words[..full].iter().any(|&w| w != pattern) { + return false; + } + let tail = entries - full * per; + tail == 0 || { + let used = (1u64 << (tail * bits)) - 1; + self.words[full] & used == pattern & used + } + } + /// `getAndSet`: stores `value` and returns the previous one. + pub fn set(&mut self, i: usize, value: u32) -> u32 { + let id = self.id_for(value); + match &self.palette { + Palette::Single(old) => *old, + Palette::List(list) => { + let old = read(&self.words, self.bits, i) as usize; + write(&mut self.words, self.bits, i, id as u32); + list[old] + } + Palette::Global => { + let old = read(&self.words, self.bits, i); + write(&mut self.words, self.bits, i, id as u32); + old + } + } + } + /// `Palette.idFor`, growing the container when the palette is full. + fn id_for(&mut self, value: u32) -> usize { + match &mut self.palette { + Palette::Single(v) if *v == value => 0, + Palette::Single(_) => self.resize(1, value), + Palette::List(list) => { + if let Some(id) = list.iter().position(|&v| v == value) { + id + } else if list.len() < 1 << self.bits { + list.push(value); + list.len() - 1 + } else { + self.resize(self.bits + 1, value) + } + } + Palette::Global => value as usize, + } + } + /// `PalettedContainer.onResize`: rebuild with the configuration for + /// `bits`, then add `last` and return its id. + fn resize(&mut self, bits: u8, last: u32) -> usize { + let strategy = self.strategy; + let config = strategy.config(bits); + let storage_bits = strategy.storage_bits(config); + let entries = strategy.entries(); + let mut palette = match config { + Config::Single => Palette::Single(last), + Config::List(_) => Palette::List(Vec::new()), + Config::Global => Palette::Global, + }; + let mut words = vec![0u64; storage_len(storage_bits, entries)]; + let id_in = |palette: &mut Palette, value: u32| -> u32 { + match palette { + Palette::Single(_) => 0, + Palette::List(list) => match list.iter().position(|&v| v == value) { + Some(id) => id as u32, + None => { + list.push(value); + list.len() as u32 - 1 + } + }, + Palette::Global => value, + } + }; + if let Palette::Single(only) = self.palette { + let id = id_in(&mut palette, only); + if storage_bits > 0 && id != 0 { + for i in 0..entries { + write(&mut words, storage_bits, i, id); + } + } + } else { + let (mut last_old, mut new_id) = (u32::MAX, 0); + for i in 0..entries { + let old = read(&self.words, self.bits, i); + if old != last_old { + last_old = old; + let value = match &self.palette { + Palette::List(list) => list[old as usize], + _ => old, + }; + new_id = id_in(&mut palette, value); + } + if storage_bits > 0 { + write(&mut words, storage_bits, i, new_id); + } + } + } + let id = id_in(&mut palette, last); + self.palette = palette; + self.bits = storage_bits; + self.words = words; + id as usize + } + /// All values in index order. + pub fn unpack(&self, out: &mut [u32]) { + assert_eq!(out.len(), self.strategy.entries()); + match &self.palette { + Palette::Single(value) => out.fill(*value), + palette => { + let bits = usize::from(self.bits); + let per = 64 / bits; + let m = mask(self.bits); + for (chunk, &word) in out.chunks_mut(per).zip(&self.words) { + for (slot, cell) in chunk.iter_mut().enumerate() { + let id = ((word >> (slot * bits)) & m) as u32; + *cell = match palette { + Palette::List(list) => list[id as usize], + _ => id, + }; + } + } + } + } + } + /// Block ids as 16-bit values (callers whose registry fits in 16 bits). + pub fn unpack_u16(&self, out: &mut [u16; 4096]) { + debug_assert_eq!(self.strategy.entries(), 4096); + match &self.palette { + Palette::Single(value) => out.fill(*value as u16), + palette => { + let bits = usize::from(self.bits); + let per = 64 / bits; + let m = mask(self.bits); + for (chunk, &word) in out.chunks_mut(per).zip(&self.words) { + for (slot, cell) in chunk.iter_mut().enumerate() { + let id = ((word >> (slot * bits)) & m) as usize; + *cell = match palette { + Palette::List(list) => list[id] as u16, + _ => id as u16, + }; + } + } + } + } + } + /// A container holding `values` in the smallest configuration for their + /// distinct count, with palette ids in first-occurrence order, like a + /// container unpacked from disk. + pub fn from_values(strategy: Strategy, values: &[u32]) -> Self { + assert_eq!(values.len(), strategy.entries()); + let mut palette: Vec = Vec::new(); + let mut small = true; + let mut lookup: std::collections::HashMap = std::collections::HashMap::new(); + let mut ids = Vec::with_capacity(values.len()); + for &value in values { + let id = if small { + match palette.iter().position(|&v| v == value) { + Some(id) => id as u32, + None => { + palette.push(value); + if palette.len() > 16 { + small = false; + lookup.extend(palette.iter().enumerate().map(|(i, &v)| (v, i as u32))); + } + palette.len() as u32 - 1 + } + } + } else { + *lookup.entry(value).or_insert_with(|| { + palette.push(value); + palette.len() as u32 - 1 + }) + }; + ids.push(id); + } + Self::build(strategy, palette, &ids) + } + /// A container over a distinct-value `palette` and an id per entry. + fn build(strategy: Strategy, palette: Vec, ids: &[u32]) -> Self { + let config = strategy.config(ceil_log2(palette.len())); + match config { + Config::Single => Self::new(strategy, palette[0]), + Config::List(bits) => Self { + strategy, + bits, + words: pack_words(bits, ids.len(), ids.iter().copied()), + palette: Palette::List(palette), + }, + Config::Global => { + let bits = strategy.global_bits; + Self { + strategy, + bits, + words: pack_words(bits, ids.len(), ids.iter().map(|&id| palette[id as usize])), + palette: Palette::Global, + } + } + } + } + /// `PalettedContainer.read`: the container as sent on the wire. `bits` is + /// the leading byte, `palette` the VarInt entries (a single one for + /// `bits == 0`, none for the global palette) and `words` the longs. + pub fn from_wire( + strategy: Strategy, + bits: u8, + palette: Vec, + words: Vec, + ) -> Result { + let config = strategy.config(bits); + let storage_bits = strategy.storage_bits(config); + let expected = storage_len(storage_bits, strategy.entries()); + if words.len() != expected { + return Err(PaletteError::BadStorage { + expected, + actual: words.len(), + }); + } + match config { + Config::Single => { + if palette.len() != 1 { + return Err(PaletteError::BadPalette { len: palette.len() }); + } + Ok(Self::new(strategy, palette[0])) + } + Config::List(bits) => { + if palette.is_empty() || palette.len() > 1 << bits { + return Err(PaletteError::BadPalette { len: palette.len() }); + } + let container = Self { + strategy, + bits, + palette: Palette::List(palette), + words, + }; + container.check_indices()?; + Ok(container) + } + Config::Global => Ok(Self { + strategy, + bits: storage_bits, + palette: Palette::Global, + words, + }), + } + } + fn check_indices(&self) -> Result<(), PaletteError> { + let Palette::List(list) = &self.palette else { + return Ok(()); + }; + if list.len() == 1 << self.bits { + return Ok(()); + } + for i in 0..self.strategy.entries() { + let index = read(&self.words, self.bits, i); + if index as usize >= list.len() { + return Err(PaletteError::BadIndex { + index, + palette: list.len(), + }); + } + } + Ok(()) + } + /// `PalettedContainer.write`: the leading bits byte, the palette entries + /// (none for the global palette) and the raw longs, exactly as resident. + pub fn to_wire(&self) -> (u8, Vec, &[u64]) { + let palette = match &self.palette { + Palette::Single(value) => vec![*value], + Palette::List(list) => list.clone(), + Palette::Global => Vec::new(), + }; + (self.bits, palette, &self.words) + } + /// `PalettedContainer.pack`: distinct values in index order, indices + /// re-encoded for the saved bit width. + pub fn pack(&self) -> Packed { + let entries = self.strategy.entries(); + let mut palette: Vec = Vec::new(); + let mut ids = Vec::with_capacity(entries); + let mut values = vec![0u32; entries]; + self.unpack(&mut values); + let mut lookup: std::collections::HashMap = std::collections::HashMap::new(); + for &value in &values { + let id = *lookup.entry(value).or_insert_with(|| { + palette.push(value); + palette.len() as u32 - 1 + }); + ids.push(id); + } + let bits = match self.strategy.config(ceil_log2(palette.len())) { + Config::Single => 0, + Config::List(bits) => bits, + Config::Global => ceil_log2(palette.len()), + }; + let data = (bits != 0).then(|| pack_words(bits, entries, ids.iter().copied())); + Packed { + palette, + bits, + data, + } + } + /// `PalettedContainer.unpack`: a container from the saved form. + pub fn from_packed( + strategy: Strategy, + palette: Vec, + data: Option<&[u64]>, + ) -> Result { + if palette.is_empty() { + return Err(PaletteError::BadPalette { len: 0 }); + } + let config = strategy.config(ceil_log2(palette.len())); + let entries = strategy.entries(); + let disc_bits = match config { + Config::Single => return Ok(Self::new(strategy, palette[0])), + Config::List(bits) => bits, + Config::Global => ceil_log2(palette.len()), + }; + let data = data.ok_or(PaletteError::BadStorage { + expected: storage_len(disc_bits, entries), + actual: 0, + })?; + let expected = storage_len(disc_bits, entries); + if data.len() != expected { + return Err(PaletteError::BadStorage { + expected, + actual: data.len(), + }); + } + let mut ids = Vec::with_capacity(entries); + for i in 0..entries { + let index = read(data, disc_bits, i); + if index as usize >= palette.len() { + return Err(PaletteError::BadIndex { + index, + palette: palette.len(), + }); + } + ids.push(index); + } + Ok(match config { + Config::List(bits) => Self { + strategy, + bits, + words: data.to_vec(), + palette: Palette::List(palette), + }, + _ => Self::build(strategy, palette, &ids), + }) + } + /// Whether any palette entry satisfies `predicate` (`maybeHas`): entries + /// no longer used by any position still count. + pub fn maybe_has(&self, mut predicate: impl FnMut(u32) -> bool) -> bool { + match &self.palette { + Palette::Single(value) => predicate(*value), + Palette::List(list) => list.iter().any(|&v| predicate(v)), + Palette::Global => (0..1u32 << self.bits.min(31)).any(predicate), + } + } + /// Heap bytes held by the palette and the storage. + pub fn heap_bytes(&self) -> usize { + self.words.capacity() * 8 + + match &self.palette { + Palette::List(list) => list.capacity() * 4, + _ => 0, + } + } +} +impl PartialEq for PalettedContainer { + /// Equal when every entry is. + fn eq(&self, other: &Self) -> bool { + let entries = self.strategy.entries(); + entries == other.strategy.entries() && (0..entries).all(|i| self.get(i) == other.get(i)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + const BLOCKS: Strategy = Strategy::blocks(16); + const BIOMES: Strategy = Strategy::biomes(7); + fn rng(state: &mut u64) -> u64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state + } + #[test] + fn grows_through_vanilla_widths() { + let mut c = PalettedContainer::new(BLOCKS, 0); + assert_eq!((c.bits(), c.palette_len()), (0, 1)); + let mut expected = Vec::new(); + for n in 1..=300u32 { + c.set(n as usize, n); + expected.push(match n + 1 { + 2..=16 => 4, + 17..=32 => 5, + 33..=64 => 6, + 65..=128 => 7, + 129..=256 => 8, + _ => 16, + }); + assert_eq!(c.bits(), *expected.last().unwrap(), "after {n} values"); + } + assert!(c.is_global()); + for n in 1..=300u32 { + assert_eq!(c.get(n as usize), n); + } + let mut b = PalettedContainer::new(BIOMES, 3); + for n in 1..=9u32 { + b.set(n as usize, n + 3); + let want = match n + 1 { + 2 => 1, + 3..=4 => 2, + 5..=8 => 3, + _ => 7, + }; + assert_eq!(b.bits(), want, "after {n} biomes"); + } + } + #[test] + fn matches_a_plain_array_under_random_edits() { + for (strategy, distinct) in [(BLOCKS, 700u64), (BLOCKS, 12), (BIOMES, 9), (BIOMES, 3)] { + let entries = strategy.entries(); + let mut state = 0x1234_5678_9ABC_DEF1u64; + let mut c = PalettedContainer::new(strategy, 5); + let mut plain = vec![5u32; entries]; + for _ in 0..20_000 { + let r = rng(&mut state); + let i = (r % entries as u64) as usize; + let v = ((r >> 20) % distinct) as u32 * 7; + assert_eq!(c.set(i, v), plain[i]); + plain[i] = v; + } + let mut out = vec![0; entries]; + c.unpack(&mut out); + assert_eq!(out, plain); + for (i, &v) in plain.iter().enumerate() { + assert_eq!(c.get(i), v); + } + let packed = c.pack(); + let loaded = PalettedContainer::from_packed(strategy, packed.palette.clone(), packed.data.as_deref()) + .unwrap(); + assert_eq!(loaded, c); + assert_eq!(loaded.pack(), packed); + let again = PalettedContainer::from_values(strategy, &plain); + assert_eq!(again, c); + assert_eq!(again.pack(), packed); + let (bits, palette, words) = c.to_wire(); + let wire = PalettedContainer::from_wire(strategy, bits, palette, words.to_vec()).unwrap(); + assert_eq!(wire, c); + } + } + #[test] + fn uniform_detection_ignores_stale_palette_entries() { + let mut c = PalettedContainer::new(BLOCKS, 0); + c.set(10, 3); + assert!(!c.is_uniform(0)); + c.set(10, 0); + assert!(c.palette_len() == 2 && c.is_uniform(0) && c.uniform() == Some(0)); + assert!(!c.is_uniform(3)); + for i in 0..4096 { + c.set(i, 3); + } + assert_eq!(c.uniform(), Some(3)); + } + #[test] + fn rejects_malformed_wire_data() { + assert!(PalettedContainer::from_wire(BLOCKS, 4, vec![1, 2], vec![0; 255]).is_err()); + assert!(PalettedContainer::from_wire(BLOCKS, 4, vec![1, 2], vec![0x3333; 256]).is_err()); + assert!(PalettedContainer::from_wire(BLOCKS, 4, vec![], vec![0; 256]).is_err()); + assert!(PalettedContainer::from_wire(BLOCKS, 0, vec![1, 2], vec![]).is_err()); + assert!(PalettedContainer::from_wire(BIOMES, 2, vec![1, 2, 3, 4, 5], vec![0; 4]).is_err()); + assert!(PalettedContainer::from_wire(BLOCKS, 0, vec![9], vec![]).is_ok()); + } + #[test] + fn division_magic_is_exact() { + for bits in 1..=32u8 { + let per = 64 / usize::from(bits); + for i in 0..4096usize { + assert_eq!(locate(bits, i).0, i / per, "bits {bits} index {i}"); + } + } + } +} diff --git a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaPalettedContainerOracle.java b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaPalettedContainerOracle.java new file mode 100644 index 000000000..a03b0eadc --- /dev/null +++ b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaPalettedContainerOracle.java @@ -0,0 +1,103 @@ +package dev.horizonlab.parity; + +import com.google.gson.*; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.nio.file.*; +import java.util.*; +import java.util.stream.LongStream; +import net.minecraft.core.IdMap; +import net.minecraft.network.FriendlyByteBuf; +import net.minecraft.world.level.chunk.PalettedContainer; +import net.minecraft.world.level.chunk.PalettedContainerRO; +import net.minecraft.world.level.chunk.Strategy; + +/** Runs the real PalettedContainer (single-value, linear, hash-map and global palettes, resize, + * network write/read, pack/unpack) over scripted edits and records hashes of everything + * observable: set results, the serialized container after every few edits, the saved form and + * the containers read back from both. */ +public final class VanillaPalettedContainerOracle { + private static final Gson JSON = new GsonBuilder().serializeNulls().create(); + /** The palettes compare entries by identity, as registry objects allow: intern the boxes. */ + private static final Map INTERNED = new HashMap<>(); + private static Integer intern(int value) { return INTERNED.computeIfAbsent(value, v -> new Integer(v)); } + private static final class Ids implements IdMap { + final int size; + Ids(int size) { this.size = size; } + public int getId(Integer value) { return value < size ? value : -1; } + public Integer byId(int id) { return id >= 0 && id < size ? intern(id) : null; } + public int size() { return size; } + public Iterator iterator() { return java.util.stream.IntStream.range(0, size).boxed().iterator(); } + } + private static long fnv(long h, int b) { return (h ^ (b & 0xFF)) * 0x100000001b3L; } + private static final long BASIS = 0xcbf29ce484222325L; + private static long hashBytes(byte[] bytes) { + long h = BASIS; + for (byte b : bytes) h = fnv(h, b); + return h; + } + private static long hashValues(PalettedContainer c, int bits) { + long h = BASIS; + int n = 1 << (3 * bits); + for (int i = 0; i < n; i++) { + int v = c.get(i & (n - 1) & ((1 << bits) - 1), i >> (2 * bits), (i >> bits) & ((1 << bits) - 1)); + for (int k = 0; k < 4; k++) h = fnv(h, v >> (8 * k)); + } + return h; + } + private static byte[] write(PalettedContainer c) { + var buffer = new FriendlyByteBuf(Unpooled.buffer()); + c.write(buffer); + byte[] bytes = new byte[buffer.readableBytes()]; + buffer.readBytes(bytes); + return bytes; + } + private static String wire(PalettedContainer c) { + byte[] bytes = write(c); + return bytes.length + ":" + hashBytes(bytes); + } + /** Input line: name kind initial every i v i v ...; output: one line of fields. */ + private static String scenario(String line) { + var t = line.trim().split(" "); + boolean blocks = t[1].equals("blocks"); + int bits = blocks ? 4 : 2; + Strategy strategy = blocks ? Strategy.createForBlockStates(new Ids(35723)) : Strategy.createForBiomes(new Ids(67)); + var c = new PalettedContainer<>(intern(Integer.parseInt(t[2])), strategy); + int every = Integer.parseInt(t[3]); + var checkpoints = new StringJoiner(","); + long oldsHash = BASIS; + int n = 0, mask = (1 << bits) - 1; + for (int k = 4; k + 1 < t.length; k += 2) { + int i = Integer.parseInt(t[k]); + int old = c.getAndSetUnchecked(i & mask, i >> (2 * bits), (i >> bits) & mask, intern(Integer.parseInt(t[k + 1]))); + for (int j = 0; j < 4; j++) oldsHash = fnv(oldsHash, old >> (8 * j)); + if (++n % every == 0) checkpoints.add(wire(c)); + } + var out = new StringJoiner(" "); + out.add(t[0]).add(Long.toString(oldsHash)).add("cp=" + checkpoints).add("bits=" + c.bitsPerEntry()); + out.add("final=" + wire(c)).add("values=" + hashValues(c, bits)); + var read = new PalettedContainer<>(intern(0), strategy); + read.read(new FriendlyByteBuf(Unpooled.wrappedBuffer(write(c)))); + out.add("read=" + hashValues(read, bits) + "/" + wire(read)); + var packed = c.pack(strategy); + long[] storage = packed.storage().map(LongStream::toArray).orElse(null); + long storageHash = BASIS; + if (storage != null) for (long word : storage) for (int j = 0; j < 8; j++) storageHash = fnv(storageHash, (int) (word >> (8 * j))); + var palette = new StringJoiner(","); + for (int v : packed.paletteEntries()) palette.add(Integer.toString(v)); + out.add("packed=" + packed.bitsPerEntry() + "/" + storageHash + "/" + palette); + var loaded = PalettedContainer.unpack(strategy, new PalettedContainerRO.PackedData<>(packed.paletteEntries(), + Optional.ofNullable(storage).map(LongStream::of), -1)).getOrThrow(); + out.add("loaded=" + hashValues(loaded, bits) + "/" + wire(loaded)); + return out.toString(); + } + public static void main(String[] args) throws Exception { + net.minecraft.SharedConstants.tryDetectVersion(); + try (var input = Files.newBufferedReader(Path.of(args[0])); var output = Files.newBufferedWriter(Path.of(args[1]))) { + for (String line; (line = input.readLine()) != null; ) { + output.write(scenario(line)); + output.newLine(); + } + } + } +} diff --git a/tools/test_paletted_container_parity.py b/tools/test_paletted_container_parity.py new file mode 100644 index 000000000..f3b2e28bc --- /dev/null +++ b/tools/test_paletted_container_parity.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Compare Horizon's resident PalettedContainer with the client's own. + +Scripted edits run through vanilla's PalettedContainer (block-state and biome +strategies: single-value, linear, hash-map and global palettes, resizing, network +write and read, pack and unpack) and through horizon_core::paletted. Both sides +print the set results, the serialized container after every few edits, the +container read back from the wire, the saved form (palette, bit width, storage) +and the container loaded from it, as hashes. +""" +import argparse +import json +import os +from pathlib import Path +import platform +import random +import subprocess +try: + from . import lab, test_rust_parity as base +except ImportError: + import lab + import test_rust_parity as base + +BUILD = lab.ROOT / 'target/paletted-container-parity' +LIMITS = {'blocks': (4096, 35723), 'biomes': (64, 67)} + + +def corpus(): + rng = random.Random(0x50414C45545445) + counts = [1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 31, 32, 33, 64, 100, 128, 129, 200, 256, 257, 300, 700, 2000] + for kind, (entries, ids) in LIMITS.items(): + for count in counts: + if kind == 'biomes' and count > 70: + continue + for trial in range(6): + pool = rng.sample(range(ids), min(count, ids)) + initial = rng.choice(pool) if trial % 2 else rng.randrange(ids) + ops = [] + for _ in range(rng.choice([entries // 4, entries, entries * 3])): + ops.append((rng.randrange(entries), rng.choice(pool))) + if trial == 5: + # Every position written once, in order, then half of them reset. + ops = [(i, pool[i % len(pool)]) for i in range(entries)] + [(i, initial) for i in range(0, entries, 2)] + yield f'{kind}_{count}_{trial}', kind, initial, rng.choice([1, 7, 37, 211]), ops + # A few edits at a time: the palette grows one value at a time from a single value. + for trial in range(20): + ops = [(rng.randrange(entries), rng.randrange(ids)) for _ in range(rng.randrange(1, 40))] + yield f'{kind}_sparse_{trial}', kind, rng.randrange(ids), 1, ops + # Overwriting with the value already there, and filling a section with one value. + yield f'{kind}_fill', kind, 3, 500, [(i, 9) for i in range(entries)] + [(i, 3) for i in range(entries)] + yield f'{kind}_same', kind, 3, 1, [(i, 3) for i in range(0, entries, 5)] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.parse_args() + lock, jars, libraries = base.verify_artifacts() + BUILD.mkdir(parents=True, exist_ok=True) + oracle = lab.ROOT / 'mods/horizon/src/test/java/dev/horizonlab/parity/VanillaPalettedContainerOracle.java' + sources = [Path(__file__), oracle, lab.ROOT / 'tools/lab.py', lab.ROOT / 'tools/test_rust_parity.py', lab.ROOT / 'Cargo.toml', lab.ROOT / 'Cargo.lock', + lab.ROOT / 'crates/horizon-core/Cargo.toml', lab.ROOT / 'crates/horizon-core/examples/paletted_fixture.rs', + *sorted((lab.ROOT / 'crates/horizon-core/src').rglob('*.rs'))] + tested = base.source_hashes(sources) + scenarios = list(corpus()) + inputs = BUILD / 'inputs.txt' + inputs.write_text(''.join(f'{name} {kind} {initial} {every} ' + ' '.join(f'{i} {v}' for i, v in ops) + '\n' + for name, kind, initial, every, ops in scenarios)) + + def run(command, name): + with (BUILD / (name + '.log')).open('w') as log: + base.run(list(map(str, command)), stdout=log, stderr=subprocess.STDOUT) + classes = BUILD / 'classes' + classes.mkdir(exist_ok=True) + dependencies = os.pathsep.join(map(str, [jars['client'], *libraries])) + run(['javac', '--release', lock['minecraft']['java_major'], '-proc:none', '-cp', dependencies, '-d', classes, oracle], 'compile-java') + run(['cargo', 'build', '--locked', '--profile', 'fast', '-j', os.environ.get('CARGO_BUILD_JOBS', '3'), '-p', 'horizon-core', '--example', 'paletted_fixture'], 'compile-rust') + metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--no-deps', '--format-version', '1'], cwd=lab.ROOT)) + binary = Path(metadata['target_directory']) / 'fast/examples/paletted_fixture' + run([binary, inputs, BUILD / 'rust.txt'], 'rust') + run(['java', '-Xmx1G', '-cp', str(classes) + os.pathsep + dependencies, 'dev.horizonlab.parity.VanillaPalettedContainerOracle', inputs, BUILD / 'client.txt'], 'client') + client = (BUILD / 'client.txt').read_text().splitlines() + rust = (BUILD / 'rust.txt').read_text().splitlines() + assert len(client) == len(rust) == len(scenarios), (len(client), len(rust), len(scenarios)) + bad = [(a, b) for a, b in zip(client, rust) if a != b] + base.ensure_sources_unchanged(sources, tested) + for a, b in bad[:5]: + print('client', a[:600]) + print('rust ', b[:600]) + if bad: + raise SystemExit(f'{len(bad)} of {len(scenarios)} scenarios differ') + edits = sum(len(s[4]) for s in scenarios) + report = {'schema': 1, 'minecraft': lock['minecraft']['version'], 'platform': platform.platform(), 'scenarios': len(scenarios), 'edits': edits, + 'sources_sha256': tested} + lab.write_json(BUILD / 'report.json', report) + print(f'PalettedContainer parity: {len(scenarios)} scenarios, {edits:,} edits, all hashes identical') + + +if __name__ == '__main__': + main()