WIP: paletted containers and mesher changes
Work in progress before merging main. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
@@ -0,0 +1,196 @@
|
||||
//! Resident chunk storage benchmark: generates a block of vanilla terrain, then
|
||||
//! reports the resident block/biome memory, the cost of lookups, meshing and
|
||||
//! lighting on it.
|
||||
//!
|
||||
//! chunk_bench <client resource dir> [--seed S] [--size N]
|
||||
#![forbid(unsafe_code)]
|
||||
use horizon_client::{
|
||||
assets::Assets,
|
||||
biome::Biomes,
|
||||
blocks::BlockFacts,
|
||||
chunk::{Chunk, MIN_Y, SECTIONS},
|
||||
light::{LightTables, light_chunk},
|
||||
mesher::{MeshTables, Mesher},
|
||||
models,
|
||||
streaming::{ChunkGenerator, Proto},
|
||||
vanillagen::VanillaTerrain,
|
||||
};
|
||||
use horizon_core::biome::BiomeZoom;
|
||||
use std::{path::PathBuf, sync::Arc, time::Instant};
|
||||
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
let root = PathBuf::from(args.first().ok_or("usage: chunk_bench RESOURCES")?);
|
||||
let flag = |name: &str, default: i64| -> i64 {
|
||||
args.iter()
|
||||
.position(|a| a == name)
|
||||
.and_then(|i| args.get(i + 1))
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(default)
|
||||
};
|
||||
let (seed, size) = (flag("--seed", 12345), flag("--size", 8) as i32);
|
||||
let assets = Assets::load(&root)?;
|
||||
let facts = BlockFacts::load(&root, assets.registry)?;
|
||||
let biomes = Biomes::load(&root)?;
|
||||
let tables = MeshTables::new(&assets, &facts, models::load_visibility(&root, assets.registry)?)?;
|
||||
let light = LightTables::load(&root, assets.registry, &facts)?;
|
||||
let terrain = VanillaTerrain::new(&root.join("worldgen"), assets.registry, &biomes, seed)?;
|
||||
let mut worker = terrain.worker();
|
||||
|
||||
// Protos for size+4 columns, decorated for the inner size+2, finished as size+2.
|
||||
let started = Instant::now();
|
||||
let outer = size + 4;
|
||||
let mut protos: Vec<Option<Proto>> = (0..outer * outer)
|
||||
.map(|i| Some(terrain.proto(&mut worker, i % outer, i / outer)))
|
||||
.collect();
|
||||
eprintln!("proto: {:.2}s", started.elapsed().as_secs_f64());
|
||||
let started = Instant::now();
|
||||
let inner = size + 2;
|
||||
for z in 1..=inner {
|
||||
for x in 1..=inner {
|
||||
let mut region: [Proto; 9] = std::array::from_fn(|k| {
|
||||
let (dx, dz) = (k as i32 % 3 - 1, k as i32 / 3 - 1);
|
||||
protos[((z + dz) * outer + x + dx) as usize].take().expect("proto")
|
||||
});
|
||||
terrain.decorate(&mut worker, &mut region);
|
||||
for (k, p) in region.into_iter().enumerate() {
|
||||
let (dx, dz) = (k as i32 % 3 - 1, k as i32 / 3 - 1);
|
||||
protos[((z + dz) * outer + x + dx) as usize] = Some(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
eprintln!("decorate: {:.2}s", started.elapsed().as_secs_f64());
|
||||
let started = Instant::now();
|
||||
let mut chunks: Vec<Chunk> = Vec::new();
|
||||
for z in 1..=inner {
|
||||
for x in 1..=inner {
|
||||
let proto = protos[(z * outer + x) as usize].take().expect("proto");
|
||||
let mut chunk = terrain.finish(proto);
|
||||
chunk.x = x;
|
||||
chunk.z = z;
|
||||
chunks.push(chunk);
|
||||
}
|
||||
}
|
||||
let finish = started.elapsed().as_secs_f64();
|
||||
println!("finish (proto -> resident chunk): {:.1} ms per chunk", finish * 1e3 / chunks.len() as f64);
|
||||
|
||||
let chunks: Vec<Arc<Chunk>> = chunks.into_iter().map(Arc::new).collect();
|
||||
let at = |x: i32, z: i32| chunks[((z - 1) * inner + x - 1) as usize].clone();
|
||||
|
||||
// Light all inner-inner columns.
|
||||
let started = Instant::now();
|
||||
let mut lit = Vec::new();
|
||||
for z in 2..inner {
|
||||
for x in 2..inner {
|
||||
let region: [[Option<Arc<Chunk>>; 3]; 3] = std::array::from_fn(|dz| {
|
||||
std::array::from_fn(|dx| Some(at(x + dx as i32 - 1, z + dz as i32 - 1)))
|
||||
});
|
||||
let borrowed = horizon_client::mesher::borrow(®ion);
|
||||
lit.push(((x, z), light_chunk(&borrowed, &light, true)));
|
||||
}
|
||||
}
|
||||
println!("light: {:.2} ms per column", started.elapsed().as_secs_f64() * 1e3 / lit.len() as f64);
|
||||
let mut chunks: Vec<Chunk> = chunks.iter().map(|c| (**c).clone()).collect();
|
||||
for ((x, z), sections) in lit {
|
||||
chunks[((z - 1) * inner + x - 1) as usize].sections = sections;
|
||||
}
|
||||
let chunks: Vec<Arc<Chunk>> = chunks.into_iter().map(Arc::new).collect();
|
||||
let at = |x: i32, z: i32| chunks[((z - 1) * inner + x - 1) as usize].clone();
|
||||
|
||||
// Memory.
|
||||
let (mut bytes, mut sections, mut uniform) = (0usize, 0usize, 0usize);
|
||||
for chunk in &chunks {
|
||||
bytes += chunk.heap_bytes();
|
||||
for s in &chunk.sections {
|
||||
sections += 1;
|
||||
uniform += usize::from(s.is_uniform(s.get(0)));
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"resident: {} chunks, {} sections ({} uniform), {:.1} KiB per chunk ({:.2} MiB total)",
|
||||
chunks.len(),
|
||||
sections,
|
||||
uniform,
|
||||
bytes as f64 / 1024.0 / chunks.len() as f64,
|
||||
bytes as f64 / 1048576.0
|
||||
);
|
||||
|
||||
// Meshing.
|
||||
let mut mesher = Mesher {
|
||||
nether_shading: false,
|
||||
assets: &assets,
|
||||
tables: &tables,
|
||||
biomes: &biomes,
|
||||
visibility: tables.visibility.clone(),
|
||||
zoom: BiomeZoom::new(7),
|
||||
blend_radius: 2,
|
||||
smooth_lighting: true,
|
||||
cutout_leaves: true,
|
||||
};
|
||||
let mut best = f64::MAX;
|
||||
let mut quads = 0usize;
|
||||
for _ in 0..3 {
|
||||
let started = Instant::now();
|
||||
let mut jobs = 0;
|
||||
quads = 0;
|
||||
for z in 2..inner {
|
||||
for x in 2..inner {
|
||||
let region: [[Option<Arc<Chunk>>; 3]; 3] = std::array::from_fn(|dz| {
|
||||
std::array::from_fn(|dx| Some(at(x + dx as i32 - 1, z + dz as i32 - 1)))
|
||||
});
|
||||
let borrowed = horizon_client::mesher::borrow(®ion);
|
||||
for sy in 0..SECTIONS as i32 {
|
||||
let mesh = mesher.mesh(&borrowed, sy);
|
||||
quads += (mesh.solid.len() + mesh.translucent.len()) / 4;
|
||||
jobs += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
best = best.min(started.elapsed().as_secs_f64() / f64::from(jobs));
|
||||
}
|
||||
println!("mesh: {:.1} us per section ({} quads total)", best * 1e6, quads);
|
||||
|
||||
// Random lookups, as collision and entity queries do.
|
||||
let mut state = 0x9E37_79B9_7F4A_7C15u64;
|
||||
let started = Instant::now();
|
||||
let mut sum = 0u64;
|
||||
let lookups = 20_000_000u64;
|
||||
for _ in 0..lookups {
|
||||
state ^= state << 13;
|
||||
state ^= state >> 7;
|
||||
state ^= state << 17;
|
||||
let cx = 2 + (state % u64::from((inner - 2) as u32)) as i32;
|
||||
let cz = 2 + ((state >> 8) % u64::from((inner - 2) as u32)) as i32;
|
||||
let chunk = &chunks[((cz - 1) * inner + cx - 1) as usize];
|
||||
let (x, y, z) = (
|
||||
(state >> 16) as i32 & 15,
|
||||
MIN_Y + ((state >> 24) % 384) as i32,
|
||||
(state >> 40) as i32 & 15,
|
||||
);
|
||||
sum += u64::from(chunk.get(x, y, z));
|
||||
}
|
||||
println!(
|
||||
"random get: {:.2} ns per lookup (checksum {sum})",
|
||||
started.elapsed().as_secs_f64() * 1e9 / lookups as f64
|
||||
);
|
||||
// Coherent scan, as a column or area query does.
|
||||
let started = Instant::now();
|
||||
let mut sum = 0u64;
|
||||
for _ in 0..4 {
|
||||
for chunk in &chunks {
|
||||
for y in MIN_Y..MIN_Y + 384 {
|
||||
for z in 0..16 {
|
||||
for x in 0..16 {
|
||||
sum += u64::from(chunk.get(x, y, z));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"scan get: {:.2} ns per lookup (checksum {sum})",
|
||||
started.elapsed().as_secs_f64() * 1e9 / (4.0 * chunks.len() as f64 * 98304.0)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -104,14 +104,21 @@ fn scene(
|
||||
blend_radius = int(&field["radius"])?;
|
||||
let default = biomes.id(field["default"].as_str().ok_or("missing default biome")?);
|
||||
for chunk in &mut chunks {
|
||||
Arc::make_mut(&mut chunk.biomes).fill(default);
|
||||
for qy in 0..96 {
|
||||
for cell in 0..16 {
|
||||
chunk.set_biome_cell(cell & 3, qy, cell >> 2, default);
|
||||
}
|
||||
}
|
||||
}
|
||||
for cell in field["cells"].as_array().ok_or("missing biome cells")? {
|
||||
let (qx, qy, qz) = (int(&cell[0])?, int(&cell[1])?, int(&cell[2])?);
|
||||
let i = chunk_index(qx * 4, qz * 4)?;
|
||||
let slot = ((qy - MIN_Y / 4) * 16 + (qz & 3) * 4 + (qx & 3)) as usize;
|
||||
Arc::make_mut(&mut chunks[i].biomes)[slot] =
|
||||
biomes.id(cell[3].as_str().ok_or("biome name")?);
|
||||
chunks[i].set_biome_cell(
|
||||
qx,
|
||||
qy - MIN_Y / 4,
|
||||
qz,
|
||||
biomes.id(cell[3].as_str().ok_or("biome name")?),
|
||||
);
|
||||
}
|
||||
}
|
||||
let region: [[Option<&Chunk>; 3]; 3] =
|
||||
|
||||
@@ -2,17 +2,18 @@
|
||||
//! threads can mesh and light immutable snapshots while the game thread edits
|
||||
//! (copy-on-write through `Arc::make_mut`).
|
||||
use horizon_world::registry::BlockStateId;
|
||||
use horizon_core::paletted::{PalettedContainer, Strategy};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub const MIN_Y: i32 = -64;
|
||||
pub const HEIGHT: i32 = 384;
|
||||
pub const SECTIONS: usize = (HEIGHT / 16) as usize;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum Blocks {
|
||||
Uniform(u16),
|
||||
Full(Box<[u16; 4096]>),
|
||||
}
|
||||
/// Block states are `PalettedContainer`s over the pinned registry (35,723
|
||||
/// states: a 16-bit global palette); biomes are 4x4x4 containers over the
|
||||
/// biome table (67 entries: a 7-bit global palette).
|
||||
pub const BLOCK_STRATEGY: Strategy = Strategy::blocks(16);
|
||||
pub const BIOME_STRATEGY: Strategy = Strategy::biomes(7);
|
||||
/// Packed light per block: sky light in the high nibble, block light in the low one.
|
||||
#[derive(Clone)]
|
||||
pub enum Light {
|
||||
@@ -25,13 +26,16 @@ pub fn index(x: i32, y: i32, z: i32) -> usize {
|
||||
}
|
||||
#[derive(Clone)]
|
||||
pub struct Section {
|
||||
pub blocks: Blocks,
|
||||
pub blocks: PalettedContainer,
|
||||
/// Biome table index per 4x4x4 cell, `(y << 2 | z) << 2 | x`.
|
||||
pub biomes: PalettedContainer,
|
||||
pub light: Light,
|
||||
}
|
||||
impl Default for Section {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
blocks: Blocks::Uniform(0),
|
||||
blocks: PalettedContainer::new(BLOCK_STRATEGY, 0),
|
||||
biomes: PalettedContainer::new(BIOME_STRATEGY, 0),
|
||||
light: Light::Uniform(0xF0),
|
||||
}
|
||||
}
|
||||
@@ -39,23 +43,19 @@ impl Default for Section {
|
||||
impl Section {
|
||||
#[inline]
|
||||
pub fn get(&self, i: usize) -> u16 {
|
||||
match &self.blocks {
|
||||
Blocks::Uniform(v) => *v,
|
||||
Blocks::Full(a) => a[i],
|
||||
}
|
||||
self.blocks.get(i) as u16
|
||||
}
|
||||
pub fn set(&mut self, i: usize, v: u16) -> u16 {
|
||||
match &mut self.blocks {
|
||||
Blocks::Uniform(u) if *u == v => v,
|
||||
Blocks::Uniform(u) => {
|
||||
let old = *u;
|
||||
let mut a = Box::new([old; 4096]);
|
||||
a[i] = v;
|
||||
self.blocks = Blocks::Full(a);
|
||||
old
|
||||
}
|
||||
Blocks::Full(a) => std::mem::replace(&mut a[i], v),
|
||||
}
|
||||
self.blocks.set(i, u32::from(v)) as u16
|
||||
}
|
||||
/// All 4,096 block states in index order.
|
||||
pub fn unpack(&self, out: &mut [u16; 4096]) {
|
||||
self.blocks.unpack_u16(out);
|
||||
}
|
||||
/// The block states of a whole section at once.
|
||||
pub fn from_blocks(values: &[u16; 4096]) -> PalettedContainer {
|
||||
let wide: Vec<u32> = values.iter().map(|&v| u32::from(v)).collect();
|
||||
PalettedContainer::from_values(BLOCK_STRATEGY, &wide)
|
||||
}
|
||||
#[inline]
|
||||
pub fn light(&self, i: usize) -> u8 {
|
||||
@@ -65,17 +65,26 @@ impl Section {
|
||||
}
|
||||
}
|
||||
pub fn is_uniform(&self, v: u16) -> bool {
|
||||
match &self.blocks {
|
||||
Blocks::Uniform(u) => *u == v,
|
||||
Blocks::Full(a) => a.iter().all(|&b| b == v),
|
||||
}
|
||||
self.blocks.is_uniform(u32::from(v))
|
||||
}
|
||||
/// Collapse storage that became uniform.
|
||||
/// Heap bytes held by the block, biome and light storage (benchmark accounting).
|
||||
pub fn heap_bytes(&self) -> usize {
|
||||
self.blocks.heap_bytes()
|
||||
+ self.biomes.heap_bytes()
|
||||
+ match &self.light {
|
||||
Light::Uniform(_) => 0,
|
||||
Light::Full(_) => 4096,
|
||||
}
|
||||
}
|
||||
/// Collapse storage that became uniform, and drop palette entries no
|
||||
/// block uses any more.
|
||||
pub fn compact(&mut self) {
|
||||
if let Blocks::Full(a) = &self.blocks
|
||||
&& a.iter().all(|&b| b == a[0])
|
||||
{
|
||||
self.blocks = Blocks::Uniform(a[0]);
|
||||
for container in [&mut self.blocks, &mut self.biomes] {
|
||||
if container.bits() != 0 {
|
||||
if let Some(v) = container.uniform() {
|
||||
*container = PalettedContainer::new(container.strategy(), v);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Light::Full(a) = &self.light
|
||||
&& a.iter().all(|&b| b == a[0])
|
||||
@@ -89,8 +98,6 @@ pub struct Chunk {
|
||||
pub x: i32,
|
||||
pub z: i32,
|
||||
pub sections: Vec<Arc<Section>>,
|
||||
/// Biome index per 4x4x4 cell (x,z in 0..4, y in 0..96) into `Biomes`.
|
||||
pub biomes: Arc<Vec<u16>>,
|
||||
/// Highest non-air, light-blocking-or-fluid Y per column (MOTION_BLOCKING-like), for rain/sky.
|
||||
pub heights: Arc<[i16; 256]>,
|
||||
pub lit: bool,
|
||||
@@ -119,7 +126,6 @@ impl Chunk {
|
||||
sections: (0..SECTIONS)
|
||||
.map(|_| Arc::new(Section::default()))
|
||||
.collect(),
|
||||
biomes: Arc::new(vec![0; 16 * (HEIGHT as usize / 4)]),
|
||||
heights: Arc::new([MIN_Y as i16; 256]),
|
||||
lit: false,
|
||||
light_loaded: false,
|
||||
@@ -161,9 +167,24 @@ impl Chunk {
|
||||
}
|
||||
self.sections[s as usize].light(index(x, y, z))
|
||||
}
|
||||
/// Heap bytes held by the sections (benchmark accounting).
|
||||
pub fn heap_bytes(&self) -> usize {
|
||||
self.sections.iter().map(|s| s.heap_bytes()).sum::<usize>()
|
||||
}
|
||||
/// Biome table index at a block position (`ChunkAccess.getNoiseBiome` with
|
||||
/// the quart y clamped to the column).
|
||||
#[inline]
|
||||
pub fn biome(&self, x: i32, y: i32, z: i32) -> u16 {
|
||||
let qy = ((y - MIN_Y) >> 2).clamp(0, HEIGHT / 4 - 1);
|
||||
self.biomes[(qy * 16 + ((z & 15) >> 2) * 4 + ((x & 15) >> 2)) as usize]
|
||||
self.sections[(qy >> 2) as usize]
|
||||
.biomes
|
||||
.get(((qy & 3) << 4 | ((z & 15) >> 2) << 2 | (x & 15) >> 2) as usize) as u16
|
||||
}
|
||||
/// Sets the biome of one quart cell (`qy` counts from the column's bottom).
|
||||
pub fn set_biome_cell(&mut self, qx: i32, qy: i32, qz: i32, biome: u16) {
|
||||
let section = Arc::make_mut(&mut self.sections[(qy >> 2) as usize]);
|
||||
section
|
||||
.biomes
|
||||
.set(((qy & 3) << 4 | (qz & 3) << 2 | (qx & 3)) as usize, u32::from(biome));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -933,9 +933,14 @@ impl Game<'_> {
|
||||
};
|
||||
let mut found = Vec::new();
|
||||
for (s, section) in chunk.sections.iter().enumerate() {
|
||||
let horizon_client::chunk::Blocks::Full(blocks) = §ion.blocks else {
|
||||
if !section
|
||||
.blocks
|
||||
.maybe_has(|v| v == u32::from(portal) || v == u32::from(gateway))
|
||||
{
|
||||
continue;
|
||||
};
|
||||
}
|
||||
let mut blocks = Box::new([0u16; 4096]);
|
||||
section.unpack(&mut blocks);
|
||||
for (i, &b) in blocks.iter().enumerate() {
|
||||
if b == portal || b == gateway {
|
||||
let y = horizon_client::chunk::MIN_Y + s as i32 * 16 + (i >> 8) as i32;
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
use super::{Dim, Game, Mining, Screen};
|
||||
use horizon_client::{
|
||||
Result,
|
||||
chunk::{Blocks, Chunk, Light, MIN_Y, SECTIONS, Section},
|
||||
chunk::{BIOME_STRATEGY, BLOCK_STRATEGY, Chunk, Light, MIN_Y, SECTIONS, Section},
|
||||
containers::{BlockEntity, Furnace, FurnaceKind},
|
||||
entities::{Experience, ItemEntity},
|
||||
hud::{HudBatch, rgba},
|
||||
@@ -22,7 +22,7 @@ use horizon_client::{
|
||||
streaming::{ChunkGenerator, Proto, Streaming},
|
||||
world::World,
|
||||
};
|
||||
use horizon_core::{geometry::Vec3, random::RandomSource};
|
||||
use horizon_core::{geometry::Vec3, paletted::PalettedContainer, random::RandomSource};
|
||||
use horizon_protocol::{
|
||||
entity_data::{ITEM_ENTITY_ITEM, Value},
|
||||
item::{Hashed, Stack},
|
||||
@@ -434,54 +434,55 @@ impl Game<'_> {
|
||||
.sections(r.dimension.sections(), 16, r.biome_bits)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut chunk = Chunk::empty(p.x, p.z);
|
||||
let mut biomes = vec![0u16; chunk.biomes.len()];
|
||||
let offset = (r.dimension.min_y - MIN_Y).div_euclid(16);
|
||||
// Biome table index per quart cell of the dimension's own sections.
|
||||
let mut biome_cells: Vec<[u32; 64]> = Vec::with_capacity(sections.len());
|
||||
for (i, s) in sections.iter().enumerate() {
|
||||
let target = offset + i as i32;
|
||||
let mut cells = [0u32; 64];
|
||||
for (j, cell) in cells.iter_mut().enumerate() {
|
||||
*cell = u32::from(
|
||||
r.biome_map
|
||||
.get(s.biomes.get(j) as usize)
|
||||
.copied()
|
||||
.unwrap_or(0),
|
||||
);
|
||||
}
|
||||
biome_cells.push(cells);
|
||||
if !(0..SECTIONS as i32).contains(&target) {
|
||||
continue;
|
||||
}
|
||||
let blocks = if s.states.bits == 0 {
|
||||
Blocks::Uniform(s.states.get(0) as u16)
|
||||
} else {
|
||||
let mut a = Box::new([0u16; 4096]);
|
||||
for (j, v) in a.iter_mut().enumerate() {
|
||||
*v = s.states.get(j) as u16;
|
||||
}
|
||||
Blocks::Full(a)
|
||||
};
|
||||
let strategy = BLOCK_STRATEGY;
|
||||
let palette = s.states.palette.iter().map(|&p| p as u32).collect();
|
||||
let words = s.states.data.iter().map(|&w| w as u64).collect();
|
||||
let blocks = PalettedContainer::from_wire(strategy, s.states.bits, palette, words)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut section = Section {
|
||||
blocks,
|
||||
biomes: PalettedContainer::from_values(BIOME_STRATEGY, &cells),
|
||||
light: Light::Uniform(0xF0),
|
||||
};
|
||||
section.compact();
|
||||
chunk.sections[target as usize] = Arc::new(section);
|
||||
for j in 0..64 {
|
||||
let biome = r
|
||||
.biome_map
|
||||
.get(s.biomes.get(j) as usize)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
if let Some(cell) = biomes.get_mut(target as usize * 64 + j) {
|
||||
*cell = biome;
|
||||
}
|
||||
// Sections outside the dimension's range take its nearest biomes
|
||||
// (the column's top or bottom cell layer repeated).
|
||||
if !biome_cells.is_empty() {
|
||||
let last = biome_cells.len() - 1;
|
||||
for si in 0..SECTIONS as i32 {
|
||||
let i = si - offset;
|
||||
if (0..sections.len() as i32).contains(&i) {
|
||||
continue;
|
||||
}
|
||||
let (edge, layer) = if i < 0 { (0, 0) } else { (last, 3) };
|
||||
let mut cells = [0u32; 64];
|
||||
for (j, cell) in cells.iter_mut().enumerate() {
|
||||
*cell = biome_cells[edge][layer * 16 + j % 16];
|
||||
}
|
||||
Arc::make_mut(&mut chunk.sections[si as usize]).biomes =
|
||||
PalettedContainer::from_values(BIOME_STRATEGY, &cells);
|
||||
}
|
||||
}
|
||||
// Columns outside the dimension's range take its nearest biome.
|
||||
let (lo, hi) = (
|
||||
offset.max(0) as usize * 64,
|
||||
((offset + sections.len() as i32).min(SECTIONS as i32)) as usize * 64,
|
||||
);
|
||||
if lo < hi {
|
||||
for j in 0..biomes.len() {
|
||||
if j < lo {
|
||||
biomes[j] = biomes[lo + j % 16];
|
||||
} else if j >= hi {
|
||||
biomes[j] = biomes[hi - 16 + j % 16];
|
||||
}
|
||||
}
|
||||
}
|
||||
chunk.biomes = Arc::new(biomes);
|
||||
Ok(chunk)
|
||||
}
|
||||
/// Keep the renderable block entities (chests, furnaces) in step with blocks.
|
||||
|
||||
@@ -216,6 +216,7 @@ pub fn light_chunk(
|
||||
let mut block = vec![0u8; W * W * h];
|
||||
let mut queue = VecDeque::new();
|
||||
let mut sky_queue = VecDeque::new();
|
||||
let mut ids = Box::new([0u16; 4096]);
|
||||
for (dz, row) in region.iter().enumerate() {
|
||||
for (dx, chunk) in row.iter().enumerate() {
|
||||
let Some(chunk) = chunk else { continue };
|
||||
@@ -223,8 +224,8 @@ pub fn light_chunk(
|
||||
if section.is_uniform(0) {
|
||||
continue;
|
||||
}
|
||||
for i in 0..4096 {
|
||||
let id = section.get(i);
|
||||
section.unpack(&mut ids);
|
||||
for (i, &id) in ids.iter().enumerate() {
|
||||
let (lx, lz, ly) = (i & 15, (i >> 4) & 15, i >> 8);
|
||||
let p = at(dx * 16 + lx, si * 16 + ly, dz * 16 + lz);
|
||||
states[p] = id;
|
||||
|
||||
@@ -383,6 +383,13 @@ impl Padded {
|
||||
let mut blocks = vec![0u16; P * P * P];
|
||||
let mut light = vec![0u8; P * P * P];
|
||||
let base_y = MIN_Y + sy * 16;
|
||||
// The section itself is unpacked in one pass; the border cells of the
|
||||
// neighbouring sections are read one by one.
|
||||
let mut own = Box::new([0u16; 4096]);
|
||||
let center = region[1][1].map(|c| &c.sections[sy as usize]);
|
||||
if let Some(section) = center {
|
||||
section.unpack(&mut own);
|
||||
}
|
||||
for y in -B..16 + B {
|
||||
let wy = base_y + y;
|
||||
for z in -B..16 + B {
|
||||
@@ -390,7 +397,14 @@ impl Padded {
|
||||
let chunk = region[((z + 16) >> 4) as usize][((x + 16) >> 4) as usize];
|
||||
let i = pi(x, y, z);
|
||||
if let Some(c) = chunk {
|
||||
blocks[i] = c.get(x & 15, wy, z & 15);
|
||||
blocks[i] = if (0..16).contains(&x)
|
||||
&& (0..16).contains(&y)
|
||||
&& (0..16).contains(&z)
|
||||
{
|
||||
own[(y << 8 | z << 4 | x) as usize]
|
||||
} else {
|
||||
c.get(x & 15, wy, z & 15)
|
||||
};
|
||||
light[i] = c.light(x & 15, wy, z & 15);
|
||||
} else {
|
||||
light[i] = 0xF0;
|
||||
|
||||
@@ -16,10 +16,11 @@
|
||||
use crate::{
|
||||
Result,
|
||||
biome::Biomes,
|
||||
chunk::{Blocks, Chunk, Light, MIN_Y, SECTIONS},
|
||||
chunk::{BIOME_STRATEGY, BLOCK_STRATEGY, Chunk, Light, MIN_Y, SECTIONS},
|
||||
streaming::ChunkGenerator,
|
||||
};
|
||||
use horizon_core::{
|
||||
paletted::PalettedContainer,
|
||||
position::{BlockPos, ChunkPos},
|
||||
ticks::{SavedTick, TickPriority, TickTypeId},
|
||||
};
|
||||
@@ -42,29 +43,6 @@ use std::{
|
||||
|
||||
pub const VERSION_NAME: &str = "26.3";
|
||||
|
||||
fn pack(indices: &[u16], bits: u32) -> Vec<i64> {
|
||||
let per_long = (64 / bits) as usize;
|
||||
let mut longs = vec![0i64; indices.len().div_ceil(per_long)];
|
||||
for (i, &v) in indices.iter().enumerate() {
|
||||
longs[i / per_long] |=
|
||||
(i64::from(v) & ((1i64 << bits) - 1)) << ((i % per_long) as u32 * bits);
|
||||
}
|
||||
longs
|
||||
}
|
||||
fn unpack(longs: &[i64], bits: u32, count: usize) -> Vec<u16> {
|
||||
let per_long = (64 / bits) as usize;
|
||||
let mask = (1u64 << bits) - 1;
|
||||
(0..count)
|
||||
.map(|i| {
|
||||
longs.get(i / per_long).map_or(0, |&l| {
|
||||
((l as u64 >> ((i % per_long) as u32 * bits)) & mask) as u16
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
fn bits_for(len: usize, min: u32) -> u32 {
|
||||
(usize::BITS - (len.max(1) - 1).leading_zeros()).max(min)
|
||||
}
|
||||
fn string(s: &str) -> Tag {
|
||||
Tag::String(s.into())
|
||||
}
|
||||
@@ -584,33 +562,22 @@ pub fn encode_with(
|
||||
});
|
||||
for (si, section) in chunk.sections.iter().enumerate() {
|
||||
let y = (MIN_Y >> 4) + si as i32;
|
||||
let ids: Vec<u16> = match §ion.blocks {
|
||||
Blocks::Uniform(v) => vec![*v],
|
||||
Blocks::Full(a) => a.to_vec(),
|
||||
};
|
||||
let mut palette: Vec<u16> = Vec::new();
|
||||
let mut lookup: HashMap<u16, u16> = HashMap::new();
|
||||
let indices: Vec<u16> = ids
|
||||
.iter()
|
||||
.map(|id| {
|
||||
*lookup.entry(*id).or_insert_with(|| {
|
||||
palette.push(*id);
|
||||
(palette.len() - 1) as u16
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
// PalettedContainer.pack: distinct states in index order, indices at
|
||||
// the saved width.
|
||||
let packed = section.blocks.pack();
|
||||
let mut states = vec![(
|
||||
"palette",
|
||||
Tag::List(
|
||||
palette
|
||||
packed
|
||||
.palette
|
||||
.iter()
|
||||
.map(|id| {
|
||||
let state = registry.state(BlockStateId(u32::from(*id)));
|
||||
.map(|&id| {
|
||||
let state = registry.state(BlockStateId(id));
|
||||
match state {
|
||||
// BlockState.CODEC: the block name for default states.
|
||||
Some(s)
|
||||
if tables.is_some_and(|t| {
|
||||
t.default_state.get(*id as usize) == Some(&true)
|
||||
t.default_state.get(id as usize) == Some(&true)
|
||||
}) =>
|
||||
{
|
||||
string(&s.name)
|
||||
@@ -622,37 +589,27 @@ pub fn encode_with(
|
||||
.collect(),
|
||||
),
|
||||
)];
|
||||
if palette.len() > 1 {
|
||||
if let Some(data) = &packed.data {
|
||||
states.push((
|
||||
"data",
|
||||
Tag::LongArray(pack(&indices, bits_for(palette.len(), 4))),
|
||||
Tag::LongArray(data.iter().map(|&w| w as i64).collect()),
|
||||
));
|
||||
}
|
||||
let quart: Vec<u16> = (0..64).map(|i| chunk.biomes[si * 64 + i]).collect();
|
||||
let mut biome_palette: Vec<u16> = Vec::new();
|
||||
let biome_indices: Vec<u16> = quart
|
||||
.iter()
|
||||
.map(|b| match biome_palette.iter().position(|p| p == b) {
|
||||
Some(i) => i as u16,
|
||||
None => {
|
||||
biome_palette.push(*b);
|
||||
(biome_palette.len() - 1) as u16
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let biome_packed = section.biomes.pack();
|
||||
let mut biome_tag = vec![(
|
||||
"palette",
|
||||
Tag::List(
|
||||
biome_palette
|
||||
biome_packed
|
||||
.palette
|
||||
.iter()
|
||||
.map(|b| string(&biomes.get(*b).name))
|
||||
.map(|&b| string(&biomes.get(b as u16).name))
|
||||
.collect(),
|
||||
),
|
||||
)];
|
||||
if biome_palette.len() > 1 {
|
||||
if let Some(data) = &biome_packed.data {
|
||||
biome_tag.push((
|
||||
"data",
|
||||
Tag::LongArray(pack(&biome_indices, bits_for(biome_palette.len(), 0))),
|
||||
Tag::LongArray(data.iter().map(|&w| w as i64).collect()),
|
||||
));
|
||||
}
|
||||
let mut entries = vec![
|
||||
@@ -811,7 +768,6 @@ pub fn decode_with(
|
||||
}
|
||||
let air = registry.default_state("minecraft:air")?.0 as u16;
|
||||
let mut cache: HashMap<String, u16> = HashMap::new();
|
||||
let mut biome_data = (*chunk.biomes).clone();
|
||||
let Some(Tag::List(sections)) = tag.get("sections") else {
|
||||
return Some(chunk);
|
||||
};
|
||||
@@ -854,15 +810,14 @@ pub fn decode_with(
|
||||
})
|
||||
.collect();
|
||||
let target = Arc::make_mut(&mut chunk.sections[si as usize]);
|
||||
if ids.len() <= 1 {
|
||||
target.blocks = Blocks::Uniform(ids.first().copied().unwrap_or(air));
|
||||
} else if let Some(Tag::LongArray(data)) = states.get("data") {
|
||||
let indices = unpack(data, bits_for(ids.len(), 4), 4096);
|
||||
let mut full = Box::new([air; 4096]);
|
||||
for (i, idx) in indices.iter().enumerate() {
|
||||
full[i] = ids.get(*idx as usize).copied().unwrap_or(air);
|
||||
}
|
||||
target.blocks = Blocks::Full(full);
|
||||
let data = match states.get("data") {
|
||||
Some(Tag::LongArray(data)) => Some(data.iter().map(|&w| w as u64).collect::<Vec<_>>()),
|
||||
_ => None,
|
||||
};
|
||||
let palette: Vec<u32> = ids.iter().map(|&id| u32::from(id)).collect();
|
||||
if let Ok(blocks) = PalettedContainer::from_packed(BLOCK_STRATEGY, palette, data.as_deref())
|
||||
{
|
||||
target.blocks = blocks;
|
||||
}
|
||||
}
|
||||
if let Some(b) = section.get("biomes")
|
||||
@@ -875,18 +830,21 @@ pub fn decode_with(
|
||||
_ => 0,
|
||||
})
|
||||
.collect();
|
||||
let indices = match b.get("data") {
|
||||
Some(Tag::LongArray(data)) if ids.len() > 1 => {
|
||||
unpack(data, bits_for(ids.len(), 0), 64)
|
||||
}
|
||||
_ => vec![0; 64],
|
||||
let data = match b.get("data") {
|
||||
Some(Tag::LongArray(data)) => Some(data.iter().map(|&w| w as u64).collect::<Vec<_>>()),
|
||||
_ => None,
|
||||
};
|
||||
for (i, idx) in indices.iter().enumerate() {
|
||||
biome_data[si as usize * 64 + i] = ids.get(*idx as usize).copied().unwrap_or(0);
|
||||
let palette: Vec<u32> = ids.iter().map(|&id| u32::from(id)).collect();
|
||||
let target = Arc::make_mut(&mut chunk.sections[si as usize]);
|
||||
if let Ok(container) =
|
||||
PalettedContainer::from_packed(BIOME_STRATEGY, palette.clone(), data.as_deref())
|
||||
{
|
||||
target.biomes = container;
|
||||
} else if let Some(&first) = palette.first() {
|
||||
target.biomes = PalettedContainer::new(BIOME_STRATEGY, first);
|
||||
}
|
||||
}
|
||||
}
|
||||
chunk.biomes = Arc::new(biome_data);
|
||||
let mut extras = ChunkExtras {
|
||||
last_update: number(tag.get("LastUpdate")).unwrap_or(0),
|
||||
..ChunkExtras::default()
|
||||
|
||||
@@ -76,7 +76,13 @@ impl ChunkGenerator for TestTerrain {
|
||||
impl TestTerrain {
|
||||
fn generate(&self, _: &mut (), cx: i32, cz: i32) -> Chunk {
|
||||
let mut chunk = Chunk::empty(cx, cz);
|
||||
chunk.biomes = std::sync::Arc::new(vec![self.plains; chunk.biomes.len()]);
|
||||
for s in &mut chunk.sections {
|
||||
std::sync::Arc::make_mut(s).biomes =
|
||||
horizon_core::paletted::PalettedContainer::new(
|
||||
crate::chunk::BIOME_STRATEGY,
|
||||
u32::from(self.plains),
|
||||
);
|
||||
}
|
||||
for z in 0..16 {
|
||||
for x in 0..16 {
|
||||
let (wx, wz) = (cx * 16 + x, cz * 16 + z);
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
//! aquifers, surface rules, carvers and placed features) to chunk streaming.
|
||||
use crate::{
|
||||
biome::Biomes,
|
||||
chunk::{Chunk, HEIGHT, MIN_Y},
|
||||
chunk::{BIOME_STRATEGY, Chunk, HEIGHT, MIN_Y, SECTIONS, Section},
|
||||
streaming::{ChunkGenerator, Proto},
|
||||
};
|
||||
use horizon_core::position::ChunkPos;
|
||||
use horizon_core::{paletted::PalettedContainer, position::ChunkPos};
|
||||
use horizon_world::{
|
||||
registry::BlockRegistry,
|
||||
worldgen::generator::{GeneratorData, GeneratorWorker, OverworldGenerator, ProtoChunk},
|
||||
@@ -177,15 +177,26 @@ impl ChunkGenerator for VanillaTerrain {
|
||||
chunk.terrain.min_y().max(MIN_Y),
|
||||
(chunk.terrain.min_y() + chunk.terrain.height()).min(MIN_Y + HEIGHT),
|
||||
);
|
||||
for y in lo..hi {
|
||||
for z in 0..16 {
|
||||
for x in 0..16 {
|
||||
let id = chunk.block(bx + x, y, bz + z).0 as u16;
|
||||
if id != 0 {
|
||||
out.set(x, y, z, id);
|
||||
let mut values = Box::new([0u16; 4096]);
|
||||
for si in 0..SECTIONS as i32 {
|
||||
let base = MIN_Y + si * 16;
|
||||
if base + 16 <= lo || base >= hi {
|
||||
continue;
|
||||
}
|
||||
values.fill(0);
|
||||
let mut any = false;
|
||||
for y in base.max(lo)..(base + 16).min(hi) {
|
||||
for z in 0..16 {
|
||||
for x in 0..16 {
|
||||
let id = chunk.block(bx + x, y, bz + z).0 as u16;
|
||||
values[((y & 15) << 8 | z << 4 | x) as usize] = id;
|
||||
any |= id != 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
if any {
|
||||
Arc::make_mut(&mut out.sections[si as usize]).blocks = Section::from_blocks(&values);
|
||||
}
|
||||
}
|
||||
let base_section = chunk.terrain.min_y() >> 4;
|
||||
for (i, marks) in chunk.terrain.postprocessing().iter().enumerate() {
|
||||
@@ -199,19 +210,17 @@ impl ChunkGenerator for VanillaTerrain {
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut biomes = vec![0u16; 16 * (HEIGHT as usize / 4)];
|
||||
for qy in 0..HEIGHT / 4 {
|
||||
for qz in 0..4 {
|
||||
for qx in 0..4 {
|
||||
let y = (MIN_Y + qy * 4).clamp(lo, hi - 1);
|
||||
let id = chunk.biome_id_at(bx + qx * 4, y, bz + qz * 4);
|
||||
let index = id.0 as usize;
|
||||
biomes[(qy * 16 + qz * 4 + qx) as usize] =
|
||||
self.biome_map.get(index).copied().unwrap_or(0);
|
||||
}
|
||||
for si in 0..SECTIONS as i32 {
|
||||
let mut cells = [0u32; 64];
|
||||
for (j, cell) in cells.iter_mut().enumerate() {
|
||||
let (qx, qz, qy) = (j as i32 & 3, (j as i32 >> 2) & 3, si * 4 + (j as i32 >> 4));
|
||||
let y = (MIN_Y + qy * 4).clamp(lo, hi - 1);
|
||||
let id = chunk.biome_id_at(bx + qx * 4, y, bz + qz * 4);
|
||||
*cell = u32::from(self.biome_map.get(id.0 as usize).copied().unwrap_or(0));
|
||||
}
|
||||
Arc::make_mut(&mut out.sections[si as usize]).biomes =
|
||||
PalettedContainer::from_values(BIOME_STRATEGY, &cells);
|
||||
}
|
||||
out.biomes = Arc::new(biomes);
|
||||
out.block_entities = crate::structure_blocks::block_entities(&chunk, self.loot.as_deref());
|
||||
out.structure_refs = chunk
|
||||
.structures()
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
//! Text transport for the PalettedContainer comparison: replays scripted edits
|
||||
//! and prints the same hashes as `VanillaPalettedContainerOracle`.
|
||||
use horizon_core::paletted::{PalettedContainer, Strategy};
|
||||
use std::{
|
||||
fs,
|
||||
io::{self, Write},
|
||||
};
|
||||
const BASIS: u64 = 0xcbf2_9ce4_8422_2325;
|
||||
fn fnv(h: u64, b: u8) -> u64 {
|
||||
(h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3)
|
||||
}
|
||||
fn hash_bytes(bytes: &[u8]) -> u64 {
|
||||
bytes.iter().fold(BASIS, |h, &b| fnv(h, b))
|
||||
}
|
||||
fn varint(out: &mut Vec<u8>, mut v: u32) {
|
||||
while v >= 0x80 {
|
||||
out.push((v & 0x7F) as u8 | 0x80);
|
||||
v >>= 7;
|
||||
}
|
||||
out.push(v as u8);
|
||||
}
|
||||
/// `PalettedContainer.write`.
|
||||
fn write(c: &PalettedContainer) -> Vec<u8> {
|
||||
let (bits, palette, words) = c.to_wire();
|
||||
let mut out = vec![bits];
|
||||
if bits == 0 {
|
||||
varint(&mut out, palette[0]);
|
||||
} else if !palette.is_empty() {
|
||||
varint(&mut out, palette.len() as u32);
|
||||
for p in palette {
|
||||
varint(&mut out, p);
|
||||
}
|
||||
}
|
||||
for w in words {
|
||||
out.extend(w.to_be_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
/// `PalettedContainer.read` for the leading byte, palette and longs.
|
||||
fn read(strategy: Strategy, bytes: &[u8]) -> PalettedContainer {
|
||||
let mut at = 1;
|
||||
let mut varint = || {
|
||||
let (mut v, mut shift) = (0u32, 0);
|
||||
loop {
|
||||
let b = bytes[at];
|
||||
at += 1;
|
||||
v |= u32::from(b & 0x7F) << shift;
|
||||
if b & 0x80 == 0 {
|
||||
return v;
|
||||
}
|
||||
shift += 7;
|
||||
}
|
||||
};
|
||||
let bits = bytes[0];
|
||||
let palette = if bits == 0 {
|
||||
vec![varint()]
|
||||
} else if strategy.wire_has_palette(bits) {
|
||||
let n = varint();
|
||||
(0..n).map(|_| varint()).collect()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let words = bytes[at..]
|
||||
.chunks_exact(8)
|
||||
.map(|c| u64::from_be_bytes(c.try_into().unwrap()))
|
||||
.collect();
|
||||
PalettedContainer::from_wire(strategy, bits, palette, words).unwrap()
|
||||
}
|
||||
fn wire(c: &PalettedContainer) -> String {
|
||||
let bytes = write(c);
|
||||
format!("{}:{}", bytes.len(), hash_bytes(&bytes) as i64)
|
||||
}
|
||||
fn values(c: &PalettedContainer) -> i64 {
|
||||
let mut h = BASIS;
|
||||
for i in 0..c.strategy().entries() {
|
||||
for b in c.get(i).to_le_bytes() {
|
||||
h = fnv(h, b);
|
||||
}
|
||||
}
|
||||
h as i64
|
||||
}
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let mut out = io::BufWriter::new(fs::File::create(&args[2])?);
|
||||
for line in fs::read_to_string(&args[1])?.lines() {
|
||||
let t: Vec<&str> = line.split_whitespace().collect();
|
||||
let strategy = if t[1] == "blocks" {
|
||||
Strategy::blocks(16)
|
||||
} else {
|
||||
Strategy::biomes(7)
|
||||
};
|
||||
let mut c = PalettedContainer::new(strategy, t[2].parse()?);
|
||||
let every: usize = t[3].parse()?;
|
||||
let (mut olds, mut n) = (BASIS, 0usize);
|
||||
let mut checkpoints = Vec::new();
|
||||
for pair in t[4..].chunks(2) {
|
||||
let old = c.set(pair[0].parse()?, pair[1].parse()?);
|
||||
for b in old.to_le_bytes() {
|
||||
olds = fnv(olds, b);
|
||||
}
|
||||
n += 1;
|
||||
if n % every == 0 {
|
||||
checkpoints.push(wire(&c));
|
||||
}
|
||||
}
|
||||
let reread = read(strategy, &write(&c));
|
||||
let packed = c.pack();
|
||||
let storage = packed.data.as_ref().map_or(BASIS, |data| {
|
||||
data.iter()
|
||||
.flat_map(|w| w.to_le_bytes())
|
||||
.fold(BASIS, |h, b| fnv(h, b))
|
||||
});
|
||||
let palette: Vec<String> = packed.palette.iter().map(u32::to_string).collect();
|
||||
let loaded =
|
||||
PalettedContainer::from_packed(strategy, packed.palette.clone(), packed.data.as_deref())?;
|
||||
writeln!(
|
||||
out,
|
||||
"{} {} cp={} bits={} final={} values={} read={}/{} packed={}/{}/{} loaded={}/{}",
|
||||
t[0],
|
||||
olds as i64,
|
||||
checkpoints.join(","),
|
||||
c.bits(),
|
||||
wire(&c),
|
||||
values(&c),
|
||||
values(&reread),
|
||||
wire(&reread),
|
||||
packed.bits,
|
||||
storage as i64,
|
||||
palette.join(","),
|
||||
values(&loaded),
|
||||
wire(&loaded),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -9,6 +9,7 @@ pub mod carving;
|
||||
pub mod density;
|
||||
pub mod geometry;
|
||||
pub mod noise;
|
||||
pub mod paletted;
|
||||
pub mod position;
|
||||
pub mod random;
|
||||
pub mod storage;
|
||||
|
||||
@@ -0,0 +1,739 @@
|
||||
//! Resident `PalettedContainer` of Java Edition 26.3: single-value, linear and
|
||||
//! hash-map palettes with vanilla's bit-width thresholds, and the global
|
||||
//! palette beyond them, over `SimpleBitStorage` words (values never span
|
||||
//! longs). One container type serves blocks (16^3 entries) and biomes (4^3).
|
||||
//!
|
||||
//! Growth follows `PalettedContainer.onResize`: a full palette asks for one
|
||||
//! more bit, the new configuration is built by re-adding the distinct values
|
||||
//! in index order, and the value that did not fit is added last. Palettes
|
||||
//! never shrink while resident; `pack` writes the canonical saved form.
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Kind {
|
||||
/// `Strategy.createForBlockStates`: 4096 entries, linear palettes up to 4
|
||||
/// bits and hash-map palettes from 5 to 8.
|
||||
Blocks,
|
||||
/// `Strategy.createForBiomes`: 64 entries, linear palettes up to 3 bits.
|
||||
Biomes,
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Strategy {
|
||||
pub kind: Kind,
|
||||
/// `ceillog2` of the registry size: the width of the global palette.
|
||||
pub global_bits: u8,
|
||||
}
|
||||
/// `getConfigurationForBitCount`.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Config {
|
||||
Single,
|
||||
/// Palette of up to `1 << bits` entries, stored in `bits` bits.
|
||||
List(u8),
|
||||
Global,
|
||||
}
|
||||
impl Strategy {
|
||||
pub const fn blocks(global_bits: u8) -> Self {
|
||||
Self {
|
||||
kind: Kind::Blocks,
|
||||
global_bits,
|
||||
}
|
||||
}
|
||||
pub const fn biomes(global_bits: u8) -> Self {
|
||||
Self {
|
||||
kind: Kind::Biomes,
|
||||
global_bits,
|
||||
}
|
||||
}
|
||||
pub const fn entries(self) -> usize {
|
||||
match self.kind {
|
||||
Kind::Blocks => 4096,
|
||||
Kind::Biomes => 64,
|
||||
}
|
||||
}
|
||||
fn config(self, bits: u8) -> Config {
|
||||
match (self.kind, bits) {
|
||||
(_, 0) => Config::Single,
|
||||
(Kind::Blocks, 1..=4) => Config::List(4),
|
||||
(Kind::Blocks, 5..=8) => Config::List(bits),
|
||||
(Kind::Biomes, 1..=3) => Config::List(bits),
|
||||
_ => Config::Global,
|
||||
}
|
||||
}
|
||||
/// Whether a container written with leading byte `bits` lists its palette
|
||||
/// (all but the global one).
|
||||
pub fn wire_has_palette(self, bits: u8) -> bool {
|
||||
self.config(bits) != Config::Global
|
||||
}
|
||||
/// Bits of the storage a configuration keeps in memory.
|
||||
fn storage_bits(self, config: Config) -> u8 {
|
||||
match config {
|
||||
Config::Single => 0,
|
||||
Config::List(bits) => bits,
|
||||
Config::Global => self.global_bits,
|
||||
}
|
||||
}
|
||||
}
|
||||
/// `Mth.ceillog2`.
|
||||
pub fn ceil_log2(n: usize) -> u8 {
|
||||
if n <= 1 {
|
||||
0
|
||||
} else {
|
||||
(usize::BITS - (n - 1).leading_zeros()) as u8
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum PaletteError {
|
||||
/// A palette bigger than its configuration allows, or empty.
|
||||
BadPalette { len: usize },
|
||||
/// Storage with the wrong number of longs.
|
||||
BadStorage { expected: usize, actual: usize },
|
||||
/// A stored index past the palette.
|
||||
BadIndex { index: u32, palette: usize },
|
||||
/// The declared bit count does not match the palette.
|
||||
BadBits { declared: u8, expected: u8 },
|
||||
}
|
||||
impl std::fmt::Display for PaletteError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{self:?}")
|
||||
}
|
||||
}
|
||||
impl std::error::Error for PaletteError {}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
enum Palette {
|
||||
Single(u32),
|
||||
/// Linear and hash-map palettes alike: ids in insertion order.
|
||||
List(Vec<u32>),
|
||||
Global,
|
||||
}
|
||||
/// `n / per_word` for `n < 4096` by multiplication: `MAGIC[bits] = ceil(2^20 /
|
||||
/// (64 / bits))` is exact while `n * (64 / bits) < 2^20`.
|
||||
const MAGIC: [u32; 33] = {
|
||||
let mut table = [0; 33];
|
||||
let mut bits = 1;
|
||||
while bits <= 32 {
|
||||
let per = 64 / bits as u32;
|
||||
table[bits] = (1u32 << 20).div_ceil(per);
|
||||
bits += 1;
|
||||
}
|
||||
table
|
||||
};
|
||||
/// Words of `bits`-wide values: `entries.div_ceil(64 / bits)`.
|
||||
pub fn storage_len(bits: u8, entries: usize) -> usize {
|
||||
if bits == 0 {
|
||||
0
|
||||
} else {
|
||||
entries.div_ceil(64 / usize::from(bits))
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
fn locate(bits: u8, i: usize) -> (usize, u32) {
|
||||
if bits.is_power_of_two() {
|
||||
// 64 / bits is a power of two: shifts instead of the multiplication.
|
||||
let log = bits.trailing_zeros();
|
||||
let word = i >> (6 - log);
|
||||
(word, ((i & ((64 >> log) - 1)) as u32) << log)
|
||||
} else {
|
||||
let per = 64 / u32::from(bits);
|
||||
let word = (i as u32 * MAGIC[usize::from(bits)]) >> 20;
|
||||
(word as usize, (i as u32 - word * per) * u32::from(bits))
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
fn mask(bits: u8) -> u64 {
|
||||
(1u64 << bits) - 1
|
||||
}
|
||||
fn read(words: &[u64], bits: u8, i: usize) -> u32 {
|
||||
let (word, shift) = locate(bits, i);
|
||||
((words[word] >> shift) & mask(bits)) as u32
|
||||
}
|
||||
fn write(words: &mut [u64], bits: u8, i: usize, value: u32) {
|
||||
let (word, shift) = locate(bits, i);
|
||||
words[word] = (words[word] & !(mask(bits) << shift)) | (u64::from(value) << shift);
|
||||
}
|
||||
/// Words holding `ids` in `bits`-wide slots.
|
||||
fn pack_words(bits: u8, entries: usize, ids: impl Iterator<Item = u32>) -> Vec<u64> {
|
||||
let mut words = vec![0u64; storage_len(bits, entries)];
|
||||
for (i, id) in ids.enumerate() {
|
||||
write(&mut words, bits, i, id);
|
||||
}
|
||||
words
|
||||
}
|
||||
/// The saved form: distinct values in index order, and the packed indices
|
||||
/// when there is more than one value.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Packed {
|
||||
pub palette: Vec<u32>,
|
||||
/// Bits per index in `data` (0 when `data` is absent).
|
||||
pub bits: u8,
|
||||
pub data: Option<Vec<u64>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PalettedContainer {
|
||||
strategy: Strategy,
|
||||
bits: u8,
|
||||
palette: Palette,
|
||||
words: Vec<u64>,
|
||||
}
|
||||
impl PalettedContainer {
|
||||
/// `new PalettedContainer(initialValue, strategy)`.
|
||||
pub fn new(strategy: Strategy, value: u32) -> Self {
|
||||
Self {
|
||||
strategy,
|
||||
bits: 0,
|
||||
palette: Palette::Single(value),
|
||||
words: Vec::new(),
|
||||
}
|
||||
}
|
||||
pub fn strategy(&self) -> Strategy {
|
||||
self.strategy
|
||||
}
|
||||
/// Bits per stored index (`BitStorage.getBits`).
|
||||
pub fn bits(&self) -> u8 {
|
||||
self.bits
|
||||
}
|
||||
/// Entries in the palette (the global palette reports its bit capacity).
|
||||
pub fn palette_len(&self) -> usize {
|
||||
match &self.palette {
|
||||
Palette::Single(_) => 1,
|
||||
Palette::List(list) => list.len(),
|
||||
Palette::Global => 1 << self.bits.min(31),
|
||||
}
|
||||
}
|
||||
pub fn is_global(&self) -> bool {
|
||||
matches!(self.palette, Palette::Global)
|
||||
}
|
||||
#[inline]
|
||||
fn raw(&self, i: usize) -> usize {
|
||||
let (word, shift) = locate(self.bits, i);
|
||||
((self.words[word] >> shift) & mask(self.bits)) as usize
|
||||
}
|
||||
#[inline]
|
||||
pub fn get(&self, i: usize) -> u32 {
|
||||
match &self.palette {
|
||||
Palette::Single(value) => *value,
|
||||
Palette::List(list) => list[self.raw(i)],
|
||||
Palette::Global => self.raw(i) as u32,
|
||||
}
|
||||
}
|
||||
/// The value when every entry is equal, without the palette having to be
|
||||
/// minimal.
|
||||
pub fn uniform(&self) -> Option<u32> {
|
||||
match &self.palette {
|
||||
Palette::Single(value) => Some(*value),
|
||||
Palette::List(list) if list.len() == 1 => Some(list[0]),
|
||||
Palette::Global => self.all_equal(self.get(0)).then(|| self.get(0)),
|
||||
Palette::List(list) => {
|
||||
let first = list[self.raw(0)];
|
||||
self.all_equal(first).then_some(first)
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Whether all entries equal `value`.
|
||||
pub fn is_uniform(&self, value: u32) -> bool {
|
||||
match &self.palette {
|
||||
Palette::Single(v) => *v == value,
|
||||
Palette::List(list) => match list.iter().position(|&v| v == value) {
|
||||
None => false,
|
||||
Some(_) if list.len() == 1 => true,
|
||||
Some(_) => self.all_equal(value),
|
||||
},
|
||||
Palette::Global => self.all_equal(value),
|
||||
}
|
||||
}
|
||||
fn all_equal(&self, value: u32) -> bool {
|
||||
let id = match &self.palette {
|
||||
Palette::Single(v) => return *v == value,
|
||||
Palette::List(list) => match list.iter().position(|&v| v == value) {
|
||||
Some(id) => id as u64,
|
||||
None => return false,
|
||||
},
|
||||
Palette::Global => u64::from(value),
|
||||
};
|
||||
let bits = usize::from(self.bits);
|
||||
let per = 64 / bits;
|
||||
let pattern = (0..per).fold(0u64, |word, slot| word | id << (slot * bits));
|
||||
let entries = self.strategy.entries();
|
||||
let full = entries / per;
|
||||
if self.words[..full].iter().any(|&w| w != pattern) {
|
||||
return false;
|
||||
}
|
||||
let tail = entries - full * per;
|
||||
tail == 0 || {
|
||||
let used = (1u64 << (tail * bits)) - 1;
|
||||
self.words[full] & used == pattern & used
|
||||
}
|
||||
}
|
||||
/// `getAndSet`: stores `value` and returns the previous one.
|
||||
pub fn set(&mut self, i: usize, value: u32) -> u32 {
|
||||
let id = self.id_for(value);
|
||||
match &self.palette {
|
||||
Palette::Single(old) => *old,
|
||||
Palette::List(list) => {
|
||||
let old = read(&self.words, self.bits, i) as usize;
|
||||
write(&mut self.words, self.bits, i, id as u32);
|
||||
list[old]
|
||||
}
|
||||
Palette::Global => {
|
||||
let old = read(&self.words, self.bits, i);
|
||||
write(&mut self.words, self.bits, i, id as u32);
|
||||
old
|
||||
}
|
||||
}
|
||||
}
|
||||
/// `Palette.idFor`, growing the container when the palette is full.
|
||||
fn id_for(&mut self, value: u32) -> usize {
|
||||
match &mut self.palette {
|
||||
Palette::Single(v) if *v == value => 0,
|
||||
Palette::Single(_) => self.resize(1, value),
|
||||
Palette::List(list) => {
|
||||
if let Some(id) = list.iter().position(|&v| v == value) {
|
||||
id
|
||||
} else if list.len() < 1 << self.bits {
|
||||
list.push(value);
|
||||
list.len() - 1
|
||||
} else {
|
||||
self.resize(self.bits + 1, value)
|
||||
}
|
||||
}
|
||||
Palette::Global => value as usize,
|
||||
}
|
||||
}
|
||||
/// `PalettedContainer.onResize`: rebuild with the configuration for
|
||||
/// `bits`, then add `last` and return its id.
|
||||
fn resize(&mut self, bits: u8, last: u32) -> usize {
|
||||
let strategy = self.strategy;
|
||||
let config = strategy.config(bits);
|
||||
let storage_bits = strategy.storage_bits(config);
|
||||
let entries = strategy.entries();
|
||||
let mut palette = match config {
|
||||
Config::Single => Palette::Single(last),
|
||||
Config::List(_) => Palette::List(Vec::new()),
|
||||
Config::Global => Palette::Global,
|
||||
};
|
||||
let mut words = vec![0u64; storage_len(storage_bits, entries)];
|
||||
let id_in = |palette: &mut Palette, value: u32| -> u32 {
|
||||
match palette {
|
||||
Palette::Single(_) => 0,
|
||||
Palette::List(list) => match list.iter().position(|&v| v == value) {
|
||||
Some(id) => id as u32,
|
||||
None => {
|
||||
list.push(value);
|
||||
list.len() as u32 - 1
|
||||
}
|
||||
},
|
||||
Palette::Global => value,
|
||||
}
|
||||
};
|
||||
if let Palette::Single(only) = self.palette {
|
||||
let id = id_in(&mut palette, only);
|
||||
if storage_bits > 0 && id != 0 {
|
||||
for i in 0..entries {
|
||||
write(&mut words, storage_bits, i, id);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let (mut last_old, mut new_id) = (u32::MAX, 0);
|
||||
for i in 0..entries {
|
||||
let old = read(&self.words, self.bits, i);
|
||||
if old != last_old {
|
||||
last_old = old;
|
||||
let value = match &self.palette {
|
||||
Palette::List(list) => list[old as usize],
|
||||
_ => old,
|
||||
};
|
||||
new_id = id_in(&mut palette, value);
|
||||
}
|
||||
if storage_bits > 0 {
|
||||
write(&mut words, storage_bits, i, new_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
let id = id_in(&mut palette, last);
|
||||
self.palette = palette;
|
||||
self.bits = storage_bits;
|
||||
self.words = words;
|
||||
id as usize
|
||||
}
|
||||
/// All values in index order.
|
||||
pub fn unpack(&self, out: &mut [u32]) {
|
||||
assert_eq!(out.len(), self.strategy.entries());
|
||||
match &self.palette {
|
||||
Palette::Single(value) => out.fill(*value),
|
||||
palette => {
|
||||
let bits = usize::from(self.bits);
|
||||
let per = 64 / bits;
|
||||
let m = mask(self.bits);
|
||||
for (chunk, &word) in out.chunks_mut(per).zip(&self.words) {
|
||||
for (slot, cell) in chunk.iter_mut().enumerate() {
|
||||
let id = ((word >> (slot * bits)) & m) as u32;
|
||||
*cell = match palette {
|
||||
Palette::List(list) => list[id as usize],
|
||||
_ => id,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Block ids as 16-bit values (callers whose registry fits in 16 bits).
|
||||
pub fn unpack_u16(&self, out: &mut [u16; 4096]) {
|
||||
debug_assert_eq!(self.strategy.entries(), 4096);
|
||||
match &self.palette {
|
||||
Palette::Single(value) => out.fill(*value as u16),
|
||||
palette => {
|
||||
let bits = usize::from(self.bits);
|
||||
let per = 64 / bits;
|
||||
let m = mask(self.bits);
|
||||
for (chunk, &word) in out.chunks_mut(per).zip(&self.words) {
|
||||
for (slot, cell) in chunk.iter_mut().enumerate() {
|
||||
let id = ((word >> (slot * bits)) & m) as usize;
|
||||
*cell = match palette {
|
||||
Palette::List(list) => list[id] as u16,
|
||||
_ => id as u16,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/// A container holding `values` in the smallest configuration for their
|
||||
/// distinct count, with palette ids in first-occurrence order, like a
|
||||
/// container unpacked from disk.
|
||||
pub fn from_values(strategy: Strategy, values: &[u32]) -> Self {
|
||||
assert_eq!(values.len(), strategy.entries());
|
||||
let mut palette: Vec<u32> = Vec::new();
|
||||
let mut small = true;
|
||||
let mut lookup: std::collections::HashMap<u32, u32> = std::collections::HashMap::new();
|
||||
let mut ids = Vec::with_capacity(values.len());
|
||||
for &value in values {
|
||||
let id = if small {
|
||||
match palette.iter().position(|&v| v == value) {
|
||||
Some(id) => id as u32,
|
||||
None => {
|
||||
palette.push(value);
|
||||
if palette.len() > 16 {
|
||||
small = false;
|
||||
lookup.extend(palette.iter().enumerate().map(|(i, &v)| (v, i as u32)));
|
||||
}
|
||||
palette.len() as u32 - 1
|
||||
}
|
||||
}
|
||||
} else {
|
||||
*lookup.entry(value).or_insert_with(|| {
|
||||
palette.push(value);
|
||||
palette.len() as u32 - 1
|
||||
})
|
||||
};
|
||||
ids.push(id);
|
||||
}
|
||||
Self::build(strategy, palette, &ids)
|
||||
}
|
||||
/// A container over a distinct-value `palette` and an id per entry.
|
||||
fn build(strategy: Strategy, palette: Vec<u32>, ids: &[u32]) -> Self {
|
||||
let config = strategy.config(ceil_log2(palette.len()));
|
||||
match config {
|
||||
Config::Single => Self::new(strategy, palette[0]),
|
||||
Config::List(bits) => Self {
|
||||
strategy,
|
||||
bits,
|
||||
words: pack_words(bits, ids.len(), ids.iter().copied()),
|
||||
palette: Palette::List(palette),
|
||||
},
|
||||
Config::Global => {
|
||||
let bits = strategy.global_bits;
|
||||
Self {
|
||||
strategy,
|
||||
bits,
|
||||
words: pack_words(bits, ids.len(), ids.iter().map(|&id| palette[id as usize])),
|
||||
palette: Palette::Global,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/// `PalettedContainer.read`: the container as sent on the wire. `bits` is
|
||||
/// the leading byte, `palette` the VarInt entries (a single one for
|
||||
/// `bits == 0`, none for the global palette) and `words` the longs.
|
||||
pub fn from_wire(
|
||||
strategy: Strategy,
|
||||
bits: u8,
|
||||
palette: Vec<u32>,
|
||||
words: Vec<u64>,
|
||||
) -> Result<Self, PaletteError> {
|
||||
let config = strategy.config(bits);
|
||||
let storage_bits = strategy.storage_bits(config);
|
||||
let expected = storage_len(storage_bits, strategy.entries());
|
||||
if words.len() != expected {
|
||||
return Err(PaletteError::BadStorage {
|
||||
expected,
|
||||
actual: words.len(),
|
||||
});
|
||||
}
|
||||
match config {
|
||||
Config::Single => {
|
||||
if palette.len() != 1 {
|
||||
return Err(PaletteError::BadPalette { len: palette.len() });
|
||||
}
|
||||
Ok(Self::new(strategy, palette[0]))
|
||||
}
|
||||
Config::List(bits) => {
|
||||
if palette.is_empty() || palette.len() > 1 << bits {
|
||||
return Err(PaletteError::BadPalette { len: palette.len() });
|
||||
}
|
||||
let container = Self {
|
||||
strategy,
|
||||
bits,
|
||||
palette: Palette::List(palette),
|
||||
words,
|
||||
};
|
||||
container.check_indices()?;
|
||||
Ok(container)
|
||||
}
|
||||
Config::Global => Ok(Self {
|
||||
strategy,
|
||||
bits: storage_bits,
|
||||
palette: Palette::Global,
|
||||
words,
|
||||
}),
|
||||
}
|
||||
}
|
||||
fn check_indices(&self) -> Result<(), PaletteError> {
|
||||
let Palette::List(list) = &self.palette else {
|
||||
return Ok(());
|
||||
};
|
||||
if list.len() == 1 << self.bits {
|
||||
return Ok(());
|
||||
}
|
||||
for i in 0..self.strategy.entries() {
|
||||
let index = read(&self.words, self.bits, i);
|
||||
if index as usize >= list.len() {
|
||||
return Err(PaletteError::BadIndex {
|
||||
index,
|
||||
palette: list.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// `PalettedContainer.write`: the leading bits byte, the palette entries
|
||||
/// (none for the global palette) and the raw longs, exactly as resident.
|
||||
pub fn to_wire(&self) -> (u8, Vec<u32>, &[u64]) {
|
||||
let palette = match &self.palette {
|
||||
Palette::Single(value) => vec![*value],
|
||||
Palette::List(list) => list.clone(),
|
||||
Palette::Global => Vec::new(),
|
||||
};
|
||||
(self.bits, palette, &self.words)
|
||||
}
|
||||
/// `PalettedContainer.pack`: distinct values in index order, indices
|
||||
/// re-encoded for the saved bit width.
|
||||
pub fn pack(&self) -> Packed {
|
||||
let entries = self.strategy.entries();
|
||||
let mut palette: Vec<u32> = Vec::new();
|
||||
let mut ids = Vec::with_capacity(entries);
|
||||
let mut values = vec![0u32; entries];
|
||||
self.unpack(&mut values);
|
||||
let mut lookup: std::collections::HashMap<u32, u32> = std::collections::HashMap::new();
|
||||
for &value in &values {
|
||||
let id = *lookup.entry(value).or_insert_with(|| {
|
||||
palette.push(value);
|
||||
palette.len() as u32 - 1
|
||||
});
|
||||
ids.push(id);
|
||||
}
|
||||
let bits = match self.strategy.config(ceil_log2(palette.len())) {
|
||||
Config::Single => 0,
|
||||
Config::List(bits) => bits,
|
||||
Config::Global => ceil_log2(palette.len()),
|
||||
};
|
||||
let data = (bits != 0).then(|| pack_words(bits, entries, ids.iter().copied()));
|
||||
Packed {
|
||||
palette,
|
||||
bits,
|
||||
data,
|
||||
}
|
||||
}
|
||||
/// `PalettedContainer.unpack`: a container from the saved form.
|
||||
pub fn from_packed(
|
||||
strategy: Strategy,
|
||||
palette: Vec<u32>,
|
||||
data: Option<&[u64]>,
|
||||
) -> Result<Self, PaletteError> {
|
||||
if palette.is_empty() {
|
||||
return Err(PaletteError::BadPalette { len: 0 });
|
||||
}
|
||||
let config = strategy.config(ceil_log2(palette.len()));
|
||||
let entries = strategy.entries();
|
||||
let disc_bits = match config {
|
||||
Config::Single => return Ok(Self::new(strategy, palette[0])),
|
||||
Config::List(bits) => bits,
|
||||
Config::Global => ceil_log2(palette.len()),
|
||||
};
|
||||
let data = data.ok_or(PaletteError::BadStorage {
|
||||
expected: storage_len(disc_bits, entries),
|
||||
actual: 0,
|
||||
})?;
|
||||
let expected = storage_len(disc_bits, entries);
|
||||
if data.len() != expected {
|
||||
return Err(PaletteError::BadStorage {
|
||||
expected,
|
||||
actual: data.len(),
|
||||
});
|
||||
}
|
||||
let mut ids = Vec::with_capacity(entries);
|
||||
for i in 0..entries {
|
||||
let index = read(data, disc_bits, i);
|
||||
if index as usize >= palette.len() {
|
||||
return Err(PaletteError::BadIndex {
|
||||
index,
|
||||
palette: palette.len(),
|
||||
});
|
||||
}
|
||||
ids.push(index);
|
||||
}
|
||||
Ok(match config {
|
||||
Config::List(bits) => Self {
|
||||
strategy,
|
||||
bits,
|
||||
words: data.to_vec(),
|
||||
palette: Palette::List(palette),
|
||||
},
|
||||
_ => Self::build(strategy, palette, &ids),
|
||||
})
|
||||
}
|
||||
/// Whether any palette entry satisfies `predicate` (`maybeHas`): entries
|
||||
/// no longer used by any position still count.
|
||||
pub fn maybe_has(&self, mut predicate: impl FnMut(u32) -> bool) -> bool {
|
||||
match &self.palette {
|
||||
Palette::Single(value) => predicate(*value),
|
||||
Palette::List(list) => list.iter().any(|&v| predicate(v)),
|
||||
Palette::Global => (0..1u32 << self.bits.min(31)).any(predicate),
|
||||
}
|
||||
}
|
||||
/// Heap bytes held by the palette and the storage.
|
||||
pub fn heap_bytes(&self) -> usize {
|
||||
self.words.capacity() * 8
|
||||
+ match &self.palette {
|
||||
Palette::List(list) => list.capacity() * 4,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl PartialEq for PalettedContainer {
|
||||
/// Equal when every entry is.
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
let entries = self.strategy.entries();
|
||||
entries == other.strategy.entries() && (0..entries).all(|i| self.get(i) == other.get(i))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
const BLOCKS: Strategy = Strategy::blocks(16);
|
||||
const BIOMES: Strategy = Strategy::biomes(7);
|
||||
fn rng(state: &mut u64) -> u64 {
|
||||
*state ^= *state << 13;
|
||||
*state ^= *state >> 7;
|
||||
*state ^= *state << 17;
|
||||
*state
|
||||
}
|
||||
#[test]
|
||||
fn grows_through_vanilla_widths() {
|
||||
let mut c = PalettedContainer::new(BLOCKS, 0);
|
||||
assert_eq!((c.bits(), c.palette_len()), (0, 1));
|
||||
let mut expected = Vec::new();
|
||||
for n in 1..=300u32 {
|
||||
c.set(n as usize, n);
|
||||
expected.push(match n + 1 {
|
||||
2..=16 => 4,
|
||||
17..=32 => 5,
|
||||
33..=64 => 6,
|
||||
65..=128 => 7,
|
||||
129..=256 => 8,
|
||||
_ => 16,
|
||||
});
|
||||
assert_eq!(c.bits(), *expected.last().unwrap(), "after {n} values");
|
||||
}
|
||||
assert!(c.is_global());
|
||||
for n in 1..=300u32 {
|
||||
assert_eq!(c.get(n as usize), n);
|
||||
}
|
||||
let mut b = PalettedContainer::new(BIOMES, 3);
|
||||
for n in 1..=9u32 {
|
||||
b.set(n as usize, n + 3);
|
||||
let want = match n + 1 {
|
||||
2 => 1,
|
||||
3..=4 => 2,
|
||||
5..=8 => 3,
|
||||
_ => 7,
|
||||
};
|
||||
assert_eq!(b.bits(), want, "after {n} biomes");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn matches_a_plain_array_under_random_edits() {
|
||||
for (strategy, distinct) in [(BLOCKS, 700u64), (BLOCKS, 12), (BIOMES, 9), (BIOMES, 3)] {
|
||||
let entries = strategy.entries();
|
||||
let mut state = 0x1234_5678_9ABC_DEF1u64;
|
||||
let mut c = PalettedContainer::new(strategy, 5);
|
||||
let mut plain = vec![5u32; entries];
|
||||
for _ in 0..20_000 {
|
||||
let r = rng(&mut state);
|
||||
let i = (r % entries as u64) as usize;
|
||||
let v = ((r >> 20) % distinct) as u32 * 7;
|
||||
assert_eq!(c.set(i, v), plain[i]);
|
||||
plain[i] = v;
|
||||
}
|
||||
let mut out = vec![0; entries];
|
||||
c.unpack(&mut out);
|
||||
assert_eq!(out, plain);
|
||||
for (i, &v) in plain.iter().enumerate() {
|
||||
assert_eq!(c.get(i), v);
|
||||
}
|
||||
let packed = c.pack();
|
||||
let loaded = PalettedContainer::from_packed(strategy, packed.palette.clone(), packed.data.as_deref())
|
||||
.unwrap();
|
||||
assert_eq!(loaded, c);
|
||||
assert_eq!(loaded.pack(), packed);
|
||||
let again = PalettedContainer::from_values(strategy, &plain);
|
||||
assert_eq!(again, c);
|
||||
assert_eq!(again.pack(), packed);
|
||||
let (bits, palette, words) = c.to_wire();
|
||||
let wire = PalettedContainer::from_wire(strategy, bits, palette, words.to_vec()).unwrap();
|
||||
assert_eq!(wire, c);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn uniform_detection_ignores_stale_palette_entries() {
|
||||
let mut c = PalettedContainer::new(BLOCKS, 0);
|
||||
c.set(10, 3);
|
||||
assert!(!c.is_uniform(0));
|
||||
c.set(10, 0);
|
||||
assert!(c.palette_len() == 2 && c.is_uniform(0) && c.uniform() == Some(0));
|
||||
assert!(!c.is_uniform(3));
|
||||
for i in 0..4096 {
|
||||
c.set(i, 3);
|
||||
}
|
||||
assert_eq!(c.uniform(), Some(3));
|
||||
}
|
||||
#[test]
|
||||
fn rejects_malformed_wire_data() {
|
||||
assert!(PalettedContainer::from_wire(BLOCKS, 4, vec![1, 2], vec![0; 255]).is_err());
|
||||
assert!(PalettedContainer::from_wire(BLOCKS, 4, vec![1, 2], vec![0x3333; 256]).is_err());
|
||||
assert!(PalettedContainer::from_wire(BLOCKS, 4, vec![], vec![0; 256]).is_err());
|
||||
assert!(PalettedContainer::from_wire(BLOCKS, 0, vec![1, 2], vec![]).is_err());
|
||||
assert!(PalettedContainer::from_wire(BIOMES, 2, vec![1, 2, 3, 4, 5], vec![0; 4]).is_err());
|
||||
assert!(PalettedContainer::from_wire(BLOCKS, 0, vec![9], vec![]).is_ok());
|
||||
}
|
||||
#[test]
|
||||
fn division_magic_is_exact() {
|
||||
for bits in 1..=32u8 {
|
||||
let per = 64 / usize::from(bits);
|
||||
for i in 0..4096usize {
|
||||
assert_eq!(locate(bits, i).0, i / per, "bits {bits} index {i}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package dev.horizonlab.parity;
|
||||
|
||||
import com.google.gson.*;
|
||||
import io.netty.buffer.ByteBuf;
|
||||
import io.netty.buffer.Unpooled;
|
||||
import java.nio.file.*;
|
||||
import java.util.*;
|
||||
import java.util.stream.LongStream;
|
||||
import net.minecraft.core.IdMap;
|
||||
import net.minecraft.network.FriendlyByteBuf;
|
||||
import net.minecraft.world.level.chunk.PalettedContainer;
|
||||
import net.minecraft.world.level.chunk.PalettedContainerRO;
|
||||
import net.minecraft.world.level.chunk.Strategy;
|
||||
|
||||
/** Runs the real PalettedContainer (single-value, linear, hash-map and global palettes, resize,
|
||||
* network write/read, pack/unpack) over scripted edits and records hashes of everything
|
||||
* observable: set results, the serialized container after every few edits, the saved form and
|
||||
* the containers read back from both. */
|
||||
public final class VanillaPalettedContainerOracle {
|
||||
private static final Gson JSON = new GsonBuilder().serializeNulls().create();
|
||||
/** The palettes compare entries by identity, as registry objects allow: intern the boxes. */
|
||||
private static final Map<Integer, Integer> INTERNED = new HashMap<>();
|
||||
private static Integer intern(int value) { return INTERNED.computeIfAbsent(value, v -> new Integer(v)); }
|
||||
private static final class Ids implements IdMap<Integer> {
|
||||
final int size;
|
||||
Ids(int size) { this.size = size; }
|
||||
public int getId(Integer value) { return value < size ? value : -1; }
|
||||
public Integer byId(int id) { return id >= 0 && id < size ? intern(id) : null; }
|
||||
public int size() { return size; }
|
||||
public Iterator<Integer> iterator() { return java.util.stream.IntStream.range(0, size).boxed().iterator(); }
|
||||
}
|
||||
private static long fnv(long h, int b) { return (h ^ (b & 0xFF)) * 0x100000001b3L; }
|
||||
private static final long BASIS = 0xcbf29ce484222325L;
|
||||
private static long hashBytes(byte[] bytes) {
|
||||
long h = BASIS;
|
||||
for (byte b : bytes) h = fnv(h, b);
|
||||
return h;
|
||||
}
|
||||
private static long hashValues(PalettedContainer<Integer> c, int bits) {
|
||||
long h = BASIS;
|
||||
int n = 1 << (3 * bits);
|
||||
for (int i = 0; i < n; i++) {
|
||||
int v = c.get(i & (n - 1) & ((1 << bits) - 1), i >> (2 * bits), (i >> bits) & ((1 << bits) - 1));
|
||||
for (int k = 0; k < 4; k++) h = fnv(h, v >> (8 * k));
|
||||
}
|
||||
return h;
|
||||
}
|
||||
private static byte[] write(PalettedContainer<Integer> c) {
|
||||
var buffer = new FriendlyByteBuf(Unpooled.buffer());
|
||||
c.write(buffer);
|
||||
byte[] bytes = new byte[buffer.readableBytes()];
|
||||
buffer.readBytes(bytes);
|
||||
return bytes;
|
||||
}
|
||||
private static String wire(PalettedContainer<Integer> c) {
|
||||
byte[] bytes = write(c);
|
||||
return bytes.length + ":" + hashBytes(bytes);
|
||||
}
|
||||
/** Input line: name kind initial every i v i v ...; output: one line of fields. */
|
||||
private static String scenario(String line) {
|
||||
var t = line.trim().split(" ");
|
||||
boolean blocks = t[1].equals("blocks");
|
||||
int bits = blocks ? 4 : 2;
|
||||
Strategy<Integer> strategy = blocks ? Strategy.createForBlockStates(new Ids(35723)) : Strategy.createForBiomes(new Ids(67));
|
||||
var c = new PalettedContainer<>(intern(Integer.parseInt(t[2])), strategy);
|
||||
int every = Integer.parseInt(t[3]);
|
||||
var checkpoints = new StringJoiner(",");
|
||||
long oldsHash = BASIS;
|
||||
int n = 0, mask = (1 << bits) - 1;
|
||||
for (int k = 4; k + 1 < t.length; k += 2) {
|
||||
int i = Integer.parseInt(t[k]);
|
||||
int old = c.getAndSetUnchecked(i & mask, i >> (2 * bits), (i >> bits) & mask, intern(Integer.parseInt(t[k + 1])));
|
||||
for (int j = 0; j < 4; j++) oldsHash = fnv(oldsHash, old >> (8 * j));
|
||||
if (++n % every == 0) checkpoints.add(wire(c));
|
||||
}
|
||||
var out = new StringJoiner(" ");
|
||||
out.add(t[0]).add(Long.toString(oldsHash)).add("cp=" + checkpoints).add("bits=" + c.bitsPerEntry());
|
||||
out.add("final=" + wire(c)).add("values=" + hashValues(c, bits));
|
||||
var read = new PalettedContainer<>(intern(0), strategy);
|
||||
read.read(new FriendlyByteBuf(Unpooled.wrappedBuffer(write(c))));
|
||||
out.add("read=" + hashValues(read, bits) + "/" + wire(read));
|
||||
var packed = c.pack(strategy);
|
||||
long[] storage = packed.storage().map(LongStream::toArray).orElse(null);
|
||||
long storageHash = BASIS;
|
||||
if (storage != null) for (long word : storage) for (int j = 0; j < 8; j++) storageHash = fnv(storageHash, (int) (word >> (8 * j)));
|
||||
var palette = new StringJoiner(",");
|
||||
for (int v : packed.paletteEntries()) palette.add(Integer.toString(v));
|
||||
out.add("packed=" + packed.bitsPerEntry() + "/" + storageHash + "/" + palette);
|
||||
var loaded = PalettedContainer.unpack(strategy, new PalettedContainerRO.PackedData<>(packed.paletteEntries(),
|
||||
Optional.ofNullable(storage).map(LongStream::of), -1)).getOrThrow();
|
||||
out.add("loaded=" + hashValues(loaded, bits) + "/" + wire(loaded));
|
||||
return out.toString();
|
||||
}
|
||||
public static void main(String[] args) throws Exception {
|
||||
net.minecraft.SharedConstants.tryDetectVersion();
|
||||
try (var input = Files.newBufferedReader(Path.of(args[0])); var output = Files.newBufferedWriter(Path.of(args[1]))) {
|
||||
for (String line; (line = input.readLine()) != null; ) {
|
||||
output.write(scenario(line));
|
||||
output.newLine();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare Horizon's resident PalettedContainer with the client's own.
|
||||
|
||||
Scripted edits run through vanilla's PalettedContainer (block-state and biome
|
||||
strategies: single-value, linear, hash-map and global palettes, resizing, network
|
||||
write and read, pack and unpack) and through horizon_core::paletted. Both sides
|
||||
print the set results, the serialized container after every few edits, the
|
||||
container read back from the wire, the saved form (palette, bit width, storage)
|
||||
and the container loaded from it, as hashes.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import random
|
||||
import subprocess
|
||||
try:
|
||||
from . import lab, test_rust_parity as base
|
||||
except ImportError:
|
||||
import lab
|
||||
import test_rust_parity as base
|
||||
|
||||
BUILD = lab.ROOT / 'target/paletted-container-parity'
|
||||
LIMITS = {'blocks': (4096, 35723), 'biomes': (64, 67)}
|
||||
|
||||
|
||||
def corpus():
|
||||
rng = random.Random(0x50414C45545445)
|
||||
counts = [1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 31, 32, 33, 64, 100, 128, 129, 200, 256, 257, 300, 700, 2000]
|
||||
for kind, (entries, ids) in LIMITS.items():
|
||||
for count in counts:
|
||||
if kind == 'biomes' and count > 70:
|
||||
continue
|
||||
for trial in range(6):
|
||||
pool = rng.sample(range(ids), min(count, ids))
|
||||
initial = rng.choice(pool) if trial % 2 else rng.randrange(ids)
|
||||
ops = []
|
||||
for _ in range(rng.choice([entries // 4, entries, entries * 3])):
|
||||
ops.append((rng.randrange(entries), rng.choice(pool)))
|
||||
if trial == 5:
|
||||
# Every position written once, in order, then half of them reset.
|
||||
ops = [(i, pool[i % len(pool)]) for i in range(entries)] + [(i, initial) for i in range(0, entries, 2)]
|
||||
yield f'{kind}_{count}_{trial}', kind, initial, rng.choice([1, 7, 37, 211]), ops
|
||||
# A few edits at a time: the palette grows one value at a time from a single value.
|
||||
for trial in range(20):
|
||||
ops = [(rng.randrange(entries), rng.randrange(ids)) for _ in range(rng.randrange(1, 40))]
|
||||
yield f'{kind}_sparse_{trial}', kind, rng.randrange(ids), 1, ops
|
||||
# Overwriting with the value already there, and filling a section with one value.
|
||||
yield f'{kind}_fill', kind, 3, 500, [(i, 9) for i in range(entries)] + [(i, 3) for i in range(entries)]
|
||||
yield f'{kind}_same', kind, 3, 1, [(i, 3) for i in range(0, entries, 5)]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.parse_args()
|
||||
lock, jars, libraries = base.verify_artifacts()
|
||||
BUILD.mkdir(parents=True, exist_ok=True)
|
||||
oracle = lab.ROOT / 'mods/horizon/src/test/java/dev/horizonlab/parity/VanillaPalettedContainerOracle.java'
|
||||
sources = [Path(__file__), oracle, lab.ROOT / 'tools/lab.py', lab.ROOT / 'tools/test_rust_parity.py', lab.ROOT / 'Cargo.toml', lab.ROOT / 'Cargo.lock',
|
||||
lab.ROOT / 'crates/horizon-core/Cargo.toml', lab.ROOT / 'crates/horizon-core/examples/paletted_fixture.rs',
|
||||
*sorted((lab.ROOT / 'crates/horizon-core/src').rglob('*.rs'))]
|
||||
tested = base.source_hashes(sources)
|
||||
scenarios = list(corpus())
|
||||
inputs = BUILD / 'inputs.txt'
|
||||
inputs.write_text(''.join(f'{name} {kind} {initial} {every} ' + ' '.join(f'{i} {v}' for i, v in ops) + '\n'
|
||||
for name, kind, initial, every, ops in scenarios))
|
||||
|
||||
def run(command, name):
|
||||
with (BUILD / (name + '.log')).open('w') as log:
|
||||
base.run(list(map(str, command)), stdout=log, stderr=subprocess.STDOUT)
|
||||
classes = BUILD / 'classes'
|
||||
classes.mkdir(exist_ok=True)
|
||||
dependencies = os.pathsep.join(map(str, [jars['client'], *libraries]))
|
||||
run(['javac', '--release', lock['minecraft']['java_major'], '-proc:none', '-cp', dependencies, '-d', classes, oracle], 'compile-java')
|
||||
run(['cargo', 'build', '--locked', '--profile', 'fast', '-j', os.environ.get('CARGO_BUILD_JOBS', '3'), '-p', 'horizon-core', '--example', 'paletted_fixture'], 'compile-rust')
|
||||
metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--no-deps', '--format-version', '1'], cwd=lab.ROOT))
|
||||
binary = Path(metadata['target_directory']) / 'fast/examples/paletted_fixture'
|
||||
run([binary, inputs, BUILD / 'rust.txt'], 'rust')
|
||||
run(['java', '-Xmx1G', '-cp', str(classes) + os.pathsep + dependencies, 'dev.horizonlab.parity.VanillaPalettedContainerOracle', inputs, BUILD / 'client.txt'], 'client')
|
||||
client = (BUILD / 'client.txt').read_text().splitlines()
|
||||
rust = (BUILD / 'rust.txt').read_text().splitlines()
|
||||
assert len(client) == len(rust) == len(scenarios), (len(client), len(rust), len(scenarios))
|
||||
bad = [(a, b) for a, b in zip(client, rust) if a != b]
|
||||
base.ensure_sources_unchanged(sources, tested)
|
||||
for a, b in bad[:5]:
|
||||
print('client', a[:600])
|
||||
print('rust ', b[:600])
|
||||
if bad:
|
||||
raise SystemExit(f'{len(bad)} of {len(scenarios)} scenarios differ')
|
||||
edits = sum(len(s[4]) for s in scenarios)
|
||||
report = {'schema': 1, 'minecraft': lock['minecraft']['version'], 'platform': platform.platform(), 'scenarios': len(scenarios), 'edits': edits,
|
||||
'sources_sha256': tested}
|
||||
lab.write_json(BUILD / 'report.json', report)
|
||||
print(f'PalettedContainer parity: {len(scenarios)} scenarios, {edits:,} edits, all hashes identical')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user