From 3667bb963c63cca07f99182cc2da7c23a2977382 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:45:48 +0200 Subject: [PATCH] Chunk batches: make the columns' heightmaps on the warm-up threads too Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/assets.rs | 20 +++ crates/horizon-client/src/lan/chunks.rs | 221 +++++++++++++++++------- crates/horizon-client/src/save.rs | 41 +++-- 3 files changed, 211 insertions(+), 71 deletions(-) diff --git a/crates/horizon-client/src/assets.rs b/crates/horizon-client/src/assets.rs index 3e7c6e909..beafa96fd 100644 --- a/crates/horizon-client/src/assets.rs +++ b/crates/horizon-client/src/assets.rs @@ -42,6 +42,26 @@ impl HeightmapState for StateFacts { } #[derive(Clone)] pub struct Properties(Arc>); +#[cfg(test)] +impl Properties { + /// Heightmap facts for tests, per state: (air, has a fluid, blocks + /// motion, blocks motion but is not leaves). State 0 is minecraft:air. + pub(crate) fn for_tests(states: &[(bool, bool, bool, bool)]) -> Self { + Self(Arc::new( + states + .iter() + .enumerate() + .map(|(i, &(air, fluid, motion, no_leaves))| StateFacts { + air, + fluid, + air_block: i == 0, + motion, + no_leaves, + }) + .collect(), + )) + } +} impl TerrainBlockStates for Properties { type State = StateFacts; fn heightmap_state(&self, id: BlockStateId) -> StateFacts { diff --git a/crates/horizon-client/src/lan/chunks.rs b/crates/horizon-client/src/lan/chunks.rs index 35339a05c..65b092881 100644 --- a/crates/horizon-client/src/lan/chunks.rs +++ b/crates/horizon-client/src/lan/chunks.rs @@ -125,6 +125,44 @@ struct Heights { /// Heightmap type ids with their packed long arrays. type ColumnHeightmaps = Arc)>>; +/// A column's cached heightmaps, while it holds the sections they were made +/// from. +fn cached_heights( + heights: &HashMap<(i32, i32), Heights>, + chunk: &Chunk, +) -> Option { + let h = heights.get(&(chunk.x, chunk.z))?; + let same = h.sections.len() == chunk.sections.len() + && h.sections + .iter() + .zip(&chunk.sections) + .all(|(w, s)| std::ptr::eq(w.as_ptr(), Arc::as_ptr(s))); + same.then(|| h.maps.clone()) +} + +/// Cache a column's heightmaps with the sections they were made from, +/// dropping the columns whose sections are gone once there are many. +fn keep_heights( + heights: &mut HashMap<(i32, i32), Heights>, + key: (i32, i32), + sections: &[Arc
], + maps: &ColumnHeightmaps, +) { + if heights.len() >= CACHE_SWEEP / 4 { + heights.retain(|_, h| h.sections.iter().all(|w| w.strong_count() > 0)); + if heights.len() >= CACHE_SWEEP / 8 { + heights.clear(); + } + } + heights.insert( + key, + Heights { + sections: sections.iter().map(Arc::downgrade).collect(), + maps: maps.clone(), + }, + ); +} + /// What a resident section gives the wire, kept while the section is /// unchanged. Resident sections are immutable behind `Arc` (edits go through /// `Arc::make_mut`, which moves an edited section to a new address while a @@ -259,20 +297,34 @@ const CACHE_SWEEP: usize = 1 << 15; /// alone: waking the warm-up threads costs more than a few sections. const WARM_MIN: usize = 16; -/// Sections a batch needs, each taken by whichever thread is free first, -/// and what each gave: its wire bytes and, with varying light, its layers. +/// The work a batch needs, each piece taken by whichever thread is free +/// first: the heightmaps of its columns not cached yet, then its sections +/// not cached yet, and what each gave (a section: its wire bytes and, with +/// varying light, its layers). struct Warm { coder: Arc, + /// The heightmaps' tables and the level's (min_y, height). + heightmaps: Option<(Arc, i32, i32)>, + columns: Vec<((i32, i32), Vec>)>, + heights: Vec>, sections: Vec>, next: AtomicUsize, done: Vec>, } type Warmed = ((Arc<[u8]>, Entities), Option>); impl Warm { - /// Encode sections until none is left to take. + /// Take work until none is left. fn run(&self) { loop { let i = self.next.fetch_add(1, Ordering::Relaxed); + if let Some((_, sections)) = self.columns.get(i) { + if let Some((tables, min_y, height)) = &self.heightmaps { + let maps = crate::save::client_heightmaps_in(sections, tables, *min_y, *height); + let _ = self.heights[i].set(Arc::new(maps)); + } + continue; + } + let i = i - self.columns.len(); let Some(s) = self.sections.get(i) else { return; }; @@ -391,40 +443,29 @@ impl ChunkEncoder { /// The client heightmaps (Heightmap.prime over the column), from the /// cache while the column's sections are unchanged. fn heightmaps(&self, chunk: &Chunk, tables: &crate::save::SaveTables) -> ColumnHeightmaps { - let same = |h: &Heights| { - h.sections.len() == chunk.sections.len() - && h.sections - .iter() - .zip(&chunk.sections) - .all(|(w, s)| std::ptr::eq(w.as_ptr(), Arc::as_ptr(s))) - }; let mut heights = self.heights.lock().unwrap(); - if let Some(h) = heights.get(&(chunk.x, chunk.z)) - && same(h) - { - return h.maps.clone(); + if let Some(maps) = cached_heights(&heights, chunk) { + return maps; } - if heights.len() >= CACHE_SWEEP / 4 { - heights.retain(|_, h| h.sections.iter().all(|w| w.strong_count() > 0)); - if heights.len() >= CACHE_SWEEP / 8 { - heights.clear(); - } - } - let min_y = MIN_Y + 16 * self.sections.start as i32; - let height = 16 * self.sections.len() as i32; + let (min_y, height) = self.level(); let maps = Arc::new(crate::save::client_heightmaps_in( - chunk, tables, min_y, height, + &chunk.sections, + tables, + min_y, + height, )); - heights.insert( - (chunk.x, chunk.z), - Heights { - sections: chunk.sections.iter().map(Arc::downgrade).collect(), - maps: maps.clone(), - }, - ); + keep_heights(&mut heights, (chunk.x, chunk.z), &chunk.sections, &maps); maps } + /// The level's min_y and height (`DimensionType`), the heightmaps' range. + fn level(&self) -> (i32, i32) { + ( + MIN_Y + 16 * self.sections.start as i32, + 16 * self.sections.len() as i32, + ) + } + /// The cached entry for a section, made with `non_air` when missing. fn cached(&self, section: &Arc
, read: impl FnOnce(&mut Encoded) -> T) -> T { read(self.entry(&mut self.cache.lock().unwrap(), section)) @@ -538,14 +579,54 @@ impl ChunkEncoder { self.body(chunk, &world.block_entities, self.around(chunk, world)) } - /// Encode the sections of these columns not cached yet before their - /// bodies are made, shared out to the warm-up threads: a batch of new - /// terrain costs mostly its sections' first encoding. This thread takes - /// sections too until none is left, then caches what is done; it never - /// waits for the other threads (they keep their results to themselves - /// until then), and a section one of them is still on is encoded again - /// by `body_in`, with the same bytes. + /// Make what these columns need and is not cached yet (their heightmaps + /// and sections) before their bodies are made, shared out to the warm-up + /// threads: a batch of new terrain costs mostly its sections' first + /// encoding. This thread takes work too until none is left, then caches + /// what is done; it never waits for the other threads (they keep their + /// results to themselves until then), and what one of them is still on + /// is made again by `body_in`, the same. pub fn warm(&self, columns: &[&Chunk]) { + let Some(job) = self.job(columns) else { + return; + }; + let Some(pool) = warm_pool() else { + return; + }; + let job = Arc::new(job); + { + let mut slot = pool.job.lock().unwrap(); + slot.0 += 1; + slot.1 = Some(job.clone()); + } + pool.ready.notify_all(); + job.run(); + // Everything is claimed: the job leaves the slot. + { + let mut slot = pool.job.lock().unwrap(); + if slot.1.as_ref().is_some_and(|j| Arc::ptr_eq(j, &job)) { + slot.1 = None; + } + } + self.keep(&job); + } + + /// The work `warm` shares out: the heightmaps of the columns and the + /// sections not cached yet; none when there is too little to wake the + /// warm-up threads for. + fn job(&self, columns: &[&Chunk]) -> Option { + let stale: Vec<((i32, i32), Vec>)> = match &self.tables { + Some(_) => { + let heights = self.heights.lock().unwrap(); + let mut seen = crate::fast_hash::FxHashSet::default(); + columns + .iter() + .filter(|c| seen.insert((c.x, c.z)) && cached_heights(&heights, c).is_none()) + .map(|c| ((c.x, c.z), c.sections.clone())) + .collect() + } + None => Vec::new(), + }; let sections: Vec> = { let cache = self.cache.lock().unwrap(); columns @@ -559,30 +640,29 @@ impl ChunkEncoder { .cloned() .collect() }; - if sections.len() < WARM_MIN { - return; + if sections.len() + stale.len() < WARM_MIN { + return None; } - let Some(pool) = warm_pool() else { - return; - }; - let job = Arc::new(Warm { + let (min_y, height) = self.level(); + Some(Warm { coder: self.coder.clone(), + heightmaps: self.tables.clone().map(|t| (t, min_y, height)), + heights: stale.iter().map(|_| OnceLock::new()).collect(), + columns: stale, done: sections.iter().map(|_| OnceLock::new()).collect(), sections, next: AtomicUsize::new(0), - }); + }) + } + + /// Cache what a job's threads have finished. + fn keep(&self, job: &Warm) { { - let mut slot = pool.job.lock().unwrap(); - slot.0 += 1; - slot.1 = Some(job.clone()); - } - pool.ready.notify_all(); - job.run(); - // Every section is claimed: the job leaves the slot. - { - let mut slot = pool.job.lock().unwrap(); - if slot.1.as_ref().is_some_and(|j| Arc::ptr_eq(j, &job)) { - slot.1 = None; + let mut heights = self.heights.lock().unwrap(); + for ((key, sections), done) in job.columns.iter().zip(&job.heights) { + if let Some(maps) = done.get() { + keep_heights(&mut heights, *key, sections, maps); + } } } let mut cache = self.cache.lock().unwrap(); @@ -1232,9 +1312,9 @@ mod tests { ); } - /// Columns whose sections the warm-up threads encoded give the bytes - /// each gives on its own, also with a column twice and its neighbours' - /// sections deciding the light layers. + /// Columns whose sections and heightmaps the warm-up threads made give + /// the bytes each gives on its own, also with a column twice and its + /// neighbours' sections deciding the light layers. #[test] fn warmed_columns_give_the_same_bodies() { let builtin = Builtin::parse( @@ -1278,15 +1358,36 @@ mod tests { .iter() .map(|k| &**world.chunks.get(k).unwrap()) .collect(); - let alone = ChunkEncoder::new(states.clone(), &builtin, &[], &[]); + // (air, fluid, blocks motion, blocks motion and is not leaves) per state. + let tables = Arc::new(crate::save::SaveTables::for_heightmaps( + crate::assets::Properties::for_tests(&[ + (true, false, false, false), + (false, false, true, true), + (false, true, true, true), + (false, false, false, false), + ]), + )); + let mut alone = ChunkEncoder::new(states.clone(), &builtin, &[], &[]); + alone.tables = Some(tables.clone()); let expected: Vec> = chunks.iter().map(|c| alone.body_in(c, &world)).collect(); - let warmed = ChunkEncoder::new(states, &builtin, &[], &[]); + let mut warmed = ChunkEncoder::new(states.clone(), &builtin, &[], &[]); + warmed.tables = Some(tables); let bodies = |encoder: &ChunkEncoder| -> Vec> { chunks.iter().map(|c| encoder.body_in(c, &world)).collect() }; assert!(chunks.len() * SECTIONS >= WARM_MIN); warmed.warm(&chunks); assert_eq!(bodies(&warmed), expected); + // Run on this thread alone, the job makes every column's heightmaps + // (once for the column there twice) and sections. + let mut cold = ChunkEncoder::new(states, &builtin, &[], &[]); + cold.tables = alone.tables.clone(); + let job = cold.job(&chunks).unwrap(); + assert_eq!((job.columns.len(), job.sections.len()), (16, 17 * SECTIONS)); + job.run(); + cold.keep(&job); + assert_eq!(cold.heights.lock().unwrap().len(), 16); + assert_eq!(bodies(&cold), expected); // Again, from the cache, with nothing left to warm. warmed.warm(&chunks); assert_eq!(bodies(&warmed), expected); diff --git a/crates/horizon-client/src/save.rs b/crates/horizon-client/src/save.rs index c407a7967..314e50c2d 100644 --- a/crates/horizon-client/src/save.rs +++ b/crates/horizon-client/src/save.rs @@ -16,7 +16,7 @@ use crate::{ Result, biome::Biomes, - chunk::{BIOME_STRATEGY, Chunk, Light, MIN_Y, SECTIONS, block_strategy}, + chunk::{BIOME_STRATEGY, Chunk, Light, MIN_Y, SECTIONS, Section, block_strategy}, streaming::ChunkGenerator, }; use horizon_core::{ @@ -269,6 +269,18 @@ impl SaveTables { air: registry.default_state("minecraft:air").map_or(0, |s| s.0), } } + /// Tables with only the heightmaps' state facts, for tests. + #[cfg(test)] + pub(crate) fn for_heightmaps(heightmaps: crate::assets::Properties) -> Self { + Self { + block_names: Vec::new(), + block_ids: HashMap::new(), + default_state: Vec::new(), + block_entity: Vec::new(), + heightmaps, + air: 0, + } + } /// Which states carry a block entity (`BlockFacts`), to drop entries for /// unknown block entities whose block is gone. pub fn with_block_entities(mut self, facts: &crate::blocks::BlockFacts) -> Self { @@ -673,8 +685,9 @@ pub fn decode_block_entity(tag: &Tag) -> Option<((i32, i32, i32), crate::contain Some((pos, entity)) } +/// A column's sections as `Heightmap.primeHeightmaps` reads them. struct HeightmapView<'a> { - chunk: &'a Chunk, + sections: &'a [Arc
], tables: &'a SaveTables, } impl HeightmapChunk for HeightmapView<'_> { @@ -687,7 +700,6 @@ impl HeightmapChunk for HeightmapView<'_> { } fn highest_section_y(&self) -> i32 { let top = self - .chunk .sections .iter() .rposition(|s| !s.is_uniform(self.tables.air)) @@ -695,9 +707,12 @@ impl HeightmapChunk for HeightmapView<'_> { MIN_Y + top as i32 * 16 } fn block_state(&self, x: i32, y: i32, z: i32) -> Self::State { - self.tables - .heightmaps - .heightmap_state(self.chunk.block(x, y, z)) + // Chunk.get: no block outside the sections. + let state = usize::try_from((y - MIN_Y) >> 4) + .ok() + .and_then(|s| self.sections.get(s)) + .map_or(0, |s| s.get(crate::chunk::index(x, y, z))); + self.tables.heightmaps.heightmap_state(BlockStateId(state)) } } /// Heightmaps kept after world generation (`ChunkStatus.FULL.heightmapsAfter`). @@ -712,7 +727,10 @@ fn heightmaps(chunk: &Chunk, tables: &SaveTables) -> Tag { .iter() .filter_map(|k| Heightmap::new(*k, MIN_Y, SECTIONS as i32 * 16).ok()) .collect(); - let view = HeightmapView { chunk, tables }; + let view = HeightmapView { + sections: &chunk.sections, + tables, + }; let mut refs: Vec<&mut Heightmap> = maps.iter_mut().collect(); if Heightmap::prime(&view, &mut refs).is_err() { return Tag::Compound(Compound::new()); @@ -728,11 +746,12 @@ fn heightmaps(chunk: &Chunk, tables: &SaveTables) -> Tag { /// sendToClient: WORLD_SURFACE, MOTION_BLOCKING, MOTION_BLOCKING_NO_LEAVES), /// as (network id, packed longs) in Heightmap.Types order. pub fn client_heightmaps(chunk: &Chunk, tables: &SaveTables) -> Vec<(i32, Vec)> { - client_heightmaps_in(chunk, tables, MIN_Y, SECTIONS as i32 * 16) + client_heightmaps_in(&chunk.sections, tables, MIN_Y, SECTIONS as i32 * 16) } -/// The client heightmaps of a level from `min_y`, `height` blocks high. +/// The client heightmaps of a column's sections, in a level from `min_y`, +/// `height` blocks high. pub fn client_heightmaps_in( - chunk: &Chunk, + sections: &[Arc
], tables: &SaveTables, min_y: i32, height: i32, @@ -746,7 +765,7 @@ pub fn client_heightmaps_in( .iter() .filter_map(|(id, k)| Heightmap::new(*k, min_y, height).ok().map(|m| (*id, m))) .collect(); - let view = HeightmapView { chunk, tables }; + let view = HeightmapView { sections, tables }; let mut refs: Vec<&mut Heightmap> = maps.iter_mut().map(|(_, m)| m).collect(); if Heightmap::prime(&view, &mut refs).is_err() { return Vec::new();