From 0eaa13020cc16264f6e0488da7e7108f013750cd Mon Sep 17 00:00:00 2001 From: rdbtCVS <87152661+rdbtCVS@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:18:39 +0100 Subject: [PATCH] perf: load uniform saved light without expanding nibble layers --- crates/horizon-client/src/save.rs | 214 +++++++++++++++++++++++++- docs/saved-light-loading-benchmark.md | 44 ++++++ 2 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 docs/saved-light-loading-benchmark.md diff --git a/crates/horizon-client/src/save.rs b/crates/horizon-client/src/save.rs index 2e1eb8455..0cfe3c585 100644 --- a/crates/horizon-client/src/save.rs +++ b/crates/horizon-client/src/save.rs @@ -1305,8 +1305,73 @@ pub fn load_light(tag: &Tag, chunk: &mut Chunk, sky_light: bool) { }; layers[si as usize] = (layer("SkyLight"), layer("BlockLight")); } - let above = [if sky_light { 15 } else { 0 }; 256]; - crate::light::light_from_layers(chunk, &layers, above, sky_light); + // Uniform saved layers need no expanded allocation. Both nibbles of + // each byte must match, not merely the bytes across the layer. + let uniform_layer = |layer: &[u8]| { + let first = layer[0]; + (first & 15 == first >> 4 && layer.iter().all(|&v| v == first)).then_some(first & 15) + }; + let mut above = [if sky_light { 15 } else { 0 }; 256]; + for (si, (sky, block)) in layers.into_iter().enumerate().rev() { + let sky = sky.filter(|_| sky_light); + let sky_uniform = sky.map_or_else( + || above.iter().all(|&v| v == above[0]).then_some(above[0]), + uniform_layer, + ); + let block_uniform = block.map_or(Some(0), uniform_layer); + let section = Arc::make_mut(&mut chunk.sections[si]); + if let (Some(s), Some(b)) = (sky_uniform, block_uniform) { + section.light = Light::Uniform(s << 4 | b); + above.fill(s); + } else { + let mut data = Box::new([0u8; 4096]); + match (sky, block) { + (Some(sky), Some(block)) => { + for ((pair, &s), &b) in + data.as_chunks_mut::<2>().0.iter_mut().zip(sky).zip(block) + { + pair[0] = s << 4 | (b & 15); + pair[1] = (s & 0xf0) | (b >> 4); + } + } + (Some(sky), None) => { + for (pair, &s) in data.as_chunks_mut::<2>().0.iter_mut().zip(sky) { + pair[0] = s << 4; + pair[1] = s & 0xf0; + } + } + (None, Some(block)) => { + for (i, (pair, &b)) in data + .as_chunks_mut::<2>() + .0 + .iter_mut() + .zip(block) + .enumerate() + { + pair[0] = above[(i * 2) & 255] << 4 | (b & 15); + pair[1] = above[(i * 2 + 1) & 255] << 4 | (b >> 4); + } + } + (None, None) => { + for layer in data.as_chunks_mut::<256>().0 { + for (v, &s) in layer.iter_mut().zip(&above) { + *v = s << 4; + } + } + } + } + if let Some(sky) = sky { + for (pair, &s) in above.as_chunks_mut::<2>().0.iter_mut().zip(sky) { + pair[0] = s & 15; + pair[1] = s >> 4; + } + } + section.light = Light::Full(data); + } + section.compact(); + } + chunk.lit = true; + chunk.light_loaded = true; } type RegionCache = Mutex>>>>; @@ -3892,6 +3957,151 @@ mod tests { assert!(!unlit.lit && !unlit.light_loaded); } #[test] + fn saved_light_matches_reference_for_layers_and_metadata() { + fn reference(tag: &Tag, chunk: &mut Chunk, sky_light: bool) { + if tag.get("isLightOn") != Some(&Tag::Byte(1)) { + return; + } + let Some(Tag::List(sections)) = tag.get("sections") else { + return; + }; + let mut layers: Vec<(Option<&[u8]>, Option<&[u8]>)> = vec![(None, None); SECTIONS]; + for section in sections { + let Some(y) = number(section.get("Y")) else { + continue; + }; + let si = y as i32 - (MIN_Y >> 4); + if !(0..SECTIONS as i32).contains(&si) { + continue; + } + let layer = |name: &str| match section.get(name) { + Some(Tag::ByteArray(a)) if a.len() == 2048 => Some(&a[..]), + _ => None, + }; + layers[si as usize] = (layer("SkyLight"), layer("BlockLight")); + } + let above = [if sky_light { 15 } else { 0 }; 256]; + crate::light::light_from_layers(chunk, &layers, above, sky_light); + } + + let section = |y: Tag, sky: Option, block: Option| { + let mut entries = vec![("Y", y)]; + if let Some(v) = sky { + entries.push(("SkyLight", v)); + } + if let Some(v) = block { + entries.push(("BlockLight", v)); + } + Tag::compound(entries) + }; + let root = |sections| { + Tag::compound([ + ("isLightOn", Tag::Byte(1)), + ("sections", Tag::List(sections)), + ]) + }; + let check = |tag: &Tag| { + for sky_light in [false, true] { + let mut expected = Chunk::empty(-3, 5); + let s = Arc::make_mut(&mut expected.sections[0]); + s.light = Light::Full(Box::new([0xab; 4096])); + s.blocks.set(0, 1); + s.blocks.set(0, 0); + s.biomes.set(0, 1); + s.biomes.set(0, 0); + let mut actual = expected.clone(); + reference(tag, &mut expected, sky_light); + load_light(tag, &mut actual, sky_light); + assert_eq!( + (actual.lit, actual.light_loaded), + (expected.lit, expected.light_loaded) + ); + for (a, b) in actual.sections.iter().zip(&expected.sections) { + assert_eq!( + std::mem::discriminant(&a.light), + std::mem::discriminant(&b.light) + ); + assert_eq!( + format!("{:?}", a.blocks.to_wire()), + format!("{:?}", b.blocks.to_wire()) + ); + assert_eq!( + format!("{:?}", a.biomes.to_wire()), + format!("{:?}", b.biomes.to_wire()) + ); + for i in 0..4096 { + assert_eq!(a.light(i), b.light(i), "index {i}, sky {sky_light}"); + } + } + } + }; + for tag in [ + Tag::compound(Vec::<(&str, Tag)>::new()), + Tag::compound([("isLightOn", Tag::Byte(0))]), + Tag::compound([("isLightOn", Tag::Byte(2))]), + Tag::compound([("isLightOn", Tag::Int(1)), ("sections", Tag::List(vec![]))]), + Tag::compound([("isLightOn", Tag::Byte(1)), ("sections", Tag::Int(0))]), + root(vec![]), + ] { + check(&tag); + } + // Cover every repeated byte, including 0xF0/0x0F: a uniform byte + // array need not hold uniform nibbles. + for byte in 0..=255u8 { + check(&root(vec![section( + Tag::Byte(19), + Some(Tag::ByteArray(vec![byte; 2048])), + Some(Tag::ByteArray(vec![255 - byte; 2048])), + )])); + } + // Every pair of uniform sky/block nibble values, with missing + // sections below inheriting the stored bottom row. + for sky in 0..16u8 { + for block in 0..16u8 { + check(&root(vec![section( + Tag::Byte(1), + Some(Tag::ByteArray(vec![sky * 17; 2048])), + Some(Tag::ByteArray(vec![block * 17; 2048])), + )])); + } + } + for length in [0, 1, 2047, 2048, 2049] { + for invalid in [ + Tag::Int(1), + Tag::List(vec![]), + Tag::ByteArray(vec![0xf0; length]), + ] { + check(&root(vec![section( + Tag::Byte(-4), + Some(invalid.clone()), + Some(invalid), + )])); + } + } + let varied: Vec = (0..2048) + .map(|i| (i as u8).wrapping_mul(73).wrapping_add(31)) + .collect(); + check(&root(vec![ + section(Tag::Byte(-5), Some(Tag::ByteArray(varied.clone())), None), + section(Tag::Byte(20), Some(Tag::ByteArray(varied.clone())), None), + section( + Tag::String("19".into()), + Some(Tag::ByteArray(varied.clone())), + None, + ), + section(Tag::Byte(19), Some(Tag::ByteArray(varied.clone())), None), + section(Tag::Byte(17), None, Some(Tag::ByteArray(varied.clone()))), + section(Tag::Byte(-4), Some(Tag::ByteArray(varied.clone())), None), + // Last duplicate wins, even when its arrays have wrong types. + section( + Tag::Byte(-4), + Some(Tag::Int(15)), + Some(Tag::ByteArray(vec![7; 2047])), + ), + section(Tag::Byte(0), None, Some(Tag::ByteArray(varied))), + ])); + } + #[test] fn level_round_trips_through_the_vanilla_files() { let dir = std::env::temp_dir().join(format!("horizon-level-{}", std::process::id())); let _ = fs::remove_dir_all(&dir); diff --git a/docs/saved-light-loading-benchmark.md b/docs/saved-light-loading-benchmark.md new file mode 100644 index 000000000..f11d72443 --- /dev/null +++ b/docs/saved-light-loading-benchmark.md @@ -0,0 +1,44 @@ +# Saved-light loading benchmark + +Saved-light reconstruction reduces complete saved-chunk loading by a median **6.179% on the normal world** and **10.981% on the flat world**, incremental to the accepted palette-loading optimization. Every measured process pair improves. The implementation changes only `save.rs::load_light` and its tests. + +## Implementation and scope + +Detect uniform stored nibble layers before allocating expanded resident light. Both nibbles must match: repeated bytes such as `0xF0` and `0x0F` remain nonuniform. Decode two resident light values per stored byte for remaining layers. Preserve section compaction, low-nibble-first indexing, missing-block darkness, missing-sky bottom-row inheritance, skyless behavior, section bounds, last duplicate section, invalid array handling, and loaded flags. Pinned Minecraft 26.3 `DataLayer` and `SkyLightSectionStorage.repeatFirstLayer` bytecode was inspected first. No saved-light optimization overlap was found among the 34 open PRs checked on 2026-10-22. + +## Measurements + +Baseline `411c5574` includes accepted palette loading and aquifer changes; the candidate differs only in saved-light reconstruction. Both loader executables were rebuilt from those sources using identical benchmark source (SHA256 `b6f5042731d7c9d5622758043f7c1e125a3f7d098c57ee0501012474c2df7325`), dependencies, resources and flags. Machine: AMD Ryzen 5 5600X, 6 cores/12 logical CPUs, 32 GiB RAM; Linux 7.2.11-zen1-1-zen; Rust 1.99.0. Release optimization, thin LTO, one codegen unit, client default features disabled; CPU boost enabled, powersave governor, ordinary desktop activity. No builds, tests or profiles overlapped timing. + +Each corpus uses one discarded process pair followed by five measured pairs in alternating baseline/candidate order. Each observation is a process's median of five rounds loading the same sorted 256 full chunks. Checksumming and setup are outside timing; allocation/destruction and region reads/decompression/NBT/native decode/saved light are inside complete loading. Storage is warm. Reductions are medians/ranges of paired reductions; independently reported medians need not give the same ratio. Separate stage loops do not sum to a timing breakdown. + +| Corpus | Complete load, baseline → candidate | Paired reduction (range) | Inflate/NBT, baseline → candidate | Native decode, baseline → candidate | +| --- | ---: | ---: | ---: | ---: | +| Normal vanilla 26.3, seed 12345 | 50.036 → 47.005 ms | 6.179% (5.217–6.836%) | 22.187 → 22.087 ms | 20.464 → 21.621 ms | +| Flat vanilla 26.3 | 12.030 → 10.688 ms | 10.981% (10.423–11.751%) | 4.562 → 4.549 ms | 4.076 → 4.183 ms | + +Values are milliseconds per complete 256-chunk batch. The separate native-decode loop slows by a paired median 4.644% on normal and 3.185% on flat; the flat observations are variable. Its source is unchanged, and the cause of this separate-loop change was not isolated. Complete `SaveDir::load`, which includes native decode and saved light, improves in all ten measured pairs. The earlier two-pair normal pilot improved 5.894% and 6.553%; final-source qualification was repeated after the iterator syntax change. All timed compressed-record and block/biome/light hashes agree. Normal compressed records: `ff9bf9e822e334e117e3d6ba5c26a88b7698456ec6d83bdbe4355ad0af1d6f55`; block/biome/light: `661dd26eed78b051afeca11d8ba69222162b5da72e5d1572a12ce2a3b3a4f764`. + +## Correctness + +All 20 save unit tests pass. The new differential test compares the original loader with the candidate for every repeated packed byte, all 256 pairs of uniform sky/block nibble values, nonuniform inherited rows, missing/wrong layer types and lengths, skyless dimensions, out-of-range/invalid section Y values and duplicate sections. It compares every resident light value, uniform/full representation, block/biome storage and loaded flags. + +A fresh pinned Java 26.3 oracle uses `DataLayer.get` and `SkyLightSectionStorage.repeatFirstLayer` over all 256 repeated byte patterns in both sky modes. Baseline and candidate exactly match all **50,331,648 light values** plus flags (SHA256 `608bab077edc713110f827740cb5d9baf2af9c1a251ea2458441e2d981f25920`). + +The same comprehensive snapshot harness runs on the actual accepted-source baseline and final candidate: **400 normal chunks, 289 flat chunks, and two synthetic unknown-NBT/nonempty-tick cases**. All 691 outputs match byte-for-byte. It covers resident blocks/biomes/light, complete reencoded NBT, heightmaps, block entities and their saved sources/order, structures, unknown NBT, ticks, postprocessing, entities, names and loaded/arrival flags. Light representation is additionally covered by the unit test. Temporary diagnostic sources were removed. + +The final source uses the fixed-size chunk iterator syntax required by Clippy. Its rebuilt executable (SHA256 `40d7031e7d3efe785815b1286af05553807bbc59f66a9275a5f89191f0ecda7e`) is the one used for final qualification; snapshot/oracle harnesses use the same production source. Earlier pilot artifacts are retained separately: section extraction normalized ELF metadata (original timed SHA256 `eb28209fd54b6deedab1ff701c291ae36b8116b2ba52a02d1ec82600b57e427f`; normalized SHA256 `78f2092b2661655fdea8d0fa2247389ccc8f48f37dd9abfbb8d8864867187297`). Those earlier artifacts are superseded by the final-source qualification. + +The required repository quick self-check passes: public-tree validation, formatting, Clippy with warnings denied, **1,443 client/core/world tests**, and the doc-test step. Command: `RUST_TEST_THREADS=1 CARGO_BUILD_JOBS=6 CARGO_TARGET_DIR=/srv/bob-data/horizon-chunk-campaign-check-target python3 tools/pr_ready.py --quick --base origin/main` (11 min 15 s). Tests were serialized after an initial client-test process SIGSEGV; the failed log is retained as `pr-ready-attempt1.txt`. Relevant oracle checks were run separately; `--quick` lists other affected parity harnesses for CI. + +## Reproduction and evidence + +Private/generated evidence: `/srv/bob-data/horizon-chunk-campaign-41055168/saved-light/`. Preserved baseline/candidate sources and executables, source/binary hashes, open-PR catalog, pinned bytecode, build logs, `final-pairs.py`, `final-raw.json`, final qualification logs and `final-summary.json`; earlier pilot/qualification artifacts are retained there. Resources: `/srv/bob-data/horizon/.local/horizon-client/26.3`. Normal corpus: `/srv/bob-data/horizon-chunk-campaign-41055168/vanilla-normal/world/dimensions/minecraft/overworld`; flat: `/srv/bob-data/horizon/.local/locator-bar-parity/generate/world/dimensions/minecraft/overworld`. + +```sh +CARGO_BUILD_JOBS=6 CARGO_TARGET_DIR=/srv/bob-data/horizon-chunk-campaign-target cargo build --locked --release -p horizon-client --no-default-features --example load_bench +load_bench RESOURCES DIMENSION_DIR --rounds 5 --limit 256 --checksum +python3 /srv/bob-data/horizon-chunk-campaign-41055168/saved-light/final-pairs.py final 5 +``` + +These results cover the declared warm saved-load workloads. Cold physical I/O, remote chunk streams and complete gameplay parity are outside this comparison.