perf: load uniform saved light without expanding nibble layers
This commit is contained in:
@@ -1305,8 +1305,73 @@ pub fn load_light(tag: &Tag, chunk: &mut Chunk, sky_light: bool) {
|
||||
};
|
||||
layers[si as usize] = (layer("SkyLight"), layer("BlockLight"));
|
||||
}
|
||||
let above = [if sky_light { 15 } else { 0 }; 256];
|
||||
crate::light::light_from_layers(chunk, &layers, above, sky_light);
|
||||
// Uniform saved layers need no expanded allocation. Both nibbles of
|
||||
// each byte must match, not merely the bytes across the layer.
|
||||
let uniform_layer = |layer: &[u8]| {
|
||||
let first = layer[0];
|
||||
(first & 15 == first >> 4 && layer.iter().all(|&v| v == first)).then_some(first & 15)
|
||||
};
|
||||
let mut above = [if sky_light { 15 } else { 0 }; 256];
|
||||
for (si, (sky, block)) in layers.into_iter().enumerate().rev() {
|
||||
let sky = sky.filter(|_| sky_light);
|
||||
let sky_uniform = sky.map_or_else(
|
||||
|| above.iter().all(|&v| v == above[0]).then_some(above[0]),
|
||||
uniform_layer,
|
||||
);
|
||||
let block_uniform = block.map_or(Some(0), uniform_layer);
|
||||
let section = Arc::make_mut(&mut chunk.sections[si]);
|
||||
if let (Some(s), Some(b)) = (sky_uniform, block_uniform) {
|
||||
section.light = Light::Uniform(s << 4 | b);
|
||||
above.fill(s);
|
||||
} else {
|
||||
let mut data = Box::new([0u8; 4096]);
|
||||
match (sky, block) {
|
||||
(Some(sky), Some(block)) => {
|
||||
for ((pair, &s), &b) in
|
||||
data.as_chunks_mut::<2>().0.iter_mut().zip(sky).zip(block)
|
||||
{
|
||||
pair[0] = s << 4 | (b & 15);
|
||||
pair[1] = (s & 0xf0) | (b >> 4);
|
||||
}
|
||||
}
|
||||
(Some(sky), None) => {
|
||||
for (pair, &s) in data.as_chunks_mut::<2>().0.iter_mut().zip(sky) {
|
||||
pair[0] = s << 4;
|
||||
pair[1] = s & 0xf0;
|
||||
}
|
||||
}
|
||||
(None, Some(block)) => {
|
||||
for (i, (pair, &b)) in data
|
||||
.as_chunks_mut::<2>()
|
||||
.0
|
||||
.iter_mut()
|
||||
.zip(block)
|
||||
.enumerate()
|
||||
{
|
||||
pair[0] = above[(i * 2) & 255] << 4 | (b & 15);
|
||||
pair[1] = above[(i * 2 + 1) & 255] << 4 | (b >> 4);
|
||||
}
|
||||
}
|
||||
(None, None) => {
|
||||
for layer in data.as_chunks_mut::<256>().0 {
|
||||
for (v, &s) in layer.iter_mut().zip(&above) {
|
||||
*v = s << 4;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(sky) = sky {
|
||||
for (pair, &s) in above.as_chunks_mut::<2>().0.iter_mut().zip(sky) {
|
||||
pair[0] = s & 15;
|
||||
pair[1] = s >> 4;
|
||||
}
|
||||
}
|
||||
section.light = Light::Full(data);
|
||||
}
|
||||
section.compact();
|
||||
}
|
||||
chunk.lit = true;
|
||||
chunk.light_loaded = true;
|
||||
}
|
||||
|
||||
type RegionCache = Mutex<HashMap<(i32, i32), Option<Arc<Mutex<RegionFile>>>>>;
|
||||
@@ -3892,6 +3957,151 @@ mod tests {
|
||||
assert!(!unlit.lit && !unlit.light_loaded);
|
||||
}
|
||||
#[test]
|
||||
fn saved_light_matches_reference_for_layers_and_metadata() {
|
||||
fn reference(tag: &Tag, chunk: &mut Chunk, sky_light: bool) {
|
||||
if tag.get("isLightOn") != Some(&Tag::Byte(1)) {
|
||||
return;
|
||||
}
|
||||
let Some(Tag::List(sections)) = tag.get("sections") else {
|
||||
return;
|
||||
};
|
||||
let mut layers: Vec<(Option<&[u8]>, Option<&[u8]>)> = vec![(None, None); SECTIONS];
|
||||
for section in sections {
|
||||
let Some(y) = number(section.get("Y")) else {
|
||||
continue;
|
||||
};
|
||||
let si = y as i32 - (MIN_Y >> 4);
|
||||
if !(0..SECTIONS as i32).contains(&si) {
|
||||
continue;
|
||||
}
|
||||
let layer = |name: &str| match section.get(name) {
|
||||
Some(Tag::ByteArray(a)) if a.len() == 2048 => Some(&a[..]),
|
||||
_ => None,
|
||||
};
|
||||
layers[si as usize] = (layer("SkyLight"), layer("BlockLight"));
|
||||
}
|
||||
let above = [if sky_light { 15 } else { 0 }; 256];
|
||||
crate::light::light_from_layers(chunk, &layers, above, sky_light);
|
||||
}
|
||||
|
||||
let section = |y: Tag, sky: Option<Tag>, block: Option<Tag>| {
|
||||
let mut entries = vec![("Y", y)];
|
||||
if let Some(v) = sky {
|
||||
entries.push(("SkyLight", v));
|
||||
}
|
||||
if let Some(v) = block {
|
||||
entries.push(("BlockLight", v));
|
||||
}
|
||||
Tag::compound(entries)
|
||||
};
|
||||
let root = |sections| {
|
||||
Tag::compound([
|
||||
("isLightOn", Tag::Byte(1)),
|
||||
("sections", Tag::List(sections)),
|
||||
])
|
||||
};
|
||||
let check = |tag: &Tag| {
|
||||
for sky_light in [false, true] {
|
||||
let mut expected = Chunk::empty(-3, 5);
|
||||
let s = Arc::make_mut(&mut expected.sections[0]);
|
||||
s.light = Light::Full(Box::new([0xab; 4096]));
|
||||
s.blocks.set(0, 1);
|
||||
s.blocks.set(0, 0);
|
||||
s.biomes.set(0, 1);
|
||||
s.biomes.set(0, 0);
|
||||
let mut actual = expected.clone();
|
||||
reference(tag, &mut expected, sky_light);
|
||||
load_light(tag, &mut actual, sky_light);
|
||||
assert_eq!(
|
||||
(actual.lit, actual.light_loaded),
|
||||
(expected.lit, expected.light_loaded)
|
||||
);
|
||||
for (a, b) in actual.sections.iter().zip(&expected.sections) {
|
||||
assert_eq!(
|
||||
std::mem::discriminant(&a.light),
|
||||
std::mem::discriminant(&b.light)
|
||||
);
|
||||
assert_eq!(
|
||||
format!("{:?}", a.blocks.to_wire()),
|
||||
format!("{:?}", b.blocks.to_wire())
|
||||
);
|
||||
assert_eq!(
|
||||
format!("{:?}", a.biomes.to_wire()),
|
||||
format!("{:?}", b.biomes.to_wire())
|
||||
);
|
||||
for i in 0..4096 {
|
||||
assert_eq!(a.light(i), b.light(i), "index {i}, sky {sky_light}");
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
for tag in [
|
||||
Tag::compound(Vec::<(&str, Tag)>::new()),
|
||||
Tag::compound([("isLightOn", Tag::Byte(0))]),
|
||||
Tag::compound([("isLightOn", Tag::Byte(2))]),
|
||||
Tag::compound([("isLightOn", Tag::Int(1)), ("sections", Tag::List(vec![]))]),
|
||||
Tag::compound([("isLightOn", Tag::Byte(1)), ("sections", Tag::Int(0))]),
|
||||
root(vec![]),
|
||||
] {
|
||||
check(&tag);
|
||||
}
|
||||
// Cover every repeated byte, including 0xF0/0x0F: a uniform byte
|
||||
// array need not hold uniform nibbles.
|
||||
for byte in 0..=255u8 {
|
||||
check(&root(vec![section(
|
||||
Tag::Byte(19),
|
||||
Some(Tag::ByteArray(vec![byte; 2048])),
|
||||
Some(Tag::ByteArray(vec![255 - byte; 2048])),
|
||||
)]));
|
||||
}
|
||||
// Every pair of uniform sky/block nibble values, with missing
|
||||
// sections below inheriting the stored bottom row.
|
||||
for sky in 0..16u8 {
|
||||
for block in 0..16u8 {
|
||||
check(&root(vec![section(
|
||||
Tag::Byte(1),
|
||||
Some(Tag::ByteArray(vec![sky * 17; 2048])),
|
||||
Some(Tag::ByteArray(vec![block * 17; 2048])),
|
||||
)]));
|
||||
}
|
||||
}
|
||||
for length in [0, 1, 2047, 2048, 2049] {
|
||||
for invalid in [
|
||||
Tag::Int(1),
|
||||
Tag::List(vec![]),
|
||||
Tag::ByteArray(vec![0xf0; length]),
|
||||
] {
|
||||
check(&root(vec![section(
|
||||
Tag::Byte(-4),
|
||||
Some(invalid.clone()),
|
||||
Some(invalid),
|
||||
)]));
|
||||
}
|
||||
}
|
||||
let varied: Vec<u8> = (0..2048)
|
||||
.map(|i| (i as u8).wrapping_mul(73).wrapping_add(31))
|
||||
.collect();
|
||||
check(&root(vec![
|
||||
section(Tag::Byte(-5), Some(Tag::ByteArray(varied.clone())), None),
|
||||
section(Tag::Byte(20), Some(Tag::ByteArray(varied.clone())), None),
|
||||
section(
|
||||
Tag::String("19".into()),
|
||||
Some(Tag::ByteArray(varied.clone())),
|
||||
None,
|
||||
),
|
||||
section(Tag::Byte(19), Some(Tag::ByteArray(varied.clone())), None),
|
||||
section(Tag::Byte(17), None, Some(Tag::ByteArray(varied.clone()))),
|
||||
section(Tag::Byte(-4), Some(Tag::ByteArray(varied.clone())), None),
|
||||
// Last duplicate wins, even when its arrays have wrong types.
|
||||
section(
|
||||
Tag::Byte(-4),
|
||||
Some(Tag::Int(15)),
|
||||
Some(Tag::ByteArray(vec![7; 2047])),
|
||||
),
|
||||
section(Tag::Byte(0), None, Some(Tag::ByteArray(varied))),
|
||||
]));
|
||||
}
|
||||
#[test]
|
||||
fn level_round_trips_through_the_vanilla_files() {
|
||||
let dir = std::env::temp_dir().join(format!("horizon-level-{}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# Saved-light loading benchmark
|
||||
|
||||
Saved-light reconstruction reduces complete saved-chunk loading by a median **6.179% on the normal world** and **10.981% on the flat world**, incremental to the accepted palette-loading optimization. Every measured process pair improves. The implementation changes only `save.rs::load_light` and its tests.
|
||||
|
||||
## Implementation and scope
|
||||
|
||||
Detect uniform stored nibble layers before allocating expanded resident light. Both nibbles must match: repeated bytes such as `0xF0` and `0x0F` remain nonuniform. Decode two resident light values per stored byte for remaining layers. Preserve section compaction, low-nibble-first indexing, missing-block darkness, missing-sky bottom-row inheritance, skyless behavior, section bounds, last duplicate section, invalid array handling, and loaded flags. Pinned Minecraft 26.3 `DataLayer` and `SkyLightSectionStorage.repeatFirstLayer` bytecode was inspected first. No saved-light optimization overlap was found among the 34 open PRs checked on 2026-10-22.
|
||||
|
||||
## Measurements
|
||||
|
||||
Baseline `411c5574` includes accepted palette loading and aquifer changes; the candidate differs only in saved-light reconstruction. Both loader executables were rebuilt from those sources using identical benchmark source (SHA256 `b6f5042731d7c9d5622758043f7c1e125a3f7d098c57ee0501012474c2df7325`), dependencies, resources and flags. Machine: AMD Ryzen 5 5600X, 6 cores/12 logical CPUs, 32 GiB RAM; Linux 7.2.11-zen1-1-zen; Rust 1.99.0. Release optimization, thin LTO, one codegen unit, client default features disabled; CPU boost enabled, powersave governor, ordinary desktop activity. No builds, tests or profiles overlapped timing.
|
||||
|
||||
Each corpus uses one discarded process pair followed by five measured pairs in alternating baseline/candidate order. Each observation is a process's median of five rounds loading the same sorted 256 full chunks. Checksumming and setup are outside timing; allocation/destruction and region reads/decompression/NBT/native decode/saved light are inside complete loading. Storage is warm. Reductions are medians/ranges of paired reductions; independently reported medians need not give the same ratio. Separate stage loops do not sum to a timing breakdown.
|
||||
|
||||
| Corpus | Complete load, baseline → candidate | Paired reduction (range) | Inflate/NBT, baseline → candidate | Native decode, baseline → candidate |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| Normal vanilla 26.3, seed 12345 | 50.036 → 47.005 ms | 6.179% (5.217–6.836%) | 22.187 → 22.087 ms | 20.464 → 21.621 ms |
|
||||
| Flat vanilla 26.3 | 12.030 → 10.688 ms | 10.981% (10.423–11.751%) | 4.562 → 4.549 ms | 4.076 → 4.183 ms |
|
||||
|
||||
Values are milliseconds per complete 256-chunk batch. The separate native-decode loop slows by a paired median 4.644% on normal and 3.185% on flat; the flat observations are variable. Its source is unchanged, and the cause of this separate-loop change was not isolated. Complete `SaveDir::load`, which includes native decode and saved light, improves in all ten measured pairs. The earlier two-pair normal pilot improved 5.894% and 6.553%; final-source qualification was repeated after the iterator syntax change. All timed compressed-record and block/biome/light hashes agree. Normal compressed records: `ff9bf9e822e334e117e3d6ba5c26a88b7698456ec6d83bdbe4355ad0af1d6f55`; block/biome/light: `661dd26eed78b051afeca11d8ba69222162b5da72e5d1572a12ce2a3b3a4f764`.
|
||||
|
||||
## Correctness
|
||||
|
||||
All 20 save unit tests pass. The new differential test compares the original loader with the candidate for every repeated packed byte, all 256 pairs of uniform sky/block nibble values, nonuniform inherited rows, missing/wrong layer types and lengths, skyless dimensions, out-of-range/invalid section Y values and duplicate sections. It compares every resident light value, uniform/full representation, block/biome storage and loaded flags.
|
||||
|
||||
A fresh pinned Java 26.3 oracle uses `DataLayer.get` and `SkyLightSectionStorage.repeatFirstLayer` over all 256 repeated byte patterns in both sky modes. Baseline and candidate exactly match all **50,331,648 light values** plus flags (SHA256 `608bab077edc713110f827740cb5d9baf2af9c1a251ea2458441e2d981f25920`).
|
||||
|
||||
The same comprehensive snapshot harness runs on the actual accepted-source baseline and final candidate: **400 normal chunks, 289 flat chunks, and two synthetic unknown-NBT/nonempty-tick cases**. All 691 outputs match byte-for-byte. It covers resident blocks/biomes/light, complete reencoded NBT, heightmaps, block entities and their saved sources/order, structures, unknown NBT, ticks, postprocessing, entities, names and loaded/arrival flags. Light representation is additionally covered by the unit test. Temporary diagnostic sources were removed.
|
||||
|
||||
The final source uses the fixed-size chunk iterator syntax required by Clippy. Its rebuilt executable (SHA256 `40d7031e7d3efe785815b1286af05553807bbc59f66a9275a5f89191f0ecda7e`) is the one used for final qualification; snapshot/oracle harnesses use the same production source. Earlier pilot artifacts are retained separately: section extraction normalized ELF metadata (original timed SHA256 `eb28209fd54b6deedab1ff701c291ae36b8116b2ba52a02d1ec82600b57e427f`; normalized SHA256 `78f2092b2661655fdea8d0fa2247389ccc8f48f37dd9abfbb8d8864867187297`). Those earlier artifacts are superseded by the final-source qualification.
|
||||
|
||||
The required repository quick self-check passes: public-tree validation, formatting, Clippy with warnings denied, **1,443 client/core/world tests**, and the doc-test step. Command: `RUST_TEST_THREADS=1 CARGO_BUILD_JOBS=6 CARGO_TARGET_DIR=/srv/bob-data/horizon-chunk-campaign-check-target python3 tools/pr_ready.py --quick --base origin/main` (11 min 15 s). Tests were serialized after an initial client-test process SIGSEGV; the failed log is retained as `pr-ready-attempt1.txt`. Relevant oracle checks were run separately; `--quick` lists other affected parity harnesses for CI.
|
||||
|
||||
## Reproduction and evidence
|
||||
|
||||
Private/generated evidence: `/srv/bob-data/horizon-chunk-campaign-41055168/saved-light/`. Preserved baseline/candidate sources and executables, source/binary hashes, open-PR catalog, pinned bytecode, build logs, `final-pairs.py`, `final-raw.json`, final qualification logs and `final-summary.json`; earlier pilot/qualification artifacts are retained there. Resources: `/srv/bob-data/horizon/.local/horizon-client/26.3`. Normal corpus: `/srv/bob-data/horizon-chunk-campaign-41055168/vanilla-normal/world/dimensions/minecraft/overworld`; flat: `/srv/bob-data/horizon/.local/locator-bar-parity/generate/world/dimensions/minecraft/overworld`.
|
||||
|
||||
```sh
|
||||
CARGO_BUILD_JOBS=6 CARGO_TARGET_DIR=/srv/bob-data/horizon-chunk-campaign-target cargo build --locked --release -p horizon-client --no-default-features --example load_bench
|
||||
load_bench RESOURCES DIMENSION_DIR --rounds 5 --limit 256 --checksum
|
||||
python3 /srv/bob-data/horizon-chunk-campaign-41055168/saved-light/final-pairs.py final 5
|
||||
```
|
||||
|
||||
These results cover the declared warm saved-load workloads. Cold physical I/O, remote chunk streams and complete gameplay parity are outside this comparison.
|
||||
Reference in New Issue
Block a user