From 90023be7c24730000aca28391e2b9bb76ca4ff74 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:02:16 +0200 Subject: [PATCH] Add a whole-chunk differential harness against vanilla generation The new oracle drives vanilla's own chunk stages (structure starts and references, biomes, terrain with surface and carvers, then biome decoration with every placed feature and structure piece) through real WorldGenRegion instances, and the chunk_dump example runs the same stages in the same order in Horizon. The Python runner compares every block state, quart biome and block entity over terrain sites and structure sites found by scanning each seed, in the overworld, Nether and End, with phases that stop before features or run structure pieces only. Co-Authored-By: Claude Sonnet 5.5 --- crates/horizon-world/examples/chunk_dump.rs | 222 ++++++++++ .../parity/VanillaChunkGenerationOracle.java | 404 ++++++++++++++++++ tools/test_chunk_generation_parity.py | 318 ++++++++++++++ 3 files changed, 944 insertions(+) create mode 100644 crates/horizon-world/examples/chunk_dump.rs create mode 100644 mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java create mode 100644 tools/test_chunk_generation_parity.py diff --git a/crates/horizon-world/examples/chunk_dump.rs b/crates/horizon-world/examples/chunk_dump.rs new file mode 100644 index 000000000..e7e440252 --- /dev/null +++ b/crates/horizon-world/examples/chunk_dump.rs @@ -0,0 +1,222 @@ +//! Whole-chunk dump in the format of the vanilla chunk-generation oracle. +//! +//! chunk_dump ... +//! With HORIZON_SCAN set, one rectangle lists the structure starts in it. +//! +//! Chunks are generated through biomes, terrain and carvers, then decorated in +//! row-major order over the target rectangle grown by one chunk, each with its +//! eight neighbours so feature spill persists, exactly as the oracle does. +//! For every target chunk this prints `chunk X Z`, one `q QY NAME...` line per +//! quart layer (16 columns, z-major), one `y Y ID...` line per block row and +//! one `be X Y Z CANONICAL_NBT` line per block entity. +use horizon_core::position::ChunkPos; +use horizon_world::{ + DATA_VERSION, + nbt::Tag, + registry::BlockRegistry, + worldgen::generator::{GeneratorData, OverworldGenerator, ProtoChunk}, +}; +use serde_json::Value; +use std::{collections::HashMap, fmt::Write as _, fs, io::Write as _, path::Path}; + +type Error = Box; + +fn tag(v: &Value) -> Tag { + match v { + Value::String(s) => Tag::String(s.as_str().into()), + Value::Number(n) => Tag::Int(n.as_i64().unwrap() as i32), + Value::Array(a) => Tag::List(a.iter().map(tag).collect()), + Value::Object(o) => { + Tag::Compound(o.iter().map(|(k, v)| (k.as_str().into(), tag(v))).collect()) + } + _ => panic!("invalid registry metadata"), + } +} +fn registry(resources: &Path) -> Result<&'static BlockRegistry, Error> { + let facts: Value = serde_json::from_str(&fs::read_to_string(resources.join("facts.json"))?)?; + let states = facts["states"].as_array().ok_or("facts states")?; + let registry = BlockRegistry::from_nbt(&Tag::compound([ + ("DataVersion", Tag::Int(DATA_VERSION)), + ( + "states", + Tag::List(states.iter().map(|v| tag(&v[4])).collect()), + ), + ("defaults", tag(&facts["defaults"])), + ]))?; + Ok(Box::leak(Box::new(registry))) +} + +/// Type-prefixed scalars with sorted compound keys, as the oracle prints them. +fn canonical(tag: &Tag, out: &mut String) { + match tag { + Tag::End => {} + Tag::Byte(v) => write!(out, "b{v}").unwrap(), + Tag::Short(v) => write!(out, "s{v}").unwrap(), + Tag::Int(v) => write!(out, "i{v}").unwrap(), + Tag::Long(v) => write!(out, "l{v}").unwrap(), + Tag::Float(v) => write!(out, "f{}", *v as i32).unwrap(), + Tag::Double(v) => write!(out, "d{}", *v as i64).unwrap(), + Tag::String(s) => { + let text = s + .to_string_lossy() + .replace('\\', "\\\\") + .replace('"', "\\\""); + write!(out, "\"{text}\"").unwrap(); + } + Tag::ByteArray(v) => { + out.push_str("B["); + for b in v { + write!(out, "{},", *b as i8).unwrap(); + } + out.push(']'); + } + Tag::IntArray(v) => { + out.push_str("I["); + for b in v { + write!(out, "{b},").unwrap(); + } + out.push(']'); + } + Tag::LongArray(v) => { + out.push_str("L["); + for b in v { + write!(out, "{b},").unwrap(); + } + out.push(']'); + } + Tag::List(v) => { + out.push('['); + for item in v { + canonical(item, out); + out.push(','); + } + out.push(']'); + } + Tag::Compound(c) => { + out.push('{'); + for (k, v) in c { + write!(out, "{}=", k.to_string_lossy()).unwrap(); + canonical(v, out); + out.push(','); + } + out.push('}'); + } + } +} + +fn main() -> Result<(), Error> { + let mut args = std::env::args().skip(1); + let resources = std::path::PathBuf::from(args.next().ok_or("client resource directory")?); + let seed: i64 = args.next().ok_or("seed")?.parse()?; + let dimension = format!("minecraft:{}", args.next().ok_or("dimension")?); + let rects = args + .map(|rect| { + let v: Vec = rect.split(',').map(str::parse).collect::>()?; + let v: [i32; 4] = v.try_into().map_err(|_| "rectangle x0,z0,x1,z1")?; + Ok(v) + }) + .collect::, Error>>()?; + let data = GeneratorData::load(&resources.join("worldgen"), registry(&resources)?)?; + let generator = OverworldGenerator::for_dimension(data, seed, "minecraft:normal", &dimension)?; + let mut worker = generator.worker(); + // HORIZON_PHASE: terrain (no decoration), structures (pieces only) or full. + let phase = std::env::var("HORIZON_PHASE").unwrap_or_else(|_| "full".into()); + worker.set_structures_only(phase == "structures"); + if std::env::var_os("HORIZON_SCAN").is_some() { + let [x0, z0, x1, z1] = rects[0]; + for x in x0..=x1 { + for z in z0..=z1 { + for s in worker.structure_starts(ChunkPos::new(x, z)) { + if s.chunk == ChunkPos::new(x, z) { + println!( + "start {} {x} {z} {} {} {} {} {} {}", + s.name, + s.bounding_box.min[0], + s.bounding_box.min[1], + s.bounding_box.min[2], + s.bounding_box.max[0], + s.bounding_box.max[1], + s.bounding_box.max[2] + ); + } + } + } + } + return Ok(()); + } + for [x0, z0, x1, z1] in rects { + let mut chunks: HashMap<(i32, i32), ProtoChunk> = HashMap::new(); + for z in z0 - 2..=z1 + 2 { + for x in x0 - 2..=x1 + 2 { + chunks.insert((x, z), worker.generate_proto(ChunkPos::new(x, z))?); + } + } + let mut failed = 0; + let margin = i32::from(std::env::var_os("HORIZON_ISOLATED").is_none()); + for z in z0 - margin..=z1 + margin { + for x in x0 - margin..=x1 + margin { + let mut region = Vec::with_capacity(9); + for dz in -1..=1 { + for dx in -1..=1 { + region.push(chunks.remove(&(x + dx, z + dz)).ok_or("missing chunk")?); + } + } + let mut region: [ProtoChunk; 9] = region.try_into().map_err(|_| "region size")?; + let stats = if phase == "terrain" { + Default::default() + } else { + worker.decorate(&mut region)? + }; + if stats.failed > 0 || stats.unsupported > 0 { + failed += 1; + eprintln!( + "chunk {x} {z}: {} failed, {} unsupported {:?}", + stats.failed, stats.unsupported, stats.skipped + ); + } + for (i, chunk) in region.into_iter().enumerate() { + chunks.insert((x + i as i32 % 3 - 1, z + i as i32 / 3 - 1), chunk); + } + } + } + if failed > 0 { + eprintln!("{failed} chunks had failed or unsupported features"); + } + let stdout = std::io::stdout(); + let mut w = std::io::BufWriter::new(stdout.lock()); + for z in z0..=z1 { + for x in x0..=x1 { + let chunk = &chunks[&(x, z)]; + writeln!(w, "chunk {x} {z}")?; + let (min, height) = (chunk.terrain.min_y(), chunk.terrain.height()); + for qy in min >> 2..=(min + height - 1) >> 2 { + let mut line = format!("q {qy}"); + for qz in 0..4 { + for qx in 0..4 { + line.push(' '); + line.push_str(chunk.biome_at(x * 16 + qx * 4, qy * 4, z * 16 + qz * 4)); + } + } + writeln!(w, "{line}")?; + } + for y in min..min + height { + let mut line = format!("y {y}"); + for zz in 0..16 { + for xx in 0..16 { + write!(line, " {}", chunk.block(x * 16 + xx, y, z * 16 + zz).0)?; + } + } + writeln!(w, "{line}")?; + } + let mut entities: Vec<_> = chunk.block_entities.iter().collect(); + entities.sort_by_key(|(p, _)| *p); + for ([bx, by, bz], nbt) in entities { + let mut text = String::new(); + canonical(&Tag::Compound(nbt.clone()), &mut text); + writeln!(w, "be {bx} {by} {bz}\t{text}")?; + } + } + } + } + Ok(()) +} diff --git a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java new file mode 100644 index 000000000..b381298ae --- /dev/null +++ b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java @@ -0,0 +1,404 @@ +package dev.horizonlab.parity; + +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import net.minecraft.SharedConstants; +import net.minecraft.core.*; +import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.core.registries.Registries; +import net.minecraft.resources.RegistryDataLoader; +import net.minecraft.resources.ResourceKey; +import net.minecraft.server.Bootstrap; +import net.minecraft.server.RegistryLayer; +import net.minecraft.server.level.GenerationChunkHolder; +import net.minecraft.server.level.ServerChunkCache; +import net.minecraft.server.level.ServerLevel; +import net.minecraft.server.level.WorldGenRegion; +import net.minecraft.server.packs.PackType; +import net.minecraft.server.packs.repository.ServerPacksSource; +import net.minecraft.server.packs.resources.MultiPackResourceManager; +import net.minecraft.tags.TagLoader; +import net.minecraft.util.StaticCache2D; +import net.minecraft.util.datafix.DataFixers; +import net.minecraft.world.level.*; +import net.minecraft.world.level.biome.*; +import net.minecraft.world.level.block.Block; +import net.minecraft.world.level.block.entity.BlockEntity; +import net.minecraft.world.level.chunk.*; +import net.minecraft.world.level.chunk.status.ChunkPyramid; +import net.minecraft.world.level.chunk.status.ChunkStatus; +import net.minecraft.world.level.levelgen.*; +import net.minecraft.world.level.levelgen.blending.Blender; +import net.minecraft.world.level.levelgen.structure.*; +import net.minecraft.world.level.levelgen.structure.templatesystem.StructureTemplateManager; +import net.minecraft.world.level.storage.LevelStorageSource; + +/** + * Original oracle: whole-chunk generation through vanilla's own pipeline. + * + * Chunks run through STRUCTURE_STARTS, STRUCTURE_REFERENCES, BIOMES, TERRAIN + * (noise, surface, carvers) and FEATURES exactly as the ChunkStatusTasks do, + * with actual WorldGenRegion instances over an allocation-only ServerLevel. + * FEATURES run in row-major order over the target rectangle grown by one chunk, + * so every target chunk has all eight neighbours decorated. Spill from + * neighbouring features therefore matches a canonical vanilla order. + * + * usage: dump OUT SEED DIMENSION X0,Z0,X1,Z1... (target chunks, inclusive) + * scan OUT SEED DIMENSION X0 Z0 X1 Z1 (structure starts per chunk) + * dump takes any number of X0,Z0,X1,Z1 rectangles, each generated from scratch. + */ +public final class VanillaChunkGenerationOracle { + private static final class OracleCache extends ServerChunkCache { + private OracleCache() { + super(null, null, null, null, null, null, 0, 0, false, null); + } + static ChunkGeneratorStructureState state; + static RandomState random; + static ChunkGenerator generator; + @Override + public ChunkGeneratorStructureState getGeneratorState() { + return state; + } + @Override + public RandomState randomState() { + return random; + } + @Override + public ChunkGenerator getGenerator() { + return generator; + } + } + + private static final class OracleLevel extends ServerLevel { + private OracleLevel() { + super(null, null, null, null, null, null, false, 0L, null, false); + } + static long seed; + static RegistryAccess access; + static net.minecraft.world.level.dimension.DimensionType type; + static StructureManager structures; + static BiomeResolver resolver; + static ServerChunkCache cache; + static ResourceKey key; + static int minY, height; + static Map chunks; + static net.minecraft.world.level.lighting.LevelLightEngine light; + @Override + public net.minecraft.world.level.lighting.LevelLightEngine getLightEngine() { + return light; + } + @Override + public long getSeed() { + return seed; + } + @Override + public net.minecraft.world.flag.FeatureFlagSet enabledFeatures() { + return net.minecraft.world.flag.FeatureFlagSet.of(); + } + @Override + public RegistryAccess registryAccess() { + return access; + } + @Override + public net.minecraft.world.level.dimension.DimensionType dimensionType() { + return type; + } + @Override + public StructureManager structureManager() { + return structures; + } + @Override + public BiomeResolver uncachedBiomeResolver() { + return resolver; + } + @Override + public ServerChunkCache getChunkSource() { + return cache; + } + @Override + public ResourceKey dimension() { + return key; + } + @Override + public net.minecraft.world.level.storage.ServerLevelData getLevelData() { + return DATA; + } + static final net.minecraft.world.level.storage.ServerLevelData DATA = (net.minecraft.world.level.storage.ServerLevelData) java.lang.reflect.Proxy.newProxyInstance( + OracleLevel.class.getClassLoader(), new Class[] {net.minecraft.world.level.storage.ServerLevelData.class}, (proxy, method, values) -> { + Class type = method.getReturnType(); + if (type == long.class) return 0L; + if (type == int.class) return 0; + if (type == boolean.class) return false; + if (type == float.class) return 0f; + if (type == double.class) return 0d; + return null; + }); + @Override + public int getMinY() { + return minY; + } + @Override + public int getHeight() { + return height; + } + @Override + public int getSeaLevel() { + return OracleCache.generator.getSeaLevel(); + } + @Override + public void updatePOIOnBlockStateChange(net.minecraft.core.BlockPos pos, net.minecraft.world.level.block.state.BlockState a, net.minecraft.world.level.block.state.BlockState b) {} + @Override + public ChunkAccess getChunk(int x, int z, ChunkStatus status, boolean load) { + return chunks.get(ChunkPos.pack(x, z)); + } + } + + private static final class ChunkHolder extends GenerationChunkHolder { + final ProtoChunk chunk; + ChunkHolder(ProtoChunk chunk) { + super(chunk.getPos()); + this.chunk = chunk; + } + @Override + public ChunkAccess getChunkIfPresentUnchecked(ChunkStatus status) { + return status.isOrBefore(chunk.getPersistedStatus()) ? chunk : null; + } + @Override + protected void addSaveDependency(CompletableFuture sync) {} + @Override + public int getTicketLevel() { + return 0; + } + @Override + public int getQueueLevel() { + return 0; + } + } + + public static void main(String[] args) throws Exception { + SharedConstants.tryDetectVersion(); + Bootstrap.bootStrap(); + String mode = args[0]; + Path out = Path.of(args[1]); + long seed = Long.parseLong(args[2]); + String dimension = args[3]; + var repo = ServerPacksSource.createVanillaTrustedRepository(); + repo.reload(); + repo.setSelected(List.of("vanilla")); + var resources = new MultiPackResourceManager(PackType.SERVER_DATA, repo.openAllSelected()); + var layers = RegistryLayer.createRegistryAccess(); + var staticTags = TagLoader.loadTagsForExistingRegistries(resources, layers.getLayer(RegistryLayer.STATIC)); + var context = TagLoader.buildUpdatedLookups(layers.getAccessForLoading(RegistryLayer.WORLD), staticTags); + RegistryAccess.Frozen world = RegistryDataLoader.load(resources, context, RegistryDataLoader.WORLD_REGISTRIES, Runnable::run).join(); + var loaded = net.minecraft.server.ReloadableServerRegistries.reload(layers.replaceFrom(RegistryLayer.WORLD, world), staticTags, resources, Runnable::run).join(); + var components = BuiltInRegistries.DATA_COMPONENT_INITIALIZERS.build(loaded.lookupWithUpdatedTags()); + staticTags.forEach(Registry.PendingTags::apply); + components.forEach(net.minecraft.core.component.DataComponentInitializers.PendingComponents::apply); + RegistryAccess access = loaded.layers().compositeAccess(); + var settings = world.lookupOrThrow(Registries.NOISE_SETTINGS).getOrThrow(switch (dimension) { + case "the_nether" -> NoiseGeneratorSettings.NETHER; + case "the_end" -> NoiseGeneratorSettings.END; + default -> NoiseGeneratorSettings.OVERWORLD; + }); + var presets = world.lookupOrThrow(Registries.MULTI_NOISE_BIOME_SOURCE_PARAMETER_LIST); + BiomeSource biomeSource = switch (dimension) { + case "the_nether" -> MultiNoiseBiomeSource.createFromPreset(presets.getOrThrow(MultiNoiseBiomeSourceParameterLists.NETHER)); + case "the_end" -> TheEndBiomeSource.create(world.lookupOrThrow(Registries.BIOME)); + default -> MultiNoiseBiomeSource.createFromPreset(presets.getOrThrow(MultiNoiseBiomeSourceParameterLists.OVERWORLD)); + }; + ResourceKey levelKey = switch (dimension) { + case "the_nether" -> Level.NETHER; + case "the_end" -> Level.END; + default -> Level.OVERWORLD; + }; + var randomState = RandomState.create(world.lookupOrThrow(Registries.NOISE), seed, settings.value()); + var generator = new NoiseBasedChunkGenerator(biomeSource, settings); + String phase = System.getenv().getOrDefault("HORIZON_PHASE", "full"); + if (phase.equals("structures")) { + // Structure pieces only: every biome keeps its carvers but loses its placed features. + java.util.function.Function, BiomeGenerationSettings> noFeatures = biome -> { + var builder = new BiomeGenerationSettings.PlainBuilder(); + for (var carver : biome.value().getGenerationSettings().getCarvers()) builder.addCarver(carver); + return builder.build(); + }; + var field = ChunkGenerator.class.getDeclaredField("generationSettingsGetter"); + field.setAccessible(true); + field.set(generator, noFeatures); + } + var state = ChunkGeneratorStructureState.createForNormal(randomState, seed, new ChunkPos(0, 0), biomeSource, world.lookupOrThrow(Registries.STRUCTURE_SET)); + Path tmp = Files.createTempDirectory("horizon-chunks"); + var storage = LevelStorageSource.createDefault(tmp).createAccess("oracle"); + var templates = new StructureTemplateManager(resources, storage, DataFixers.getDataFixer(), BuiltInRegistries.BLOCK); + var dimensionType = world.lookupOrThrow(Registries.DIMENSION_TYPE).getOrThrow(switch (dimension) { + case "the_nether" -> net.minecraft.world.level.dimension.BuiltinDimensionTypes.NETHER; + case "the_end" -> net.minecraft.world.level.dimension.BuiltinDimensionTypes.END; + default -> net.minecraft.world.level.dimension.BuiltinDimensionTypes.OVERWORLD; + }).value(); + LevelHeightAccessor height = dimension.equals("overworld") ? LevelHeightAccessor.create(-64, 384) : LevelHeightAccessor.create(0, 256); + var factory = PalettedContainerFactory.create(access); + var unsafeField = Class.forName("sun.misc.Unsafe").getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + Object unsafe = unsafeField.get(null); + var allocate = unsafe.getClass().getMethod("allocateInstance", Class.class); + Map chunks = new HashMap<>(); + OracleCache.state = state; + OracleCache.random = randomState; + OracleCache.generator = generator; + OracleLevel.seed = seed; + OracleLevel.access = access; + OracleLevel.type = dimensionType; + OracleLevel.resolver = biomeSource.createUncachedResolver(randomState); + OracleLevel.cache = (ServerChunkCache) allocate.invoke(unsafe, OracleCache.class); + OracleLevel.key = levelKey; + OracleLevel.minY = height.getMinY(); + OracleLevel.height = height.getHeight(); + OracleLevel.chunks = chunks; + var blockGetter = (BlockGetter) java.lang.reflect.Proxy.newProxyInstance(BlockGetter.class.getClassLoader(), new Class[] {BlockGetter.class}, (proxy, method, values) -> { + switch (method.getName()) { + case "getMinY": return height.getMinY(); + case "getHeight": return height.getHeight(); + } + if (method.isDefault()) return java.lang.reflect.InvocationHandler.invokeDefault(proxy, method, values); + return null; + }); + var lightGetter = new LightChunkGetter() { + public LightChunk getChunkForLighting(int x, int z) { return null; } + public BlockGetter getLevel() { return blockGetter; } + }; + OracleLevel.light = new net.minecraft.world.level.lighting.LevelLightEngine(lightGetter, true, dimensionType.hasSkyLight()); + var check = new StructureCheck((pos, visitor) -> CompletableFuture.completedFuture(null), access, templates, levelKey, generator, randomState, height, biomeSource, seed, DataFixers.getDataFixer()); + var level = (ServerLevel) allocate.invoke(unsafe, OracleLevel.class); + OracleLevel.structures = new StructureManager(level, new WorldOptions(seed, true, false), check); + var pyramid = ChunkPyramid.GENERATION_PYRAMID; + java.util.function.BiFunction chunkAt = (x, z) -> chunks.computeIfAbsent(ChunkPos.pack(x, z), k -> new ProtoChunk(new ChunkPos(x, z), UpgradeData.EMPTY, height, factory, null)); + java.util.function.BiFunction regionFor = (status, chunk) -> { + var step = pyramid.getStepTo(status); + int radius = step.directDependencies().size() - 1; + var cache = StaticCache2D.create(chunk.getPos().x(), chunk.getPos().z(), radius, (x, z) -> new ChunkHolder(chunkAt.apply(x, z))); + return new WorldGenRegion(level, cache, step, chunk); + }; + Runnable noop = () -> {}; + try (var w = new PrintWriter(Files.newBufferedWriter(out, StandardCharsets.UTF_8))) { + if (mode.equals("scan")) { + int x0 = Integer.parseInt(args[4]), z0 = Integer.parseInt(args[5]), x1 = Integer.parseInt(args[6]), z1 = Integer.parseInt(args[7]); + for (int x = x0; x <= x1; x++) for (int z = z0; z <= z1; z++) { + var chunk = chunkAt.apply(x, z); + generator.createStructures(access, state, OracleLevel.structures, chunk, templates, levelKey); + for (var start : chunk.getAllStarts().values()) { + if (!start.isValid()) continue; + var box = start.getBoundingBox(); + var id = access.lookupOrThrow(Registries.STRUCTURE).getKey(start.getStructure()); + w.println("start " + id + " " + x + " " + z + " " + box.minX() + " " + box.minY() + " " + box.minZ() + " " + box.maxX() + " " + box.maxY() + " " + box.maxZ()); + } + } + return; + } + for (int rect = 4; rect < args.length; rect++) { + chunks.clear(); + String[] corners = args[rect].split(","); + int x0 = Integer.parseInt(corners[0]), z0 = Integer.parseInt(corners[1]), x1 = Integer.parseInt(corners[2]), z1 = Integer.parseInt(corners[3]); + // Margins: FEATURES need TERRAIN at 1, which needs BIOMES at 1 more; STRUCTURE_STARTS reach 8 further. + int m = System.getenv("HORIZON_ISOLATED") != null ? 0 : 1; // diagnostics: decorate the targets only + int fx0 = x0 - m, fz0 = z0 - m, fx1 = x1 + m, fz1 = z1 + m; + int tx0 = fx0 - 1, tz0 = fz0 - 1, tx1 = fx1 + 1, tz1 = fz1 + 1; + int bx0 = tx0 - 1, bz0 = tz0 - 1, bx1 = tx1 + 1, bz1 = tz1 + 1; + int sx0 = tx0 - 8, sz0 = tz0 - 8, sx1 = tx1 + 8, sz1 = tz1 + 8; + for (int z = sz0; z <= sz1; z++) for (int x = sx0; x <= sx1; x++) { + var chunk = chunkAt.apply(x, z); + generator.createStructures(access, state, OracleLevel.structures, chunk, templates, levelKey); + chunk.setPersistedStatus(ChunkStatus.STRUCTURE_STARTS); + } + for (int z = tz0; z <= tz1; z++) for (int x = tx0; x <= tx1; x++) { + var chunk = chunkAt.apply(x, z); + var region = regionFor.apply(ChunkStatus.STRUCTURE_REFERENCES, chunk); + generator.createReferences(region, OracleLevel.structures.forWorldGenRegion(region), chunk); + chunk.setPersistedStatus(ChunkStatus.STRUCTURE_REFERENCES); + } + for (int z = bz0; z <= bz1; z++) for (int x = bx0; x <= bx1; x++) { + var chunk = chunkAt.apply(x, z); + if (chunk.getPersistedStatus() != ChunkStatus.STRUCTURE_STARTS && chunk.getPersistedStatus() != ChunkStatus.EMPTY && chunk.getPersistedStatus() != ChunkStatus.STRUCTURE_REFERENCES) continue; + var region = regionFor.apply(ChunkStatus.BIOMES, chunk); + generator.createBiomes(randomState, Blender.empty(), OracleLevel.structures.forWorldGenRegion(region), chunk).join(); + chunk.setPersistedStatus(ChunkStatus.BIOMES); + } + for (int z = tz0; z <= tz1; z++) for (int x = tx0; x <= tx1; x++) { + var chunk = chunkAt.apply(x, z); + var region = regionFor.apply(ChunkStatus.TERRAIN, chunk); + Set> possible = new HashSet<>(); + for (int dz = -1; dz <= 1; dz++) for (int dx = -1; dx <= 1; dx++) region.getChunk(x + dx, z + dz).collectBiomesInPalette(possible); + generator.buildTerrain(chunk, Blender.empty(), randomState, OracleLevel.structures.forWorldGenRegion(region), region.getBiomeManager(), region, possible).join(); + Heightmap.primeHeightmaps(chunk, EnumSet.of(Heightmap.Types.MOTION_BLOCKING, Heightmap.Types.MOTION_BLOCKING_NO_LEAVES, Heightmap.Types.OCEAN_FLOOR, Heightmap.Types.WORLD_SURFACE)); + chunk.setPersistedStatus(ChunkStatus.TERRAIN); + } + for (int z = fz0; z <= fz1 && !phase.equals("terrain"); z++) for (int x = fx0; x <= fx1; x++) { + var chunk = chunkAt.apply(x, z); + var region = regionFor.apply(ChunkStatus.FEATURES, chunk); + generator.applyBiomeDecoration(region, chunk, OracleLevel.structures.forWorldGenRegion(region)); + chunk.setPersistedStatus(ChunkStatus.FEATURES); + } + var registry = access; + for (int z = z0; z <= z1; z++) for (int x = x0; x <= x1; x++) { + var target = chunkAt.apply(x, z); + w.println("chunk " + x + " " + z); + // Biomes: one line per quart layer, 16 columns in z-major order. + for (int qy = height.getMinY() >> 2; qy <= height.getMaxY() >> 2; qy++) { + StringBuilder line = new StringBuilder("q " + qy); + for (int qz = 0; qz < 4; qz++) for (int qx = 0; qx < 4; qx++) + line.append(' ').append(target.getNoiseBiome((x << 2) + qx, qy, (z << 2) + qz).unwrapKey().orElseThrow().identifier()); + w.println(line); + } + for (int y = height.getMinY(); y <= height.getMaxY(); y++) { + StringBuilder line = new StringBuilder("y " + y); + for (int zz = 0; zz < 16; zz++) for (int xx = 0; xx < 16; xx++) + line.append(' ').append(Block.getId(target.getBlockState(new BlockPos((x << 4) + xx, y, (z << 4) + zz)))); + w.println(line); + } + var positions = new TreeSet(Comparator.comparingInt(BlockPos::getX).thenComparingInt(BlockPos::getY).thenComparingInt(BlockPos::getZ)); + positions.addAll(target.getBlockEntitiesPos()); + for (var pos : positions) { + var tag = target.getBlockEntityNbtForSaving(pos, registry); + if (tag == null) continue; + var st = target.getBlockState(pos); + var fresh = ((net.minecraft.world.level.block.EntityBlock) st.getBlock()).newBlockEntity(pos, st).saveWithFullMetadata(registry); + if ("DUMMY".equals(tag.getStringOr("id", ""))) tag = fresh; + w.println("be " + pos.getX() + " " + pos.getY() + " " + pos.getZ() + "\t" + canonicalOf(tag) + "\t" + canonicalOf(fresh)); + } + } + } + } + storage.close(); + System.exit(0); + } + + private static String canonicalOf(net.minecraft.nbt.Tag tag) { + var out = new StringBuilder(); + canonical(tag, out); + return out.toString(); + } + + /** Canonical single-line form of a tag: type-prefixed scalars, sorted compound keys. */ + private static void canonical(net.minecraft.nbt.Tag tag, StringBuilder out) { + switch (tag) { + case net.minecraft.nbt.ByteTag t -> out.append("b").append(t.byteValue()); + case net.minecraft.nbt.ShortTag t -> out.append("s").append(t.shortValue()); + case net.minecraft.nbt.IntTag t -> out.append("i").append(t.intValue()); + case net.minecraft.nbt.LongTag t -> out.append("l").append(t.longValue()); + case net.minecraft.nbt.FloatTag t -> out.append("f").append(Float.floatToRawIntBits(t.floatValue())); + case net.minecraft.nbt.DoubleTag t -> out.append("d").append(Double.doubleToRawLongBits(t.doubleValue())); + case net.minecraft.nbt.StringTag t -> out.append('"').append(t.value().replace("\\", "\\\\").replace("\"", "\\\"")).append('"'); + case net.minecraft.nbt.ByteArrayTag t -> { out.append("B["); for (byte v : t.getAsByteArray()) out.append(v).append(','); out.append(']'); } + case net.minecraft.nbt.IntArrayTag t -> { out.append("I["); for (int v : t.getAsIntArray()) out.append(v).append(','); out.append(']'); } + case net.minecraft.nbt.LongArrayTag t -> { out.append("L["); for (long v : t.getAsLongArray()) out.append(v).append(','); out.append(']'); } + case net.minecraft.nbt.ListTag t -> { out.append('['); for (var item : t) { canonical(item, out); out.append(','); } out.append(']'); } + case net.minecraft.nbt.CompoundTag t -> { + out.append('{'); + for (String key : new TreeSet<>(t.keySet())) { out.append(key).append('='); canonical(t.get(key), out); out.append(','); } + out.append('}'); + } + default -> throw new IllegalStateException(tag.getClass().getName()); + } + } +} diff --git a/tools/test_chunk_generation_parity.py b/tools/test_chunk_generation_parity.py new file mode 100644 index 000000000..e7ee17aba --- /dev/null +++ b/tools/test_chunk_generation_parity.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Compare whole generated chunks block-for-block with vanilla. + +Vanilla side: VanillaChunkGenerationOracle drives the actual ChunkGenerator +stages (structure starts and references, biomes, terrain with surface and +carvers, then biome decoration with every placed feature and structure piece) +through real WorldGenRegion instances. Horizon side: the chunk_dump example +runs the same stages in the same order. Every block state, quart biome and +block entity (canonical NBT) of each target chunk is compared. + +Cases: terrain sites for several seeds across the overworld, the Nether and the +End, plus rectangles around vanilla structure starts found by scanning each +seed (villages, mineshafts, strongholds, ancient cities, trial chambers, +monuments, mansions, ...). + +usage: test_chunk_generation_parity.py [--seeds N] [--per-structure K] + [--filter REGEX] [--dimension overworld|the_nether|the_end] [--keep] +""" +import argparse +import json +import os +import random +import re +import subprocess +import sys +import time +from collections import Counter, defaultdict +from pathlib import Path + +try: + from . import lab, test_rust_parity as base +except ImportError: + import lab + import test_rust_parity as base + +SEEDS = [12345, -7046029254386353131, 987654321987, 4242, -1, 0x5DEECE66D, 20260607, 777] +RESOURCES = lab.LOCAL / 'horizon-client/26.3' +ORACLE = lab.ROOT / 'mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java' + + +def build_dir(): + path = lab.LOCAL / 'chunk-generation-parity' + path.mkdir(parents=True, exist_ok=True) + return path + + +class Oracle: + def __init__(self): + self.lock, self.jars, self.libraries = base.verify_artifacts() + self.dependencies = os.pathsep.join(map(str, [self.jars['server'], *self.libraries])) + self.classes = build_dir() / 'classes' + self.classes.mkdir(exist_ok=True) + result = subprocess.run(['javac', '--release', str(self.lock['minecraft']['java_major']), '-proc:none', '-cp', self.dependencies, + '-d', str(self.classes), str(ORACLE)], capture_output=True, text=True) + if result.returncode: + raise SystemExit('javac failed:\n' + result.stderr[-4000:]) + + def run(self, out, args): + result = subprocess.run(['java', '-Xmx8G', '-cp', str(self.classes) + os.pathsep + self.dependencies, + 'dev.horizonlab.parity.VanillaChunkGenerationOracle', *map(str, [args[0], out, *args[1:]])], + capture_output=True, text=True, env=dict(os.environ)) + if result.returncode: + lines = [l for l in (result.stdout + result.stderr).splitlines() if 'Exception' in l or 'Caused' in l or '\tat ' in l] + raise SystemExit('oracle failed: ' + ' | '.join(lines[:10])) + + +def rust_binary(): + subprocess.run(['cargo', 'build', '--release', '-q', '-p', 'horizon-world', '--example', 'chunk_dump'], check=True, cwd=lab.ROOT) + metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--no-deps', '--format-version', '1'], cwd=lab.ROOT, text=True)) + return Path(metadata['target_directory']) / 'release/examples/chunk_dump' + + +def run_rust(binary, out, seed, dimension, rects, scan=False): + env = dict(os.environ) + if scan: + env['HORIZON_SCAN'] = '1' + with open(out, 'w') as handle: + result = subprocess.run([str(binary), str(RESOURCES), str(seed), dimension, *rects], stdout=handle, stderr=subprocess.PIPE, text=True, env=env, cwd=lab.ROOT) + if result.returncode: + raise SystemExit('chunk_dump failed: ' + result.stderr[-2000:]) + return result.stderr + + +def parse(path): + chunks, current = {}, None + for line in Path(path).read_text().splitlines(): + if line.startswith('chunk '): + _, x, z = line.split() + current = (int(x), int(z)) + chunks[current] = {'q': {}, 'y': {}, 'be': {}} + elif line.startswith('q '): + _, q, row = line.split(' ', 2) + chunks[current]['q'][int(q)] = row + elif line.startswith('y '): + _, y, row = line.split(' ', 2) + chunks[current]['y'][int(y)] = row + elif line.startswith('be '): + head, *forms = line.split('\t') + parts = head.split(' ') + chunks[current]['be'][(int(parts[1]), int(parts[2]), int(parts[3]))] = forms + return chunks + + +def parse_nbt(text, i=0): + """Canonical NBT text -> Python values (typed scalars as (type, text) tuples).""" + c = text[i] + if c == '{': + i += 1 + value = {} + while text[i] != '}': + j = text.index('=', i) + key = text[i:j] + value[key], i = parse_nbt(text, j + 1) + i += 1 # comma + return value, i + 1 + if c == '[': + i += 1 + items = [] + while text[i] != ']': + item, i = parse_nbt(text, i) + items.append(item) + i += 1 + return items, i + 1 + if c == '"': + i += 1 + out = [] + while text[i] != '"': + if text[i] == '\\': + i += 1 + out.append(text[i]) + i += 1 + return ('str', ''.join(out)), i + 1 + if c in 'BIL' and text[i + 1] == '[': + j = text.index(']', i) + return (c, text[i + 2:j]), j + 1 + j = i + 1 + while j < len(text) and text[j] not in ',}]': + j += 1 + return (c, text[i + 1:j]), j + + +def nbt_matches(vanilla, rust, default): + """Horizon stores the placed data only; vanilla saves defaults too. Every key + Horizon sets must match, and every other vanilla key must be the fresh default.""" + if isinstance(vanilla, dict) and isinstance(rust, dict): + default = default if isinstance(default, dict) else {} + for key, value in vanilla.items(): + if key in ('x', 'y', 'z') or (key == 'components' and value == {}): + continue + if key in rust: + if not nbt_matches(value, rust[key], default.get(key)): + return False + elif value != default.get(key): + return False + return all(key in vanilla or key in ('x', 'y', 'z') for key in rust) + return vanilla == rust + + +def entity_matches(vanilla, rust): + if rust is None or vanilla is None: + return False + return nbt_matches(parse_nbt(vanilla[0])[0], parse_nbt(rust[0])[0], parse_nbt(vanilla[1])[0]) + + +def state_names(): + facts = json.loads((RESOURCES / 'facts.json').read_text()) + return [s[3].removeprefix('Block{').replace('}', '', 1) for s in facts['states']] + + +def compare(vanilla, rust, names, label, limit=6, quiet=False): + """Returns (differing chunks, differing blocks, differing biome cells, differing block entities).""" + chunks = blocks = biomes = entities = 0 + for pos in sorted(vanilla): + a, b = vanilla[pos], rust.get(pos) + if b is None: + print(f' {label} chunk {pos}: missing from Horizon output') + chunks += 1 + continue + block_diffs, biome_diffs, entity_diffs = [], [], [] + for y in sorted(a['y']): + ra, rb = a['y'][y], b['y'].get(y) + if ra == rb: + continue + for i, (u, v) in enumerate(zip(ra.split(' '), (rb or '').split(' '))): + if u != v: + block_diffs.append((pos[0] * 16 + i % 16, y, pos[1] * 16 + i // 16, names[int(u)], names[int(v)])) + for q in sorted(a['q']): + if a['q'][q] != b['q'].get(q): + for i, (u, v) in enumerate(zip(a['q'][q].split(' '), b['q'].get(q, '').split(' '))): + if u != v: + biome_diffs.append((pos[0] * 4 + i % 4, q, pos[1] * 4 + i // 4, u, v)) + for key in sorted(set(a['be']) | set(b['be'])): + if not entity_matches(a['be'].get(key), b['be'].get(key)): + entity_diffs.append((key, a['be'].get(key, [None])[0], (b['be'].get(key) or [None])[0])) + if block_diffs or biome_diffs or entity_diffs: + chunks += 1 + blocks += len(block_diffs) + biomes += len(biome_diffs) + entities += len(entity_diffs) + if quiet: + continue + print(f' {label} chunk {pos}: {len(block_diffs)} blocks, {len(biome_diffs)} biome cells, {len(entity_diffs)} block entities') + for d in block_diffs[:limit]: + print(f' block {d[0]} {d[1]} {d[2]}: vanilla {d[3]} / horizon {d[4]}') + for d in biome_diffs[:2]: + print(f' biome quart {d[0]} {d[1]} {d[2]}: vanilla {d[3]} / horizon {d[4]}') + for d in entity_diffs[:3]: + print(f' block entity {d[0]}: vanilla {d[1]} / horizon {d[2]}') + return chunks, blocks, biomes, entities + + +def scan(oracle, binary, seed, dimension, radius): + out = build_dir() / f'scan-{dimension}-{seed}.txt' + oracle.run(out, ['scan', seed, dimension, -radius, -radius, radius, radius]) + starts = [] + for line in out.read_text().splitlines(): + t = line.split() + if t and t[0] == 'start': + starts.append((t[1], int(t[2]), int(t[3]), tuple(map(int, t[4:10])))) + return starts + + +def pick_structures(starts, per_kind, rng): + by_kind = defaultdict(list) + for s in starts: + by_kind[s[0]].append(s) + picked = [] + for kind, items in sorted(by_kind.items()): + items.sort(key=lambda s: abs(s[1]) + abs(s[2])) + # Nearest first, then random ones, so that variants (village types) differ. + chosen = items[:1] + rng.sample(items[1:], min(per_kind - 1, len(items) - 1)) + picked.extend(chosen[:per_kind]) + return picked + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--seeds', type=int, default=3) + parser.add_argument('--per-structure', type=int, default=2) + parser.add_argument('--sites', type=int, default=6, help='terrain sites per overworld seed') + parser.add_argument('--radius', type=int, default=1, help='chunk radius around each site') + parser.add_argument('--scan-radius', type=int, default=180) + parser.add_argument('--limit', type=int, default=6, help='block differences shown per chunk') + parser.add_argument('--summary', action='store_true', help='one line per differing site') + parser.add_argument('--rect', action='append', help='compare only this x0,z0,x1,z1 chunk rectangle (repeatable)') + parser.add_argument('--filter', help='regex over structure names') + parser.add_argument('--dimension', action='append') + parser.add_argument('--phase', choices=['terrain', 'structures', 'full'], default='full', help='stop before features, or run structure pieces only') + parser.add_argument('--terrain-only', action='store_true', help='skip the structure sites') + parser.add_argument('--structures-only', action='store_true') + args = parser.parse_args() + os.environ['HORIZON_PHASE'] = args.phase + dimensions = args.dimension or ['overworld', 'the_nether', 'the_end'] + oracle = Oracle() + binary = rust_binary() + names = state_names() + build = build_dir() + totals = Counter() + biome_seen = set() + structure_seen = Counter() + started = time.time() + for seed in SEEDS[:args.seeds]: + rng = random.Random(seed) + for dimension in dimensions: + rects, labels = [], [] + if args.rect: + rects, labels = list(args.rect), [f'rect {r}' for r in args.rect] + r = args.radius + if not args.structures_only and not args.rect: + spread = {'overworld': 5000, 'the_nether': 1500, 'the_end': 1800}[dimension] + sites = [(0, 0)] + [(rng.randint(-spread, spread) // 16, rng.randint(-spread, spread) // 16) for _ in range(args.sites if dimension == 'overworld' else 2)] + for cx, cz in sites: + rects.append(f'{cx - r},{cz - r},{cx + r},{cz + r}') + labels.append(f'terrain {cx},{cz}') + if not args.terrain_only and not args.rect: + starts = scan(oracle, binary, seed, dimension, args.scan_radius if dimension == 'overworld' else 80) + for kind, cx, cz, box in pick_structures(starts, args.per_structure, rng): + if args.filter and not re.search(args.filter, kind): + continue + rects.append(f'{cx - r},{cz - r},{cx + r},{cz + r}') + labels.append(f'{kind} {cx},{cz}') + structure_seen[kind] += 1 + if not rects: + continue + tag = f'{dimension}-{seed}-{args.phase}' + ('-iso' if os.environ.get('HORIZON_ISOLATED') else '') + ('-rect' if args.rect else '') + vanilla_path, rust_path = build / f'vanilla-{tag}.txt', build / f'rust-{tag}.txt' + t0 = time.time() + if not (os.environ.get('REUSE_VANILLA') and vanilla_path.exists()): + oracle.run(vanilla_path, ['dump', seed, dimension, *rects]) + t1 = time.time() + stderr = run_rust(binary, rust_path, seed, dimension, rects) + t2 = time.time() + vanilla, rust = parse(vanilla_path), parse(rust_path) + for line in stderr.splitlines()[:5]: + print(' horizon:', line) + counts = [0, 0, 0, 0] + for rect, label in zip(rects, labels): + x0, z0, x1, z1 = map(int, rect.split(',')) + part = {k: v for k, v in vanilla.items() if x0 <= k[0] <= x1 and z0 <= k[1] <= z1} + one = compare(part, rust, names, f'{tag} {label}', limit=args.limit, quiet=args.summary) + if one[0]: + print(f'{tag} {label}: {one[0]} chunks differ ({one[1]} blocks, {one[2]} biome cells, {one[3]} block entities)') + counts = [a + b for a, b in zip(counts, one)] + totals.update(dict(zip(['chunks', 'blocks', 'biomes', 'entities'], counts))) + totals['compared'] += len(vanilla) + for chunk in vanilla.values(): + for row in chunk['q'].values(): + biome_seen.update(row) + print(f'{tag}: {len(rects)} sites, {len(vanilla)} chunks, differing chunks {counts[0]} (blocks {counts[1]}, biome cells {counts[2]}, block entities {counts[3]}); vanilla {t1 - t0:.0f}s, horizon {t2 - t1:.0f}s', flush=True) + print(f'Compared {totals["compared"]} chunks across {len(biome_seen)} biomes and {sum(structure_seen.values())} structure sites in {time.time() - started:.0f}s') + print('structure sites:', dict(sorted(structure_seen.items()))) + print(f'Differences: {totals["chunks"]} chunks, {totals["blocks"]} blocks, {totals["biomes"]} biome cells, {totals["entities"]} block entities') + if totals['chunks']: + sys.exit(1) + + +if __name__ == '__main__': + main()