diff --git a/crates/horizon-world/examples/chunk_dump.rs b/crates/horizon-world/examples/chunk_dump.rs new file mode 100644 index 000000000..e7e440252 --- /dev/null +++ b/crates/horizon-world/examples/chunk_dump.rs @@ -0,0 +1,222 @@ +//! Whole-chunk dump in the format of the vanilla chunk-generation oracle. +//! +//! chunk_dump ... +//! With HORIZON_SCAN set, one rectangle lists the structure starts in it. +//! +//! Chunks are generated through biomes, terrain and carvers, then decorated in +//! row-major order over the target rectangle grown by one chunk, each with its +//! eight neighbours so feature spill persists, exactly as the oracle does. +//! For every target chunk this prints `chunk X Z`, one `q QY NAME...` line per +//! quart layer (16 columns, z-major), one `y Y ID...` line per block row and +//! one `be X Y Z CANONICAL_NBT` line per block entity. +use horizon_core::position::ChunkPos; +use horizon_world::{ + DATA_VERSION, + nbt::Tag, + registry::BlockRegistry, + worldgen::generator::{GeneratorData, OverworldGenerator, ProtoChunk}, +}; +use serde_json::Value; +use std::{collections::HashMap, fmt::Write as _, fs, io::Write as _, path::Path}; + +type Error = Box; + +fn tag(v: &Value) -> Tag { + match v { + Value::String(s) => Tag::String(s.as_str().into()), + Value::Number(n) => Tag::Int(n.as_i64().unwrap() as i32), + Value::Array(a) => Tag::List(a.iter().map(tag).collect()), + Value::Object(o) => { + Tag::Compound(o.iter().map(|(k, v)| (k.as_str().into(), tag(v))).collect()) + } + _ => panic!("invalid registry metadata"), + } +} +fn registry(resources: &Path) -> Result<&'static BlockRegistry, Error> { + let facts: Value = serde_json::from_str(&fs::read_to_string(resources.join("facts.json"))?)?; + let states = facts["states"].as_array().ok_or("facts states")?; + let registry = BlockRegistry::from_nbt(&Tag::compound([ + ("DataVersion", Tag::Int(DATA_VERSION)), + ( + "states", + Tag::List(states.iter().map(|v| tag(&v[4])).collect()), + ), + ("defaults", tag(&facts["defaults"])), + ]))?; + Ok(Box::leak(Box::new(registry))) +} + +/// Type-prefixed scalars with sorted compound keys, as the oracle prints them. +fn canonical(tag: &Tag, out: &mut String) { + match tag { + Tag::End => {} + Tag::Byte(v) => write!(out, "b{v}").unwrap(), + Tag::Short(v) => write!(out, "s{v}").unwrap(), + Tag::Int(v) => write!(out, "i{v}").unwrap(), + Tag::Long(v) => write!(out, "l{v}").unwrap(), + Tag::Float(v) => write!(out, "f{}", *v as i32).unwrap(), + Tag::Double(v) => write!(out, "d{}", *v as i64).unwrap(), + Tag::String(s) => { + let text = s + .to_string_lossy() + .replace('\\', "\\\\") + .replace('"', "\\\""); + write!(out, "\"{text}\"").unwrap(); + } + Tag::ByteArray(v) => { + out.push_str("B["); + for b in v { + write!(out, "{},", *b as i8).unwrap(); + } + out.push(']'); + } + Tag::IntArray(v) => { + out.push_str("I["); + for b in v { + write!(out, "{b},").unwrap(); + } + out.push(']'); + } + Tag::LongArray(v) => { + out.push_str("L["); + for b in v { + write!(out, "{b},").unwrap(); + } + out.push(']'); + } + Tag::List(v) => { + out.push('['); + for item in v { + canonical(item, out); + out.push(','); + } + out.push(']'); + } + Tag::Compound(c) => { + out.push('{'); + for (k, v) in c { + write!(out, "{}=", k.to_string_lossy()).unwrap(); + canonical(v, out); + out.push(','); + } + out.push('}'); + } + } +} + +fn main() -> Result<(), Error> { + let mut args = std::env::args().skip(1); + let resources = std::path::PathBuf::from(args.next().ok_or("client resource directory")?); + let seed: i64 = args.next().ok_or("seed")?.parse()?; + let dimension = format!("minecraft:{}", args.next().ok_or("dimension")?); + let rects = args + .map(|rect| { + let v: Vec = rect.split(',').map(str::parse).collect::>()?; + let v: [i32; 4] = v.try_into().map_err(|_| "rectangle x0,z0,x1,z1")?; + Ok(v) + }) + .collect::, Error>>()?; + let data = GeneratorData::load(&resources.join("worldgen"), registry(&resources)?)?; + let generator = OverworldGenerator::for_dimension(data, seed, "minecraft:normal", &dimension)?; + let mut worker = generator.worker(); + // HORIZON_PHASE: terrain (no decoration), structures (pieces only) or full. + let phase = std::env::var("HORIZON_PHASE").unwrap_or_else(|_| "full".into()); + worker.set_structures_only(phase == "structures"); + if std::env::var_os("HORIZON_SCAN").is_some() { + let [x0, z0, x1, z1] = rects[0]; + for x in x0..=x1 { + for z in z0..=z1 { + for s in worker.structure_starts(ChunkPos::new(x, z)) { + if s.chunk == ChunkPos::new(x, z) { + println!( + "start {} {x} {z} {} {} {} {} {} {}", + s.name, + s.bounding_box.min[0], + s.bounding_box.min[1], + s.bounding_box.min[2], + s.bounding_box.max[0], + s.bounding_box.max[1], + s.bounding_box.max[2] + ); + } + } + } + } + return Ok(()); + } + for [x0, z0, x1, z1] in rects { + let mut chunks: HashMap<(i32, i32), ProtoChunk> = HashMap::new(); + for z in z0 - 2..=z1 + 2 { + for x in x0 - 2..=x1 + 2 { + chunks.insert((x, z), worker.generate_proto(ChunkPos::new(x, z))?); + } + } + let mut failed = 0; + let margin = i32::from(std::env::var_os("HORIZON_ISOLATED").is_none()); + for z in z0 - margin..=z1 + margin { + for x in x0 - margin..=x1 + margin { + let mut region = Vec::with_capacity(9); + for dz in -1..=1 { + for dx in -1..=1 { + region.push(chunks.remove(&(x + dx, z + dz)).ok_or("missing chunk")?); + } + } + let mut region: [ProtoChunk; 9] = region.try_into().map_err(|_| "region size")?; + let stats = if phase == "terrain" { + Default::default() + } else { + worker.decorate(&mut region)? + }; + if stats.failed > 0 || stats.unsupported > 0 { + failed += 1; + eprintln!( + "chunk {x} {z}: {} failed, {} unsupported {:?}", + stats.failed, stats.unsupported, stats.skipped + ); + } + for (i, chunk) in region.into_iter().enumerate() { + chunks.insert((x + i as i32 % 3 - 1, z + i as i32 / 3 - 1), chunk); + } + } + } + if failed > 0 { + eprintln!("{failed} chunks had failed or unsupported features"); + } + let stdout = std::io::stdout(); + let mut w = std::io::BufWriter::new(stdout.lock()); + for z in z0..=z1 { + for x in x0..=x1 { + let chunk = &chunks[&(x, z)]; + writeln!(w, "chunk {x} {z}")?; + let (min, height) = (chunk.terrain.min_y(), chunk.terrain.height()); + for qy in min >> 2..=(min + height - 1) >> 2 { + let mut line = format!("q {qy}"); + for qz in 0..4 { + for qx in 0..4 { + line.push(' '); + line.push_str(chunk.biome_at(x * 16 + qx * 4, qy * 4, z * 16 + qz * 4)); + } + } + writeln!(w, "{line}")?; + } + for y in min..min + height { + let mut line = format!("y {y}"); + for zz in 0..16 { + for xx in 0..16 { + write!(line, " {}", chunk.block(x * 16 + xx, y, z * 16 + zz).0)?; + } + } + writeln!(w, "{line}")?; + } + let mut entities: Vec<_> = chunk.block_entities.iter().collect(); + entities.sort_by_key(|(p, _)| *p); + for ([bx, by, bz], nbt) in entities { + let mut text = String::new(); + canonical(&Tag::Compound(nbt.clone()), &mut text); + writeln!(w, "be {bx} {by} {bz}\t{text}")?; + } + } + } + } + Ok(()) +} diff --git a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java new file mode 100644 index 000000000..b381298ae --- /dev/null +++ b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java @@ -0,0 +1,404 @@ +package dev.horizonlab.parity; + +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import net.minecraft.SharedConstants; +import net.minecraft.core.*; +import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.core.registries.Registries; +import net.minecraft.resources.RegistryDataLoader; +import net.minecraft.resources.ResourceKey; +import net.minecraft.server.Bootstrap; +import net.minecraft.server.RegistryLayer; +import net.minecraft.server.level.GenerationChunkHolder; +import net.minecraft.server.level.ServerChunkCache; +import net.minecraft.server.level.ServerLevel; +import net.minecraft.server.level.WorldGenRegion; +import net.minecraft.server.packs.PackType; +import net.minecraft.server.packs.repository.ServerPacksSource; +import net.minecraft.server.packs.resources.MultiPackResourceManager; +import net.minecraft.tags.TagLoader; +import net.minecraft.util.StaticCache2D; +import net.minecraft.util.datafix.DataFixers; +import net.minecraft.world.level.*; +import net.minecraft.world.level.biome.*; +import net.minecraft.world.level.block.Block; +import net.minecraft.world.level.block.entity.BlockEntity; +import net.minecraft.world.level.chunk.*; +import net.minecraft.world.level.chunk.status.ChunkPyramid; +import net.minecraft.world.level.chunk.status.ChunkStatus; +import net.minecraft.world.level.levelgen.*; +import net.minecraft.world.level.levelgen.blending.Blender; +import net.minecraft.world.level.levelgen.structure.*; +import net.minecraft.world.level.levelgen.structure.templatesystem.StructureTemplateManager; +import net.minecraft.world.level.storage.LevelStorageSource; + +/** + * Original oracle: whole-chunk generation through vanilla's own pipeline. + * + * Chunks run through STRUCTURE_STARTS, STRUCTURE_REFERENCES, BIOMES, TERRAIN + * (noise, surface, carvers) and FEATURES exactly as the ChunkStatusTasks do, + * with actual WorldGenRegion instances over an allocation-only ServerLevel. + * FEATURES run in row-major order over the target rectangle grown by one chunk, + * so every target chunk has all eight neighbours decorated. Spill from + * neighbouring features therefore matches a canonical vanilla order. + * + * usage: dump OUT SEED DIMENSION X0,Z0,X1,Z1... (target chunks, inclusive) + * scan OUT SEED DIMENSION X0 Z0 X1 Z1 (structure starts per chunk) + * dump takes any number of X0,Z0,X1,Z1 rectangles, each generated from scratch. + */ +public final class VanillaChunkGenerationOracle { + private static final class OracleCache extends ServerChunkCache { + private OracleCache() { + super(null, null, null, null, null, null, 0, 0, false, null); + } + static ChunkGeneratorStructureState state; + static RandomState random; + static ChunkGenerator generator; + @Override + public ChunkGeneratorStructureState getGeneratorState() { + return state; + } + @Override + public RandomState randomState() { + return random; + } + @Override + public ChunkGenerator getGenerator() { + return generator; + } + } + + private static final class OracleLevel extends ServerLevel { + private OracleLevel() { + super(null, null, null, null, null, null, false, 0L, null, false); + } + static long seed; + static RegistryAccess access; + static net.minecraft.world.level.dimension.DimensionType type; + static StructureManager structures; + static BiomeResolver resolver; + static ServerChunkCache cache; + static ResourceKey key; + static int minY, height; + static Map chunks; + static net.minecraft.world.level.lighting.LevelLightEngine light; + @Override + public net.minecraft.world.level.lighting.LevelLightEngine getLightEngine() { + return light; + } + @Override + public long getSeed() { + return seed; + } + @Override + public net.minecraft.world.flag.FeatureFlagSet enabledFeatures() { + return net.minecraft.world.flag.FeatureFlagSet.of(); + } + @Override + public RegistryAccess registryAccess() { + return access; + } + @Override + public net.minecraft.world.level.dimension.DimensionType dimensionType() { + return type; + } + @Override + public StructureManager structureManager() { + return structures; + } + @Override + public BiomeResolver uncachedBiomeResolver() { + return resolver; + } + @Override + public ServerChunkCache getChunkSource() { + return cache; + } + @Override + public ResourceKey dimension() { + return key; + } + @Override + public net.minecraft.world.level.storage.ServerLevelData getLevelData() { + return DATA; + } + static final net.minecraft.world.level.storage.ServerLevelData DATA = (net.minecraft.world.level.storage.ServerLevelData) java.lang.reflect.Proxy.newProxyInstance( + OracleLevel.class.getClassLoader(), new Class[] {net.minecraft.world.level.storage.ServerLevelData.class}, (proxy, method, values) -> { + Class type = method.getReturnType(); + if (type == long.class) return 0L; + if (type == int.class) return 0; + if (type == boolean.class) return false; + if (type == float.class) return 0f; + if (type == double.class) return 0d; + return null; + }); + @Override + public int getMinY() { + return minY; + } + @Override + public int getHeight() { + return height; + } + @Override + public int getSeaLevel() { + return OracleCache.generator.getSeaLevel(); + } + @Override + public void updatePOIOnBlockStateChange(net.minecraft.core.BlockPos pos, net.minecraft.world.level.block.state.BlockState a, net.minecraft.world.level.block.state.BlockState b) {} + @Override + public ChunkAccess getChunk(int x, int z, ChunkStatus status, boolean load) { + return chunks.get(ChunkPos.pack(x, z)); + } + } + + private static final class ChunkHolder extends GenerationChunkHolder { + final ProtoChunk chunk; + ChunkHolder(ProtoChunk chunk) { + super(chunk.getPos()); + this.chunk = chunk; + } + @Override + public ChunkAccess getChunkIfPresentUnchecked(ChunkStatus status) { + return status.isOrBefore(chunk.getPersistedStatus()) ? chunk : null; + } + @Override + protected void addSaveDependency(CompletableFuture sync) {} + @Override + public int getTicketLevel() { + return 0; + } + @Override + public int getQueueLevel() { + return 0; + } + } + + public static void main(String[] args) throws Exception { + SharedConstants.tryDetectVersion(); + Bootstrap.bootStrap(); + String mode = args[0]; + Path out = Path.of(args[1]); + long seed = Long.parseLong(args[2]); + String dimension = args[3]; + var repo = ServerPacksSource.createVanillaTrustedRepository(); + repo.reload(); + repo.setSelected(List.of("vanilla")); + var resources = new MultiPackResourceManager(PackType.SERVER_DATA, repo.openAllSelected()); + var layers = RegistryLayer.createRegistryAccess(); + var staticTags = TagLoader.loadTagsForExistingRegistries(resources, layers.getLayer(RegistryLayer.STATIC)); + var context = TagLoader.buildUpdatedLookups(layers.getAccessForLoading(RegistryLayer.WORLD), staticTags); + RegistryAccess.Frozen world = RegistryDataLoader.load(resources, context, RegistryDataLoader.WORLD_REGISTRIES, Runnable::run).join(); + var loaded = net.minecraft.server.ReloadableServerRegistries.reload(layers.replaceFrom(RegistryLayer.WORLD, world), staticTags, resources, Runnable::run).join(); + var components = BuiltInRegistries.DATA_COMPONENT_INITIALIZERS.build(loaded.lookupWithUpdatedTags()); + staticTags.forEach(Registry.PendingTags::apply); + components.forEach(net.minecraft.core.component.DataComponentInitializers.PendingComponents::apply); + RegistryAccess access = loaded.layers().compositeAccess(); + var settings = world.lookupOrThrow(Registries.NOISE_SETTINGS).getOrThrow(switch (dimension) { + case "the_nether" -> NoiseGeneratorSettings.NETHER; + case "the_end" -> NoiseGeneratorSettings.END; + default -> NoiseGeneratorSettings.OVERWORLD; + }); + var presets = world.lookupOrThrow(Registries.MULTI_NOISE_BIOME_SOURCE_PARAMETER_LIST); + BiomeSource biomeSource = switch (dimension) { + case "the_nether" -> MultiNoiseBiomeSource.createFromPreset(presets.getOrThrow(MultiNoiseBiomeSourceParameterLists.NETHER)); + case "the_end" -> TheEndBiomeSource.create(world.lookupOrThrow(Registries.BIOME)); + default -> MultiNoiseBiomeSource.createFromPreset(presets.getOrThrow(MultiNoiseBiomeSourceParameterLists.OVERWORLD)); + }; + ResourceKey levelKey = switch (dimension) { + case "the_nether" -> Level.NETHER; + case "the_end" -> Level.END; + default -> Level.OVERWORLD; + }; + var randomState = RandomState.create(world.lookupOrThrow(Registries.NOISE), seed, settings.value()); + var generator = new NoiseBasedChunkGenerator(biomeSource, settings); + String phase = System.getenv().getOrDefault("HORIZON_PHASE", "full"); + if (phase.equals("structures")) { + // Structure pieces only: every biome keeps its carvers but loses its placed features. + java.util.function.Function, BiomeGenerationSettings> noFeatures = biome -> { + var builder = new BiomeGenerationSettings.PlainBuilder(); + for (var carver : biome.value().getGenerationSettings().getCarvers()) builder.addCarver(carver); + return builder.build(); + }; + var field = ChunkGenerator.class.getDeclaredField("generationSettingsGetter"); + field.setAccessible(true); + field.set(generator, noFeatures); + } + var state = ChunkGeneratorStructureState.createForNormal(randomState, seed, new ChunkPos(0, 0), biomeSource, world.lookupOrThrow(Registries.STRUCTURE_SET)); + Path tmp = Files.createTempDirectory("horizon-chunks"); + var storage = LevelStorageSource.createDefault(tmp).createAccess("oracle"); + var templates = new StructureTemplateManager(resources, storage, DataFixers.getDataFixer(), BuiltInRegistries.BLOCK); + var dimensionType = world.lookupOrThrow(Registries.DIMENSION_TYPE).getOrThrow(switch (dimension) { + case "the_nether" -> net.minecraft.world.level.dimension.BuiltinDimensionTypes.NETHER; + case "the_end" -> net.minecraft.world.level.dimension.BuiltinDimensionTypes.END; + default -> net.minecraft.world.level.dimension.BuiltinDimensionTypes.OVERWORLD; + }).value(); + LevelHeightAccessor height = dimension.equals("overworld") ? LevelHeightAccessor.create(-64, 384) : LevelHeightAccessor.create(0, 256); + var factory = PalettedContainerFactory.create(access); + var unsafeField = Class.forName("sun.misc.Unsafe").getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + Object unsafe = unsafeField.get(null); + var allocate = unsafe.getClass().getMethod("allocateInstance", Class.class); + Map chunks = new HashMap<>(); + OracleCache.state = state; + OracleCache.random = randomState; + OracleCache.generator = generator; + OracleLevel.seed = seed; + OracleLevel.access = access; + OracleLevel.type = dimensionType; + OracleLevel.resolver = biomeSource.createUncachedResolver(randomState); + OracleLevel.cache = (ServerChunkCache) allocate.invoke(unsafe, OracleCache.class); + OracleLevel.key = levelKey; + OracleLevel.minY = height.getMinY(); + OracleLevel.height = height.getHeight(); + OracleLevel.chunks = chunks; + var blockGetter = (BlockGetter) java.lang.reflect.Proxy.newProxyInstance(BlockGetter.class.getClassLoader(), new Class[] {BlockGetter.class}, (proxy, method, values) -> { + switch (method.getName()) { + case "getMinY": return height.getMinY(); + case "getHeight": return height.getHeight(); + } + if (method.isDefault()) return java.lang.reflect.InvocationHandler.invokeDefault(proxy, method, values); + return null; + }); + var lightGetter = new LightChunkGetter() { + public LightChunk getChunkForLighting(int x, int z) { return null; } + public BlockGetter getLevel() { return blockGetter; } + }; + OracleLevel.light = new net.minecraft.world.level.lighting.LevelLightEngine(lightGetter, true, dimensionType.hasSkyLight()); + var check = new StructureCheck((pos, visitor) -> CompletableFuture.completedFuture(null), access, templates, levelKey, generator, randomState, height, biomeSource, seed, DataFixers.getDataFixer()); + var level = (ServerLevel) allocate.invoke(unsafe, OracleLevel.class); + OracleLevel.structures = new StructureManager(level, new WorldOptions(seed, true, false), check); + var pyramid = ChunkPyramid.GENERATION_PYRAMID; + java.util.function.BiFunction chunkAt = (x, z) -> chunks.computeIfAbsent(ChunkPos.pack(x, z), k -> new ProtoChunk(new ChunkPos(x, z), UpgradeData.EMPTY, height, factory, null)); + java.util.function.BiFunction regionFor = (status, chunk) -> { + var step = pyramid.getStepTo(status); + int radius = step.directDependencies().size() - 1; + var cache = StaticCache2D.create(chunk.getPos().x(), chunk.getPos().z(), radius, (x, z) -> new ChunkHolder(chunkAt.apply(x, z))); + return new WorldGenRegion(level, cache, step, chunk); + }; + Runnable noop = () -> {}; + try (var w = new PrintWriter(Files.newBufferedWriter(out, StandardCharsets.UTF_8))) { + if (mode.equals("scan")) { + int x0 = Integer.parseInt(args[4]), z0 = Integer.parseInt(args[5]), x1 = Integer.parseInt(args[6]), z1 = Integer.parseInt(args[7]); + for (int x = x0; x <= x1; x++) for (int z = z0; z <= z1; z++) { + var chunk = chunkAt.apply(x, z); + generator.createStructures(access, state, OracleLevel.structures, chunk, templates, levelKey); + for (var start : chunk.getAllStarts().values()) { + if (!start.isValid()) continue; + var box = start.getBoundingBox(); + var id = access.lookupOrThrow(Registries.STRUCTURE).getKey(start.getStructure()); + w.println("start " + id + " " + x + " " + z + " " + box.minX() + " " + box.minY() + " " + box.minZ() + " " + box.maxX() + " " + box.maxY() + " " + box.maxZ()); + } + } + return; + } + for (int rect = 4; rect < args.length; rect++) { + chunks.clear(); + String[] corners = args[rect].split(","); + int x0 = Integer.parseInt(corners[0]), z0 = Integer.parseInt(corners[1]), x1 = Integer.parseInt(corners[2]), z1 = Integer.parseInt(corners[3]); + // Margins: FEATURES need TERRAIN at 1, which needs BIOMES at 1 more; STRUCTURE_STARTS reach 8 further. + int m = System.getenv("HORIZON_ISOLATED") != null ? 0 : 1; // diagnostics: decorate the targets only + int fx0 = x0 - m, fz0 = z0 - m, fx1 = x1 + m, fz1 = z1 + m; + int tx0 = fx0 - 1, tz0 = fz0 - 1, tx1 = fx1 + 1, tz1 = fz1 + 1; + int bx0 = tx0 - 1, bz0 = tz0 - 1, bx1 = tx1 + 1, bz1 = tz1 + 1; + int sx0 = tx0 - 8, sz0 = tz0 - 8, sx1 = tx1 + 8, sz1 = tz1 + 8; + for (int z = sz0; z <= sz1; z++) for (int x = sx0; x <= sx1; x++) { + var chunk = chunkAt.apply(x, z); + generator.createStructures(access, state, OracleLevel.structures, chunk, templates, levelKey); + chunk.setPersistedStatus(ChunkStatus.STRUCTURE_STARTS); + } + for (int z = tz0; z <= tz1; z++) for (int x = tx0; x <= tx1; x++) { + var chunk = chunkAt.apply(x, z); + var region = regionFor.apply(ChunkStatus.STRUCTURE_REFERENCES, chunk); + generator.createReferences(region, OracleLevel.structures.forWorldGenRegion(region), chunk); + chunk.setPersistedStatus(ChunkStatus.STRUCTURE_REFERENCES); + } + for (int z = bz0; z <= bz1; z++) for (int x = bx0; x <= bx1; x++) { + var chunk = chunkAt.apply(x, z); + if (chunk.getPersistedStatus() != ChunkStatus.STRUCTURE_STARTS && chunk.getPersistedStatus() != ChunkStatus.EMPTY && chunk.getPersistedStatus() != ChunkStatus.STRUCTURE_REFERENCES) continue; + var region = regionFor.apply(ChunkStatus.BIOMES, chunk); + generator.createBiomes(randomState, Blender.empty(), OracleLevel.structures.forWorldGenRegion(region), chunk).join(); + chunk.setPersistedStatus(ChunkStatus.BIOMES); + } + for (int z = tz0; z <= tz1; z++) for (int x = tx0; x <= tx1; x++) { + var chunk = chunkAt.apply(x, z); + var region = regionFor.apply(ChunkStatus.TERRAIN, chunk); + Set> possible = new HashSet<>(); + for (int dz = -1; dz <= 1; dz++) for (int dx = -1; dx <= 1; dx++) region.getChunk(x + dx, z + dz).collectBiomesInPalette(possible); + generator.buildTerrain(chunk, Blender.empty(), randomState, OracleLevel.structures.forWorldGenRegion(region), region.getBiomeManager(), region, possible).join(); + Heightmap.primeHeightmaps(chunk, EnumSet.of(Heightmap.Types.MOTION_BLOCKING, Heightmap.Types.MOTION_BLOCKING_NO_LEAVES, Heightmap.Types.OCEAN_FLOOR, Heightmap.Types.WORLD_SURFACE)); + chunk.setPersistedStatus(ChunkStatus.TERRAIN); + } + for (int z = fz0; z <= fz1 && !phase.equals("terrain"); z++) for (int x = fx0; x <= fx1; x++) { + var chunk = chunkAt.apply(x, z); + var region = regionFor.apply(ChunkStatus.FEATURES, chunk); + generator.applyBiomeDecoration(region, chunk, OracleLevel.structures.forWorldGenRegion(region)); + chunk.setPersistedStatus(ChunkStatus.FEATURES); + } + var registry = access; + for (int z = z0; z <= z1; z++) for (int x = x0; x <= x1; x++) { + var target = chunkAt.apply(x, z); + w.println("chunk " + x + " " + z); + // Biomes: one line per quart layer, 16 columns in z-major order. + for (int qy = height.getMinY() >> 2; qy <= height.getMaxY() >> 2; qy++) { + StringBuilder line = new StringBuilder("q " + qy); + for (int qz = 0; qz < 4; qz++) for (int qx = 0; qx < 4; qx++) + line.append(' ').append(target.getNoiseBiome((x << 2) + qx, qy, (z << 2) + qz).unwrapKey().orElseThrow().identifier()); + w.println(line); + } + for (int y = height.getMinY(); y <= height.getMaxY(); y++) { + StringBuilder line = new StringBuilder("y " + y); + for (int zz = 0; zz < 16; zz++) for (int xx = 0; xx < 16; xx++) + line.append(' ').append(Block.getId(target.getBlockState(new BlockPos((x << 4) + xx, y, (z << 4) + zz)))); + w.println(line); + } + var positions = new TreeSet(Comparator.comparingInt(BlockPos::getX).thenComparingInt(BlockPos::getY).thenComparingInt(BlockPos::getZ)); + positions.addAll(target.getBlockEntitiesPos()); + for (var pos : positions) { + var tag = target.getBlockEntityNbtForSaving(pos, registry); + if (tag == null) continue; + var st = target.getBlockState(pos); + var fresh = ((net.minecraft.world.level.block.EntityBlock) st.getBlock()).newBlockEntity(pos, st).saveWithFullMetadata(registry); + if ("DUMMY".equals(tag.getStringOr("id", ""))) tag = fresh; + w.println("be " + pos.getX() + " " + pos.getY() + " " + pos.getZ() + "\t" + canonicalOf(tag) + "\t" + canonicalOf(fresh)); + } + } + } + } + storage.close(); + System.exit(0); + } + + private static String canonicalOf(net.minecraft.nbt.Tag tag) { + var out = new StringBuilder(); + canonical(tag, out); + return out.toString(); + } + + /** Canonical single-line form of a tag: type-prefixed scalars, sorted compound keys. */ + private static void canonical(net.minecraft.nbt.Tag tag, StringBuilder out) { + switch (tag) { + case net.minecraft.nbt.ByteTag t -> out.append("b").append(t.byteValue()); + case net.minecraft.nbt.ShortTag t -> out.append("s").append(t.shortValue()); + case net.minecraft.nbt.IntTag t -> out.append("i").append(t.intValue()); + case net.minecraft.nbt.LongTag t -> out.append("l").append(t.longValue()); + case net.minecraft.nbt.FloatTag t -> out.append("f").append(Float.floatToRawIntBits(t.floatValue())); + case net.minecraft.nbt.DoubleTag t -> out.append("d").append(Double.doubleToRawLongBits(t.doubleValue())); + case net.minecraft.nbt.StringTag t -> out.append('"').append(t.value().replace("\\", "\\\\").replace("\"", "\\\"")).append('"'); + case net.minecraft.nbt.ByteArrayTag t -> { out.append("B["); for (byte v : t.getAsByteArray()) out.append(v).append(','); out.append(']'); } + case net.minecraft.nbt.IntArrayTag t -> { out.append("I["); for (int v : t.getAsIntArray()) out.append(v).append(','); out.append(']'); } + case net.minecraft.nbt.LongArrayTag t -> { out.append("L["); for (long v : t.getAsLongArray()) out.append(v).append(','); out.append(']'); } + case net.minecraft.nbt.ListTag t -> { out.append('['); for (var item : t) { canonical(item, out); out.append(','); } out.append(']'); } + case net.minecraft.nbt.CompoundTag t -> { + out.append('{'); + for (String key : new TreeSet<>(t.keySet())) { out.append(key).append('='); canonical(t.get(key), out); out.append(','); } + out.append('}'); + } + default -> throw new IllegalStateException(tag.getClass().getName()); + } + } +} diff --git a/tools/test_chunk_generation_parity.py b/tools/test_chunk_generation_parity.py new file mode 100644 index 000000000..e7ee17aba --- /dev/null +++ b/tools/test_chunk_generation_parity.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Compare whole generated chunks block-for-block with vanilla. + +Vanilla side: VanillaChunkGenerationOracle drives the actual ChunkGenerator +stages (structure starts and references, biomes, terrain with surface and +carvers, then biome decoration with every placed feature and structure piece) +through real WorldGenRegion instances. Horizon side: the chunk_dump example +runs the same stages in the same order. Every block state, quart biome and +block entity (canonical NBT) of each target chunk is compared. + +Cases: terrain sites for several seeds across the overworld, the Nether and the +End, plus rectangles around vanilla structure starts found by scanning each +seed (villages, mineshafts, strongholds, ancient cities, trial chambers, +monuments, mansions, ...). + +usage: test_chunk_generation_parity.py [--seeds N] [--per-structure K] + [--filter REGEX] [--dimension overworld|the_nether|the_end] [--keep] +""" +import argparse +import json +import os +import random +import re +import subprocess +import sys +import time +from collections import Counter, defaultdict +from pathlib import Path + +try: + from . import lab, test_rust_parity as base +except ImportError: + import lab + import test_rust_parity as base + +SEEDS = [12345, -7046029254386353131, 987654321987, 4242, -1, 0x5DEECE66D, 20260607, 777] +RESOURCES = lab.LOCAL / 'horizon-client/26.3' +ORACLE = lab.ROOT / 'mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java' + + +def build_dir(): + path = lab.LOCAL / 'chunk-generation-parity' + path.mkdir(parents=True, exist_ok=True) + return path + + +class Oracle: + def __init__(self): + self.lock, self.jars, self.libraries = base.verify_artifacts() + self.dependencies = os.pathsep.join(map(str, [self.jars['server'], *self.libraries])) + self.classes = build_dir() / 'classes' + self.classes.mkdir(exist_ok=True) + result = subprocess.run(['javac', '--release', str(self.lock['minecraft']['java_major']), '-proc:none', '-cp', self.dependencies, + '-d', str(self.classes), str(ORACLE)], capture_output=True, text=True) + if result.returncode: + raise SystemExit('javac failed:\n' + result.stderr[-4000:]) + + def run(self, out, args): + result = subprocess.run(['java', '-Xmx8G', '-cp', str(self.classes) + os.pathsep + self.dependencies, + 'dev.horizonlab.parity.VanillaChunkGenerationOracle', *map(str, [args[0], out, *args[1:]])], + capture_output=True, text=True, env=dict(os.environ)) + if result.returncode: + lines = [l for l in (result.stdout + result.stderr).splitlines() if 'Exception' in l or 'Caused' in l or '\tat ' in l] + raise SystemExit('oracle failed: ' + ' | '.join(lines[:10])) + + +def rust_binary(): + subprocess.run(['cargo', 'build', '--release', '-q', '-p', 'horizon-world', '--example', 'chunk_dump'], check=True, cwd=lab.ROOT) + metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--no-deps', '--format-version', '1'], cwd=lab.ROOT, text=True)) + return Path(metadata['target_directory']) / 'release/examples/chunk_dump' + + +def run_rust(binary, out, seed, dimension, rects, scan=False): + env = dict(os.environ) + if scan: + env['HORIZON_SCAN'] = '1' + with open(out, 'w') as handle: + result = subprocess.run([str(binary), str(RESOURCES), str(seed), dimension, *rects], stdout=handle, stderr=subprocess.PIPE, text=True, env=env, cwd=lab.ROOT) + if result.returncode: + raise SystemExit('chunk_dump failed: ' + result.stderr[-2000:]) + return result.stderr + + +def parse(path): + chunks, current = {}, None + for line in Path(path).read_text().splitlines(): + if line.startswith('chunk '): + _, x, z = line.split() + current = (int(x), int(z)) + chunks[current] = {'q': {}, 'y': {}, 'be': {}} + elif line.startswith('q '): + _, q, row = line.split(' ', 2) + chunks[current]['q'][int(q)] = row + elif line.startswith('y '): + _, y, row = line.split(' ', 2) + chunks[current]['y'][int(y)] = row + elif line.startswith('be '): + head, *forms = line.split('\t') + parts = head.split(' ') + chunks[current]['be'][(int(parts[1]), int(parts[2]), int(parts[3]))] = forms + return chunks + + +def parse_nbt(text, i=0): + """Canonical NBT text -> Python values (typed scalars as (type, text) tuples).""" + c = text[i] + if c == '{': + i += 1 + value = {} + while text[i] != '}': + j = text.index('=', i) + key = text[i:j] + value[key], i = parse_nbt(text, j + 1) + i += 1 # comma + return value, i + 1 + if c == '[': + i += 1 + items = [] + while text[i] != ']': + item, i = parse_nbt(text, i) + items.append(item) + i += 1 + return items, i + 1 + if c == '"': + i += 1 + out = [] + while text[i] != '"': + if text[i] == '\\': + i += 1 + out.append(text[i]) + i += 1 + return ('str', ''.join(out)), i + 1 + if c in 'BIL' and text[i + 1] == '[': + j = text.index(']', i) + return (c, text[i + 2:j]), j + 1 + j = i + 1 + while j < len(text) and text[j] not in ',}]': + j += 1 + return (c, text[i + 1:j]), j + + +def nbt_matches(vanilla, rust, default): + """Horizon stores the placed data only; vanilla saves defaults too. Every key + Horizon sets must match, and every other vanilla key must be the fresh default.""" + if isinstance(vanilla, dict) and isinstance(rust, dict): + default = default if isinstance(default, dict) else {} + for key, value in vanilla.items(): + if key in ('x', 'y', 'z') or (key == 'components' and value == {}): + continue + if key in rust: + if not nbt_matches(value, rust[key], default.get(key)): + return False + elif value != default.get(key): + return False + return all(key in vanilla or key in ('x', 'y', 'z') for key in rust) + return vanilla == rust + + +def entity_matches(vanilla, rust): + if rust is None or vanilla is None: + return False + return nbt_matches(parse_nbt(vanilla[0])[0], parse_nbt(rust[0])[0], parse_nbt(vanilla[1])[0]) + + +def state_names(): + facts = json.loads((RESOURCES / 'facts.json').read_text()) + return [s[3].removeprefix('Block{').replace('}', '', 1) for s in facts['states']] + + +def compare(vanilla, rust, names, label, limit=6, quiet=False): + """Returns (differing chunks, differing blocks, differing biome cells, differing block entities).""" + chunks = blocks = biomes = entities = 0 + for pos in sorted(vanilla): + a, b = vanilla[pos], rust.get(pos) + if b is None: + print(f' {label} chunk {pos}: missing from Horizon output') + chunks += 1 + continue + block_diffs, biome_diffs, entity_diffs = [], [], [] + for y in sorted(a['y']): + ra, rb = a['y'][y], b['y'].get(y) + if ra == rb: + continue + for i, (u, v) in enumerate(zip(ra.split(' '), (rb or '').split(' '))): + if u != v: + block_diffs.append((pos[0] * 16 + i % 16, y, pos[1] * 16 + i // 16, names[int(u)], names[int(v)])) + for q in sorted(a['q']): + if a['q'][q] != b['q'].get(q): + for i, (u, v) in enumerate(zip(a['q'][q].split(' '), b['q'].get(q, '').split(' '))): + if u != v: + biome_diffs.append((pos[0] * 4 + i % 4, q, pos[1] * 4 + i // 4, u, v)) + for key in sorted(set(a['be']) | set(b['be'])): + if not entity_matches(a['be'].get(key), b['be'].get(key)): + entity_diffs.append((key, a['be'].get(key, [None])[0], (b['be'].get(key) or [None])[0])) + if block_diffs or biome_diffs or entity_diffs: + chunks += 1 + blocks += len(block_diffs) + biomes += len(biome_diffs) + entities += len(entity_diffs) + if quiet: + continue + print(f' {label} chunk {pos}: {len(block_diffs)} blocks, {len(biome_diffs)} biome cells, {len(entity_diffs)} block entities') + for d in block_diffs[:limit]: + print(f' block {d[0]} {d[1]} {d[2]}: vanilla {d[3]} / horizon {d[4]}') + for d in biome_diffs[:2]: + print(f' biome quart {d[0]} {d[1]} {d[2]}: vanilla {d[3]} / horizon {d[4]}') + for d in entity_diffs[:3]: + print(f' block entity {d[0]}: vanilla {d[1]} / horizon {d[2]}') + return chunks, blocks, biomes, entities + + +def scan(oracle, binary, seed, dimension, radius): + out = build_dir() / f'scan-{dimension}-{seed}.txt' + oracle.run(out, ['scan', seed, dimension, -radius, -radius, radius, radius]) + starts = [] + for line in out.read_text().splitlines(): + t = line.split() + if t and t[0] == 'start': + starts.append((t[1], int(t[2]), int(t[3]), tuple(map(int, t[4:10])))) + return starts + + +def pick_structures(starts, per_kind, rng): + by_kind = defaultdict(list) + for s in starts: + by_kind[s[0]].append(s) + picked = [] + for kind, items in sorted(by_kind.items()): + items.sort(key=lambda s: abs(s[1]) + abs(s[2])) + # Nearest first, then random ones, so that variants (village types) differ. + chosen = items[:1] + rng.sample(items[1:], min(per_kind - 1, len(items) - 1)) + picked.extend(chosen[:per_kind]) + return picked + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--seeds', type=int, default=3) + parser.add_argument('--per-structure', type=int, default=2) + parser.add_argument('--sites', type=int, default=6, help='terrain sites per overworld seed') + parser.add_argument('--radius', type=int, default=1, help='chunk radius around each site') + parser.add_argument('--scan-radius', type=int, default=180) + parser.add_argument('--limit', type=int, default=6, help='block differences shown per chunk') + parser.add_argument('--summary', action='store_true', help='one line per differing site') + parser.add_argument('--rect', action='append', help='compare only this x0,z0,x1,z1 chunk rectangle (repeatable)') + parser.add_argument('--filter', help='regex over structure names') + parser.add_argument('--dimension', action='append') + parser.add_argument('--phase', choices=['terrain', 'structures', 'full'], default='full', help='stop before features, or run structure pieces only') + parser.add_argument('--terrain-only', action='store_true', help='skip the structure sites') + parser.add_argument('--structures-only', action='store_true') + args = parser.parse_args() + os.environ['HORIZON_PHASE'] = args.phase + dimensions = args.dimension or ['overworld', 'the_nether', 'the_end'] + oracle = Oracle() + binary = rust_binary() + names = state_names() + build = build_dir() + totals = Counter() + biome_seen = set() + structure_seen = Counter() + started = time.time() + for seed in SEEDS[:args.seeds]: + rng = random.Random(seed) + for dimension in dimensions: + rects, labels = [], [] + if args.rect: + rects, labels = list(args.rect), [f'rect {r}' for r in args.rect] + r = args.radius + if not args.structures_only and not args.rect: + spread = {'overworld': 5000, 'the_nether': 1500, 'the_end': 1800}[dimension] + sites = [(0, 0)] + [(rng.randint(-spread, spread) // 16, rng.randint(-spread, spread) // 16) for _ in range(args.sites if dimension == 'overworld' else 2)] + for cx, cz in sites: + rects.append(f'{cx - r},{cz - r},{cx + r},{cz + r}') + labels.append(f'terrain {cx},{cz}') + if not args.terrain_only and not args.rect: + starts = scan(oracle, binary, seed, dimension, args.scan_radius if dimension == 'overworld' else 80) + for kind, cx, cz, box in pick_structures(starts, args.per_structure, rng): + if args.filter and not re.search(args.filter, kind): + continue + rects.append(f'{cx - r},{cz - r},{cx + r},{cz + r}') + labels.append(f'{kind} {cx},{cz}') + structure_seen[kind] += 1 + if not rects: + continue + tag = f'{dimension}-{seed}-{args.phase}' + ('-iso' if os.environ.get('HORIZON_ISOLATED') else '') + ('-rect' if args.rect else '') + vanilla_path, rust_path = build / f'vanilla-{tag}.txt', build / f'rust-{tag}.txt' + t0 = time.time() + if not (os.environ.get('REUSE_VANILLA') and vanilla_path.exists()): + oracle.run(vanilla_path, ['dump', seed, dimension, *rects]) + t1 = time.time() + stderr = run_rust(binary, rust_path, seed, dimension, rects) + t2 = time.time() + vanilla, rust = parse(vanilla_path), parse(rust_path) + for line in stderr.splitlines()[:5]: + print(' horizon:', line) + counts = [0, 0, 0, 0] + for rect, label in zip(rects, labels): + x0, z0, x1, z1 = map(int, rect.split(',')) + part = {k: v for k, v in vanilla.items() if x0 <= k[0] <= x1 and z0 <= k[1] <= z1} + one = compare(part, rust, names, f'{tag} {label}', limit=args.limit, quiet=args.summary) + if one[0]: + print(f'{tag} {label}: {one[0]} chunks differ ({one[1]} blocks, {one[2]} biome cells, {one[3]} block entities)') + counts = [a + b for a, b in zip(counts, one)] + totals.update(dict(zip(['chunks', 'blocks', 'biomes', 'entities'], counts))) + totals['compared'] += len(vanilla) + for chunk in vanilla.values(): + for row in chunk['q'].values(): + biome_seen.update(row) + print(f'{tag}: {len(rects)} sites, {len(vanilla)} chunks, differing chunks {counts[0]} (blocks {counts[1]}, biome cells {counts[2]}, block entities {counts[3]}); vanilla {t1 - t0:.0f}s, horizon {t2 - t1:.0f}s', flush=True) + print(f'Compared {totals["compared"]} chunks across {len(biome_seen)} biomes and {sum(structure_seen.values())} structure sites in {time.time() - started:.0f}s') + print('structure sites:', dict(sorted(structure_seen.items()))) + print(f'Differences: {totals["chunks"]} chunks, {totals["blocks"]} blocks, {totals["biomes"]} biome cells, {totals["entities"]} block entities') + if totals['chunks']: + sys.exit(1) + + +if __name__ == '__main__': + main()