Add a whole-chunk differential harness against vanilla generation

The new oracle drives vanilla's own chunk stages (structure starts and references, biomes, terrain with surface and carvers, then biome decoration with every placed feature and structure piece) through real WorldGenRegion instances, and the chunk_dump example runs the same stages in the same order in Horizon. The Python runner compares every block state, quart biome and block entity over terrain sites and structure sites found by scanning each seed, in the overworld, Nether and End, with phases that stop before features or run structure pieces only.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-09 14:02:16 +02:00
parent 2b618b1dfe
commit 90023be7c2
3 changed files with 944 additions and 0 deletions
+222
View File
@@ -0,0 +1,222 @@
//! Whole-chunk dump in the format of the vanilla chunk-generation oracle.
//!
//! chunk_dump <client resource dir> <seed> <dimension> <x0,z0,x1,z1>...
//! With HORIZON_SCAN set, one rectangle lists the structure starts in it.
//!
//! Chunks are generated through biomes, terrain and carvers, then decorated in
//! row-major order over the target rectangle grown by one chunk, each with its
//! eight neighbours so feature spill persists, exactly as the oracle does.
//! For every target chunk this prints `chunk X Z`, one `q QY NAME...` line per
//! quart layer (16 columns, z-major), one `y Y ID...` line per block row and
//! one `be X Y Z CANONICAL_NBT` line per block entity.
use horizon_core::position::ChunkPos;
use horizon_world::{
DATA_VERSION,
nbt::Tag,
registry::BlockRegistry,
worldgen::generator::{GeneratorData, OverworldGenerator, ProtoChunk},
};
use serde_json::Value;
use std::{collections::HashMap, fmt::Write as _, fs, io::Write as _, path::Path};
type Error = Box<dyn std::error::Error + Send + Sync>;
fn tag(v: &Value) -> Tag {
match v {
Value::String(s) => Tag::String(s.as_str().into()),
Value::Number(n) => Tag::Int(n.as_i64().unwrap() as i32),
Value::Array(a) => Tag::List(a.iter().map(tag).collect()),
Value::Object(o) => {
Tag::Compound(o.iter().map(|(k, v)| (k.as_str().into(), tag(v))).collect())
}
_ => panic!("invalid registry metadata"),
}
}
fn registry(resources: &Path) -> Result<&'static BlockRegistry, Error> {
let facts: Value = serde_json::from_str(&fs::read_to_string(resources.join("facts.json"))?)?;
let states = facts["states"].as_array().ok_or("facts states")?;
let registry = BlockRegistry::from_nbt(&Tag::compound([
("DataVersion", Tag::Int(DATA_VERSION)),
(
"states",
Tag::List(states.iter().map(|v| tag(&v[4])).collect()),
),
("defaults", tag(&facts["defaults"])),
]))?;
Ok(Box::leak(Box::new(registry)))
}
/// Type-prefixed scalars with sorted compound keys, as the oracle prints them.
fn canonical(tag: &Tag, out: &mut String) {
match tag {
Tag::End => {}
Tag::Byte(v) => write!(out, "b{v}").unwrap(),
Tag::Short(v) => write!(out, "s{v}").unwrap(),
Tag::Int(v) => write!(out, "i{v}").unwrap(),
Tag::Long(v) => write!(out, "l{v}").unwrap(),
Tag::Float(v) => write!(out, "f{}", *v as i32).unwrap(),
Tag::Double(v) => write!(out, "d{}", *v as i64).unwrap(),
Tag::String(s) => {
let text = s
.to_string_lossy()
.replace('\\', "\\\\")
.replace('"', "\\\"");
write!(out, "\"{text}\"").unwrap();
}
Tag::ByteArray(v) => {
out.push_str("B[");
for b in v {
write!(out, "{},", *b as i8).unwrap();
}
out.push(']');
}
Tag::IntArray(v) => {
out.push_str("I[");
for b in v {
write!(out, "{b},").unwrap();
}
out.push(']');
}
Tag::LongArray(v) => {
out.push_str("L[");
for b in v {
write!(out, "{b},").unwrap();
}
out.push(']');
}
Tag::List(v) => {
out.push('[');
for item in v {
canonical(item, out);
out.push(',');
}
out.push(']');
}
Tag::Compound(c) => {
out.push('{');
for (k, v) in c {
write!(out, "{}=", k.to_string_lossy()).unwrap();
canonical(v, out);
out.push(',');
}
out.push('}');
}
}
}
fn main() -> Result<(), Error> {
let mut args = std::env::args().skip(1);
let resources = std::path::PathBuf::from(args.next().ok_or("client resource directory")?);
let seed: i64 = args.next().ok_or("seed")?.parse()?;
let dimension = format!("minecraft:{}", args.next().ok_or("dimension")?);
let rects = args
.map(|rect| {
let v: Vec<i32> = rect.split(',').map(str::parse).collect::<Result<_, _>>()?;
let v: [i32; 4] = v.try_into().map_err(|_| "rectangle x0,z0,x1,z1")?;
Ok(v)
})
.collect::<Result<Vec<_>, Error>>()?;
let data = GeneratorData::load(&resources.join("worldgen"), registry(&resources)?)?;
let generator = OverworldGenerator::for_dimension(data, seed, "minecraft:normal", &dimension)?;
let mut worker = generator.worker();
// HORIZON_PHASE: terrain (no decoration), structures (pieces only) or full.
let phase = std::env::var("HORIZON_PHASE").unwrap_or_else(|_| "full".into());
worker.set_structures_only(phase == "structures");
if std::env::var_os("HORIZON_SCAN").is_some() {
let [x0, z0, x1, z1] = rects[0];
for x in x0..=x1 {
for z in z0..=z1 {
for s in worker.structure_starts(ChunkPos::new(x, z)) {
if s.chunk == ChunkPos::new(x, z) {
println!(
"start {} {x} {z} {} {} {} {} {} {}",
s.name,
s.bounding_box.min[0],
s.bounding_box.min[1],
s.bounding_box.min[2],
s.bounding_box.max[0],
s.bounding_box.max[1],
s.bounding_box.max[2]
);
}
}
}
}
return Ok(());
}
for [x0, z0, x1, z1] in rects {
let mut chunks: HashMap<(i32, i32), ProtoChunk> = HashMap::new();
for z in z0 - 2..=z1 + 2 {
for x in x0 - 2..=x1 + 2 {
chunks.insert((x, z), worker.generate_proto(ChunkPos::new(x, z))?);
}
}
let mut failed = 0;
let margin = i32::from(std::env::var_os("HORIZON_ISOLATED").is_none());
for z in z0 - margin..=z1 + margin {
for x in x0 - margin..=x1 + margin {
let mut region = Vec::with_capacity(9);
for dz in -1..=1 {
for dx in -1..=1 {
region.push(chunks.remove(&(x + dx, z + dz)).ok_or("missing chunk")?);
}
}
let mut region: [ProtoChunk; 9] = region.try_into().map_err(|_| "region size")?;
let stats = if phase == "terrain" {
Default::default()
} else {
worker.decorate(&mut region)?
};
if stats.failed > 0 || stats.unsupported > 0 {
failed += 1;
eprintln!(
"chunk {x} {z}: {} failed, {} unsupported {:?}",
stats.failed, stats.unsupported, stats.skipped
);
}
for (i, chunk) in region.into_iter().enumerate() {
chunks.insert((x + i as i32 % 3 - 1, z + i as i32 / 3 - 1), chunk);
}
}
}
if failed > 0 {
eprintln!("{failed} chunks had failed or unsupported features");
}
let stdout = std::io::stdout();
let mut w = std::io::BufWriter::new(stdout.lock());
for z in z0..=z1 {
for x in x0..=x1 {
let chunk = &chunks[&(x, z)];
writeln!(w, "chunk {x} {z}")?;
let (min, height) = (chunk.terrain.min_y(), chunk.terrain.height());
for qy in min >> 2..=(min + height - 1) >> 2 {
let mut line = format!("q {qy}");
for qz in 0..4 {
for qx in 0..4 {
line.push(' ');
line.push_str(chunk.biome_at(x * 16 + qx * 4, qy * 4, z * 16 + qz * 4));
}
}
writeln!(w, "{line}")?;
}
for y in min..min + height {
let mut line = format!("y {y}");
for zz in 0..16 {
for xx in 0..16 {
write!(line, " {}", chunk.block(x * 16 + xx, y, z * 16 + zz).0)?;
}
}
writeln!(w, "{line}")?;
}
let mut entities: Vec<_> = chunk.block_entities.iter().collect();
entities.sort_by_key(|(p, _)| *p);
for ([bx, by, bz], nbt) in entities {
let mut text = String::new();
canonical(&Tag::Compound(nbt.clone()), &mut text);
writeln!(w, "be {bx} {by} {bz}\t{text}")?;
}
}
}
}
Ok(())
}
@@ -0,0 +1,404 @@
package dev.horizonlab.parity;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import net.minecraft.SharedConstants;
import net.minecraft.core.*;
import net.minecraft.core.registries.BuiltInRegistries;
import net.minecraft.core.registries.Registries;
import net.minecraft.resources.RegistryDataLoader;
import net.minecraft.resources.ResourceKey;
import net.minecraft.server.Bootstrap;
import net.minecraft.server.RegistryLayer;
import net.minecraft.server.level.GenerationChunkHolder;
import net.minecraft.server.level.ServerChunkCache;
import net.minecraft.server.level.ServerLevel;
import net.minecraft.server.level.WorldGenRegion;
import net.minecraft.server.packs.PackType;
import net.minecraft.server.packs.repository.ServerPacksSource;
import net.minecraft.server.packs.resources.MultiPackResourceManager;
import net.minecraft.tags.TagLoader;
import net.minecraft.util.StaticCache2D;
import net.minecraft.util.datafix.DataFixers;
import net.minecraft.world.level.*;
import net.minecraft.world.level.biome.*;
import net.minecraft.world.level.block.Block;
import net.minecraft.world.level.block.entity.BlockEntity;
import net.minecraft.world.level.chunk.*;
import net.minecraft.world.level.chunk.status.ChunkPyramid;
import net.minecraft.world.level.chunk.status.ChunkStatus;
import net.minecraft.world.level.levelgen.*;
import net.minecraft.world.level.levelgen.blending.Blender;
import net.minecraft.world.level.levelgen.structure.*;
import net.minecraft.world.level.levelgen.structure.templatesystem.StructureTemplateManager;
import net.minecraft.world.level.storage.LevelStorageSource;
/**
* Original oracle: whole-chunk generation through vanilla's own pipeline.
*
* Chunks run through STRUCTURE_STARTS, STRUCTURE_REFERENCES, BIOMES, TERRAIN
* (noise, surface, carvers) and FEATURES exactly as the ChunkStatusTasks do,
* with actual WorldGenRegion instances over an allocation-only ServerLevel.
* FEATURES run in row-major order over the target rectangle grown by one chunk,
* so every target chunk has all eight neighbours decorated. Spill from
* neighbouring features therefore matches a canonical vanilla order.
*
* usage: dump OUT SEED DIMENSION X0,Z0,X1,Z1... (target chunks, inclusive)
* scan OUT SEED DIMENSION X0 Z0 X1 Z1 (structure starts per chunk)
* dump takes any number of X0,Z0,X1,Z1 rectangles, each generated from scratch.
*/
public final class VanillaChunkGenerationOracle {
private static final class OracleCache extends ServerChunkCache {
private OracleCache() {
super(null, null, null, null, null, null, 0, 0, false, null);
}
static ChunkGeneratorStructureState state;
static RandomState random;
static ChunkGenerator generator;
@Override
public ChunkGeneratorStructureState getGeneratorState() {
return state;
}
@Override
public RandomState randomState() {
return random;
}
@Override
public ChunkGenerator getGenerator() {
return generator;
}
}
private static final class OracleLevel extends ServerLevel {
private OracleLevel() {
super(null, null, null, null, null, null, false, 0L, null, false);
}
static long seed;
static RegistryAccess access;
static net.minecraft.world.level.dimension.DimensionType type;
static StructureManager structures;
static BiomeResolver resolver;
static ServerChunkCache cache;
static ResourceKey<Level> key;
static int minY, height;
static Map<Long, ProtoChunk> chunks;
static net.minecraft.world.level.lighting.LevelLightEngine light;
@Override
public net.minecraft.world.level.lighting.LevelLightEngine getLightEngine() {
return light;
}
@Override
public long getSeed() {
return seed;
}
@Override
public net.minecraft.world.flag.FeatureFlagSet enabledFeatures() {
return net.minecraft.world.flag.FeatureFlagSet.of();
}
@Override
public RegistryAccess registryAccess() {
return access;
}
@Override
public net.minecraft.world.level.dimension.DimensionType dimensionType() {
return type;
}
@Override
public StructureManager structureManager() {
return structures;
}
@Override
public BiomeResolver uncachedBiomeResolver() {
return resolver;
}
@Override
public ServerChunkCache getChunkSource() {
return cache;
}
@Override
public ResourceKey<Level> dimension() {
return key;
}
@Override
public net.minecraft.world.level.storage.ServerLevelData getLevelData() {
return DATA;
}
static final net.minecraft.world.level.storage.ServerLevelData DATA = (net.minecraft.world.level.storage.ServerLevelData) java.lang.reflect.Proxy.newProxyInstance(
OracleLevel.class.getClassLoader(), new Class<?>[] {net.minecraft.world.level.storage.ServerLevelData.class}, (proxy, method, values) -> {
Class<?> type = method.getReturnType();
if (type == long.class) return 0L;
if (type == int.class) return 0;
if (type == boolean.class) return false;
if (type == float.class) return 0f;
if (type == double.class) return 0d;
return null;
});
@Override
public int getMinY() {
return minY;
}
@Override
public int getHeight() {
return height;
}
@Override
public int getSeaLevel() {
return OracleCache.generator.getSeaLevel();
}
@Override
public void updatePOIOnBlockStateChange(net.minecraft.core.BlockPos pos, net.minecraft.world.level.block.state.BlockState a, net.minecraft.world.level.block.state.BlockState b) {}
@Override
public ChunkAccess getChunk(int x, int z, ChunkStatus status, boolean load) {
return chunks.get(ChunkPos.pack(x, z));
}
}
private static final class ChunkHolder extends GenerationChunkHolder {
final ProtoChunk chunk;
ChunkHolder(ProtoChunk chunk) {
super(chunk.getPos());
this.chunk = chunk;
}
@Override
public ChunkAccess getChunkIfPresentUnchecked(ChunkStatus status) {
return status.isOrBefore(chunk.getPersistedStatus()) ? chunk : null;
}
@Override
protected void addSaveDependency(CompletableFuture<?> sync) {}
@Override
public int getTicketLevel() {
return 0;
}
@Override
public int getQueueLevel() {
return 0;
}
}
public static void main(String[] args) throws Exception {
SharedConstants.tryDetectVersion();
Bootstrap.bootStrap();
String mode = args[0];
Path out = Path.of(args[1]);
long seed = Long.parseLong(args[2]);
String dimension = args[3];
var repo = ServerPacksSource.createVanillaTrustedRepository();
repo.reload();
repo.setSelected(List.of("vanilla"));
var resources = new MultiPackResourceManager(PackType.SERVER_DATA, repo.openAllSelected());
var layers = RegistryLayer.createRegistryAccess();
var staticTags = TagLoader.loadTagsForExistingRegistries(resources, layers.getLayer(RegistryLayer.STATIC));
var context = TagLoader.buildUpdatedLookups(layers.getAccessForLoading(RegistryLayer.WORLD), staticTags);
RegistryAccess.Frozen world = RegistryDataLoader.load(resources, context, RegistryDataLoader.WORLD_REGISTRIES, Runnable::run).join();
var loaded = net.minecraft.server.ReloadableServerRegistries.reload(layers.replaceFrom(RegistryLayer.WORLD, world), staticTags, resources, Runnable::run).join();
var components = BuiltInRegistries.DATA_COMPONENT_INITIALIZERS.build(loaded.lookupWithUpdatedTags());
staticTags.forEach(Registry.PendingTags::apply);
components.forEach(net.minecraft.core.component.DataComponentInitializers.PendingComponents::apply);
RegistryAccess access = loaded.layers().compositeAccess();
var settings = world.lookupOrThrow(Registries.NOISE_SETTINGS).getOrThrow(switch (dimension) {
case "the_nether" -> NoiseGeneratorSettings.NETHER;
case "the_end" -> NoiseGeneratorSettings.END;
default -> NoiseGeneratorSettings.OVERWORLD;
});
var presets = world.lookupOrThrow(Registries.MULTI_NOISE_BIOME_SOURCE_PARAMETER_LIST);
BiomeSource biomeSource = switch (dimension) {
case "the_nether" -> MultiNoiseBiomeSource.createFromPreset(presets.getOrThrow(MultiNoiseBiomeSourceParameterLists.NETHER));
case "the_end" -> TheEndBiomeSource.create(world.lookupOrThrow(Registries.BIOME));
default -> MultiNoiseBiomeSource.createFromPreset(presets.getOrThrow(MultiNoiseBiomeSourceParameterLists.OVERWORLD));
};
ResourceKey<Level> levelKey = switch (dimension) {
case "the_nether" -> Level.NETHER;
case "the_end" -> Level.END;
default -> Level.OVERWORLD;
};
var randomState = RandomState.create(world.lookupOrThrow(Registries.NOISE), seed, settings.value());
var generator = new NoiseBasedChunkGenerator(biomeSource, settings);
String phase = System.getenv().getOrDefault("HORIZON_PHASE", "full");
if (phase.equals("structures")) {
// Structure pieces only: every biome keeps its carvers but loses its placed features.
java.util.function.Function<Holder<Biome>, BiomeGenerationSettings> noFeatures = biome -> {
var builder = new BiomeGenerationSettings.PlainBuilder();
for (var carver : biome.value().getGenerationSettings().getCarvers()) builder.addCarver(carver);
return builder.build();
};
var field = ChunkGenerator.class.getDeclaredField("generationSettingsGetter");
field.setAccessible(true);
field.set(generator, noFeatures);
}
var state = ChunkGeneratorStructureState.createForNormal(randomState, seed, new ChunkPos(0, 0), biomeSource, world.lookupOrThrow(Registries.STRUCTURE_SET));
Path tmp = Files.createTempDirectory("horizon-chunks");
var storage = LevelStorageSource.createDefault(tmp).createAccess("oracle");
var templates = new StructureTemplateManager(resources, storage, DataFixers.getDataFixer(), BuiltInRegistries.BLOCK);
var dimensionType = world.lookupOrThrow(Registries.DIMENSION_TYPE).getOrThrow(switch (dimension) {
case "the_nether" -> net.minecraft.world.level.dimension.BuiltinDimensionTypes.NETHER;
case "the_end" -> net.minecraft.world.level.dimension.BuiltinDimensionTypes.END;
default -> net.minecraft.world.level.dimension.BuiltinDimensionTypes.OVERWORLD;
}).value();
LevelHeightAccessor height = dimension.equals("overworld") ? LevelHeightAccessor.create(-64, 384) : LevelHeightAccessor.create(0, 256);
var factory = PalettedContainerFactory.create(access);
var unsafeField = Class.forName("sun.misc.Unsafe").getDeclaredField("theUnsafe");
unsafeField.setAccessible(true);
Object unsafe = unsafeField.get(null);
var allocate = unsafe.getClass().getMethod("allocateInstance", Class.class);
Map<Long, ProtoChunk> chunks = new HashMap<>();
OracleCache.state = state;
OracleCache.random = randomState;
OracleCache.generator = generator;
OracleLevel.seed = seed;
OracleLevel.access = access;
OracleLevel.type = dimensionType;
OracleLevel.resolver = biomeSource.createUncachedResolver(randomState);
OracleLevel.cache = (ServerChunkCache) allocate.invoke(unsafe, OracleCache.class);
OracleLevel.key = levelKey;
OracleLevel.minY = height.getMinY();
OracleLevel.height = height.getHeight();
OracleLevel.chunks = chunks;
var blockGetter = (BlockGetter) java.lang.reflect.Proxy.newProxyInstance(BlockGetter.class.getClassLoader(), new Class<?>[] {BlockGetter.class}, (proxy, method, values) -> {
switch (method.getName()) {
case "getMinY": return height.getMinY();
case "getHeight": return height.getHeight();
}
if (method.isDefault()) return java.lang.reflect.InvocationHandler.invokeDefault(proxy, method, values);
return null;
});
var lightGetter = new LightChunkGetter() {
public LightChunk getChunkForLighting(int x, int z) { return null; }
public BlockGetter getLevel() { return blockGetter; }
};
OracleLevel.light = new net.minecraft.world.level.lighting.LevelLightEngine(lightGetter, true, dimensionType.hasSkyLight());
var check = new StructureCheck((pos, visitor) -> CompletableFuture.completedFuture(null), access, templates, levelKey, generator, randomState, height, biomeSource, seed, DataFixers.getDataFixer());
var level = (ServerLevel) allocate.invoke(unsafe, OracleLevel.class);
OracleLevel.structures = new StructureManager(level, new WorldOptions(seed, true, false), check);
var pyramid = ChunkPyramid.GENERATION_PYRAMID;
java.util.function.BiFunction<Integer, Integer, ProtoChunk> chunkAt = (x, z) -> chunks.computeIfAbsent(ChunkPos.pack(x, z), k -> new ProtoChunk(new ChunkPos(x, z), UpgradeData.EMPTY, height, factory, null));
java.util.function.BiFunction<ChunkStatus, ProtoChunk, WorldGenRegion> regionFor = (status, chunk) -> {
var step = pyramid.getStepTo(status);
int radius = step.directDependencies().size() - 1;
var cache = StaticCache2D.<GenerationChunkHolder>create(chunk.getPos().x(), chunk.getPos().z(), radius, (x, z) -> new ChunkHolder(chunkAt.apply(x, z)));
return new WorldGenRegion(level, cache, step, chunk);
};
Runnable noop = () -> {};
try (var w = new PrintWriter(Files.newBufferedWriter(out, StandardCharsets.UTF_8))) {
if (mode.equals("scan")) {
int x0 = Integer.parseInt(args[4]), z0 = Integer.parseInt(args[5]), x1 = Integer.parseInt(args[6]), z1 = Integer.parseInt(args[7]);
for (int x = x0; x <= x1; x++) for (int z = z0; z <= z1; z++) {
var chunk = chunkAt.apply(x, z);
generator.createStructures(access, state, OracleLevel.structures, chunk, templates, levelKey);
for (var start : chunk.getAllStarts().values()) {
if (!start.isValid()) continue;
var box = start.getBoundingBox();
var id = access.lookupOrThrow(Registries.STRUCTURE).getKey(start.getStructure());
w.println("start " + id + " " + x + " " + z + " " + box.minX() + " " + box.minY() + " " + box.minZ() + " " + box.maxX() + " " + box.maxY() + " " + box.maxZ());
}
}
return;
}
for (int rect = 4; rect < args.length; rect++) {
chunks.clear();
String[] corners = args[rect].split(",");
int x0 = Integer.parseInt(corners[0]), z0 = Integer.parseInt(corners[1]), x1 = Integer.parseInt(corners[2]), z1 = Integer.parseInt(corners[3]);
// Margins: FEATURES need TERRAIN at 1, which needs BIOMES at 1 more; STRUCTURE_STARTS reach 8 further.
int m = System.getenv("HORIZON_ISOLATED") != null ? 0 : 1; // diagnostics: decorate the targets only
int fx0 = x0 - m, fz0 = z0 - m, fx1 = x1 + m, fz1 = z1 + m;
int tx0 = fx0 - 1, tz0 = fz0 - 1, tx1 = fx1 + 1, tz1 = fz1 + 1;
int bx0 = tx0 - 1, bz0 = tz0 - 1, bx1 = tx1 + 1, bz1 = tz1 + 1;
int sx0 = tx0 - 8, sz0 = tz0 - 8, sx1 = tx1 + 8, sz1 = tz1 + 8;
for (int z = sz0; z <= sz1; z++) for (int x = sx0; x <= sx1; x++) {
var chunk = chunkAt.apply(x, z);
generator.createStructures(access, state, OracleLevel.structures, chunk, templates, levelKey);
chunk.setPersistedStatus(ChunkStatus.STRUCTURE_STARTS);
}
for (int z = tz0; z <= tz1; z++) for (int x = tx0; x <= tx1; x++) {
var chunk = chunkAt.apply(x, z);
var region = regionFor.apply(ChunkStatus.STRUCTURE_REFERENCES, chunk);
generator.createReferences(region, OracleLevel.structures.forWorldGenRegion(region), chunk);
chunk.setPersistedStatus(ChunkStatus.STRUCTURE_REFERENCES);
}
for (int z = bz0; z <= bz1; z++) for (int x = bx0; x <= bx1; x++) {
var chunk = chunkAt.apply(x, z);
if (chunk.getPersistedStatus() != ChunkStatus.STRUCTURE_STARTS && chunk.getPersistedStatus() != ChunkStatus.EMPTY && chunk.getPersistedStatus() != ChunkStatus.STRUCTURE_REFERENCES) continue;
var region = regionFor.apply(ChunkStatus.BIOMES, chunk);
generator.createBiomes(randomState, Blender.empty(), OracleLevel.structures.forWorldGenRegion(region), chunk).join();
chunk.setPersistedStatus(ChunkStatus.BIOMES);
}
for (int z = tz0; z <= tz1; z++) for (int x = tx0; x <= tx1; x++) {
var chunk = chunkAt.apply(x, z);
var region = regionFor.apply(ChunkStatus.TERRAIN, chunk);
Set<Holder<Biome>> possible = new HashSet<>();
for (int dz = -1; dz <= 1; dz++) for (int dx = -1; dx <= 1; dx++) region.getChunk(x + dx, z + dz).collectBiomesInPalette(possible);
generator.buildTerrain(chunk, Blender.empty(), randomState, OracleLevel.structures.forWorldGenRegion(region), region.getBiomeManager(), region, possible).join();
Heightmap.primeHeightmaps(chunk, EnumSet.of(Heightmap.Types.MOTION_BLOCKING, Heightmap.Types.MOTION_BLOCKING_NO_LEAVES, Heightmap.Types.OCEAN_FLOOR, Heightmap.Types.WORLD_SURFACE));
chunk.setPersistedStatus(ChunkStatus.TERRAIN);
}
for (int z = fz0; z <= fz1 && !phase.equals("terrain"); z++) for (int x = fx0; x <= fx1; x++) {
var chunk = chunkAt.apply(x, z);
var region = regionFor.apply(ChunkStatus.FEATURES, chunk);
generator.applyBiomeDecoration(region, chunk, OracleLevel.structures.forWorldGenRegion(region));
chunk.setPersistedStatus(ChunkStatus.FEATURES);
}
var registry = access;
for (int z = z0; z <= z1; z++) for (int x = x0; x <= x1; x++) {
var target = chunkAt.apply(x, z);
w.println("chunk " + x + " " + z);
// Biomes: one line per quart layer, 16 columns in z-major order.
for (int qy = height.getMinY() >> 2; qy <= height.getMaxY() >> 2; qy++) {
StringBuilder line = new StringBuilder("q " + qy);
for (int qz = 0; qz < 4; qz++) for (int qx = 0; qx < 4; qx++)
line.append(' ').append(target.getNoiseBiome((x << 2) + qx, qy, (z << 2) + qz).unwrapKey().orElseThrow().identifier());
w.println(line);
}
for (int y = height.getMinY(); y <= height.getMaxY(); y++) {
StringBuilder line = new StringBuilder("y " + y);
for (int zz = 0; zz < 16; zz++) for (int xx = 0; xx < 16; xx++)
line.append(' ').append(Block.getId(target.getBlockState(new BlockPos((x << 4) + xx, y, (z << 4) + zz))));
w.println(line);
}
var positions = new TreeSet<BlockPos>(Comparator.<BlockPos>comparingInt(BlockPos::getX).thenComparingInt(BlockPos::getY).thenComparingInt(BlockPos::getZ));
positions.addAll(target.getBlockEntitiesPos());
for (var pos : positions) {
var tag = target.getBlockEntityNbtForSaving(pos, registry);
if (tag == null) continue;
var st = target.getBlockState(pos);
var fresh = ((net.minecraft.world.level.block.EntityBlock) st.getBlock()).newBlockEntity(pos, st).saveWithFullMetadata(registry);
if ("DUMMY".equals(tag.getStringOr("id", ""))) tag = fresh;
w.println("be " + pos.getX() + " " + pos.getY() + " " + pos.getZ() + "\t" + canonicalOf(tag) + "\t" + canonicalOf(fresh));
}
}
}
}
storage.close();
System.exit(0);
}
private static String canonicalOf(net.minecraft.nbt.Tag tag) {
var out = new StringBuilder();
canonical(tag, out);
return out.toString();
}
/** Canonical single-line form of a tag: type-prefixed scalars, sorted compound keys. */
private static void canonical(net.minecraft.nbt.Tag tag, StringBuilder out) {
switch (tag) {
case net.minecraft.nbt.ByteTag t -> out.append("b").append(t.byteValue());
case net.minecraft.nbt.ShortTag t -> out.append("s").append(t.shortValue());
case net.minecraft.nbt.IntTag t -> out.append("i").append(t.intValue());
case net.minecraft.nbt.LongTag t -> out.append("l").append(t.longValue());
case net.minecraft.nbt.FloatTag t -> out.append("f").append(Float.floatToRawIntBits(t.floatValue()));
case net.minecraft.nbt.DoubleTag t -> out.append("d").append(Double.doubleToRawLongBits(t.doubleValue()));
case net.minecraft.nbt.StringTag t -> out.append('"').append(t.value().replace("\\", "\\\\").replace("\"", "\\\"")).append('"');
case net.minecraft.nbt.ByteArrayTag t -> { out.append("B["); for (byte v : t.getAsByteArray()) out.append(v).append(','); out.append(']'); }
case net.minecraft.nbt.IntArrayTag t -> { out.append("I["); for (int v : t.getAsIntArray()) out.append(v).append(','); out.append(']'); }
case net.minecraft.nbt.LongArrayTag t -> { out.append("L["); for (long v : t.getAsLongArray()) out.append(v).append(','); out.append(']'); }
case net.minecraft.nbt.ListTag t -> { out.append('['); for (var item : t) { canonical(item, out); out.append(','); } out.append(']'); }
case net.minecraft.nbt.CompoundTag t -> {
out.append('{');
for (String key : new TreeSet<>(t.keySet())) { out.append(key).append('='); canonical(t.get(key), out); out.append(','); }
out.append('}');
}
default -> throw new IllegalStateException(tag.getClass().getName());
}
}
}
+318
View File
@@ -0,0 +1,318 @@
#!/usr/bin/env python3
"""Compare whole generated chunks block-for-block with vanilla.
Vanilla side: VanillaChunkGenerationOracle drives the actual ChunkGenerator
stages (structure starts and references, biomes, terrain with surface and
carvers, then biome decoration with every placed feature and structure piece)
through real WorldGenRegion instances. Horizon side: the chunk_dump example
runs the same stages in the same order. Every block state, quart biome and
block entity (canonical NBT) of each target chunk is compared.
Cases: terrain sites for several seeds across the overworld, the Nether and the
End, plus rectangles around vanilla structure starts found by scanning each
seed (villages, mineshafts, strongholds, ancient cities, trial chambers,
monuments, mansions, ...).
usage: test_chunk_generation_parity.py [--seeds N] [--per-structure K]
[--filter REGEX] [--dimension overworld|the_nether|the_end] [--keep]
"""
import argparse
import json
import os
import random
import re
import subprocess
import sys
import time
from collections import Counter, defaultdict
from pathlib import Path
try:
from . import lab, test_rust_parity as base
except ImportError:
import lab
import test_rust_parity as base
SEEDS = [12345, -7046029254386353131, 987654321987, 4242, -1, 0x5DEECE66D, 20260607, 777]
RESOURCES = lab.LOCAL / 'horizon-client/26.3'
ORACLE = lab.ROOT / 'mods/horizon/src/test/java/dev/horizonlab/parity/VanillaChunkGenerationOracle.java'
def build_dir():
path = lab.LOCAL / 'chunk-generation-parity'
path.mkdir(parents=True, exist_ok=True)
return path
class Oracle:
def __init__(self):
self.lock, self.jars, self.libraries = base.verify_artifacts()
self.dependencies = os.pathsep.join(map(str, [self.jars['server'], *self.libraries]))
self.classes = build_dir() / 'classes'
self.classes.mkdir(exist_ok=True)
result = subprocess.run(['javac', '--release', str(self.lock['minecraft']['java_major']), '-proc:none', '-cp', self.dependencies,
'-d', str(self.classes), str(ORACLE)], capture_output=True, text=True)
if result.returncode:
raise SystemExit('javac failed:\n' + result.stderr[-4000:])
def run(self, out, args):
result = subprocess.run(['java', '-Xmx8G', '-cp', str(self.classes) + os.pathsep + self.dependencies,
'dev.horizonlab.parity.VanillaChunkGenerationOracle', *map(str, [args[0], out, *args[1:]])],
capture_output=True, text=True, env=dict(os.environ))
if result.returncode:
lines = [l for l in (result.stdout + result.stderr).splitlines() if 'Exception' in l or 'Caused' in l or '\tat ' in l]
raise SystemExit('oracle failed: ' + ' | '.join(lines[:10]))
def rust_binary():
subprocess.run(['cargo', 'build', '--release', '-q', '-p', 'horizon-world', '--example', 'chunk_dump'], check=True, cwd=lab.ROOT)
metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--no-deps', '--format-version', '1'], cwd=lab.ROOT, text=True))
return Path(metadata['target_directory']) / 'release/examples/chunk_dump'
def run_rust(binary, out, seed, dimension, rects, scan=False):
env = dict(os.environ)
if scan:
env['HORIZON_SCAN'] = '1'
with open(out, 'w') as handle:
result = subprocess.run([str(binary), str(RESOURCES), str(seed), dimension, *rects], stdout=handle, stderr=subprocess.PIPE, text=True, env=env, cwd=lab.ROOT)
if result.returncode:
raise SystemExit('chunk_dump failed: ' + result.stderr[-2000:])
return result.stderr
def parse(path):
chunks, current = {}, None
for line in Path(path).read_text().splitlines():
if line.startswith('chunk '):
_, x, z = line.split()
current = (int(x), int(z))
chunks[current] = {'q': {}, 'y': {}, 'be': {}}
elif line.startswith('q '):
_, q, row = line.split(' ', 2)
chunks[current]['q'][int(q)] = row
elif line.startswith('y '):
_, y, row = line.split(' ', 2)
chunks[current]['y'][int(y)] = row
elif line.startswith('be '):
head, *forms = line.split('\t')
parts = head.split(' ')
chunks[current]['be'][(int(parts[1]), int(parts[2]), int(parts[3]))] = forms
return chunks
def parse_nbt(text, i=0):
"""Canonical NBT text -> Python values (typed scalars as (type, text) tuples)."""
c = text[i]
if c == '{':
i += 1
value = {}
while text[i] != '}':
j = text.index('=', i)
key = text[i:j]
value[key], i = parse_nbt(text, j + 1)
i += 1 # comma
return value, i + 1
if c == '[':
i += 1
items = []
while text[i] != ']':
item, i = parse_nbt(text, i)
items.append(item)
i += 1
return items, i + 1
if c == '"':
i += 1
out = []
while text[i] != '"':
if text[i] == '\\':
i += 1
out.append(text[i])
i += 1
return ('str', ''.join(out)), i + 1
if c in 'BIL' and text[i + 1] == '[':
j = text.index(']', i)
return (c, text[i + 2:j]), j + 1
j = i + 1
while j < len(text) and text[j] not in ',}]':
j += 1
return (c, text[i + 1:j]), j
def nbt_matches(vanilla, rust, default):
"""Horizon stores the placed data only; vanilla saves defaults too. Every key
Horizon sets must match, and every other vanilla key must be the fresh default."""
if isinstance(vanilla, dict) and isinstance(rust, dict):
default = default if isinstance(default, dict) else {}
for key, value in vanilla.items():
if key in ('x', 'y', 'z') or (key == 'components' and value == {}):
continue
if key in rust:
if not nbt_matches(value, rust[key], default.get(key)):
return False
elif value != default.get(key):
return False
return all(key in vanilla or key in ('x', 'y', 'z') for key in rust)
return vanilla == rust
def entity_matches(vanilla, rust):
if rust is None or vanilla is None:
return False
return nbt_matches(parse_nbt(vanilla[0])[0], parse_nbt(rust[0])[0], parse_nbt(vanilla[1])[0])
def state_names():
facts = json.loads((RESOURCES / 'facts.json').read_text())
return [s[3].removeprefix('Block{').replace('}', '', 1) for s in facts['states']]
def compare(vanilla, rust, names, label, limit=6, quiet=False):
"""Returns (differing chunks, differing blocks, differing biome cells, differing block entities)."""
chunks = blocks = biomes = entities = 0
for pos in sorted(vanilla):
a, b = vanilla[pos], rust.get(pos)
if b is None:
print(f' {label} chunk {pos}: missing from Horizon output')
chunks += 1
continue
block_diffs, biome_diffs, entity_diffs = [], [], []
for y in sorted(a['y']):
ra, rb = a['y'][y], b['y'].get(y)
if ra == rb:
continue
for i, (u, v) in enumerate(zip(ra.split(' '), (rb or '').split(' '))):
if u != v:
block_diffs.append((pos[0] * 16 + i % 16, y, pos[1] * 16 + i // 16, names[int(u)], names[int(v)]))
for q in sorted(a['q']):
if a['q'][q] != b['q'].get(q):
for i, (u, v) in enumerate(zip(a['q'][q].split(' '), b['q'].get(q, '').split(' '))):
if u != v:
biome_diffs.append((pos[0] * 4 + i % 4, q, pos[1] * 4 + i // 4, u, v))
for key in sorted(set(a['be']) | set(b['be'])):
if not entity_matches(a['be'].get(key), b['be'].get(key)):
entity_diffs.append((key, a['be'].get(key, [None])[0], (b['be'].get(key) or [None])[0]))
if block_diffs or biome_diffs or entity_diffs:
chunks += 1
blocks += len(block_diffs)
biomes += len(biome_diffs)
entities += len(entity_diffs)
if quiet:
continue
print(f' {label} chunk {pos}: {len(block_diffs)} blocks, {len(biome_diffs)} biome cells, {len(entity_diffs)} block entities')
for d in block_diffs[:limit]:
print(f' block {d[0]} {d[1]} {d[2]}: vanilla {d[3]} / horizon {d[4]}')
for d in biome_diffs[:2]:
print(f' biome quart {d[0]} {d[1]} {d[2]}: vanilla {d[3]} / horizon {d[4]}')
for d in entity_diffs[:3]:
print(f' block entity {d[0]}: vanilla {d[1]} / horizon {d[2]}')
return chunks, blocks, biomes, entities
def scan(oracle, binary, seed, dimension, radius):
out = build_dir() / f'scan-{dimension}-{seed}.txt'
oracle.run(out, ['scan', seed, dimension, -radius, -radius, radius, radius])
starts = []
for line in out.read_text().splitlines():
t = line.split()
if t and t[0] == 'start':
starts.append((t[1], int(t[2]), int(t[3]), tuple(map(int, t[4:10]))))
return starts
def pick_structures(starts, per_kind, rng):
by_kind = defaultdict(list)
for s in starts:
by_kind[s[0]].append(s)
picked = []
for kind, items in sorted(by_kind.items()):
items.sort(key=lambda s: abs(s[1]) + abs(s[2]))
# Nearest first, then random ones, so that variants (village types) differ.
chosen = items[:1] + rng.sample(items[1:], min(per_kind - 1, len(items) - 1))
picked.extend(chosen[:per_kind])
return picked
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--seeds', type=int, default=3)
parser.add_argument('--per-structure', type=int, default=2)
parser.add_argument('--sites', type=int, default=6, help='terrain sites per overworld seed')
parser.add_argument('--radius', type=int, default=1, help='chunk radius around each site')
parser.add_argument('--scan-radius', type=int, default=180)
parser.add_argument('--limit', type=int, default=6, help='block differences shown per chunk')
parser.add_argument('--summary', action='store_true', help='one line per differing site')
parser.add_argument('--rect', action='append', help='compare only this x0,z0,x1,z1 chunk rectangle (repeatable)')
parser.add_argument('--filter', help='regex over structure names')
parser.add_argument('--dimension', action='append')
parser.add_argument('--phase', choices=['terrain', 'structures', 'full'], default='full', help='stop before features, or run structure pieces only')
parser.add_argument('--terrain-only', action='store_true', help='skip the structure sites')
parser.add_argument('--structures-only', action='store_true')
args = parser.parse_args()
os.environ['HORIZON_PHASE'] = args.phase
dimensions = args.dimension or ['overworld', 'the_nether', 'the_end']
oracle = Oracle()
binary = rust_binary()
names = state_names()
build = build_dir()
totals = Counter()
biome_seen = set()
structure_seen = Counter()
started = time.time()
for seed in SEEDS[:args.seeds]:
rng = random.Random(seed)
for dimension in dimensions:
rects, labels = [], []
if args.rect:
rects, labels = list(args.rect), [f'rect {r}' for r in args.rect]
r = args.radius
if not args.structures_only and not args.rect:
spread = {'overworld': 5000, 'the_nether': 1500, 'the_end': 1800}[dimension]
sites = [(0, 0)] + [(rng.randint(-spread, spread) // 16, rng.randint(-spread, spread) // 16) for _ in range(args.sites if dimension == 'overworld' else 2)]
for cx, cz in sites:
rects.append(f'{cx - r},{cz - r},{cx + r},{cz + r}')
labels.append(f'terrain {cx},{cz}')
if not args.terrain_only and not args.rect:
starts = scan(oracle, binary, seed, dimension, args.scan_radius if dimension == 'overworld' else 80)
for kind, cx, cz, box in pick_structures(starts, args.per_structure, rng):
if args.filter and not re.search(args.filter, kind):
continue
rects.append(f'{cx - r},{cz - r},{cx + r},{cz + r}')
labels.append(f'{kind} {cx},{cz}')
structure_seen[kind] += 1
if not rects:
continue
tag = f'{dimension}-{seed}-{args.phase}' + ('-iso' if os.environ.get('HORIZON_ISOLATED') else '') + ('-rect' if args.rect else '')
vanilla_path, rust_path = build / f'vanilla-{tag}.txt', build / f'rust-{tag}.txt'
t0 = time.time()
if not (os.environ.get('REUSE_VANILLA') and vanilla_path.exists()):
oracle.run(vanilla_path, ['dump', seed, dimension, *rects])
t1 = time.time()
stderr = run_rust(binary, rust_path, seed, dimension, rects)
t2 = time.time()
vanilla, rust = parse(vanilla_path), parse(rust_path)
for line in stderr.splitlines()[:5]:
print(' horizon:', line)
counts = [0, 0, 0, 0]
for rect, label in zip(rects, labels):
x0, z0, x1, z1 = map(int, rect.split(','))
part = {k: v for k, v in vanilla.items() if x0 <= k[0] <= x1 and z0 <= k[1] <= z1}
one = compare(part, rust, names, f'{tag} {label}', limit=args.limit, quiet=args.summary)
if one[0]:
print(f'{tag} {label}: {one[0]} chunks differ ({one[1]} blocks, {one[2]} biome cells, {one[3]} block entities)')
counts = [a + b for a, b in zip(counts, one)]
totals.update(dict(zip(['chunks', 'blocks', 'biomes', 'entities'], counts)))
totals['compared'] += len(vanilla)
for chunk in vanilla.values():
for row in chunk['q'].values():
biome_seen.update(row)
print(f'{tag}: {len(rects)} sites, {len(vanilla)} chunks, differing chunks {counts[0]} (blocks {counts[1]}, biome cells {counts[2]}, block entities {counts[3]}); vanilla {t1 - t0:.0f}s, horizon {t2 - t1:.0f}s', flush=True)
print(f'Compared {totals["compared"]} chunks across {len(biome_seen)} biomes and {sum(structure_seen.values())} structure sites in {time.time() - started:.0f}s')
print('structure sites:', dict(sorted(structure_seen.items())))
print(f'Differences: {totals["chunks"]} chunks, {totals["blocks"]} blocks, {totals["biomes"]} biome cells, {totals["entities"]} block entities')
if totals['chunks']:
sys.exit(1)
if __name__ == '__main__':
main()