Make the datafix harness take 40 seconds instead of 25 minutes

The oracle spent 58 of every world's 62 seconds waiting for the idle
threads of the game's IO pool to time out; it now exits once its outputs are
written (same outputs on all 22 worlds). The harness runs in 6 shards, keeps
the vanilla outputs so the oracle's results are reused, and declares its Rust
scope (datafix, NBT, region files, compression and horizon-core positions):
other horizon-world changes no longer rerun it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-12 16:41:51 +02:00
parent f5ab40d415
commit cb19f5e7d6
2 changed files with 44 additions and 4 deletions
@@ -40,6 +40,10 @@ import net.minecraft.world.level.storage.LevelStorageSource;
*
* Files the game rewrites after loading get the current DataVersion so that the
* output compares with what the game would save. Nothing is answered by hand.
*
* OUT_WORLD is replaced. The oracle exits as soon as everything is written:
* the level storage starts the game's IO worker pool, whose idle threads would
* otherwise keep the JVM alive for a minute.
*/
public final class VanillaDataFixOracle {
private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
@@ -49,6 +53,7 @@ public final class VanillaDataFixOracle {
Bootstrap.bootStrap();
Path in = Path.of(args[0]);
Path out = Path.of(args[1]);
delete(out);
copy(in, out);
int current = SharedConstants.getCurrentVersion().dataVersion().version();
var fixer = DataFixers.getDataFixer();
@@ -104,6 +109,7 @@ public final class VanillaDataFixOracle {
fixed = fixed.set("DataVersion", fixed.createInt(current));
Files.writeString(file, GSON.toJson(fixed.getValue()), StandardCharsets.UTF_8);
}
System.exit(0);
}
/** Chunk generator type per dimension, from the saved world generation settings. */
@@ -231,6 +237,17 @@ public final class VanillaDataFixOracle {
}
}
private static void delete(Path path) throws Exception {
if (!Files.exists(path)) {
return;
}
try (Stream<Path> s = Files.walk(path)) {
for (Path p : s.sorted(Comparator.reverseOrder()).toList()) {
Files.delete(p);
}
}
}
private static void copy(Path from, Path to) throws Exception {
try (Stream<Path> s = Files.walk(from)) {
for (Path p : s.sorted().toList()) {
+27 -4
View File
@@ -16,6 +16,10 @@ Then every file is compared: region files chunk by chunk, NBT files and JSON
files value by value (compound and object key order does not matter). The only
tolerated differences are listed in IGNORED (wall-clock time).
tools/parity.py runs it in PARITY_SHARDS parts, each upgrading its share of
the worlds (PARITY_SHARD=i/n). The vanilla outputs stay in .local/datafix/run
so that the oracle's results are reused while its inputs are unchanged.
test_datafix_parity.py [--keep] [--skip-build] [VERSION...]
"""
import argparse
@@ -35,6 +39,15 @@ LOCAL = ROOT / '.local' / 'datafix'
WORLDS = (ROOT / '.local' / 'minecraft').resolve().parent / 'datafix' / 'worlds'
RUN = LOCAL / 'run'
PARITY_SHARDS = 6
# What the datafix_world probe can reach, checked by tools/parity.py: it reruns
# this harness only when these sources (or the harness, oracle and data) change.
PARITY_RUST_SCOPE = [
'crates/horizon-world/src/datafix/', 'crates/horizon-world/src/nbt.rs', 'crates/horizon-world/src/region.rs',
'crates/horizon-world/src/compression.rs', 'crates/horizon-core/src/position.rs', 'crates/horizon-core/src/position/',
'crates/horizon-core/src/geometry/',
]
IGNORED = {
r'^level\.dat\.Data\.LastPlayed$': 'wall-clock time of the upgrade',
}
@@ -46,7 +59,7 @@ def java_classpath():
import test_model_baking_parity as baking
lock, jars, _ = base.verify_artifacts()
libraries = baking.client_libraries()
build = LOCAL / 'build'
build = LOCAL / 'build' / os.environ.get('PARITY_SHARD', 'all').replace('/', '-') # shards compile apart
build.mkdir(parents=True, exist_ok=True)
classes, dependencies, _ = harness.compile_oracle(build, 'VanillaDataFixOracle', jars, libraries,
lock['minecraft']['java_major'])
@@ -61,8 +74,17 @@ def build_rust():
return Path(meta['target_directory']) / 'release' / 'examples' / 'datafix_world'
def shard(items):
"""This run's share of items when tools/parity.py runs the harness in parts (PARITY_SHARD=i/n)."""
spec = os.environ.get('PARITY_SHARD')
if not spec:
return items
index, count = map(int, spec.split('/'))
return items[index - 1::count]
def run_vanilla(classpath, world, out):
shutil.rmtree(out, ignore_errors=True)
# The oracle replaces OUT itself: left in place, an unchanged run's outputs can be reused.
log = out.parent / (out.name + '.log')
with log.open('w') as f:
subprocess.run(['java', '-Xmx3g', '-Djava.awt.headless=true', '-cp', classpath,
@@ -195,7 +217,7 @@ def main():
print('generating sample worlds:', ' '.join(missing), flush=True)
subprocess.run([sys.executable, str(ROOT / 'tools' / 'datafix_worlds.py'), *missing], cwd=ROOT, check=True)
worlds = sorted(p.name for p in WORLDS.iterdir() if p.is_dir())
versions = args.versions or worlds
versions = shard(args.versions or worlds)
classpath = java_classpath()
binary = None if args.vanilla_only else Path(args.binary) if args.binary else build_rust()
total = 0
@@ -214,7 +236,8 @@ def main():
if binary is not None:
print('datafix parity', 'FAILED' if total else 'ok', f'({total} differences over {len(versions)} worlds)')
if not total and not args.keep:
shutil.rmtree(RUN, ignore_errors=True)
for version in versions: # vanilla outputs stay, for reuse
shutil.rmtree(RUN / f'{version}-horizon', ignore_errors=True)
sys.exit(1 if total else 0)