From cb19f5e7d6b11ecef9a8080e861ddf857b0a5dbd Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:41:51 +0200 Subject: [PATCH] Make the datafix harness take 40 seconds instead of 25 minutes The oracle spent 58 of every world's 62 seconds waiting for the idle threads of the game's IO pool to time out; it now exits once its outputs are written (same outputs on all 22 worlds). The harness runs in 6 shards, keeps the vanilla outputs so the oracle's results are reused, and declares its Rust scope (datafix, NBT, region files, compression and horizon-core positions): other horizon-world changes no longer rerun it. Co-Authored-By: Claude Opus 5.5 --- .../parity/VanillaDataFixOracle.java | 17 ++++++++++ tools/test_datafix_parity.py | 31 ++++++++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaDataFixOracle.java b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaDataFixOracle.java index 2bb7028f9..462da8388 100644 --- a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaDataFixOracle.java +++ b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaDataFixOracle.java @@ -40,6 +40,10 @@ import net.minecraft.world.level.storage.LevelStorageSource; * * Files the game rewrites after loading get the current DataVersion so that the * output compares with what the game would save. Nothing is answered by hand. + * + * OUT_WORLD is replaced. The oracle exits as soon as everything is written: + * the level storage starts the game's IO worker pool, whose idle threads would + * otherwise keep the JVM alive for a minute. */ public final class VanillaDataFixOracle { private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); @@ -49,6 +53,7 @@ public final class VanillaDataFixOracle { Bootstrap.bootStrap(); Path in = Path.of(args[0]); Path out = Path.of(args[1]); + delete(out); copy(in, out); int current = SharedConstants.getCurrentVersion().dataVersion().version(); var fixer = DataFixers.getDataFixer(); @@ -104,6 +109,7 @@ public final class VanillaDataFixOracle { fixed = fixed.set("DataVersion", fixed.createInt(current)); Files.writeString(file, GSON.toJson(fixed.getValue()), StandardCharsets.UTF_8); } + System.exit(0); } /** Chunk generator type per dimension, from the saved world generation settings. */ @@ -231,6 +237,17 @@ public final class VanillaDataFixOracle { } } + private static void delete(Path path) throws Exception { + if (!Files.exists(path)) { + return; + } + try (Stream s = Files.walk(path)) { + for (Path p : s.sorted(Comparator.reverseOrder()).toList()) { + Files.delete(p); + } + } + } + private static void copy(Path from, Path to) throws Exception { try (Stream s = Files.walk(from)) { for (Path p : s.sorted().toList()) { diff --git a/tools/test_datafix_parity.py b/tools/test_datafix_parity.py index d3fd6957e..f03183ae0 100644 --- a/tools/test_datafix_parity.py +++ b/tools/test_datafix_parity.py @@ -16,6 +16,10 @@ Then every file is compared: region files chunk by chunk, NBT files and JSON files value by value (compound and object key order does not matter). The only tolerated differences are listed in IGNORED (wall-clock time). +tools/parity.py runs it in PARITY_SHARDS parts, each upgrading its share of +the worlds (PARITY_SHARD=i/n). The vanilla outputs stay in .local/datafix/run +so that the oracle's results are reused while its inputs are unchanged. + test_datafix_parity.py [--keep] [--skip-build] [VERSION...] """ import argparse @@ -35,6 +39,15 @@ LOCAL = ROOT / '.local' / 'datafix' WORLDS = (ROOT / '.local' / 'minecraft').resolve().parent / 'datafix' / 'worlds' RUN = LOCAL / 'run' +PARITY_SHARDS = 6 +# What the datafix_world probe can reach, checked by tools/parity.py: it reruns +# this harness only when these sources (or the harness, oracle and data) change. +PARITY_RUST_SCOPE = [ + 'crates/horizon-world/src/datafix/', 'crates/horizon-world/src/nbt.rs', 'crates/horizon-world/src/region.rs', + 'crates/horizon-world/src/compression.rs', 'crates/horizon-core/src/position.rs', 'crates/horizon-core/src/position/', + 'crates/horizon-core/src/geometry/', +] + IGNORED = { r'^level\.dat\.Data\.LastPlayed$': 'wall-clock time of the upgrade', } @@ -46,7 +59,7 @@ def java_classpath(): import test_model_baking_parity as baking lock, jars, _ = base.verify_artifacts() libraries = baking.client_libraries() - build = LOCAL / 'build' + build = LOCAL / 'build' / os.environ.get('PARITY_SHARD', 'all').replace('/', '-') # shards compile apart build.mkdir(parents=True, exist_ok=True) classes, dependencies, _ = harness.compile_oracle(build, 'VanillaDataFixOracle', jars, libraries, lock['minecraft']['java_major']) @@ -61,8 +74,17 @@ def build_rust(): return Path(meta['target_directory']) / 'release' / 'examples' / 'datafix_world' +def shard(items): + """This run's share of items when tools/parity.py runs the harness in parts (PARITY_SHARD=i/n).""" + spec = os.environ.get('PARITY_SHARD') + if not spec: + return items + index, count = map(int, spec.split('/')) + return items[index - 1::count] + + def run_vanilla(classpath, world, out): - shutil.rmtree(out, ignore_errors=True) + # The oracle replaces OUT itself: left in place, an unchanged run's outputs can be reused. log = out.parent / (out.name + '.log') with log.open('w') as f: subprocess.run(['java', '-Xmx3g', '-Djava.awt.headless=true', '-cp', classpath, @@ -195,7 +217,7 @@ def main(): print('generating sample worlds:', ' '.join(missing), flush=True) subprocess.run([sys.executable, str(ROOT / 'tools' / 'datafix_worlds.py'), *missing], cwd=ROOT, check=True) worlds = sorted(p.name for p in WORLDS.iterdir() if p.is_dir()) - versions = args.versions or worlds + versions = shard(args.versions or worlds) classpath = java_classpath() binary = None if args.vanilla_only else Path(args.binary) if args.binary else build_rust() total = 0 @@ -214,7 +236,8 @@ def main(): if binary is not None: print('datafix parity', 'FAILED' if total else 'ok', f'({total} differences over {len(versions)} worlds)') if not total and not args.keep: - shutil.rmtree(RUN, ignore_errors=True) + for version in versions: # vanilla outputs stay, for reuse + shutil.rmtree(RUN / f'{version}-horizon', ignore_errors=True) sys.exit(1 if total else 0)