Keep the datafix sample worlds with the shared vanilla data, and generate them when missing

The old server jars and the 22 worlds made from them are inputs, but they
lived in each worktree's .local and were lost when its outputs were
cleared. They now sit beside the shared vanilla data, written atomically,
and the harness generates any that are missing (under a lock, since
several worktrees may run it at once).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-11 19:36:09 +02:00
parent 4eb2f53ffb
commit 82dceec5e7
2 changed files with 29 additions and 13 deletions
+15 -7
View File
@@ -2,10 +2,11 @@
"""Make sample worlds with older vanilla servers, for the DataFixer harness.
For each listed version, the official server jar (cached under
.local/datafix/servers/VERSION/server.jar) generates a default world around spawn,
.local/datafix/servers/VERSION/server.jar, beside the shared vanilla data) generates a default world around spawn,
gets a scene of block entities, entities and items in that version's own save
format (tools/datafix_scene.py), and a bot player joins (with its inventory filled)
so that player data exists. The saved world is kept in .local/datafix/worlds/VERSION.
so that player data exists. The saved world is kept in datafix/worlds/VERSION beside the shared vanilla data
(.local/datafix in a plain checkout), where every worktree finds it.
datafix_worlds.py [--force] [VERSION...] (default: every version below)
"""
@@ -33,6 +34,9 @@ import datafix_scene as scene # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
LOCAL = ROOT / '.local' / 'datafix'
# Old server jars and their worlds are inputs, not outputs: keep them beside the
# vanilla data, which parallel worktrees share (.local/minecraft links there).
SHARED = (ROOT / '.local' / 'minecraft').resolve().parent / 'datafix'
VERSIONS = ['1.18.2', '1.19', '1.19.2', '1.19.3', '1.19.4', '1.20.1', '1.20.2', '1.20.4', '1.20.5', '1.20.6', '1.21.1',
'1.21.3', '1.21.4', '1.21.5', '1.21.6', '1.21.8', '1.21.9', '1.21.10', '1.21.11', '26.1', '26.1.2', '26.2']
MANIFEST = 'https://piston-meta.mojang.com/mc/game/version_manifest_v2.json'
@@ -41,7 +45,7 @@ UUID_BASE = 0x6000
def server_jar(version):
path = LOCAL / 'servers' / version / 'server.jar'
path = SHARED / 'servers' / version / 'server.jar'
if not path.exists():
path.parent.mkdir(parents=True, exist_ok=True)
manifest = json.load(urllib.request.urlopen(MANIFEST))
@@ -271,7 +275,7 @@ def uuid_nbt(i):
def build(version, force=False):
jar, data_version, protocol = server_jar(version)
target = LOCAL / 'worlds' / version
target = SHARED / 'worlds' / version
if target.exists() and not force:
print(f'{version}: kept {target}')
return []
@@ -310,11 +314,15 @@ def build(version, force=False):
bot.leave()
time.sleep(2)
server.stop()
shutil.rmtree(target, ignore_errors=True)
# Copy aside, then rename: an interrupted copy must not look like a finished world.
partial = target.with_name(target.name + '.partial')
shutil.rmtree(partial, ignore_errors=True)
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(work / 'world', target)
(target / 'datafix-source.json').write_text(json.dumps(
shutil.copytree(work / 'world', partial)
(partial / 'datafix-source.json').write_text(json.dumps(
{'version': version, 'data_version': data_version, 'problems': problems}, indent=1))
shutil.rmtree(target, ignore_errors=True)
partial.rename(target)
for command, out in problems:
print(f'{version}: command failed: {command}\n {out}')
print(f'{version}: DataVersion {data_version}, world in {target}')
+14 -6
View File
@@ -2,8 +2,9 @@
"""DataFixer differential test: worlds saved by older game versions, upgraded by
vanilla 26.3 and by Horizon, must give the same data.
For every sample world in .local/datafix/worlds (made by tools/datafix_worlds.py
with the official server jars of 1.20.1 to 26.2), two upgrades run on a copy:
For every sample world in datafix/worlds beside the shared vanilla data (made by
tools/datafix_worlds.py with the official server jars of 1.18.2 to 26.2, and
generated automatically when missing), two upgrades run on a copy:
vanilla VanillaDataFixOracle: FileFixerUpper, then DataFixers on level.dat, every
chunk, entity chunk and POI chunk, player, saved data, advancements and
@@ -31,7 +32,7 @@ import anvil_dump # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
LOCAL = ROOT / '.local' / 'datafix'
WORLDS = LOCAL / 'worlds'
WORLDS = (ROOT / '.local' / 'minecraft').resolve().parent / 'datafix' / 'worlds'
RUN = LOCAL / 'run'
IGNORED = {
@@ -184,10 +185,17 @@ def main():
p.add_argument('--vanilla-only', action='store_true', help='only run the oracle')
p.add_argument('--binary', help='a prebuilt datafix_world example')
args = p.parse_args()
worlds = sorted(p.name for p in WORLDS.iterdir()) if WORLDS.is_dir() else []
import datafix_worlds
import fcntl
WORLDS.parent.mkdir(parents=True, exist_ok=True)
with open(WORLDS.parent / 'generate.lock', 'w') as lock:
fcntl.flock(lock, fcntl.LOCK_EX) # another worktree may be generating them
missing = [v for v in datafix_worlds.VERSIONS if not (WORLDS / v / 'level.dat').exists()]
if missing:
print('generating sample worlds:', ' '.join(missing), flush=True)
subprocess.run([sys.executable, str(ROOT / 'tools' / 'datafix_worlds.py'), *missing], cwd=ROOT, check=True)
worlds = sorted(p.name for p in WORLDS.iterdir() if p.is_dir())
versions = args.versions or worlds
if not versions:
sys.exit('no sample worlds: run tools/datafix_worlds.py first')
classpath = java_classpath()
binary = None if args.vanilla_only else Path(args.binary) if args.binary else build_rust()
total = 0