From 82eda1d8bf9adfe6eee7d14e49039126db529ab0 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:30:48 +0200 Subject: [PATCH] Port the 1.18 structure fixes (2967, 2970) and add a 1.18.1 sample world Old server jars run on Mojang's own Java runtimes, pinned in research-lock.json and installed beside the shared vanilla data. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/save.rs | 2 +- .../horizon-world/src/datafix/fixes_cliffs.rs | 266 ++++++++++++++++++ crates/horizon-world/src/datafix/mod.rs | 8 +- research-lock.json | 32 +++ tools/datafix_worlds.py | 95 ++++++- 5 files changed, 389 insertions(+), 14 deletions(-) create mode 100644 crates/horizon-world/src/datafix/fixes_cliffs.rs diff --git a/crates/horizon-client/src/save.rs b/crates/horizon-client/src/save.rs index 3e1940d10..25c077400 100644 --- a/crates/horizon-client/src/save.rs +++ b/crates/horizon-client/src/save.rs @@ -130,7 +130,7 @@ pub fn upgrade_older_world(world: &Path) -> Result<()> { } if version < i64::from(horizon_world::datafix::OLDEST_SUPPORTED) { return Err(format!( - "{} was saved with DataVersion {version}; worlds older than DataVersion {} (1.18.2) \ + "{} was saved with DataVersion {version}; worlds older than DataVersion {} \ cannot be upgraded yet (tools/upgrade_world.py runs vanilla's own upgrader)", world.display(), horizon_world::datafix::OLDEST_SUPPORTED diff --git a/crates/horizon-world/src/datafix/fixes_cliffs.rs b/crates/horizon-world/src/datafix/fixes_cliffs.rs new file mode 100644 index 000000000..8ae12be1b --- /dev/null +++ b/crates/horizon-world/src/datafix/fixes_cliffs.rs @@ -0,0 +1,266 @@ +//! Fixes of 1.17 and 1.18, Caves & Cliffs (keys 2679 to 2970): what worlds +//! saved by 1.16.5 to 1.18.1 need before the fixes of [`super::fixes_old`]. +use super::dynamic::{as_str, empty_map, set, stream, string}; +use super::schema::{Key, Ty, key}; +use super::{Cx, FixFn}; +use crate::nbt::{Compound, JavaString, Tag}; + +pub(super) fn register(out: &mut Vec<(Key, FixFn)>) { + out.push((key(2967, 0), structure_settings_flatten)); + out.push((key(2970, 0), structures_become_configured)); +} + +// ------------------------------------------------------------------ 2967 + +/// `StructureSettingsFlattenFix`. +fn structure_settings_flatten(cx: &mut Cx) { + cx.each(Ty::WorldGenSettings, |v| { + let Some(Tag::Compound(dims)) = v.get_mut("dimensions") else { + return; + }; + for dim in dims.values_mut() { + let Some(settings) = dim.get_mut("generator").and_then(|g| g.get_mut("settings")) + else { + continue; + }; + if let Some(structures) = settings.get_mut("structures") { + *structures = flatten_structures(structures); + } + } + }); +} + +fn flatten_structures(input: &Tag) -> Tag { + let mut structures = match input.get("structures") { + Some(Tag::Compound(c)) => { + let mut out = Compound::new(); + for (k, s) in c { + let mut s = s.clone(); + set(&mut s, "type", string("minecraft:random_spread")); + out.insert(k.clone(), s); + } + Tag::Compound(out) + } + _ => empty_map(), + }; + if let Some(stronghold) = input.get("stronghold") { + let mut stronghold = stronghold.clone(); + set( + &mut stronghold, + "type", + string("minecraft:concentric_rings"), + ); + set(&mut structures, "minecraft:stronghold", stronghold); + } + structures +} + +// ------------------------------------------------------------------ 2970 + +struct Conversion { + biomes: &'static [(&'static [&'static str], &'static str)], + fallback: &'static str, +} + +const fn trivial(fallback: &'static str) -> Conversion { + Conversion { + biomes: &[], + fallback, + } +} + +fn conversion(key: &str) -> Option { + Some(match key { + "mineshaft" => Conversion { + biomes: &[( + &[ + "minecraft:badlands", + "minecraft:eroded_badlands", + "minecraft:wooded_badlands", + ], + "minecraft:mineshaft_mesa", + )], + fallback: "minecraft:mineshaft", + }, + "shipwreck" => Conversion { + biomes: &[( + &["minecraft:beach", "minecraft:snowy_beach"], + "minecraft:shipwreck_beached", + )], + fallback: "minecraft:shipwreck", + }, + "ocean_ruin" => Conversion { + biomes: &[( + &[ + "minecraft:warm_ocean", + "minecraft:lukewarm_ocean", + "minecraft:deep_lukewarm_ocean", + ], + "minecraft:ocean_ruin_warm", + )], + fallback: "minecraft:ocean_ruin_cold", + }, + "village" => Conversion { + biomes: &[ + (&["minecraft:desert"], "minecraft:village_desert"), + (&["minecraft:savanna"], "minecraft:village_savanna"), + (&["minecraft:snowy_plains"], "minecraft:village_snowy"), + (&["minecraft:taiga"], "minecraft:village_taiga"), + ], + fallback: "minecraft:village_plains", + }, + "ruined_portal" => Conversion { + biomes: &[ + (&["minecraft:desert"], "minecraft:ruined_portal_desert"), + ( + &[ + "minecraft:badlands", + "minecraft:eroded_badlands", + "minecraft:wooded_badlands", + "minecraft:windswept_hills", + "minecraft:windswept_forest", + "minecraft:windswept_gravelly_hills", + "minecraft:savanna_plateau", + "minecraft:windswept_savanna", + "minecraft:stony_shore", + "minecraft:meadow", + "minecraft:frozen_peaks", + "minecraft:jagged_peaks", + "minecraft:stony_peaks", + "minecraft:snowy_slopes", + ], + "minecraft:ruined_portal_mountain", + ), + ( + &[ + "minecraft:bamboo_jungle", + "minecraft:jungle", + "minecraft:sparse_jungle", + ], + "minecraft:ruined_portal_jungle", + ), + ( + &[ + "minecraft:deep_frozen_ocean", + "minecraft:deep_cold_ocean", + "minecraft:deep_ocean", + "minecraft:deep_lukewarm_ocean", + "minecraft:frozen_ocean", + "minecraft:ocean", + "minecraft:cold_ocean", + "minecraft:lukewarm_ocean", + "minecraft:warm_ocean", + ], + "minecraft:ruined_portal_ocean", + ), + ], + fallback: "minecraft:ruined_portal", + }, + "pillager_outpost" => trivial("minecraft:pillager_outpost"), + "mansion" => trivial("minecraft:mansion"), + "jungle_pyramid" => trivial("minecraft:jungle_pyramid"), + "desert_pyramid" => trivial("minecraft:desert_pyramid"), + "igloo" => trivial("minecraft:igloo"), + "swamp_hut" => trivial("minecraft:swamp_hut"), + "stronghold" => trivial("minecraft:stronghold"), + "monument" => trivial("minecraft:monument"), + "fortress" => trivial("minecraft:fortress"), + "endcity" => trivial("minecraft:end_city"), + "buried_treasure" => trivial("minecraft:buried_treasure"), + "nether_fossil" => trivial("minecraft:nether_fossil"), + "bastion_remnant" => trivial("minecraft:bastion_remnant"), + _ => return None, + }) +} + +/// `StructuresBecomeConfiguredFix.findUpdatedStructureType`. +fn updated_structure(key: &JavaString, chunk: &Tag) -> Option { + let conv = conversion(&key.to_string_lossy().to_lowercase())?; + if conv.biomes.is_empty() { + return Some(conv.fallback.to_owned()); + } + // `guessConfiguration`: palette biome counts per mapping, in first-seen + // order; the most frequent wins, the first on ties. + let mapping = |biome: &str| { + conv.biomes + .iter() + .find(|(list, _)| list.contains(&biome)) + .map(|(_, to)| *to) + }; + let mut matches: Vec<(&str, i32)> = Vec::new(); + for s in stream(chunk.get("sections")) { + let palette = s.get("biomes").and_then(|b| b.get("palette")).cloned(); + for biome in stream(palette.as_ref()) { + if let Some(m) = mapping(&as_str(Some(&biome)).unwrap_or_default()) { + match matches.iter_mut().find(|(k, _)| *k == m) { + Some((_, n)) => *n += 1, + None => matches.push((m, 1)), + } + } + } + } + let best = matches + .iter() + .fold(None::<(&str, i32)>, |best, &(k, n)| match best { + Some((_, b)) if b >= n => best, + _ => Some((k, n)), + }); + Some(best.map_or(conv.fallback, |(k, _)| k).to_owned()) +} + +/// `StructuresBecomeConfiguredFix`. +fn structures_become_configured(cx: &mut Cx) { + cx.each(Ty::Chunk, |chunk| { + let Some(structures) = chunk.get("structures").cloned() else { + return; + }; + let mut structures = structures; + if let Some(Tag::Compound(starts)) = structures.get("starts") { + let mut out: Vec<(String, Tag)> = Vec::new(); + for (k, start) in starts { + if as_str(start.get("id")).as_deref().unwrap_or("INVALID") == "INVALID" { + continue; + } + if let Some(new) = updated_structure(k, chunk) + && !out.iter().any(|(n, _)| *n == new) + { + let mut start = start.clone(); + set(&mut start, "id", string(&new)); + out.push((new, start)); + } + } + let map = out + .into_iter() + .map(|(k, v)| (JavaString::from(k.as_str()), v)); + set(&mut structures, "starts", Tag::Compound(map.collect())); + } else if structures.get("starts").is_some() { + set(&mut structures, "starts", empty_map()); + } + if let Some(refs) = structures.get("References") { + let mut out: Vec<(String, Vec)> = Vec::new(); + if let Tag::Compound(refs) = refs { + for (k, list) in refs { + let list = stream(Some(list)); + if list.is_empty() { + continue; + } + if let Some(new) = updated_structure(k, chunk) { + match out.iter_mut().find(|(n, _)| *n == new) { + Some((_, old)) => old.extend(list), + None => out.push((new, list)), + } + } + } + } + let map = out.into_iter().map(|(k, v)| { + let longs = v + .iter() + .map(|t| super::dynamic::as_long(Some(t), 0)) + .collect(); + (JavaString::from(k.as_str()), Tag::LongArray(longs)) + }); + set(&mut structures, "References", Tag::Compound(map.collect())); + } + set(chunk, "structures", structures); + }); +} diff --git a/crates/horizon-world/src/datafix/mod.rs b/crates/horizon-world/src/datafix/mod.rs index ad5c6f342..e52d1e621 100644 --- a/crates/horizon-world/src/datafix/mod.rs +++ b/crates/horizon-world/src/datafix/mod.rs @@ -14,6 +14,7 @@ pub mod files; mod fixes_a; mod fixes_b; mod fixes_c; +mod fixes_cliffs; mod fixes_old; mod items; pub mod json; @@ -26,9 +27,9 @@ use schema::{Key, Ty, key, walk}; /// The DataVersion of the game. pub const CURRENT: i32 = crate::DATA_VERSION; -/// Data saved before this DataVersion (1.18.2 is 2975) needs fixes that are not -/// ported (2967 and 2970, and everything before). -pub const OLDEST_SUPPORTED: i32 = 2970; +/// Data saved before this DataVersion (1.18 is 2860) needs fixes that are not +/// ported (2852 and everything before). +pub const OLDEST_SUPPORTED: i32 = 2852; /// `Identifier.tryParse` validity, as `NamespacedSchema.ensureNamespaced` uses it. pub fn identifier(id: &str) -> Option<(String, String)> { @@ -176,6 +177,7 @@ type FixFn = fn(&mut Cx); /// Every ported fix (from the 1.19 schemas on), in DataFixers order. fn fixes() -> Vec<(Key, FixFn)> { let mut out = Vec::new(); + fixes_cliffs::register(&mut out); fixes_old::register(&mut out); fixes_a::register(&mut out); fixes_b::register(&mut out); diff --git a/research-lock.json b/research-lock.json index 8d5fe6eda..7537e6f81 100644 --- a/research-lock.json +++ b/research-lock.json @@ -18,5 +18,37 @@ "version": "0.7.1", "url": "https://github.com/1013503897/jadx-headless-mcp/releases/download/v0.7.1/jadx-headless-mcp-0.7.1-all.jar", "sha256": "6e5eacf500b64292bfb73c49797c1958f6ee44646e43e868039ae7feb573ff75" + }, + "java_runtimes": { + "note": "Mojang's Java runtimes for the old server jars that make the datafix sample worlds (tools/datafix_worlds.py). A component a platform lacks falls back to java-runtime-gamma.", + "linux": { + "jre-legacy": { + "version": "8u202", + "manifest_url": "https://piston-meta.mojang.com/v1/packages/c529f68a6febc042e71835a579f44d55a3717c46/manifest.json", + "manifest_sha1": "c529f68a6febc042e71835a579f44d55a3717c46" + }, + "java-runtime-alpha": { + "version": "16.0.1.9.1", + "manifest_url": "https://piston-meta.mojang.com/v1/packages/3bfc5fdcc28d8897aa12f372ea98a9afeb11a813/manifest.json", + "manifest_sha1": "3bfc5fdcc28d8897aa12f372ea98a9afeb11a813" + }, + "java-runtime-beta": { + "version": "17.0.15", + "manifest_url": "https://piston-meta.mojang.com/v1/packages/4be835c1384c4bf1ebcb5c3bfe999ad2b1fe88ac/manifest.json", + "manifest_sha1": "4be835c1384c4bf1ebcb5c3bfe999ad2b1fe88ac" + }, + "java-runtime-gamma": { + "version": "17.0.15", + "manifest_url": "https://piston-meta.mojang.com/v1/packages/71651340afda9fe593026fa7c56b1d9e2dbc1292/manifest.json", + "manifest_sha1": "71651340afda9fe593026fa7c56b1d9e2dbc1292" + } + }, + "mac-os-arm64": { + "java-runtime-gamma": { + "version": "17.0.15", + "manifest_url": "https://piston-meta.mojang.com/v1/packages/e0c8d5ccfe8cfb38075ff1b7f6352d7b03fd7868/manifest.json", + "manifest_sha1": "e0c8d5ccfe8cfb38075ff1b7f6352d7b03fd7868" + } + } } } diff --git a/tools/datafix_worlds.py b/tools/datafix_worlds.py index a89581e84..6c442f172 100644 --- a/tools/datafix_worlds.py +++ b/tools/datafix_worlds.py @@ -37,31 +37,106 @@ LOCAL = ROOT / '.local' / 'datafix' # Old server jars and their worlds are inputs, not outputs: keep them beside the # vanilla data, which parallel worktrees share (.local/minecraft links there). SHARED = (ROOT / '.local' / 'minecraft').resolve().parent / 'datafix' -VERSIONS = ['1.18.2', '1.19', '1.19.2', '1.19.3', '1.19.4', '1.20.1', '1.20.2', '1.20.4', '1.20.5', '1.20.6', '1.21.1', +VERSIONS = ['1.18.1', '1.18.2', '1.19', '1.19.2', '1.19.3', '1.19.4', '1.20.1', '1.20.2', '1.20.4', '1.20.5', '1.20.6', '1.21.1', '1.21.3', '1.21.4', '1.21.5', '1.21.6', '1.21.8', '1.21.9', '1.21.10', '1.21.11', '26.1', '26.1.2', '26.2'] MANIFEST = 'https://piston-meta.mojang.com/mc/game/version_manifest_v2.json' +LOCK = json.loads((ROOT / 'research-lock.json').read_text())['java_runtimes'] BOT = 'Upgrader' UUID_BASE = 0x6000 -def server_jar(version): - path = SHARED / 'servers' / version / 'server.jar' +def version_meta(version): + path = SHARED / 'servers' / version / 'version.json' if not path.exists(): path.parent.mkdir(parents=True, exist_ok=True) manifest = json.load(urllib.request.urlopen(MANIFEST)) entry = next(v for v in manifest['versions'] if v['id'] == version) - meta = json.load(urllib.request.urlopen(entry['url'])) - download = meta['downloads']['server'] + data = urllib.request.urlopen(entry['url']).read() + if hashlib.sha1(data).hexdigest() != entry['sha1']: + sys.exit(f'{version}: version metadata checksum mismatch') + path.write_bytes(data) + return json.loads(path.read_text()) + + +def server_jar(version): + """The server jar, its DataVersion (None before 1.9: no DataVersion) and protocol.""" + path = SHARED / 'servers' / version / 'server.jar' + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + download = version_meta(version)['downloads']['server'] data = urllib.request.urlopen(download['url']).read() if hashlib.sha1(data).hexdigest() != download['sha1']: sys.exit(f'{version}: server.jar checksum mismatch') path.write_bytes(data) - info = json.loads(zipfile.ZipFile(path).read('version.json')) - return path, info['world_version'], info['protocol_version'] + jar = zipfile.ZipFile(path) + if 'version.json' in jar.namelist(): + info = json.loads(jar.read('version.json')) + return path, info['world_version'], info['protocol_version'] + return path, OLD_VERSIONS[version][0], OLD_VERSIONS[version][1] + + +# DataVersion and protocol of servers whose jar has no version.json (before 1.14). +OLD_VERSIONS = { + '1.13.2': (1631, 404), '1.12.2': (1343, 340), '1.11.2': (922, 316), '1.10.2': (512, 210), + '1.9.4': (184, 110), '1.8.9': (None, 47), +} + + +def platform(): + import platform as p + if sys.platform == 'darwin': + return 'mac-os-arm64' if p.machine() == 'arm64' else 'mac-os' + return 'linux' + + +def java(version): + """The java executable for a server version: Mojang's runtime of the version's + component (pinned in research-lock.json), installed beside the shared data.""" + meta = version_meta(version) + component = meta.get('javaVersion', {}).get('component', 'jre-legacy') + if component in ('java-runtime-delta', 'java-runtime-epsilon'): + return 'java' # 1.20.5 on: the JDK that runs 26.3 runs them + runtimes = LOCK[platform()] + if component not in runtimes: + component = 'java-runtime-gamma' + pin = runtimes[component] + home = SHARED / 'java' / f'{component}-{platform()}' + stamp = home / '.complete' + if not stamp.exists() or stamp.read_text() != pin['manifest_sha1']: + install_runtime(pin, home) + stamp.write_text(pin['manifest_sha1']) + exe = next(home.glob('**/bin/java')) + return str(exe) + + +def install_runtime(pin, home): + data = urllib.request.urlopen(pin['manifest_url']).read() + if hashlib.sha1(data).hexdigest() != pin['manifest_sha1']: + sys.exit(f'{pin["manifest_url"]}: checksum mismatch') + shutil.rmtree(home, ignore_errors=True) + links = [] + for name, entry in sorted(json.loads(data)['files'].items()): + target = home / name + if entry['type'] == 'directory': + target.mkdir(parents=True, exist_ok=True) + elif entry['type'] == 'link': + links.append((target, entry['target'])) + else: + raw = entry['downloads']['raw'] + body = urllib.request.urlopen(raw['url']).read() + if hashlib.sha1(body).hexdigest() != raw['sha1']: + sys.exit(f'{name}: checksum mismatch') + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(body) + if entry.get('executable'): + target.chmod(0o755) + for target, to in links: + target.parent.mkdir(parents=True, exist_ok=True) + target.symlink_to(to) class Server: - def __init__(self, directory, jar, port, packs=()): + def __init__(self, directory, jar, port, packs=(), java='java'): directory.mkdir(parents=True, exist_ok=True) (directory / 'eula.txt').write_text('eula=true\n') props = {'server-port': port, 'online-mode': 'false', 'level-name': 'world', 'level-seed': '4242', @@ -70,7 +145,7 @@ class Server: if packs: props['initial-enabled-packs'] = ','.join(['vanilla', *packs]) (directory / 'server.properties').write_text(''.join(f'{k}={v}\n' for k, v in props.items())) - self.process = subprocess.Popen(['java', '-Xmx2G', '-jar', str(jar), 'nogui'], cwd=directory, + self.process = subprocess.Popen([java, '-Xmx2G', '-jar', str(jar), 'nogui'], cwd=directory, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) self.lines = [] self.queue = queue.Queue() @@ -283,7 +358,7 @@ def build(version, force=False): work = LOCAL / 'gen' / version shutil.rmtree(work, ignore_errors=True) port = 30000 + (os.getpid() + data_version) % 20000 - server = Server(work, jar, port, scene.experiments(data_version)) + server = Server(work, jar, port, scene.experiments(data_version), java(version)) problems = [] def run(command):