From 9fa3a684c6bc5c45d456b54fb5ec7d388d7abb42 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:03:26 +0200 Subject: [PATCH] horizon-prepare: the resource preparation in Rust A port of tools/prepare_native_client.py and the helpers it imported from lab.py and the test_*_parity.py modules: the player's pinned game files are imported and checked against research-lock.json, the client's assets and data copied, the vanilla exporters compiled and run with the JDK (JAVA_HOME, the PATH, or java_home on macOS, at the pinned java_major), and the resources and their manifest written byte for byte as the script wrote them, with its sha256-keyed skips. tools/compare_prepare.py checks that against the Python preparer. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 184 ++++ Cargo.toml | 2 +- crates/horizon-prepare/Cargo.toml | 19 + crates/horizon-prepare/build.rs | 38 + crates/horizon-prepare/src/artifacts.rs | 681 ++++++++++++++ crates/horizon-prepare/src/inventory.rs | 177 ++++ crates/horizon-prepare/src/java.rs | 179 ++++ crates/horizon-prepare/src/lib.rs | 1134 +++++++++++++++++++++++ crates/horizon-prepare/src/main.rs | 109 +++ crates/horizon-prepare/src/pyjson.rs | 541 +++++++++++ crates/horizon-prepare/src/survival.rs | 146 +++ tools/compare_prepare.py | 149 +++ 12 files changed, 3358 insertions(+), 1 deletion(-) create mode 100644 crates/horizon-prepare/Cargo.toml create mode 100644 crates/horizon-prepare/build.rs create mode 100644 crates/horizon-prepare/src/artifacts.rs create mode 100644 crates/horizon-prepare/src/inventory.rs create mode 100644 crates/horizon-prepare/src/java.rs create mode 100644 crates/horizon-prepare/src/lib.rs create mode 100644 crates/horizon-prepare/src/main.rs create mode 100644 crates/horizon-prepare/src/pyjson.rs create mode 100644 crates/horizon-prepare/src/survival.rs create mode 100644 tools/compare_prepare.py diff --git a/Cargo.lock b/Cargo.lock index f0e8c5987..b95c1ab9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -99,6 +108,15 @@ dependencies = [ "libc", ] +[[package]] +name = "arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arboard" version = "3.6.1" @@ -183,6 +201,15 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.5.1" @@ -412,6 +439,15 @@ dependencies = [ "windows 0.54.0", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -433,6 +469,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "cursor-icon" version = "1.2.0" @@ -445,6 +491,27 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "derive_arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dispatch" version = "0.2.0" @@ -461,6 +528,17 @@ dependencies = [ "objc2 0.6.5", ] +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "dlib" version = "0.5.3" @@ -622,6 +700,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -761,6 +849,16 @@ dependencies = [ "libm", ] +[[package]] +name = "horizon-prepare" +version = "0.1.0" +dependencies = [ + "regex", + "sha1", + "sha2", + "zip", +] + [[package]] name = "horizon-protocol" version = "0.1.0" @@ -1754,6 +1852,35 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "renderdoc-sys" version = "1.1.0" @@ -1889,6 +2016,28 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -2214,6 +2363,12 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.26" @@ -3068,6 +3223,23 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror 2.0.21", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.8" @@ -3079,3 +3251,15 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index a0eb0c0f2..46f2b68fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] exclude = ["launcher"] -members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme"] +members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare"] resolver = "2" [workspace.package] diff --git a/crates/horizon-prepare/Cargo.toml b/crates/horizon-prepare/Cargo.toml new file mode 100644 index 000000000..3d62202bd --- /dev/null +++ b/crates/horizon-prepare/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "horizon-prepare" +version = "0.1.0" +edition = "2021" +license = "MIT" +description = "Prepares Horizon's resources from the player's own Minecraft: imports and checks the pinned jars, runs the vanilla exporters, writes the resource folder" + +[dependencies] +regex = { version = "1", default-features = false, features = ["std", "unicode-perl"] } +sha1 = "0.10" +sha2 = "0.10" +zip = { version = "2", default-features = false, features = ["deflate"] } + +[build-dependencies] +sha2 = "0.10" + +[[bin]] +name = "horizon-prepare" +path = "src/main.rs" diff --git a/crates/horizon-prepare/build.rs b/crates/horizon-prepare/build.rs new file mode 100644 index 000000000..fa2ef8a11 --- /dev/null +++ b/crates/horizon-prepare/build.rs @@ -0,0 +1,38 @@ +//! HORIZON_PREPARE_SOURCES_SHA256: a digest of this crate's sources, which the launcher compares +//! with a version folder's copy to know whether it may prepare that version in-process. Keep the +//! algorithm the same as `horizon_prepare::sources_sha256`. + +use sha2::Digest; +use std::fs; +use std::path::Path; + +fn main() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + for name in ["Cargo.toml", "build.rs"] { + println!("cargo:rerun-if-changed={name}"); + files.push((name.to_string(), fs::read(dir.join(name)).unwrap())); + } + println!("cargo:rerun-if-changed=src"); + let mut sources: Vec = fs::read_dir(dir.join("src")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|n| n.ends_with(".rs")) + .collect(); + sources.sort(); + for name in sources { + println!("cargo:rerun-if-changed=src/{name}"); + files.push(( + format!("src/{name}"), + fs::read(dir.join("src").join(&name)).unwrap(), + )); + } + let mut hash = sha2::Sha256::new(); + for (name, data) in files { + let data: Vec = data.into_iter().filter(|b| *b != b'\r').collect(); + hash.update(format!("{name}\0{}\0", data.len())); + hash.update(&data); + } + let hex: String = hash.finalize().iter().map(|b| format!("{b:02x}")).collect(); + println!("cargo:rustc-env=HORIZON_PREPARE_SOURCES_SHA256={hex}"); +} diff --git a/crates/horizon-prepare/src/artifacts.rs b/crates/horizon-prepare/src/artifacts.rs new file mode 100644 index 000000000..c3aa10280 --- /dev/null +++ b/crates/horizon-prepare/src/artifacts.rs @@ -0,0 +1,681 @@ +//! The player's own game files: the pinned version JSON, client jar and libraries from their +//! official launcher install, and the server jar they downloaded, each checked against +//! research-lock.json before it is copied to `.local/minecraft/` (tools/lab.py's +//! `prepare`, which the Python preparer ran first), then checked again on every run +//! (tools/test_rust_parity.py's `verify_artifacts`). + +use crate::pyjson::{self, Value}; +use crate::{digest, read_json, relative_display, write_json, Log}; +use std::collections::HashMap; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +const LAUNCHER_HINT: &str = + "Horizon does not download the game: install Minecraft Java Edition with the official \ +launcher, play {version} once, then rerun. If the launcher uses another game directory, \ +pass --minecraft-dir or set HORIZON_MINECRAFT_DIR."; +const SERVER_HINT: &str = "Horizon does not download the game: download the {version} server jar yourself (minecraft.net/download/server \ +while {version} is the current release, otherwise the downloads.server.url in your launcher's \ +versions/{version}/{version}.json), then pass --server-jar PATH or set HORIZON_SERVER_JAR."; + +fn hint(text: &str, version: &str) -> String { + text.replace("{version}", version) +} + +/// research-lock.json and the folder the pinned version's files go to. +pub struct Lock { + pub json: Value, + pub version: String, + pub java_major: i64, + pub directory: PathBuf, +} + +impl Lock { + pub fn read(root: &Path) -> Result { + let json = read_json(&root.join("research-lock.json"))?; + let game = json + .get("minecraft") + .ok_or("research-lock.json has no minecraft")?; + let version = game + .get("version") + .and_then(Value::as_str) + .ok_or("research-lock.json has no minecraft version")? + .to_string(); + if version.is_empty() + || !version + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)) + || version == "." + || version == ".." + { + return Err("Unsafe version identifier in lock file".into()); + } + let java_major = game + .get("java_major") + .and_then(Value::as_i64) + .ok_or("research-lock.json has no java_major")?; + let directory = root.join(".local/minecraft").join(&version); + Ok(Lock { + json, + version, + java_major, + directory, + }) + } + + fn pin(&self, key: &str) -> Result<&str, String> { + self.json + .get("minecraft") + .and_then(|m| m.get(key)) + .and_then(Value::as_str) + .ok_or_else(|| format!("research-lock.json has no minecraft {key}")) + } + + /// Mojang republishes a version JSON for asset-index changes; every known one is accepted. + fn manifest_accepted(&self, checksum: &str) -> bool { + let game = self.json.get("minecraft"); + self.pin("manifest_sha1").is_ok_and(|pin| pin == checksum) + || game + .and_then(|g| g.get("accepted_manifest_sha1")) + .and_then(Value::as_array) + .is_some_and(|list| list.iter().any(|v| v.as_str() == Some(checksum))) + } +} + +fn home() -> PathBuf { + let var = if cfg!(windows) { "USERPROFILE" } else { "HOME" }; + std::env::var_os(var) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(".")) +} + +/// Python's `Path.expanduser` for the forms players type: `~` and `~/...`. +fn expand_user(path: &Path) -> PathBuf { + let mut components = path.components(); + match components.next() { + Some(Component::Normal(first)) if first == "~" => home().join(components.as_path()), + _ => path.to_path_buf(), + } +} + +/// The official launcher's game directory, which holds the player's own copy of the game. +pub fn minecraft_dir(chosen: Option<&Path>) -> PathBuf { + if let Some(dir) = chosen.filter(|d| !d.as_os_str().is_empty()) { + return expand_user(dir); + } + if let Some(dir) = std::env::var_os("HORIZON_MINECRAFT_DIR").filter(|d| !d.is_empty()) { + return expand_user(Path::new(&dir)); + } + if cfg!(target_os = "macos") { + home().join("Library/Application Support/minecraft") + } else if cfg!(windows) { + std::env::var_os("APPDATA") + .map(PathBuf::from) + .unwrap_or_else(home) + .join(".minecraft") + } else { + home().join(".minecraft") + } +} + +/// Copy one file from the player's install, checked like a download; an existing verified +/// copy is kept. +fn import_copy( + source: &Path, + destination: &Path, + checksum: &str, + algorithm: &str, + hint: &str, +) -> Result { + fs::create_dir_all(destination.parent().unwrap()).map_err(|e| e.to_string())?; + if destination.exists() && digest(destination, algorithm)? == checksum { + return Ok(destination.to_path_buf()); + } + if !source.is_file() { + return Err(format!("Missing {}. {hint}", source.display()) + .trim() + .to_string()); + } + let actual = digest(source, algorithm)?; + if actual != checksum { + return Err(format!( + "{} is not the pinned file: expected {algorithm} {checksum}, got {actual}", + source.display() + )); + } + let temporary = destination.with_file_name(format!( + ".{}.partial-{}", + destination.file_name().unwrap().to_string_lossy(), + std::process::id() + )); + let copied = fs::copy(source, &temporary).and_then(|_| fs::rename(&temporary, destination)); + let _ = fs::remove_file(&temporary); + copied.map_err(|e| format!("{}: {e}", destination.display()))?; + Ok(destination.to_path_buf()) +} + +/// The pinned version JSON from the launcher's versions/ directory. The launcher's copy wins +/// over the cached one so the manifest matches the asset index installed alongside it. +fn import_manifest(lock: &Lock, minecraft: &Path) -> Result { + let version = &lock.version; + let destination = lock.directory.join("manifest.json"); + let source = minecraft + .join("versions") + .join(version) + .join(format!("{version}.json")); + if source.is_file() { + let manifest = read_json(&source)?; + let downloads = manifest.get("downloads"); + let sha1 = |side: &str| { + downloads + .and_then(|d| d.get(side)) + .and_then(|d| d.get("sha1")) + .and_then(Value::as_str) + }; + if manifest.get("id").and_then(Value::as_str) != Some(version) + || sha1("client") != Some(lock.pin("client_sha1")?) + || sha1("server") != Some(lock.pin("server_sha1")?) + { + return Err(format!( + "{} does not describe the pinned {version} release", + source.display() + )); + } + let checksum = digest(&source, "sha1")?; + if !lock.manifest_accepted(&checksum) { + return Err(format!( + "{} is a {version} version file Horizon does not know yet (sha1 {checksum}); \ + Mojang may have republished it. Update Horizon, or report it if this is the latest release.", + source.display() + )); + } + if !(destination.exists() && digest(&destination, "sha1")? == checksum) { + fs::create_dir_all(&lock.directory).map_err(|e| e.to_string())?; + fs::copy(&source, &destination).map_err(|e| e.to_string())?; + } + } else if !(destination.exists() && lock.manifest_accepted(&digest(&destination, "sha1")?)) { + return Err(format!( + "Missing {}. {}", + source.display(), + hint(LAUNCHER_HINT, version) + )); + } + let manifest = read_json(&destination)?; + let java = manifest + .get("javaVersion") + .and_then(|j| j.get("majorVersion")) + .and_then(Value::as_i64); + if java != Some(lock.java_major) { + return Err("Manifest Java version does not match lock file".into()); + } + Ok(manifest) +} + +fn import_server(lock: &Lock, server_jar: Option<&Path>) -> Result { + let version = &lock.version; + let destination = lock.directory.join("jars/server.jar"); + let checksum = lock.pin("server_sha1")?; + let hint = hint(SERVER_HINT, version); + let source = server_jar + .map(Path::to_path_buf) + .or_else(|| std::env::var_os("HORIZON_SERVER_JAR").map(PathBuf::from)); + let Some(source) = source else { + if destination.exists() && digest(&destination, "sha1")? == checksum { + return Ok(destination); + } + return Err(format!("No {version} server jar. {hint}")); + }; + import_copy(&expand_user(&source), &destination, checksum, "sha1", &hint) +} + +/// The launcher's `os.name` for this machine. +fn rule_system() -> &'static str { + if cfg!(target_os = "macos") { + "osx" + } else if cfg!(windows) { + "windows" + } else { + "linux" + } +} + +/// This machine's architecture and its aliases, as Python's `platform.machine()` names them. +fn rule_architectures() -> Vec<&'static str> { + match std::env::consts::ARCH { + "x86_64" => vec!["x86_64", "amd64", "x64"], + "aarch64" => vec!["arm64", "aarch64"], + "x86" => vec!["i686"], + other => vec![other], + } +} + +/// The kernel release Python's `platform.release()` reports (only old version JSONs ask). +fn os_release() -> String { + #[cfg(unix)] + { + if let Ok(out) = std::process::Command::new("uname").arg("-r").output() { + return String::from_utf8_lossy(&out.stdout).trim().to_string(); + } + } + String::new() +} + +/// The official launcher's rules: the last matching rule decides, no rules means allowed. +fn library_allowed(library: &Value) -> Result { + let system = rule_system(); + let architectures = rule_architectures(); + let rules = library + .get("rules") + .and_then(Value::as_array) + .unwrap_or(&[]); + let mut allowed = rules.is_empty(); + for rule in rules { + let empty = Value::Object(Vec::new()); + let os = rule.get("os").unwrap_or(&empty); + if os.get("name").and_then(Value::as_str).unwrap_or(system) != system { + continue; + } + if let Some(arch) = os.get("arch") { + let pattern = regex::Regex::new(&format!("^(?:{})$", arch.as_str().unwrap_or(""))) + .map_err(|e| e.to_string())?; + if !architectures.iter().any(|name| pattern.is_match(name)) { + continue; + } + } + if let Some(version) = os.get("version") { + let pattern = + regex::Regex::new(version.as_str().unwrap_or("")).map_err(|e| e.to_string())?; + if !pattern.is_match(&os_release()) { + continue; + } + } + if rule.get("features").is_some_and(|f| match f { + Value::Object(o) => !o.is_empty(), + Value::Array(a) => !a.is_empty(), + Value::Null | Value::Bool(false) => false, + Value::Str(s) => !s.is_empty(), + _ => true, + }) { + continue; + } + allowed = rule.get("action").and_then(Value::as_str) == Some("allow"); + } + Ok(allowed) +} + +fn classifier_ends_with(classifier: &str, names: &[&str]) -> bool { + names.iter().any(|name| { + classifier == *name + || classifier + .strip_suffix(name) + .is_some_and(|rest| rest.ends_with('-')) + }) +} + +/// Modern manifests list x64 and ARM natives with OS-only rules; keep this machine's. A plain +/// natives- jar is the x64 build when the manifest also lists an ARM variant of it. +fn native_library_matches(library: &Value, names: &[&str]) -> bool { + let arm = rule_architectures().contains(&"arm64"); + let name = library.get("name").and_then(Value::as_str).unwrap_or(""); + let classifier = name.split(':').nth(3).unwrap_or(""); + if classifier_ends_with(classifier, &["arm64", "aarch64", "aarch_64"]) { + return arm; + } + if classifier_ends_with(classifier, &["x86_64", "x64"]) { + return !arm; + } + if arm && classifier.starts_with("natives-") { + return !["arm64", "aarch64"] + .iter() + .any(|suffix| names.contains(&format!("{name}-{suffix}").as_str())); + } + true +} + +/// The client libraries the official launcher installs on this OS and architecture. +fn client_library_artifacts(manifest: &Value) -> Result, String> { + let libraries = manifest + .get("libraries") + .and_then(Value::as_array) + .ok_or("The version JSON has no libraries")?; + let names: Vec<&str> = libraries + .iter() + .filter_map(|l| l.get("name").and_then(Value::as_str)) + .collect(); + let mut artifacts = Vec::new(); + for library in libraries { + let Some(artifact) = library.get("downloads").and_then(|d| d.get("artifact")) else { + continue; + }; + if library_allowed(library)? && native_library_matches(library, &names) { + artifacts.push(artifact); + } + } + Ok(artifacts) +} + +/// A relative path with only normal components and forward slashes, from a manifest or bundle. +fn safe_relative(path: &str) -> Option { + if path.contains('\\') || path.starts_with('/') { + return None; + } + let parts: Vec<&str> = path + .split('/') + .filter(|p| !p.is_empty() && *p != ".") + .collect(); + if parts.contains(&"..") || (cfg!(windows) && parts.first().is_some_and(|p| p.contains(':'))) { + return None; + } + Some(parts.iter().collect()) +} + +fn artifact_fields(artifact: &Value) -> Result<(&str, &str), String> { + let path = artifact.get("path").and_then(Value::as_str); + let sha1 = artifact.get("sha1").and_then(Value::as_str); + path.zip(sha1) + .ok_or_else(|| "A library in the version JSON has no path or sha1".to_string()) +} + +/// `(sha256, identifier, path)` lines of a server bundle's versions.list or libraries.list. +fn bundle_entries( + archive: &mut zip::ZipArchive, + name: &str, +) -> Result, String> { + let text = crate::zip_text(archive, name)?; + let mut entries = Vec::new(); + for line in crate::split_lines(&text) { + if line.trim().is_empty() { + continue; + } + let fields: Vec<&str> = line.split('\t').collect(); + let [checksum, identifier, relative] = fields[..] else { + return Err(format!("Malformed line in {name}")); + }; + let path = + safe_relative(relative).ok_or_else(|| format!("Unsafe bundled path: {relative}"))?; + // PurePosixPath's text: empty and `.` parts dropped. + let posix = relative + .split('/') + .filter(|p| !p.is_empty() && *p != ".") + .collect::>() + .join("/"); + entries.push((checksum.to_string(), identifier.to_string(), path, posix)); + } + Ok(entries) +} + +fn extract_verified( + archive: &mut zip::ZipArchive, + member: &str, + destination: &Path, + checksum: &str, +) -> Result<(), String> { + let data = crate::zip_bytes(archive, member)?; + if crate::sha256_hex(&data) != checksum { + return Err(format!("Bundled artifact checksum mismatch: {member}")); + } + fs::create_dir_all(destination.parent().unwrap()).map_err(|e| e.to_string())?; + fs::write(destination, data).map_err(|e| format!("{}: {e}", destination.display())) +} + +/// Modern server downloads are launchers: the game jar and its libraries are inside. +fn unpack_server( + server: PathBuf, + directory: &Path, + version: &str, +) -> Result<(PathBuf, Vec), String> { + let mut archive = crate::open_zip(&server)?; + if archive.index_for_name("META-INF/versions.list").is_none() { + return Ok((server, Vec::new())); + } + let versions = bundle_entries(&mut archive, "META-INF/versions.list")?; + if versions.len() != 1 || versions[0].1 != version { + return Err("Expected one bundled server matching the pinned version".into()); + } + let (checksum, _, _, relative) = &versions[0]; + let game = directory.join("jars/server-game.jar"); + extract_verified( + &mut archive, + &format!("META-INF/versions/{relative}"), + &game, + checksum, + )?; + let mut libraries = Vec::new(); + for (checksum, _, path, relative) in bundle_entries(&mut archive, "META-INF/libraries.list")? { + let destination = directory.join("libraries/server").join(path); + extract_verified( + &mut archive, + &format!("META-INF/libraries/{relative}"), + &destination, + checksum.as_str(), + )?; + libraries.push(destination); + } + Ok((game, libraries)) +} + +/// Imports the player's own pinned game files into `.local/minecraft/` and writes the +/// class inventories and prepared.json (tools/lab.py `prepare`). The decompiler lab.py also +/// fetches is a development tool, not something preparing needs; `tools/lab.py prepare` gets it. +pub fn import( + root: &Path, + lock: &Lock, + minecraft: Option<&Path>, + server_jar: Option<&Path>, + log: Log, +) -> Result<(), String> { + let minecraft = minecraft_dir(minecraft); + let version = &lock.version; + let manifest = import_manifest(lock, &minecraft)?; + let client = import_copy( + &minecraft + .join("versions") + .join(version) + .join(format!("{version}.jar")), + &lock.directory.join("jars/client.jar"), + lock.pin("client_sha1")?, + "sha1", + &hint(LAUNCHER_HINT, version), + )?; + let downloads = manifest.get("downloads"); + if downloads.and_then(|d| d.get("client_mappings")).is_some() + || downloads.and_then(|d| d.get("server_mappings")).is_some() + { + return Err( + "This workflow targets unobfuscated releases. Older releases need a remapping stage." + .into(), + ); + } + let id = manifest + .get("id") + .and_then(Value::as_str) + .ok_or("The version JSON has no id")?; + log(&format!( + "Imported client {id} from {}", + minecraft.display() + )); + let server = import_server(lock, server_jar)?; + let (server, server_libraries) = unpack_server(server, &lock.directory, id)?; + let mut client_libraries = Vec::new(); + for artifact in client_library_artifacts(&manifest)? { + let (path, sha1) = artifact_fields(artifact)?; + let relative = safe_relative(path).ok_or("Unsafe library path")?; + client_libraries.push(import_copy( + &minecraft.join("libraries").join(&relative), + &lock.directory.join("libraries/client").join(&relative), + sha1, + "sha1", + &hint(LAUNCHER_HINT, version), + )?); + } + for (side, jar) in [("client", &client), ("server", &server)] { + let data = crate::inventory::inventory(jar)?; + let count = |key| data.get(key).cloned().unwrap_or(Value::Null); + write_json( + &lock + .directory + .join(format!("reports/{side}-inventory.json")), + &data, + )?; + log(&format!( + "{side}: {} classes, {} methods", + pyjson::dumps(&count("class_count"), &pyjson::DEFAULT), + pyjson::dumps(&count("method_count"), &pyjson::DEFAULT) + )); + } + let paths = |list: &[PathBuf]| -> Result { + Ok(Value::Array( + list.iter() + .map(|p| relative_display(p, root).map(Value::Str)) + .collect::>()?, + )) + }; + write_json( + &lock.directory.join("prepared.json"), + &Value::Object(vec![ + ("version".into(), Value::str(id)), + ( + "jars".into(), + Value::Object(vec![ + ( + "client".into(), + Value::Str(relative_display(&client, root)?), + ), + ( + "server".into(), + Value::Str(relative_display(&server, root)?), + ), + ]), + ), + ( + "libraries".into(), + Value::Object(vec![ + ("client".into(), paths(&client_libraries)?), + ("server".into(), paths(&server_libraries)?), + ]), + ), + ]), + )?; + log(&format!("Artifacts: {}", lock.directory.display())); + Ok(()) +} + +/// The checked jars (`client`, `server` = the bundle's inner game jar) and the server +/// libraries, authenticated against the lock and the bundle's own checksums on every run. +pub struct Artifacts { + pub manifest: Value, + pub jars: HashMap<&'static str, PathBuf>, + pub libraries: Vec, +} + +pub fn verify(lock: &Lock) -> Result { + let manifest_path = lock.directory.join("manifest.json"); + if !manifest_path.exists() { + return Err(format!( + "{} is missing: prepare with --prepare first", + manifest_path.display() + )); + } + if !lock.manifest_accepted(&digest(&manifest_path, "sha1")?) { + return Err("Manifest checksum changed; prepare again with --prepare".into()); + } + let manifest = read_json(&manifest_path)?; + let id = manifest.get("id").and_then(Value::as_str).unwrap_or(""); + if id != lock.version { + return Err("Manifest version differs from the lock".into()); + } + let mut jars = HashMap::new(); + for side in ["client", "server"] { + let jar = lock.directory.join(format!("jars/{side}.jar")); + let pinned = manifest + .get("downloads") + .and_then(|d| d.get(side)) + .and_then(|d| d.get("sha1")) + .and_then(Value::as_str); + if !jar.exists() || Some(digest(&jar, "sha1")?.as_str()) != pinned { + return Err(format!( + "{side} artifact checksum changed; prepare again with --prepare" + )); + } + jars.insert(side, jar); + } + let mut archive = crate::open_zip(&jars["server"])?; + let versions = bundle_entries(&mut archive, "META-INF/versions.list")?; + if versions.len() != 1 || versions[0].1 != id { + return Err("Unexpected server bundle".into()); + } + let game = lock.directory.join("jars/server-game.jar"); + if !game.exists() || digest(&game, "sha256")? != versions[0].0 { + return Err("Inner server artifact checksum changed".into()); + } + jars.insert("server", game); + let mut libraries = Vec::new(); + for (checksum, _, path, relative) in bundle_entries(&mut archive, "META-INF/libraries.list")? { + let library = lock.directory.join("libraries/server").join(path); + if !library.exists() || digest(&library, "sha256")? != checksum { + return Err(format!("Dependency checksum changed: {relative}")); + } + libraries.push(library); + } + Ok(Artifacts { + manifest, + jars, + libraries, + }) +} + +/// The client libraries, checked against the version JSON (tools/test_model_baking_parity.py's +/// `client_libraries`); the client-side exporters compile against them. +pub fn client_libraries(lock: &Lock, manifest: &Value) -> Result, String> { + let mut libraries = Vec::new(); + for artifact in client_library_artifacts(manifest)? { + let (path, sha1) = artifact_fields(artifact)?; + let relative = safe_relative(path).ok_or("Unsafe client library path")?; + let library = lock.directory.join("libraries/client").join(&relative); + if !library.exists() || digest(&library, "sha1")? != sha1 { + return Err(format!("Client dependency checksum changed: {path}")); + } + libraries.push(library); + } + Ok(libraries) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn natives_follow_the_architecture() { + let names = ["a:b:1:natives-macos", "a:b:1:natives-macos-arm64"]; + let lib = |name: &str| pyjson::loads(&format!("{{\"name\":\"{name}\"}}")).unwrap(); + let arm = rule_architectures().contains(&"arm64"); + assert_eq!( + native_library_matches(&lib("a:b:1:natives-macos-arm64"), &names), + arm + ); + assert_eq!( + native_library_matches(&lib("x:y:1:linux-x86_64"), &names), + !arm + ); + assert_eq!( + native_library_matches(&lib("x:y:1:linux-aarch_64"), &names), + arm + ); + assert_eq!( + native_library_matches(&lib("a:b:1:natives-macos"), &names), + !arm + ); + assert!(native_library_matches(&lib("a:b:1"), &names)); + } + + #[test] + fn unsafe_paths_are_refused() { + assert!(safe_relative("a/../b").is_none()); + assert!(safe_relative("/a").is_none()); + assert!(safe_relative("a\\b").is_none()); + assert_eq!( + safe_relative("a/b.jar").unwrap(), + Path::new("a").join("b.jar") + ); + } +} diff --git a/crates/horizon-prepare/src/inventory.rs b/crates/horizon-prepare/src/inventory.rs new file mode 100644 index 000000000..fbf9097b7 --- /dev/null +++ b/crates/horizon-prepare/src/inventory.rs @@ -0,0 +1,177 @@ +//! Class inventories of the imported jars (tools/lab.py's `inventory` and `ClassReader`): each +//! class's SourceFile and method count, read from the bytecode; no Java text is published. + +use crate::pyjson::Value; +use std::path::Path; + +struct ClassReader<'a> { + data: &'a [u8], + offset: usize, +} + +impl ClassReader<'_> { + fn take(&mut self, count: usize) -> Result<&[u8], String> { + let value = self + .data + .get(self.offset..self.offset + count) + .ok_or("Truncated class file")?; + self.offset += count; + Ok(value) + } + + fn number(&mut self, count: usize) -> Result { + Ok(self + .take(count)? + .iter() + .fold(0usize, |n, b| (n << 8) | *b as usize)) + } + + fn attributes(&mut self) -> Result)>, String> { + let count = self.number(2)?; + let mut attributes = Vec::with_capacity(count); + for _ in 0..count { + let name = self.number(2)?; + let length = self.number(4)?; + attributes.push((name, self.take(length)?.to_vec())); + } + Ok(attributes) + } + + /// `(SourceFile, method count)`. + fn metadata(&mut self) -> Result<(Option, usize), String> { + if self.take(4)? != b"\xca\xfe\xba\xbe" { + return Err("Invalid class file magic".into()); + } + self.take(4)?; // Minor and major version. + let mut pool: Vec> = vec![None; self.number(2)?]; + let mut i = 1; + while i < pool.len() { + let tag = self.number(1)?; + if tag == 1 { + let length = self.number(2)?; + pool[i] = Some(String::from_utf8_lossy(self.take(length)?).into_owned()); + } else { + let size = match tag { + 3 | 4 | 9 | 10 | 11 | 12 | 17 | 18 => 4, + 5 | 6 => 8, + 7 | 8 | 16 | 19 | 20 => 2, + 15 => 3, + _ => return Err(format!("Unknown constant pool tag: {tag}")), + }; + self.take(size)?; + } + i += if tag == 5 || tag == 6 { 2 } else { 1 }; + } + self.take(6)?; // Access, this class, super class. + let interfaces = self.number(2)?; + self.take(interfaces * 2)?; + for _ in 0..self.number(2)? { + self.take(6)?; // Fields. + self.attributes()?; + } + let methods = self.number(2)?; + for _ in 0..methods { + self.take(6)?; + self.attributes()?; + } + let mut source = None; + for (name, data) in self.attributes()? { + if pool.get(name).and_then(Option::as_deref) == Some("SourceFile") { + let [high, low] = data[..] else { + return Err("Malformed SourceFile attribute".into()); + }; + let index = ((high as usize) << 8) | low as usize; + source = Some( + pool.get(index) + .cloned() + .flatten() + .ok_or("SourceFile points outside the constant pool")?, + ); + } + } + Ok((source, methods)) + } +} + +/// PurePosixPath(name).parent as text. +fn parent(name: &str) -> &str { + match name.trim_end_matches('/').rsplit_once('/') { + Some((parent, _)) if !parent.is_empty() => parent, + Some(_) => "/", + None => ".", + } +} + +pub fn inventory(jar: &Path) -> Result { + let mut archive = crate::open_zip(jar)?; + let mut classes = Vec::new(); + let mut packages: Vec<(String, usize)> = Vec::new(); + let mut sources = std::collections::BTreeSet::new(); + let mut method_count = 0; + for index in 0..archive.len() { + let mut item = archive.by_index(index).map_err(|e| e.to_string())?; + let name = item.name().to_string(); + if !name.ends_with(".class") { + continue; + } + let size = item.size(); + let mut data = Vec::with_capacity(size as usize); + std::io::Read::read_to_end(&mut item, &mut data).map_err(|e| e.to_string())?; + let (source, methods) = ClassReader { + data: &data, + offset: 0, + } + .metadata() + .map_err(|e| format!("{name}: {e}"))?; + let file = name.rsplit('/').next().unwrap(); + let source = source.filter(|s| !s.is_empty()).unwrap_or_else(|| { + let stem = file.split('$').next().unwrap(); + format!("{}.java", stem.strip_suffix(".class").unwrap_or(stem)) + }); + if source.contains('/') || source.contains('\\') || source == "." { + return Err(format!("Unsafe SourceFile attribute in {name}")); + } + let directory = parent(&name); + let source_path = if directory == "." { + source + } else { + format!("{directory}/{source}") + }; + match packages.iter_mut().find(|(p, _)| p == directory) { + Some(entry) => entry.1 += 1, + None => packages.push((directory.to_string(), 1)), + } + sources.insert(source_path.clone()); + method_count += methods; + classes.push(Value::Object(vec![ + ("class".into(), Value::Str(name)), + ("source".into(), Value::Str(source_path)), + ("bytecode_bytes".into(), Value::Int(size.to_string())), + ("methods".into(), Value::Int(methods.to_string())), + ])); + } + // Counter.most_common: by count, ties in first-seen order. + packages.sort_by_key(|(_, count)| std::cmp::Reverse(*count)); + Ok(Value::Object(vec![ + ( + "jar_sha256".into(), + Value::Str(crate::digest(jar, "sha256")?), + ), + ("class_count".into(), Value::Int(classes.len().to_string())), + ( + "expected_source_files".into(), + Value::Array(sources.into_iter().map(Value::Str).collect()), + ), + ("method_count".into(), Value::Int(method_count.to_string())), + ( + "packages".into(), + Value::Object( + packages + .into_iter() + .map(|(p, n)| (p, Value::Int(n.to_string()))) + .collect(), + ), + ), + ("classes".into(), Value::Array(classes)), + ])) +} diff --git a/crates/horizon-prepare/src/java.rs b/crates/horizon-prepare/src/java.rs new file mode 100644 index 000000000..860f4835a --- /dev/null +++ b/crates/horizon-prepare/src/java.rs @@ -0,0 +1,179 @@ +//! The JDK that compiles and runs the vanilla exporters. They call Mojang's own code, so they +//! stay Java; everything around them is here. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +pub struct Jdk { + pub javac: PathBuf, + pub java: PathBuf, +} + +const TIMEOUT: Duration = Duration::from_secs(3600); + +fn executable(dir: &Path, name: &str) -> PathBuf { + dir.join(if cfg!(windows) { + format!("{name}.exe") + } else { + name.to_string() + }) +} + +fn on_path(name: &str) -> Option { + std::env::split_paths(&std::env::var_os("PATH")?) + .map(|dir| executable(&dir, name)) + .find(|p| p.is_file()) +} + +/// The major version `javac -version` reports ("javac 25.0.1" → 25; "javac 1.8.0" → 8). +fn javac_major(javac: &Path) -> Result { + let output = Command::new(javac) + .arg("-version") + .stdin(Stdio::null()) + .output() + .map_err(|e| format!("{}: {e}", javac.display()))?; + let text = String::from_utf8_lossy(&output.stdout).into_owned() + + &String::from_utf8_lossy(&output.stderr); + let version = text + .split_whitespace() + .skip_while(|w| *w != "javac") + .nth(1) + .ok_or_else(|| { + format!( + "{} -version printed no version: {}", + javac.display(), + text.trim() + ) + })?; + let mut parts = version.split(['.', '-', '+', '_']); + let first: u32 = parts.next().and_then(|p| p.parse().ok()).unwrap_or(0); + Ok(if first == 1 { + parts.next().and_then(|p| p.parse().ok()).unwrap_or(0) + } else { + first + }) +} + +/// Finds a JDK with `javac` of at least `major`, the version research-lock.json pins: +/// `JAVA_HOME` when it holds one, else `javac` and `java` on the PATH (as the Python preparer +/// ran them), else, on macOS, the JDK `/usr/libexec/java_home -v ` reports. +pub fn find(major: i64) -> Result { + let need = format!( + "Java {major} is needed (a JDK with javac, e.g. Eclipse Temurin {major}). \ + Install it, or set JAVA_HOME to it, and try again." + ); + let mut candidates = Vec::new(); + if let Some(home) = std::env::var_os("JAVA_HOME").filter(|h| !h.is_empty()) { + let bin = Path::new(&home).join("bin"); + candidates.push(( + executable(&bin, "javac"), + executable(&bin, "java"), + "JAVA_HOME", + )); + } + if let (Some(javac), Some(java)) = (on_path("javac"), on_path("java")) { + candidates.push((javac, java, "PATH")); + } + if cfg!(target_os = "macos") { + if let Ok(out) = Command::new("/usr/libexec/java_home") + .args(["-v", &major.to_string()]) + .stderr(Stdio::null()) + .output() + { + let home = String::from_utf8_lossy(&out.stdout).trim().to_string(); + if out.status.success() && !home.is_empty() { + let bin = Path::new(&home).join("bin"); + candidates.push(( + executable(&bin, "javac"), + executable(&bin, "java"), + "/usr/libexec/java_home", + )); + } + } + } + let mut problems = Vec::new(); + for (javac, java, from) in candidates { + if !javac.is_file() || !java.is_file() { + problems.push(format!( + "{from} ({}) has no javac and java", + javac.parent().unwrap().display() + )); + continue; + } + match javac_major(&javac) { + Ok(found) if i64::from(found) >= major => return Ok(Jdk { javac, java }), + Ok(found) => problems.push(format!("{from} has Java {found} ({})", javac.display())), + Err(e) => problems.push(e), + } + } + if problems.is_empty() { + Err(format!("{need} No javac was found.")) + } else { + Err(format!("{need} Found: {}.", problems.join("; "))) + } +} + +/// A classpath with the platform's separator. +pub fn classpath<'a>(paths: impl IntoIterator) -> String { + let separator = if cfg!(windows) { ";" } else { ":" }; + paths + .into_iter() + .map(|p| p.to_string_lossy().into_owned()) + .collect::>() + .join(separator) +} + +impl Jdk { + /// `javac --release -proc:none` with Python's `subprocess.run(..., check=True, + /// stdout=log, stderr=STDOUT, timeout=3600)`. + pub fn javac(&self, major: i64, args: &[String], log: &fs::File) -> Result<(), String> { + let mut command = Command::new(&self.javac); + command + .args(["--release", &major.to_string(), "-proc:none"]) + .args(args); + run(command, log) + } + + pub fn java(&self, args: &[String], cwd: Option<&Path>, log: &fs::File) -> Result<(), String> { + let mut command = Command::new(&self.java); + command.args(args); + if let Some(dir) = cwd { + command.current_dir(dir); + } + run(command, log) + } +} + +fn run(mut command: Command, log: &fs::File) -> Result<(), String> { + let out = log.try_clone().map_err(|e| e.to_string())?; + let err = log.try_clone().map_err(|e| e.to_string())?; + let program = Path::new(command.get_program()) + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(); + let mut child = command + .stdin(Stdio::null()) + .stdout(out) + .stderr(err) + .spawn() + .map_err(|e| format!("Cannot start {program}: {e}"))?; + let start = Instant::now(); + loop { + if let Some(status) = child.try_wait().map_err(|e| e.to_string())? { + return if status.success() { + Ok(()) + } else { + Err(format!("{program} failed ({status})")) + }; + } + if start.elapsed() > TIMEOUT { + let _ = child.kill(); + let _ = child.wait(); + return Err(format!("{program} took over an hour and was stopped")); + } + std::thread::sleep(Duration::from_millis(50)); + } +} diff --git a/crates/horizon-prepare/src/lib.rs b/crates/horizon-prepare/src/lib.rs new file mode 100644 index 000000000..e09b529eb --- /dev/null +++ b/crates/horizon-prepare/src/lib.rs @@ -0,0 +1,1134 @@ +//! Prepares Horizon's resources from the player's own Minecraft Java Edition: imports the pinned +//! client jar, libraries and version JSON from their official launcher install and the server +//! jar they downloaded (each checked against research-lock.json), copies the game's assets and +//! data, and runs the vanilla exporters (mods/horizon/src/test/java/dev/horizonlab/parity), which +//! call Mojang's own code and so stay Java. The result, `.local/horizon-client/` with its +//! `horizon-resources.json` manifest, is what the game's `--resources` and the launcher read. +//! +//! This is the Rust port of the Python preparer (tools/prepare_native_client.py and the helpers +//! it imported); its files are byte for byte the ones the script wrote (tools/compare_prepare.py +//! checks it), so the manifest's `schema` is unchanged. Unchanged steps are skipped on the next +//! run, keyed by the sha256 of their inputs and outputs recorded in the manifest, as before. + +pub mod artifacts; +mod inventory; +pub mod java; +pub mod pyjson; +mod survival; + +use pyjson::Value; +use sha2::Digest; +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; + +/// Where progress lines go: the terminal for the binary, the log pane for the launcher. +pub type Log<'a> = &'a dyn Fn(&str); + +/// The manifest's `schema`: the prepared files' layout, which the launcher compares. +pub const SCHEMA: i64 = 7; + +/// Digest of this crate's sources (build.rs): the launcher prepares a version in-process when +/// that version ships these same sources, and runs the version's own binary otherwise. +pub const SOURCES_SHA256: &str = env!("HORIZON_PREPARE_SOURCES_SHA256"); + +/// `SOURCES_SHA256` of the sources in `crate_dir` (a version folder's crates/horizon-prepare), +/// computed as build.rs computes it; None when the folder has none. +pub fn sources_sha256(crate_dir: &Path) -> Option { + let mut files = Vec::new(); + for name in ["Cargo.toml", "build.rs"] { + files.push((name.to_string(), fs::read(crate_dir.join(name)).ok()?)); + } + let mut sources: Vec = fs::read_dir(crate_dir.join("src")) + .ok()? + .filter_map(Result::ok) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| n.ends_with(".rs")) + .collect(); + sources.sort(); + for name in sources { + let data = fs::read(crate_dir.join("src").join(&name)).ok()?; + files.push((format!("src/{name}"), data)); + } + let mut hash = sha2::Sha256::new(); + for (name, data) in files { + // Line endings as checked out: a Windows checkout's CRLF is the same source. + let data: Vec = data.into_iter().filter(|b| *b != b'\r').collect(); + hash.update(format!("{name}\0{}\0", data.len())); + hash.update(&data); + } + Some(hex(&hash.finalize())) +} + +pub struct Options { + /// The version folder (a release folder, or a checkout): research-lock.json, the exporters' + /// Java sources, and `.local/` where everything is written. + pub root: PathBuf, + /// Import the pinned game files from the player's own copy first (`--prepare`). + pub import: bool, + /// The official launcher's game directory (`--minecraft-dir`, else HORIZON_MINECRAFT_DIR, + /// else the launcher's own). + pub minecraft_dir: Option, + /// The pinned server jar the player downloaded (`--server-jar`, else HORIZON_SERVER_JAR). + pub server_jar: Option, +} + +pub(crate) fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +pub(crate) fn sha256_hex(data: &[u8]) -> String { + hex(&sha2::Sha256::digest(data)) +} + +pub(crate) fn digest(path: &Path, algorithm: &str) -> Result { + fn stream(mut file: fs::File) -> std::io::Result { + let mut hash = D::new(); + let mut buffer = vec![0; 1 << 20]; + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + return Ok(hex(&hash.finalize())); + } + hash.update(&buffer[..n]); + } + } + let file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + match algorithm { + "sha1" => stream::(file), + _ => stream::(file), + } + .map_err(|e| format!("{}: {e}", path.display())) +} + +fn read(path: &Path) -> Result, String> { + fs::read(path).map_err(|e| format!("{}: {e}", path.display())) +} + +fn write(path: &Path, data: &[u8]) -> Result<(), String> { + fs::write(path, data).map_err(|e| format!("{}: {e}", path.display())) +} + +fn mkdir(path: &Path) -> Result<(), String> { + fs::create_dir_all(path).map_err(|e| format!("{}: {e}", path.display())) +} + +pub(crate) fn read_json(path: &Path) -> Result { + let text = String::from_utf8(read(path)?).map_err(|e| format!("{}: {e}", path.display()))?; + pyjson::loads(&text).map_err(|e| format!("{}: {e}", path.display())) +} + +/// Python's `Path.write_text`: text mode, so `\n` is the platform's line ending. +fn write_text(path: &Path, text: &str) -> Result<(), String> { + if cfg!(windows) { + write(path, text.replace('\n', "\r\n").as_bytes()) + } else { + write(path, text.as_bytes()) + } +} + +/// tools/lab.py's `write_json`: indented by two, with a final newline. +pub(crate) fn write_json(path: &Path, value: &Value) -> Result<(), String> { + mkdir(path.parent().unwrap())?; + write_text(path, &(pyjson::dumps(value, &pyjson::INDENT2) + "\n")) +} + +/// Writes `data` unless the file already holds it, so unchanged files keep their times. +fn write_if_changed(target: &Path, data: &[u8]) -> Result<(), String> { + if !(target.exists() && read(target)? == data) { + mkdir(target.parent().unwrap())?; + write(target, data)?; + } + Ok(()) +} + +fn copy(from: &Path, to: &Path) -> Result<(), String> { + write(to, &read(from)?) +} + +/// `path` relative to `root`, with the platform's separators, as Python's +/// `str(path.relative_to(ROOT))` prints it. +pub(crate) fn relative_display(path: &Path, root: &Path) -> Result { + let relative = path + .strip_prefix(root) + .map_err(|_| format!("{} is outside {}", path.display(), root.display()))?; + Ok(relative + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join(std::path::MAIN_SEPARATOR_STR)) +} + +/// Python's `str.splitlines`. +pub(crate) fn split_lines(text: &str) -> Vec<&str> { + let mut lines = Vec::new(); + let mut start = 0; + let mut chars = text.char_indices().peekable(); + while let Some((i, c)) = chars.next() { + if matches!( + c, + '\n' | '\r' + | '\u{b}' + | '\u{c}' + | '\u{1c}' + | '\u{1d}' + | '\u{1e}' + | '\u{85}' + | '\u{2028}' + | '\u{2029}' + ) { + lines.push(&text[start..i]); + let mut end = i + c.len_utf8(); + if c == '\r' && chars.peek().is_some_and(|(_, n)| *n == '\n') { + chars.next(); + end += 1; + } + start = end; + } + } + if start < text.len() { + lines.push(&text[start..]); + } + lines +} + +pub(crate) fn open_zip(path: &Path) -> Result, String> { + let file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + zip::ZipArchive::new(file).map_err(|e| format!("{}: {e}", path.display())) +} + +pub(crate) fn zip_bytes( + archive: &mut zip::ZipArchive, + name: &str, +) -> Result, String> { + let mut entry = archive.by_name(name).map_err(|e| format!("{name}: {e}"))?; + let mut data = Vec::with_capacity(entry.size() as usize); + entry + .read_to_end(&mut data) + .map_err(|e| format!("{name}: {e}"))?; + Ok(data) +} + +pub(crate) fn zip_text( + archive: &mut zip::ZipArchive, + name: &str, +) -> Result { + String::from_utf8(zip_bytes(archive, name)?).map_err(|e| format!("{name}: {e}")) +} + +/// The archive's member names in its own order (Python's `namelist`). +fn zip_names(archive: &zip::ZipArchive) -> Vec { + (0..archive.len()) + .filter_map(|i| archive.name_for_index(i).map(str::to_string)) + .collect() +} + +/// `{"minecraft:": text}` for every `.json`, sorted by member name +/// (the comprehension the Python preparer used for tags, worldgen and damage resources). +fn json_resources(archive: &mut zip::ZipArchive, prefix: &str) -> Result { + let mut names: Vec = zip_names(archive) + .into_iter() + .filter(|n| n.starts_with(prefix) && n.ends_with(".json")) + .collect(); + names.sort(); + let mut resources = Value::Object(Vec::new()); + for name in names { + let text = zip_text(archive, &name)?; + let key = format!("minecraft:{}", &name[prefix.len()..name.len() - 5]); + resources.insert(&key, Value::Str(text)); + } + Ok(resources) +} + +/// A relative member path with no `..` and not absolute (Python's `Path(name)` checks). +fn member_path(name: &str, what: &str) -> Result { + let path = Path::new(name); + if path.is_absolute() + || name.starts_with('/') + || path.components().any(|c| { + !matches!( + c, + std::path::Component::Normal(_) | std::path::Component::CurDir + ) + }) + { + return Err(what.to_string()); + } + Ok(path.to_path_buf()) +} + +/// `{relative path: sha256}` of exporter sources, in the order given +/// (tools/test_rust_parity.py's `source_hashes`). +fn source_hashes(root: &Path, sources: &[&Path]) -> Result { + let mut hashes = Value::Object(Vec::new()); + for source in sources { + hashes.insert( + &relative_display(source, root)?, + Value::Str(digest(source, "sha256")?), + ); + } + Ok(hashes) +} + +/// A classpath entry list with the platform's separator. +fn classpath_join(head: &Path, rest: &[PathBuf]) -> String { + java::classpath(std::iter::once(head).chain(rest.iter().map(PathBuf::as_path))) +} + +struct Run<'a> { + root: PathBuf, + local: PathBuf, + resources: PathBuf, + jdk: java::Jdk, + java_major: i64, + server: PathBuf, + client: PathBuf, + /// The server jar and its libraries: the classpath of the server-side exporters. + server_classpath: String, + /// The client jar and its libraries: the classpath of the client-side exporters. + client_classpath: String, + previous: Value, + origin: Value, + log: Log<'a>, +} + +const PARITY: &str = "mods/horizon/src/test/java/dev/horizonlab/parity"; + +impl Run<'_> { + fn oracle(&self, name: &str) -> PathBuf { + self.root.join(PARITY).join(name) + } + + fn previous(&self, key: &str) -> Option<&Value> { + self.previous.get(key) + } + + fn same(&self, key: &str) -> bool { + self.previous(key) == self.origin.get(key) + } + + fn file_hash(&self, name: &str) -> Result { + Ok(Value::Str(digest(&self.resources.join(name), "sha256")?)) + } + + /// Every file exists and has the hash the previous manifest recorded under `key`. + fn files_unchanged(&self, key: &str, names: &[&str]) -> Result { + for name in names { + if !self.resources.join(name).exists() + || self.previous(key).and_then(|h| h.get(name)) != Some(&self.file_hash(name)?) + { + return Ok(false); + } + } + Ok(true) + } + + fn log_file(&self, path: &Path) -> Result { + fs::File::create(path).map_err(|e| format!("{}: {e}", path.display())) + } + + /// `javac --release -proc:none -cp [extra] -d `. + fn compile( + &self, + classpath: &str, + extra: &[String], + classes: &Path, + sources: &[&Path], + log: &fs::File, + ) -> Result<(), String> { + let mut args = vec!["-cp".to_string(), classpath.to_string()]; + args.extend_from_slice(extra); + args.push("-d".into()); + args.push(classes.display().to_string()); + args.extend(sources.iter().map(|s| s.display().to_string())); + self.jdk.javac(self.java_major, &args, log) + } + + /// `java -Xmx -cp : dev.horizonlab.parity.
`. + fn run( + &self, + heap: &str, + classes: &Path, + classpath: &str, + main: &str, + args: &[String], + log: &fs::File, + ) -> Result<(), String> { + let separator = if cfg!(windows) { ";" } else { ":" }; + let mut all = vec![ + format!("-Xmx{heap}"), + "-cp".into(), + format!("{}{separator}{classpath}", classes.display()), + format!("dev.horizonlab.parity.{main}"), + ]; + all.extend_from_slice(args); + self.jdk.java(&all, None, log) + } + + /// Compiles `sources` into `classes` and runs each of `mains` with `args`, their output in + /// the log at `log`: the shape of every exporter step. + #[allow(clippy::too_many_arguments)] + fn export( + &self, + what: &str, + classpath: &str, + classes: &Path, + sources: &[&Path], + mains: &[&str], + heap: &str, + args: &[String], + log: &Path, + ) -> Result<(), String> { + (self.log)(&format!("Exporting {what} ({})", mains.join(", "))); + mkdir(classes)?; + let file = self.log_file(log)?; + let fail = |e: String| format!("{e} while exporting {what}; see {}", log.display()); + self.compile(classpath, &[], classes, sources, &file) + .map_err(fail)?; + for main in mains { + self.run(heap, classes, classpath, main, args, &file) + .map_err(fail)?; + } + Ok(()) + } +} + +/// Prepares the resources; returns their folder (`.local/horizon-client/`). +pub fn prepare(options: &Options, log: Log) -> Result { + let root = &options.root; + let lock = artifacts::Lock::read(root)?; + // Java first: nothing below can finish without it, and the player should hear it at once. + let jdk = java::find(lock.java_major)?; + if options.import { + artifacts::import( + root, + &lock, + options.minecraft_dir.as_deref(), + options.server_jar.as_deref(), + log, + )?; + } + let found = artifacts::verify(&lock)?; + let local = root.join(".local"); + let resources = local.join("horizon-client").join(&lock.version); + mkdir(&resources)?; + let manifest_path = resources.join("horizon-resources.json"); + let oracle = root.join(PARITY).join("VanillaHeightmapOracle.java"); + let origin = Value::Object(vec![ + ("minecraft".into(), Value::Str(lock.version.clone())), + ( + "client_sha256".into(), + Value::Str(digest(&found.jars["client"], "sha256")?), + ), + ( + "server_sha256".into(), + Value::Str(digest(&found.jars["server"], "sha256")?), + ), + ( + "exporter_sha256".into(), + Value::Str(digest(&oracle, "sha256")?), + ), + ]); + let previous = if manifest_path.exists() { + read_json(&manifest_path)? + } else { + Value::Object(Vec::new()) + }; + let client_libraries = artifacts::client_libraries(&lock, &found.manifest)?; + let run = Run { + root: root.clone(), + local, + resources: resources.clone(), + jdk, + java_major: lock.java_major, + server: found.jars["server"].clone(), + client: found.jars["client"].clone(), + server_classpath: classpath_join(&found.jars["server"], &found.libraries), + client_classpath: classpath_join(&found.jars["client"], &client_libraries), + previous, + origin, + log, + }; + steps(&run, &lock, &oracle)?; + log(&format!( + "Prepared Horizon native-client resources: {}", + resources.display() + )); + Ok(resources) +} + +fn arg(path: &Path) -> String { + path.display().to_string() +} + +fn steps(run: &Run, lock: &artifacts::Lock, oracle: &Path) -> Result<(), String> { + let root = &run.resources; + let repo = &run.root; + let server_cp = run.server_classpath.as_str(); + let client_cp = run.client_classpath.as_str(); + + // Every block state and its facts. + let facts = root.join("facts.json"); + let origin_same = run + .origin + .as_object() + .unwrap() + .iter() + .all(|(k, v)| run.previous(k) == Some(v)); + if !(origin_same + && facts.exists() + && run.previous("facts_sha256") == Some(&run.file_hash("facts.json")?)) + { + run.export( + "block states", + server_cp, + &root.join("classes"), + &[oracle], + &["VanillaHeightmapOracle"], + "512M", + &["export".into(), arg(&facts)], + &root.join("export.log"), + )?; + } + + // Block state to model bindings, from the client. + let binding = run.oracle("VanillaBlockStateBindingOracle.java"); + let helper = run.oracle("VanillaCuboidModelOracle.java"); + let binding_sources = source_hashes(repo, &[&binding, &helper])?; + let model_states = root.join("model-states.json"); + if !(run.previous("model_state_exporters_sha256") == Some(&binding_sources) + && run.same("client_sha256") + && model_states.exists() + && run.previous("model_states_sha256") == Some(&run.file_hash("model-states.json")?)) + { + run.export( + "model bindings", + client_cp, + &root.join("client-classes"), + &[&binding, &helper], + &["VanillaBlockStateBindingOracle"], + "512M", + &["export".into(), arg(&model_states)], + &root.join("model-state-export.log"), + )?; + } + + let block_tags = "data/minecraft/tags/block/"; + let tags = json_resources(&mut open_zip(&run.server)?, block_tags)?; + if tags != json_resources(&mut open_zip(&run.client)?, block_tags)? { + return Err("Reference block tags disagree".into()); + } + write_json(&root.join("block-tags.json"), &tags)?; + + let visibility = run.oracle("VanillaFaceVisibilityOracle.java"); + let collision = run.oracle("VanillaCollisionOracle.java"); + let collision_sources = source_hashes(repo, &[&collision])?; + let movement = run.oracle("VanillaMovementRegistryOracle.java"); + let attributes = run.oracle("VanillaEntityAttributeOracle.java"); + let landing = run.oracle("VanillaBlockLandingOracle.java"); + let motion_sources = source_hashes( + repo, + &[&movement, &attributes, oracle, &landing, &collision], + )?; + let motion_files = [ + "movement-properties.nbt", + "attributes.json", + "landing-definitions.json", + ]; + if !(run.previous("motion_exporters_sha256") == Some(&motion_sources) + && run.same("server_sha256") + && run.previous("block_tags_sha256") == Some(&run.file_hash("block-tags.json")?) + && run.files_unchanged("motion_files_sha256", &motion_files)?) + { + let export = run.local.join("horizon-client-motion"); + mkdir(&export)?; + write_json(&export.join("tags.json"), &tags)?; + run.export( + "movement and attributes", + server_cp, + &export.join("classes"), + &[&movement, &attributes, oracle, &landing, &collision], + &["VanillaBlockLandingOracle", "VanillaEntityAttributeOracle"], + "768M", + &["export".into(), arg(&export)], + &export.join("export.log"), + )?; + copy( + &export.join("movement-properties.nbt"), + &root.join("movement-properties.nbt"), + )?; + copy(&export.join("catalog.json"), &root.join("attributes.json"))?; + copy( + &export.join("landing-definitions.json"), + &root.join("landing-definitions.json"), + )?; + } + + let damage = run.oracle("VanillaDamageSourceOracle.java"); + let health = run.oracle("VanillaPlayerHealthOracle.java"); + let health_sources = source_hashes(repo, &[&damage, &health, &attributes])?; + let health_files = [ + "damage-types.json", + "damage-tags.json", + "damage-catalog.json", + "player-float-ids.json", + ]; + if !(run.previous("health_exporters_sha256") == Some(&health_sources) + && run.same("server_sha256") + && run.files_unchanged("health_files_sha256", &health_files)?) + { + let export = run.local.join("horizon-client-health"); + mkdir(&export)?; + let mut server = open_zip(&run.server)?; + let mut client = open_zip(&run.client)?; + for (file, prefix) in [ + ("types.json", "data/minecraft/damage_type/"), + ("tags.json", "data/minecraft/tags/damage_type/"), + ] { + let resources = json_resources(&mut server, prefix)?; + if resources != json_resources(&mut client, prefix)? { + return Err("Reference damage resources disagree".into()); + } + write_json(&export.join(file), &resources)?; + } + run.export( + "damage and health", + server_cp, + &export.join("classes"), + &[&damage, &health, &attributes], + &["VanillaPlayerHealthOracle", "VanillaDamageSourceOracle"], + "768M", + &["export".into(), arg(&export)], + &export.join("export.log"), + )?; + for (source, target) in [ + ("types.json", "damage-types.json"), + ("tags.json", "damage-tags.json"), + ("catalog.json", "damage-catalog.json"), + ("ids.json", "player-float-ids.json"), + ] { + copy(&export.join(source), &root.join(target))?; + } + } + + let collision_file = root.join("cached-collision.nbt"); + if !(run.previous("collision_exporters_sha256") == Some(&collision_sources) + && run.same("server_sha256") + && collision_file.exists() + && run.previous("collision_sha256") == Some(&run.file_hash("cached-collision.nbt")?)) + { + let export = run.local.join("horizon-client-collision"); + mkdir(&export)?; + run.export( + "collision shapes", + server_cp, + &export.join("classes"), + &[&collision], + &["VanillaCollisionOracle"], + "768M", + &["generate".into(), arg(&export)], + &export.join("export.log"), + )?; + copy(&export.join("cached-collision.nbt"), &collision_file)?; + } + + let render_sources = source_hashes(repo, &[&visibility, &collision, oracle])?; + let render_files = ["render-definitions.json", "occlusion.nbt"]; + if !(run.previous("render_exporters_sha256") == Some(&render_sources) + && run.same("server_sha256") + && run.previous("block_tags_sha256") == Some(&run.file_hash("block-tags.json")?) + && run.files_unchanged("render_files_sha256", &render_files)?) + { + let export = run.local.join("horizon-client-render"); + mkdir(&export)?; + write_json(&export.join("tags.json"), &tags)?; + run.export( + "face visibility", + server_cp, + &export.join("classes"), + &[&visibility, &collision, oracle], + &["VanillaFaceVisibilityOracle"], + "768M", + &["export".into(), arg(&export)], + &export.join("export.log"), + )?; + copy( + &export.join("definitions.json"), + &root.join("render-definitions.json"), + )?; + copy(&export.join("occlusion.nbt"), &root.join("occlusion.nbt"))?; + } + + // The playable client needs every client asset (textures, models, fonts, GUI) + // and every data-pack resource (worldgen, recipes, loot tables, tags). + (run.log)("Copying the client's assets and data"); + let mut count = 0usize; + let mut client = open_zip(&run.client)?; + let version = pyjson::loads(&zip_text(&mut client, "version.json")?)?; + for name in zip_names(&client) { + if !(name.starts_with("assets/") || name.starts_with("data/")) || name.ends_with('/') { + continue; + } + let relative = member_path(&name, "Invalid resource path")?; + let data = zip_bytes(&mut client, &name)?; + write_if_changed(&root.join(relative), &data)?; + count += 1; + } + + // One file each, exported again when the exporter or the jar it reads changed. + let single = |what: &str, + file: &str, + oracles: &[&str], + key: &str, + jar: &str, + classes: &str, + heap: &str, + extra: &[String], + log: &str| + -> Result { + let target = root.join(file); + let sources: Vec = oracles.iter().map(|o| run.oracle(o)).collect(); + let sources: Vec<&Path> = sources.iter().map(PathBuf::as_path).collect(); + let hashes = source_hashes(repo, &sources)?; + if !(run.previous(key) == Some(&hashes) && target.exists() && run.same(jar)) { + let main = oracles[0].strip_suffix(".java").unwrap(); + let mut args = vec![arg(&target)]; + args.extend_from_slice(extra); + run.export( + what, + if jar == "client_sha256" { + client_cp + } else { + server_cp + }, + &run.local.join(classes), + &sources, + &[main], + heap, + &args, + &root.join(log), + )?; + } + Ok(hashes) + }; + let gameplay_sources = single( + "gameplay facts", + "gameplay-facts.json", + &["VanillaGameplayFactsExporter.java"], + "gameplay_exporters_sha256", + "server_sha256", + "gameplay-facts/classes", + "1G", + &[], + "gameplay-export.log", + )?; + // Entity dimensions, categories and default attributes for the client's mobs. + let entity_types_sources = single( + "entity types", + "entity-types.json", + &["VanillaEntityTypesExporter.java"], + "entity_types_exporters_sha256", + "server_sha256", + "entity-types/classes", + "1G", + &[], + "entity-types-export.log", + )?; + // Block classes, instance configuration and per-state support/signal/fluid + // facts for horizon-world's level behaviour (level::Behaviours::load). + let behaviour_sources = single( + "block behaviours", + "block-behaviours.json", + &["VanillaBlockBehaviourExporter.java"], + "behaviour_exporters_sha256", + "server_sha256", + "block-behaviours/classes", + "1G", + &[], + "behaviour-export.log", + )?; + + let items = root.join("items.json"); + if !items.exists() || !run.same("server_sha256") { + // Vanilla's own data generator reports every item's default components. + (run.log)("Running vanilla's data generator reports"); + let datagen = run.local.join("datagen"); + mkdir(&datagen)?; + let log_path = datagen.join("datagen.log"); + let log = run.log_file(&log_path)?; + run.jdk + .java( + &[ + "-DbundlerMainClass=net.minecraft.data.Main".into(), + "-jar".into(), + arg(&lock.directory.join("jars").join("server.jar")), + "--reports".into(), + "--output".into(), + "out".into(), + ], + Some(&datagen), + &log, + ) + .map_err(|e| format!("{e} in the data generator; see {}", log_path.display()))?; + let reports = datagen.join("out").join("reports"); + let mut components = Value::Object(Vec::new()); + for file in json_files(&reports.join("minecraft/components/item"))? { + let stem = file.file_stem().unwrap().to_string_lossy().into_owned(); + let report = read_json(&file)?; + let component = report + .get("components") + .ok_or_else(|| format!("{} has no components", file.display()))?; + components.insert(&format!("minecraft:{stem}"), component.clone()); + } + write_text(&items, &pyjson::dumps(&components, &pyjson::DEFAULT))?; + let biome_parameters = root.join("biome-parameters"); + mkdir(&biome_parameters)?; + for file in json_files(&reports.join("biome_parameters/minecraft"))? { + copy(&file, &biome_parameters.join(file.file_name().unwrap()))?; + } + } + + // Biome parameter tables come from the data generator reports above. + prepare_worldgen(run)?; + + let entity_sources = single( + "entity models", + "entity-models.json", + &["VanillaEntityModelExporter.java"], + "entity_model_exporters_sha256", + "client_sha256", + "entity-models/classes", + "1G", + &[], + "entity-model-export.log", + )?; + // Mob AI and animation facts: per-state path types, spawn placements and the + // client's keyframe animation definitions (needs the client jar). + let mob_sources = single( + "mob data", + "mob-data.json", + &["VanillaMobDataExporter.java"], + "mob_data_exporters_sha256", + "client_sha256", + "mob-data/classes", + "1G", + &[arg(&run.client)], + "mob-data-export.log", + )?; + + // Vanilla's built-in registries for in-game command parsing (mob effects, + // data component types, dialogs, block entity and tag members). + let command_registries = root.join("command-registries.json"); + let command_oracle = run.oracle("VanillaCommandParseOracle.java"); + let command_sources = source_hashes(repo, &[&command_oracle])?; + if !(run.previous("command_registries_exporters_sha256") == Some(&command_sources) + && command_registries.exists() + && run.same("server_sha256")) + { + let export = run.local.join("command-registries"); + run.export( + "command registries", + server_cp, + &export.join("classes"), + &[&command_oracle], + &["VanillaCommandParseOracle"], + "1G", + &["registries".into(), arg(&export)], + &root.join("command-registries-export.log"), + )?; + copy(&export.join("registries.json"), &command_registries)?; + } + + // The creative inventory (creative-inventory.json): every tab built by vanilla's + // CreativeModeTabs over the vanilla data pack's registries, with the search + // tab's tooltip lines and tags. + let creative_sources = single( + "the creative inventory", + "creative-inventory.json", + &["VanillaCreativeTabsExporter.java", "ParityData.java"], + "creative_exporters_sha256", + "server_sha256", + "creative-inventory/classes", + "2G", + &[], + "creative-export.log", + )?; + + let file_hashes = |names: &[&str]| -> Result { + Ok(Value::Object( + names + .iter() + .map(|n| Ok((n.to_string(), run.file_hash(n)?))) + .collect::>()?, + )) + }; + let mut manifest = vec![("schema".to_string(), Value::Int(SCHEMA.to_string()))]; + manifest.extend(run.origin.as_object().unwrap().iter().cloned()); + manifest.extend([ + ("health_exporters_sha256".into(), health_sources), + ("health_files_sha256".into(), file_hashes(&health_files)?), + ("motion_exporters_sha256".into(), motion_sources), + ("motion_files_sha256".into(), file_hashes(&motion_files)?), + ("collision_exporters_sha256".into(), collision_sources), + ( + "collision_sha256".into(), + run.file_hash("cached-collision.nbt")?, + ), + ("render_exporters_sha256".into(), render_sources), + ("render_files_sha256".into(), file_hashes(&render_files)?), + ("model_state_exporters_sha256".into(), binding_sources), + ("gameplay_exporters_sha256".into(), gameplay_sources), + ("entity_model_exporters_sha256".into(), entity_sources), + ("creative_exporters_sha256".into(), creative_sources), + ("mob_data_exporters_sha256".into(), mob_sources), + ("entity_types_exporters_sha256".into(), entity_types_sources), + ("behaviour_exporters_sha256".into(), behaviour_sources), + ( + "command_registries_exporters_sha256".into(), + command_sources, + ), + ( + "model_states_sha256".into(), + run.file_hash("model-states.json")?, + ), + ( + "data_version".into(), + version + .get("world_version") + .cloned() + .ok_or("version.json has no world_version")?, + ), + ("facts_sha256".into(), run.file_hash("facts.json")?), + ( + "block_tags_sha256".into(), + run.file_hash("block-tags.json")?, + ), + ("resource_files".into(), Value::Int(count.to_string())), + ]); + write_json( + &root.join("horizon-resources.json"), + &Value::Object(manifest), + ) +} + +/// `sorted(directory.glob('*.json'))`: glob skips names starting with a dot. +fn json_files(directory: &Path) -> Result, String> { + let mut files: Vec = fs::read_dir(directory) + .map_err(|e| format!("{}: {e}", directory.display()))? + .filter_map(Result::ok) + .filter(|e| { + let name = e.file_name().to_string_lossy().into_owned(); + name.ends_with(".json") && !name.starts_with('.') + }) + .map(|e| e.path()) + .collect(); + files.sort(); + Ok(files) +} + +const WORLDGEN_BUNDLES: [(&str, &str); 19] = [ + ("noises", "worldgen/noise"), + ("densities", "worldgen/density_function"), + ("rules", "worldgen/material_rule"), + ("conditions", "worldgen/material_condition"), + ("settings", "worldgen/noise_settings"), + ("carvers", "worldgen/carver"), + ("biomes", "worldgen/biome"), + ("world_presets", "worldgen/world_preset"), + ("features", "worldgen/feature"), + ("placed_features", "worldgen/placed_feature"), + ("state_providers", "worldgen/block_state_provider"), + ("biome_tags", "tags/worldgen/biome"), + ("feature_tags", "tags/worldgen/feature"), + ("block_tags", "tags/block"), + ("fluid_tags", "tags/fluid"), + ("processor_lists", "worldgen/processor_list"), + ("template_pools", "worldgen/template_pool"), + ("structures", "worldgen/structure"), + ("structure_sets", "worldgen/structure_set"), +]; + +const BLOCK_BEHAVIOR_FILES: [&str; 6] = [ + "block-properties.nbt", + "cached-collision.nbt", + "cached-support.nbt", + "survival-rules.json", + "survival-extra-definitions.json", + "placement-definitions.json", +]; + +/// The chunk generator's resource bundle in `/worldgen` (see GeneratorData::load). +fn prepare_worldgen(run: &Run) -> Result<(), String> { + (run.log)("Writing the world generator's bundle"); + let root = &run.resources; + let local = &run.local; + let out = root.join("worldgen"); + mkdir(&out)?; + // Structure templates: gzip NBT files under structure//.nbt. + let mut server = open_zip(&run.server)?; + let prefix = "data/minecraft/structure/"; + for name in zip_names(&server) { + if name.starts_with(prefix) && name.ends_with(".nbt") { + let relative = member_path(&name[prefix.len()..], "Invalid structure path")?; + let data = zip_bytes(&mut server, &name)?; + write_if_changed(&out.join("structure/minecraft").join(relative), &data)?; + } + } + let mut client = open_zip(&run.client)?; + for (name, path) in WORLDGEN_BUNDLES { + let prefix = format!("data/minecraft/{path}/"); + let resources = json_resources(&mut server, &prefix)?; + if resources.as_object().is_none_or(<[_]>::is_empty) + || resources != json_resources(&mut client, &prefix)? + { + return Err(format!("Reference worldgen resources disagree: {path}")); + } + write_text( + &out.join(format!("{name}.json")), + &pyjson::dumps(&resources, &pyjson::COMPACT), + )?; + } + let facts = read_json(&root.join("facts.json"))?; + let short = || "A block state in facts.json is short".to_string(); + let flags = facts + .get("states") + .and_then(Value::as_array) + .ok_or("facts.json has no states")? + .iter() + .map(|s| { + Ok(Value::Array(vec![ + s.index(1).cloned().ok_or_else(short)?, + s.index(2).cloned().ok_or_else(short)?, + ])) + }) + .collect::, String>>()?; + write_text( + &out.join("block_flags.json"), + &pyjson::dumps(&Value::Array(flags), &pyjson::COMPACT), + )?; + let reports = out.join("biome_parameters"); + mkdir(&reports)?; + let mut presets = false; + for source in [ + root.join("biome-parameters"), + local.join("datagen/out/reports/biome_parameters/minecraft"), + ] { + if source.join("overworld.json").exists() { + for report in json_files(&source)? { + copy(&report, &reports.join(report.file_name().unwrap()))?; + } + presets = true; + break; + } + } + // Block behavior used by feature placement checks (survival, support, collision). + // These come from the Java exporters of the simple-block parity run, or a direct export. + let all_present = |dir: &Path| BLOCK_BEHAVIOR_FILES.iter().all(|n| dir.join(n).exists()); + let mut behavior = local.join("rust-simple-block-parity"); + if !all_present(&behavior) { + behavior = local.join("horizon-client-block-behavior"); + export_block_behavior(run, &behavior)?; + } + let blocks = out.join("blocks"); + if all_present(&behavior) { + mkdir(&blocks)?; + for name in BLOCK_BEHAVIOR_FILES { + copy(&behavior.join(name), &blocks.join(name))?; + } + } + mkdir(&blocks)?; + let structures = local.join("horizon-client-structures"); + run.export( + "structure blocks", + &run.server_classpath, + &structures.join("classes"), + &[&run.oracle("VanillaStructureBlockExporter.java")], + &["VanillaStructureBlockExporter"], + "1G", + &[arg(&blocks.join("structure-blocks.json"))], + &structures.join("export.log"), + )?; + if !presets { + // Fall back to the Java-exported quantized tables from the material-stage parity run. + let tables = local.join("rust-material-stage-parity/presets.json"); + if !tables.exists() { + return Err( + "No multi-noise biome parameter tables; run the data generator reports".into(), + ); + } + copy(&tables, &out.join("presets.json"))?; + } + Ok(()) +} + +/// The vanilla exporters of block properties, support, collision and survival definitions. +/// Feature placement needs them: without these files the generator skips every placed feature +/// whose blocks are checked for survival or support (trees, flowers, grass, kelp, ...). +fn export_block_behavior(run: &Run, export: &Path) -> Result<(), String> { + (run.log)("Exporting block behavior (VanillaSimpleBlockOracle)"); + let tests = run.root.join("mods/horizon/src/test/java"); + let classes = export.join("classes"); + mkdir(&classes)?; + let log_path = export.join("export.log"); + let log = run.log_file(&log_path)?; + let fail = |e: String| { + format!( + "{e} while exporting block behavior; see {}", + log_path.display() + ) + }; + // The oracle's helper classes are compiled on demand from the source path. + run.compile( + &run.server_classpath, + &["-sourcepath".into(), arg(&tests)], + &classes, + &[&tests.join("dev/horizonlab/parity/VanillaSimpleBlockOracle.java")], + &log, + ) + .map_err(fail)?; + run.run( + "1G", + &classes, + &run.server_classpath, + "VanillaSimpleBlockOracle", + &["export".into(), arg(export)], + &log, + ) + .map_err(fail)?; + let rules = survival::bind_definitions( + &read_json(&export.join("survival-classes.json"))?, + &read_json(&export.join("survival-placement-classes.json"))?, + &read_json(&export.join("survival-support-tags.json"))?, + )?; + write_text( + &export.join("survival-rules.json"), + &pyjson::dumps(&rules, &pyjson::COMPACT), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_source_digest_is_the_one_build_rs_computed() { + assert_eq!( + sources_sha256(Path::new(env!("CARGO_MANIFEST_DIR"))).as_deref(), + Some(SOURCES_SHA256) + ); + assert_eq!(sources_sha256(Path::new("/nonexistent")), None); + } + + #[test] + fn lines_split_as_python_splits_them() { + assert_eq!(split_lines("a\r\nb\rc\n\nd"), ["a", "b", "c", "", "d"]); + assert_eq!(split_lines("a\n"), ["a"]); + } + + #[test] + fn member_paths_stay_inside() { + assert!(member_path("assets/a.png", "bad").is_ok()); + assert!(member_path("../a", "bad").is_err()); + assert!(member_path("/a", "bad").is_err()); + } +} diff --git a/crates/horizon-prepare/src/main.rs b/crates/horizon-prepare/src/main.rs new file mode 100644 index 000000000..d56ee8175 --- /dev/null +++ b/crates/horizon-prepare/src/main.rs @@ -0,0 +1,109 @@ +//! `horizon-prepare`: prepares Horizon's resources from your own Minecraft Java Edition. +//! Its flags are the ones tools/prepare_native_client.py took. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +const HELP: &str = "\ +horizon-prepare: prepare Horizon's resources from your own Minecraft Java Edition + +Usage: horizon-prepare [--prepare] [--server-jar PATH] [--minecraft-dir PATH] [--root PATH] + + --prepare Import the pinned game files from your own copy first + --server-jar PATH The pinned server jar you downloaded yourself + (default: HORIZON_SERVER_JAR, or the copy imported before) + --minecraft-dir PATH The official launcher's game directory with your installed copy + (default: HORIZON_MINECRAFT_DIR, or the launcher's own) + --root PATH The Horizon folder to prepare (default: the folder this program is + in, else the nearest folder above the current one with research-lock.json) + +Needs Minecraft played once from the official launcher, and Java (a JDK) of the version +research-lock.json pins: JAVA_HOME, else javac and java on the PATH. +The resources go to /.local/horizon-client/; pass that to horizon-client --resources. +"; + +/// The folder with research-lock.json: beside the executable in a release folder, else the +/// current folder or one above it (a checkout, with the binary in target/). +fn default_root() -> Option { + let beside = std::env::current_exe() + .ok() + .and_then(|exe| exe.parent().map(Path::to_path_buf)); + if let Some(dir) = beside.filter(|d| d.join("research-lock.json").is_file()) { + return Some(dir); + } + let cwd = std::env::current_dir().ok()?; + cwd.ancestors() + .find(|d| d.join("research-lock.json").is_file()) + .map(Path::to_path_buf) +} + +fn main() -> ExitCode { + let mut options = horizon_prepare::Options { + root: PathBuf::new(), + import: false, + minecraft_dir: None, + server_jar: None, + }; + let mut root = None; + let mut args = std::env::args_os().skip(1); + while let Some(arg) = args.next() { + let text = arg.to_string_lossy().into_owned(); + let (flag, inline) = match text.split_once('=') { + Some((flag, value)) if flag.starts_with("--") => { + (flag.to_string(), Some(value.to_string())) + } + _ => (text.clone(), None), + }; + let mut value = |name: &str| -> Result { + match &inline { + Some(v) => Ok(PathBuf::from(v)), + None => args + .next() + .map(PathBuf::from) + .ok_or_else(|| format!("{name} needs a path")), + } + }; + let parsed = match flag.as_str() { + "--prepare" if inline.is_none() => { + options.import = true; + Ok(()) + } + "--server-jar" => value("--server-jar").map(|v| options.server_jar = Some(v)), + "--minecraft-dir" => value("--minecraft-dir").map(|v| options.minecraft_dir = Some(v)), + "--root" => value("--root").map(|v| root = Some(v)), + "-h" | "--help" => { + print!("{HELP}"); + return ExitCode::SUCCESS; + } + "-V" | "--version" => { + println!("horizon-prepare {}", env!("CARGO_PKG_VERSION")); + return ExitCode::SUCCESS; + } + _ => Err(format!("unrecognized argument: {text}")), + }; + if let Err(e) = parsed { + eprintln!("horizon-prepare: error: {e}\n\n{HELP}"); + return ExitCode::from(2); + } + } + let Some(root) = root.or_else(default_root) else { + eprintln!( + "horizon-prepare: error: no research-lock.json beside this program or above the current folder; pass --root" + ); + return ExitCode::from(2); + }; + options.root = root; + let log = |line: &str| { + let mut out = std::io::stdout().lock(); + let _ = writeln!(out, "{line}"); + let _ = out.flush(); + }; + match horizon_prepare::prepare(&options, &log) { + Ok(_) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} diff --git a/crates/horizon-prepare/src/pyjson.rs b/crates/horizon-prepare/src/pyjson.rs new file mode 100644 index 000000000..f8e98787c --- /dev/null +++ b/crates/horizon-prepare/src/pyjson.rs @@ -0,0 +1,541 @@ +//! JSON read and written exactly as Python's `json` module does, so the files this crate writes +//! are byte for byte the ones `tools/prepare_native_client.py` wrote: objects keep their key +//! order (a repeated key keeps its first place and its last value), integers keep every digit, +//! floats print as Python's `repr`, and strings are escaped as `ensure_ascii=True` escapes them. + +use std::fmt::Write; + +#[derive(Clone, Debug)] +pub enum Value { + Null, + Bool(bool), + /// A JSON integer, kept as its decimal digits (Python's integers have no size limit). + Int(String), + Float(f64), + Str(String), + Array(Vec), + Object(Vec<(String, Value)>), +} + +impl Value { + pub fn str(s: impl Into) -> Value { + Value::Str(s.into()) + } + + pub fn get(&self, key: &str) -> Option<&Value> { + match self { + Value::Object(fields) => fields.iter().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + + pub fn index(&self, i: usize) -> Option<&Value> { + match self { + Value::Array(items) => items.get(i), + _ => None, + } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + Value::Str(s) => Some(s), + _ => None, + } + } + + pub fn as_array(&self) -> Option<&[Value]> { + match self { + Value::Array(items) => Some(items), + _ => None, + } + } + + pub fn as_object(&self) -> Option<&[(String, Value)]> { + match self { + Value::Object(fields) => Some(fields), + _ => None, + } + } + + pub fn as_i64(&self) -> Option { + match self { + Value::Int(digits) => digits.parse().ok(), + _ => None, + } + } + + /// `dict[key] = value`: a new key goes last, an existing one keeps its place. + pub fn insert(&mut self, key: &str, value: Value) { + if let Value::Object(fields) = self { + match fields.iter_mut().find(|(k, _)| k == key) { + Some(field) => field.1 = value, + None => fields.push((key.to_string(), value)), + } + } + } +} + +/// Python's `==`: objects compare without regard to key order, and 1 == 1.0. +impl PartialEq for Value { + fn eq(&self, other: &Value) -> bool { + match (self, other) { + (Value::Null, Value::Null) => true, + (Value::Bool(a), Value::Bool(b)) => a == b, + (Value::Int(a), Value::Int(b)) => a == b, + (Value::Float(a), Value::Float(b)) => a == b, + (Value::Int(a), Value::Float(b)) | (Value::Float(b), Value::Int(a)) => { + a.parse::().is_ok_and(|a| a == *b && b.fract() == 0.0) + } + (Value::Str(a), Value::Str(b)) => a == b, + (Value::Array(a), Value::Array(b)) => a == b, + (Value::Object(a), Value::Object(b)) => { + a.len() == b.len() && a.iter().all(|(k, v)| other.get(k) == Some(v)) + } + _ => false, + } + } +} + +pub fn loads(text: &str) -> Result { + let mut parser = Parser { + text: text.as_bytes(), + at: 0, + }; + parser.space(); + let value = parser.value()?; + parser.space(); + if parser.at != parser.text.len() { + return Err(parser.error("Extra data")); + } + Ok(value) +} + +struct Parser<'a> { + text: &'a [u8], + at: usize, +} + +impl Parser<'_> { + fn error(&self, what: &str) -> String { + format!("{what} at byte {}", self.at) + } + + fn space(&mut self) { + while matches!(self.text.get(self.at), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.at += 1; + } + } + + fn literal(&mut self, word: &str) -> bool { + if self.text[self.at..].starts_with(word.as_bytes()) { + self.at += word.len(); + true + } else { + false + } + } + + fn value(&mut self) -> Result { + match self.text.get(self.at) { + Some(b'{') => { + self.at += 1; + let mut object = Value::Object(Vec::new()); + self.space(); + if self.text.get(self.at) == Some(&b'}') { + self.at += 1; + return Ok(object); + } + loop { + self.space(); + if self.text.get(self.at) != Some(&b'"') { + return Err(self.error("Expecting property name enclosed in double quotes")); + } + let key = self.string()?; + self.space(); + if self.text.get(self.at) != Some(&b':') { + return Err(self.error("Expecting ':' delimiter")); + } + self.at += 1; + self.space(); + let value = self.value()?; + object.insert(&key, value); + self.space(); + match self.text.get(self.at) { + Some(b',') => self.at += 1, + Some(b'}') => { + self.at += 1; + return Ok(object); + } + _ => return Err(self.error("Expecting ',' delimiter")), + } + } + } + Some(b'[') => { + self.at += 1; + let mut items = Vec::new(); + self.space(); + if self.text.get(self.at) == Some(&b']') { + self.at += 1; + return Ok(Value::Array(items)); + } + loop { + self.space(); + items.push(self.value()?); + self.space(); + match self.text.get(self.at) { + Some(b',') => self.at += 1, + Some(b']') => { + self.at += 1; + return Ok(Value::Array(items)); + } + _ => return Err(self.error("Expecting ',' delimiter")), + } + } + } + Some(b'"') => Ok(Value::Str(self.string()?)), + Some(b'n') if self.literal("null") => Ok(Value::Null), + Some(b't') if self.literal("true") => Ok(Value::Bool(true)), + Some(b'f') if self.literal("false") => Ok(Value::Bool(false)), + Some(b'N') if self.literal("NaN") => Ok(Value::Float(f64::NAN)), + Some(b'I') if self.literal("Infinity") => Ok(Value::Float(f64::INFINITY)), + Some(b'-') if self.literal("-Infinity") => Ok(Value::Float(f64::NEG_INFINITY)), + Some(b'-' | b'0'..=b'9') => self.number(), + _ => Err(self.error("Expecting value")), + } + } + + /// Python's NUMBER_RE: `(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?`. + fn number(&mut self) -> Result { + let start = self.at; + let digits = |p: &mut Self| { + let from = p.at; + while p.text.get(p.at).is_some_and(u8::is_ascii_digit) { + p.at += 1; + } + p.at - from + }; + if self.text.get(self.at) == Some(&b'-') { + self.at += 1; + } + if self.text.get(self.at) == Some(&b'0') { + self.at += 1; + } else if digits(self) == 0 { + self.at = start; + return Err(self.error("Expecting value")); + } + let integer_end = self.at; + let mut float = false; + if self.text.get(self.at) == Some(&b'.') { + let dot = self.at; + self.at += 1; + if digits(self) == 0 { + self.at = dot; + } else { + float = true; + } + } + if matches!(self.text.get(self.at), Some(b'e' | b'E')) { + let mark = self.at; + self.at += 1; + if matches!(self.text.get(self.at), Some(b'-' | b'+')) { + self.at += 1; + } + if digits(self) == 0 { + self.at = mark; + } else { + float = true; + } + } + let text = std::str::from_utf8(&self.text[start..self.at]).unwrap(); + if float { + text.parse::() + .map(Value::Float) + .map_err(|e| e.to_string()) + } else { + debug_assert_eq!(self.at, integer_end); + Ok(Value::Int(if text == "-0" { + "0".into() + } else { + text.into() + })) + } + } + + fn hex4(&mut self) -> Result { + let digits = self + .text + .get(self.at..self.at + 4) + .and_then(|d| std::str::from_utf8(d).ok()) + .filter(|d| d.bytes().all(|b| b.is_ascii_hexdigit())) + .ok_or_else(|| self.error("Invalid \\uXXXX escape"))?; + self.at += 4; + Ok(u32::from_str_radix(digits, 16).unwrap()) + } + + fn string(&mut self) -> Result { + self.at += 1; + let mut out = String::new(); + loop { + let run = self.at; + while self + .text + .get(self.at) + .is_some_and(|&b| b != b'"' && b != b'\\' && b >= 0x20) + { + self.at += 1; + } + out.push_str(std::str::from_utf8(&self.text[run..self.at]).map_err(|e| e.to_string())?); + match self.text.get(self.at) { + Some(b'"') => { + self.at += 1; + return Ok(out); + } + Some(b'\\') => { + self.at += 1; + let escape = *self + .text + .get(self.at) + .ok_or_else(|| self.error("Unterminated string"))?; + self.at += 1; + match escape { + b'"' => out.push('"'), + b'\\' => out.push('\\'), + b'/' => out.push('/'), + b'b' => out.push('\u{8}'), + b'f' => out.push('\u{c}'), + b'n' => out.push('\n'), + b'r' => out.push('\r'), + b't' => out.push('\t'), + b'u' => { + let mut code = self.hex4()?; + if (0xd800..0xdc00).contains(&code) + && self.text[self.at..].starts_with(b"\\u") + { + let back = self.at; + self.at += 2; + let low = self.hex4()?; + if (0xdc00..0xe000).contains(&low) { + code = 0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00); + } else { + self.at = back; + } + } + out.push(char::from_u32(code).ok_or_else(|| { + self.error( + "Lone surrogate (Python keeps it; Horizon's files have none)", + ) + })?); + } + _ => return Err(self.error("Invalid \\escape")), + } + } + _ => return Err(self.error("Unterminated string or control character")), + } + } + } +} + +/// `json.dumps(value, indent=indent, separators=(item, key))` with `ensure_ascii=True`. +/// Python's defaults are `(", ", ": ")` without an indent and `(",", ": ")` with one. +pub struct Format { + pub indent: Option, + pub item: &'static str, + pub key: &'static str, +} + +/// `json.dumps(value)`. +pub const DEFAULT: Format = Format { + indent: None, + item: ", ", + key: ": ", +}; +/// `json.dumps(value, separators=(',', ':'))`. +pub const COMPACT: Format = Format { + indent: None, + item: ",", + key: ":", +}; +/// `json.dumps(value, indent=2)`. +pub const INDENT2: Format = Format { + indent: Some(2), + item: ",", + key: ": ", +}; + +pub fn dumps(value: &Value, format: &Format) -> String { + let mut out = String::new(); + write_value(&mut out, value, format, 0); + out +} + +fn newline(out: &mut String, format: &Format, level: usize) { + if let Some(indent) = format.indent { + out.push('\n'); + out.extend(std::iter::repeat_n(' ', indent * level)); + } +} + +fn write_value(out: &mut String, value: &Value, format: &Format, level: usize) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }), + Value::Int(digits) => out.push_str(digits), + Value::Float(f) => out.push_str(&float_repr(*f)), + Value::Str(s) => write_string(out, s), + Value::Array(items) => { + if items.is_empty() { + out.push_str("[]"); + return; + } + out.push('['); + for (i, item) in items.iter().enumerate() { + if i > 0 { + out.push_str(format.item); + } + newline(out, format, level + 1); + write_value(out, item, format, level + 1); + } + newline(out, format, level); + out.push(']'); + } + Value::Object(fields) => { + if fields.is_empty() { + out.push_str("{}"); + return; + } + out.push('{'); + for (i, (key, item)) in fields.iter().enumerate() { + if i > 0 { + out.push_str(format.item); + } + newline(out, format, level + 1); + write_string(out, key); + out.push_str(format.key); + write_value(out, item, format, level + 1); + } + newline(out, format, level); + out.push('}'); + } + } +} + +/// Python's `py_encode_basestring_ascii`: every character outside space..`~` is escaped, +/// with lowercase hex and surrogate pairs above the BMP. +pub fn write_string(out: &mut String, s: &str) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{8}' => out.push_str("\\b"), + '\u{c}' => out.push_str("\\f"), + ' '..='~' => out.push(c), + _ => { + let mut units = [0u16; 2]; + for unit in c.encode_utf16(&mut units) { + let _ = write!(out, "\\u{unit:04x}"); + } + } + } + } + out.push('"'); +} + +/// Python's `repr(float)`: the shortest digits that read back the same, in fixed notation when +/// the decimal point falls between 10^-4 and 10^16, with `.0` on whole numbers. +pub fn float_repr(f: f64) -> String { + if f.is_nan() { + return "NaN".into(); + } + if f.is_infinite() { + return if f > 0.0 { "Infinity" } else { "-Infinity" }.into(); + } + // Rust's `{:e}` gives the same shortest round-trip digits: "d.ddde-x". + let scientific = format!("{:e}", f.abs()); + let (mantissa, exponent) = scientific.split_once('e').unwrap(); + let digits: String = mantissa.chars().filter(|c| *c != '.').collect(); + let exponent: i32 = exponent.parse().unwrap(); + let point = exponent + 1; // The value is 0.digits * 10^point. + let sign = if f.is_sign_negative() { "-" } else { "" }; + let n = digits.len() as i32; + if f == 0.0 { + return format!("{sign}0.0"); + } + if -4 < point && point <= 16 { + if point <= 0 { + format!("{sign}0.{}{digits}", "0".repeat((-point) as usize)) + } else if point >= n { + format!("{sign}{digits}{}.0", "0".repeat((point - n) as usize)) + } else { + let (whole, fraction) = digits.split_at(point as usize); + format!("{sign}{whole}.{fraction}") + } + } else { + let (first, rest) = digits.split_at(1); + let mantissa = if rest.is_empty() { + first.to_string() + } else { + format!("{first}.{rest}") + }; + let e = point - 1; + format!( + "{sign}{mantissa}e{}{:02}", + if e < 0 { '-' } else { '+' }, + e.abs() + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn floats_print_as_python_repr() { + for (f, repr) in [ + (0.0, "0.0"), + (-0.0, "-0.0"), + (1.0, "1.0"), + (0.1, "0.1"), + (1.5e-5, "1.5e-05"), + (0.0001, "0.0001"), + (1e16, "1e+16"), + (1234567890123456.0, "1234567890123456.0"), + (12345678901234567.0, "1.2345678901234568e+16"), + (-2.5, "-2.5"), + (1e-310, "1e-310"), + (f64::MAX, "1.7976931348623157e+308"), + (100.0, "100.0"), + (0.30000000000000004, "0.30000000000000004"), + ] { + assert_eq!(float_repr(f), repr, "{f}"); + } + } + + #[test] + fn strings_escape_as_ensure_ascii() { + let mut out = String::new(); + write_string(&mut out, "a\"\\\n\u{7f}é\u{1F600}\u{1}"); + assert_eq!(out, "\"a\\\"\\\\\\n\\u007f\\u00e9\\ud83d\\ude00\\u0001\""); + } + + #[test] + fn objects_keep_first_place_and_last_value() { + let v = loads(r#"{"b": 1, "a": [1.0, -0, 1e2, true, null], "b": 2}"#).unwrap(); + assert_eq!( + dumps(&v, &DEFAULT), + r#"{"b": 2, "a": [1.0, 0, 100.0, true, null]}"# + ); + assert_eq!( + dumps(&v, &INDENT2), + "{\n \"b\": 2,\n \"a\": [\n 1.0,\n 0,\n 100.0,\n true,\n null\n ]\n}" + ); + assert_eq!( + dumps(&loads("{\"x\":{},\"y\":[]}").unwrap(), &INDENT2), + "{\n \"x\": {},\n \"y\": []\n}" + ); + } +} diff --git a/crates/horizon-prepare/src/survival.rs b/crates/horizon-prepare/src/survival.rs new file mode 100644 index 000000000..16c451c32 --- /dev/null +++ b/crates/horizon-prepare/src/survival.rs @@ -0,0 +1,146 @@ +//! Survival rules of the world generator's feature placement: each block's rule from the class +//! vanilla gives it and the class of its placement check (tools/test_block_survival_parity.py's +//! `bind_definitions` and its tables). + +use crate::pyjson::Value; + +const CLASS_RULES: &[(&str, &str)] = &[ + ("FireBlock", "Fire"), + ("CactusBlock", "Cactus"), + ("PathBlock", "Path"), + ("MushroomBlock", "Mushroom"), + ("MultifaceBlock", "Multiface"), + ("VineBlock", "Vine"), + ("HangingMossBlock", "HangingMoss"), + ("SeaPickleBlock", "SeaPickle"), + ("SnowLayerBlock", "SnowLayer"), + ("BellBlock", "Bell"), + ("ShelfMushroomBlock", "FullBehind"), + ("GrowingPlantBlock", "GrowingPlant"), + ("MangrovePropaguleBlock", "MangrovePropagule"), + ("SpeleothemBlock", "Speleothem"), + ("ScaffoldingBlock", "Scaffolding"), + ("PistonHeadBlock", "PistonHead"), + ("ChorusPlantBlock", "ChorusPlant"), + ("ChorusFlowerBlock", "ChorusFlower"), + ("PitcherCropBlock", "PitcherCrop"), + ("BigDripleafBlock", "BigDripleaf"), + ("BigDripleafStemBlock", "BigDripleafStem"), + ("FarmlandBlock", "Farmland"), + ("CocoaBlock", "Cocoa"), + ("SugarCaneBlock", "SugarCane"), + ("SporeBlossomBlock", "SporeBlossom"), + ("FrogspawnBlock", "Frogspawn"), + ("SmallDripleafBlock", "SmallDripleaf"), + ("TallSeagrassBlock", "TallSeagrass"), + ("DoorBlock", "Door"), + ("BaseRailBlock", "RigidBelow"), + ("LeafLitterBlock", "FullBelow"), + ("WallSignBlock", "SolidBehind"), + ("GrindstoneBlock", "Always"), + ("RedstoneWallTorchBlock", "FullBehind"), + ("BubbleColumnBlock", "BubbleColumn"), + ("RedstoneWireBlock", "RedstoneWire"), + ("SoulFireBlock", "TagBelow:minecraft:soul_fire_base_blocks"), + ("BlockBehaviour", "Always"), + ("CakeBlock", "SolidBelow"), + ("CandleCakeBlock", "SolidBelow"), + ("BannerBlock", "SolidBelow"), + ("StandingSignBlock", "SolidBelow"), + ("WallBannerBlock", "SolidBehind"), + ("CarpetBlock", "NonAirBelow"), + ("MossyCarpetBlock", "MossCarpet"), + ("BaseTorchBlock", "CenterBelow"), + ("CandleBlock", "CenterBelow"), + ("LanternBlock", "Lantern"), + ("HangingRootsBlock", "FullAbove"), + ("LadderBlock", "FullBehind"), + ("AmethystClusterBlock", "FullBehind"), + ("TripWireHookBlock", "FullBehind"), + ("BaseCoralWallFanBlock", "FullBehind"), + ("WallTorchBlock", "FullBehind"), + ("BasePressurePlateBlock", "RigidOrCenterBelow"), + ("DiodeBlock", "RigidBelow"), + ("BaseCoralPlantTypeBlock", "FullBelow"), + ("CeilingHangingSignBlock", "CenterAbove"), + ("FaceAttachedHorizontalDirectionalBlock", "FaceAttached"), + ("BambooSaplingBlock", "SupportsBamboo"), + ("BambooStalkBlock", "SupportsBamboo"), +]; + +const VEGETATION_RULES: &[(&str, &str)] = &[ + ("LilyPadBlock", "LilyPad"), + ("VegetationBlock", "TagBelow:minecraft:supports_vegetation"), + ("AzaleaBlock", "TagBelow:minecraft:supports_azalea"), + ( + "DryVegetationBlock", + "TagBelow:minecraft:supports_dry_vegetation", + ), + ( + "NetherSproutsBlock", + "TagBelow:minecraft:supports_nether_sprouts", + ), + ("NetherWartBlock", "TagBelow:minecraft:supports_nether_wart"), + ("WitherRoseBlock", "TagBelow:minecraft:supports_wither_rose"), + ("CactusFlowerBlock", "CactusFlower"), + ("SeagrassBlock", "Seagrass"), +]; + +fn lookup(table: &[(&str, &'static str)], key: &str) -> Option<&'static str> { + table.iter().find(|(k, _)| *k == key).map(|(_, v)| *v) +} + +/// `{block: rule}` from the exporter's `{block: class}`, `{block: placement class}` and +/// `{block: {field: support tag}}`, in the order Python's dict built it. +pub fn bind_definitions( + classes: &Value, + placement: &Value, + support_tags: &Value, +) -> Result { + let classes = classes + .as_object() + .ok_or("survival-classes.json is not an object")?; + let mut definitions = Value::Object(Vec::new()); + for (name, owner) in classes { + if let Some(rule) = owner.as_str().and_then(|o| lookup(CLASS_RULES, o)) { + definitions.insert(name, Value::str(rule)); + } + } + for (name, owner) in classes { + let owner = owner.as_str().unwrap_or(""); + let placed = placement.get(name).and_then(Value::as_str); + if owner == "CropBlock" && placed == Some("CropBlock") { + definitions.insert(name, Value::str("Crop")); + } + if owner == "DoublePlantBlock" && placed == Some("VegetationBlock") { + definitions.insert(name, Value::str("DoublePlant")); + } + if owner != "VegetationBlock" { + continue; + } + let method = placed.ok_or_else(|| format!("No placement class for {name}"))?; + if let Some(rule) = lookup(VEGETATION_RULES, method) { + definitions.insert(name, Value::str(rule)); + } else if [ + "AttachedStemBlock", + "StemBlock", + "NetherFungusBlock", + "NetherRootsBlock", + ] + .contains(&method) + { + let field = if method == "StemBlock" { + "stemSupportBlocks" + } else { + "supportBlocks" + }; + let tag = support_tags + .get(name) + .and_then(|t| t.get(field)) + .and_then(Value::as_str) + .ok_or_else(|| format!("No {field} support tag for {name}"))?; + definitions.insert(name, Value::Str(format!("TagBelow:{tag}"))); + } + } + Ok(definitions) +} diff --git a/tools/compare_prepare.py b/tools/compare_prepare.py new file mode 100644 index 000000000..58216af00 --- /dev/null +++ b/tools/compare_prepare.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Check that horizon-prepare (Rust) writes what the Python preparer wrote. + +Prepares the same jars three times in fresh folders made from the same tree (research-lock.json +and the vanilla exporters of --python-ref): twice with that ref's tools/prepare_native_client.py +and once with horizon-prepare. Every file under .local/ is compared by sha256: + +- identical: the same bytes in all three runs; +- exporter noise: differs between the two Python runs too, so the vanilla Java exporter itself + writes it differently each run (lambda addresses, hash-set order), not the preparer; +- port differences: identical between the Python runs but not in the Rust run. There must be none. + +The manifest (horizon-resources.json) is compared field by field: only the recorded hashes of +exporter-noise files may differ, and each must be the hash of its own run's file. + +Then each preparer runs again over the other's output (Rust over a Python tree, Python over the +Rust tree): the manifests are read the same way, so nothing is exported again and no resource +changes. Development tool: it needs Python 3.11+, Java and git. Report: /report.json. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +# The last commit whose tools/prepare_native_client.py is the Python preparer itself. +PYTHON_REF = '66ce394d2d6dc19ed5f5c53a97b650741af9be76' +# Written by tools the comparison is not about: the decompiler only lab.py fetches, logs with +# times in them, compiled classes, and the data generator's own dated cache. +IGNORED_PARTS = {'classes', 'client-classes', 'logs', '.cache'} + + +def tree(ref, destination): + destination.mkdir(parents=True) + archive = subprocess.run(['git', '-C', str(ROOT), 'archive', ref, 'tools', 'mods/horizon/src/test/java', + 'research-lock.json'], check=True, capture_output=True).stdout + subprocess.run(['tar', '-x', '-C', str(destination)], input=archive, check=True) + + +def hashes(folder): + local = folder / '.local' + result = {} + for path in sorted(local.rglob('*')): + relative = path.relative_to(local) + if path.is_file() and not (set(relative.parts) & IGNORED_PARTS) and path.suffix != '.log': + result[relative.as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + return result + + +def run(name, command, cwd, log): + started = time.monotonic() + with log.open('w') as stream: + subprocess.run(command, cwd=cwd, check=True, stdout=stream, stderr=subprocess.STDOUT) + seconds = round(time.monotonic() - started, 1) + print(f'{name}: {seconds} s', flush=True) + return seconds + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--server-jar', type=Path, required=True) + parser.add_argument('--minecraft-dir', type=Path, required=True) + parser.add_argument('--python-ref', default=PYTHON_REF) + parser.add_argument('--binary', type=Path, help='horizon-prepare to test (default: cargo build --release)') + parser.add_argument('--work', type=Path, default=ROOT / '.local/compare-prepare') + parser.add_argument('--keep', action='store_true', help='keep the prepared folders') + args = parser.parse_args() + binary = args.binary + if binary is None: + subprocess.run(['cargo', 'build', '--release', '-p', 'horizon-prepare'], cwd=ROOT, check=True) + metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--no-deps', '--format-version', '1'], + cwd=ROOT, text=True)) + binary = Path(metadata['target_directory']) / 'release' / ( + 'horizon-prepare.exe' if os.name == 'nt' else 'horizon-prepare') + binary = binary.resolve() + work = args.work.resolve() + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + game = ['--prepare', '--server-jar', str(args.server_jar.resolve()), + '--minecraft-dir', str(args.minecraft_dir.resolve())] + python = [sys.executable, '-u', 'tools/prepare_native_client.py', *game] + folders = {name: work / name for name in ['python-1', 'python-2', 'rust']} + times = {} + for name, folder in folders.items(): + tree(args.python_ref, folder) + command = [str(binary), '--root', str(folder), *game] if name == 'rust' else python + times[name] = run(name, command, folder, work / f'{name}.log') + first, second, rust = (hashes(folders[n]) for n in ['python-1', 'python-2', 'rust']) + names = sorted(first.keys() | second.keys() | rust.keys()) + identical = [n for n in names if first.get(n) == second.get(n) == rust.get(n) is not None] + noise = [n for n in names if first.get(n) != second.get(n)] + port = [n for n in names if n not in identical and n not in noise] + # Only lab.py fetches the decompiler; preparing does not need it. + port = [n for n in port if not (n.startswith('tools/vineflower-') and n not in rust)] + manifest = 'horizon-client/' + json.loads((folders['rust'] / 'research-lock.json').read_text())[ + 'minecraft']['version'] + '/horizon-resources.json' + noisy = {n.rsplit('/', 1)[-1] for n in noise if n.startswith(manifest.rsplit('/', 1)[0] + '/')} + manifests = {n: json.loads((folders[n] / '.local' / manifest).read_text()) for n in folders} + manifest_differences = [] + + def walk(python, candidate, path): + if isinstance(python, dict) and isinstance(candidate, dict) and list(python) == list(candidate): + for key in python: + walk(python[key], candidate[key], path + [key]) + elif python != candidate: + name = path[-1] + expected = hashlib.sha256((folders['rust'] / '.local' / manifest).with_name(name).read_bytes()).hexdigest() \ + if name in noisy else None + if not (name in noisy and candidate == expected): + manifest_differences.append('.'.join(path)) + + walk(manifests['python-1'], manifests['rust'], []) + if manifest in port and not manifest_differences: + port.remove(manifest) + # Each preparer over the other's output: nothing to export again, no resource changes. + resources = 'horizon-client/' + before = {'python-2': second, 'rust': rust} + times['rust over python'] = run('rust over python', [str(binary), '--root', str(folders['python-2']), *game], + folders['python-2'], work / 'rust-over-python.log') + times['python over rust'] = run('python over rust', python, folders['rust'], work / 'python-over-rust.log') + rerun_changes = {} + for name in before: + after = hashes(folders[name]) + changed = [n for n in sorted(before[name].keys() | after.keys()) + if n.startswith(resources) and before[name].get(n) != after.get(n)] + rerun_changes[name] = changed + exported_again = {name: [line.strip() for line in (work / log).read_text().splitlines() if 'Exporting' in line] + for name, log in [('rust over python', 'rust-over-python.log')]} + report = {'python_ref': args.python_ref, 'binary': str(binary), 'seconds': times, + 'files': len(names), 'identical': len(identical), 'exporter_noise': noise, + 'port_differences': port, 'manifest_differences': manifest_differences, + 'rerun_resource_changes': rerun_changes, 'exported_again': exported_again} + (work / 'report.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) + if not args.keep: + for folder in folders.values(): + shutil.rmtree(folder, ignore_errors=True) + ok = not port and not manifest_differences + print('identical output' if ok else 'DIFFERENT OUTPUT', flush=True) + return 0 if ok else 1 + + +if __name__ == '__main__': + sys.exit(main())