horizon-prepare: the resource preparation in Rust

A port of tools/prepare_native_client.py and the helpers it imported from
lab.py and the test_*_parity.py modules: the player's pinned game files are
imported and checked against research-lock.json, the client's assets and data
copied, the vanilla exporters compiled and run with the JDK (JAVA_HOME, the
PATH, or java_home on macOS, at the pinned java_major), and the resources and
their manifest written byte for byte as the script wrote them, with its
sha256-keyed skips. tools/compare_prepare.py checks that against the Python
preparer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-14 16:03:26 +02:00
parent 27fc327195
commit 9fa3a684c6
12 changed files with 3358 additions and 1 deletions
Generated
+184
View File
@@ -37,6 +37,15 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "aho-corasick"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
dependencies = [
"memchr",
]
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -99,6 +108,15 @@ dependencies = [
"libc",
]
[[package]]
name = "arbitrary"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed"
dependencies = [
"derive_arbitrary",
]
[[package]]
name = "arboard"
version = "3.6.1"
@@ -183,6 +201,15 @@ version = "2.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "block2"
version = "0.5.1"
@@ -412,6 +439,15 @@ dependencies = [
"windows 0.54.0",
]
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]]
name = "crc32fast"
version = "1.5.2"
@@ -433,6 +469,16 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "cursor-icon"
version = "1.2.0"
@@ -445,6 +491,27 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
[[package]]
name = "derive_arbitrary"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.6",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]]
name = "dispatch"
version = "0.2.0"
@@ -461,6 +528,17 @@ dependencies = [
"objc2 0.6.5",
]
[[package]]
name = "displaydoc"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.6",
]
[[package]]
name = "dlib"
version = "0.5.3"
@@ -622,6 +700,16 @@ dependencies = [
"slab",
]
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "gethostname"
version = "1.1.0"
@@ -761,6 +849,16 @@ dependencies = [
"libm",
]
[[package]]
name = "horizon-prepare"
version = "0.1.0"
dependencies = [
"regex",
"sha1",
"sha2",
"zip",
]
[[package]]
name = "horizon-protocol"
version = "0.1.0"
@@ -1754,6 +1852,35 @@ dependencies = [
"bitflags 2.13.2",
]
[[package]]
name = "regex"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]]
name = "renderdoc-sys"
version = "1.1.0"
@@ -1889,6 +2016,28 @@ dependencies = [
"zmij",
]
[[package]]
name = "sha1"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "shlex"
version = "2.0.1"
@@ -2214,6 +2363,12 @@ version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"
[[package]]
name = "typenum"
version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
[[package]]
name = "unicode-ident"
version = "1.0.26"
@@ -3068,6 +3223,23 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "zip"
version = "2.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50"
dependencies = [
"arbitrary",
"crc32fast",
"crossbeam-utils",
"displaydoc",
"flate2",
"indexmap",
"memchr",
"thiserror 2.0.21",
"zopfli",
]
[[package]]
name = "zlib-rs"
version = "0.6.8"
@@ -3079,3 +3251,15 @@ name = "zmij"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
[[package]]
name = "zopfli"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11"
dependencies = [
"bumpalo",
"crc32fast",
"log",
"simd-adler32",
]
+1 -1
View File
@@ -1,6 +1,6 @@
[workspace]
exclude = ["launcher"]
members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme"]
members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare"]
resolver = "2"
[workspace.package]
+19
View File
@@ -0,0 +1,19 @@
[package]
name = "horizon-prepare"
version = "0.1.0"
edition = "2021"
license = "MIT"
description = "Prepares Horizon's resources from the player's own Minecraft: imports and checks the pinned jars, runs the vanilla exporters, writes the resource folder"
[dependencies]
regex = { version = "1", default-features = false, features = ["std", "unicode-perl"] }
sha1 = "0.10"
sha2 = "0.10"
zip = { version = "2", default-features = false, features = ["deflate"] }
[build-dependencies]
sha2 = "0.10"
[[bin]]
name = "horizon-prepare"
path = "src/main.rs"
+38
View File
@@ -0,0 +1,38 @@
//! HORIZON_PREPARE_SOURCES_SHA256: a digest of this crate's sources, which the launcher compares
//! with a version folder's copy to know whether it may prepare that version in-process. Keep the
//! algorithm the same as `horizon_prepare::sources_sha256`.
use sha2::Digest;
use std::fs;
use std::path::Path;
fn main() {
let dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let mut files = Vec::new();
for name in ["Cargo.toml", "build.rs"] {
println!("cargo:rerun-if-changed={name}");
files.push((name.to_string(), fs::read(dir.join(name)).unwrap()));
}
println!("cargo:rerun-if-changed=src");
let mut sources: Vec<String> = fs::read_dir(dir.join("src"))
.unwrap()
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
.filter(|n| n.ends_with(".rs"))
.collect();
sources.sort();
for name in sources {
println!("cargo:rerun-if-changed=src/{name}");
files.push((
format!("src/{name}"),
fs::read(dir.join("src").join(&name)).unwrap(),
));
}
let mut hash = sha2::Sha256::new();
for (name, data) in files {
let data: Vec<u8> = data.into_iter().filter(|b| *b != b'\r').collect();
hash.update(format!("{name}\0{}\0", data.len()));
hash.update(&data);
}
let hex: String = hash.finalize().iter().map(|b| format!("{b:02x}")).collect();
println!("cargo:rustc-env=HORIZON_PREPARE_SOURCES_SHA256={hex}");
}
+681
View File
@@ -0,0 +1,681 @@
//! The player's own game files: the pinned version JSON, client jar and libraries from their
//! official launcher install, and the server jar they downloaded, each checked against
//! research-lock.json before it is copied to `.local/minecraft/<version>` (tools/lab.py's
//! `prepare`, which the Python preparer ran first), then checked again on every run
//! (tools/test_rust_parity.py's `verify_artifacts`).
use crate::pyjson::{self, Value};
use crate::{digest, read_json, relative_display, write_json, Log};
use std::collections::HashMap;
use std::fs;
use std::path::{Component, Path, PathBuf};
const LAUNCHER_HINT: &str =
"Horizon does not download the game: install Minecraft Java Edition with the official \
launcher, play {version} once, then rerun. If the launcher uses another game directory, \
pass --minecraft-dir or set HORIZON_MINECRAFT_DIR.";
const SERVER_HINT: &str = "Horizon does not download the game: download the {version} server jar yourself (minecraft.net/download/server \
while {version} is the current release, otherwise the downloads.server.url in your launcher's \
versions/{version}/{version}.json), then pass --server-jar PATH or set HORIZON_SERVER_JAR.";
fn hint(text: &str, version: &str) -> String {
text.replace("{version}", version)
}
/// research-lock.json and the folder the pinned version's files go to.
pub struct Lock {
pub json: Value,
pub version: String,
pub java_major: i64,
pub directory: PathBuf,
}
impl Lock {
pub fn read(root: &Path) -> Result<Lock, String> {
let json = read_json(&root.join("research-lock.json"))?;
let game = json
.get("minecraft")
.ok_or("research-lock.json has no minecraft")?;
let version = game
.get("version")
.and_then(Value::as_str)
.ok_or("research-lock.json has no minecraft version")?
.to_string();
if version.is_empty()
|| !version
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b))
|| version == "."
|| version == ".."
{
return Err("Unsafe version identifier in lock file".into());
}
let java_major = game
.get("java_major")
.and_then(Value::as_i64)
.ok_or("research-lock.json has no java_major")?;
let directory = root.join(".local/minecraft").join(&version);
Ok(Lock {
json,
version,
java_major,
directory,
})
}
fn pin(&self, key: &str) -> Result<&str, String> {
self.json
.get("minecraft")
.and_then(|m| m.get(key))
.and_then(Value::as_str)
.ok_or_else(|| format!("research-lock.json has no minecraft {key}"))
}
/// Mojang republishes a version JSON for asset-index changes; every known one is accepted.
fn manifest_accepted(&self, checksum: &str) -> bool {
let game = self.json.get("minecraft");
self.pin("manifest_sha1").is_ok_and(|pin| pin == checksum)
|| game
.and_then(|g| g.get("accepted_manifest_sha1"))
.and_then(Value::as_array)
.is_some_and(|list| list.iter().any(|v| v.as_str() == Some(checksum)))
}
}
fn home() -> PathBuf {
let var = if cfg!(windows) { "USERPROFILE" } else { "HOME" };
std::env::var_os(var)
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("."))
}
/// Python's `Path.expanduser` for the forms players type: `~` and `~/...`.
fn expand_user(path: &Path) -> PathBuf {
let mut components = path.components();
match components.next() {
Some(Component::Normal(first)) if first == "~" => home().join(components.as_path()),
_ => path.to_path_buf(),
}
}
/// The official launcher's game directory, which holds the player's own copy of the game.
pub fn minecraft_dir(chosen: Option<&Path>) -> PathBuf {
if let Some(dir) = chosen.filter(|d| !d.as_os_str().is_empty()) {
return expand_user(dir);
}
if let Some(dir) = std::env::var_os("HORIZON_MINECRAFT_DIR").filter(|d| !d.is_empty()) {
return expand_user(Path::new(&dir));
}
if cfg!(target_os = "macos") {
home().join("Library/Application Support/minecraft")
} else if cfg!(windows) {
std::env::var_os("APPDATA")
.map(PathBuf::from)
.unwrap_or_else(home)
.join(".minecraft")
} else {
home().join(".minecraft")
}
}
/// Copy one file from the player's install, checked like a download; an existing verified
/// copy is kept.
fn import_copy(
source: &Path,
destination: &Path,
checksum: &str,
algorithm: &str,
hint: &str,
) -> Result<PathBuf, String> {
fs::create_dir_all(destination.parent().unwrap()).map_err(|e| e.to_string())?;
if destination.exists() && digest(destination, algorithm)? == checksum {
return Ok(destination.to_path_buf());
}
if !source.is_file() {
return Err(format!("Missing {}. {hint}", source.display())
.trim()
.to_string());
}
let actual = digest(source, algorithm)?;
if actual != checksum {
return Err(format!(
"{} is not the pinned file: expected {algorithm} {checksum}, got {actual}",
source.display()
));
}
let temporary = destination.with_file_name(format!(
".{}.partial-{}",
destination.file_name().unwrap().to_string_lossy(),
std::process::id()
));
let copied = fs::copy(source, &temporary).and_then(|_| fs::rename(&temporary, destination));
let _ = fs::remove_file(&temporary);
copied.map_err(|e| format!("{}: {e}", destination.display()))?;
Ok(destination.to_path_buf())
}
/// The pinned version JSON from the launcher's versions/ directory. The launcher's copy wins
/// over the cached one so the manifest matches the asset index installed alongside it.
fn import_manifest(lock: &Lock, minecraft: &Path) -> Result<Value, String> {
let version = &lock.version;
let destination = lock.directory.join("manifest.json");
let source = minecraft
.join("versions")
.join(version)
.join(format!("{version}.json"));
if source.is_file() {
let manifest = read_json(&source)?;
let downloads = manifest.get("downloads");
let sha1 = |side: &str| {
downloads
.and_then(|d| d.get(side))
.and_then(|d| d.get("sha1"))
.and_then(Value::as_str)
};
if manifest.get("id").and_then(Value::as_str) != Some(version)
|| sha1("client") != Some(lock.pin("client_sha1")?)
|| sha1("server") != Some(lock.pin("server_sha1")?)
{
return Err(format!(
"{} does not describe the pinned {version} release",
source.display()
));
}
let checksum = digest(&source, "sha1")?;
if !lock.manifest_accepted(&checksum) {
return Err(format!(
"{} is a {version} version file Horizon does not know yet (sha1 {checksum}); \
Mojang may have republished it. Update Horizon, or report it if this is the latest release.",
source.display()
));
}
if !(destination.exists() && digest(&destination, "sha1")? == checksum) {
fs::create_dir_all(&lock.directory).map_err(|e| e.to_string())?;
fs::copy(&source, &destination).map_err(|e| e.to_string())?;
}
} else if !(destination.exists() && lock.manifest_accepted(&digest(&destination, "sha1")?)) {
return Err(format!(
"Missing {}. {}",
source.display(),
hint(LAUNCHER_HINT, version)
));
}
let manifest = read_json(&destination)?;
let java = manifest
.get("javaVersion")
.and_then(|j| j.get("majorVersion"))
.and_then(Value::as_i64);
if java != Some(lock.java_major) {
return Err("Manifest Java version does not match lock file".into());
}
Ok(manifest)
}
fn import_server(lock: &Lock, server_jar: Option<&Path>) -> Result<PathBuf, String> {
let version = &lock.version;
let destination = lock.directory.join("jars/server.jar");
let checksum = lock.pin("server_sha1")?;
let hint = hint(SERVER_HINT, version);
let source = server_jar
.map(Path::to_path_buf)
.or_else(|| std::env::var_os("HORIZON_SERVER_JAR").map(PathBuf::from));
let Some(source) = source else {
if destination.exists() && digest(&destination, "sha1")? == checksum {
return Ok(destination);
}
return Err(format!("No {version} server jar. {hint}"));
};
import_copy(&expand_user(&source), &destination, checksum, "sha1", &hint)
}
/// The launcher's `os.name` for this machine.
fn rule_system() -> &'static str {
if cfg!(target_os = "macos") {
"osx"
} else if cfg!(windows) {
"windows"
} else {
"linux"
}
}
/// This machine's architecture and its aliases, as Python's `platform.machine()` names them.
fn rule_architectures() -> Vec<&'static str> {
match std::env::consts::ARCH {
"x86_64" => vec!["x86_64", "amd64", "x64"],
"aarch64" => vec!["arm64", "aarch64"],
"x86" => vec!["i686"],
other => vec![other],
}
}
/// The kernel release Python's `platform.release()` reports (only old version JSONs ask).
fn os_release() -> String {
#[cfg(unix)]
{
if let Ok(out) = std::process::Command::new("uname").arg("-r").output() {
return String::from_utf8_lossy(&out.stdout).trim().to_string();
}
}
String::new()
}
/// The official launcher's rules: the last matching rule decides, no rules means allowed.
fn library_allowed(library: &Value) -> Result<bool, String> {
let system = rule_system();
let architectures = rule_architectures();
let rules = library
.get("rules")
.and_then(Value::as_array)
.unwrap_or(&[]);
let mut allowed = rules.is_empty();
for rule in rules {
let empty = Value::Object(Vec::new());
let os = rule.get("os").unwrap_or(&empty);
if os.get("name").and_then(Value::as_str).unwrap_or(system) != system {
continue;
}
if let Some(arch) = os.get("arch") {
let pattern = regex::Regex::new(&format!("^(?:{})$", arch.as_str().unwrap_or("")))
.map_err(|e| e.to_string())?;
if !architectures.iter().any(|name| pattern.is_match(name)) {
continue;
}
}
if let Some(version) = os.get("version") {
let pattern =
regex::Regex::new(version.as_str().unwrap_or("")).map_err(|e| e.to_string())?;
if !pattern.is_match(&os_release()) {
continue;
}
}
if rule.get("features").is_some_and(|f| match f {
Value::Object(o) => !o.is_empty(),
Value::Array(a) => !a.is_empty(),
Value::Null | Value::Bool(false) => false,
Value::Str(s) => !s.is_empty(),
_ => true,
}) {
continue;
}
allowed = rule.get("action").and_then(Value::as_str) == Some("allow");
}
Ok(allowed)
}
fn classifier_ends_with(classifier: &str, names: &[&str]) -> bool {
names.iter().any(|name| {
classifier == *name
|| classifier
.strip_suffix(name)
.is_some_and(|rest| rest.ends_with('-'))
})
}
/// Modern manifests list x64 and ARM natives with OS-only rules; keep this machine's. A plain
/// natives-<os> jar is the x64 build when the manifest also lists an ARM variant of it.
fn native_library_matches(library: &Value, names: &[&str]) -> bool {
let arm = rule_architectures().contains(&"arm64");
let name = library.get("name").and_then(Value::as_str).unwrap_or("");
let classifier = name.split(':').nth(3).unwrap_or("");
if classifier_ends_with(classifier, &["arm64", "aarch64", "aarch_64"]) {
return arm;
}
if classifier_ends_with(classifier, &["x86_64", "x64"]) {
return !arm;
}
if arm && classifier.starts_with("natives-") {
return !["arm64", "aarch64"]
.iter()
.any(|suffix| names.contains(&format!("{name}-{suffix}").as_str()));
}
true
}
/// The client libraries the official launcher installs on this OS and architecture.
fn client_library_artifacts(manifest: &Value) -> Result<Vec<&Value>, String> {
let libraries = manifest
.get("libraries")
.and_then(Value::as_array)
.ok_or("The version JSON has no libraries")?;
let names: Vec<&str> = libraries
.iter()
.filter_map(|l| l.get("name").and_then(Value::as_str))
.collect();
let mut artifacts = Vec::new();
for library in libraries {
let Some(artifact) = library.get("downloads").and_then(|d| d.get("artifact")) else {
continue;
};
if library_allowed(library)? && native_library_matches(library, &names) {
artifacts.push(artifact);
}
}
Ok(artifacts)
}
/// A relative path with only normal components and forward slashes, from a manifest or bundle.
fn safe_relative(path: &str) -> Option<PathBuf> {
if path.contains('\\') || path.starts_with('/') {
return None;
}
let parts: Vec<&str> = path
.split('/')
.filter(|p| !p.is_empty() && *p != ".")
.collect();
if parts.contains(&"..") || (cfg!(windows) && parts.first().is_some_and(|p| p.contains(':'))) {
return None;
}
Some(parts.iter().collect())
}
fn artifact_fields(artifact: &Value) -> Result<(&str, &str), String> {
let path = artifact.get("path").and_then(Value::as_str);
let sha1 = artifact.get("sha1").and_then(Value::as_str);
path.zip(sha1)
.ok_or_else(|| "A library in the version JSON has no path or sha1".to_string())
}
/// `(sha256, identifier, path)` lines of a server bundle's versions.list or libraries.list.
fn bundle_entries(
archive: &mut zip::ZipArchive<fs::File>,
name: &str,
) -> Result<Vec<(String, String, PathBuf, String)>, String> {
let text = crate::zip_text(archive, name)?;
let mut entries = Vec::new();
for line in crate::split_lines(&text) {
if line.trim().is_empty() {
continue;
}
let fields: Vec<&str> = line.split('\t').collect();
let [checksum, identifier, relative] = fields[..] else {
return Err(format!("Malformed line in {name}"));
};
let path =
safe_relative(relative).ok_or_else(|| format!("Unsafe bundled path: {relative}"))?;
// PurePosixPath's text: empty and `.` parts dropped.
let posix = relative
.split('/')
.filter(|p| !p.is_empty() && *p != ".")
.collect::<Vec<_>>()
.join("/");
entries.push((checksum.to_string(), identifier.to_string(), path, posix));
}
Ok(entries)
}
fn extract_verified(
archive: &mut zip::ZipArchive<fs::File>,
member: &str,
destination: &Path,
checksum: &str,
) -> Result<(), String> {
let data = crate::zip_bytes(archive, member)?;
if crate::sha256_hex(&data) != checksum {
return Err(format!("Bundled artifact checksum mismatch: {member}"));
}
fs::create_dir_all(destination.parent().unwrap()).map_err(|e| e.to_string())?;
fs::write(destination, data).map_err(|e| format!("{}: {e}", destination.display()))
}
/// Modern server downloads are launchers: the game jar and its libraries are inside.
fn unpack_server(
server: PathBuf,
directory: &Path,
version: &str,
) -> Result<(PathBuf, Vec<PathBuf>), String> {
let mut archive = crate::open_zip(&server)?;
if archive.index_for_name("META-INF/versions.list").is_none() {
return Ok((server, Vec::new()));
}
let versions = bundle_entries(&mut archive, "META-INF/versions.list")?;
if versions.len() != 1 || versions[0].1 != version {
return Err("Expected one bundled server matching the pinned version".into());
}
let (checksum, _, _, relative) = &versions[0];
let game = directory.join("jars/server-game.jar");
extract_verified(
&mut archive,
&format!("META-INF/versions/{relative}"),
&game,
checksum,
)?;
let mut libraries = Vec::new();
for (checksum, _, path, relative) in bundle_entries(&mut archive, "META-INF/libraries.list")? {
let destination = directory.join("libraries/server").join(path);
extract_verified(
&mut archive,
&format!("META-INF/libraries/{relative}"),
&destination,
checksum.as_str(),
)?;
libraries.push(destination);
}
Ok((game, libraries))
}
/// Imports the player's own pinned game files into `.local/minecraft/<version>` and writes the
/// class inventories and prepared.json (tools/lab.py `prepare`). The decompiler lab.py also
/// fetches is a development tool, not something preparing needs; `tools/lab.py prepare` gets it.
pub fn import(
root: &Path,
lock: &Lock,
minecraft: Option<&Path>,
server_jar: Option<&Path>,
log: Log,
) -> Result<(), String> {
let minecraft = minecraft_dir(minecraft);
let version = &lock.version;
let manifest = import_manifest(lock, &minecraft)?;
let client = import_copy(
&minecraft
.join("versions")
.join(version)
.join(format!("{version}.jar")),
&lock.directory.join("jars/client.jar"),
lock.pin("client_sha1")?,
"sha1",
&hint(LAUNCHER_HINT, version),
)?;
let downloads = manifest.get("downloads");
if downloads.and_then(|d| d.get("client_mappings")).is_some()
|| downloads.and_then(|d| d.get("server_mappings")).is_some()
{
return Err(
"This workflow targets unobfuscated releases. Older releases need a remapping stage."
.into(),
);
}
let id = manifest
.get("id")
.and_then(Value::as_str)
.ok_or("The version JSON has no id")?;
log(&format!(
"Imported client {id} from {}",
minecraft.display()
));
let server = import_server(lock, server_jar)?;
let (server, server_libraries) = unpack_server(server, &lock.directory, id)?;
let mut client_libraries = Vec::new();
for artifact in client_library_artifacts(&manifest)? {
let (path, sha1) = artifact_fields(artifact)?;
let relative = safe_relative(path).ok_or("Unsafe library path")?;
client_libraries.push(import_copy(
&minecraft.join("libraries").join(&relative),
&lock.directory.join("libraries/client").join(&relative),
sha1,
"sha1",
&hint(LAUNCHER_HINT, version),
)?);
}
for (side, jar) in [("client", &client), ("server", &server)] {
let data = crate::inventory::inventory(jar)?;
let count = |key| data.get(key).cloned().unwrap_or(Value::Null);
write_json(
&lock
.directory
.join(format!("reports/{side}-inventory.json")),
&data,
)?;
log(&format!(
"{side}: {} classes, {} methods",
pyjson::dumps(&count("class_count"), &pyjson::DEFAULT),
pyjson::dumps(&count("method_count"), &pyjson::DEFAULT)
));
}
let paths = |list: &[PathBuf]| -> Result<Value, String> {
Ok(Value::Array(
list.iter()
.map(|p| relative_display(p, root).map(Value::Str))
.collect::<Result<_, _>>()?,
))
};
write_json(
&lock.directory.join("prepared.json"),
&Value::Object(vec![
("version".into(), Value::str(id)),
(
"jars".into(),
Value::Object(vec![
(
"client".into(),
Value::Str(relative_display(&client, root)?),
),
(
"server".into(),
Value::Str(relative_display(&server, root)?),
),
]),
),
(
"libraries".into(),
Value::Object(vec![
("client".into(), paths(&client_libraries)?),
("server".into(), paths(&server_libraries)?),
]),
),
]),
)?;
log(&format!("Artifacts: {}", lock.directory.display()));
Ok(())
}
/// The checked jars (`client`, `server` = the bundle's inner game jar) and the server
/// libraries, authenticated against the lock and the bundle's own checksums on every run.
pub struct Artifacts {
pub manifest: Value,
pub jars: HashMap<&'static str, PathBuf>,
pub libraries: Vec<PathBuf>,
}
pub fn verify(lock: &Lock) -> Result<Artifacts, String> {
let manifest_path = lock.directory.join("manifest.json");
if !manifest_path.exists() {
return Err(format!(
"{} is missing: prepare with --prepare first",
manifest_path.display()
));
}
if !lock.manifest_accepted(&digest(&manifest_path, "sha1")?) {
return Err("Manifest checksum changed; prepare again with --prepare".into());
}
let manifest = read_json(&manifest_path)?;
let id = manifest.get("id").and_then(Value::as_str).unwrap_or("");
if id != lock.version {
return Err("Manifest version differs from the lock".into());
}
let mut jars = HashMap::new();
for side in ["client", "server"] {
let jar = lock.directory.join(format!("jars/{side}.jar"));
let pinned = manifest
.get("downloads")
.and_then(|d| d.get(side))
.and_then(|d| d.get("sha1"))
.and_then(Value::as_str);
if !jar.exists() || Some(digest(&jar, "sha1")?.as_str()) != pinned {
return Err(format!(
"{side} artifact checksum changed; prepare again with --prepare"
));
}
jars.insert(side, jar);
}
let mut archive = crate::open_zip(&jars["server"])?;
let versions = bundle_entries(&mut archive, "META-INF/versions.list")?;
if versions.len() != 1 || versions[0].1 != id {
return Err("Unexpected server bundle".into());
}
let game = lock.directory.join("jars/server-game.jar");
if !game.exists() || digest(&game, "sha256")? != versions[0].0 {
return Err("Inner server artifact checksum changed".into());
}
jars.insert("server", game);
let mut libraries = Vec::new();
for (checksum, _, path, relative) in bundle_entries(&mut archive, "META-INF/libraries.list")? {
let library = lock.directory.join("libraries/server").join(path);
if !library.exists() || digest(&library, "sha256")? != checksum {
return Err(format!("Dependency checksum changed: {relative}"));
}
libraries.push(library);
}
Ok(Artifacts {
manifest,
jars,
libraries,
})
}
/// The client libraries, checked against the version JSON (tools/test_model_baking_parity.py's
/// `client_libraries`); the client-side exporters compile against them.
pub fn client_libraries(lock: &Lock, manifest: &Value) -> Result<Vec<PathBuf>, String> {
let mut libraries = Vec::new();
for artifact in client_library_artifacts(manifest)? {
let (path, sha1) = artifact_fields(artifact)?;
let relative = safe_relative(path).ok_or("Unsafe client library path")?;
let library = lock.directory.join("libraries/client").join(&relative);
if !library.exists() || digest(&library, "sha1")? != sha1 {
return Err(format!("Client dependency checksum changed: {path}"));
}
libraries.push(library);
}
Ok(libraries)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn natives_follow_the_architecture() {
let names = ["a:b:1:natives-macos", "a:b:1:natives-macos-arm64"];
let lib = |name: &str| pyjson::loads(&format!("{{\"name\":\"{name}\"}}")).unwrap();
let arm = rule_architectures().contains(&"arm64");
assert_eq!(
native_library_matches(&lib("a:b:1:natives-macos-arm64"), &names),
arm
);
assert_eq!(
native_library_matches(&lib("x:y:1:linux-x86_64"), &names),
!arm
);
assert_eq!(
native_library_matches(&lib("x:y:1:linux-aarch_64"), &names),
arm
);
assert_eq!(
native_library_matches(&lib("a:b:1:natives-macos"), &names),
!arm
);
assert!(native_library_matches(&lib("a:b:1"), &names));
}
#[test]
fn unsafe_paths_are_refused() {
assert!(safe_relative("a/../b").is_none());
assert!(safe_relative("/a").is_none());
assert!(safe_relative("a\\b").is_none());
assert_eq!(
safe_relative("a/b.jar").unwrap(),
Path::new("a").join("b.jar")
);
}
}
+177
View File
@@ -0,0 +1,177 @@
//! Class inventories of the imported jars (tools/lab.py's `inventory` and `ClassReader`): each
//! class's SourceFile and method count, read from the bytecode; no Java text is published.
use crate::pyjson::Value;
use std::path::Path;
struct ClassReader<'a> {
data: &'a [u8],
offset: usize,
}
impl ClassReader<'_> {
fn take(&mut self, count: usize) -> Result<&[u8], String> {
let value = self
.data
.get(self.offset..self.offset + count)
.ok_or("Truncated class file")?;
self.offset += count;
Ok(value)
}
fn number(&mut self, count: usize) -> Result<usize, String> {
Ok(self
.take(count)?
.iter()
.fold(0usize, |n, b| (n << 8) | *b as usize))
}
fn attributes(&mut self) -> Result<Vec<(usize, Vec<u8>)>, String> {
let count = self.number(2)?;
let mut attributes = Vec::with_capacity(count);
for _ in 0..count {
let name = self.number(2)?;
let length = self.number(4)?;
attributes.push((name, self.take(length)?.to_vec()));
}
Ok(attributes)
}
/// `(SourceFile, method count)`.
fn metadata(&mut self) -> Result<(Option<String>, usize), String> {
if self.take(4)? != b"\xca\xfe\xba\xbe" {
return Err("Invalid class file magic".into());
}
self.take(4)?; // Minor and major version.
let mut pool: Vec<Option<String>> = vec![None; self.number(2)?];
let mut i = 1;
while i < pool.len() {
let tag = self.number(1)?;
if tag == 1 {
let length = self.number(2)?;
pool[i] = Some(String::from_utf8_lossy(self.take(length)?).into_owned());
} else {
let size = match tag {
3 | 4 | 9 | 10 | 11 | 12 | 17 | 18 => 4,
5 | 6 => 8,
7 | 8 | 16 | 19 | 20 => 2,
15 => 3,
_ => return Err(format!("Unknown constant pool tag: {tag}")),
};
self.take(size)?;
}
i += if tag == 5 || tag == 6 { 2 } else { 1 };
}
self.take(6)?; // Access, this class, super class.
let interfaces = self.number(2)?;
self.take(interfaces * 2)?;
for _ in 0..self.number(2)? {
self.take(6)?; // Fields.
self.attributes()?;
}
let methods = self.number(2)?;
for _ in 0..methods {
self.take(6)?;
self.attributes()?;
}
let mut source = None;
for (name, data) in self.attributes()? {
if pool.get(name).and_then(Option::as_deref) == Some("SourceFile") {
let [high, low] = data[..] else {
return Err("Malformed SourceFile attribute".into());
};
let index = ((high as usize) << 8) | low as usize;
source = Some(
pool.get(index)
.cloned()
.flatten()
.ok_or("SourceFile points outside the constant pool")?,
);
}
}
Ok((source, methods))
}
}
/// PurePosixPath(name).parent as text.
fn parent(name: &str) -> &str {
match name.trim_end_matches('/').rsplit_once('/') {
Some((parent, _)) if !parent.is_empty() => parent,
Some(_) => "/",
None => ".",
}
}
pub fn inventory(jar: &Path) -> Result<Value, String> {
let mut archive = crate::open_zip(jar)?;
let mut classes = Vec::new();
let mut packages: Vec<(String, usize)> = Vec::new();
let mut sources = std::collections::BTreeSet::new();
let mut method_count = 0;
for index in 0..archive.len() {
let mut item = archive.by_index(index).map_err(|e| e.to_string())?;
let name = item.name().to_string();
if !name.ends_with(".class") {
continue;
}
let size = item.size();
let mut data = Vec::with_capacity(size as usize);
std::io::Read::read_to_end(&mut item, &mut data).map_err(|e| e.to_string())?;
let (source, methods) = ClassReader {
data: &data,
offset: 0,
}
.metadata()
.map_err(|e| format!("{name}: {e}"))?;
let file = name.rsplit('/').next().unwrap();
let source = source.filter(|s| !s.is_empty()).unwrap_or_else(|| {
let stem = file.split('$').next().unwrap();
format!("{}.java", stem.strip_suffix(".class").unwrap_or(stem))
});
if source.contains('/') || source.contains('\\') || source == "." {
return Err(format!("Unsafe SourceFile attribute in {name}"));
}
let directory = parent(&name);
let source_path = if directory == "." {
source
} else {
format!("{directory}/{source}")
};
match packages.iter_mut().find(|(p, _)| p == directory) {
Some(entry) => entry.1 += 1,
None => packages.push((directory.to_string(), 1)),
}
sources.insert(source_path.clone());
method_count += methods;
classes.push(Value::Object(vec![
("class".into(), Value::Str(name)),
("source".into(), Value::Str(source_path)),
("bytecode_bytes".into(), Value::Int(size.to_string())),
("methods".into(), Value::Int(methods.to_string())),
]));
}
// Counter.most_common: by count, ties in first-seen order.
packages.sort_by_key(|(_, count)| std::cmp::Reverse(*count));
Ok(Value::Object(vec![
(
"jar_sha256".into(),
Value::Str(crate::digest(jar, "sha256")?),
),
("class_count".into(), Value::Int(classes.len().to_string())),
(
"expected_source_files".into(),
Value::Array(sources.into_iter().map(Value::Str).collect()),
),
("method_count".into(), Value::Int(method_count.to_string())),
(
"packages".into(),
Value::Object(
packages
.into_iter()
.map(|(p, n)| (p, Value::Int(n.to_string())))
.collect(),
),
),
("classes".into(), Value::Array(classes)),
]))
}
+179
View File
@@ -0,0 +1,179 @@
//! The JDK that compiles and runs the vanilla exporters. They call Mojang's own code, so they
//! stay Java; everything around them is here.
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
pub struct Jdk {
pub javac: PathBuf,
pub java: PathBuf,
}
const TIMEOUT: Duration = Duration::from_secs(3600);
fn executable(dir: &Path, name: &str) -> PathBuf {
dir.join(if cfg!(windows) {
format!("{name}.exe")
} else {
name.to_string()
})
}
fn on_path(name: &str) -> Option<PathBuf> {
std::env::split_paths(&std::env::var_os("PATH")?)
.map(|dir| executable(&dir, name))
.find(|p| p.is_file())
}
/// The major version `javac -version` reports ("javac 25.0.1" → 25; "javac 1.8.0" → 8).
fn javac_major(javac: &Path) -> Result<u32, String> {
let output = Command::new(javac)
.arg("-version")
.stdin(Stdio::null())
.output()
.map_err(|e| format!("{}: {e}", javac.display()))?;
let text = String::from_utf8_lossy(&output.stdout).into_owned()
+ &String::from_utf8_lossy(&output.stderr);
let version = text
.split_whitespace()
.skip_while(|w| *w != "javac")
.nth(1)
.ok_or_else(|| {
format!(
"{} -version printed no version: {}",
javac.display(),
text.trim()
)
})?;
let mut parts = version.split(['.', '-', '+', '_']);
let first: u32 = parts.next().and_then(|p| p.parse().ok()).unwrap_or(0);
Ok(if first == 1 {
parts.next().and_then(|p| p.parse().ok()).unwrap_or(0)
} else {
first
})
}
/// Finds a JDK with `javac` of at least `major`, the version research-lock.json pins:
/// `JAVA_HOME` when it holds one, else `javac` and `java` on the PATH (as the Python preparer
/// ran them), else, on macOS, the JDK `/usr/libexec/java_home -v <major>` reports.
pub fn find(major: i64) -> Result<Jdk, String> {
let need = format!(
"Java {major} is needed (a JDK with javac, e.g. Eclipse Temurin {major}). \
Install it, or set JAVA_HOME to it, and try again."
);
let mut candidates = Vec::new();
if let Some(home) = std::env::var_os("JAVA_HOME").filter(|h| !h.is_empty()) {
let bin = Path::new(&home).join("bin");
candidates.push((
executable(&bin, "javac"),
executable(&bin, "java"),
"JAVA_HOME",
));
}
if let (Some(javac), Some(java)) = (on_path("javac"), on_path("java")) {
candidates.push((javac, java, "PATH"));
}
if cfg!(target_os = "macos") {
if let Ok(out) = Command::new("/usr/libexec/java_home")
.args(["-v", &major.to_string()])
.stderr(Stdio::null())
.output()
{
let home = String::from_utf8_lossy(&out.stdout).trim().to_string();
if out.status.success() && !home.is_empty() {
let bin = Path::new(&home).join("bin");
candidates.push((
executable(&bin, "javac"),
executable(&bin, "java"),
"/usr/libexec/java_home",
));
}
}
}
let mut problems = Vec::new();
for (javac, java, from) in candidates {
if !javac.is_file() || !java.is_file() {
problems.push(format!(
"{from} ({}) has no javac and java",
javac.parent().unwrap().display()
));
continue;
}
match javac_major(&javac) {
Ok(found) if i64::from(found) >= major => return Ok(Jdk { javac, java }),
Ok(found) => problems.push(format!("{from} has Java {found} ({})", javac.display())),
Err(e) => problems.push(e),
}
}
if problems.is_empty() {
Err(format!("{need} No javac was found."))
} else {
Err(format!("{need} Found: {}.", problems.join("; ")))
}
}
/// A classpath with the platform's separator.
pub fn classpath<'a>(paths: impl IntoIterator<Item = &'a Path>) -> String {
let separator = if cfg!(windows) { ";" } else { ":" };
paths
.into_iter()
.map(|p| p.to_string_lossy().into_owned())
.collect::<Vec<_>>()
.join(separator)
}
impl Jdk {
/// `javac --release <major> -proc:none` with Python's `subprocess.run(..., check=True,
/// stdout=log, stderr=STDOUT, timeout=3600)`.
pub fn javac(&self, major: i64, args: &[String], log: &fs::File) -> Result<(), String> {
let mut command = Command::new(&self.javac);
command
.args(["--release", &major.to_string(), "-proc:none"])
.args(args);
run(command, log)
}
pub fn java(&self, args: &[String], cwd: Option<&Path>, log: &fs::File) -> Result<(), String> {
let mut command = Command::new(&self.java);
command.args(args);
if let Some(dir) = cwd {
command.current_dir(dir);
}
run(command, log)
}
}
fn run(mut command: Command, log: &fs::File) -> Result<(), String> {
let out = log.try_clone().map_err(|e| e.to_string())?;
let err = log.try_clone().map_err(|e| e.to_string())?;
let program = Path::new(command.get_program())
.file_name()
.unwrap_or_default()
.to_string_lossy()
.into_owned();
let mut child = command
.stdin(Stdio::null())
.stdout(out)
.stderr(err)
.spawn()
.map_err(|e| format!("Cannot start {program}: {e}"))?;
let start = Instant::now();
loop {
if let Some(status) = child.try_wait().map_err(|e| e.to_string())? {
return if status.success() {
Ok(())
} else {
Err(format!("{program} failed ({status})"))
};
}
if start.elapsed() > TIMEOUT {
let _ = child.kill();
let _ = child.wait();
return Err(format!("{program} took over an hour and was stopped"));
}
std::thread::sleep(Duration::from_millis(50));
}
}
File diff suppressed because it is too large Load Diff
+109
View File
@@ -0,0 +1,109 @@
//! `horizon-prepare`: prepares Horizon's resources from your own Minecraft Java Edition.
//! Its flags are the ones tools/prepare_native_client.py took.
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::ExitCode;
const HELP: &str = "\
horizon-prepare: prepare Horizon's resources from your own Minecraft Java Edition
Usage: horizon-prepare [--prepare] [--server-jar PATH] [--minecraft-dir PATH] [--root PATH]
--prepare Import the pinned game files from your own copy first
--server-jar PATH The pinned server jar you downloaded yourself
(default: HORIZON_SERVER_JAR, or the copy imported before)
--minecraft-dir PATH The official launcher's game directory with your installed copy
(default: HORIZON_MINECRAFT_DIR, or the launcher's own)
--root PATH The Horizon folder to prepare (default: the folder this program is
in, else the nearest folder above the current one with research-lock.json)
Needs Minecraft played once from the official launcher, and Java (a JDK) of the version
research-lock.json pins: JAVA_HOME, else javac and java on the PATH.
The resources go to <root>/.local/horizon-client/<version>; pass that to horizon-client --resources.
";
/// The folder with research-lock.json: beside the executable in a release folder, else the
/// current folder or one above it (a checkout, with the binary in target/).
fn default_root() -> Option<PathBuf> {
let beside = std::env::current_exe()
.ok()
.and_then(|exe| exe.parent().map(Path::to_path_buf));
if let Some(dir) = beside.filter(|d| d.join("research-lock.json").is_file()) {
return Some(dir);
}
let cwd = std::env::current_dir().ok()?;
cwd.ancestors()
.find(|d| d.join("research-lock.json").is_file())
.map(Path::to_path_buf)
}
fn main() -> ExitCode {
let mut options = horizon_prepare::Options {
root: PathBuf::new(),
import: false,
minecraft_dir: None,
server_jar: None,
};
let mut root = None;
let mut args = std::env::args_os().skip(1);
while let Some(arg) = args.next() {
let text = arg.to_string_lossy().into_owned();
let (flag, inline) = match text.split_once('=') {
Some((flag, value)) if flag.starts_with("--") => {
(flag.to_string(), Some(value.to_string()))
}
_ => (text.clone(), None),
};
let mut value = |name: &str| -> Result<PathBuf, String> {
match &inline {
Some(v) => Ok(PathBuf::from(v)),
None => args
.next()
.map(PathBuf::from)
.ok_or_else(|| format!("{name} needs a path")),
}
};
let parsed = match flag.as_str() {
"--prepare" if inline.is_none() => {
options.import = true;
Ok(())
}
"--server-jar" => value("--server-jar").map(|v| options.server_jar = Some(v)),
"--minecraft-dir" => value("--minecraft-dir").map(|v| options.minecraft_dir = Some(v)),
"--root" => value("--root").map(|v| root = Some(v)),
"-h" | "--help" => {
print!("{HELP}");
return ExitCode::SUCCESS;
}
"-V" | "--version" => {
println!("horizon-prepare {}", env!("CARGO_PKG_VERSION"));
return ExitCode::SUCCESS;
}
_ => Err(format!("unrecognized argument: {text}")),
};
if let Err(e) = parsed {
eprintln!("horizon-prepare: error: {e}\n\n{HELP}");
return ExitCode::from(2);
}
}
let Some(root) = root.or_else(default_root) else {
eprintln!(
"horizon-prepare: error: no research-lock.json beside this program or above the current folder; pass --root"
);
return ExitCode::from(2);
};
options.root = root;
let log = |line: &str| {
let mut out = std::io::stdout().lock();
let _ = writeln!(out, "{line}");
let _ = out.flush();
};
match horizon_prepare::prepare(&options, &log) {
Ok(_) => ExitCode::SUCCESS,
Err(e) => {
eprintln!("error: {e}");
ExitCode::FAILURE
}
}
}
+541
View File
@@ -0,0 +1,541 @@
//! JSON read and written exactly as Python's `json` module does, so the files this crate writes
//! are byte for byte the ones `tools/prepare_native_client.py` wrote: objects keep their key
//! order (a repeated key keeps its first place and its last value), integers keep every digit,
//! floats print as Python's `repr`, and strings are escaped as `ensure_ascii=True` escapes them.
use std::fmt::Write;
#[derive(Clone, Debug)]
pub enum Value {
Null,
Bool(bool),
/// A JSON integer, kept as its decimal digits (Python's integers have no size limit).
Int(String),
Float(f64),
Str(String),
Array(Vec<Value>),
Object(Vec<(String, Value)>),
}
impl Value {
pub fn str(s: impl Into<String>) -> Value {
Value::Str(s.into())
}
pub fn get(&self, key: &str) -> Option<&Value> {
match self {
Value::Object(fields) => fields.iter().find(|(k, _)| k == key).map(|(_, v)| v),
_ => None,
}
}
pub fn index(&self, i: usize) -> Option<&Value> {
match self {
Value::Array(items) => items.get(i),
_ => None,
}
}
pub fn as_str(&self) -> Option<&str> {
match self {
Value::Str(s) => Some(s),
_ => None,
}
}
pub fn as_array(&self) -> Option<&[Value]> {
match self {
Value::Array(items) => Some(items),
_ => None,
}
}
pub fn as_object(&self) -> Option<&[(String, Value)]> {
match self {
Value::Object(fields) => Some(fields),
_ => None,
}
}
pub fn as_i64(&self) -> Option<i64> {
match self {
Value::Int(digits) => digits.parse().ok(),
_ => None,
}
}
/// `dict[key] = value`: a new key goes last, an existing one keeps its place.
pub fn insert(&mut self, key: &str, value: Value) {
if let Value::Object(fields) = self {
match fields.iter_mut().find(|(k, _)| k == key) {
Some(field) => field.1 = value,
None => fields.push((key.to_string(), value)),
}
}
}
}
/// Python's `==`: objects compare without regard to key order, and 1 == 1.0.
impl PartialEq for Value {
fn eq(&self, other: &Value) -> bool {
match (self, other) {
(Value::Null, Value::Null) => true,
(Value::Bool(a), Value::Bool(b)) => a == b,
(Value::Int(a), Value::Int(b)) => a == b,
(Value::Float(a), Value::Float(b)) => a == b,
(Value::Int(a), Value::Float(b)) | (Value::Float(b), Value::Int(a)) => {
a.parse::<f64>().is_ok_and(|a| a == *b && b.fract() == 0.0)
}
(Value::Str(a), Value::Str(b)) => a == b,
(Value::Array(a), Value::Array(b)) => a == b,
(Value::Object(a), Value::Object(b)) => {
a.len() == b.len() && a.iter().all(|(k, v)| other.get(k) == Some(v))
}
_ => false,
}
}
}
pub fn loads(text: &str) -> Result<Value, String> {
let mut parser = Parser {
text: text.as_bytes(),
at: 0,
};
parser.space();
let value = parser.value()?;
parser.space();
if parser.at != parser.text.len() {
return Err(parser.error("Extra data"));
}
Ok(value)
}
struct Parser<'a> {
text: &'a [u8],
at: usize,
}
impl Parser<'_> {
fn error(&self, what: &str) -> String {
format!("{what} at byte {}", self.at)
}
fn space(&mut self) {
while matches!(self.text.get(self.at), Some(b' ' | b'\t' | b'\n' | b'\r')) {
self.at += 1;
}
}
fn literal(&mut self, word: &str) -> bool {
if self.text[self.at..].starts_with(word.as_bytes()) {
self.at += word.len();
true
} else {
false
}
}
fn value(&mut self) -> Result<Value, String> {
match self.text.get(self.at) {
Some(b'{') => {
self.at += 1;
let mut object = Value::Object(Vec::new());
self.space();
if self.text.get(self.at) == Some(&b'}') {
self.at += 1;
return Ok(object);
}
loop {
self.space();
if self.text.get(self.at) != Some(&b'"') {
return Err(self.error("Expecting property name enclosed in double quotes"));
}
let key = self.string()?;
self.space();
if self.text.get(self.at) != Some(&b':') {
return Err(self.error("Expecting ':' delimiter"));
}
self.at += 1;
self.space();
let value = self.value()?;
object.insert(&key, value);
self.space();
match self.text.get(self.at) {
Some(b',') => self.at += 1,
Some(b'}') => {
self.at += 1;
return Ok(object);
}
_ => return Err(self.error("Expecting ',' delimiter")),
}
}
}
Some(b'[') => {
self.at += 1;
let mut items = Vec::new();
self.space();
if self.text.get(self.at) == Some(&b']') {
self.at += 1;
return Ok(Value::Array(items));
}
loop {
self.space();
items.push(self.value()?);
self.space();
match self.text.get(self.at) {
Some(b',') => self.at += 1,
Some(b']') => {
self.at += 1;
return Ok(Value::Array(items));
}
_ => return Err(self.error("Expecting ',' delimiter")),
}
}
}
Some(b'"') => Ok(Value::Str(self.string()?)),
Some(b'n') if self.literal("null") => Ok(Value::Null),
Some(b't') if self.literal("true") => Ok(Value::Bool(true)),
Some(b'f') if self.literal("false") => Ok(Value::Bool(false)),
Some(b'N') if self.literal("NaN") => Ok(Value::Float(f64::NAN)),
Some(b'I') if self.literal("Infinity") => Ok(Value::Float(f64::INFINITY)),
Some(b'-') if self.literal("-Infinity") => Ok(Value::Float(f64::NEG_INFINITY)),
Some(b'-' | b'0'..=b'9') => self.number(),
_ => Err(self.error("Expecting value")),
}
}
/// Python's NUMBER_RE: `(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?`.
fn number(&mut self) -> Result<Value, String> {
let start = self.at;
let digits = |p: &mut Self| {
let from = p.at;
while p.text.get(p.at).is_some_and(u8::is_ascii_digit) {
p.at += 1;
}
p.at - from
};
if self.text.get(self.at) == Some(&b'-') {
self.at += 1;
}
if self.text.get(self.at) == Some(&b'0') {
self.at += 1;
} else if digits(self) == 0 {
self.at = start;
return Err(self.error("Expecting value"));
}
let integer_end = self.at;
let mut float = false;
if self.text.get(self.at) == Some(&b'.') {
let dot = self.at;
self.at += 1;
if digits(self) == 0 {
self.at = dot;
} else {
float = true;
}
}
if matches!(self.text.get(self.at), Some(b'e' | b'E')) {
let mark = self.at;
self.at += 1;
if matches!(self.text.get(self.at), Some(b'-' | b'+')) {
self.at += 1;
}
if digits(self) == 0 {
self.at = mark;
} else {
float = true;
}
}
let text = std::str::from_utf8(&self.text[start..self.at]).unwrap();
if float {
text.parse::<f64>()
.map(Value::Float)
.map_err(|e| e.to_string())
} else {
debug_assert_eq!(self.at, integer_end);
Ok(Value::Int(if text == "-0" {
"0".into()
} else {
text.into()
}))
}
}
fn hex4(&mut self) -> Result<u32, String> {
let digits = self
.text
.get(self.at..self.at + 4)
.and_then(|d| std::str::from_utf8(d).ok())
.filter(|d| d.bytes().all(|b| b.is_ascii_hexdigit()))
.ok_or_else(|| self.error("Invalid \\uXXXX escape"))?;
self.at += 4;
Ok(u32::from_str_radix(digits, 16).unwrap())
}
fn string(&mut self) -> Result<String, String> {
self.at += 1;
let mut out = String::new();
loop {
let run = self.at;
while self
.text
.get(self.at)
.is_some_and(|&b| b != b'"' && b != b'\\' && b >= 0x20)
{
self.at += 1;
}
out.push_str(std::str::from_utf8(&self.text[run..self.at]).map_err(|e| e.to_string())?);
match self.text.get(self.at) {
Some(b'"') => {
self.at += 1;
return Ok(out);
}
Some(b'\\') => {
self.at += 1;
let escape = *self
.text
.get(self.at)
.ok_or_else(|| self.error("Unterminated string"))?;
self.at += 1;
match escape {
b'"' => out.push('"'),
b'\\' => out.push('\\'),
b'/' => out.push('/'),
b'b' => out.push('\u{8}'),
b'f' => out.push('\u{c}'),
b'n' => out.push('\n'),
b'r' => out.push('\r'),
b't' => out.push('\t'),
b'u' => {
let mut code = self.hex4()?;
if (0xd800..0xdc00).contains(&code)
&& self.text[self.at..].starts_with(b"\\u")
{
let back = self.at;
self.at += 2;
let low = self.hex4()?;
if (0xdc00..0xe000).contains(&low) {
code = 0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00);
} else {
self.at = back;
}
}
out.push(char::from_u32(code).ok_or_else(|| {
self.error(
"Lone surrogate (Python keeps it; Horizon's files have none)",
)
})?);
}
_ => return Err(self.error("Invalid \\escape")),
}
}
_ => return Err(self.error("Unterminated string or control character")),
}
}
}
}
/// `json.dumps(value, indent=indent, separators=(item, key))` with `ensure_ascii=True`.
/// Python's defaults are `(", ", ": ")` without an indent and `(",", ": ")` with one.
pub struct Format {
pub indent: Option<usize>,
pub item: &'static str,
pub key: &'static str,
}
/// `json.dumps(value)`.
pub const DEFAULT: Format = Format {
indent: None,
item: ", ",
key: ": ",
};
/// `json.dumps(value, separators=(',', ':'))`.
pub const COMPACT: Format = Format {
indent: None,
item: ",",
key: ":",
};
/// `json.dumps(value, indent=2)`.
pub const INDENT2: Format = Format {
indent: Some(2),
item: ",",
key: ": ",
};
pub fn dumps(value: &Value, format: &Format) -> String {
let mut out = String::new();
write_value(&mut out, value, format, 0);
out
}
fn newline(out: &mut String, format: &Format, level: usize) {
if let Some(indent) = format.indent {
out.push('\n');
out.extend(std::iter::repeat_n(' ', indent * level));
}
}
fn write_value(out: &mut String, value: &Value, format: &Format, level: usize) {
match value {
Value::Null => out.push_str("null"),
Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
Value::Int(digits) => out.push_str(digits),
Value::Float(f) => out.push_str(&float_repr(*f)),
Value::Str(s) => write_string(out, s),
Value::Array(items) => {
if items.is_empty() {
out.push_str("[]");
return;
}
out.push('[');
for (i, item) in items.iter().enumerate() {
if i > 0 {
out.push_str(format.item);
}
newline(out, format, level + 1);
write_value(out, item, format, level + 1);
}
newline(out, format, level);
out.push(']');
}
Value::Object(fields) => {
if fields.is_empty() {
out.push_str("{}");
return;
}
out.push('{');
for (i, (key, item)) in fields.iter().enumerate() {
if i > 0 {
out.push_str(format.item);
}
newline(out, format, level + 1);
write_string(out, key);
out.push_str(format.key);
write_value(out, item, format, level + 1);
}
newline(out, format, level);
out.push('}');
}
}
}
/// Python's `py_encode_basestring_ascii`: every character outside space..`~` is escaped,
/// with lowercase hex and surrogate pairs above the BMP.
pub fn write_string(out: &mut String, s: &str) {
out.push('"');
for c in s.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
'\u{8}' => out.push_str("\\b"),
'\u{c}' => out.push_str("\\f"),
' '..='~' => out.push(c),
_ => {
let mut units = [0u16; 2];
for unit in c.encode_utf16(&mut units) {
let _ = write!(out, "\\u{unit:04x}");
}
}
}
}
out.push('"');
}
/// Python's `repr(float)`: the shortest digits that read back the same, in fixed notation when
/// the decimal point falls between 10^-4 and 10^16, with `.0` on whole numbers.
pub fn float_repr(f: f64) -> String {
if f.is_nan() {
return "NaN".into();
}
if f.is_infinite() {
return if f > 0.0 { "Infinity" } else { "-Infinity" }.into();
}
// Rust's `{:e}` gives the same shortest round-trip digits: "d.ddde-x".
let scientific = format!("{:e}", f.abs());
let (mantissa, exponent) = scientific.split_once('e').unwrap();
let digits: String = mantissa.chars().filter(|c| *c != '.').collect();
let exponent: i32 = exponent.parse().unwrap();
let point = exponent + 1; // The value is 0.digits * 10^point.
let sign = if f.is_sign_negative() { "-" } else { "" };
let n = digits.len() as i32;
if f == 0.0 {
return format!("{sign}0.0");
}
if -4 < point && point <= 16 {
if point <= 0 {
format!("{sign}0.{}{digits}", "0".repeat((-point) as usize))
} else if point >= n {
format!("{sign}{digits}{}.0", "0".repeat((point - n) as usize))
} else {
let (whole, fraction) = digits.split_at(point as usize);
format!("{sign}{whole}.{fraction}")
}
} else {
let (first, rest) = digits.split_at(1);
let mantissa = if rest.is_empty() {
first.to_string()
} else {
format!("{first}.{rest}")
};
let e = point - 1;
format!(
"{sign}{mantissa}e{}{:02}",
if e < 0 { '-' } else { '+' },
e.abs()
)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn floats_print_as_python_repr() {
for (f, repr) in [
(0.0, "0.0"),
(-0.0, "-0.0"),
(1.0, "1.0"),
(0.1, "0.1"),
(1.5e-5, "1.5e-05"),
(0.0001, "0.0001"),
(1e16, "1e+16"),
(1234567890123456.0, "1234567890123456.0"),
(12345678901234567.0, "1.2345678901234568e+16"),
(-2.5, "-2.5"),
(1e-310, "1e-310"),
(f64::MAX, "1.7976931348623157e+308"),
(100.0, "100.0"),
(0.30000000000000004, "0.30000000000000004"),
] {
assert_eq!(float_repr(f), repr, "{f}");
}
}
#[test]
fn strings_escape_as_ensure_ascii() {
let mut out = String::new();
write_string(&mut out, "a\"\\\n\u{7f}é\u{1F600}\u{1}");
assert_eq!(out, "\"a\\\"\\\\\\n\\u007f\\u00e9\\ud83d\\ude00\\u0001\"");
}
#[test]
fn objects_keep_first_place_and_last_value() {
let v = loads(r#"{"b": 1, "a": [1.0, -0, 1e2, true, null], "b": 2}"#).unwrap();
assert_eq!(
dumps(&v, &DEFAULT),
r#"{"b": 2, "a": [1.0, 0, 100.0, true, null]}"#
);
assert_eq!(
dumps(&v, &INDENT2),
"{\n \"b\": 2,\n \"a\": [\n 1.0,\n 0,\n 100.0,\n true,\n null\n ]\n}"
);
assert_eq!(
dumps(&loads("{\"x\":{},\"y\":[]}").unwrap(), &INDENT2),
"{\n \"x\": {},\n \"y\": []\n}"
);
}
}
+146
View File
@@ -0,0 +1,146 @@
//! Survival rules of the world generator's feature placement: each block's rule from the class
//! vanilla gives it and the class of its placement check (tools/test_block_survival_parity.py's
//! `bind_definitions` and its tables).
use crate::pyjson::Value;
const CLASS_RULES: &[(&str, &str)] = &[
("FireBlock", "Fire"),
("CactusBlock", "Cactus"),
("PathBlock", "Path"),
("MushroomBlock", "Mushroom"),
("MultifaceBlock", "Multiface"),
("VineBlock", "Vine"),
("HangingMossBlock", "HangingMoss"),
("SeaPickleBlock", "SeaPickle"),
("SnowLayerBlock", "SnowLayer"),
("BellBlock", "Bell"),
("ShelfMushroomBlock", "FullBehind"),
("GrowingPlantBlock", "GrowingPlant"),
("MangrovePropaguleBlock", "MangrovePropagule"),
("SpeleothemBlock", "Speleothem"),
("ScaffoldingBlock", "Scaffolding"),
("PistonHeadBlock", "PistonHead"),
("ChorusPlantBlock", "ChorusPlant"),
("ChorusFlowerBlock", "ChorusFlower"),
("PitcherCropBlock", "PitcherCrop"),
("BigDripleafBlock", "BigDripleaf"),
("BigDripleafStemBlock", "BigDripleafStem"),
("FarmlandBlock", "Farmland"),
("CocoaBlock", "Cocoa"),
("SugarCaneBlock", "SugarCane"),
("SporeBlossomBlock", "SporeBlossom"),
("FrogspawnBlock", "Frogspawn"),
("SmallDripleafBlock", "SmallDripleaf"),
("TallSeagrassBlock", "TallSeagrass"),
("DoorBlock", "Door"),
("BaseRailBlock", "RigidBelow"),
("LeafLitterBlock", "FullBelow"),
("WallSignBlock", "SolidBehind"),
("GrindstoneBlock", "Always"),
("RedstoneWallTorchBlock", "FullBehind"),
("BubbleColumnBlock", "BubbleColumn"),
("RedstoneWireBlock", "RedstoneWire"),
("SoulFireBlock", "TagBelow:minecraft:soul_fire_base_blocks"),
("BlockBehaviour", "Always"),
("CakeBlock", "SolidBelow"),
("CandleCakeBlock", "SolidBelow"),
("BannerBlock", "SolidBelow"),
("StandingSignBlock", "SolidBelow"),
("WallBannerBlock", "SolidBehind"),
("CarpetBlock", "NonAirBelow"),
("MossyCarpetBlock", "MossCarpet"),
("BaseTorchBlock", "CenterBelow"),
("CandleBlock", "CenterBelow"),
("LanternBlock", "Lantern"),
("HangingRootsBlock", "FullAbove"),
("LadderBlock", "FullBehind"),
("AmethystClusterBlock", "FullBehind"),
("TripWireHookBlock", "FullBehind"),
("BaseCoralWallFanBlock", "FullBehind"),
("WallTorchBlock", "FullBehind"),
("BasePressurePlateBlock", "RigidOrCenterBelow"),
("DiodeBlock", "RigidBelow"),
("BaseCoralPlantTypeBlock", "FullBelow"),
("CeilingHangingSignBlock", "CenterAbove"),
("FaceAttachedHorizontalDirectionalBlock", "FaceAttached"),
("BambooSaplingBlock", "SupportsBamboo"),
("BambooStalkBlock", "SupportsBamboo"),
];
const VEGETATION_RULES: &[(&str, &str)] = &[
("LilyPadBlock", "LilyPad"),
("VegetationBlock", "TagBelow:minecraft:supports_vegetation"),
("AzaleaBlock", "TagBelow:minecraft:supports_azalea"),
(
"DryVegetationBlock",
"TagBelow:minecraft:supports_dry_vegetation",
),
(
"NetherSproutsBlock",
"TagBelow:minecraft:supports_nether_sprouts",
),
("NetherWartBlock", "TagBelow:minecraft:supports_nether_wart"),
("WitherRoseBlock", "TagBelow:minecraft:supports_wither_rose"),
("CactusFlowerBlock", "CactusFlower"),
("SeagrassBlock", "Seagrass"),
];
fn lookup(table: &[(&str, &'static str)], key: &str) -> Option<&'static str> {
table.iter().find(|(k, _)| *k == key).map(|(_, v)| *v)
}
/// `{block: rule}` from the exporter's `{block: class}`, `{block: placement class}` and
/// `{block: {field: support tag}}`, in the order Python's dict built it.
pub fn bind_definitions(
classes: &Value,
placement: &Value,
support_tags: &Value,
) -> Result<Value, String> {
let classes = classes
.as_object()
.ok_or("survival-classes.json is not an object")?;
let mut definitions = Value::Object(Vec::new());
for (name, owner) in classes {
if let Some(rule) = owner.as_str().and_then(|o| lookup(CLASS_RULES, o)) {
definitions.insert(name, Value::str(rule));
}
}
for (name, owner) in classes {
let owner = owner.as_str().unwrap_or("");
let placed = placement.get(name).and_then(Value::as_str);
if owner == "CropBlock" && placed == Some("CropBlock") {
definitions.insert(name, Value::str("Crop"));
}
if owner == "DoublePlantBlock" && placed == Some("VegetationBlock") {
definitions.insert(name, Value::str("DoublePlant"));
}
if owner != "VegetationBlock" {
continue;
}
let method = placed.ok_or_else(|| format!("No placement class for {name}"))?;
if let Some(rule) = lookup(VEGETATION_RULES, method) {
definitions.insert(name, Value::str(rule));
} else if [
"AttachedStemBlock",
"StemBlock",
"NetherFungusBlock",
"NetherRootsBlock",
]
.contains(&method)
{
let field = if method == "StemBlock" {
"stemSupportBlocks"
} else {
"supportBlocks"
};
let tag = support_tags
.get(name)
.and_then(|t| t.get(field))
.and_then(Value::as_str)
.ok_or_else(|| format!("No {field} support tag for {name}"))?;
definitions.insert(name, Value::Str(format!("TagBelow:{tag}")));
}
}
Ok(definitions)
}
+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
"""Check that horizon-prepare (Rust) writes what the Python preparer wrote.
Prepares the same jars three times in fresh folders made from the same tree (research-lock.json
and the vanilla exporters of --python-ref): twice with that ref's tools/prepare_native_client.py
and once with horizon-prepare. Every file under .local/ is compared by sha256:
- identical: the same bytes in all three runs;
- exporter noise: differs between the two Python runs too, so the vanilla Java exporter itself
writes it differently each run (lambda addresses, hash-set order), not the preparer;
- port differences: identical between the Python runs but not in the Rust run. There must be none.
The manifest (horizon-resources.json) is compared field by field: only the recorded hashes of
exporter-noise files may differ, and each must be the hash of its own run's file.
Then each preparer runs again over the other's output (Rust over a Python tree, Python over the
Rust tree): the manifests are read the same way, so nothing is exported again and no resource
changes. Development tool: it needs Python 3.11+, Java and git. Report: <work>/report.json.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
# The last commit whose tools/prepare_native_client.py is the Python preparer itself.
PYTHON_REF = '66ce394d2d6dc19ed5f5c53a97b650741af9be76'
# Written by tools the comparison is not about: the decompiler only lab.py fetches, logs with
# times in them, compiled classes, and the data generator's own dated cache.
IGNORED_PARTS = {'classes', 'client-classes', 'logs', '.cache'}
def tree(ref, destination):
destination.mkdir(parents=True)
archive = subprocess.run(['git', '-C', str(ROOT), 'archive', ref, 'tools', 'mods/horizon/src/test/java',
'research-lock.json'], check=True, capture_output=True).stdout
subprocess.run(['tar', '-x', '-C', str(destination)], input=archive, check=True)
def hashes(folder):
local = folder / '.local'
result = {}
for path in sorted(local.rglob('*')):
relative = path.relative_to(local)
if path.is_file() and not (set(relative.parts) & IGNORED_PARTS) and path.suffix != '.log':
result[relative.as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest()
return result
def run(name, command, cwd, log):
started = time.monotonic()
with log.open('w') as stream:
subprocess.run(command, cwd=cwd, check=True, stdout=stream, stderr=subprocess.STDOUT)
seconds = round(time.monotonic() - started, 1)
print(f'{name}: {seconds} s', flush=True)
return seconds
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--server-jar', type=Path, required=True)
parser.add_argument('--minecraft-dir', type=Path, required=True)
parser.add_argument('--python-ref', default=PYTHON_REF)
parser.add_argument('--binary', type=Path, help='horizon-prepare to test (default: cargo build --release)')
parser.add_argument('--work', type=Path, default=ROOT / '.local/compare-prepare')
parser.add_argument('--keep', action='store_true', help='keep the prepared folders')
args = parser.parse_args()
binary = args.binary
if binary is None:
subprocess.run(['cargo', 'build', '--release', '-p', 'horizon-prepare'], cwd=ROOT, check=True)
metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--no-deps', '--format-version', '1'],
cwd=ROOT, text=True))
binary = Path(metadata['target_directory']) / 'release' / (
'horizon-prepare.exe' if os.name == 'nt' else 'horizon-prepare')
binary = binary.resolve()
work = args.work.resolve()
shutil.rmtree(work, ignore_errors=True)
work.mkdir(parents=True)
game = ['--prepare', '--server-jar', str(args.server_jar.resolve()),
'--minecraft-dir', str(args.minecraft_dir.resolve())]
python = [sys.executable, '-u', 'tools/prepare_native_client.py', *game]
folders = {name: work / name for name in ['python-1', 'python-2', 'rust']}
times = {}
for name, folder in folders.items():
tree(args.python_ref, folder)
command = [str(binary), '--root', str(folder), *game] if name == 'rust' else python
times[name] = run(name, command, folder, work / f'{name}.log')
first, second, rust = (hashes(folders[n]) for n in ['python-1', 'python-2', 'rust'])
names = sorted(first.keys() | second.keys() | rust.keys())
identical = [n for n in names if first.get(n) == second.get(n) == rust.get(n) is not None]
noise = [n for n in names if first.get(n) != second.get(n)]
port = [n for n in names if n not in identical and n not in noise]
# Only lab.py fetches the decompiler; preparing does not need it.
port = [n for n in port if not (n.startswith('tools/vineflower-') and n not in rust)]
manifest = 'horizon-client/' + json.loads((folders['rust'] / 'research-lock.json').read_text())[
'minecraft']['version'] + '/horizon-resources.json'
noisy = {n.rsplit('/', 1)[-1] for n in noise if n.startswith(manifest.rsplit('/', 1)[0] + '/')}
manifests = {n: json.loads((folders[n] / '.local' / manifest).read_text()) for n in folders}
manifest_differences = []
def walk(python, candidate, path):
if isinstance(python, dict) and isinstance(candidate, dict) and list(python) == list(candidate):
for key in python:
walk(python[key], candidate[key], path + [key])
elif python != candidate:
name = path[-1]
expected = hashlib.sha256((folders['rust'] / '.local' / manifest).with_name(name).read_bytes()).hexdigest() \
if name in noisy else None
if not (name in noisy and candidate == expected):
manifest_differences.append('.'.join(path))
walk(manifests['python-1'], manifests['rust'], [])
if manifest in port and not manifest_differences:
port.remove(manifest)
# Each preparer over the other's output: nothing to export again, no resource changes.
resources = 'horizon-client/'
before = {'python-2': second, 'rust': rust}
times['rust over python'] = run('rust over python', [str(binary), '--root', str(folders['python-2']), *game],
folders['python-2'], work / 'rust-over-python.log')
times['python over rust'] = run('python over rust', python, folders['rust'], work / 'python-over-rust.log')
rerun_changes = {}
for name in before:
after = hashes(folders[name])
changed = [n for n in sorted(before[name].keys() | after.keys())
if n.startswith(resources) and before[name].get(n) != after.get(n)]
rerun_changes[name] = changed
exported_again = {name: [line.strip() for line in (work / log).read_text().splitlines() if 'Exporting' in line]
for name, log in [('rust over python', 'rust-over-python.log')]}
report = {'python_ref': args.python_ref, 'binary': str(binary), 'seconds': times,
'files': len(names), 'identical': len(identical), 'exporter_noise': noise,
'port_differences': port, 'manifest_differences': manifest_differences,
'rerun_resource_changes': rerun_changes, 'exported_again': exported_again}
(work / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps(report, indent=2))
if not args.keep:
for folder in folders.values():
shutil.rmtree(folder, ignore_errors=True)
ok = not port and not manifest_differences
print('identical output' if ok else 'DIFFERENT OUTPUT', flush=True)
return 0 if ok else 1
if __name__ == '__main__':
sys.exit(main())