diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4cba851fa..4d20941d4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -77,9 +77,39 @@ jobs: # A dry run builds whatever the decision, to test the builds. if [[ "${{ steps.mode.outputs.dry_run }}" == true ]]; then echo "release=true" >> "$GITHUB_OUTPUT"; fi - build: + exporters: needs: decide if: needs.decide.outputs.release == 'true' + # This runner holds the maintainer's own pinned Minecraft jars. Only our classes leave it. + runs-on: ${{ vars.MAC_RUNNER || 'ludovics-mac-mini' }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.decide.outputs.head }} + persist-credentials: false + clean: true + - uses: dtolnay/rust-toolchain@stable + - run: bash tools/ci/runner-env.sh + - run: bash tools/ci/vanilla-data.sh + - name: Compile and validate original exporters against the pinned jars + shell: bash + run: | + set -euo pipefail + bundle=$(target/release/horizon-prepare compile-exporters --root "$PWD" | tail -1) + mkdir -p exporters + cp -R "$bundle/server" "$bundle/client" "$bundle/manifest.json" exporters/ + python3 tools/check_prepare_exporters.py --binary target/release/horizon-prepare + - uses: actions/upload-artifact@v4 + with: + name: original-exporters + path: exporters/ + if-no-files-found: error + retention-days: 3 + + build: + needs: [decide, exporters] + if: needs.decide.outputs.release == 'true' name: build ${{ matrix.name }} strategy: fail-fast: false @@ -218,6 +248,10 @@ jobs: path: ${{ runner.temp }}/smoke-test.json if-no-files-found: error retention-days: 3 + - uses: actions/download-artifact@v4 + with: + name: original-exporters + path: exporters - name: Package run: | set -euo pipefail diff --git a/.gitignore b/.gitignore index 7fffd6a99..6bdae649a 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ /.horizon-cache/ /crash-reports/ /debug/ + +# Release-only compiled original exporter bundle. +/exporters/ diff --git a/README.md b/README.md index ceee29249..65078464e 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Horizon is Minecraft: Java Edition 26.3 rewritten in Rust: a native game that re Horizon is installed through **the Horizon launcher**. Get it from [playhorizon.dev](https://playhorizon.dev) (the Download button picks your system) or from [the launcher releases](https://github.com/ludovic111/horizon/releases?q=launcher-v&expanded=true), for Linux x86-64, macOS on Apple Silicon or Windows x64. The launcher installs and updates the game, prepares its files from your own copy of Minecraft, and updates itself. -**You need** Minecraft: Java Edition 26.3, played once (the official launcher, Prism, MultiMC and other launchers all work); a Java 25 JDK, for the one-time preparation; and a GPU with Vulkan on Linux and Windows, or any Apple Silicon Mac. Step by step, with macOS's first-open prompt and the first launch: [Install and first launch](https://playhorizon.dev/docs/install). +**You need** Minecraft: Java Edition 26.3, played once (the official launcher, Prism, MultiMC and other launchers all work); a Java 25 runtime, for the one-time preparation; and a GPU with Vulkan on Linux and Windows, or any Apple Silicon Mac. Step by step, with macOS's first-open prompt and the first launch: [Install and first launch](https://playhorizon.dev/docs/install). diff --git a/crates/horizon-prepare/src/exporters.rs b/crates/horizon-prepare/src/exporters.rs new file mode 100644 index 000000000..35992c8b5 --- /dev/null +++ b/crates/horizon-prepare/src/exporters.rs @@ -0,0 +1,218 @@ +//! Only original Horizon classes are packaged. The pinned jars remain player supplied. +use crate::*; + +const SETTINGS: &str = "exporters-v1 --release -proc:none -encoding UTF-8"; +const SERVER: &[&str] = &[ + "VanillaHeightmapOracle", + "VanillaBlockLandingOracle", + "VanillaEntityAttributeOracle", + "VanillaPlayerHealthOracle", + "VanillaDamageSourceOracle", + "VanillaCollisionOracle", + "VanillaFaceVisibilityOracle", + "VanillaGameplayFactsExporter", + "VanillaEntityTypesExporter", + "VanillaBlockBehaviourExporter", + "VanillaCommandParseOracle", + "VanillaCreativeTabsExporter", + "VanillaSimpleBlockOracle", + "VanillaStructureBlockExporter", + "PrepareWorker", +]; +const CLIENT: &[&str] = &[ + "VanillaBlockStateBindingOracle", + "VanillaEntityModelExporter", + "VanillaMobDataExporter", + "PrepareWorker", +]; + +fn files(dir: &Path) -> Result, String> { + let mut result = Vec::new(); + for entry in fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))? { + let path = entry.map_err(|e| e.to_string())?.path(); + if path.is_dir() { + result.extend(files(&path)?); + } else { + result.push(path); + } + } + result.sort(); + Ok(result) +} + +// research-lock pins both jars and every accepted client manifest (which pins its libraries). +// artifacts::verify/client_libraries check those dependencies before any bundle is used. +fn key(root: &Path) -> Result { + let mut hash = sha2::Sha256::new(); + hash.update(SETTINGS); + // The launcher's bundled preparer may differ from the selected game's. Hash that + // version's compiler settings so a newer release still selects a runtime correctly. + let compiler = fs::read_to_string(root.join("crates/horizon-prepare/src/exporters.rs")) + .unwrap_or_else(|_| include_str!("exporters.rs").into()); + hash.update(compiler.replace('\r', "")); + let mut sources = files(&root.join(PARITY))?; + sources.retain(|p| p.extension().is_some_and(|e| e == "java")); + sources.push(root.join("research-lock.json")); + for path in sources { + let data: Vec<_> = read(&path)?.into_iter().filter(|b| *b != b'\r').collect(); + hash.update(format!( + "{}\0{}\0", + relative_display(&path, root)?.replace('\\', "/"), + data.len() + )); + hash.update(data); + } + Ok(hex(&hash.finalize())) +} + +fn outputs(dir: &Path) -> Result { + let mut hashes = Value::Object(Vec::new()); + for path in files(dir)? { + if path.extension().is_some_and(|e| e == "class") { + let name = relative_display(&path, dir)?.replace('\\', "/"); + // javac's sourcepath contains our sources only; enforce that at packaging too. + if !name.starts_with("server/dev/horizonlab/parity/") + && !name.starts_with("client/dev/horizonlab/parity/") + { + return Err(format!("Non-Horizon class in exporter bundle: {name}")); + } + hashes.insert(&name, Value::Str(digest(&path, "sha256")?)); + } + } + Ok(hashes) +} + +fn valid(dir: &Path, key: &str) -> bool { + let Ok(manifest) = read_json(&dir.join("manifest.json")) else { + return false; + }; + manifest.get("key").and_then(Value::as_str) == Some(key) + && outputs(dir).is_ok_and(|hashes| { + !hashes.as_object().unwrap().is_empty() && manifest.get("classes") == Some(&hashes) + }) +} + +/// Whether the version ships a matching, intact bundle and needs only a Java runtime. +pub fn available(root: &Path) -> bool { + key(root).is_ok_and(|key| valid(&root.join("exporters"), &key)) +} + +pub(crate) fn classes(run: &Run) -> Result { + let key = key(&run.root)?; + let shipped = run.root.join("exporters"); + if valid(&shipped, &key) { + return Ok(shipped); + } + let cached = run.local.join("exporters").join(&key); + if valid(&cached, &key) { + return Ok(cached); + } + // Build privately: an interrupted compiler or another preparer cannot publish partial classes. + let destination = cached; + let cached = run + .local + .join("exporters") + .join(format!(".{key}-{}", std::process::id())); + if cached.exists() { + fs::remove_dir_all(&cached).map_err(|e| e.to_string())?; + } + mkdir(&cached)?; + let jdk = java::find(run.java_major, &[run.jdk.home()])?; + for (side, mains, cp) in [ + ("server", SERVER, &run.server_classpath), + ("client", CLIENT, &run.client_classpath), + ] { + (run.log)(&format!("Compiling {side} exporters (shared helpers once)")); + let classes = cached.join(side); + mkdir(&classes)?; + let log = run.log_file(&cached.join(format!("{side}.log")))?; + let mut args = vec![ + "-encoding".into(), + "UTF-8".into(), + "-cp".into(), + cp.clone(), + "-sourcepath".into(), + arg(&run.root.join("mods/horizon/src/test/java")), + "-d".into(), + arg(&classes), + ]; + args.extend(mains.iter().map(|n| arg(&run.oracle(&format!("{n}.java"))))); + jdk.javac(run.java_major, &args, &log)?; + let validate = vec![ + "-Xverify:all".into(), + "-cp".into(), + format!( + "{}{}{}", + classes.display(), + if cfg!(windows) { ";" } else { ":" }, + cp + ), + "dev.horizonlab.parity.PrepareWorker".into(), + "validate".into(), + arg(&classes), + ]; + // No exporter executes here: resolve every class and its signatures against pinned jars. + run.jdk.java(&validate, None, &log)?; + } + let manifest = Value::Object(vec![ + ("key".into(), Value::Str(key.clone())), + ("classes".into(), outputs(&cached)?), + ]); + write_json(&cached.join("manifest.json"), &manifest)?; + if valid(&destination, &key) { + fs::remove_dir_all(&cached).map_err(|e| e.to_string())?; + } else { + if destination.exists() { + fs::remove_dir_all(&destination).map_err(|e| e.to_string())?; + } + fs::rename(&cached, &destination).map_err(|e| e.to_string())?; + } + Ok(destination) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bundles_invalidate_sources_pins_and_damaged_classes() { + let root = std::env::temp_dir().join(format!("prepare-bundle-{}", std::process::id())); + let source = root.join(PARITY).join("VanillaHeightmapOracle.java"); + mkdir(source.parent().unwrap()).unwrap(); + write(&source, b"original\n").unwrap(); + let compiler = root.join("crates/horizon-prepare/src/exporters.rs"); + mkdir(compiler.parent().unwrap()).unwrap(); + write(&compiler, b"--release 25 -proc:none").unwrap(); + write(&root.join("research-lock.json"), b"pins").unwrap(); + let original = key(&root).unwrap(); + write(&compiler, b"--release 26 -proc:none").unwrap(); + assert_ne!(key(&root).unwrap(), original); + write(&compiler, b"--release 25 -proc:none").unwrap(); + write(&source, b"original\r\n").unwrap(); + assert_eq!(key(&root).unwrap(), original); + write(&source, b"changed").unwrap(); + assert_ne!(key(&root).unwrap(), original); + write(&source, b"original\n").unwrap(); + write(&root.join("research-lock.json"), b"new dependencies").unwrap(); + assert_ne!(key(&root).unwrap(), original); + write(&root.join("research-lock.json"), b"pins").unwrap(); + let bundle = root.join("exporters"); + let class = bundle.join("server/dev/horizonlab/parity/Helper.class"); + mkdir(class.parent().unwrap()).unwrap(); + write(&class, b"compiled").unwrap(); + write_json( + &bundle.join("manifest.json"), + &Value::Object(vec![ + ("key".into(), Value::Str(original)), + ("classes".into(), outputs(&bundle).unwrap()), + ]), + ) + .unwrap(); + assert!(available(&root)); + write(&class, b"damaged").unwrap(); + assert!(!available(&root)); + fs::remove_file(&class).unwrap(); + assert!(!available(&root)); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/crates/horizon-prepare/src/java.rs b/crates/horizon-prepare/src/java.rs index d46cb9428..6343d8433 100644 --- a/crates/horizon-prepare/src/java.rs +++ b/crates/horizon-prepare/src/java.rs @@ -1,4 +1,4 @@ -//! The JDK that compiles and runs the vanilla exporters. They call Mojang's own code, so they +//! Java discovery and execution for the vanilla exporters. They call Mojang's own code, so they //! stay Java; everything around them is here. use std::ffi::OsStr; @@ -15,7 +15,7 @@ pub struct Jdk { const TIMEOUT: Duration = Duration::from_secs(3600); /// Exporters and JDK probes write to pipes or log files: on Windows they open no terminal. -fn command(program: impl AsRef) -> Command { +pub(crate) fn command(program: impl AsRef) -> Command { let command = Command::new(program); #[cfg(windows)] let command = { @@ -56,9 +56,9 @@ fn without_verbatim(path: PathBuf) -> PathBuf { /// Resolve compiler symlinks before picking its sibling runtime. The first `java` on the /// PATH may be an older, runtime-only install; it must not be paired with a newer `javac`. -fn path_jdks(path: &OsStr) -> Vec { +fn path_homes(path: &OsStr, tool: &str) -> Vec { std::env::split_paths(path) - .map(|dir| executable(&dir, "javac")) + .map(|dir| executable(&dir, tool)) .filter(|javac| javac.is_file()) .map(real_path) .filter_map(|javac| { @@ -162,7 +162,7 @@ fn jdk_parents() -> Vec { /// The JDKs (folders with `bin/javac`) one level below `parents`; a macOS bundle's home is /// its `Contents/Home`. Those named for `major` come first, then the others newest name first. -fn installed_jdks(parents: &[PathBuf], major: i64) -> Vec { +fn installed_jdks(parents: &[PathBuf], major: i64, tool: &str) -> Vec { let mut homes: Vec = parents .iter() .filter_map(|parent| fs::read_dir(parent).ok()) @@ -176,7 +176,7 @@ fn installed_jdks(parents: &[PathBuf], major: i64) -> Vec { entry.path() } }) - .filter(|home| executable(&home.join("bin"), "javac").is_file()) + .filter(|home| executable(&home.join("bin"), tool).is_file()) .collect(); let named = |home: &Path| { home.iter().any(|part| { @@ -191,8 +191,7 @@ fn installed_jdks(parents: &[PathBuf], major: i64) -> Vec { } /// The Java runtimes the official launcher downloaded into `runtime` (a game directory's -/// `runtime` folder: `///`, a macOS one in `jre.bundle`). Mojang's -/// runtimes are whole JDKs, javac included, so a player who has run Minecraft has one already. +/// `runtime` folder: `///`, a macOS one in `jre.bundle`). pub fn minecraft_runtimes(runtime: &Path) -> Vec { let children = |dir: &Path| -> Vec { let mut paths: Vec = fs::read_dir(dir) @@ -206,7 +205,7 @@ pub fn minecraft_runtimes(runtime: &Path) -> Vec { for platform in children(&component) { let home = platform.join(component.file_name().unwrap_or_default()); for home in [home.join("jre.bundle/Contents/Home"), home] { - if executable(&home.join("bin"), "javac").is_file() { + if executable(&home.join("bin"), "java").is_file() { homes.push(home); break; } @@ -221,8 +220,18 @@ pub fn minecraft_runtimes(runtime: &Path) -> Vec { /// `/usr/libexec/java_home -v ` reports, else the homes in `more` (the launcher's pick, /// Minecraft's own runtimes), else the folders installers use (`jdk_parents`). pub fn find(major: i64, more: &[PathBuf]) -> Result { + find_tools(major, more, true) +} + +/// Finds a compatible runtime; precompiled exporters do not need javac. +pub fn find_runtime(major: i64, more: &[PathBuf]) -> Result { + find_tools(major, more, false) +} + +fn find_tools(major: i64, more: &[PathBuf], compiler: bool) -> Result { + let tool = if compiler { "javac" } else { "java" }; let need = format!( - "Java {major} is needed (a JDK with javac, e.g. Eclipse Temurin {major}). \ + "Java {major} is needed ({tool}, e.g. Eclipse Temurin {major}). \ Install it, or set JAVA_HOME to it, and try again." ); let mut candidates = Vec::new(); @@ -235,7 +244,7 @@ pub fn find(major: i64, more: &[PathBuf]) -> Result { )); } if let Some(path) = std::env::var_os("PATH") { - for home in path_jdks(&path) { + for home in path_homes(&path, tool) { let bin = home.join("bin"); candidates.push((executable(&bin, "javac"), executable(&bin, "java"), "PATH")); } @@ -265,7 +274,7 @@ pub fn find(major: i64, more: &[PathBuf]) -> Result { "Minecraft's Java", )); } - for home in installed_jdks(&jdk_parents(), major) { + for home in installed_jdks(&jdk_parents(), major, tool) { let bin = home.join("bin"); candidates.push(( executable(&bin, "javac"), @@ -275,15 +284,16 @@ pub fn find(major: i64, more: &[PathBuf]) -> Result { } let mut problems = Vec::new(); for (javac, java, from) in candidates { - if !javac.is_file() || !java.is_file() { + if (compiler && !javac.is_file()) || !java.is_file() { problems.push(format!( - "{from} ({}) has no javac and java", + "{from} ({}) has no compatible Java tools", javac.parent().unwrap().display() )); continue; } let checked = [(&javac, "javac"), (&java, "java")] .into_iter() + .filter(|(_, name)| compiler || *name == "java") .try_for_each(|(program, name)| { let found = tool_major(program, name)?; if i64::from(found) >= major { @@ -298,7 +308,7 @@ pub fn find(major: i64, more: &[PathBuf]) -> Result { } } if problems.is_empty() { - Err(format!("{need} No javac was found.")) + Err(format!("{need} No {tool} was found.")) } else { Err(format!("{need} Found: {}.", problems.join("; "))) } @@ -317,7 +327,7 @@ pub fn classpath<'a>(paths: impl IntoIterator) -> String { impl Jdk { /// The JDK's home, the folder above `bin` (what `JAVA_HOME` names). pub fn home(&self) -> PathBuf { - self.javac + self.java .parent() .and_then(Path::parent) .map(Path::to_path_buf) @@ -424,6 +434,7 @@ mod tests { dir.join("missing"), ], 25, + "javac", ); assert_eq!( found, @@ -445,7 +456,7 @@ mod tests { let make = |home: PathBuf, javac: bool| { fs::create_dir_all(home.join("bin")).unwrap(); if javac { - fs::write(executable(&home.join("bin"), "javac"), b"").unwrap(); + fs::write(executable(&home.join("bin"), "java"), b"").unwrap(); } home }; diff --git a/crates/horizon-prepare/src/lib.rs b/crates/horizon-prepare/src/lib.rs index c7e443c7d..09787e476 100644 --- a/crates/horizon-prepare/src/lib.rs +++ b/crates/horizon-prepare/src/lib.rs @@ -11,11 +11,13 @@ //! run, keyed by the sha256 of their inputs and outputs recorded in the manifest, as before. pub mod artifacts; +pub mod exporters; mod inventory; pub mod java; pub mod pyjson; mod sounds; mod survival; +mod worker; pub use sounds::{fonts, languages, packs, panorama, sounds, BUILT_IN_PACKS}; @@ -65,7 +67,7 @@ pub fn sources_sha256(crate_dir: &Path) -> Option { pub struct Options { /// The version folder (a release folder, or a checkout): research-lock.json, the exporters' - /// Java sources, and `.local/` where everything is written. + /// Java sources, optional precompiled `exporters/`, and `.local/` where everything is written. pub root: PathBuf, /// Import the pinned game files from the player's own copy first (`--prepare`). pub import: bool, @@ -74,7 +76,7 @@ pub struct Options { pub minecraft_dir: Option, /// The pinned server jar the player downloaded (`--server-jar`, else HORIZON_SERVER_JAR). pub server_jar: Option, - /// The JDK to try after JAVA_HOME and the PATH (the launcher's pick, e.g. Minecraft's + /// The Java runtime to try after JAVA_HOME and the PATH (the launcher's pick, e.g. Minecraft's /// own Java it downloaded from Mojang). pub java_home: Option, } @@ -293,6 +295,8 @@ struct Run<'a> { server_classpath: String, /// The client jar and its libraries: the classpath of the client-side exporters. client_classpath: String, + bundles: std::cell::OnceCell, + workers: std::cell::RefCell<[Option; 2]>, previous: Value, origin: Value, log: Log<'a>, @@ -301,6 +305,13 @@ struct Run<'a> { const PARITY: &str = "mods/horizon/src/test/java/dev/horizonlab/parity"; impl Run<'_> { + fn bundle(&self) -> Result<&Path, String> { + if self.bundles.get().is_none() { + self.bundles.set(exporters::classes(self)?).unwrap(); + } + Ok(self.bundles.get().unwrap()) + } + fn oracle(&self, name: &str) -> PathBuf { self.root.join(PARITY).join(name) } @@ -333,33 +344,31 @@ impl Run<'_> { fs::File::create(path).map_err(|e| format!("{}: {e}", path.display())) } - /// `javac --release -proc:none -cp [extra] -d `. - fn compile( - &self, - classpath: &str, - extra: &[String], - classes: &Path, - sources: &[&Path], - log: &fs::File, - ) -> Result<(), String> { - let mut args = vec!["-cp".to_string(), classpath.to_string()]; - args.extend_from_slice(extra); - args.push("-d".into()); - args.push(classes.display().to_string()); - args.extend(sources.iter().map(|s| s.display().to_string())); - self.jdk.javac(self.java_major, &args, log) - } - /// `java -Xmx -cp : dev.horizonlab.parity.
`. fn run( &self, heap: &str, - classes: &Path, classpath: &str, main: &str, args: &[String], log: &fs::File, + log_path: &Path, ) -> Result<(), String> { + let side = usize::from(classpath == self.client_classpath); + let classes = self + .bundle()? + .join(if side == 1 { "client" } else { "server" }); + if !matches!( + main, + "VanillaCommandParseOracle" | "VanillaCreativeTabsExporter" + ) && std::env::var_os("HORIZON_PREPARE_ISOLATED").is_none() + { + let mut workers = self.workers.borrow_mut(); + if workers[side].is_none() { + workers[side] = Some(worker::Worker::new(self, &classes, classpath)?); + } + return workers[side].as_mut().unwrap().run(main, args, log_path); + } let separator = if cfg!(windows) { ";" } else { ":" }; let mut all = vec![ format!("-Xmx{heap}"), @@ -371,7 +380,7 @@ impl Run<'_> { self.jdk.java(&all, None, log) } - /// Compiles `sources` into `classes` and runs each of `mains` with `args`, their output in + /// Runs each exporter from the verified shared bundle, with output in /// the log at `log`: the shape of every exporter step. #[allow(clippy::too_many_arguments)] fn export( @@ -379,7 +388,7 @@ impl Run<'_> { what: &str, classpath: &str, classes: &Path, - sources: &[&Path], + _sources: &[&Path], mains: &[&str], heap: &str, args: &[String], @@ -389,10 +398,8 @@ impl Run<'_> { mkdir(classes)?; let file = self.log_file(log)?; let fail = |e: String| format!("{e} while exporting {what}; see {}", log.display()); - self.compile(classpath, &[], classes, sources, &file) - .map_err(fail)?; for main in mains { - self.run(heap, classes, classpath, main, args, &file) + self.run(heap, classpath, main, args, &file, log) .map_err(fail)?; } Ok(()) @@ -401,6 +408,15 @@ impl Run<'_> { /// Prepares the resources; returns their folder (`.local/horizon-client/`). pub fn prepare(options: &Options, log: Log) -> Result { + prepare_impl(options, log, false) +} + +/// Release build tool: compile and link-check only Horizon's original exporter classes. +pub fn compile_exporters(options: &Options, log: Log) -> Result { + prepare_impl(options, log, true) +} + +fn prepare_impl(options: &Options, log: Log, compile_only: bool) -> Result { let root = &std::path::absolute(&options.root) .map_err(|e| format!("{}: {e}", options.root.display()))?; let lock = artifacts::Lock::read(root)?; @@ -409,7 +425,7 @@ pub fn prepare(options: &Options, log: Log) -> Result { homes.extend(java::minecraft_runtimes( &artifacts::minecraft_dir(options.minecraft_dir.as_deref()).join("runtime"), )); - let jdk = java::find(lock.java_major, &homes)?; + let jdk = java::find_runtime(lock.java_major, &homes)?; if options.import && !artifacts::reuse_imported(root, log)? { artifacts::import( root, @@ -456,10 +472,15 @@ pub fn prepare(options: &Options, log: Log) -> Result { client: found.jars["client"].clone(), server_classpath: classpath_join(&found.jars["server"], &found.libraries), client_classpath: classpath_join(&found.jars["client"], &client_libraries), + bundles: std::cell::OnceCell::new(), + workers: std::cell::RefCell::new([None, None]), previous, origin, log, }; + if compile_only { + return Ok(run.bundle()?.to_path_buf()); + } steps(&run, &lock, &oracle, options.minecraft_dir.as_deref())?; log(&format!( "Prepared Horizon native-client resources: {}", @@ -1117,7 +1138,6 @@ fn prepare_worldgen(run: &Run) -> Result<(), String> { /// whose blocks are checked for survival or support (trees, flowers, grass, kelp, ...). fn export_block_behavior(run: &Run, export: &Path) -> Result<(), String> { (run.log)("Exporting block behavior (VanillaSimpleBlockOracle)"); - let tests = run.root.join("mods/horizon/src/test/java"); let classes = export.join("classes"); mkdir(&classes)?; let log_path = export.join("export.log"); @@ -1128,22 +1148,13 @@ fn export_block_behavior(run: &Run, export: &Path) -> Result<(), String> { log_path.display() ) }; - // The oracle's helper classes are compiled on demand from the source path. - run.compile( - &run.server_classpath, - &["-sourcepath".into(), arg(&tests)], - &classes, - &[&tests.join("dev/horizonlab/parity/VanillaSimpleBlockOracle.java")], - &log, - ) - .map_err(fail)?; run.run( "1G", - &classes, &run.server_classpath, "VanillaSimpleBlockOracle", &["export".into(), arg(export)], &log, + &log_path, ) .map_err(fail)?; let rules = survival::bind_definitions( diff --git a/crates/horizon-prepare/src/main.rs b/crates/horizon-prepare/src/main.rs index db64ca19d..10e153a04 100644 --- a/crates/horizon-prepare/src/main.rs +++ b/crates/horizon-prepare/src/main.rs @@ -16,6 +16,7 @@ const HELP: &str = "\ horizon-prepare: prepare Horizon's resources from your own Minecraft Java Edition Usage: horizon-prepare [--prepare] [--server-jar PATH] [--minecraft-dir PATH] [--root PATH] + horizon-prepare compile-exporters [--root PATH] horizon-prepare sounds [--minecraft-dir PATH] [--root PATH] horizon-prepare panorama [--minecraft-dir PATH] [--root PATH] horizon-prepare fonts [--minecraft-dir PATH] [--root PATH] @@ -30,8 +31,9 @@ Usage: horizon-prepare [--prepare] [--server-jar PATH] [--minecraft-dir PATH] [- --root PATH The Horizon folder to prepare (default: the folder this program is in, else the nearest folder above the current one with research-lock.json) -Needs Minecraft played once from the official launcher, and Java (a JDK) of the version -research-lock.json pins: JAVA_HOME, else javac and java on the PATH. +Needs Minecraft played once from the official launcher, and Java of the version +research-lock.json pins. Releases use a runtime; development or changed exporters need a JDK. +compile-exporters builds and validates a release bundle from the already imported pinned jars. The resources go to /.local/horizon-client/; pass that to horizon-client --resources. horizon-prepare panorama repairs the classic menu background in existing resources without Java. @@ -73,7 +75,7 @@ fn main() -> ExitCode { let assets = args.next_if(|a| { matches!( a.to_str(), - Some("sounds" | "panorama" | "fonts" | "packs" | "languages") + Some("sounds" | "panorama" | "fonts" | "packs" | "languages" | "compile-exporters") ) }); while let Some(arg) = args.next() { @@ -132,6 +134,8 @@ fn main() -> ExitCode { let _ = out.flush(); }; let result = match assets.as_deref().and_then(|a| a.to_str()) { + Some("compile-exporters") => horizon_prepare::compile_exporters(&options, &log) + .inspect(|dir| log(&dir.display().to_string())), Some("sounds") => { horizon_prepare::sounds(&options.root, options.minecraft_dir.as_deref(), &log) } diff --git a/crates/horizon-prepare/src/worker.rs b/crates/horizon-prepare/src/worker.rs new file mode 100644 index 000000000..564b11746 --- /dev/null +++ b/crates/horizon-prepare/src/worker.rs @@ -0,0 +1,79 @@ +//! A bounded request/reply channel to one initialized exporter JVM. +use crate::*; +use std::io::{BufRead, BufReader, Write}; +use std::process::{Child, ChildStdin, Stdio}; +use std::sync::mpsc::{self, Receiver}; + +pub(crate) struct Worker { + child: Child, + input: ChildStdin, + replies: Receiver>, +} + +impl Worker { + pub(crate) fn new(run: &Run, classes: &Path, cp: &str) -> Result { + let log = run.log_file(&run.local.join(format!( + "{}-worker.log", + classes.file_name().unwrap().to_string_lossy() + )))?; + let mut child = java::command(&run.jdk.java) + .args([ + "-Xmx2G", + "-cp", + &format!( + "{}{}{}", + classes.display(), + if cfg!(windows) { ";" } else { ":" }, + cp + ), + "dev.horizonlab.parity.PrepareWorker", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(log) + .spawn() + .map_err(|e| e.to_string())?; + let input = child.stdin.take().unwrap(); + let output = child.stdout.take().unwrap(); + let (send, replies) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + if send.send(line).is_err() { + break; + } + } + }); + Ok(Self { + child, + input, + replies, + }) + } + + pub(crate) fn run(&mut self, main: &str, args: &[String], log: &Path) -> Result<(), String> { + let task = Value::Array(vec![ + Value::Str(arg(log)), + Value::Str(main.into()), + Value::Array(args.iter().cloned().map(Value::Str).collect()), + ]); + writeln!(self.input, "{}", pyjson::dumps(&task, &pyjson::COMPACT)) + .map_err(|e| e.to_string())?; + self.input.flush().map_err(|e| e.to_string())?; + let reply = self + .replies + .recv_timeout(std::time::Duration::from_secs(3600)) + .map_err(|e| format!("Exporter JVM stopped or timed out: {e}"))? + .map_err(|e| e.to_string())?; + match pyjson::loads(&reply)?.as_str() { + Some("") => Ok(()), + _ => Err(format!("Exporter {main} failed: {reply}")), + } + } +} + +impl Drop for Worker { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/crates/horizon-prepare/tests/java_selection.rs b/crates/horizon-prepare/tests/java_selection.rs index ba15304e6..4e8d60f55 100644 --- a/crates/horizon-prepare/tests/java_selection.rs +++ b/crates/horizon-prepare/tests/java_selection.rs @@ -25,13 +25,23 @@ fn selects_a_compatible_compiler_and_runtime() { assert!(error.contains(&format!("java {runtime} (")), "{error}"); return; } - let jdk = java::find(25, &[]).unwrap(); - let expected = root.join(if scenario == "target-release" { + let jdk = if scenario == "runtime-only" { + java::find_runtime(25, &[]) + } else { + java::find(25, &[]) + } + .unwrap(); + let expected = root.join(if scenario == "runtime-only" { + "jre25" + } else if scenario == "target-release" { "newer" } else { "jdk25" }); for (actual, name) in [(&jdk.javac, "javac"), (&jdk.java, "java")] { + if scenario == "runtime-only" && name == "javac" { + continue; + } assert_eq!( fs::canonicalize(actual).unwrap(), fs::canonicalize(tool(&expected.join("bin"), name)).unwrap() @@ -64,6 +74,7 @@ fn selects_a_compatible_compiler_and_runtime() { ); for (home, compiler, runtime) in [ ("jre21", None, "21.0.1"), + ("jre25", None, "25.0.1"), ("jdk21", Some("21.0.1"), "21.0.1"), ("jdk25", Some("25.0.1"), "25.0.1"), ("mixed", Some("25.0.1"), "21.0.1"), @@ -83,6 +94,7 @@ fn selects_a_compatible_compiler_and_runtime() { } fs::write(root.join("failing/bin/java.exit"), "7").unwrap(); let scenarios = vec![ + ("runtime-only", Some("jre25"), vec!["jre21", "jre25"]), ("older-runtime", None, vec!["jre21", "jdk25"]), ("older-jdk", None, vec!["jdk21", "jdk25"]), ("incompatible-home", Some("mixed"), vec!["jdk25"]), diff --git a/docs/hosting.md b/docs/hosting.md index bf431cada..778dcbaab 100644 --- a/docs/hosting.md +++ b/docs/hosting.md @@ -71,7 +71,7 @@ from there too. ## 2. Prepare the game files Horizon reads Minecraft's data from your own copy of the game. You need Minecraft Java Edition 26.3, played -once from the official launcher on this computer, Java 25 (a JDK), and the 26.3 `server.jar`: it's the +once from the official launcher on this computer, a Java 25 runtime, and the 26.3 `server.jar`: it's the `downloads.server.url` in your launcher's `versions/26.3/26.3.json`. Then: ```sh diff --git a/docs/launcher-preparation.md b/docs/launcher-preparation.md new file mode 100644 index 000000000..7df5b9131 --- /dev/null +++ b/docs/launcher-preparation.md @@ -0,0 +1,53 @@ +# Launcher resource preparation + +Release folders include `exporters/server`, `exporters/client` and an integrity manifest. +These contain only Horizon's original Java exporter classes. Minecraft jars, libraries, +assets and generated resources remain local and are verified against `research-lock.json`. +The same platform-independent class bundle is used in game and dedicated-server releases. + +The release runner compiles the server and client source sets separately with +`javac --release -proc:none -encoding UTF-8`, resolving shared helpers once +per classpath. It checks every class and its signatures with `-Xverify:all` against the +pinned jars. The release job also compares two fresh isolated preparations with a fresh +batched preparation using a runtime-only Java home: + +```sh +cargo build --release -p horizon-prepare +# Requires already imported Minecraft files in .local/minecraft/. +target/release/horizon-prepare compile-exporters +# Copy server/, client/ and manifest.json from the printed folder into exporters/. +python3 tools/check_prepare_exporters.py --binary target/release/horizon-prepare +``` + +Development checkouts and changed sources compile on demand, into +`.local/exporters/`. The key covers Java sources (including implicit helpers), +pinned dependency checksums and compiler settings. Class checksums detect missing or damaged +outputs. Compilation happens in a private staging folder before publishing a complete cache +entry. Sources and paths are normalized so one release bundle works on all three platforms. + +Compatible exporters run sequentially through one server JVM and one client JVM. Both keep +Minecraft's initialized built-in registries. Every task restores the original registry tags +before replying; failures stop preparation. Collision exports share the first generated +registry and collision files, encoded block states and shape grids within the server worker. +Ordinary parity oracle processes keep their existing behavior. + +Command-registry and creative-tab exporters remain separate processes: they install data +components as well as tags. The data generator also remains separate. Inspection of pinned +`net.minecraft.data.Main.main` showed that it calls `Util.shutdownExecutors` after running +its providers. Calling it inside a worker would shut down shared executors; incorporating +its providers directly needs a separate comparison of registry and executor lifetimes. +Fresh release preparation therefore starts five exporter/data-generator JVMs, down from 18, +and invokes no compiler. `HORIZON_PREPARE_ISOLATED=1` retains separate exporter JVMs for +output comparisons and diagnosis. + +`tools/check_prepare_exporters.py` requires identical resource file sets and identical bytes +for every file stable across the two isolated runs. For the three known noisy JSON files, +it also requires semantic equality after normalizing object key order, opaque JVM lambda +addresses and creative-tab tag set order. Other variation fails the release check. + +On Linux with Java 25, two comparisons produced 23,204 matching resource files each. +The first isolated runs took 178.95 s and 104.25 s, versus 45.05 s batched; the second took +100.03 s and 176.06 s, versus 120.09 s batched. All used precompiled exporters. These are fresh output directories with a warm OS file cache, on a +machine with other workloads; the spread between isolated runs prevents treating this as a +controlled speedup estimate. The compiler saving is separate: release preparation removes +the previously measured 15 compiler invocations (29–52 s). diff --git a/launcher/src/backend.rs b/launcher/src/backend.rs index 7e5c4babb..dfe264300 100644 --- a/launcher/src/backend.rs +++ b/launcher/src/backend.rs @@ -1096,17 +1096,23 @@ fn download_for_owner( } } -/// The home of a JDK whose compiler and runtime support `major`. Forward even an installed -/// JDK to external preparers: older versions may otherwise rediscover incompatible PATH tools. +/// The Java home for this version: a runtime for precompiled exporters, a JDK for older +/// preparers or changed sources. Forward the selection to external preparers too. /// Else try the official launcher's Java in `games`, then download the runtime named by /// `manifest` into Horizon's game folder (Mojang's runtimes are whole JDKs). fn java( major: i64, + compiler: bool, games: &[PathBuf], manifest: &Path, log: &dyn Fn(String), ) -> Result { - if let Ok(jdk) = horizon_prepare::java::find(major, &[]) { + let find = if compiler { + horizon_prepare::java::find + } else { + horizon_prepare::java::find_runtime + }; + if let Ok(jdk) = find(major, &[]) { return Ok(jdk.home()); } let mut runtimes: Vec = Vec::new(); @@ -1118,7 +1124,7 @@ fn java( runtimes.push(home); } } - let missing = match horizon_prepare::java::find(major, &runtimes) { + let missing = match find(major, &runtimes) { Ok(jdk) => return Ok(jdk.home()), Err(e) => e, }; @@ -1129,7 +1135,7 @@ fn java( // musl distributions) it cannot start, so don't download it. if cfg!(all(target_os = "linux", target_arch = "x86_64")) && !Path::new(GLIBC_LOADER).exists() { return Err(format!( - "{missing}\n • Minecraft's own Java from Mojang needs glibc ({GLIBC_LOADER} is missing; on NixOS, enable programs.nix-ld), so install a JDK instead." + "{missing}\n • Minecraft's own Java from Mojang needs glibc ({GLIBC_LOADER} is missing; on NixOS, enable programs.nix-ld), so install compatible Java instead." )); } let fetch = |url: &str| -> Result, String> { @@ -1144,13 +1150,13 @@ fn java( let home = mojang::java_runtime(&component, &runtime, 8, &fetch, log).map_err(|e| { format!("{missing}\n • Horizon also tried to download Minecraft's own Java from Mojang, but: {e}") })?; - horizon_prepare::java::find(major, &[home]).map(|jdk| jdk.home()) + find(major, &[home]).map(|jdk| jdk.home()) } /// The dynamic loader glibc programs on x86-64 Linux start through. const GLIBC_LOADER: &str = "/lib64/ld-linux-x86-64.so.2"; -/// Runs `cmd` with the JDK at `home`: JAVA_HOME names it and its `bin` leads the PATH, as the +/// Runs `cmd` with Java at `home`: JAVA_HOME names it and its `bin` leads the PATH, as the /// versions' own preparers look for Java. fn with_java(cmd: &mut Command, home: &Path) { let mut path = vec![home.join("bin")]; @@ -1253,7 +1259,7 @@ pub fn preparer( "This version prepares with Python: Python 3.10 or newer is needed. Install it, or pick a newer version.", )?), }; - // The JDK the version pins, found as horizon-prepare finds it, so a missing one is + // The Java the version pins, found as horizon-prepare finds it, so a missing one is // reported (or downloaded) before the server jar is downloaded. Versions from before // java_major ran Java 25. let major = read_json(&dir.join("research-lock.json")) @@ -1267,7 +1273,13 @@ pub fn preparer( .into_iter() .map(|(_, dir)| dir), ); - let java_home = java(major, &games, &manifest, log)?; + let java_home = java( + major, + !horizon_prepare::exporters::available(&dir), + &games, + &manifest, + log, + )?; let jar = data_dir() .join("server-jars") .join(format!("{version}.jar")); diff --git a/launcher/src/backend/java_tests.rs b/launcher/src/backend/java_tests.rs index 75914a1b0..3f46123b8 100644 --- a/launcher/src/backend/java_tests.rs +++ b/launcher/src/backend/java_tests.rs @@ -5,7 +5,7 @@ fn selected_jdk_reaches_external_preparers() { const CHILD: &str = "HORIZON_LAUNCHER_JAVA_TEST"; if let Some(root) = std::env::var_os(CHILD) { let root = PathBuf::from(root); - let home = java(25, &[], &root.join("missing.json"), &|_| {}).unwrap(); + let home = java(25, true, &[], &root.join("missing.json"), &|_| {}).unwrap(); assert_eq!( fs::canonicalize(&home).unwrap(), fs::canonicalize(root.join("jdk25")).unwrap() diff --git a/launcher/src/mojang.rs b/launcher/src/mojang.rs index 27f9c3f0b..f242672b9 100644 --- a/launcher/src/mojang.rs +++ b/launcher/src/mojang.rs @@ -103,8 +103,8 @@ fn java_platform() -> Option<&'static str> { /// Downloads Mojang's Java runtime `component` (the version JSON's `javaVersion.component`, e.g. /// `java-runtime-epsilon`) into the game directory's `runtime` folder, as the official launcher /// lays it out: `///`. Each file is checked against its sha1; -/// files already there and whole are kept. Mojang's runtimes are whole JDKs, javac included, -/// which is what the exporters need. +/// files already there and whole are kept. These also include javac for older release preparers; +/// current releases can use an already installed runtime without it. pub fn java_runtime( component: &str, runtime: &Path, diff --git a/mods/horizon/src/test/java/dev/horizonlab/parity/PrepareWorker.java b/mods/horizon/src/test/java/dev/horizonlab/parity/PrepareWorker.java new file mode 100644 index 000000000..26104bbe1 --- /dev/null +++ b/mods/horizon/src/test/java/dev/horizonlab/parity/PrepareWorker.java @@ -0,0 +1,69 @@ +package dev.horizonlab.parity; + +import com.google.gson.*; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.util.*; +import net.minecraft.core.*; +import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.tags.*; + +/** Original launcher transport. Each worker owns one server OR client classpath. */ +public final class PrepareWorker { + private static Runnable restoreTags(Registry registry) { + var tags = new HashMap, List>>(); + registry.getTags().forEach(tag -> tags.put(tag.key(), tag.stream().toList())); + return () -> registry.prepareTagReload(new TagLoader.LoadResult<>(registry.key(), tags)).apply(); + } + + public static void main(String[] args) throws Exception { + if (args.length > 0 && args[0].equals("validate")) { + try (var paths = Files.walk(Path.of(args[1]))) { + for (var path : paths.filter(p -> p.toString().endsWith(".class")).toList()) { + String name = Path.of(args[1]).relativize(path).toString() + .replace(File.separatorChar, '.').replaceAll("\\.class$", ""); + var type = Class.forName(name, false, PrepareWorker.class.getClassLoader()); + type.getDeclaredMethods(); type.getDeclaredConstructors(); type.getDeclaredFields(); + } + } + return; + } + // Bootstrap replaces System.out. Keep the protocol on the original descriptor. + var replies = new PrintWriter(new OutputStreamWriter(new FileOutputStream(FileDescriptor.out), StandardCharsets.UTF_8), true); + System.setOut(System.err); + net.minecraft.SharedConstants.tryDetectVersion(); + net.minecraft.server.Bootstrap.bootStrap(); + var resets = new ArrayList(); + for (var registry : BuiltInRegistries.REGISTRY) resets.add(restoreTags(registry)); + System.setProperty("horizon.prepare.reuse", "true"); + try (var input = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8))) { + for (String line; (line = input.readLine()) != null;) { + var task = JsonParser.parseString(line).getAsJsonArray(); + var previousOut = System.out; var previousErr = System.err; + String error = ""; + try (var log = new PrintStream(Files.newOutputStream(Path.of(task.get(0).getAsString()), + StandardOpenOption.CREATE, StandardOpenOption.APPEND), true, StandardCharsets.UTF_8)) { + System.setOut(log); System.setErr(log); + try { + var values = task.get(2).getAsJsonArray(); + var argv = new String[values.size()]; + for (int i = 0; i < argv.length; i++) argv[i] = values.get(i).getAsString(); + Class.forName("dev.horizonlab.parity." + task.get(1).getAsString()) + .getMethod("main", String[].class).invoke(null, (Object) argv); + } catch (Throwable failure) { + failure.printStackTrace(log); error = failure.toString(); + } finally { + for (var reset : resets) reset.run(); + } + } finally { + System.setOut(previousOut); System.setErr(previousErr); + } + replies.println(new Gson().toJson(error)); + if (!error.isEmpty()) System.exit(1); + } + } + // Vanilla may have started non-daemon executors. All task outputs are closed now. + System.exit(0); + } +} diff --git a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java index 3466606cd..6ac41ff47 100644 --- a/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java +++ b/mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java @@ -19,6 +19,14 @@ import net.minecraft.world.phys.shapes.*; /** Exports private runtime data locally and invokes actual vanilla cached-shape queries. */ public final class VanillaCollisionOracle { static { SharedConstants.tryDetectVersion(); Bootstrap.bootStrap(); } + private static final boolean REUSE = Boolean.getBoolean("horizon.prepare.reuse"); + private static Map generated; + private static final Map encodedStates = new IdentityHashMap<>(); + private static final Map encodedGrids = new IdentityHashMap<>(); + private static Tag encode(BlockState state) { + if (!REUSE) return BlockState.FULL_CODEC.encodeStart(NbtOps.INSTANCE,state).getOrThrow(); + return encodedStates.computeIfAbsent(state, s -> BlockState.FULL_CODEC.encodeStart(NbtOps.INSTANCE,s).getOrThrow()).copy(); + } private static final Field CACHE, COLLISION, DISCRETE; private static final Method INDEX; static { @@ -37,6 +45,7 @@ public final class VanillaCollisionOracle { return shape; } private static CompoundTag grid(VoxelShape shape) throws ReflectiveOperationException { + if (REUSE && encodedGrids.containsKey(shape)) return encodedGrids.get(shape).copy(); CompoundTag tag = new CompoundTag(); tag.putByte("canonical",(byte)(shape==Shapes.empty()?1:shape==Shapes.block()?2:0)); tag.putByte("cube", (byte)(shape instanceof CubeVoxelShape ? 1 : 0)); @@ -52,13 +61,14 @@ public final class VanillaCollisionOracle { byte[] bits = new byte[discrete.getXSize() * discrete.getYSize() * discrete.getZSize()]; for (int x = 0; x < discrete.getXSize(); x++) for (int y = 0; y < discrete.getYSize(); y++) for (int z = 0; z < discrete.getZSize(); z++) bits[(x * discrete.getYSize() + y) * discrete.getZSize() + z] = (byte)(discrete.isFull(x, y, z) ? 1 : 0); tag.putByteArray("occupied", bits); + if (REUSE) encodedGrids.put(shape, tag.copy()); return tag; } public static CompoundTag occlusionDefinitions() throws Exception { var method=BlockBehaviour.class.getDeclaredMethod("getOcclusionShape",BlockState.class);method.setAccessible(true); var states=new ListTag();var shapes=new ListTag();var ids=new IdentityHashMap();int[] mapping=new int[Block.BLOCK_STATE_REGISTRY.size()];int index=0; for(var state:Block.BLOCK_STATE_REGISTRY) { - states.add(BlockState.FULL_CODEC.encodeStart(NbtOps.INSTANCE,state).getOrThrow()); + states.add(encode(state)); var shape=(VoxelShape)method.invoke(state.getBlock(),state);Integer id=ids.get(shape); if(id==null){id=shapes.size();ids.put(shape,id);shapes.add(grid(shape));} mapping[index++]=id; @@ -70,12 +80,16 @@ public final class VanillaCollisionOracle { } private static void inputValues(PrintWriter out, double... values) { for (double value : values) out.printf(" %016x", Double.doubleToRawLongBits(value)); } private static void generate(Path base) throws Exception { + if (REUSE && generated != null) { + for (var file : generated.entrySet()) Files.write(base.resolve(file.getKey()), file.getValue()); + return; + } ListTag states = new ListTag(), shapes = new ListTag(); CompoundTag defaults = new CompoundTag(); Map ids = new IdentityHashMap<>();Set grids=new HashSet<>(); int[] mapping = new int[Block.BLOCK_STATE_REGISTRY.size()]; int cachedCount = 0; for (BlockState state : Block.BLOCK_STATE_REGISTRY) { int id = states.size(); if (Block.BLOCK_STATE_REGISTRY.getId(state) != id) throw new AssertionError("non-dense states"); - Tag encoded = BlockState.FULL_CODEC.encodeStart(NbtOps.INSTANCE, state).getOrThrow(); states.add(encoded); + Tag encoded = encode(state); states.add(encoded); if (state == state.getBlock().defaultBlockState()) defaults.putInt(((CompoundTag)encoded).getStringOr("id", ""), id); VoxelShape shape = cached(state); if (shape == null) mapping[id] = -1; @@ -109,6 +123,12 @@ public final class VanillaCollisionOracle { } } } + if (REUSE) { + // Consumers can overwrite block-registry.nbt; preserve the original bytes. + generated = new HashMap<>(); + for (String file : List.of("block-registry.nbt", "cached-collision.nbt", "inputs.txt")) + generated.put(file, Files.readAllBytes(base.resolve(file))); + } System.out.printf("HORIZON_COLLISION_RESULT {\"registry_states\":%d,\"cached_states\":%d,\"uncached_states\":%d,\"unique_shapes\":%d,\"unique_grids\":%d,\"queries\":%d}%n", mapping.length,cachedCount,mapping.length-cachedCount,shapes.size(),grids.size(),cachedCount*6); } private static void values(PrintWriter out, String label, double... values) { out.print(label); for (double value : values) out.printf(" %016x",Double.doubleToLongBits(value)); out.println(); } diff --git a/packaging/PLAY.txt b/packaging/PLAY.txt index 20ea402fb..6d696b16c 100644 --- a/packaging/PLAY.txt +++ b/packaging/PLAY.txt @@ -17,7 +17,7 @@ Downloaded this archive directly? More about Horizon: https://playhorizon.dev To prepare this folder by hand you need Minecraft Java Edition 26.3, played - once from the official launcher, and Java 25 (a JDK). Then, in this folder: + once from the official launcher, and a Java 25 runtime. Then, in this folder: ./horizon-prepare --prepare --server-jar PATH/TO/server.jar ./horizon-prepare sounds (optional, for sound) (Windows: horizon-prepare.exe.) The 26.3 server jar is the downloads.server.url diff --git a/tests/test_prepare_exporters.py b/tests/test_prepare_exporters.py new file mode 100644 index 000000000..56416100a --- /dev/null +++ b/tests/test_prepare_exporters.py @@ -0,0 +1,37 @@ +"""The release comparison must tolerate only known variation and still catch regressions.""" +import contextlib +import io +import json +from pathlib import Path +import tempfile +import unittest + +from tools.check_prepare_exporters import verify + + +class PrepareExportersTests(unittest.TestCase): + def test_known_noise_does_not_mask_changed_resources(self): + with tempfile.TemporaryDirectory() as temporary: + folders = [Path(temporary) / str(i) for i in range(3)] + for i, folder in enumerate(folders): + folder.mkdir() + (folder / 'stable.nbt').write_bytes(b'unchanged') + (folder / 'horizon-resources.json').write_text('{"schema":7}') + (folder / 'block-behaviours.json').write_text(json.dumps({ + 'fields': {'shape': f'', 'solid': True}})) + (folder / 'creative-inventory.json').write_text(json.dumps({ + 'tabs': [{}, {'tags': [['a', 'b'] if i < 2 else ['b', 'a']], 'items': ['stone']}]})) + with contextlib.redirect_stdout(io.StringIO()): + verify(folders) + for name, changed in [ + ('stable.nbt', b'changed'), + ('block-behaviours.json', b'{"fields":{"shape":"","solid":false}}'), + ('creative-inventory.json', b'{"tabs":[{},{"tags":[["a","b"]],"items":["dirt"]}]}'), + ('horizon-resources.json', b'{"schema":8}'), + ]: + path = folders[2] / name + original = path.read_bytes() + path.write_bytes(changed) + with self.subTest(resource=name), self.assertRaises(SystemExit): + verify(folders) + path.write_bytes(original) diff --git a/tools/check_prepare_exporters.py b/tools/check_prepare_exporters.py new file mode 100644 index 000000000..6d37dfe93 --- /dev/null +++ b/tools/check_prepare_exporters.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Compare two isolated preparations with a batched, runtime-only release preparation. + +Uses only locally imported pinned Minecraft files. Run after compile-exporters and copy its +server/, client/ and manifest.json to exporters/. Byte differences are accepted only where +both isolated runs also differ and normalize identically (known exporter noise). +""" +import argparse +import hashlib +import json +import os +import re +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import time + +ROOT = Path(__file__).resolve().parents[1] + + +def hashes(folder): + return {str(p.relative_to(folder)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in folder.rglob('*') if p.is_file() and p.suffix != '.log' + and not {'classes', 'client-classes'} & set(p.relative_to(folder).parts) + and p.name != 'horizon-resources.json'} + + +def normalize(path): + value = json.loads(path.read_text()) + if path.name == 'block-behaviours.json': + # JVM-assigned lambda class addresses are printed as opaque field values. + def clean(node): + if isinstance(node, str): + return re.sub(r'\$\$Lambda/0x[0-9a-fA-F]+', '$$Lambda/
', node) + if isinstance(node, list): + return [clean(item) for item in node] + if isinstance(node, dict): + return {key: clean(item) for key, item in node.items()} + return node + value = clean(value) + elif path.name == 'creative-inventory.json': + for tab in value['tabs']: + if 'tags' in tab: + tab['tags'] = [sorted(tags) for tags in tab['tags']] + return value + + +def verify(folders): + a, b, batched = [hashes(folder) for folder in folders] + if a.keys() != b.keys() or a.keys() != batched.keys(): + raise SystemExit('Resource file sets differ') + known = {'render-definitions.json', 'block-behaviours.json', 'creative-inventory.json'} + noise = [name for name in a if a[name] != b[name] or a[name] != batched[name]] + differences = [] + for name in noise: + if name not in known: + differences.append(name) + else: + x, y, z = [normalize(folder / name) for folder in folders] + if x != y or x != z: + differences.append(name) + manifests = [] + for folder, raw in zip(folders, [a, b, batched]): + replacements = {raw[name]: f'normalized:{name}' for name in noise} + def clean(node): + if isinstance(node, str): + return replacements.get(node, node) + if isinstance(node, list): + return [clean(item) for item in node] + if isinstance(node, dict): + return {key: clean(item) for key, item in node.items()} + return node + manifests.append(clean(json.loads((folder / 'horizon-resources.json').read_text()))) + if manifests[0] != manifests[1] or manifests[0] != manifests[2]: + differences.append('horizon-resources.json') + print(f'Normalized exporter noise: {noise}') + if differences: + raise SystemExit(f'Resources differ: {differences}') + print(f'PASS: all {len(a)} resource files match, with only known noise normalized') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--work', type=Path) + args = parser.parse_args() + binary = args.binary.resolve() + with tempfile.TemporaryDirectory(prefix='prepare-exporters-') as temporary: + work = args.work.resolve() if args.work else Path(temporary) + work.mkdir(parents=True, exist_ok=True) + # Freeze the tested executable even if a developer rebuilds concurrently. + frozen = work / binary.name + shutil.copy2(binary, frozen) + binary = frozen + results = [] + times = [] + for name in ['isolated-1', 'isolated-2', 'batched']: + folder = work / name + folder.mkdir() + shutil.copyfile(ROOT / 'research-lock.json', folder / 'research-lock.json') + shutil.copytree(ROOT / 'mods/horizon/src/test/java', folder / 'mods/horizon/src/test/java') + shutil.copytree(ROOT / 'exporters', folder / 'exporters') + (folder / '.local').mkdir() + (folder / '.local/minecraft').symlink_to((ROOT / '.local/minecraft').resolve(), target_is_directory=True) + env = dict(os.environ) + env.pop('HORIZON_PREPARE_ISOLATED', None) + if name.startswith('isolated'): + env['HORIZON_PREPARE_ISOLATED'] = '1' + else: + # An actual runtime-only home: no javac in bin. Java resolves its own libraries. + java = Path(shutil.which('java')).resolve() + runtime = folder / 'runtime' + (runtime / 'bin').mkdir(parents=True) + trace = folder / 'java-invocations.txt' + wrapper = runtime / 'bin' / java.name + wrapper.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> ' + + shlex.quote(str(trace)) + '\nexec ' + + shlex.quote(str(java)) + ' "$@"\n') + wrapper.chmod(0o755) + env['JAVA_HOME'] = str(runtime) + env['PATH'] = str(runtime / 'bin') + started = time.monotonic() + with (work / f'{name}.log').open('w') as log: + subprocess.run([str(binary), '--root', str(folder)], cwd=folder, env=env, + stdout=log, stderr=subprocess.STDOUT, check=True, timeout=1200) + if name == 'batched': + launches = [line for line in trace.read_text().splitlines() if line != '-version'] + if len(launches) != 5: + raise SystemExit(f'Expected 5 exporter/data-generator JVMs, got {len(launches)}') + print(f'Runtime-only preparation: {len(launches)} JVM launches', flush=True) + text = (work / f'{name}.log').read_text() + if 'Compiling ' in text: + raise SystemExit(f'{name} unexpectedly needed a compiler') + resources = next((folder / '.local/horizon-client').iterdir()) + results.append(resources) + times.append(round(time.monotonic() - started, 2)) + print(f'{name}: {times[-1]}s, prepared', flush=True) + verify(results) + print(f'Isolated {times[:2]}s, batched {times[2]}s') + + +if __name__ == '__main__': + main() diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 28b591626..2682b1f82 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -222,16 +222,40 @@ fn package(binary: &Path, out: &Path) -> Result<(), String> { let repo = new_folder(out)?; copy_into(binary, out)?; copy_beside(binary, &TOOLS, out)?; + copy_exporters(&repo.join("exporters"), &out.join("exporters"))?; let play = repo.join("packaging/PLAY.txt"); copy(&play, &out.join("PLAY.txt")) } fn package_server(engine: &Path, out: &Path) -> Result<(), String> { - new_folder(out)?; + let repo = new_folder(out)?; + copy_exporters(&repo.join("exporters"), &out.join("exporters"))?; copy(engine, &out.join(program_name(engine, "horizon-engine")))?; copy_beside(engine, &SERVER_TOOLS, out) } +/// Release builds supply only our compiled classes and their integrity manifest. +fn copy_exporters(from: &Path, to: &Path) -> Result<(), String> { + if !from.join("manifest.json").is_file() { + return Err("Missing exporters/manifest.json; run horizon-prepare compile-exporters and copy its bundle to exporters/".into()); + } + copy_exporter_tree(from, to) +} + +fn copy_exporter_tree(from: &Path, to: &Path) -> Result<(), String> { + fs::create_dir_all(to).map_err(|e| e.to_string())?; + for entry in fs::read_dir(from).map_err(|e| e.to_string())? { + let path = entry.map_err(|e| e.to_string())?.path(); + let target = to.join(path.file_name().unwrap()); + if path.is_dir() { + copy_exporter_tree(&path, &target)?; + } else { + copy(&path, &target)?; + } + } + Ok(()) +} + /// `OUT`, emptied, with the sources the release folders take (`ARCHIVED`); the repository's path. fn new_folder(out: &Path) -> Result { let top = git(