From 32a4383459471bc121c0fdbeacd8f55eaef6042c Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:08:33 +0200 Subject: [PATCH] Build local Minecraft 26.3 decompilation and Rust optimization lab --- .githooks/pre-commit | 2 + .githooks/pre-push | 2 + .github/workflows/ci.yml | 27 +++ .gitignore | 15 ++ Cargo.lock | 7 + Cargo.toml | 12 ++ LICENSE | 21 ++ README.md | 80 ++++++++ crates/voxel-kernels/Cargo.toml | 10 + crates/voxel-kernels/src/lib.rs | 136 +++++++++++++ crates/voxel-kernels/src/main.rs | 92 +++++++++ docs/decompilation-status.md | 23 +++ docs/optimization-plan.md | 67 +++++++ research-lock.json | 13 ++ tests/test_lab.py | 124 ++++++++++++ tools/check_public_tree.py | 41 ++++ tools/lab.py | 322 +++++++++++++++++++++++++++++++ 17 files changed, 994 insertions(+) create mode 100755 .githooks/pre-commit create mode 100755 .githooks/pre-push create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 crates/voxel-kernels/Cargo.toml create mode 100644 crates/voxel-kernels/src/lib.rs create mode 100644 crates/voxel-kernels/src/main.rs create mode 100644 docs/decompilation-status.md create mode 100644 docs/optimization-plan.md create mode 100644 research-lock.json create mode 100644 tests/test_lab.py create mode 100644 tools/check_public_tree.py create mode 100644 tools/lab.py diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 000000000..2d7f3fac6 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,2 @@ +#!/bin/sh +exec python3 tools/check_public_tree.py diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 000000000..a886ccf0c --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,2 @@ +#!/bin/sh +exec python3 tools/check_public_tree.py --history diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..fb9bc85b9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: checks +on: + push: + pull_request: +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Keep game artifacts out of Git + run: | + python3 tools/check_public_tree.py + python3 tools/check_public_tree.py --history + - name: Test offline research tooling + run: python3 -m unittest discover -s tests -v + - name: Install Rust toolchain + run: rustup toolchain install stable --profile minimal --component rustfmt --component clippy + - name: Check formatting and lint + run: | + cargo +stable fmt --all -- --check + cargo +stable clippy --workspace --all-targets -- -D warnings + - name: Test voxel kernel correctness + run: cargo +stable test --workspace diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..732d42083 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +/.local/ +/target/ +**/__pycache__/ +*.py[cod] +/.venv/ +# Downloaded/decompiled game content and private profiling artifacts. +*.jar +*.class +*.jfr +*.hprof +/decompiled/ +/minecraft/ +/assets/ +/run/ +/logs/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..c676f4e42 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "voxel-kernels" +version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..26510443b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,12 @@ +[workspace] +members = ["crates/voxel-kernels"] +resolver = "2" + +[workspace.package] +edition = "2024" +license = "MIT" +rust-version = "1.85" + +[profile.release] +lto = "thin" +codegen-units = 1 diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..df2363379 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Ludovic + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 000000000..2a7c089b0 --- /dev/null +++ b/README.md @@ -0,0 +1,80 @@ +# Minecraft Optimization Lab + +A personal research workspace for understanding Minecraft Java Edition and testing original performance improvements, including Rust kernels. Not affiliated with Mojang or Microsoft. + +**Current scope:** complete local client/server decompilation for pinned Java Edition **26.3**, class inventories and output checks, and a tested Rust occupancy experiment. This is not a playable Rust port, a rebuilt Minecraft distribution, or proof of a game-level speedup. + +The game JARs, libraries, decompiled Java, inventories, and logs live **inside this checkout** under **`.local/`**, which is gitignored. GitHub contains only our tooling, original Rust code, and research documentation. A fresh clone regenerates the local game workspace from official downloads. + +## Prepare and inspect the game + +Requires Python 3.11+, JDK 25+, and Git. No Microsoft account credentials are needed for these downloads or decompilation. Use your normal licensed launcher and sign-in when playing or collecting client performance measurements. + +```sh +git clone https://github.com/ludovic111/minecraft-optimization-lab.git +cd minecraft-optimization-lab +git config core.hooksPath .githooks +python3 tools/lab.py prepare +python3 tools/lab.py decompile +python3 tools/lab.py status +``` + +`research-lock.json` pins the version manifest and Vineflower 1.12.0 with checksums. Preparation verifies official artifact hashes, extracts the inner server game JAR from its launcher bundle, and supplies client/server libraries to the decompiler. No game process is launched, no game authentication is implemented, and no server `eula.txt` is written by the tooling. + +The default decompilation uses four threads and a 6 GiB Java heap, with one side processed at a time: + +```sh +python3 tools/lab.py decompile --side client --threads 4 --heap 6G +python3 tools/lab.py decompile --side server --java /path/to/jdk-25/bin/java +``` + +Outputs: + +```text +.local/minecraft/26.3/ + jars/client.jar + jars/server.jar # Downloaded launcher bundle + jars/server-game.jar # Actual server bytecode + libraries/{client,server}/ + sources/client/ # Decompiled client, including shared game logic + sources/server/ # Decompiled dedicated server + reports/*-inventory.json # Every class, SourceFile, method count, size + reports/*-coverage.json # Missing files and detected failure markers + reports/*-decompiler.log +``` + +Inner classes generally share a `.java` file with their enclosing class. Source coverage compares class-file `SourceFile` metadata with emitted Java, and scans Vineflower error markers. An exit code of 2 from `decompile` or `status` means coverage/failure markers need investigation. This does not establish semantic equivalence, preservation of every synthetic method, or recompilability; keep the original bytecode as the reference. Completed source directories are never overwritten automatically. Move a directory aside before repeating decompilation. + +For older obfuscated versions, preparation deliberately stops: a version-specific remapping stage is needed. The tool does not silently present obfuscated output as readable source. The current pin can be updated by reviewing an official version manifest and its checksums. + +## Run the original Rust experiment + +Requires Rust 1.85+; there are no third-party crate dependencies. + +```sh +cargo test --workspace +cargo run --release --bin voxel-bench -- 100 > .local/voxel-benchmark.csv +``` + +`voxel-kernels` compares a scalar reference with a row-bitset algorithm for counting exposed faces in synthetic 16×16×16 sections. It checks empty, full, flat, checkerboard, and seeded random occupancy at three densities. Each result is the median of 21 samples, with warm-up, alternating measurement order, and checks against the reference. The repeat count is configurable. + +All cubes are treated as opaque/full, and external neighbors as air. There is no mesh output, texture lookup, AO, lighting, GPU work, cross-section neighborhood, JNI bridge, or Minecraft integration yet. A speedup here applies only to this kernel relative to our scalar reference, not to Minecraft's renderer or its frame rate. See [the optimization plan](docs/optimization-plan.md). + +## Develop + +```sh +python3 -m unittest discover -s tests -v +cargo fmt --all -- --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +python3 tools/check_public_tree.py +python3 tools/check_public_tree.py --history +``` + +CI is offline with respect to Minecraft: it tests the original tooling and Rust kernels without downloading or publishing game files. The optional hooks and CI reject `.local/`, game/profiling binaries, and `.java` files even if force-added. Original Java mod sources would need an explicit reviewed exception before adding them. The path check is a guardrail, not a substitute for reviewing a public diff. + +## Ownership and references + +The [MIT license](LICENSE) applies only to the original code and documentation tracked in this repository. It does not relicense downloaded game content, local decompiled Java, dependency JARs, or Vineflower. Their respective licenses still apply. + +Mojang [announced removal of Java Edition obfuscation](https://www.minecraft.net/en-us/article/removing-obfuscation-in-java-edition) without changing the [Minecraft EULA](https://www.minecraft.net/en-us/eula). Keep game code and modified game builds local; publish original tools and independent optimization work. Decompiler behavior is documented by [Vineflower](https://vineflower.org/usage/). diff --git a/crates/voxel-kernels/Cargo.toml b/crates/voxel-kernels/Cargo.toml new file mode 100644 index 000000000..f629b4019 --- /dev/null +++ b/crates/voxel-kernels/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "voxel-kernels" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[[bin]] +name = "voxel-bench" +path = "src/main.rs" diff --git a/crates/voxel-kernels/src/lib.rs b/crates/voxel-kernels/src/lib.rs new file mode 100644 index 000000000..caf72879f --- /dev/null +++ b/crates/voxel-kernels/src/lib.rs @@ -0,0 +1,136 @@ +//! Original occupancy kernels for synthetic 16³ voxel sections. +//! A set bit means an opaque full cube; all neighbors outside the section are air. +//! This is not a Minecraft renderer or a model of partial/transparent blocks. + +pub const SIDE: usize = 16; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Section { + rows: [u16; SIDE * SIDE], +} + +impl Default for Section { + fn default() -> Self { + Self { + rows: [0; SIDE * SIDE], + } + } +} + +impl Section { + pub fn from_fn(mut occupied: impl FnMut(usize, usize, usize) -> bool) -> Self { + let mut section = Self::default(); + for z in 0..SIDE { + for y in 0..SIDE { + for x in 0..SIDE { + if occupied(x, y, z) { + section.rows[z * SIDE + y] |= 1 << x; + } + } + } + } + section + } + + pub fn occupied(&self, x: usize, y: usize, z: usize) -> bool { + assert!(x < SIDE && y < SIDE && z < SIDE); + self.rows[z * SIDE + y] & (1 << x) != 0 + } + + /// Simple per-voxel reference. Count each opaque cube's exposed unit faces. + pub fn exposed_faces_scalar(&self) -> u32 { + let mut faces = 0; + for z in 0..SIDE { + for y in 0..SIDE { + for x in 0..SIDE { + if !self.occupied(x, y, z) { + continue; + } + faces += u32::from(x == 0 || !self.occupied(x - 1, y, z)); + faces += u32::from(x + 1 == SIDE || !self.occupied(x + 1, y, z)); + faces += u32::from(y == 0 || !self.occupied(x, y - 1, z)); + faces += u32::from(y + 1 == SIDE || !self.occupied(x, y + 1, z)); + faces += u32::from(z == 0 || !self.occupied(x, y, z - 1)); + faces += u32::from(z + 1 == SIDE || !self.occupied(x, y, z + 1)); + } + } + } + faces + } + + /// Evaluate sixteen voxels at once with row masks and population counts. + pub fn exposed_faces_bitset(&self) -> u32 { + let mut faces = 0; + for z in 0..SIDE { + for y in 0..SIDE { + let i = z * SIDE + y; + let row = self.rows[i]; + let below = if y == 0 { 0 } else { self.rows[i - 1] }; + let above = if y + 1 == SIDE { 0 } else { self.rows[i + 1] }; + let back = if z == 0 { 0 } else { self.rows[i - SIDE] }; + let front = if z + 1 == SIDE { + 0 + } else { + self.rows[i + SIDE] + }; + faces += (row & !(row << 1)).count_ones(); + faces += (row & !(row >> 1)).count_ones(); + faces += (row & !below).count_ones(); + faces += (row & !above).count_ones(); + faces += (row & !back).count_ones(); + faces += (row & !front).count_ones(); + } + } + faces + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check(section: &Section, expected: u32) { + assert_eq!(section.exposed_faces_scalar(), expected); + assert_eq!(section.exposed_faces_bitset(), expected); + } + + #[test] + fn known_geometry() { + check(&Section::default(), 0); + check(&Section::from_fn(|_, _, _| true), 6 * 16 * 16); + check(&Section::from_fn(|x, y, z| x == 3 && y == 4 && z == 5), 6); + check(&Section::from_fn(|x, y, z| x < 2 && y == 4 && z == 5), 10); + check( + &Section::from_fn(|x, y, z| (x + y + z) % 2 == 0), + 4096 / 2 * 6, + ); + } + + #[test] + fn every_single_voxel_including_boundaries() { + for i in 0..SIDE.pow(3) { + let mut section = Section::default(); + section.rows[i / SIDE] = 1 << (i % SIDE); + check(§ion, 6); + } + } + + #[test] + fn random_occupancy_matches_reference_at_multiple_densities() { + let mut state = 0x1234_5678_9abc_def0_u64; + for threshold in [0, 1, 16, 64, 128, 192, 254, 255, 256] { + for _ in 0..32 { + let section = Section::from_fn(|_, _, _| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 255) < threshold + }); + assert_eq!( + section.exposed_faces_scalar(), + section.exposed_faces_bitset() + ); + } + } + } +} diff --git a/crates/voxel-kernels/src/main.rs b/crates/voxel-kernels/src/main.rs new file mode 100644 index 000000000..e2443958b --- /dev/null +++ b/crates/voxel-kernels/src/main.rs @@ -0,0 +1,92 @@ +use std::{hint::black_box, time::Instant}; +use voxel_kernels::Section; + +fn sample(corpus: &[Section], kernel: fn(&Section) -> u32, repeats: usize) -> f64 { + let start = Instant::now(); + let mut checksum = 0_u64; + for _ in 0..repeats { + for section in corpus { + checksum = checksum.wrapping_add(u64::from(kernel(black_box(section)))); + } + } + black_box(checksum); + start.elapsed().as_nanos() as f64 / (corpus.len() * repeats) as f64 +} + +fn median(samples: &mut [f64]) -> f64 { + samples.sort_by(f64::total_cmp); + samples[samples.len() / 2] +} + +fn main() { + let repeats: usize = match std::env::args().nth(1) { + None => 100, + Some(value) => match value.parse() { + Ok(n) if n > 0 => n, + _ => { + eprintln!("usage: voxel-bench [positive repeat count]"); + std::process::exit(2); + } + }, + }; + let mut rng = 0x7065_7273_6f6e_616c_u64; + let random = |rng: &mut u64| { + *rng ^= *rng << 13; + *rng ^= *rng >> 7; + *rng ^= *rng << 17; + *rng + }; + println!( + "workload,sections,samples,repeats,scalar_ns_per_section,bitset_ns_per_section,speedup" + ); + for workload in [ + "empty", + "solid", + "flat", + "checkerboard", + "random_10", + "random_50", + "random_90", + ] { + let corpus: Vec<_> = (0..32) + .map(|i| { + Section::from_fn(|x, y, z| match workload { + "empty" => false, + "solid" => true, + "flat" => y < 1 + i % 15, + "checkerboard" => (x + y + z + i) % 2 == 0, + "random_10" => random(&mut rng) % 100 < 10, + "random_50" => random(&mut rng) % 100 < 50, + "random_90" => random(&mut rng) % 100 < 90, + _ => unreachable!(), + }) + }) + .collect(); + for section in &corpus { + assert_eq!( + section.exposed_faces_scalar(), + section.exposed_faces_bitset() + ); + } + for kernel in [Section::exposed_faces_scalar, Section::exposed_faces_bitset] { + sample(&corpus, kernel, 10); + } + let mut scalar = Vec::new(); + let mut bitset = Vec::new(); + for trial in 0..21 { + if trial % 2 == 0 { + scalar.push(sample(&corpus, Section::exposed_faces_scalar, repeats)); + bitset.push(sample(&corpus, Section::exposed_faces_bitset, repeats)); + } else { + bitset.push(sample(&corpus, Section::exposed_faces_bitset, repeats)); + scalar.push(sample(&corpus, Section::exposed_faces_scalar, repeats)); + } + } + let scalar = median(&mut scalar); + let bitset = median(&mut bitset); + println!( + "{workload},32,21,{repeats},{scalar:.2},{bitset:.2},{:.2}", + scalar / bitset + ); + } +} diff --git a/docs/decompilation-status.md b/docs/decompilation-status.md new file mode 100644 index 000000000..f7c72d7f7 --- /dev/null +++ b/docs/decompilation-status.md @@ -0,0 +1,23 @@ +# Local decompilation status + +Verified on 2026-10-03 using the pinned official Java Edition 26.3 artifacts, Vineflower 1.12.0, and OpenJDK 25. + +| Artifact | Input classes | Input methods | Expected Java files | Emitted Java files | Missing files | Files with detected failed-decompilation markers | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Client | 11,383 | 98,739 | 7,301 | 7,301 | 0 | 0 | +| Dedicated server (inner game JAR) | 7,762 | 72,269 | 5,037 | 5,037 | 0 | 0 | + +Counts include all class files in each game JAR. Shared code appears in both sides; do not add the class/method counts to estimate unique game code. Nested classes share their enclosing Java source file. All input classes were passed to the decompiler without package filters. Third-party dependency JARs were supplied as references rather than decompiled; runtime/native libraries and the original development/build environment are outside this source recovery. + +The full outputs and machine-readable reports are local and gitignored: + +```text +.local/minecraft/26.3/sources/client/ +.local/minecraft/26.3/sources/server/ +.local/minecraft/26.3/reports/client-inventory.json +.local/minecraft/26.3/reports/server-inventory.json +.local/minecraft/26.3/reports/client-coverage.json +.local/minecraft/26.3/reports/server-coverage.json +``` + +Recheck with `python3 tools/lab.py status`. File coverage and the absence of detected failure markers do not prove every recovered method behaves identically, or that this source can be rebuilt. Original bytecode remains the behavior reference. There has been no gameplay baseline, full Rust port, or measured real-game optimization yet. diff --git a/docs/optimization-plan.md b/docs/optimization-plan.md new file mode 100644 index 000000000..cd27fb72e --- /dev/null +++ b/docs/optimization-plan.md @@ -0,0 +1,67 @@ +# Optimization research plan + +## Starting point + +Inspect both pinned game JARs locally, retain bytecode as the reference, and measure actual Minecraft workloads before selecting a subsystem to replace. The Rust face-count experiment establishes a small correctness/benchmark workflow; it does not identify a Minecraft bottleneck. + +Whole-game translation is a long-term research objective. Decompiled Java is an inspection artifact, not the original source tree: build configuration, comments, tooling history, and some source-level constructs cannot be recovered faithfully. A mechanically translated Rust game would also have to preserve behavior, data formats, networking, rendering, and mod compatibility. An isolated hot-path replacement is a useful first experiment. + +## Local code map + +These entry points were verified present in the decompiled 26.3 client. Paths below are relative to `.local/minecraft/26.3/sources/client/net/minecraft/`; shared/server logic can also be inspected in the dedicated-server tree. + +| Area | Local entry points | Questions to measure | +| --- | --- | --- | +| Client loop | `client/Minecraft.java` | CPU frame-time distribution, allocation/GC, render-thread stalls | +| Server loop | `server/MinecraftServer.java`, `server/level/ServerLevel.java` | Tick time, entity/block tick work, waiting on chunk jobs | +| Section compilation | `client/renderer/chunk/SectionCompiler.java`, `SectionRenderDispatcher.java` in that directory | Work per rebuild, queue latency, allocations, upload overhead | +| World generation | `world/level/levelgen/NoiseChunk.java`, `NoiseBasedChunkGenerator.java` in that directory | Time per newly generated chunk, cache reuse, generation scheduling | +| Lighting | `world/level/lighting/LevelLightEngine.java` | Update propagation work and queue contention | +| Block storage | `world/level/chunk/PalettedContainer.java` | Palette access costs, locality, mutation behavior | +| Region I/O | `world/level/chunk/storage/RegionFile.java` | Serialization/compression cost versus disk waits | + +The local inventory contains every class, bytecode size, method count, and source-file relationship. Large classes are navigation hints; bytecode size does not establish execution cost. Generated registries can be large while irrelevant to steady-state frame time. + +## Capture a real baseline + +Use a licensed launcher installation and a copy of a test world. Keep raw captures and world data in `.local/`. Record game version/hash, mod versions, Java version/flags, CPU/GPU, OS, resolution, render/simulation distance, world seed, route, and warm-up duration. + +Use at least these separate scenarios: + +1. A fixed stationary view after chunks settle. +2. A reproducible movement route through already generated chunks. +3. The same route through newly generated terrain. +4. A controlled entity/redstone workload on the dedicated server. +5. World loading/saving as a separate latency test. + +For a running local Java process, JDK tooling can capture a bounded recording: + +```sh +jcmd -l +# Replace 12345 with the Minecraft JVM PID, and use an absolute capture path. +jcmd 12345 JFR.start name=mc-baseline settings=profile duration=120s filename=/absolute/path/to/repo/.local/baseline.jfr +jfr summary .local/baseline.jfr +jfr print --events jdk.ExecutionSample,jdk.GarbageCollection .local/baseline.jfr > .local/baseline-events.txt +``` + +Collect CPU profiles, allocation/GC information, frame-time percentiles, server tick-time percentiles, and relevant GPU timings. JFR samples alone do not measure FPS or GPU cost. JVM compilation/warm-up, profiling overhead, world generation, and background activity must be controlled when comparing runs. No gameplay baseline has been collected by this repository's initial setup. + +## First native integration experiment + +Choose a kernel only after a profile shows enough inclusive cost to justify it. A candidate should have a clear input/output boundary and deterministic reference tests. Candidate areas include occupancy preprocessing, palette scans, or batched generation calculations; these are hypotheses to investigate. + +Measure an improved Java implementation first. Then compare an original Rust implementation with equivalent work and account for Java/native transition cost, data marshaling, allocation, and scheduling. Use coarse batches rather than a call per block. Preserve Minecraft behavior for boundaries, partial/transparent blocks, lighting, saved data, and deterministic world generation wherever the selected kernel touches them. + +Ship any integration as an original mod/native library loaded into the user's local game. Do not bundle a modified Minecraft client/server. Add a fallback to the reference implementation and differential tests using locally generated fixtures. + +## Gates before expanding scope + +| Gate | Evidence required | +| --- | --- | +| Bottleneck selected | Reproducible real-game profile and workload definition | +| Kernel equivalent | Boundary cases and randomized differential tests, including real input shapes | +| Native call worthwhile | End-to-end measurements including copying and transitions | +| Game improves | Repeated real-game frame/tick distributions, allocations, memory, and CPU/GPU cost | +| More Rust justified | Improvement persists across workloads without changing gameplay | + +If a subsystem occupies fraction `p` of total measured time, even eliminating its cost gives a theoretical maximum overall speedup of `1 / (1 - p)`. For example, a subsystem consuming 10% of frame time cannot alone yield a 2× whole-game speedup. Apply this before undertaking a large rewrite. diff --git a/research-lock.json b/research-lock.json new file mode 100644 index 000000000..1afd0035b --- /dev/null +++ b/research-lock.json @@ -0,0 +1,13 @@ +{ + "minecraft": { + "version": "26.3", + "manifest_url": "https://piston-meta.mojang.com/v1/packages/4fe1aa1ef8da1cb95c5bad1fb98890ca56dd8ca3/26.3.json", + "manifest_sha1": "4fe1aa1ef8da1cb95c5bad1fb98890ca56dd8ca3", + "java_major": 25 + }, + "vineflower": { + "version": "1.12.0", + "url": "https://github.com/Vineflower/vineflower/releases/download/1.12.0/vineflower-1.12.0.jar", + "sha256": "1dfcfe974395734fa467ce620661c7623d05ba83670de0529b1fbd63ff548b9d" + } +} diff --git a/tests/test_lab.py b/tests/test_lab.py new file mode 100644 index 000000000..0a635c8be --- /dev/null +++ b/tests/test_lab.py @@ -0,0 +1,124 @@ +import hashlib +import io +import json +from pathlib import Path +import struct +import subprocess +import tempfile +import unittest +from unittest import mock +import zipfile + +from tools import check_public_tree, lab + + +def minimal_class(source="Example.java"): + # A valid class-file structure with no fields/methods and one SourceFile attribute. + def utf8(value): + data = value.encode() + return b"\x01" + struct.pack(">H", len(data)) + data + return (b"\xca\xfe\xba\xbe" + struct.pack(">HHH", 0, 69, 5) + + utf8("SourceFile") + utf8(source) + utf8("Example") + b"\x07\x00\x03" + + struct.pack(">HHHHHHHHIH", 1, 4, 0, 0, 0, 0, 1, 1, 2, 2)) + + +class ArtifactTests(unittest.TestCase): + def test_download_rejects_corruption_without_overwriting_existing_file(self): + with tempfile.TemporaryDirectory() as temp: + destination = Path(temp) / "game.jar" + destination.write_bytes(b"existing") + with mock.patch("urllib.request.urlopen", return_value=io.BytesIO(b"corrupt")): + with self.assertRaisesRegex(ValueError, "Checksum mismatch"): + lab.download("https://example.invalid/game.jar", destination, "0" * 40) + self.assertEqual(destination.read_bytes(), b"existing") + self.assertEqual([p.name for p in Path(temp).iterdir()], ["game.jar"]) + + def test_cached_download_is_verified(self): + with tempfile.TemporaryDirectory() as temp: + destination = Path(temp) / "game.jar" + destination.write_bytes(b"verified") + with mock.patch("urllib.request.urlopen") as request: + lab.download("https://example.invalid", destination, hashlib.sha1(b"verified").hexdigest()) + request.assert_not_called() + + def test_sourcefile_metadata_covers_inner_classes(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "game.jar" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("demo/Example.class", minimal_class()) + archive.writestr("demo/Example$Inner.class", minimal_class()) + result = lab.inventory(path) + self.assertEqual(result["class_count"], 2) + self.assertEqual(result["expected_source_files"], ["demo/Example.java"]) + + def test_invalid_class_is_rejected(self): + for data in [b"oops", b"\xca\xfe\xba\xbe"]: + with self.assertRaises(ValueError): + lab.ClassReader(data).metadata() + + def test_server_bundler_extracts_actual_game_and_checks_checksum(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + path = directory / "server.jar" + game = b"inner game jar" + library = b"dependency" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("META-INF/versions.list", f"{hashlib.sha256(game).hexdigest()}\t26.3\t26.3/game.jar\n") + archive.writestr("META-INF/versions/26.3/game.jar", game) + archive.writestr("META-INF/libraries.list", f"{hashlib.sha256(library).hexdigest()}\tdemo:lib:1\tdemo/lib.jar\n") + archive.writestr("META-INF/libraries/demo/lib.jar", library) + extracted, libraries = lab.unpack_server(path, directory, "26.3") + self.assertEqual(extracted.read_bytes(), game) + self.assertEqual(libraries[0].read_bytes(), library) + with self.assertRaises(ValueError): + lab.unpack_server(path, directory, "wrong-version") + + def test_server_bundler_rejects_path_traversal(self): + for relative in ["../../escape.jar", "/escape.jar", "..\\escape.jar"]: + with zipfile.ZipFile(io.BytesIO(), "w") as archive: + archive.writestr("entries", f"{'0' * 64}\tdemo\t{relative}") + with self.assertRaisesRegex(ValueError, "Unsafe bundled path"): + list(lab.bundle_entries(archive, "entries")) + + def test_coverage_reports_missing_files_and_failed_method_markers(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + lab.write_json(directory / "reports" / "client-inventory.json", { + "class_count": 3, "expected_source_files": ["demo/Example.java", "demo/Missing.java"] + }) + sources = directory / "sources" / "client" / "demo" + sources.mkdir(parents=True) + (sources / "Example.java").write_text("// $VF: Couldn't be decompiled\n") + report = lab.coverage(directory, "client") + self.assertEqual(report["missing_source_files"], ["demo/Missing.java"]) + self.assertEqual(report["files_with_decompiler_error_markers"], ["demo/Example.java"]) + + +class PublicationTests(unittest.TestCase): + def test_private_paths_rejected(self): + for path in [".local/minecraft/source.java", "decompiled/Foo.txt", "game.JAR", "capture.jfr", "source/Foo.java"]: + self.assertTrue(check_public_tree.prohibited(path), path) + for path in ["tools/lab.py", "crates/voxel-kernels/src/lib.rs", "Cargo.lock", "docs/analysis.md"]: + self.assertFalse(check_public_tree.prohibited(path), path) + + def test_force_added_ignored_file_is_detected_in_index_and_history(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + real_check_output = subprocess.check_output + def git(*args): + return real_check_output(["git", *args], cwd=directory, stderr=subprocess.DEVNULL) + git("init", "-b", "main") + (directory / ".gitignore").write_text("/.local/\n") + (directory / ".local").mkdir() + (directory / ".local" / "artifact.txt").write_text("private fixture") + git("add", ".gitignore") + git("add", "-f", ".local/artifact.txt") + with mock.patch("tools.check_public_tree.subprocess.check_output", + side_effect=lambda command: git(*command[1:])): + self.assertIn(".local/artifact.txt", check_public_tree.paths_in_repository()) + git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-m", "fixture") + self.assertIn(".local/artifact.txt", check_public_tree.paths_in_repository(history=True)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/check_public_tree.py b/tools/check_public_tree.py new file mode 100644 index 000000000..ff0f5ab52 --- /dev/null +++ b/tools/check_public_tree.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Reject local game artifacts even when somebody uses git add -f.""" + +import argparse +from pathlib import PurePosixPath +import subprocess +import sys + +PRIVATE_ROOTS = {".local", "decompiled", "minecraft", "assets", "run", "logs"} +PRIVATE_EXTENSIONS = {".jar", ".class", ".jfr", ".hprof", ".java"} + + +def prohibited(path): + parsed = PurePosixPath(path) + return bool(parsed.parts and parsed.parts[0] in PRIVATE_ROOTS) or parsed.suffix.lower() in PRIVATE_EXTENSIONS + + +def paths_in_repository(history=False): + if not history: + result = subprocess.check_output(["git", "ls-files", "-z"]) + return result.decode("utf-8").split("\0") + result = subprocess.check_output(["git", "log", "--all", "--format=", "--name-only", "-z"]) + return [path.lstrip("\n") for path in result.decode("utf-8").split("\0")] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--history", action="store_true", help="Also inspect paths in all local Git history") + args = parser.parse_args() + rejected = sorted({path for path in paths_in_repository(args.history) if path and prohibited(path)}) + if rejected: + print("Private game/profiling artifacts must stay outside Git:", file=sys.stderr) + for path in rejected: + print(f" {path}", file=sys.stderr) + return 1 + print("Public tree check passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/lab.py b/tools/lab.py new file mode 100644 index 000000000..31537d666 --- /dev/null +++ b/tools/lab.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Pinned Java Edition acquisition/decompilation. All generated files stay in .local/.""" + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path, PurePosixPath +import re +import shutil +import struct +import subprocess +import sys +import tempfile +import urllib.request +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +LOCAL = ROOT / ".local" +LOCK = ROOT / "research-lock.json" + + +def write_json(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +def digest(path, algorithm): + with path.open("rb") as stream: + return hashlib.file_digest(stream, algorithm).hexdigest() + + +def download(url, destination, checksum, algorithm="sha1"): + """Never promote an unverified or partial download into the artifact cache.""" + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists() and digest(destination, algorithm) == checksum: + return destination + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=destination.parent, delete=False) as stream: + temporary = Path(stream.name) + request = urllib.request.Request(url, headers={"User-Agent": "minecraft-optimization-lab/0.1"}) + with urllib.request.urlopen(request, timeout=120) as response: + shutil.copyfileobj(response, stream) + actual = digest(temporary, algorithm) + if actual != checksum: + raise ValueError(f"Checksum mismatch for {destination.name}: expected {checksum}, got {actual}") + temporary.replace(destination) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return destination + + +def config(): + lock = json.loads(LOCK.read_text(encoding="utf-8")) + version = lock["minecraft"]["version"] + if not re.fullmatch(r"[A-Za-z0-9_.-]+", version) or version in {".", ".."}: + raise ValueError("Unsafe version identifier in lock file") + return lock, LOCAL / "minecraft" / version + + +def bundle_entries(archive, name): + for line in archive.read(name).decode("utf-8").splitlines(): + if not line.strip(): + continue + checksum, identifier, relative = line.split("\t") + path = PurePosixPath(relative) + if path.is_absolute() or ".." in path.parts or "\\" in relative: + raise ValueError(f"Unsafe bundled path: {relative}") + yield checksum, identifier, path + + +def extract_verified(archive, member, destination, checksum): + data = archive.read(member) + if hashlib.sha256(data).hexdigest() != checksum: + raise ValueError(f"Bundled artifact checksum mismatch: {member}") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(data) + + +def unpack_server(server, directory, version): + """Modern server downloads are launchers; decompile their inner game JAR.""" + with zipfile.ZipFile(server) as archive: + if "META-INF/versions.list" not in archive.namelist(): + return server, [] + versions = list(bundle_entries(archive, "META-INF/versions.list")) + if len(versions) != 1 or versions[0][1] != version: + raise ValueError("Expected one bundled server matching the pinned version") + checksum, _, relative = versions[0] + game = directory / "jars" / "server-game.jar" + extract_verified(archive, f"META-INF/versions/{relative}", game, checksum) + libraries = [] + for checksum, _, relative in bundle_entries(archive, "META-INF/libraries.list"): + destination = directory / "libraries" / "server" / relative + extract_verified(archive, f"META-INF/libraries/{relative}", destination, checksum) + libraries.append(destination) + return game, libraries + + +class ClassReader: + """Read SourceFile and method count; no Java text needs to be published.""" + + def __init__(self, data): + self.data = data + self.offset = 0 + + def take(self, count): + value = self.data[self.offset:self.offset + count] + if len(value) != count: + raise ValueError("Truncated class file") + self.offset += count + return value + + def number(self, count): + return int.from_bytes(self.take(count), "big") + + def attributes(self): + return [(self.number(2), self.take(self.number(4))) for _ in range(self.number(2))] + + def metadata(self): + if self.take(4) != b"\xca\xfe\xba\xbe": + raise ValueError("Invalid class file magic") + self.take(4) # Minor and major version. + pool = [None] * self.number(2) + sizes = {3: 4, 4: 4, 5: 8, 6: 8, 7: 2, 8: 2, 9: 4, 10: 4, + 11: 4, 12: 4, 15: 3, 16: 2, 17: 4, 18: 4, 19: 2, 20: 2} + i = 1 + while i < len(pool): + tag = self.number(1) + if tag == 1: + pool[i] = self.take(self.number(2)).decode("utf-8", errors="replace") + else: + if tag not in sizes: + raise ValueError(f"Unknown constant pool tag: {tag}") + self.take(sizes[tag]) + i += 2 if tag in {5, 6} else 1 + self.take(6) # Access, this class, super class. + self.take(self.number(2) * 2) # Interfaces. + for _ in range(self.number(2)): # Fields. + self.take(6) + self.attributes() + methods = self.number(2) + for _ in range(methods): + self.take(6) + self.attributes() + source = None + for name, data in self.attributes(): + if pool[name] == "SourceFile": + source = pool[struct.unpack(">H", data)[0]] + return {"source_file": source, "methods": methods} + + +def inventory(jar): + classes = [] + packages = Counter() + with zipfile.ZipFile(jar) as archive: + for item in archive.infolist(): + if not item.filename.endswith(".class"): + continue + metadata = ClassReader(archive.read(item)).metadata() + path = PurePosixPath(item.filename) + source = metadata["source_file"] or path.name.split("$")[0].removesuffix(".class") + ".java" + if PurePosixPath(source).name != source or "\\" in source: + raise ValueError(f"Unsafe SourceFile attribute in {item.filename}") + source_path = str(path.parent / source) + packages[str(path.parent)] += 1 + classes.append({"class": item.filename, "source": source_path, + "bytecode_bytes": item.file_size, "methods": metadata["methods"]}) + return {"jar_sha256": digest(jar, "sha256"), "class_count": len(classes), + "expected_source_files": sorted({c["source"] for c in classes}), + "method_count": sum(c["methods"] for c in classes), + "packages": dict(packages.most_common()), "classes": classes} + + +def prepare(): + lock, directory = config() + minecraft = lock["minecraft"] + manifest_path = download(minecraft["manifest_url"], directory / "manifest.json", + minecraft["manifest_sha1"]) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest["id"] != minecraft["version"]: + raise ValueError("Manifest version does not match lock file") + if "client_mappings" in manifest["downloads"] or "server_mappings" in manifest["downloads"]: + raise ValueError("This workflow targets unobfuscated releases. Older releases need a remapping stage.") + if manifest["javaVersion"]["majorVersion"] != minecraft["java_major"]: + raise ValueError("Manifest Java version does not match lock file") + jars = {} + for side in ["client", "server"]: + artifact = manifest["downloads"][side] + print(f"Preparing {side} {manifest['id']}...", flush=True) + jars[side] = download(artifact["url"], directory / "jars" / f"{side}.jar", artifact["sha1"]) + jars["server"], server_libraries = unpack_server(jars["server"], directory, manifest["id"]) + client_artifacts = [library["downloads"]["artifact"] for library in manifest["libraries"] + if "artifact" in library.get("downloads", {})] + + def client_library(artifact): + relative = PurePosixPath(artifact["path"]) + if relative.is_absolute() or ".." in relative.parts or "\\" in artifact["path"]: + raise ValueError("Unsafe library path") + return download(artifact["url"], directory / "libraries" / "client" / relative, artifact["sha1"]) + + with ThreadPoolExecutor(max_workers=4) as pool: + client_libraries = list(pool.map(client_library, client_artifacts)) + vf = lock["vineflower"] + download(vf["url"], LOCAL / "tools" / f"vineflower-{vf['version']}.jar", vf["sha256"], "sha256") + for side, jar in jars.items(): + data = inventory(jar) + write_json(directory / "reports" / f"{side}-inventory.json", data) + print(f"{side}: {data['class_count']} classes, {data['method_count']} methods", flush=True) + write_json(directory / "prepared.json", { + "version": manifest["id"], "jars": {k: str(v.relative_to(ROOT)) for k, v in jars.items()}, + "libraries": {"client": [str(p.relative_to(ROOT)) for p in client_libraries], + "server": [str(p.relative_to(ROOT)) for p in server_libraries]}, + }) + print(f"Artifacts: {directory}", flush=True) + + +def coverage(directory, side, source_root=None): + source_root = source_root or directory / "sources" / side + data = json.loads((directory / "reports" / f"{side}-inventory.json").read_text(encoding="utf-8")) + actual = {str(path.relative_to(source_root)) for path in source_root.rglob("*.java")} + expected = set(data["expected_source_files"]) + placeholders = [] + for path in source_root.rglob("*.java"): + contents = path.read_text(encoding="utf-8", errors="replace") + if re.search(r"\$VF:.*(?:couldn't|could not|failed)", contents, re.IGNORECASE): + placeholders.append(str(path.relative_to(source_root))) + return {"side": side, "input_classes": data["class_count"], + "expected_source_files": len(expected), "emitted_source_files": len(actual), + "missing_source_files": sorted(expected - actual), + "files_with_decompiler_error_markers": sorted(placeholders), + "coverage_note": "Inner classes share their SourceFile; file coverage does not prove method fidelity or recompilability."} + + +def decompile(args): + lock, directory = config() + prepared = json.loads((directory / "prepared.json").read_text(encoding="utf-8")) + vf = lock["vineflower"] + tool = LOCAL / "tools" / f"vineflower-{vf['version']}.jar" + if digest(tool, "sha256") != vf["sha256"]: + raise ValueError("Decompiler checksum changed; rerun prepare") + java = shutil.which(args.java) + if not java: + raise ValueError("Java not found; install JDK 25 or provide --java /path/to/java") + version = subprocess.run([java, "-version"], capture_output=True, text=True, check=True) + match = re.search(r'version "(\d+)', version.stderr + version.stdout) + if not match or int(match[1]) < lock["minecraft"]["java_major"]: + raise ValueError(f"JDK {lock['minecraft']['java_major']} or newer is required") + sides = ["client", "server"] if args.side == "both" else [args.side] + incomplete = False + for side in sides: + destination = directory / "sources" / side + jar = ROOT / prepared["jars"][side] + data = json.loads((directory / "reports" / f"{side}-inventory.json").read_text(encoding="utf-8")) + if digest(jar, "sha256") != data["jar_sha256"]: + raise ValueError(f"{side} input changed; rerun prepare") + if destination.exists(): + raise ValueError(f"Output already exists: {destination}. Move it aside before rerunning; it may contain your edits.") + destination.parent.mkdir(parents=True, exist_ok=True) + stage = Path(tempfile.mkdtemp(prefix=f"{side}-incomplete-", dir=destination.parent)) + command = [java, f"-Xmx{args.heap}", "-jar", str(tool), "--folder", + "--skip-extra-files=1", "--log-level=warn", f"--thread-count={args.threads}"] + command += [f"--add-external={ROOT / p}" for p in prepared["libraries"][side]] + command += [str(jar), str(stage)] + log = directory / "reports" / f"{side}-decompiler.log" + print(f"Decompiling {side}; log: {log}", flush=True) + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, stdout=stream, stderr=subprocess.STDOUT) + if result.returncode: + raise ValueError(f"Decompiler exited {result.returncode}. Partial output retained at {stage}; see {log}") + stage.rename(destination) + report = coverage(directory, side) + report["java_version"] = (version.stderr + version.stdout).strip() + report["vineflower_version"] = vf["version"] + report["command"] = command + write_json(directory / "reports" / f"{side}-coverage.json", report) + print(json.dumps({k: v for k, v in report.items() if k not in {"command", "java_version"}}, indent=2), flush=True) + incomplete |= bool(report["missing_source_files"] or report["files_with_decompiler_error_markers"]) + return 2 if incomplete else 0 + + +def status(): + _, directory = config() + incomplete = False + for side in ["client", "server"]: + report = coverage(directory, side) + print(json.dumps(report, indent=2)) + incomplete |= bool(report["missing_source_files"] or report["files_with_decompiler_error_markers"]) + return 2 if incomplete else 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("prepare", help="Download pinned official JARs, libraries, and decompiler; inventory classes") + decompiler = commands.add_parser("decompile", help="Decompile locally and check source-file coverage") + decompiler.add_argument("--side", choices=["client", "server", "both"], default="both") + decompiler.add_argument("--java", default="java") + decompiler.add_argument("--threads", type=int, default=4) + decompiler.add_argument("--heap", default="6G") + commands.add_parser("status", help="Check source coverage and decompiler error markers") + args = parser.parse_args() + if getattr(args, "threads", 1) < 1: + parser.error("--threads must be positive") + if not re.fullmatch(r"[1-9][0-9]*[MGmg]", getattr(args, "heap", "6G")): + parser.error("--heap must look like 6G or 512M") + try: + if args.command == "prepare": + prepare() + return 0 + if args.command == "decompile": + return decompile(args) + return status() + except (OSError, ValueError, KeyError, subprocess.CalledProcessError, zipfile.BadZipFile) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main())