diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ccf4654b..87949a259 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,12 +8,7 @@ permissions: contents: read jobs: verify: - strategy: - fail-fast: false - matrix: - # Linux only. macOS only when started by hand: gh workflow run checks --ref . - os: ${{ fromJSON(github.event_name == 'workflow_dispatch' && '["ubuntu-latest", "macos-26"]' || '["ubuntu-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -35,6 +30,20 @@ jobs: cargo +stable clippy --workspace --all-targets -- -D warnings - name: Test Rust core and voxel kernel correctness run: cargo +stable test --workspace + # Horizon never downloads the game. These jobs need a self-hosted Linux runner + # (label horizon-game) holding the maintainer's own copy: the official launcher's + # game directory in HORIZON_MINECRAFT_DIR and the pinned server jar in + # HORIZON_SERVER_JAR. Start them by hand: gh workflow run checks --ref . + vanilla-parity: + if: github.event_name == 'workflow_dispatch' + runs-on: [self-hosted, linux, horizon-game] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install Rust toolchain + run: rustup toolchain install stable --profile minimal - uses: actions/setup-java@v4 with: distribution: temurin @@ -374,12 +383,8 @@ jobs: RUSTUP_TOOLCHAIN: stable run: python tools/test_float_log_parity.py portable-mod: - strategy: - fail-fast: false - matrix: - # Linux only. macOS only when started by hand: gh workflow run checks --ref . - os: ${{ fromJSON(github.event_name == 'workflow_dispatch' && '["ubuntu-latest", "macos-26"]' || '["ubuntu-latest"]') }} - runs-on: ${{ matrix.os }} + if: github.event_name == 'workflow_dispatch' + runs-on: [self-hosted, linux, horizon-game] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -466,7 +471,7 @@ jobs: - name: Upload original mod and pack manifests only uses: actions/upload-artifact@v4 with: - name: horizon-${{ matrix.os }} + name: horizon-linux path: | dist/*.mrpack dist/horizon-*.jar diff --git a/README.md b/README.md index 9498acab8..59ae1d232 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,21 @@ memory; the failures are new harnesses that still find divergences. 5. The missing commands, the tab list and world border rendering. 6. Test LAN hosting with a vanilla client, and upgrade saves from older DataVersions. +### Bring your own copy + +Horizon contains no game files and never downloads them. It builds its resources from **your own copy** of Minecraft Java Edition 26.3, verified against the checksums in `research-lock.json`: + +1. **Client:** install Java Edition with the official launcher and play 26.3 once. Horizon reads `versions/26.3/`, `libraries/` and `assets/` from the launcher's game directory (`~/.minecraft` on Linux). If yours lives elsewhere, pass `--minecraft-dir` or set `HORIZON_MINECRAFT_DIR`. +2. **Server jar:** download the 26.3 server jar yourself, from [minecraft.net/download/server](https://www.minecraft.net/en-us/download/server) while 26.3 is the current release, or otherwise from the `downloads.server.url` in your launcher's `versions/26.3/26.3.json`. Pass it with `--server-jar` or set `HORIZON_SERVER_JAR`. It is only needed the first time. + ```sh -python3 tools/prepare_native_client.py --prepare # needs JDK 25; extracts private vanilla resources -python3 tools/fetch_sounds.py # optional: vanilla sounds +python3 tools/prepare_native_client.py --prepare --server-jar ~/Downloads/server.jar # needs JDK 25 +python3 tools/fetch_sounds.py # optional: sounds from your launcher install cargo run --release -p horizon-client ``` +Everything prepared lands in the gitignored `.local/`. Never commit it or share it. + **Target machine:** the Rust rewrite targets one machine, **ludodesktop**: Ubuntu 26.04, Intel i7-13700KF, NVIDIA RTX 4080, GNOME on Wayland. The final deliverable is a playable native Linux build validated there. macOS (including the M2 Pro Mac) and Windows are out of scope. Track implementation and acceptance criteria in [Rust rewrite status](docs/rust-parity.md). **Java performance packs (no longer developed):** the earlier track of pinned Java performance packs, distant terrain, an original frame/tick recorder and client/server starter downloads. No release downloads are published; build the packs and starters locally with `python3 tools/build_release.py` (see [Build and run](#build-and-run)). They are no longer a development target. @@ -236,18 +245,18 @@ this launcher correction is not a measured graphics performance improvement. ## Local game inspection -Game JARs, decompiled Java, libraries, inventories, worlds and logs live **inside this checkout** under gitignored **`.local/`**. No game code is in Git history or release downloads. Fresh clones regenerate the inspection workspace from official downloads; Microsoft credentials are not needed for this step. +Game JARs, decompiled Java, libraries, inventories, worlds and logs live **inside this checkout** under gitignored **`.local/`**. No game code is in Git history or release downloads. Fresh clones rebuild the inspection workspace from your own copy of the game (see [Bring your own copy](#bring-your-own-copy)); nothing of the game is downloaded. ```sh git clone https://github.com/ludovic111/horizon.git cd horizon git config core.hooksPath .githooks -python3 tools/lab.py prepare +python3 tools/lab.py prepare --server-jar ~/Downloads/server.jar python3 tools/lab.py decompile python3 tools/lab.py status ``` -`research-lock.json` pins the official manifest and Vineflower. The inner server JAR is extracted and checked; client/server libraries provide decompiler context. Output: `.local/minecraft/26.3/sources/{client,server}/` and coverage/inventory reports. Current inventories cover 11,383 client classes and 7,762 server classes, producing 7,301/5,037 Java files. See [decompilation status](docs/decompilation-status.md). +`research-lock.json` pins the official manifest, the client and server jar checksums, and Vineflower. The inner server JAR is extracted and checked; client/server libraries provide decompiler context. Output: `.local/minecraft/26.3/sources/{client,server}/` and coverage/inventory reports. Current inventories cover 11,383 client classes and 7,762 server classes, producing 7,301/5,037 Java files. See [decompilation status](docs/decompilation-status.md). Class-file SourceFile metadata covers inner classes, which often share a source file. Coverage does not prove semantic equivalence or recompilability; original bytecode remains the reference. Completed source directories are not overwritten automatically. Older obfuscated releases require a separate remapping stage. @@ -329,7 +338,7 @@ python3 tools/check_public_tree.py python3 tools/check_public_tree.py --history ``` -CI checks offline tools/Rust and compiles the original mod/builds packs on Linux and Apple Silicon macOS. Game dependencies download to ephemeral `.local/`; CI uploads only our original JAR and packs. Compilation is not Mac GPU gameplay testing. Publication checks allow Java only under our original `dev/horizonlab` namespace; review diffs as well. +CI runs the offline checks (publication, Python tooling, Rust format, lint and tests) on GitHub's Linux runners. The vanilla comparisons and the original mod build need the game, which Horizon never downloads, so they run only when started by hand on a self-hosted runner holding the maintainer's own copy; CI uploads only our original JAR and packs. Publication checks allow Java only under our original `dev/horizonlab` namespace; review diffs as well. [MIT](LICENSE) covers only our original tracked work. Dependencies retain their licenses. Mojang [removed obfuscation](https://www.minecraft.net/en-us/article/removing-obfuscation-in-java-edition) without changing [Minecraft's EULA](https://www.minecraft.net/en-us/eula). Decompiled game source remains local; published code implements independent tooling and instrumentation. diff --git a/docs/native-client.md b/docs/native-client.md index 1bf41b10f..62c1d8b48 100644 --- a/docs/native-client.md +++ b/docs/native-client.md @@ -16,14 +16,17 @@ most structures and mobs, and several containers are still missing. See ## Run from a checkout -Use current stable Rust (1.90+), Python 3.10+ and JDK 25. Resource preparation -verifies the pinned game artifacts, runs vanilla's data generator and small -exporters against the reference jars, and extracts the jar's assets and data -into ignored `.local/horizon-client/26.3/`. Sounds come from Mojang's asset -index (about 380 MB). Prepared resources are private; never commit them. +Use current stable Rust (1.90+), Python 3.10+ and JDK 25. Horizon never +downloads the game: resource preparation imports **your own copy** (the 26.3 +client from your official launcher install, and the 26.3 server jar you +downloaded yourself; see [Bring your own copy](../README.md#bring-your-own-copy)), +verifies it against the pinned checksums, runs vanilla's data generator and small +exporters against those jars, and extracts the jar's assets and data into ignored +`.local/horizon-client/26.3/`. Sounds (about 380 MB) are copied from your +launcher's asset store. Prepared resources are private; never commit them. ```sh -python3 tools/prepare_native_client.py --prepare +python3 tools/prepare_native_client.py --prepare --server-jar ~/Downloads/server.jar python3 tools/fetch_sounds.py cargo run --release -p horizon-client -- --render-distance 12 ``` diff --git a/docs/server-quickstart.txt b/docs/server-quickstart.txt index e4488145b..84c325884 100644 --- a/docs/server-quickstart.txt +++ b/docs/server-quickstart.txt @@ -1,16 +1,18 @@ Horizon server @PACK_VERSION@ — Minecraft 26.3, Fabric 0.19.5, Java 25 Install Python 3.10+ and native Java 25. Extract this ZIP and open a terminal here. +This starter never downloads the game: download the Minecraft 26.3 server jar +yourself (https://www.minecraft.net/download/server while 26.3 is current), then: - python3 tools/server.py prepare + python3 tools/server.py prepare --server-jar PATH/TO/server.jar Read Minecraft's EULA at https://www.minecraft.net/eula. Create/edit .local/server/optimized/eula.txt to contain eula=true if you agree, then: python3 tools/server.py run --heap 4G -Windows: use python in place of python3 if needed. Server downloads and assets -stay in .local. No game code, third-party mod JARs or credentials are bundled. +Windows: use python in place of python3 if needed. Your server jar, mods and +assets stay in .local. No game code, third-party mod JARs or credentials are bundled. Lithium and FerriteCore optimize the original simulation/storage; tick rate and vanilla gameplay rules are retained. No Paper-style gameplay configuration. diff --git a/research-lock.json b/research-lock.json index 1afd0035b..ba558485a 100644 --- a/research-lock.json +++ b/research-lock.json @@ -1,8 +1,9 @@ { "minecraft": { "version": "26.3", - "manifest_url": "https://piston-meta.mojang.com/v1/packages/4fe1aa1ef8da1cb95c5bad1fb98890ca56dd8ca3/26.3.json", "manifest_sha1": "4fe1aa1ef8da1cb95c5bad1fb98890ca56dd8ca3", + "client_sha1": "e877b6a07acd633fb3bb475002175cec036e7b87", + "server_sha1": "33680f5f2ac32864d6d7cf5e56a705fdb3e05f4c", "java_major": 25 }, "vineflower": { diff --git a/tests/test_lab.py b/tests/test_lab.py index eddd5835d..5c8940dbc 100644 --- a/tests/test_lab.py +++ b/tests/test_lab.py @@ -41,6 +41,56 @@ class ArtifactTests(unittest.TestCase): lab.download("https://example.invalid", destination, hashlib.sha1(b"verified").hexdigest()) request.assert_not_called() + def test_import_copies_only_the_pinned_file_from_the_users_install(self): + with tempfile.TemporaryDirectory() as temp: + source, destination = Path(temp) / "own.jar", Path(temp) / "cache" / "client.jar" + source.write_bytes(b"modified") + with self.assertRaisesRegex(ValueError, "not the pinned file"): + lab.import_copy(source, destination, hashlib.sha1(b"vanilla").hexdigest()) + self.assertFalse(destination.exists()) + source.write_bytes(b"vanilla") + lab.import_copy(source, destination, hashlib.sha1(b"vanilla").hexdigest()) + self.assertEqual(destination.read_bytes(), b"vanilla") + with self.assertRaisesRegex(FileNotFoundError, "official launcher"): + lab.import_copy(Path(temp) / "absent.jar", Path(temp) / "other.jar", "0" * 40, + hint=lab.LAUNCHER_HINT.format(version="26.3")) + + def test_game_files_come_from_the_launcher_install_without_network(self): + client, server = b"client", b"server" + lock = {"minecraft": {"version": "26.3", "manifest_sha1": "0" * 40, "java_major": 25, + "client_sha1": hashlib.sha1(client).hexdigest(), + "server_sha1": hashlib.sha1(server).hexdigest()}} + manifest = {"id": "26.3", "javaVersion": {"majorVersion": 25}, + "downloads": {"client": {"sha1": lock["minecraft"]["client_sha1"]}, + "server": {"sha1": lock["minecraft"]["server_sha1"]}}} + with tempfile.TemporaryDirectory() as temp, mock.patch("urllib.request.urlopen") as request, \ + mock.patch.dict("os.environ", {}, clear=False) as environment: + environment.pop("HORIZON_SERVER_JAR", None) + launcher, cache = Path(temp) / "minecraft", Path(temp) / "cache" + version = launcher / "versions" / "26.3" + version.mkdir(parents=True) + (version / "26.3.json").write_text(json.dumps(manifest)) + (version / "26.3.jar").write_bytes(client) + loaded, jar = lab.import_client(lock, cache, launcher) + self.assertEqual((loaded["id"], jar.read_bytes()), ("26.3", client)) + with self.assertRaisesRegex(FileNotFoundError, "server jar yourself"): + lab.import_server(lock, cache) + (Path(temp) / "server.jar").write_bytes(server) + self.assertEqual(lab.import_server(lock, cache, Path(temp) / "server.jar").read_bytes(), server) + self.assertEqual(lab.import_server(lock, cache).read_bytes(), server) + request.assert_not_called() + + def test_launcher_manifest_for_another_release_is_rejected(self): + lock = {"minecraft": {"version": "26.3", "manifest_sha1": "0" * 40, "java_major": 25, + "client_sha1": "1" * 40, "server_sha1": "2" * 40}} + with tempfile.TemporaryDirectory() as temp: + version = Path(temp) / "versions" / "26.3" + version.mkdir(parents=True) + (version / "26.3.json").write_text(json.dumps({"id": "26.3", "downloads": { + "client": {"sha1": "3" * 40}, "server": {"sha1": "2" * 40}}})) + with self.assertRaisesRegex(ValueError, "pinned 26.3 release"): + lab.import_manifest(lock, Path(temp) / "cache", Path(temp)) + def test_sourcefile_metadata_covers_inner_classes(self): with tempfile.TemporaryDirectory() as temp: path = Path(temp) / "game.jar" diff --git a/tools/fetch_client_assets.py b/tools/fetch_client_assets.py index 02373cd91..513925374 100644 --- a/tools/fetch_client_assets.py +++ b/tools/fetch_client_assets.py @@ -1,23 +1,15 @@ #!/usr/bin/env python3 -"""Fetch the pinned version's indexed client assets that are not in the game -jar: language files and the language list (pack.mcmeta), the unifont glyph -archives and their font definitions, and the built-in Programmer Art and High -Contrast resource packs. Every object's SHA-1 is verified. Files go to the -private client resource directory (or --target) and are never committed.""" +"""Copy the pinned version's indexed client assets that are not in the game +jar from your own official launcher install: language files and the language +list (pack.mcmeta), the unifont glyph archives and their font definitions, and +the built-in Programmer Art and High Contrast resource packs. Every object's +SHA-1 is verified. Nothing is downloaded; files go to the private client +resource directory (or --target) and are never committed.""" import argparse -import concurrent.futures -import hashlib -import json from pathlib import Path -import urllib.request +import shutil -ROOT = Path(__file__).resolve().parent.parent -LOCAL = ROOT / '.local' - - -def fetch(url): - with urllib.request.urlopen(url, timeout=120) as response: - return response.read() +import lab def wanted(name): @@ -29,40 +21,25 @@ def wanted(name): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--target', type=Path, default=LOCAL / 'horizon-client/26.3', + parser.add_argument('--target', type=Path, default=lab.LOCAL / 'horizon-client/26.3', help='prepared resource directory (default: .local/horizon-client/26.3)') + parser.add_argument('--minecraft-dir', type=Path, + help="official launcher game directory (default: the launcher's own)") args = parser.parse_args() - manifest = json.loads((LOCAL / 'minecraft/26.3/manifest.json').read_text()) - index_info = manifest['assetIndex'] + lock, directory = lab.config() + manifest = lab.import_manifest(lock, directory, args.minecraft_dir) + store, index = lab.launcher_assets(manifest, args.minecraft_dir) target = args.target - index_path = target / 'asset-index.json' - if not index_path.exists() or hashlib.sha1(index_path.read_bytes()).hexdigest() != index_info['sha1']: - data = fetch(index_info['url']) - if hashlib.sha1(data).hexdigest() != index_info['sha1']: - raise ValueError('asset index checksum mismatch') - target.mkdir(parents=True, exist_ok=True) - index_path.write_bytes(data) - objects = json.loads(index_path.read_text())['objects'] + target.mkdir(parents=True, exist_ok=True) + shutil.copyfile(index, target / 'asset-index.json') + objects = lab.json.loads(index.read_text())['objects'] selected = {name: info for name, info in objects.items() if wanted(name)} - - def download(item): - name, info = item - path = target / 'assets' / name - if path.exists() and hashlib.sha1(path.read_bytes()).hexdigest() == info['hash']: - return 0 - h = info['hash'] - data = fetch(f'https://resources.download.minecraft.net/{h[:2]}/{h}') - if hashlib.sha1(data).hexdigest() != h: - raise ValueError(f'checksum mismatch for {name}') - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - return len(data) - total = 0 - with concurrent.futures.ThreadPoolExecutor(16) as pool: - for size in pool.map(download, selected.items()): - total += size - print(f'Client assets ready: {len(selected)} files, {total / 1e6:.1f} MB downloaded') + for name, info in selected.items(): + source = lab.launcher_object(store, info['hash']) + lab.import_copy(source, target / 'assets' / name, info['hash']) + total += info['size'] + print(f'Client assets ready: {len(selected)} files, {total / 1e6:.1f} MB') if __name__ == '__main__': diff --git a/tools/fetch_sounds.py b/tools/fetch_sounds.py index 394c03db6..7349b3eb9 100644 --- a/tools/fetch_sounds.py +++ b/tools/fetch_sounds.py @@ -1,54 +1,35 @@ #!/usr/bin/env python3 -"""Fetch the pinned version's sound assets (sounds.json and .ogg files) from -Mojang's asset index into the private client resource directory, verifying -every object's SHA-1. The files stay in ignored .local/ and are never committed.""" -import concurrent.futures -import hashlib -import json +"""Copy the pinned version's sound assets (sounds.json and .ogg files) from your +own official launcher install into the private client resource directory, +verifying every object's SHA-1. Nothing is downloaded; the files stay in +ignored .local/ and are never committed.""" +import argparse from pathlib import Path -import urllib.request +import shutil -ROOT = Path(__file__).resolve().parent.parent -LOCAL = ROOT / '.local' - - -def fetch(url): - with urllib.request.urlopen(url, timeout=120) as response: - return response.read() +import lab def main(): - manifest = json.loads((LOCAL / 'minecraft/26.3/manifest.json').read_text()) - index_info = manifest['assetIndex'] - target = LOCAL / 'horizon-client/26.3' - index_path = target / 'asset-index.json' - if not index_path.exists() or hashlib.sha1(index_path.read_bytes()).hexdigest() != index_info['sha1']: - data = fetch(index_info['url']) - if hashlib.sha1(data).hexdigest() != index_info['sha1']: - raise ValueError('asset index checksum mismatch') - index_path.write_bytes(data) - objects = json.loads(index_path.read_text())['objects'] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--minecraft-dir', type=Path, + help="official launcher game directory (default: the launcher's own)") + args = parser.parse_args() + lock, directory = lab.config() + manifest = lab.import_manifest(lock, directory, args.minecraft_dir) + store, index = lab.launcher_assets(manifest, args.minecraft_dir) + target = lab.LOCAL / 'horizon-client' / manifest['id'] + target.mkdir(parents=True, exist_ok=True) + shutil.copyfile(index, target / 'asset-index.json') + objects = lab.json.loads(index.read_text())['objects'] wanted = {name: info for name, info in objects.items() if name == 'minecraft/sounds.json' or name.startswith('minecraft/sounds/')} - - def download(item): - name, info = item - path = target / 'assets' / name - if path.exists() and hashlib.sha1(path.read_bytes()).hexdigest() == info['hash']: - return 0 - h = info['hash'] - data = fetch(f'https://resources.download.minecraft.net/{h[:2]}/{h}') - if hashlib.sha1(data).hexdigest() != h: - raise ValueError(f'checksum mismatch for {name}') - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - return len(data) - total = 0 - with concurrent.futures.ThreadPoolExecutor(16) as pool: - for size in pool.map(download, wanted.items()): - total += size - print(f'Sound assets ready: {len(wanted)} files, {total / 1e6:.1f} MB downloaded') + for name, info in wanted.items(): + source = lab.launcher_object(store, info['hash']) + lab.import_copy(source, target / 'assets' / name, info['hash']) + total += info['size'] + print(f'Sound assets ready: {len(wanted)} files, {total / 1e6:.1f} MB') if __name__ == '__main__': diff --git a/tools/lab.py b/tools/lab.py index 77d2b8a87..ae208a528 100644 --- a/tools/lab.py +++ b/tools/lab.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -"""Pinned Java Edition acquisition/decompilation. All generated files stay in .local/.""" +"""Pinned Java Edition import/decompilation. Game files come from the user's own +copy (their official launcher install and the server jar they downloaded); all +generated files stay in .local/.""" import argparse from collections import Counter -from concurrent.futures import ThreadPoolExecutor import hashlib import json +import os from pathlib import Path, PurePosixPath import re import shutil @@ -53,6 +55,124 @@ def download(url, destination, checksum, algorithm="sha1"): return destination +LAUNCHER_HINT = ("Horizon does not download the game: install Minecraft Java Edition with the official " + "launcher, play {version} once, then rerun. If the launcher uses another game directory, " + "pass --minecraft-dir or set HORIZON_MINECRAFT_DIR.") +SERVER_HINT = ("Horizon does not download the game: download the {version} server jar yourself (minecraft.net/download/server " + "while {version} is the current release, otherwise the downloads.server.url in your launcher's " + "versions/{version}/{version}.json), then pass --server-jar PATH or set HORIZON_SERVER_JAR.") + + +def minecraft_dir(override=None): + """The official launcher's game directory, which holds the user's own copy of the game.""" + if override or os.environ.get("HORIZON_MINECRAFT_DIR"): + return Path(override or os.environ["HORIZON_MINECRAFT_DIR"]).expanduser() + if sys.platform == "darwin": + return Path.home() / "Library" / "Application Support" / "minecraft" + if sys.platform == "win32": + return Path(os.environ.get("APPDATA", Path.home())) / ".minecraft" + return Path.home() / ".minecraft" + + +def import_copy(source, destination, checksum, algorithm="sha1", hint=""): + """Copy one file from the user's own install, verified like a download.""" + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists() and digest(destination, algorithm) == checksum: + return destination + if not source.is_file(): + raise FileNotFoundError(f"Missing {source}. {hint}".strip()) + actual = digest(source, algorithm) + if actual != checksum: + raise ValueError(f"{source} is not the pinned file: expected {algorithm} {checksum}, got {actual}") + with tempfile.NamedTemporaryFile(dir=destination.parent, delete=False) as stream: + temporary = Path(stream.name) + try: + shutil.copyfile(source, temporary) + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + return destination + + +def import_manifest(lock, directory, minecraft=None): + """The pinned version JSON, from the launcher's versions/ directory.""" + game = lock["minecraft"] + version = game["version"] + destination = directory / "manifest.json" + source = minecraft_dir(minecraft) / "versions" / version / f"{version}.json" + if not (destination.exists() and digest(destination, "sha1") == game["manifest_sha1"]): + if not source.is_file(): + raise FileNotFoundError(f"Missing {source}. {LAUNCHER_HINT.format(version=version)}") + manifest = json.loads(source.read_text(encoding="utf-8")) + downloads = manifest.get("downloads", {}) + if (manifest.get("id") != version + or downloads.get("client", {}).get("sha1") != game["client_sha1"] + or downloads.get("server", {}).get("sha1") != game["server_sha1"]): + raise ValueError(f"{source} does not describe the pinned {version} release") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + manifest = json.loads(destination.read_text(encoding="utf-8")) + if manifest["javaVersion"]["majorVersion"] != game["java_major"]: + raise ValueError("Manifest Java version does not match lock file") + return manifest + + +def import_client(lock, directory, minecraft=None): + """The user's client jar from their launcher install, checked against the lock.""" + version = lock["minecraft"]["version"] + manifest = import_manifest(lock, directory, minecraft) + source = minecraft_dir(minecraft) / "versions" / version / f"{version}.jar" + jar = import_copy(source, directory / "jars" / "client.jar", lock["minecraft"]["client_sha1"], + hint=LAUNCHER_HINT.format(version=version)) + return manifest, jar + + +def import_server(lock, directory, server_jar=None): + """The server jar the user downloaded themselves, checked against the lock.""" + version = lock["minecraft"]["version"] + destination = directory / "jars" / "server.jar" + checksum = lock["minecraft"]["server_sha1"] + source = server_jar or os.environ.get("HORIZON_SERVER_JAR") + hint = SERVER_HINT.format(version=version) + if source is None: + if destination.exists() and digest(destination, "sha1") == checksum: + return destination + raise FileNotFoundError(f"No {version} server jar. {hint}") + return import_copy(Path(source).expanduser(), destination, checksum, hint=hint) + + +def import_library(artifact, destination_root, minecraft=None): + """One library jar from the launcher's libraries/ directory.""" + relative = PurePosixPath(artifact["path"]) + if relative.is_absolute() or ".." in relative.parts or "\\" in artifact["path"]: + raise ValueError("Unsafe library path") + source = minecraft_dir(minecraft) / "libraries" / relative + hint = LAUNCHER_HINT.format(version=json.loads(LOCK.read_text(encoding="utf-8"))["minecraft"]["version"]) + return import_copy(source, destination_root / relative, artifact["sha1"], hint=hint) + + +def launcher_assets(manifest, minecraft=None): + """The launcher's asset directory and the pinned version's verified asset index.""" + assets = minecraft_dir(minecraft) / "assets" + info = manifest["assetIndex"] + index = assets / "indexes" / f"{info['id']}.json" + if not index.is_file(): + raise FileNotFoundError(f"Missing {index}. {LAUNCHER_HINT.format(version=manifest['id'])}") + if digest(index, "sha1") != info["sha1"]: + raise ValueError(f"{index} is not the pinned asset index") + return assets, index + + +def launcher_object(assets, checksum): + """One verified object from the launcher's asset store.""" + path = assets / "objects" / checksum[:2] / checksum + if not path.is_file(): + raise FileNotFoundError(f"Missing asset object {path}. Play the pinned version once in the official launcher.") + if digest(path, "sha1") != checksum: + raise ValueError(f"Corrupt asset object {path}; let the launcher repair it") + return path + + def config(): lock = json.loads(LOCK.read_text(encoding="utf-8")) version = lock["minecraft"]["version"] @@ -174,35 +294,18 @@ def inventory(jar): "packages": dict(packages.most_common()), "classes": classes} -def prepare(): +def prepare(minecraft=None, server_jar=None): lock, directory = config() - minecraft = lock["minecraft"] - manifest_path = download(minecraft["manifest_url"], directory / "manifest.json", - minecraft["manifest_sha1"]) - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - if manifest["id"] != minecraft["version"]: - raise ValueError("Manifest version does not match lock file") + manifest, client = import_client(lock, directory, minecraft) if "client_mappings" in manifest["downloads"] or "server_mappings" in manifest["downloads"]: raise ValueError("This workflow targets unobfuscated releases. Older releases need a remapping stage.") - if manifest["javaVersion"]["majorVersion"] != minecraft["java_major"]: - raise ValueError("Manifest Java version does not match lock file") - jars = {} - for side in ["client", "server"]: - artifact = manifest["downloads"][side] - print(f"Preparing {side} {manifest['id']}...", flush=True) - jars[side] = download(artifact["url"], directory / "jars" / f"{side}.jar", artifact["sha1"]) + print(f"Imported client {manifest['id']} from {minecraft_dir(minecraft)}", flush=True) + jars = {"client": client, "server": import_server(lock, directory, server_jar)} jars["server"], server_libraries = unpack_server(jars["server"], directory, manifest["id"]) client_artifacts = [library["downloads"]["artifact"] for library in manifest["libraries"] if "artifact" in library.get("downloads", {})] - - def client_library(artifact): - relative = PurePosixPath(artifact["path"]) - if relative.is_absolute() or ".." in relative.parts or "\\" in artifact["path"]: - raise ValueError("Unsafe library path") - return download(artifact["url"], directory / "libraries" / "client" / relative, artifact["sha1"]) - - with ThreadPoolExecutor(max_workers=4) as pool: - client_libraries = list(pool.map(client_library, client_artifacts)) + client_libraries = [import_library(artifact, directory / "libraries" / "client", minecraft) + for artifact in client_artifacts] vf = lock["vineflower"] download(vf["url"], LOCAL / "tools" / f"vineflower-{vf['version']}.jar", vf["sha256"], "sha256") for side, jar in jars.items(): @@ -291,10 +394,17 @@ def status(): return 2 if incomplete else 0 +def add_game_arguments(parser): + parser.add_argument("--minecraft-dir", type=Path, + help="official launcher game directory with your installed copy (default: the launcher's own)") + parser.add_argument("--server-jar", type=Path, help="the pinned server jar you downloaded yourself") + + def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) - commands.add_parser("prepare", help="Download pinned official JARs, libraries, and decompiler; inventory classes") + preparer = commands.add_parser("prepare", help="Import your own pinned game files, fetch the decompiler; inventory classes") + add_game_arguments(preparer) decompiler = commands.add_parser("decompile", help="Decompile locally and check source-file coverage") decompiler.add_argument("--side", choices=["client", "server", "both"], default="both") decompiler.add_argument("--java", default="java") @@ -308,7 +418,7 @@ def main(): parser.error("--heap must look like 6G or 512M") try: if args.command == "prepare": - prepare() + prepare(args.minecraft_dir, args.server_jar) return 0 if args.command == "decompile": return decompile(args) diff --git a/tools/prepare_native_client.py b/tools/prepare_native_client.py index a44bd4f47..6fb0a049b 100644 --- a/tools/prepare_native_client.py +++ b/tools/prepare_native_client.py @@ -141,10 +141,12 @@ def prepare_worldgen(root, jars, local, libraries=None): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--prepare', action='store_true', help='Fetch pinned reference artifacts first') + parser.add_argument('--prepare', action='store_true', + help='Import the pinned game files from your own copy first') + lab.add_game_arguments(parser) args = parser.parse_args() if args.prepare: - lab.prepare() + lab.prepare(args.minecraft_dir, args.server_jar) lock, jars, libraries = base.verify_artifacts() root = lab.LOCAL / 'horizon-client' / lock['minecraft']['version'] root.mkdir(parents=True, exist_ok=True) diff --git a/tools/runtime.py b/tools/runtime.py index 1a40378c8..71d55bf94 100644 --- a/tools/runtime.py +++ b/tools/runtime.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Prepare an isolated, authenticated-by-launcher game runtime; local smoke tests use demo mode.""" +"""Prepare an isolated game runtime from your own launcher install; local smoke tests use demo mode.""" from concurrent.futures import ThreadPoolExecutor import argparse @@ -120,14 +120,10 @@ def native_library_matches(library, architecture=None): def prepare_client(assets=True): + """Classpath and natives from the user's own launcher install; nothing of the game is downloaded.""" lock, directory = lab.config() - game = lock["minecraft"] - manifest_file = fetch(game["manifest_url"], directory / "manifest.json", game["manifest_sha1"]) - manifest = json.loads(manifest_file.read_text()) - if manifest["id"] != game["version"] or manifest["javaVersion"]["majorVersion"] != game["java_major"]: - raise ValueError("Game manifest does not match research lock") - artifact = manifest["downloads"]["client"] - jars = [fetch(artifact["url"], directory / "jars/client.jar", artifact["sha1"])] + manifest, client = lab.import_client(lock, directory) + jars = [client] natives = CACHE / "natives" natives.mkdir(parents=True, exist_ok=True) for library in manifest["libraries"]: @@ -136,7 +132,7 @@ def prepare_client(assets=True): artifact = library.get("downloads", {}).get("artifact") if not artifact: continue - jar = fetch(artifact["url"], directory / "libraries/client" / artifact["path"], artifact["sha1"]) + jar = lab.import_library(artifact, directory / "libraries/client") jars.append(jar) with zipfile.ZipFile(jar) as archive: for name in archive.namelist(): @@ -144,16 +140,11 @@ def prepare_client(assets=True): (natives / Path(name).name).write_bytes(archive.read(name)) jars.extend(fabric_libraries()) if assets: - info = manifest["assetIndex"] - index = fetch(info["url"], CACHE / "assets/indexes" / f"{info['id']}.json", info["sha1"]) - objects = json.loads(index.read_text())["objects"] - unique = {v["hash"] for v in objects.values()} - def asset(checksum): - return fetch(f"https://resources.download.minecraft.net/{checksum[:2]}/{checksum}", - CACHE / "assets/objects" / checksum[:2] / checksum, checksum) - print(f"Verifying/downloading {len(unique)} unique game assets...", flush=True) - with ThreadPoolExecutor(max_workers=12) as pool: - list(pool.map(asset, sorted(unique))) + store, index = lab.launcher_assets(manifest) + unique = {v["hash"] for v in json.loads(index.read_text())["objects"].values()} + print(f"Verifying {len(unique)} game assets in {store}...", flush=True) + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(lambda checksum: lab.launcher_object(store, checksum), sorted(unique))) return manifest, jars, natives @@ -179,7 +170,7 @@ def main(): "-Dfabric.log.disableAnsi=true", "-cp", os.pathsep.join(map(str, jars)), "net.fabricmc.loader.impl.launch.knot.KnotClient", "--username", "HorizonDemo", "--version", manifest["id"], "--gameDir", str(instance), - "--assetsDir", str(CACHE / "assets"), "--assetIndex", manifest["assetIndex"]["id"], + "--assetsDir", str(lab.minecraft_dir() / "assets"), "--assetIndex", manifest["assetIndex"]["id"], "--accessToken", "0", "--uuid", "00000000000000000000000000000000", "--demo", "--width", "1280", "--height", "800"] subprocess.run(command, cwd=instance, check=True) diff --git a/tools/server.py b/tools/server.py index d9d2facc7..82308289e 100644 --- a/tools/server.py +++ b/tools/server.py @@ -18,14 +18,10 @@ except ImportError: ROOT = lab.ROOT -def prepare(instance, baseline=False): +def prepare(instance, baseline=False, server_jar=None): lock, directory = lab.config() - game = lock["minecraft"] - manifest_file = runtime.fetch(game["manifest_url"], directory / "manifest.json", game["manifest_sha1"]) - manifest = json.loads(manifest_file.read_text()) - artifact = manifest["downloads"]["server"] - launcher = runtime.fetch(artifact["url"], directory / "jars/server.jar", artifact["sha1"]) - server, libraries = lab.unpack_server(launcher, directory, game["version"]) + launcher = lab.import_server(lock, directory, server_jar) + server, libraries = lab.unpack_server(launcher, directory, lock["minecraft"]["version"]) libraries = [server, *libraries, *runtime.fabric_libraries()] pack_lock = json.loads((ROOT / "packs/mods.lock.json").read_text()) mods = pack_lock["projects"] @@ -49,6 +45,7 @@ def main(): parser.add_argument("command", choices=["prepare", "run"]) parser.add_argument("--instance", type=Path, default=ROOT / ".local/server/optimized") parser.add_argument("--heap", default="4G") + parser.add_argument("--server-jar", type=Path, help="the pinned server jar you downloaded yourself") parser.add_argument("--baseline", action="store_true", help="Use a separate instance for the baseline") parser.add_argument("--compact-headers", action="store_true", help="Enable Java 25 compact object headers for this server process") parser.add_argument("--gc", choices=["default", "g1", "zgc"], default="default", help="Select a collector for this process; default leaves JVM policy in use") @@ -58,7 +55,7 @@ def main(): instance = args.instance.resolve() if args.baseline and args.instance == parser.get_default("instance"): instance = ROOT / ".local/server/baseline" - libraries = prepare(instance, args.baseline) + libraries = prepare(instance, args.baseline, args.server_jar) if args.command == "prepare": print(f"Prepared server at {instance}. Read https://www.minecraft.net/eula before setting eula=true locally.") return