diff --git a/research-lock.json b/research-lock.json index b057a79e1..219a59066 100644 --- a/research-lock.json +++ b/research-lock.json @@ -1,7 +1,12 @@ { "minecraft": { "version": "26.3", - "manifest_sha1": "4fe1aa1ef8da1cb95c5bad1fb98890ca56dd8ca3", + "manifest_sha1": "11f1aefe473eefba0af62e3286c33b692807b2eb", + "accepted_manifest_sha1": [ + "4fe1aa1ef8da1cb95c5bad1fb98890ca56dd8ca3", + "702fe59163c6ee6578607daa85811d9bc9c7cc40" + ], + "accepted_manifest_note": "Earlier 26.3 version JSONs Mojang republished: only assetIndex differs (non-English translations), jars and libraries are identical. A launcher may still hold any of them.", "client_sha1": "e877b6a07acd633fb3bb475002175cec036e7b87", "server_sha1": "33680f5f2ac32864d6d7cf5e56a705fdb3e05f4c", "java_major": 25 diff --git a/tests/test_lab.py b/tests/test_lab.py index 5c8940dbc..648160ce1 100644 --- a/tests/test_lab.py +++ b/tests/test_lab.py @@ -71,6 +71,7 @@ class ArtifactTests(unittest.TestCase): version.mkdir(parents=True) (version / "26.3.json").write_text(json.dumps(manifest)) (version / "26.3.jar").write_bytes(client) + lock["minecraft"]["manifest_sha1"] = lab.digest(version / "26.3.json", "sha1") loaded, jar = lab.import_client(lock, cache, launcher) self.assertEqual((loaded["id"], jar.read_bytes()), ("26.3", client)) with self.assertRaisesRegex(FileNotFoundError, "server jar yourself"): @@ -91,6 +92,29 @@ class ArtifactTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "pinned 26.3 release"): lab.import_manifest(lock, Path(temp) / "cache", Path(temp)) + def test_republished_launcher_manifests_are_accepted_and_unknown_ones_rejected(self): + manifest = {"id": "26.3", "javaVersion": {"majorVersion": 25}, "assetIndex": {"sha1": "a"}, + "downloads": {"client": {"sha1": "1" * 40}, "server": {"sha1": "2" * 40}}} + older = json.dumps({**manifest, "assetIndex": {"sha1": "b"}}) + lock = {"minecraft": {"version": "26.3", "java_major": 25, "client_sha1": "1" * 40, "server_sha1": "2" * 40, + "manifest_sha1": hashlib.sha1(json.dumps(manifest).encode()).hexdigest(), + "accepted_manifest_sha1": [hashlib.sha1(older.encode()).hexdigest()]}} + with tempfile.TemporaryDirectory() as temp: + launcher, cache = Path(temp) / "minecraft", Path(temp) / "cache" + source = launcher / "versions" / "26.3" / "26.3.json" + source.parent.mkdir(parents=True) + source.write_text(older) + self.assertEqual(lab.import_manifest(lock, cache, launcher)["assetIndex"], {"sha1": "b"}) + # The launcher's newer copy replaces the cached one, keeping it in step with its asset index. + source.write_text(json.dumps(manifest)) + self.assertEqual(lab.import_manifest(lock, cache, launcher)["assetIndex"], {"sha1": "a"}) + source.unlink() + self.assertEqual(lab.import_manifest(lock, cache, launcher)["assetIndex"], {"sha1": "a"}) + source.write_text(json.dumps({**manifest, "assetIndex": {"sha1": "c"}})) + with self.assertRaisesRegex(ValueError, "does not know yet"): + lab.import_manifest(lock, cache, launcher) + + def test_sourcefile_metadata_covers_inner_classes(self): with tempfile.TemporaryDirectory() as temp: path = Path(temp) / "game.jar" @@ -177,5 +201,89 @@ class PublicationTests(unittest.TestCase): self.assertIn(".local/artifact.txt", check_public_tree.paths_in_repository(history=True)) +def library(name, os_name=None): + entry = {"name": name, "downloads": {"artifact": {"path": name.replace(":", "/") + ".jar", + "sha1": hashlib.sha1(name.encode()).hexdigest()}}} + if os_name: + entry["rules"] = [{"action": "allow", "os": {"name": os_name}}] + return entry + + +# The shapes of 26.3's libraries: OS-only rules, ARM and x64 natives told apart by classifier. +PLATFORM_MANIFEST = {"libraries": [ + library("com.mojang:brigadier:1.3.10"), + library("ca.weblite:java-objc-bridge:1.1", "osx"), + library("org.lwjgl:lwjgl:3.4.3"), + library("org.lwjgl:lwjgl:3.4.3:natives-linux", "linux"), + library("org.lwjgl:lwjgl:3.4.3:natives-macos", "osx"), + library("org.lwjgl:lwjgl:3.4.3:natives-macos-arm64", "osx"), + library("org.lwjgl:lwjgl:3.4.3:natives-windows", "windows"), + library("org.lwjgl:lwjgl:3.4.3:natives-windows-arm64", "windows"), + library("io.netty:netty-transport-native-epoll:4.2.16.Final:linux-aarch_64", "linux"), + library("io.netty:netty-transport-native-epoll:4.2.16.Final:linux-x86_64", "linux"), + library("io.netty:netty-transport-native-kqueue:4.2.16.Final:osx-aarch_64", "osx"), + library("io.netty:netty-transport-native-kqueue:4.2.16.Final:osx-x86_64", "osx"), + {"name": "org.example:no-artifact:1.0", "downloads": {}}, +]} + + +class PlatformLibraryTests(unittest.TestCase): + def selected(self, system, architecture): + names = {entry["downloads"]["artifact"]["path"]: entry["name"] + for entry in PLATFORM_MANIFEST["libraries"] if entry["downloads"]} + return {names[artifact["path"]] + for artifact in lab.client_library_artifacts(PLATFORM_MANIFEST, system, architecture)} + + def test_each_platform_requires_only_what_the_official_launcher_installs(self): + common = {"com.mojang:brigadier:1.3.10", "org.lwjgl:lwjgl:3.4.3"} + expected = { + ("linux", "x86_64"): {"org.lwjgl:lwjgl:3.4.3:natives-linux", + "io.netty:netty-transport-native-epoll:4.2.16.Final:linux-x86_64"}, + # No ARM build of natives-linux is listed, so the plain one is the only one. + ("linux", "aarch64"): {"org.lwjgl:lwjgl:3.4.3:natives-linux", + "io.netty:netty-transport-native-epoll:4.2.16.Final:linux-aarch_64"}, + ("osx", "arm64"): {"ca.weblite:java-objc-bridge:1.1", "org.lwjgl:lwjgl:3.4.3:natives-macos-arm64", + "io.netty:netty-transport-native-kqueue:4.2.16.Final:osx-aarch_64"}, + ("osx", "x86_64"): {"ca.weblite:java-objc-bridge:1.1", "org.lwjgl:lwjgl:3.4.3:natives-macos", + "io.netty:netty-transport-native-kqueue:4.2.16.Final:osx-x86_64"}, + ("windows", "AMD64"): {"org.lwjgl:lwjgl:3.4.3:natives-windows"}, + ("windows", "ARM64"): {"org.lwjgl:lwjgl:3.4.3:natives-windows-arm64"}, + } + for (system, architecture), specific in expected.items(): + with self.subTest(system=system, architecture=architecture): + self.assertEqual(self.selected(system, architecture), common | specific) + + def test_rules_follow_the_launcher_last_match_and_arch_aliases(self): + rules = [{"action": "allow"}, {"action": "disallow", "os": {"name": "osx", "arch": "aarch64"}}] + self.assertTrue(lab.library_allowed({"rules": rules}, "osx", "x86_64")) + self.assertFalse(lab.library_allowed({"rules": rules}, "osx", "arm64")) + self.assertFalse(lab.library_allowed({"rules": rules}, "osx", "aarch64")) + self.assertTrue(lab.library_allowed({"rules": rules}, "linux", "aarch64")) + self.assertTrue(lab.library_allowed({"rules": [{"action": "allow", "os": {"arch": "x86_64"}}]}, "windows", "AMD64")) + self.assertFalse(lab.library_allowed({"rules": [{"action": "allow", "features": {"is_demo_user": True}}]}, "linux")) + + def test_prepare_on_linux_skips_macos_libraries_and_still_checks_required_ones(self): + with tempfile.TemporaryDirectory() as temp: + launcher, cache = Path(temp) / "minecraft", Path(temp) / "cache" + artifacts = lab.client_library_artifacts(PLATFORM_MANIFEST, "linux", "x86_64") + # A Linux install: the launcher never fetched the macOS bridge. Each fixture + # jar's content is its Maven name, which is what its sha1 covers. + for entry in PLATFORM_MANIFEST["libraries"]: + artifact = entry["downloads"].get("artifact") + if artifact in artifacts: + path = launcher / "libraries" / artifact["path"] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(entry["name"].encode()) + imported = [lab.import_library(a, cache, launcher) for a in artifacts] + self.assertFalse(any("objc" in str(path) for path in imported)) + required = next(a for a in artifacts if "natives-linux" in a["path"]) + (launcher / "libraries" / required["path"]).write_bytes(b"tampered") + with self.assertRaisesRegex(ValueError, "not the pinned file"): + lab.import_library(required, Path(temp) / "other", launcher) + (launcher / "libraries" / required["path"]).unlink() + with self.assertRaisesRegex(FileNotFoundError, "official launcher"): + lab.import_library(required, Path(temp) / "other", launcher) + + if __name__ == "__main__": unittest.main() diff --git a/tools/lab.py b/tools/lab.py index 5b37f84e2..43770091b 100644 --- a/tools/lab.py +++ b/tools/lab.py @@ -8,6 +8,7 @@ from collections import Counter import hashlib import json import os +import platform from pathlib import Path, PurePosixPath import re import shutil @@ -99,23 +100,37 @@ def import_copy(source, destination, checksum, algorithm="sha1", hint=""): return destination +def manifest_accepted(lock, checksum): + """Mojang republishes a version JSON for asset-index changes; every known one is accepted.""" + game = lock["minecraft"] + return checksum == game["manifest_sha1"] or checksum in game.get("accepted_manifest_sha1", []) + + def import_manifest(lock, directory, minecraft=None): - """The pinned version JSON, from the launcher's versions/ directory.""" + """The pinned version JSON, from the launcher's versions/ directory. + + The launcher's copy wins over the cached one so the manifest matches the asset index the + launcher installed alongside it.""" game = lock["minecraft"] version = game["version"] destination = directory / "manifest.json" source = minecraft_dir(minecraft) / "versions" / version / f"{version}.json" - if not (destination.exists() and digest(destination, "sha1") == game["manifest_sha1"]): - if not source.is_file(): - raise FileNotFoundError(f"Missing {source}. {LAUNCHER_HINT.format(version=version)}") + if source.is_file(): manifest = json.loads(source.read_text(encoding="utf-8")) downloads = manifest.get("downloads", {}) if (manifest.get("id") != version or downloads.get("client", {}).get("sha1") != game["client_sha1"] or downloads.get("server", {}).get("sha1") != game["server_sha1"]): raise ValueError(f"{source} does not describe the pinned {version} release") - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(source, destination) + checksum = digest(source, "sha1") + if not manifest_accepted(lock, checksum): + raise ValueError(f"{source} is a {version} version file Horizon does not know yet (sha1 {checksum}); " + f"Mojang may have republished it. Update Horizon, or report it if this is the latest release.") + if not (destination.exists() and digest(destination, "sha1") == checksum): + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + elif not (destination.exists() and manifest_accepted(lock, digest(destination, "sha1"))): + raise FileNotFoundError(f"Missing {source}. {LAUNCHER_HINT.format(version=version)}") manifest = json.loads(destination.read_text(encoding="utf-8")) if manifest["javaVersion"]["majorVersion"] != game["java_major"]: raise ValueError("Manifest Java version does not match lock file") @@ -146,6 +161,71 @@ def import_server(lock, directory, server_jar=None): return import_copy(Path(source).expanduser(), destination, checksum, hint=hint) +def rule_system(system=None): + """The launcher's os.name for this machine: linux, osx or windows.""" + return system or {"Darwin": "osx", "Linux": "linux", "Windows": "windows"}[platform.system()] + + +ARCHITECTURE_ALIASES = [{"x86_64", "amd64", "x64"}, {"arm64", "aarch64"}] + + +def rule_architectures(architecture=None): + """platform.machine() and its aliases: Windows reports AMD64/ARM64, Linux aarch64, macOS arm64.""" + architecture = (architecture or platform.machine()).lower() + return next((names for names in ARCHITECTURE_ALIASES if architecture in names), {architecture}) + + +def library_allowed(library, system=None, architecture=None): + """The official launcher's rules: the last matching rule decides, no rules means allowed.""" + system, architectures = rule_system(system), rule_architectures(architecture) + rules = library.get("rules", []) + allowed = not rules + for rule in rules: + operating_system = rule.get("os", {}) + if operating_system.get("name", system) != system: + continue + if "arch" in operating_system and not any(re.fullmatch(operating_system["arch"], name) + for name in architectures): + continue + if "version" in operating_system and not re.search(operating_system["version"], platform.release()): + continue + if rule.get("features"): + continue + allowed = rule["action"] == "allow" + return allowed + + +ARM_CLASSIFIER = re.compile(r"(?:^|-)(?:arm64|aarch64|aarch_64)$") +X86_CLASSIFIER = re.compile(r"(?:^|-)(?:x86_64|x64)$") + + +def native_library_matches(library, architecture=None, names=None): + """Modern manifests list x64 and ARM natives with OS-only rules; keep this machine's. + + A plain natives- jar is the x64 build when the manifest's library names also hold + an ARM variant of it (assumed when names is None), and the only build otherwise.""" + arm = "arm64" in rule_architectures(architecture) + parts = library["name"].split(":") + classifier = parts[3] if len(parts) > 3 else "" + if ARM_CLASSIFIER.search(classifier): + return arm + if X86_CLASSIFIER.search(classifier): + return not arm + if arm and classifier.startswith("natives-"): + return names is not None and not any(f"{library['name']}-{suffix}" in names + for suffix in ("arm64", "aarch64")) + return True + + +def client_library_artifacts(manifest, system=None, architecture=None): + """The client libraries the official launcher installs on this OS and architecture.""" + names = {library["name"] for library in manifest["libraries"]} + return [library["downloads"]["artifact"] for library in manifest["libraries"] + if "artifact" in library.get("downloads", {}) + and library_allowed(library, system, architecture) + and native_library_matches(library, architecture, names)] + + def import_library(artifact, destination_root, minecraft=None): """One library jar from the launcher's libraries/ directory.""" relative = PurePosixPath(artifact["path"]) @@ -307,10 +387,8 @@ def prepare(minecraft=None, server_jar=None): print(f"Imported client {manifest['id']} from {minecraft_dir(minecraft)}", flush=True) jars = {"client": client, "server": import_server(lock, directory, server_jar)} jars["server"], server_libraries = unpack_server(jars["server"], directory, manifest["id"]) - client_artifacts = [library["downloads"]["artifact"] for library in manifest["libraries"] - if "artifact" in library.get("downloads", {})] client_libraries = [import_library(artifact, directory / "libraries" / "client", minecraft) - for artifact in client_artifacts] + for artifact in client_library_artifacts(manifest)] vf = lock["vineflower"] download(vf["url"], LOCAL / "tools" / f"vineflower-{vf['version']}.jar", vf["sha256"], "sha256") for side, jar in jars.items(): diff --git a/tools/runtime.py b/tools/runtime.py index 71d55bf94..c46b19bed 100644 --- a/tools/runtime.py +++ b/tools/runtime.py @@ -21,6 +21,8 @@ except ImportError: import lab ROOT = lab.ROOT +library_allowed = lab.library_allowed +native_library_matches = lab.native_library_matches def install_original_mod(source, mod_dir, filename=None): @@ -74,25 +76,6 @@ def fabric_libraries(): return jars -def library_allowed(library, system=None, architecture=None): - system = system or {"Darwin": "osx", "Linux": "linux", "Windows": "windows"}[platform.system()] - architecture = (architecture or platform.machine()).lower() - rules = library.get("rules", []) - allowed = not rules - for rule in rules: - operating_system = rule.get("os", {}) - if operating_system.get("name", system) != system: - continue - if "arch" in operating_system and not re.fullmatch(operating_system["arch"], architecture): - continue - if "version" in operating_system and not re.search(operating_system["version"], platform.release()): - continue - if rule.get("features"): - continue - allowed = rule["action"] == "allow" - return allowed - - def conditional_client_jvm_options(manifest, system=None, architecture=None): """Use official OS/architecture flags alongside our managed JVM/classpath options.""" options = [] @@ -107,18 +90,6 @@ def conditional_client_jvm_options(manifest, system=None, architecture=None): return options -def native_library_matches(library, architecture=None): - """Modern manifests list both x64 and ARM natives with OS-only rules.""" - architecture = (architecture or platform.machine()).lower() - name = library["name"] - if ":natives-" not in name: - return True - arm = architecture in {"arm64", "aarch64"} - if "-arm64" in name or "-aarch64" in name: - return arm - return not arm - - def prepare_client(assets=True): """Classpath and natives from the user's own launcher install; nothing of the game is downloaded.""" lock, directory = lab.config() @@ -126,12 +97,7 @@ def prepare_client(assets=True): jars = [client] natives = CACHE / "natives" natives.mkdir(parents=True, exist_ok=True) - for library in manifest["libraries"]: - if not library_allowed(library) or not native_library_matches(library): - continue - artifact = library.get("downloads", {}).get("artifact") - if not artifact: - continue + for artifact in lab.client_library_artifacts(manifest): jar = lab.import_library(artifact, directory / "libraries/client") jars.append(jar) with zipfile.ZipFile(jar) as archive: diff --git a/tools/test_model_baking_parity.py b/tools/test_model_baking_parity.py index 58c095fbe..ac7737444 100644 --- a/tools/test_model_baking_parity.py +++ b/tools/test_model_baking_parity.py @@ -59,14 +59,12 @@ def client_libraries(): _, directory=lab.config() manifest=json.loads((directory/'manifest.json').read_text()) libraries=[] - for library in manifest['libraries']: - artifact=library.get('downloads',{}).get('artifact') - if artifact: - relative=Path(artifact['path']) - if relative.is_absolute() or '..' in relative.parts: raise ValueError('Unsafe client library path') - path=directory/'libraries/client'/relative - if lab.digest(path,'sha1')!=artifact['sha1']: raise ValueError('Client dependency checksum changed: '+str(relative)) - libraries.append(path) + for artifact in lab.client_library_artifacts(manifest): + relative=Path(artifact['path']) + if relative.is_absolute() or '..' in relative.parts: raise ValueError('Unsafe client library path') + path=directory/'libraries/client'/relative + if lab.digest(path,'sha1')!=artifact['sha1']: raise ValueError('Client dependency checksum changed: '+str(relative)) + libraries.append(path) return libraries diff --git a/tools/test_rust_parity.py b/tools/test_rust_parity.py index 90725e1a5..3f1a45eb5 100644 --- a/tools/test_rust_parity.py +++ b/tools/test_rust_parity.py @@ -74,7 +74,7 @@ def verify_artifacts(): """Authenticate against the lock and the bundled checksums, not generated reports.""" lock, directory = lab.config() manifest_path = directory / "manifest.json" - if lab.digest(manifest_path, "sha1") != lock["minecraft"]["manifest_sha1"]: + if not lab.manifest_accepted(lock, lab.digest(manifest_path, "sha1")): raise ValueError("Manifest checksum changed; run tools/lab.py prepare") manifest = json.loads(manifest_path.read_text()) if manifest["id"] != lock["minecraft"]["version"]: