diff --git a/Cargo.lock b/Cargo.lock index 90b288a8e..9c3ad04ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1498,6 +1498,18 @@ dependencies = [ "horizon-mod-sdk", ] +[[package]] +name = "horizon-port-pack" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "sha1", + "toml", + "ureq", + "zip", +] + [[package]] name = "horizon-prepare" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 49ef5785f..1b4cb92d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] exclude = ["launcher", "crates/horizon-telemetry-server"] -members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-shaderpack", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"] +members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-shaderpack", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"] resolver = "2" [workspace.package] diff --git a/crates/horizon-port-pack/Cargo.toml b/crates/horizon-port-pack/Cargo.toml new file mode 100644 index 000000000..1b33b6c01 --- /dev/null +++ b/crates/horizon-port-pack/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "horizon-port-pack" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version = "1.86" +description = "Port a Java modpack to Horizon: import, inventory, classification and data carry-over (docs/modpacks.md)" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha1 = "0.10" +toml = "1.1.6" +ureq = { version = "2", default-features = false, features = ["tls"] } +zip = { version = "2", default-features = false, features = ["deflate"] } + +[lints] +workspace = true diff --git a/crates/horizon-port-pack/src/classify.rs b/crates/horizon-port-pack/src/classify.rs new file mode 100644 index 000000000..7ebb65644 --- /dev/null +++ b/crates/horizon-port-pack/src/classify.rs @@ -0,0 +1,233 @@ +//! Step 2 and 3: classify each mod (class and API tier, `docs/modpacks.md`) +//! and order them by required dependencies. +use std::collections::{BTreeMap, HashMap}; + +/// The reviewed table: ATM10's 495 rows (`docs/modpacks/atm10-mods.csv`). +/// It wins over the heuristics. +const KNOWN_CSV: &str = include_str!("../../../docs/modpacks/atm10-mods.csv"); + +pub struct Known { + by_key: HashMap, +} + +/// Lowercase letters and digits only: `Get It Together, Drops!` -> +/// `getittogetherdrops`. +pub fn normalise(s: &str) -> String { + s.chars() + .filter(char::is_ascii_alphanumeric) + .map(|c| c.to_ascii_lowercase()) + .collect() +} + +/// One CSV line, with `"quoted, fields"`. +fn csv_fields(line: &str) -> Vec { + let (mut out, mut field, mut quoted) = (Vec::new(), String::new(), false); + let mut chars = line.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '"' if quoted && chars.peek() == Some(&'"') => { + field.push('"'); + chars.next(); + } + '"' => quoted = !quoted, + ',' if !quoted => out.push(std::mem::take(&mut field)), + c => field.push(c), + } + } + out.push(field); + out +} + +impl Known { + pub fn atm10() -> Self { + let mut by_key = HashMap::new(); + for line in KNOWN_CSV.lines().skip(1) { + let f = csv_fields(line); + if f.len() < 6 || f[5] == "guess" { + continue; + } + let value = (f[3].clone(), f[4].clone()); + by_key.insert(normalise(&f[0]), value.clone()); + by_key.entry(normalise(&f[1])).or_insert(value); + } + Known { by_key } + } + + /// By mod id, Modrinth slug or name, in that order. + pub fn get(&self, keys: &[&str]) -> Option<&(String, String)> { + keys.iter() + .filter(|k| !k.is_empty()) + .find_map(|k| self.by_key.get(&normalise(k))) + } +} + +/// What the rules look at. +#[derive(Default)] +pub struct Facts<'a> { + /// `mods`, `resourcepacks`, `shaderpacks`, `datapacks`. + pub kind: &'a str, + pub client: &'a str, + pub server: &'a str, + /// Modrinth categories. + pub categories: &'a [String], + /// From the jar: classes, and whether it has blocks, items or data. + pub classes: Option, + pub has_content: bool, +} + +/// Class, tier and confidence (`known` is set by the caller). +pub fn rules(f: &Facts) -> (&'static str, &'static str, &'static str) { + let has = |c: &str| f.categories.iter().any(|x| x == c); + match f.kind { + "resourcepacks" => return ("other", "data", "kind"), + "shaderpacks" => return ("other", "none", "kind"), + "datapacks" => return ("content", "data", "kind"), + _ => {} + } + if has("library") { + return ("library", "none", "categories"); + } + if has("optimization") { + return ("performance", "none", "categories"); + } + if f.client != "unsupported" && f.server == "unsupported" { + return ("client-ui", "client", "sides"); + } + if f.client == "unsupported" && f.server != "unsupported" && !f.server.is_empty() { + return ("server-only", "logic", "sides"); + } + if f.classes == Some(0) && f.has_content { + return ("content", "data", "jar"); + } + let content = [ + "adventure", + "decoration", + "equipment", + "food", + "game-mechanics", + "magic", + "mobs", + "storage", + "technology", + "transportation", + "worldgen", + ]; + if f.categories.iter().any(|c| content.contains(&c.as_str())) { + let tier = if has("mobs") || has("magic") || has("adventure") { + "deep" + } else if has("technology") || has("storage") || has("transportation") { + "machines" + } else if f.categories.iter().all(|c| { + ["worldgen", "decoration", "food"].contains(&c.as_str()) + || !content.contains(&c.as_str()) + }) { + "data" + } else { + "logic" + }; + return ("content", tier, "categories"); + } + if has("utility") || has("management") || has("social") { + return ("other", "logic", "guess"); + } + ("unknown", "none", "unknown") +} + +/// Layer of each mod: 0 needs nothing in the pack, n needs layer n-1. +/// Mods in a cycle get one layer past the rest; dependencies outside the +/// pack are left out (the caller lists them). +pub fn layers(depends: &BTreeMap>) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + loop { + let mut progressed = false; + for (id, deps) in depends { + if out.contains_key(id) { + continue; + } + let inside = deps.iter().filter(|d| depends.contains_key(*d) && *d != id); + let mut layer = 0; + let mut ready = true; + for d in inside { + match out.get(d) { + Some(l) => layer = layer.max(l + 1), + None => ready = false, + } + } + if ready { + out.insert(id.clone(), layer); + progressed = true; + } + } + if !progressed { + break; + } + } + let past = out.values().max().map_or(0, |m| m + 1); + for id in depends.keys() { + out.entry(id.clone()).or_insert(past); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn known_table_matches_ids_and_names() { + let k = Known::atm10(); + assert_eq!(k.get(&["create"]).map(|v| v.0.as_str()), Some("content")); + assert_eq!( + k.get(&["", "Get It Together, Drops!"]) + .map(|v| v.0.as_str()), + Some("performance") + ); + assert_eq!(k.get(&["sodium"]).map(|v| v.1.as_str()), Some("none")); + assert!(k.get(&["not-a-mod"]).is_none()); + } + + #[test] + fn rules_follow_sides_then_categories() { + let cats = |c: &[&str]| c.iter().map(|s| s.to_string()).collect::>(); + let f = |client, server, c: &[String]| { + rules(&Facts { + kind: "mods", + client, + server, + categories: c, + ..Facts::default() + }) + }; + assert_eq!(f("required", "unsupported", &[]).0, "client-ui"); + assert_eq!(f("unsupported", "required", &[]).0, "server-only"); + assert_eq!(f("required", "required", &cats(&["library"])).0, "library"); + assert_eq!( + f("required", "unsupported", &cats(&["optimization"])).0, + "performance" + ); + assert_eq!( + f("required", "required", &cats(&["technology", "utility"])), + ("content", "machines", "categories") + ); + assert_eq!( + f("required", "required", &cats(&["worldgen", "neoforge"])), + ("content", "data", "categories") + ); + assert_eq!(f("", "", &[]).0, "unknown"); + } + + #[test] + fn layers_put_dependencies_first() { + let d = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::>(); + let deps = BTreeMap::from([ + ("lib".to_string(), d(&[])), + ("create".to_string(), d(&["lib", "outside"])), + ("addon".to_string(), d(&["create", "lib"])), + ("a".to_string(), d(&["b"])), + ("b".to_string(), d(&["a"])), + ]); + let l = layers(&deps); + assert_eq!((l["lib"], l["create"], l["addon"]), (0, 1, 2)); + assert_eq!((l["a"], l["b"]), (3, 3)); + } +} diff --git a/crates/horizon-port-pack/src/import.rs b/crates/horizon-port-pack/src/import.rs new file mode 100644 index 000000000..b0f5d04c2 --- /dev/null +++ b/crates/horizon-port-pack/src/import.rs @@ -0,0 +1,335 @@ +//! Step 1, import: a Modrinth `.mrpack`, a CurseForge zip, or an instance +//! folder (Prism / MultiMC, or a plain `mods/` folder) to `pack.json`. +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{ + collections::BTreeMap, + fs, + io::Read, + path::{Path, PathBuf}, +}; + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct Pack { + pub name: String, + /// `mrpack`, `curseforge` or `instance`. + pub format: String, + pub minecraft: String, + /// `neoforge`, `forge`, `fabric` or `quilt`. + pub loader: String, + pub loader_version: String, + pub files: Vec, + /// Override files by top folder (`config`, `kubejs`, ...). + pub overrides: BTreeMap, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct PackFile { + /// Path in the instance (`mods/create-1.21.1-6.0.4.jar`). + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sha1: Option, + /// `required`, `optional` or `unsupported`; empty when not stated. + pub client: String, + pub server: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub downloads: Vec, + /// CurseForge project and file ids. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub curseforge: Option<(u64, u64)>, + /// The jar on disk (instances only). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub jar: Option, +} + +impl PackFile { + /// `mods`, `resourcepacks`, `shaderpacks`, `datapacks`... + pub fn kind(&self) -> &str { + self.path.split('/').next().unwrap_or("") + } + pub fn file_name(&self) -> &str { + self.path.rsplit('/').next().unwrap_or(&self.path) + } +} + +const LOADERS: [(&str, &str); 4] = [ + ("neoforge", "neoforge"), + ("forge", "forge"), + ("fabric-loader", "fabric"), + ("quilt-loader", "quilt"), +]; + +pub fn import(path: &Path) -> Result { + if path.is_dir() { + return instance(path); + } + let file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut zip = zip::ZipArchive::new(file).map_err(|e| format!("{}: {e}", path.display()))?; + if let Some(index) = read_json(&mut zip, "modrinth.index.json")? { + return Ok(mrpack(&index, &names(&zip))); + } + if let Some(manifest) = read_json(&mut zip, "manifest.json")? { + return Ok(curseforge(&manifest, &names(&zip))); + } + Err(format!( + "{}: neither modrinth.index.json nor manifest.json", + path.display() + )) +} + +fn names(zip: &zip::ZipArchive) -> Vec { + zip.file_names().map(str::to_owned).collect() +} + +fn read_json(zip: &mut zip::ZipArchive, name: &str) -> Result, String> { + let Ok(mut entry) = zip.by_name(name) else { + return Ok(None); + }; + let mut text = String::new(); + entry + .read_to_string(&mut text) + .map_err(|e| format!("{name}: {e}"))?; + serde_json::from_str(&text) + .map(Some) + .map_err(|e| format!("{name}: {e}")) +} + +fn str_of(v: &Value, key: &str) -> String { + v.get(key).and_then(Value::as_str).unwrap_or("").to_owned() +} + +/// Override files by top folder, below any of `roots`. +fn overrides(names: &[String], roots: &[&str]) -> BTreeMap { + let mut out = BTreeMap::new(); + for name in names.iter().filter(|n| !n.ends_with('/')) { + for root in roots { + if let Some(rest) = name.strip_prefix(root).and_then(|r| r.strip_prefix('/')) { + let top = rest.split('/').next().unwrap_or(rest); + *out.entry(top.to_owned()).or_default() += 1; + } + } + } + out +} + +pub fn mrpack(index: &Value, names: &[String]) -> Pack { + let deps = &index["dependencies"]; + let (loader, loader_version) = LOADERS + .iter() + .find_map(|(key, name)| Some((name.to_string(), deps.get(*key)?.as_str()?.to_owned()))) + .unwrap_or_default(); + let files = index["files"] + .as_array() + .into_iter() + .flatten() + .map(|f| PackFile { + path: str_of(f, "path"), + sha1: f["hashes"]["sha1"].as_str().map(str::to_owned), + client: str_of(&f["env"], "client"), + server: str_of(&f["env"], "server"), + downloads: f["downloads"] + .as_array() + .into_iter() + .flatten() + .filter_map(|d| d.as_str().map(str::to_owned)) + .collect(), + ..PackFile::default() + }) + .collect(); + Pack { + name: str_of(index, "name"), + format: "mrpack".into(), + minecraft: str_of(deps, "minecraft"), + loader, + loader_version, + files, + overrides: overrides( + names, + &["overrides", "client-overrides", "server-overrides"], + ), + } +} + +pub fn curseforge(manifest: &Value, names: &[String]) -> Pack { + let primary = manifest["minecraft"]["modLoaders"] + .as_array() + .into_iter() + .flatten() + .find(|l| l["primary"].as_bool().unwrap_or(true)) + .map(|l| str_of(l, "id")) + .unwrap_or_default(); + let (loader, loader_version) = primary.split_once('-').unwrap_or((&primary, "")); + let files = manifest["files"] + .as_array() + .into_iter() + .flatten() + .map(|f| { + let (p, id) = (f["projectID"].as_u64(), f["fileID"].as_u64()); + PackFile { + path: format!("mods/curseforge-{}-{}", p.unwrap_or(0), id.unwrap_or(0)), + // CurseForge states no side; its API needs a key, so names + // come from the user's installed instance (import that). + curseforge: p.zip(id), + ..PackFile::default() + } + }) + .collect(); + let root = manifest["overrides"].as_str().unwrap_or("overrides"); + Pack { + name: str_of(manifest, "name"), + format: "curseforge".into(), + minecraft: str_of(&manifest["minecraft"], "version"), + loader: loader.to_owned(), + loader_version: loader_version.to_owned(), + files, + overrides: overrides(names, &[root]), + } +} + +/// A Prism / MultiMC instance (`mmc-pack.json`), a game folder, or a +/// folder of jars: the jars on disk, hashed. +fn instance(dir: &Path) -> Result { + let mut pack = Pack { + name: dir + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(), + format: "instance".into(), + ..Pack::default() + }; + if let Ok(text) = fs::read_to_string(dir.join("mmc-pack.json")) { + let mmc: Value = serde_json::from_str(&text).map_err(|e| format!("mmc-pack.json: {e}"))?; + for c in mmc["components"].as_array().into_iter().flatten() { + let version = str_of(c, "version"); + match c["uid"].as_str().unwrap_or("") { + "net.minecraft" => pack.minecraft = version, + "net.neoforged" => { + (pack.loader, pack.loader_version) = ("neoforge".into(), version) + } + "net.minecraftforge" => { + (pack.loader, pack.loader_version) = ("forge".into(), version) + } + "net.fabricmc.fabric-loader" => { + (pack.loader, pack.loader_version) = ("fabric".into(), version) + } + "org.quiltmc.quilt-loader" => { + (pack.loader, pack.loader_version) = ("quilt".into(), version) + } + _ => {} + } + } + } + let game = [".minecraft", "minecraft", "."] + .iter() + .map(|g| dir.join(g)) + .find(|g| g.join("mods").is_dir()) + .or_else(|| dir.read_dir().is_ok().then(|| dir.to_path_buf())) + .ok_or_else(|| format!("{}: no mods folder", dir.display()))?; + let mods = if game.join("mods").is_dir() { + game.join("mods") + } else { + game.clone() + }; + let mut jars: Vec = fs::read_dir(&mods) + .map_err(|e| format!("{}: {e}", mods.display()))? + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x == "jar")) + .collect(); + jars.sort(); + for jar in jars { + let bytes = fs::read(&jar).map_err(|e| format!("{}: {e}", jar.display()))?; + pack.files.push(PackFile { + path: format!( + "mods/{}", + jar.file_name().unwrap_or_default().to_string_lossy() + ), + sha1: Some(sha1_hex(&bytes)), + jar: Some(jar), + ..PackFile::default() + }); + } + for top in ["config", "kubejs", "defaultconfigs", "datapacks"] { + let n = count_files(&game.join(top)); + if n > 0 { + pack.overrides.insert(top.into(), n); + } + } + Ok(pack) +} + +fn count_files(dir: &Path) -> usize { + let Ok(entries) = fs::read_dir(dir) else { + return 0; + }; + entries + .flatten() + .map(|e| { + let p = e.path(); + if p.is_dir() { count_files(&p) } else { 1 } + }) + .sum() +} + +pub fn sha1_hex(bytes: &[u8]) -> String { + use sha1::Digest; + sha1::Sha1::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn reads_an_mrpack_index() { + let index = json!({"name": "Small", "dependencies": {"minecraft": "1.21.1", "neoforge": "21.1.233"}, + "files": [{"path": "mods/create.jar", "hashes": {"sha1": "ab"}, "env": {"client": "required", "server": "required"}, + "downloads": ["https://cdn.modrinth.com/create.jar"]}, + {"path": "resourcepacks/look.zip", "hashes": {"sha1": "cd"}}]}); + let names = [ + "overrides/config/a.toml", + "overrides/config/b/c.toml", + "client-overrides/kubejs/x.js", + "overrides/", + ] + .map(String::from); + let p = mrpack(&index, &names); + assert_eq!( + ( + p.minecraft.as_str(), + p.loader.as_str(), + p.loader_version.as_str() + ), + ("1.21.1", "neoforge", "21.1.233") + ); + assert_eq!(p.files.len(), 2); + assert_eq!( + (p.files[0].kind(), p.files[0].file_name()), + ("mods", "create.jar") + ); + assert_eq!(p.files[0].sha1.as_deref(), Some("ab")); + assert_eq!(p.files[1].kind(), "resourcepacks"); + assert_eq!( + p.overrides, + BTreeMap::from([("config".into(), 2), ("kubejs".into(), 1)]) + ); + } + + #[test] + fn reads_a_curseforge_manifest() { + let manifest = json!({"name": "CF", "overrides": "overrides", + "minecraft": {"version": "1.21.1", "modLoaders": [{"id": "neoforge-21.1.259", "primary": true}]}, + "files": [{"projectID": 238222, "fileID": 5101366, "required": true}]}); + let p = curseforge(&manifest, &["overrides/kubejs/server_scripts/a.js".into()]); + assert_eq!( + (p.loader.as_str(), p.loader_version.as_str()), + ("neoforge", "21.1.259") + ); + assert_eq!(p.files[0].curseforge, Some((238222, 5101366))); + assert_eq!(p.overrides["kubejs"], 1); + } +} diff --git a/crates/horizon-port-pack/src/inventory.rs b/crates/horizon-port-pack/src/inventory.rs new file mode 100644 index 000000000..49661f1ae --- /dev/null +++ b/crates/horizon-port-pack/src/inventory.rs @@ -0,0 +1,225 @@ +//! Step 2, inventory: one record per pack file, from the jar (when the +//! user's instance has it) and Modrinth's metadata, classified and ordered. +use crate::{ + classify::{self, Facts, Known}, + import::Pack, + jar::{self, JarInfo}, + modrinth::Cache, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, HashMap}, + path::{Path, PathBuf}, +}; + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct Record { + /// The mod id from the jar, else the Modrinth slug, else the file name. + pub key: String, + pub name: String, + pub version: String, + pub path: String, + pub class: String, + pub tier: String, + /// `known` (reviewed table), `sides`, `categories`, `jar`, `kind`, + /// `guess` or `unknown`. + pub confidence: String, + /// `modrinth`, `curseforge` or `file`. + pub source: String, + pub license: String, + pub client: String, + pub server: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub categories: Vec, + /// Required dependencies inside the pack, by key. + pub depends: Vec, + /// Required dependencies the pack does not have. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub missing: Vec, + pub layer: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub jar: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub info: Option, +} + +/// The jar of a pack file: the instance's own, or one the user put in +/// `/jars/`. +fn jar_of(pack_jar: Option<&PathBuf>, file_name: &str, work: &Path) -> Option { + pack_jar + .cloned() + .or_else(|| Some(work.join("jars").join(file_name))) + .filter(|p| p.is_file()) +} + +pub fn build(pack: &Pack, cache: &Cache, work: &Path) -> Vec { + let known = Known::atm10(); + let mut records = Vec::new(); + // Modrinth project id -> record key, for dependencies. + let mut by_project: HashMap = HashMap::new(); + let mut project_deps: Vec> = Vec::new(); + for f in &pack.files { + let jar = jar_of(f.jar.as_ref(), f.file_name(), work); + let info = jar.as_deref().and_then(|j| jar::read(j).ok()); + let sha1 = f.sha1.as_deref().unwrap_or(""); + let project = cache.project_of(sha1); + let p = |k: &str| project.and_then(|p| p[k].as_str()).unwrap_or("").to_owned(); + let categories: Vec = project + .into_iter() + .flat_map(|p| { + ["categories", "additional_categories"] + .iter() + .flat_map(|k| p[*k].as_array().cloned().unwrap_or_default()) + }) + .filter_map(|c| c.as_str().map(str::to_owned)) + .collect(); + let stem = f.file_name().trim_end_matches(".jar").to_owned(); + let key = info + .as_ref() + .map(|i| i.mod_id.clone()) + .filter(|k| !k.is_empty()) + .or_else(|| Some(p("slug")).filter(|s| !s.is_empty())) + .unwrap_or(stem); + let (client, server) = match project { + Some(_) if f.client.is_empty() => (p("client_side"), p("server_side")), + _ => (f.client.clone(), f.server.clone()), + }; + let census = info.as_ref().map(|i| &i.census); + let facts = Facts { + kind: f.kind(), + client: &client, + server: &server, + categories: &categories, + classes: census.map(|c| c.classes), + has_content: census.is_some_and(|c| c.blockstates + c.item_models + c.recipes > 0), + }; + let name = info + .as_ref() + .map(|i| i.name.clone()) + .filter(|n| !n.is_empty()) + .unwrap_or_else(|| p("title")); + let (class, tier, confidence) = + match known.get(&[key.as_str(), p("slug").as_str(), name.as_str()]) { + Some((c, t)) => (c.clone(), t.clone(), "known".to_owned()), + None => { + let (c, t, conf) = classify::rules(&facts); + (c.into(), t.into(), conf.into()) + } + }; + let license = info + .as_ref() + .map(|i| i.license.clone()) + .filter(|l| !l.is_empty()) + .or_else(|| project.and_then(|p| p["license"]["id"].as_str().map(str::to_owned))) + .unwrap_or_else(|| "unknown".into()); + if let Some(id) = project.and_then(|p| p["id"].as_str()) { + by_project.insert(id.to_owned(), key.clone()); + } + project_deps.push(cache.required(sha1)); + records.push(Record { + name, + version: info.as_ref().map(|i| i.version.clone()).unwrap_or_else(|| { + cache + .versions + .get(sha1) + .and_then(|v| v["version_number"].as_str()) + .unwrap_or("") + .to_owned() + }), + path: f.path.clone(), + class, + tier, + confidence, + source: if project.is_some() { + "modrinth".into() + } else if f.curseforge.is_some() { + "curseforge".into() + } else { + "file".into() + }, + license, + client, + server, + categories, + depends: info.as_ref().map(|i| i.depends.clone()).unwrap_or_default(), + jar, + info, + key, + ..Record::default() + }); + } + // Dependencies: the jar's mod ids, else Modrinth's required projects. + let keys: std::collections::HashSet = records.iter().map(|r| r.key.clone()).collect(); + for (r, deps) in records.iter_mut().zip(project_deps) { + if r.depends.is_empty() { + r.depends = deps + .iter() + .map(|p| { + by_project + .get(p) + .cloned() + .unwrap_or_else(|| format!("modrinth:{p}")) + }) + .collect(); + } + let (inside, missing): (Vec, Vec) = + r.depends.drain(..).partition(|d| keys.contains(d)); + r.depends = inside; + r.missing = missing; + } + let graph: BTreeMap> = records + .iter() + .map(|r| (r.key.clone(), r.depends.clone())) + .collect(); + let layers = classify::layers(&graph); + for r in &mut records { + r.layer = layers.get(&r.key).copied().unwrap_or(0); + } + records.sort_by(|a, b| (a.layer, &a.key).cmp(&(b.layer, &b.key))); + records +} + +/// Counts by a field, largest first. +pub fn counts<'a>( + records: &'a [Record], + field: impl Fn(&'a Record) -> &'a str, +) -> Vec<(&'a str, usize)> { + let mut m: BTreeMap<&str, usize> = BTreeMap::new(); + for r in records { + *m.entry(field(r)).or_default() += 1; + } + let mut v: Vec<_> = m.into_iter().collect(); + v.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0))); + v +} + +/// `inventory.csv`: the columns of `docs/modpacks/atm10-mods.csv`, plus the layer. +pub fn csv(records: &[Record]) -> String { + let field = |s: &str| { + if s.contains([',', '"', '\n']) { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.to_owned() + } + }; + let mut out = String::from( + "mod_id,name,version,class,tier,confidence,source,license,client,server,layer\n", + ); + for r in records { + let row = [ + &r.key, + &r.name, + &r.version, + &r.class, + &r.tier, + &r.confidence, + &r.source, + &r.license, + &r.client, + &r.server, + ]; + let cells: Vec = row.iter().map(|s| field(s)).collect(); + out.push_str(&format!("{},{}\n", cells.join(","), r.layer)); + } + out +} diff --git a/crates/horizon-port-pack/src/jar.rs b/crates/horizon-port-pack/src/jar.rs new file mode 100644 index 000000000..67028755e --- /dev/null +++ b/crates/horizon-port-pack/src/jar.rs @@ -0,0 +1,245 @@ +//! What a mod jar holds, read without decompiling: its metadata +//! (`neoforge.mods.toml`, `mods.toml`, `fabric.mod.json`) and a census of +//! its data, assets and classes. +use serde::{Deserialize, Serialize}; +use std::{fs, io::Read, path::Path}; + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct JarInfo { + pub mod_id: String, + pub name: String, + pub version: String, + pub license: String, + /// Required dependencies by mod id (loaders and `minecraft` left out). + pub depends: Vec, + pub census: Census, +} + +#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] +pub struct Census { + pub blockstates: usize, + pub item_models: usize, + pub recipes: usize, + pub loot_tables: usize, + pub tags: usize, + pub worldgen: usize, + pub structures: usize, + pub languages: usize, + pub textures: usize, + pub sounds: usize, + pub classes: usize, + pub mixin_configs: usize, +} + +const NOT_MODS: [&str; 5] = ["minecraft", "neoforge", "forge", "fabricloader", "java"]; + +pub fn read(path: &Path) -> Result { + let file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut zip = zip::ZipArchive::new(file).map_err(|e| format!("{}: {e}", path.display()))?; + let text = |zip: &mut zip::ZipArchive, name: &str| -> Option { + let mut s = String::new(); + zip.by_name(name).ok()?.read_to_string(&mut s).ok()?; + Some(s) + }; + let mut info = if let Some(t) = text(&mut zip, "META-INF/neoforge.mods.toml") + .or_else(|| text(&mut zip, "META-INF/mods.toml")) + { + mods_toml(&t)? + } else if let Some(t) = text(&mut zip, "fabric.mod.json") { + fabric(&t)? + } else { + JarInfo::default() + }; + let names: Vec = zip.file_names().map(str::to_owned).collect(); + info.census = census(&names); + if info.mod_id.is_empty() { + info.mod_id = path + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_default(); + } + Ok(info) +} + +/// `[[mods]]` (the first one is the jar's mod), the top-level `license`, and +/// `[[dependencies.]]` with `type = "required"` (or the older +/// `mandatory = true`). +pub fn mods_toml(text: &str) -> Result { + let doc: toml::Table = toml::from_str(text).map_err(|e| format!("mods.toml: {e}"))?; + let first = doc + .get("mods") + .and_then(|m| m.as_array()) + .and_then(|m| m.first()) + .and_then(|m| m.as_table()); + let get = |t: Option<&toml::Table>, k: &str| { + t.and_then(|t| t.get(k)) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_owned() + }; + let mod_id = get(first, "modId"); + let mut depends = Vec::new(); + if let Some(deps) = doc + .get("dependencies") + .and_then(|d| d.get(&mod_id)) + .and_then(|d| d.as_array()) + { + for d in deps.iter().filter_map(|d| d.as_table()) { + let required = d.get("type").and_then(|t| t.as_str()).map_or_else( + || d.get("mandatory").and_then(|m| m.as_bool()) == Some(true), + |t| t.eq_ignore_ascii_case("required"), + ); + let id = get(Some(d), "modId"); + if required && !NOT_MODS.contains(&id.as_str()) && !id.is_empty() { + depends.push(id); + } + } + } + Ok(JarInfo { + name: get(first, "displayName"), + version: get(first, "version"), + license: get(Some(&doc), "license"), + mod_id, + depends, + census: Census::default(), + }) +} + +pub fn fabric(text: &str) -> Result { + let v: serde_json::Value = + serde_json::from_str(text).map_err(|e| format!("fabric.mod.json: {e}"))?; + let s = |k: &str| v[k].as_str().unwrap_or("").to_owned(); + let license = match &v["license"] { + serde_json::Value::Array(l) => l + .iter() + .filter_map(|x| x.as_str()) + .collect::>() + .join(" OR "), + l => l.as_str().unwrap_or("").to_owned(), + }; + let depends = v["depends"] + .as_object() + .into_iter() + .flatten() + .map(|(id, _)| id.clone()) + .filter(|id| !NOT_MODS.contains(&id.as_str()) && !id.starts_with("fabric")) + .collect(); + Ok(JarInfo { + mod_id: s("id"), + name: s("name"), + version: s("version"), + license, + depends, + census: Census::default(), + }) +} + +pub fn census(names: &[String]) -> Census { + let mut c = Census::default(); + for n in names { + let parts: Vec<&str> = n.split('/').collect(); + if n.ends_with(".class") { + c.classes += 1; + } else if n.ends_with(".mixins.json") || (parts.len() == 1 && n.contains("mixins")) { + c.mixin_configs += 1; + } + // assets///... and data///... + let (Some(root), Some(kind)) = (parts.first(), parts.get(2)) else { + continue; + }; + let json = n.ends_with(".json"); + match (*root, *kind) { + ("assets", "blockstates") if json => c.blockstates += 1, + ("assets", "models") if json && parts.get(3) == Some(&"item") => c.item_models += 1, + ("assets", "lang") => c.languages += 1, + ("assets", "textures") if n.ends_with(".png") => c.textures += 1, + ("assets", "sounds.json") => c.sounds += 1, + ("data", "recipe" | "recipes") if json => c.recipes += 1, + ("data", "loot_table" | "loot_tables") if json => c.loot_tables += 1, + ("data", "tags") if json => c.tags += 1, + ("data", "worldgen") if json => c.worldgen += 1, + ("data", "structure" | "structures") if n.ends_with(".nbt") => c.structures += 1, + _ => {} + } + } + c +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_neoforge_metadata() { + let info = mods_toml( + r#"modLoader = "javafml" +license = "MIT" +[[mods]] +modId = "ruby" +version = "1.2.0" +displayName = "Ruby" +[[dependencies.ruby]] +modId = "neoforge" +type = "required" +[[dependencies.ruby]] +modId = "geckolib" +type = "required" +[[dependencies.ruby]] +modId = "jei" +type = "optional" +[[dependencies.ruby]] +modId = "curios" +mandatory = true +"#, + ) + .unwrap(); + assert_eq!( + ( + info.mod_id.as_str(), + info.version.as_str(), + info.license.as_str() + ), + ("ruby", "1.2.0", "MIT") + ); + assert_eq!(info.depends, ["geckolib", "curios"]); + } + + #[test] + fn reads_fabric_metadata() { + let info = fabric( + r#"{"id": "gem", "version": "3", "name": "Gem", "license": ["MIT", "Apache-2.0"], + "depends": {"fabricloader": "*", "fabric-api": "*", "minecraft": "1.21.1", "cloth-config": "*"}}"#, + ) + .unwrap(); + assert_eq!(info.license, "MIT OR Apache-2.0"); + assert_eq!(info.depends, ["cloth-config"]); + } + + #[test] + fn counts_files_by_kind() { + let names = [ + "assets/ruby/blockstates/ruby_block.json", + "assets/ruby/models/item/ruby.json", + "assets/ruby/models/block/ruby_block.json", + "assets/ruby/textures/item/ruby.png", + "assets/ruby/lang/en_us.json", + "data/ruby/recipe/ruby.json", + "data/ruby/loot_table/blocks/ruby_block.json", + "data/minecraft/tags/block/mineable/pickaxe.json", + "data/ruby/structure/hut.nbt", + "com/example/Ruby.class", + "ruby.mixins.json", + ] + .map(String::from); + let c = census(&names); + assert_eq!( + (c.blockstates, c.item_models, c.textures, c.languages), + (1, 1, 1, 1) + ); + assert_eq!( + (c.recipes, c.loot_tables, c.tags, c.structures), + (1, 1, 1, 1) + ); + assert_eq!((c.classes, c.mixin_configs), (1, 1)); + } +} diff --git a/crates/horizon-port-pack/src/main.rs b/crates/horizon-port-pack/src/main.rs new file mode 100644 index 000000000..200e42334 --- /dev/null +++ b/crates/horizon-port-pack/src/main.rs @@ -0,0 +1,104 @@ +//! `horizon-port-pack`: port a Java modpack to Horizon (`docs/modpacks.md`). +//! +//! - `import `: a `.mrpack`, CurseForge zip or instance folder +//! to `/pack.json`. +//! - `inventory [--offline]`: Modrinth metadata (cached in +//! `/modrinth.json`), the jars in the instance or `/jars/`, +//! classes, tiers and dependency layers in `/inventory.json` and +//! `inventory.csv`. +//! +//! The work folder belongs to the user: jars stay there and are never +//! uploaded or committed. +mod classify; +mod import; +mod inventory; +mod jar; +mod modrinth; + +use std::{fs, path::Path, process::ExitCode}; + +const USAGE: &str = "usage: horizon-port-pack import | inventory [--offline]"; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + let path = |i: usize| args.get(i).map(Path::new).ok_or(USAGE.to_owned()); + let result = match args.first().map(String::as_str) { + Some("import") => path(1) + .and_then(|p| Ok((p, path(2)?))) + .and_then(|(p, w)| import(p, w)), + Some("inventory") => { + path(1).and_then(|w| inventory(w, args.iter().any(|a| a == "--offline"))) + } + _ => Err(USAGE.to_owned()), + }; + match result { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} + +fn write_json(path: &Path, value: &impl serde::Serialize) -> Result<(), String> { + let text = serde_json::to_string_pretty(value).map_err(|e| e.to_string())?; + fs::write(path, text).map_err(|e| format!("{}: {e}", path.display())) +} + +fn read_json(path: &Path) -> Result { + let text = fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display())) +} + +fn import(pack: &Path, work: &Path) -> Result<(), String> { + let pack = import::import(pack)?; + fs::create_dir_all(work).map_err(|e| format!("{}: {e}", work.display()))?; + write_json(&work.join("pack.json"), &pack)?; + let mut kinds = std::collections::BTreeMap::<&str, usize>::new(); + for f in &pack.files { + *kinds.entry(f.kind()).or_default() += 1; + } + println!( + "{} ({}): Minecraft {}, {} {}; files {kinds:?}; overrides {:?}", + pack.name, pack.format, pack.minecraft, pack.loader, pack.loader_version, pack.overrides + ); + Ok(()) +} + +fn inventory(work: &Path, offline: bool) -> Result<(), String> { + let pack: import::Pack = read_json(&work.join("pack.json"))?; + let cache_path = work.join("modrinth.json"); + let mut cache = modrinth::Cache::load(&cache_path); + if !offline { + let hashes: Vec = pack.files.iter().filter_map(|f| f.sha1.clone()).collect(); + cache.fetch(&hashes)?; + write_json(&cache_path, &cache)?; + } + let records = inventory::build(&pack, &cache, work); + write_json(&work.join("inventory.json"), &records)?; + fs::write(work.join("inventory.csv"), inventory::csv(&records)) + .map_err(|e| format!("inventory.csv: {e}"))?; + let jars = records.iter().filter(|r| r.jar.is_some()).count(); + println!("{} files, {jars} jars read", records.len()); + for (title, field) in [ + ( + "class", + (|r: &inventory::Record| r.class.as_str()) as fn(&inventory::Record) -> &str, + ), + ("tier", |r| r.tier.as_str()), + ("confidence", |r| r.confidence.as_str()), + ("source", |r| r.source.as_str()), + ] { + println!("{title}: {:?}", inventory::counts(&records, field)); + } + let layers = records.iter().map(|r| r.layer).max().map_or(0, |m| m + 1); + let per_layer: Vec = (0..layers) + .map(|l| records.iter().filter(|r| r.layer == l).count()) + .collect(); + println!("layers: {per_layer:?}"); + let missing: usize = records.iter().map(|r| r.missing.len()).sum(); + if missing > 0 { + println!("{missing} required dependencies are not in the pack (see inventory.json)"); + } + Ok(()) +} diff --git a/crates/horizon-port-pack/src/modrinth.rs b/crates/horizon-port-pack/src/modrinth.rs new file mode 100644 index 000000000..8c34a2004 --- /dev/null +++ b/crates/horizon-port-pack/src/modrinth.rs @@ -0,0 +1,113 @@ +//! Modrinth metadata for the pack's files: the version of each file by +//! SHA-1, and its project (categories, licence, sides). Only metadata: no +//! jar is downloaded. Answers are cached in `/modrinth.json`, so a +//! rerun works offline. +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{collections::BTreeMap, path::Path}; + +const API: &str = "https://api.modrinth.com/v2"; +const AGENT: &str = "ludovic111/horizon horizon-port-pack/0.1 (docs/modpacks.md)"; + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct Cache { + /// Version objects by file SHA-1. + pub versions: BTreeMap, + /// Project objects by project id. + pub projects: BTreeMap, +} + +impl Cache { + pub fn load(path: &Path) -> Self { + std::fs::read_to_string(path) + .ok() + .and_then(|t| serde_json::from_str(&t).ok()) + .unwrap_or_default() + } + + /// Ask Modrinth about the hashes and projects not cached yet. + pub fn fetch(&mut self, hashes: &[String]) -> Result<(), String> { + let missing: Vec<&String> = hashes + .iter() + .filter(|h| !self.versions.contains_key(*h)) + .collect(); + for chunk in missing.chunks(500) { + let body = serde_json::json!({"hashes": chunk, "algorithm": "sha1"}); + let found = post(&format!("{API}/version_files"), &body)?; + for (hash, version) in found.as_object().into_iter().flatten() { + self.versions.insert(hash.clone(), version.clone()); + } + } + let mut ids: Vec = self + .versions + .values() + .filter_map(|v| v["project_id"].as_str().map(str::to_owned)) + .filter(|id| !self.projects.contains_key(id)) + .collect(); + ids.sort(); + ids.dedup(); + for chunk in ids.chunks(100) { + let list = serde_json::to_string(chunk).unwrap_or_default(); + let found = get(&format!("{API}/projects?ids={}", encode(&list)))?; + for project in found.as_array().into_iter().flatten() { + if let Some(id) = project["id"].as_str() { + self.projects.insert(id.to_owned(), project.clone()); + } + } + } + Ok(()) + } + + pub fn project_of(&self, sha1: &str) -> Option<&Value> { + let id = self.versions.get(sha1)?["project_id"].as_str()?; + self.projects.get(id) + } + + /// Required dependencies of a file's version, as project ids. + pub fn required(&self, sha1: &str) -> Vec { + let Some(v) = self.versions.get(sha1) else { + return Vec::new(); + }; + v["dependencies"] + .as_array() + .into_iter() + .flatten() + .filter(|d| d["dependency_type"] == "required") + .filter_map(|d| d["project_id"].as_str().map(str::to_owned)) + .collect() + } +} + +fn post(url: &str, body: &Value) -> Result { + let response = ureq::post(url) + .set("User-Agent", AGENT) + .set("Content-Type", "application/json") + .send_string(&body.to_string()) + .map_err(|e| format!("{url}: {e}"))?; + read(url, response) +} + +fn get(url: &str) -> Result { + let response = ureq::get(url) + .set("User-Agent", AGENT) + .call() + .map_err(|e| format!("{url}: {e}"))?; + read(url, response) +} + +fn read(url: &str, response: ureq::Response) -> Result { + let text = response.into_string().map_err(|e| format!("{url}: {e}"))?; + serde_json::from_str(&text).map_err(|e| format!("{url}: {e}")) +} + +/// Percent-encoding for a query value. +fn encode(s: &str) -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + (b as char).to_string() + } + _ => format!("%{b:02X}"), + }) + .collect() +} diff --git a/docs/modpacks.md b/docs/modpacks.md index 54dc43bdc..753299fda 100644 --- a/docs/modpacks.md +++ b/docs/modpacks.md @@ -262,6 +262,20 @@ inspected: └── coverage.json, coverage.md ``` +**Status.** `import` and `inventory` run today, `carry` follows (see below): + +``` +cargo run -p horizon-port-pack -- import ~/Downloads/pack.mrpack ~/.horizon/ports/pack +cargo run -p horizon-port-pack -- inventory ~/.horizon/ports/pack # --offline reuses modrinth.json +``` + +`inventory` reads metadata only: Modrinth answers by file hash, and the jars only if the user's +instance has them (an imported instance folder, or jars put in `/jars/`). Without jars, +classes come from the reviewed table, Modrinth categories and sides; with jars, from their +`mods.toml` / `fabric.mod.json` and file census too. On Create+ 6.0.0 alpha f (155 files, metadata +only): content 54, library 33, client-ui 26, performance 25, other 14, unknown 2, server-only 1; +40 rows matched the reviewed ATM10 table; three dependency layers (93 / 57 / 5). + ### 1. Import | Format | Read | Jars come from |