diff --git a/crates/horizon-port-pack/src/fetch.rs b/crates/horizon-port-pack/src/fetch.rs new file mode 100644 index 000000000..011e99e1f --- /dev/null +++ b/crates/horizon-port-pack/src/fetch.rs @@ -0,0 +1,389 @@ +//! Step 1b, the jars: for the mods the player asked to port, their own +//! download of each jar into `/jars/`. Only from where the pack itself +//! points (a Modrinth pack's download URLs) or the same file on Modrinth's +//! public CDN (found by its SHA-1), and only kept when the hash matches; or +//! copied from a folder the player already has (an installed instance, a +//! downloads folder). CurseForge's own downloads need an API key, so a file +//! only CurseForge has is listed as needing one. `/fetch.json` says +//! what happened to each file. +use crate::{ + import::{Pack, PackFile, sha1_hex}, + inventory::Record, + modrinth::{self, Cache}, + pipeline::{Log, read_json, write_json}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{ + collections::HashMap, + fs, + io::Read, + path::{Path, PathBuf}, +}; + +/// Hosts a Modrinth pack may download from (the `.mrpack` format's list). +const HOSTS: [&str; 4] = [ + "https://cdn.modrinth.com/", + "https://github.com/", + "https://raw.githubusercontent.com/", + "https://gitlab.com/", +]; +/// No mod jar is this large; a bigger answer is not one. +const MAX_JAR: u64 = 512 << 20; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum How { + /// Already in `jars/` (or the imported instance). + Present, + /// Downloaded from the pack's URL or Modrinth's CDN, hash checked. + Downloaded, + /// Copied from the player's folder. + Local, + /// Only on CurseForge, whose downloads need an API key. + NeedsKey, + NotFound, + Failed, +} + +impl How { + pub fn label(self) -> &'static str { + match self { + How::Present => "already here", + How::Downloaded => "downloaded", + How::Local => "copied from your folder", + How::NeedsKey => "needs a CurseForge key", + How::NotFound => "not found", + How::Failed => "failed", + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Outcome { + pub key: String, + pub file: String, + pub how: How, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub detail: String, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct Fetched { + pub files: Vec, +} + +impl Fetched { + pub fn count(&self, how: How) -> usize { + self.files.iter().filter(|o| o.how == how).count() + } + /// `/fetch.json`, empty when the step has not run. + pub fn load(work: &Path) -> Self { + read_json(&work.join("fetch.json")).unwrap_or_default() + } +} + +/// The jars of the records `want` picks that the work folder lacks. +pub fn fetch( + work: &Path, + records: &[Record], + from: Option<&Path>, + want: &dyn Fn(&Record) -> bool, + log: Log, +) -> Result { + let pack: Pack = read_json(&work.join("pack.json"))?; + let cache = Cache::load(&work.join("modrinth.json")); + let files: HashMap<&str, &PackFile> = pack.files.iter().map(|f| (f.path.as_str(), f)).collect(); + let jars = work.join("jars"); + fs::create_dir_all(&jars).map_err(|e| format!("{}: {e}", jars.display()))?; + let local = from.map(Local::index); + let wanted: Vec<&Record> = records.iter().filter(|r| want(r)).collect(); + let mut fetched = Fetched::default(); + for (i, r) in wanted.iter().enumerate() { + let Some(file) = files.get(r.path.as_str()) else { + continue; + }; + let name = file.file_name(); + let outcome = |how, detail: String| Outcome { + key: r.key.clone(), + file: name.to_owned(), + how, + detail, + }; + let dest = jars.join(name); + let sha1 = file.sha1.as_deref(); + let o = if r.jar.is_some() || matches(&dest, sha1) { + outcome(How::Present, String::new()) + } else if let Some(path) = local.as_ref().and_then(|l| l.find(file)) { + match fs::copy(&path, &dest) { + Ok(_) => outcome(How::Local, path.display().to_string()), + Err(e) => outcome(How::Failed, format!("{}: {e}", path.display())), + } + } else if let Some(url) = url_of(file, &cache) { + log(format!("{}/{} {name}", i + 1, wanted.len())); + match download(&url, sha1, &dest) { + Ok(()) => outcome(How::Downloaded, url), + Err(e) => outcome(How::Failed, e), + } + } else if pack.format == "curseforge" || file.curseforge.is_some() { + outcome( + How::NeedsKey, + "on CurseForge only: choose your installed instance, or put the jar in jars/" + .into(), + ) + } else { + outcome(How::NotFound, "no download link in the pack".into()) + }; + if matches!(o.how, How::Failed | How::NeedsKey | How::NotFound) { + log(format!("{}: {} ({})", o.key, o.how.label(), o.detail)); + } + fetched.files.push(o); + } + write_json(&work.join("fetch.json"), &fetched)?; + log(format!( + "jars: {} downloaded, {} copied, {} already here, {} need a CurseForge key, {} not found, {} failed", + fetched.count(How::Downloaded), + fetched.count(How::Local), + fetched.count(How::Present), + fetched.count(How::NeedsKey), + fetched.count(How::NotFound), + fetched.count(How::Failed) + )); + Ok(fetched) +} + +/// Whether `path` holds the file with this SHA-1 (any file when none is known). +fn matches(path: &Path, sha1: Option<&str>) -> bool { + match (fs::read(path), sha1) { + (Ok(bytes), Some(h)) => sha1_hex(&bytes).eq_ignore_ascii_case(h), + (Ok(_), None) => true, + (Err(_), _) => false, + } +} + +/// The pack's own download link, else the same file on Modrinth. +pub fn url_of(file: &PackFile, cache: &Cache) -> Option { + let allowed = |u: &&str| HOSTS.iter().any(|h| u.starts_with(h)); + if let Some(u) = file.downloads.iter().map(String::as_str).find(allowed) { + return Some(u.to_owned()); + } + let sha1 = file.sha1.as_deref()?; + cache.versions.get(sha1)?["files"] + .as_array()? + .iter() + .find(|f| f["hashes"]["sha1"].as_str() == Some(sha1))?["url"] + .as_str() + .filter(allowed) + .map(str::to_owned) +} + +/// Downloads `url` to `dest`, kept only if its SHA-1 is `sha1`. +fn download(url: &str, sha1: Option<&str>, dest: &Path) -> Result<(), String> { + let response = ureq::get(url) + .set("User-Agent", modrinth::AGENT) + .call() + .map_err(|e| format!("{url}: {e}"))?; + let mut bytes = Vec::new(); + response + .into_reader() + .take(MAX_JAR) + .read_to_end(&mut bytes) + .map_err(|e| format!("{url}: {e}"))?; + let got = sha1_hex(&bytes); + if let Some(want) = sha1.filter(|w| !w.eq_ignore_ascii_case(&got)) { + return Err(format!("{url}: SHA-1 {got}, the pack says {want}")); + } + let part = dest.with_extension("part"); + fs::write(&part, &bytes).map_err(|e| format!("{}: {e}", part.display()))?; + fs::rename(&part, dest).map_err(|e| format!("{}: {e}", dest.display())) +} + +/// The jars in a folder of the player's, by SHA-1 and by name, and what a +/// CurseForge app instance (`minecraftinstance.json`) says each +/// project's file is called. +#[derive(Default)] +struct Local { + by_sha1: HashMap, + by_name: HashMap, + curseforge: HashMap<(u64, u64), String>, +} + +impl Local { + fn index(dir: &Path) -> Local { + let mut local = Local::default(); + local.walk(dir, 3); + if let Ok(text) = fs::read_to_string(dir.join("minecraftinstance.json")) { + if let Ok(v) = serde_json::from_str::(&text) { + local.curseforge = curseforge_names(&v); + } + } + local + } + + fn walk(&mut self, dir: &Path, depth: usize) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for path in entries.flatten().map(|e| e.path()) { + if path.is_dir() { + if depth > 0 { + self.walk(&path, depth - 1); + } + } else if path.extension().is_some_and(|x| x == "jar") { + if let Ok(bytes) = fs::read(&path) { + self.by_sha1.insert(sha1_hex(&bytes), path.clone()); + } + if let Some(n) = path.file_name() { + self.by_name + .insert(n.to_string_lossy().into_owned(), path.clone()); + } + } + } + } + + fn find(&self, file: &PackFile) -> Option { + if let Some(h) = &file.sha1 { + return self.by_sha1.get(&h.to_ascii_lowercase()).cloned(); + } + let name = file + .curseforge + .and_then(|ids| self.curseforge.get(&ids)) + .map_or(file.file_name(), String::as_str); + self.by_name.get(name).cloned() + } +} + +/// `installedAddons[]`: (project, file) → the jar's file name. +pub fn curseforge_names(instance: &Value) -> HashMap<(u64, u64), String> { + instance["installedAddons"] + .as_array() + .into_iter() + .flatten() + .filter_map(|a| { + let file = &a["installedFile"]; + Some(( + (a["addonID"].as_u64()?, file["id"].as_u64()?), + file["fileName"].as_str()?.to_owned(), + )) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn file(sha1: Option<&str>, downloads: &[&str]) -> PackFile { + PackFile { + path: "mods/a.jar".into(), + sha1: sha1.map(str::to_owned), + downloads: downloads.iter().map(|d| (*d).to_owned()).collect(), + ..PackFile::default() + } + } + + #[test] + fn links_come_from_the_pack_or_modrinth_and_only_allowed_hosts() { + let mut cache = Cache::default(); + cache.versions.insert( + "ab".into(), + json!({"files": [ + {"hashes": {"sha1": "cd"}, "url": "https://cdn.modrinth.com/other.jar"}, + {"hashes": {"sha1": "ab"}, "url": "https://cdn.modrinth.com/a.jar"}]}), + ); + let pack_link = file( + Some("ab"), + &["https://evil.example/a.jar", "https://github.com/a.jar"], + ); + assert_eq!( + url_of(&pack_link, &cache).as_deref(), + Some("https://github.com/a.jar") + ); + let modrinth = file(Some("ab"), &["https://evil.example/a.jar"]); + assert_eq!( + url_of(&modrinth, &cache).as_deref(), + Some("https://cdn.modrinth.com/a.jar") + ); + assert_eq!(url_of(&file(Some("ef"), &[]), &cache), None); + assert_eq!(url_of(&file(None, &[]), &cache), None); + } + + #[test] + fn reads_a_curseforge_instance() { + let instance = json!({"installedAddons": [ + {"addonID": 238222, "installedFile": {"id": 5101366, "fileName": "jei-1.21.1.jar"}}, + {"addonID": 1, "installedFile": {"fileName": "no-id.jar"}}]}); + let names = curseforge_names(&instance); + assert_eq!(names.len(), 1); + assert_eq!(names[&(238222, 5101366)], "jei-1.21.1.jar"); + } + + #[test] + fn copies_the_players_jars_and_lists_what_needs_a_key() { + let root = std::env::temp_dir().join(format!("horizon-fetch-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + let (work, mine) = (root.join("work"), root.join("instance/mods")); + fs::create_dir_all(&mine).unwrap(); + fs::create_dir_all(&work).unwrap(); + fs::write(mine.join("renamed.jar"), b"jar a").unwrap(); + fs::write(mine.join("cf.jar"), b"jar b").unwrap(); + fs::write( + root.join("instance/minecraftinstance.json"), + json!({"installedAddons": [{"addonID": 7, "installedFile": {"id": 8, "fileName": "cf.jar"}}]}) + .to_string(), + ) + .unwrap(); + let pack = Pack { + format: "curseforge".into(), + files: vec![ + PackFile { + path: "mods/a.jar".into(), + sha1: Some(sha1_hex(b"jar a")), + ..PackFile::default() + }, + PackFile { + path: "mods/curseforge-7-8".into(), + curseforge: Some((7, 8)), + ..PackFile::default() + }, + PackFile { + path: "mods/curseforge-9-9".into(), + curseforge: Some((9, 9)), + ..PackFile::default() + }, + ], + ..Pack::default() + }; + write_json(&work.join("pack.json"), &pack).unwrap(); + let records: Vec = pack + .files + .iter() + .map(|f| Record { + key: f.file_name().into(), + path: f.path.clone(), + ..Record::default() + }) + .collect(); + let fetched = fetch( + &work, + &records, + Some(&root.join("instance")), + &|_| true, + &|_| {}, + ) + .unwrap(); + let hows: Vec = fetched.files.iter().map(|o| o.how).collect(); + assert_eq!(hows, [How::Local, How::Local, How::NeedsKey]); + assert_eq!(fs::read(work.join("jars/a.jar")).unwrap(), b"jar a"); + assert_eq!( + fs::read(work.join("jars/curseforge-7-8")).unwrap(), + b"jar b" + ); + // A second run finds them in place. + let again = fetch(&work, &records, None, &|_| true, &|_| {}).unwrap(); + assert_eq!(again.count(How::Present), 2); + assert_eq!(Fetched::load(&work).count(How::Present), 2); + fs::remove_dir_all(&root).unwrap(); + } +} diff --git a/crates/horizon-port-pack/src/import.rs b/crates/horizon-port-pack/src/import.rs index dd712acf1..0a3aa8d06 100644 --- a/crates/horizon-port-pack/src/import.rs +++ b/crates/horizon-port-pack/src/import.rs @@ -22,8 +22,20 @@ pub struct Pack { pub files: Vec, /// Override files by top folder (`config`, `kubejs`, ...). pub overrides: BTreeMap, + /// A CurseForge zip's files by project and file id only: how many its + /// manifest lists, when the files came from the pack's own mod list. + #[serde(default, skip_serializing_if = "is_zero")] + pub curseforge_files: usize, } +fn is_zero(n: &usize) -> bool { + *n == 0 +} + +/// Where Crash Assistant keeps its list in an instance (and in the +/// overrides of packs that ship it, such as All the Mods 10). +pub const MODLIST: &str = "config/crash_assistant/modlist.json"; + #[derive(Debug, Default, Serialize, Deserialize)] pub struct PackFile { /// Path in the instance (`mods/create-1.21.1-6.0.4.jar`). @@ -83,7 +95,16 @@ pub fn import(path: &Path) -> Result { return Ok(mrpack(&index, &names(&zip))); } if let Some(manifest) = read_json(&mut zip, "manifest.json")? { - return Ok(curseforge(&manifest, &names(&zip))); + let mut pack = curseforge(&manifest, &names(&zip)); + // The manifest names files only by CurseForge ids, which need a key + // to resolve; the pack's own mod list names them and gives their + // SHA-1s, so Modrinth can answer for most. + let root = manifest["overrides"].as_str().unwrap_or("overrides"); + if let Some(list) = read_json(&mut zip, &format!("{root}/{MODLIST}"))? { + pack.curseforge_files = pack.files.len(); + pack.files = modlist(&list)?.files; + } + return Ok(pack); } Err(format!( "{}: neither modrinth.index.json nor manifest.json", @@ -103,7 +124,7 @@ fn read_json(zip: &mut zip::ZipArchive, name: &str) -> Result Pack { names, &["overrides", "client-overrides", "server-overrides"], ), + ..Pack::default() } } @@ -197,6 +219,7 @@ pub fn curseforge(manifest: &Value, names: &[String]) -> Pack { loader_version: loader_version.to_owned(), files, overrides: overrides(names, &[root]), + ..Pack::default() } } @@ -418,4 +441,36 @@ mod tests { assert_eq!(p.files[0].curseforge, Some((238222, 5101366))); assert_eq!(p.overrides["kubejs"], 1); } + + #[test] + fn a_curseforge_zip_takes_names_from_its_mod_list() { + use std::io::Write; + let dir = std::env::temp_dir().join(format!("horizon-cf-zip-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let path = dir.join("pack.zip"); + let mut zip = zip::ZipWriter::new(fs::File::create(&path).unwrap()); + let opts = zip::write::SimpleFileOptions::default(); + let manifest = json!({"name": "Pack", "overrides": "overrides", + "minecraft": {"version": "1.21.1", "modLoaders": [{"id": "neoforge-21.1.259"}]}, + "files": [{"projectID": 1, "fileID": 2}, {"projectID": 3, "fileID": 4}]}); + zip.start_file("manifest.json", opts).unwrap(); + zip.write_all(manifest.to_string().as_bytes()).unwrap(); + zip.start_file(format!("overrides/{MODLIST}"), opts) + .unwrap(); + let list = json!({"mcw-doors-1.1.2.jar": {"jarName": "mcw-doors-1.1.2.jar", + "modId": "mcwdoors", "modrinthHash": "0123456789abcdef0123456789abcdef01234567"}}); + // Crash Assistant writes a byte order mark. + zip.write_all(format!("\u{feff}{list}").as_bytes()).unwrap(); + zip.finish().unwrap(); + let pack = import(&path).unwrap(); + assert_eq!( + (pack.format.as_str(), pack.name.as_str()), + ("curseforge", "Pack") + ); + assert_eq!(pack.curseforge_files, 2); + assert_eq!(pack.files.len(), 1); + assert_eq!(pack.files[0].path, "mods/mcw-doors-1.1.2.jar"); + assert_eq!(pack.overrides["config"], 1); + fs::remove_dir_all(&dir).unwrap(); + } } diff --git a/crates/horizon-port-pack/src/lib.rs b/crates/horizon-port-pack/src/lib.rs new file mode 100644 index 000000000..ecd61956c --- /dev/null +++ b/crates/horizon-port-pack/src/lib.rs @@ -0,0 +1,17 @@ +//! Port a Java modpack to Horizon (`docs/modpacks.md`): the steps behind the +//! `horizon-port-pack` command, and the launcher's "Import modpack". +//! +//! Every step reads and writes files in a work folder (`pipeline`), so it can +//! be rerun, resumed and inspected. The work folder belongs to the user: the +//! jars they downloaded and what is carried over from them stay there and +//! are never uploaded or committed. +pub mod carry; +pub mod classify; +pub mod fetch; +pub mod import; +pub mod inventory; +pub mod jar; +pub mod modrinth; +pub mod oracle; +pub mod pipeline; +pub mod report; diff --git a/crates/horizon-port-pack/src/main.rs b/crates/horizon-port-pack/src/main.rs index 3e6cdf4d7..e766a442d 100644 --- a/crates/horizon-port-pack/src/main.rs +++ b/crates/horizon-port-pack/src/main.rs @@ -6,57 +6,77 @@ //! `/modrinth.json`), the jars in the instance or `/jars/`, //! classes, tiers and dependency layers in `/inventory.json` and //! `inventory.csv`. +//! - `fetch [--from ] [...]`: the jars of the +//! portable mods (content, data tier; or the ones named) into +//! `/jars/`: copied from the player's folder, or downloaded from the +//! pack's own links or Modrinth's CDN, hash checked. What only CurseForge +//! has needs a key and is listed in `/fetch.json`. //! - `carry [--resources ] [...]`: the content //! mods' data and assets into `/mods//carry/`, a folder mod for //! local use. With the NeoForge oracle's `/oracle/facts.json` //! (`tools/neoforge_oracle.py`), blocks and items come from its facts; //! `--resources` limits templates to the blocks Horizon's tables have. -//! //! - `report `: the coverage report, per mod ported, partial, not yet //! or not needed with the reason, in `/coverage.json` and //! `coverage.md`. +//! - `port [--from ] [--resources ]`: every +//! step for the portable mods, as the launcher's "Import modpack" runs it. //! //! The work folder belongs to the user: jars and carried-over files stay //! there and are never uploaded or committed. -mod carry; -mod classify; -mod import; -mod inventory; -mod jar; -mod modrinth; -mod oracle; -mod report; +use horizon_port_pack::{fetch, inventory::Record, pipeline}; +use std::{ + path::{Path, PathBuf}, + process::ExitCode, +}; -use std::{fs, path::Path, process::ExitCode}; - -const USAGE: &str = "usage: horizon-port-pack import | inventory [--offline] | carry [--resources ] [...] | report "; +const USAGE: &str = "usage: horizon-port-pack import | inventory [--offline] | fetch [--from ] [...] | carry [--resources ] [...] | report | port [--from ] [--resources ]"; fn main() -> ExitCode { - let args: Vec = std::env::args().skip(1).collect(); - let path = |i: usize| args.get(i).map(Path::new).ok_or(USAGE.to_owned()); - let result = match args.first().map(String::as_str) { - Some("import") => path(1) - .and_then(|p| Ok((p, path(2)?))) - .and_then(|(p, w)| import(p, w)), - Some("inventory") => { - path(1).and_then(|w| inventory(w, args.iter().any(|a| a == "--offline"))) + let mut args: Vec = std::env::args().skip(1).collect(); + let result = (|| { + let from = option(&mut args, "--from")?; + let resources = option(&mut args, "--resources")?; + let offline = take_flag(&mut args, "--offline"); + let path = |i: usize| args.get(i).map(Path::new).ok_or(USAGE.to_owned()); + let rest = || args.get(2..).unwrap_or_default().to_vec(); + let log = &|line: String| println!("{line}"); + match args.first().map(String::as_str) { + Some("import") => pipeline::import(path(1)?, path(2)?, log).map(drop), + Some("inventory") => pipeline::inventory(path(1)?, offline, log).map(drop), + Some("fetch") => { + let work = path(1)?; + let records: Vec = pipeline::read_json(&work.join("inventory.json"))?; + let only = rest(); + let want = |r: &Record| { + if only.is_empty() { + pipeline::portable(r) + } else { + only.contains(&r.key) + } + }; + fetch::fetch(work, &records, from.as_deref(), &want, log).map(drop) + } + Some("carry") => { + pipeline::carry(path(1)?, resources.as_deref(), &rest(), log).map(drop) + } + Some("report") => { + let work = path(1)?; + pipeline::report(work, log)?; + println!("{}", work.join("coverage.md").display()); + Ok(()) + } + Some("port") => { + let work = path(2)?; + pipeline::import(path(1)?, work, log)?; + pipeline::inventory(work, offline, log)?; + pipeline::port(work, resources.as_deref(), from.as_deref(), log)?; + println!("{}", work.join("coverage.md").display()); + Ok(()) + } + _ => Err(USAGE.to_owned()), } - Some("carry") => path(1).and_then(|w| { - let mut rest = args[2..].to_vec(); - let resources = match rest.iter().position(|a| a == "--resources") { - Some(i) if i + 1 < rest.len() => { - let r = rest.remove(i + 1); - rest.remove(i); - Some(std::path::PathBuf::from(r)) - } - Some(_) => return Err(USAGE.to_owned()), - None => None, - }; - carry(w, resources.as_deref(), &rest) - }), - Some("report") => path(1).and_then(report), - _ => Err(USAGE.to_owned()), - }; + })(); match result { Ok(()) => ExitCode::SUCCESS, Err(e) => { @@ -66,158 +86,21 @@ fn main() -> ExitCode { } } -fn write_json(path: &Path, value: &impl serde::Serialize) -> Result<(), String> { - let text = serde_json::to_string_pretty(value).map_err(|e| e.to_string())?; - fs::write(path, text).map_err(|e| format!("{}: {e}", path.display())) -} - -fn read_json(path: &Path) -> Result { - let text = fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; - serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display())) -} - -fn report(work: &Path) -> Result<(), String> { - let pack: import::Pack = read_json(&work.join("pack.json"))?; - let records: Vec = read_json(&work.join("inventory.json"))?; - let rows = report::build(&records, work); - write_json(&work.join("coverage.json"), &rows)?; - let title = format!( - "{} ({} {}, Minecraft {})", - pack.name, pack.loader, pack.loader_version, pack.minecraft - ); - fs::write(work.join("coverage.md"), report::markdown(&title, &rows)) - .map_err(|e| format!("coverage.md: {e}"))?; - println!("{title}: {} files: {:?}", rows.len(), report::counts(&rows)); - println!("{}", work.join("coverage.md").display()); - Ok(()) -} - -fn import(pack: &Path, work: &Path) -> Result<(), String> { - let pack = import::import(pack)?; - fs::create_dir_all(work).map_err(|e| format!("{}: {e}", work.display()))?; - write_json(&work.join("pack.json"), &pack)?; - let mut kinds = std::collections::BTreeMap::<&str, usize>::new(); - for f in &pack.files { - *kinds.entry(f.kind()).or_default() += 1; - } - println!( - "{} ({}): Minecraft {}, {} {}; files {kinds:?}; overrides {:?}", - pack.name, pack.format, pack.minecraft, pack.loader, pack.loader_version, pack.overrides - ); - Ok(()) -} - -fn inventory(work: &Path, offline: bool) -> Result<(), String> { - let pack: import::Pack = read_json(&work.join("pack.json"))?; - let cache_path = work.join("modrinth.json"); - let mut cache = modrinth::Cache::load(&cache_path); - if !offline { - let hashes: Vec = pack.files.iter().filter_map(|f| f.sha1.clone()).collect(); - cache.fetch(&hashes)?; - write_json(&cache_path, &cache)?; - } - let records = inventory::build(&pack, &cache, work); - write_json(&work.join("inventory.json"), &records)?; - fs::write(work.join("inventory.csv"), inventory::csv(&records)) - .map_err(|e| format!("inventory.csv: {e}"))?; - let jars = records.iter().filter(|r| r.jar.is_some()).count(); - println!("{} files, {jars} jars read", records.len()); - for (title, field) in [ - ( - "class", - (|r: &inventory::Record| r.class.as_str()) as fn(&inventory::Record) -> &str, - ), - ("tier", |r| r.tier.as_str()), - ("confidence", |r| r.confidence.as_str()), - ("source", |r| r.source.as_str()), - ] { - println!("{title}: {:?}", inventory::counts(&records, field)); - } - let layers = records.iter().map(|r| r.layer).max().map_or(0, |m| m + 1); - let per_layer: Vec = (0..layers) - .map(|l| records.iter().filter(|r| r.layer == l).count()) - .collect(); - println!("layers: {per_layer:?}"); - let missing: usize = records.iter().map(|r| r.missing.len()).sum(); - if missing > 0 { - println!("{missing} required dependencies are not in the pack (see inventory.json)"); - } - Ok(()) -} - -/// The vanilla blocks a prepared resource root has (`facts.json`'s defaults). -fn vanilla_blocks(resources: &Path) -> Result, String> { - let facts: serde_json::Value = read_json(&resources.join("facts.json"))?; - let defaults = facts["defaults"] - .as_object() - .ok_or("facts.json: no defaults")?; - Ok(defaults.keys().cloned().collect()) -} - -fn carry(work: &Path, resources: Option<&Path>, only: &[String]) -> Result<(), String> { - let records: Vec = read_json(&work.join("inventory.json"))?; - let facts_path = work.join("oracle/facts.json"); - let facts = if facts_path.exists() { - let f = oracle::Facts::read(&facts_path)?; - println!( - "oracle facts: {} blocks, {} items", - f.blocks.len(), - f.items.len() - ); - Some(f) - } else { - None - }; - let vanilla = resources.map(vanilla_blocks).transpose()?; - let allowed = |b: &str| vanilla.as_ref().is_none_or(|v| v.contains(b)); - let mut done = 0; - for r in &records { - let chosen = if only.is_empty() { - r.class == "content" - } else { - only.contains(&r.key) - }; - let (Some(jar), Some(info)) = (&r.jar, &r.info) else { - if chosen && !only.is_empty() { - eprintln!("{}: no jar (put it in {}/jars/)", r.key, work.display()); - } - continue; - }; - if !chosen { - continue; - } - match carry::carry( - jar, - info, - &work.join("mods").join(&r.key), - facts.as_ref(), - &allowed, - ) { - Ok(report) => { - done += 1; - println!( - "{}: {} files, {} recipes upgraded ({} kept), {} loot tables upgraded, {} item definitions added, {} blocks ({} left out, {} with differences), {} items ({} left out), {} convention tags, {} outside", - r.key, - report.copied, - report.recipes_upgraded, - report.recipes_kept.len(), - report.loot_tables_upgraded, - report.item_definitions_added, - report.blocks.len(), - report.blocks_unknown.len(), - report.differences.len(), - report.items, - report.items_unknown.len(), - report.convention_tags, - report.outside.len() - ); - } - Err(e) => eprintln!("{}: {e}", r.key), +/// Removes `--name ` from `args`. +fn option(args: &mut Vec, name: &str) -> Result, String> { + match args.iter().position(|a| a == name) { + Some(i) if i + 1 < args.len() => { + let value = args.remove(i + 1); + args.remove(i); + Ok(Some(PathBuf::from(value))) } + Some(_) => Err(USAGE.to_owned()), + None => Ok(None), } - println!( - "{done} mods carried over into {}", - work.join("mods").display() - ); - Ok(()) +} + +fn take_flag(args: &mut Vec, name: &str) -> bool { + let before = args.len(); + args.retain(|a| a != name); + args.len() != before } diff --git a/crates/horizon-port-pack/src/modrinth.rs b/crates/horizon-port-pack/src/modrinth.rs index 8c34a2004..4e90c871a 100644 --- a/crates/horizon-port-pack/src/modrinth.rs +++ b/crates/horizon-port-pack/src/modrinth.rs @@ -1,13 +1,13 @@ //! Modrinth metadata for the pack's files: the version of each file by -//! SHA-1, and its project (categories, licence, sides). Only metadata: no -//! jar is downloaded. Answers are cached in `/modrinth.json`, so a +//! SHA-1, and its project (categories, licence, sides). Only metadata (the +//! jars are `fetch`'s). Answers are cached in `/modrinth.json`, so a //! rerun works offline. use serde::{Deserialize, Serialize}; use serde_json::Value; use std::{collections::BTreeMap, path::Path}; const API: &str = "https://api.modrinth.com/v2"; -const AGENT: &str = "ludovic111/horizon horizon-port-pack/0.1 (docs/modpacks.md)"; +pub(crate) const AGENT: &str = "ludovic111/horizon horizon-port-pack/0.1 (docs/modpacks.md)"; #[derive(Debug, Default, Serialize, Deserialize)] pub struct Cache { diff --git a/crates/horizon-port-pack/src/pipeline.rs b/crates/horizon-port-pack/src/pipeline.rs new file mode 100644 index 000000000..ddb39bf66 --- /dev/null +++ b/crates/horizon-port-pack/src/pipeline.rs @@ -0,0 +1,237 @@ +//! The steps on a work folder, as the command line and the launcher run +//! them. Each one reads what the earlier steps wrote and says what it did +//! through `log`, one line at a time. +use crate::{carry, fetch, import, inventory, modrinth, oracle, report}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + path::Path, +}; + +/// Where a step's lines go: the terminal, or the launcher's log. +pub type Log<'a> = &'a dyn Fn(String); + +pub fn write_json(path: &Path, value: &impl serde::Serialize) -> Result<(), String> { + let text = serde_json::to_string_pretty(value).map_err(|e| e.to_string())?; + fs::write(path, text).map_err(|e| format!("{}: {e}", path.display())) +} + +pub fn read_json(path: &Path) -> Result { + let text = fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display())) +} + +/// Step 1: the pack at `pack` to `/pack.json`. +pub fn import(pack: &Path, work: &Path, log: Log) -> Result { + let pack = import::import(pack)?; + fs::create_dir_all(work).map_err(|e| format!("{}: {e}", work.display()))?; + write_json(&work.join("pack.json"), &pack)?; + let mut kinds = BTreeMap::<&str, usize>::new(); + for f in &pack.files { + *kinds.entry(f.kind()).or_default() += 1; + } + log(format!( + "{} ({}): Minecraft {}, {} {}; files {kinds:?}; overrides {:?}", + pack.name, pack.format, pack.minecraft, pack.loader, pack.loader_version, pack.overrides + )); + Ok(pack) +} + +/// Step 2: `/inventory.json` and `inventory.csv`. Online, Modrinth is +/// asked about the files it has not answered for yet; when it cannot be +/// reached, the cached answers and the reviewed table classify alone. +pub fn inventory(work: &Path, offline: bool, log: Log) -> Result, String> { + let pack: import::Pack = read_json(&work.join("pack.json"))?; + let cache_path = work.join("modrinth.json"); + let mut cache = modrinth::Cache::load(&cache_path); + if !offline { + let hashes: Vec = pack.files.iter().filter_map(|f| f.sha1.clone()).collect(); + match cache.fetch(&hashes) { + Ok(()) => write_json(&cache_path, &cache)?, + Err(e) => log(format!("Modrinth not reached, classifying offline: {e}")), + } + } + let records = inventory::build(&pack, &cache, work); + write_json(&work.join("inventory.json"), &records)?; + fs::write(work.join("inventory.csv"), inventory::csv(&records)) + .map_err(|e| format!("inventory.csv: {e}"))?; + let jars = records.iter().filter(|r| r.jar.is_some()).count(); + log(format!("{} files, {jars} jars read", records.len())); + for (title, field) in [ + ( + "class", + (|r: &inventory::Record| r.class.as_str()) as fn(&inventory::Record) -> &str, + ), + ("tier", |r| r.tier.as_str()), + ("confidence", |r| r.confidence.as_str()), + ("source", |r| r.source.as_str()), + ] { + log(format!("{title}: {:?}", inventory::counts(&records, field))); + } + let layers = records.iter().map(|r| r.layer).max().map_or(0, |m| m + 1); + let per_layer: Vec = (0..layers) + .map(|l| records.iter().filter(|r| r.layer == l).count()) + .collect(); + log(format!("layers: {per_layer:?}")); + let missing: usize = records.iter().map(|r| r.missing.len()).sum(); + if missing > 0 { + log(format!( + "{missing} required dependencies are not in the pack (see inventory.json)" + )); + } + Ok(records) +} + +/// The vanilla blocks a prepared resource root has (`facts.json`'s defaults). +fn vanilla_blocks(resources: &Path) -> Result, String> { + let facts: serde_json::Value = read_json(&resources.join("facts.json"))?; + let defaults = facts["defaults"] + .as_object() + .ok_or("facts.json: no defaults")?; + Ok(defaults.keys().cloned().collect()) +} + +/// Step 4: the content mods with a jar (or the ones in `only`) carried over +/// into `/mods//carry/`. Returns how many were. +pub fn carry( + work: &Path, + resources: Option<&Path>, + only: &[String], + log: Log, +) -> Result { + let records: Vec = read_json(&work.join("inventory.json"))?; + let facts_path = work.join("oracle/facts.json"); + let facts = if facts_path.exists() { + let f = oracle::Facts::read(&facts_path)?; + log(format!( + "oracle facts: {} blocks, {} items", + f.blocks.len(), + f.items.len() + )); + Some(f) + } else { + None + }; + // Without the prepared tables, templates are not limited to what the game has. + let vanilla = resources.and_then(|r| { + vanilla_blocks(r) + .inspect_err(|e| log(format!("templates not limited to the game's blocks: {e}"))) + .ok() + }); + let allowed = |b: &str| vanilla.as_ref().is_none_or(|v| v.contains(b)); + let mut done = 0; + for r in &records { + let chosen = if only.is_empty() { + r.class == "content" + } else { + only.contains(&r.key) + }; + let (Some(jar), Some(info)) = (&r.jar, &r.info) else { + if chosen && !only.is_empty() { + log(format!( + "{}: no jar (put it in {}/jars/)", + r.key, + work.display() + )); + } + continue; + }; + if !chosen { + continue; + } + match carry::carry( + jar, + info, + &work.join("mods").join(&r.key), + facts.as_ref(), + &allowed, + ) { + Ok(report) => { + done += 1; + log(format!( + "{}: {} files, {} recipes upgraded ({} kept), {} loot tables upgraded, {} item definitions added, {} blocks ({} left out, {} with differences), {} items ({} left out), {} convention tags, {} outside", + r.key, + report.copied, + report.recipes_upgraded, + report.recipes_kept.len(), + report.loot_tables_upgraded, + report.item_definitions_added, + report.blocks.len(), + report.blocks_unknown.len(), + report.differences.len(), + report.items, + report.items_unknown.len(), + report.convention_tags, + report.outside.len() + )); + } + Err(e) => log(format!("{}: {e}", r.key)), + } + } + log(format!( + "{done} mods carried over into {}", + work.join("mods").display() + )); + Ok(done) +} + +/// The pack's title as the report heads it. +pub fn title(pack: &import::Pack) -> String { + format!( + "{} ({} {}, Minecraft {})", + pack.name, pack.loader, pack.loader_version, pack.minecraft + ) +} + +/// Step 7: `/coverage.json` and `coverage.md`. +pub fn report(work: &Path, log: Log) -> Result, String> { + let pack: import::Pack = read_json(&work.join("pack.json"))?; + let records: Vec = read_json(&work.join("inventory.json"))?; + let rows = report::build(&records, work); + write_json(&work.join("coverage.json"), &rows)?; + let title = title(&pack); + fs::write(work.join("coverage.md"), report::markdown(&title, &rows)) + .map_err(|e| format!("coverage.md: {e}"))?; + log(format!( + "{title}: {} files: {:?}", + rows.len(), + report::counts(&rows) + )); + Ok(rows) +} + +/// What a player's machine can port today without an agent: content mods +/// of the data tier (carry-over only). +pub fn portable(r: &inventory::Record) -> bool { + r.class == "content" && r.tier == "data" +} + +/// Everything after the import, for the portable mods: their jars (`fetch`, +/// from `from` when the player has them), the inventory again with the jars +/// read, the carry-over and the report. `resources` is the prepared +/// version the ports will play on. +pub fn port( + work: &Path, + resources: Option<&Path>, + from: Option<&Path>, + log: Log, +) -> Result<(fetch::Fetched, Vec), String> { + let records = match read_json::>(&work.join("inventory.json")) { + Ok(r) => r, + Err(_) => inventory(work, false, log)?, + }; + let fetched = fetch::fetch(work, &records, from, &portable, log)?; + // The jars tell classes and tiers better than metadata alone. + let records = inventory(work, true, log)?; + let only: Vec = records + .iter() + .filter(|r| portable(r) && r.jar.is_some()) + .map(|r| r.key.clone()) + .collect(); + if only.is_empty() { + log("no portable mod has its jar: nothing to carry over".into()); + } else { + carry(work, resources, &only, log)?; + } + Ok((fetched, report(work, log)?)) +} diff --git a/crates/horizon-port-pack/src/report.rs b/crates/horizon-port-pack/src/report.rs index ffa025c79..a98dd0432 100644 --- a/crates/horizon-port-pack/src/report.rs +++ b/crates/horizon-port-pack/src/report.rs @@ -83,7 +83,7 @@ pub fn license_kind(license: &str) -> &'static str { /// What still blocks a mod of this tier that has not been carried over. fn not_yet(tier: &str) -> &'static str { match tier { - "data" => "data tier: carry-over only, no agent; its jar is not in the work folder", + "data" => "data tier: carry-over only, no agent; its jar is not here yet (fetch gets it)", "logic" => { "needs mod API 0.3: behaviour callbacks (use, place, break, ticks) and server events" } diff --git a/docs/modpacks.md b/docs/modpacks.md index 1d8609cef..fb9bca5f4 100644 --- a/docs/modpacks.md +++ b/docs/modpacks.md @@ -267,16 +267,32 @@ inspected: └── coverage.json, coverage.md ``` -**Status.** `import`, `inventory`, `carry` and `report` run today: +**Status.** `import`, `inventory`, `fetch`, `carry` and `report` run today, and `port` runs +them all for what a player's machine can port (content mods of the data tier), as the +launcher's "Import modpack" does: ``` cargo run -p horizon-port-pack -- import ~/Downloads/pack.mrpack ~/.horizon/ports/pack cargo run -p horizon-port-pack -- inventory ~/.horizon/ports/pack # --offline reuses modrinth.json +cargo run -p horizon-port-pack -- fetch ~/.horizon/ports/pack # the portable mods' jars, or name them +# --from : copy the player's own jars (an installed instance, a downloads folder) cargo run -p horizon-port-pack -- carry ~/.horizon/ports/pack # content mods with a jar, or name them # --resources : only templates Horizon's tables have cargo run -p horizon-port-pack -- report ~/.horizon/ports/pack # coverage.json and coverage.md +cargo run -p horizon-port-pack -- port ~/Downloads/pack.zip ~/.horizon/ports/pack [--from ] [--resources ] ``` +`fetch` gets each jar the player's way, never from us: copied from a folder they have (matched +by SHA-1, or through a CurseForge app instance's `minecraftinstance.json`), else downloaded from +the link the pack itself gives (a `.mrpack`'s, on the format's hosts only) or, for a CurseForge +pack, the same file on Modrinth's public CDN, found by its SHA-1. A jar is kept only when its +hash matches. A file only CurseForge has is listed as needing a CurseForge API key (Ludovic's +decision; open question below), with the player's installed instance as the way round it. +`/fetch.json` says what happened to each file. On All the Mods 10 8.3's CurseForge zip +(2026-10-17): of the 54 data-tier content mods, 42 come from Modrinth's CDN (84 MB) and 12 need +the key or the instance; `port` then carries the 42 over (0 ported, 42 partial: the harness is a +dev step). + `inventory` reads metadata only: Modrinth answers by file hash, and the jars only if the user's instance has them (an imported instance folder, or jars put in `/jars/`). Without jars, classes come from the reviewed table, Modrinth categories and sides; with jars, from their @@ -310,7 +326,7 @@ block's name breaking ties (`_pane`, `_gate`, `_trapdoor`). | Format | Read | Jars come from | |---|---|---| | Modrinth `.mrpack` | `modrinth.index.json`: `dependencies` (minecraft, neoforge / fabric-loader / forge / quilt-loader), `files[]` with path, sha1/sha512, `env` (client/server), download URLs; `overrides/`, `client-overrides/`, `server-overrides/` | downloaded from the listed Modrinth URLs, hash-checked | -| CurseForge zip | `manifest.json`: `minecraft.version`, `modLoaders[]`, `files[]` (`projectID`, `fileID`, `required`), `overrides` | the user's installed instance (CurseForge app `minecraftinstance.json` + `mods/`). CurseForge's API needs a key, so we take what the user already downloaded | +| CurseForge zip | `manifest.json`: `minecraft.version`, `modLoaders[]`, `files[]` (`projectID`, `fileID`, `required`), `overrides`. When the overrides hold Crash Assistant's `config/crash_assistant/modlist.json` (All the Mods 10 does), its jar names, mod ids and SHA-1s replace the bare ids | the user's installed instance (CurseForge app `minecraftinstance.json` + `mods/`), or the same file on Modrinth by SHA-1. CurseForge's API needs a key, so we take what the user already downloaded | | Prism / MultiMC instance | `mmc-pack.json` components (`net.minecraft`, `net.neoforged`, `net.fabricmc.fabric-loader`), `instance.cfg`, `.minecraft/mods/*.jar`, packwiz `.pw.toml` files if present | the instance folder | | packwiz | `pack.toml`, `index.toml`, `mods/*.pw.toml` | as listed | | Crash Assistant `modlist.json` | jar name → mod id, name, version, Modrinth SHA-1; the loader entry (NeoForge `21.1.x` is Minecraft 1.21.1). Packs such as ATM10 publish it in their repository | metadata only; jars put in `/jars/` |