horizon-port-pack: import a modpack and classify its mods

A CLI for the first steps of docs/modpacks.md's pipeline:

- import: a Modrinth .mrpack, a CurseForge zip or a Prism / MultiMC
  instance (or a folder of jars) to pack.json: Minecraft and loader
  versions, each file with its hash and sides, and the overrides.
- inventory: Modrinth metadata by SHA-1 (cached in modrinth.json, so
  reruns work offline), and the jars when the user's instance has them
  (neoforge.mods.toml / mods.toml / fabric.mod.json and a file census,
  without decompiling). Each mod gets a class and API tier from the
  reviewed ATM10 table, then rules on categories, sides and the census,
  and a dependency layer. Writes inventory.json and inventory.csv.

Only metadata is downloaded; jars are never fetched.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-16 19:24:49 +02:00
parent 746c888cd3
commit 85858610bf
10 changed files with 1300 additions and 1 deletions
Generated
+12
View File
@@ -1498,6 +1498,18 @@ dependencies = [
"horizon-mod-sdk",
]
[[package]]
name = "horizon-port-pack"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"sha1",
"toml",
"ureq",
"zip",
]
[[package]]
name = "horizon-prepare"
version = "0.1.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[workspace]
exclude = ["launcher", "crates/horizon-telemetry-server"]
members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-shaderpack", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"]
members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-shaderpack", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"]
resolver = "2"
[workspace.package]
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "horizon-port-pack"
version = "0.1.0"
edition.workspace = true
license.workspace = true
rust-version = "1.86"
description = "Port a Java modpack to Horizon: import, inventory, classification and data carry-over (docs/modpacks.md)"
[dependencies]
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha1 = "0.10"
toml = "1.1.6"
ureq = { version = "2", default-features = false, features = ["tls"] }
zip = { version = "2", default-features = false, features = ["deflate"] }
[lints]
workspace = true
+233
View File
@@ -0,0 +1,233 @@
//! Step 2 and 3: classify each mod (class and API tier, `docs/modpacks.md`)
//! and order them by required dependencies.
use std::collections::{BTreeMap, HashMap};
/// The reviewed table: ATM10's 495 rows (`docs/modpacks/atm10-mods.csv`).
/// It wins over the heuristics.
const KNOWN_CSV: &str = include_str!("../../../docs/modpacks/atm10-mods.csv");
pub struct Known {
by_key: HashMap<String, (String, String)>,
}
/// Lowercase letters and digits only: `Get It Together, Drops!` ->
/// `getittogetherdrops`.
pub fn normalise(s: &str) -> String {
s.chars()
.filter(char::is_ascii_alphanumeric)
.map(|c| c.to_ascii_lowercase())
.collect()
}
/// One CSV line, with `"quoted, fields"`.
fn csv_fields(line: &str) -> Vec<String> {
let (mut out, mut field, mut quoted) = (Vec::new(), String::new(), false);
let mut chars = line.chars().peekable();
while let Some(c) = chars.next() {
match c {
'"' if quoted && chars.peek() == Some(&'"') => {
field.push('"');
chars.next();
}
'"' => quoted = !quoted,
',' if !quoted => out.push(std::mem::take(&mut field)),
c => field.push(c),
}
}
out.push(field);
out
}
impl Known {
pub fn atm10() -> Self {
let mut by_key = HashMap::new();
for line in KNOWN_CSV.lines().skip(1) {
let f = csv_fields(line);
if f.len() < 6 || f[5] == "guess" {
continue;
}
let value = (f[3].clone(), f[4].clone());
by_key.insert(normalise(&f[0]), value.clone());
by_key.entry(normalise(&f[1])).or_insert(value);
}
Known { by_key }
}
/// By mod id, Modrinth slug or name, in that order.
pub fn get(&self, keys: &[&str]) -> Option<&(String, String)> {
keys.iter()
.filter(|k| !k.is_empty())
.find_map(|k| self.by_key.get(&normalise(k)))
}
}
/// What the rules look at.
#[derive(Default)]
pub struct Facts<'a> {
/// `mods`, `resourcepacks`, `shaderpacks`, `datapacks`.
pub kind: &'a str,
pub client: &'a str,
pub server: &'a str,
/// Modrinth categories.
pub categories: &'a [String],
/// From the jar: classes, and whether it has blocks, items or data.
pub classes: Option<usize>,
pub has_content: bool,
}
/// Class, tier and confidence (`known` is set by the caller).
pub fn rules(f: &Facts) -> (&'static str, &'static str, &'static str) {
let has = |c: &str| f.categories.iter().any(|x| x == c);
match f.kind {
"resourcepacks" => return ("other", "data", "kind"),
"shaderpacks" => return ("other", "none", "kind"),
"datapacks" => return ("content", "data", "kind"),
_ => {}
}
if has("library") {
return ("library", "none", "categories");
}
if has("optimization") {
return ("performance", "none", "categories");
}
if f.client != "unsupported" && f.server == "unsupported" {
return ("client-ui", "client", "sides");
}
if f.client == "unsupported" && f.server != "unsupported" && !f.server.is_empty() {
return ("server-only", "logic", "sides");
}
if f.classes == Some(0) && f.has_content {
return ("content", "data", "jar");
}
let content = [
"adventure",
"decoration",
"equipment",
"food",
"game-mechanics",
"magic",
"mobs",
"storage",
"technology",
"transportation",
"worldgen",
];
if f.categories.iter().any(|c| content.contains(&c.as_str())) {
let tier = if has("mobs") || has("magic") || has("adventure") {
"deep"
} else if has("technology") || has("storage") || has("transportation") {
"machines"
} else if f.categories.iter().all(|c| {
["worldgen", "decoration", "food"].contains(&c.as_str())
|| !content.contains(&c.as_str())
}) {
"data"
} else {
"logic"
};
return ("content", tier, "categories");
}
if has("utility") || has("management") || has("social") {
return ("other", "logic", "guess");
}
("unknown", "none", "unknown")
}
/// Layer of each mod: 0 needs nothing in the pack, n needs layer n-1.
/// Mods in a cycle get one layer past the rest; dependencies outside the
/// pack are left out (the caller lists them).
pub fn layers(depends: &BTreeMap<String, Vec<String>>) -> BTreeMap<String, usize> {
let mut out: BTreeMap<String, usize> = BTreeMap::new();
loop {
let mut progressed = false;
for (id, deps) in depends {
if out.contains_key(id) {
continue;
}
let inside = deps.iter().filter(|d| depends.contains_key(*d) && *d != id);
let mut layer = 0;
let mut ready = true;
for d in inside {
match out.get(d) {
Some(l) => layer = layer.max(l + 1),
None => ready = false,
}
}
if ready {
out.insert(id.clone(), layer);
progressed = true;
}
}
if !progressed {
break;
}
}
let past = out.values().max().map_or(0, |m| m + 1);
for id in depends.keys() {
out.entry(id.clone()).or_insert(past);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn known_table_matches_ids_and_names() {
let k = Known::atm10();
assert_eq!(k.get(&["create"]).map(|v| v.0.as_str()), Some("content"));
assert_eq!(
k.get(&["", "Get It Together, Drops!"])
.map(|v| v.0.as_str()),
Some("performance")
);
assert_eq!(k.get(&["sodium"]).map(|v| v.1.as_str()), Some("none"));
assert!(k.get(&["not-a-mod"]).is_none());
}
#[test]
fn rules_follow_sides_then_categories() {
let cats = |c: &[&str]| c.iter().map(|s| s.to_string()).collect::<Vec<_>>();
let f = |client, server, c: &[String]| {
rules(&Facts {
kind: "mods",
client,
server,
categories: c,
..Facts::default()
})
};
assert_eq!(f("required", "unsupported", &[]).0, "client-ui");
assert_eq!(f("unsupported", "required", &[]).0, "server-only");
assert_eq!(f("required", "required", &cats(&["library"])).0, "library");
assert_eq!(
f("required", "unsupported", &cats(&["optimization"])).0,
"performance"
);
assert_eq!(
f("required", "required", &cats(&["technology", "utility"])),
("content", "machines", "categories")
);
assert_eq!(
f("required", "required", &cats(&["worldgen", "neoforge"])),
("content", "data", "categories")
);
assert_eq!(f("", "", &[]).0, "unknown");
}
#[test]
fn layers_put_dependencies_first() {
let d = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::<Vec<_>>();
let deps = BTreeMap::from([
("lib".to_string(), d(&[])),
("create".to_string(), d(&["lib", "outside"])),
("addon".to_string(), d(&["create", "lib"])),
("a".to_string(), d(&["b"])),
("b".to_string(), d(&["a"])),
]);
let l = layers(&deps);
assert_eq!((l["lib"], l["create"], l["addon"]), (0, 1, 2));
assert_eq!((l["a"], l["b"]), (3, 3));
}
}
+335
View File
@@ -0,0 +1,335 @@
//! Step 1, import: a Modrinth `.mrpack`, a CurseForge zip, or an instance
//! folder (Prism / MultiMC, or a plain `mods/` folder) to `pack.json`.
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::{
collections::BTreeMap,
fs,
io::Read,
path::{Path, PathBuf},
};
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct Pack {
pub name: String,
/// `mrpack`, `curseforge` or `instance`.
pub format: String,
pub minecraft: String,
/// `neoforge`, `forge`, `fabric` or `quilt`.
pub loader: String,
pub loader_version: String,
pub files: Vec<PackFile>,
/// Override files by top folder (`config`, `kubejs`, ...).
pub overrides: BTreeMap<String, usize>,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct PackFile {
/// Path in the instance (`mods/create-1.21.1-6.0.4.jar`).
pub path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sha1: Option<String>,
/// `required`, `optional` or `unsupported`; empty when not stated.
pub client: String,
pub server: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub downloads: Vec<String>,
/// CurseForge project and file ids.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub curseforge: Option<(u64, u64)>,
/// The jar on disk (instances only).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub jar: Option<PathBuf>,
}
impl PackFile {
/// `mods`, `resourcepacks`, `shaderpacks`, `datapacks`...
pub fn kind(&self) -> &str {
self.path.split('/').next().unwrap_or("")
}
pub fn file_name(&self) -> &str {
self.path.rsplit('/').next().unwrap_or(&self.path)
}
}
const LOADERS: [(&str, &str); 4] = [
("neoforge", "neoforge"),
("forge", "forge"),
("fabric-loader", "fabric"),
("quilt-loader", "quilt"),
];
pub fn import(path: &Path) -> Result<Pack, String> {
if path.is_dir() {
return instance(path);
}
let file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
let mut zip = zip::ZipArchive::new(file).map_err(|e| format!("{}: {e}", path.display()))?;
if let Some(index) = read_json(&mut zip, "modrinth.index.json")? {
return Ok(mrpack(&index, &names(&zip)));
}
if let Some(manifest) = read_json(&mut zip, "manifest.json")? {
return Ok(curseforge(&manifest, &names(&zip)));
}
Err(format!(
"{}: neither modrinth.index.json nor manifest.json",
path.display()
))
}
fn names(zip: &zip::ZipArchive<fs::File>) -> Vec<String> {
zip.file_names().map(str::to_owned).collect()
}
fn read_json(zip: &mut zip::ZipArchive<fs::File>, name: &str) -> Result<Option<Value>, String> {
let Ok(mut entry) = zip.by_name(name) else {
return Ok(None);
};
let mut text = String::new();
entry
.read_to_string(&mut text)
.map_err(|e| format!("{name}: {e}"))?;
serde_json::from_str(&text)
.map(Some)
.map_err(|e| format!("{name}: {e}"))
}
fn str_of(v: &Value, key: &str) -> String {
v.get(key).and_then(Value::as_str).unwrap_or("").to_owned()
}
/// Override files by top folder, below any of `roots`.
fn overrides(names: &[String], roots: &[&str]) -> BTreeMap<String, usize> {
let mut out = BTreeMap::new();
for name in names.iter().filter(|n| !n.ends_with('/')) {
for root in roots {
if let Some(rest) = name.strip_prefix(root).and_then(|r| r.strip_prefix('/')) {
let top = rest.split('/').next().unwrap_or(rest);
*out.entry(top.to_owned()).or_default() += 1;
}
}
}
out
}
pub fn mrpack(index: &Value, names: &[String]) -> Pack {
let deps = &index["dependencies"];
let (loader, loader_version) = LOADERS
.iter()
.find_map(|(key, name)| Some((name.to_string(), deps.get(*key)?.as_str()?.to_owned())))
.unwrap_or_default();
let files = index["files"]
.as_array()
.into_iter()
.flatten()
.map(|f| PackFile {
path: str_of(f, "path"),
sha1: f["hashes"]["sha1"].as_str().map(str::to_owned),
client: str_of(&f["env"], "client"),
server: str_of(&f["env"], "server"),
downloads: f["downloads"]
.as_array()
.into_iter()
.flatten()
.filter_map(|d| d.as_str().map(str::to_owned))
.collect(),
..PackFile::default()
})
.collect();
Pack {
name: str_of(index, "name"),
format: "mrpack".into(),
minecraft: str_of(deps, "minecraft"),
loader,
loader_version,
files,
overrides: overrides(
names,
&["overrides", "client-overrides", "server-overrides"],
),
}
}
pub fn curseforge(manifest: &Value, names: &[String]) -> Pack {
let primary = manifest["minecraft"]["modLoaders"]
.as_array()
.into_iter()
.flatten()
.find(|l| l["primary"].as_bool().unwrap_or(true))
.map(|l| str_of(l, "id"))
.unwrap_or_default();
let (loader, loader_version) = primary.split_once('-').unwrap_or((&primary, ""));
let files = manifest["files"]
.as_array()
.into_iter()
.flatten()
.map(|f| {
let (p, id) = (f["projectID"].as_u64(), f["fileID"].as_u64());
PackFile {
path: format!("mods/curseforge-{}-{}", p.unwrap_or(0), id.unwrap_or(0)),
// CurseForge states no side; its API needs a key, so names
// come from the user's installed instance (import that).
curseforge: p.zip(id),
..PackFile::default()
}
})
.collect();
let root = manifest["overrides"].as_str().unwrap_or("overrides");
Pack {
name: str_of(manifest, "name"),
format: "curseforge".into(),
minecraft: str_of(&manifest["minecraft"], "version"),
loader: loader.to_owned(),
loader_version: loader_version.to_owned(),
files,
overrides: overrides(names, &[root]),
}
}
/// A Prism / MultiMC instance (`mmc-pack.json`), a game folder, or a
/// folder of jars: the jars on disk, hashed.
fn instance(dir: &Path) -> Result<Pack, String> {
let mut pack = Pack {
name: dir
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default(),
format: "instance".into(),
..Pack::default()
};
if let Ok(text) = fs::read_to_string(dir.join("mmc-pack.json")) {
let mmc: Value = serde_json::from_str(&text).map_err(|e| format!("mmc-pack.json: {e}"))?;
for c in mmc["components"].as_array().into_iter().flatten() {
let version = str_of(c, "version");
match c["uid"].as_str().unwrap_or("") {
"net.minecraft" => pack.minecraft = version,
"net.neoforged" => {
(pack.loader, pack.loader_version) = ("neoforge".into(), version)
}
"net.minecraftforge" => {
(pack.loader, pack.loader_version) = ("forge".into(), version)
}
"net.fabricmc.fabric-loader" => {
(pack.loader, pack.loader_version) = ("fabric".into(), version)
}
"org.quiltmc.quilt-loader" => {
(pack.loader, pack.loader_version) = ("quilt".into(), version)
}
_ => {}
}
}
}
let game = [".minecraft", "minecraft", "."]
.iter()
.map(|g| dir.join(g))
.find(|g| g.join("mods").is_dir())
.or_else(|| dir.read_dir().is_ok().then(|| dir.to_path_buf()))
.ok_or_else(|| format!("{}: no mods folder", dir.display()))?;
let mods = if game.join("mods").is_dir() {
game.join("mods")
} else {
game.clone()
};
let mut jars: Vec<PathBuf> = fs::read_dir(&mods)
.map_err(|e| format!("{}: {e}", mods.display()))?
.flatten()
.map(|e| e.path())
.filter(|p| p.extension().is_some_and(|x| x == "jar"))
.collect();
jars.sort();
for jar in jars {
let bytes = fs::read(&jar).map_err(|e| format!("{}: {e}", jar.display()))?;
pack.files.push(PackFile {
path: format!(
"mods/{}",
jar.file_name().unwrap_or_default().to_string_lossy()
),
sha1: Some(sha1_hex(&bytes)),
jar: Some(jar),
..PackFile::default()
});
}
for top in ["config", "kubejs", "defaultconfigs", "datapacks"] {
let n = count_files(&game.join(top));
if n > 0 {
pack.overrides.insert(top.into(), n);
}
}
Ok(pack)
}
fn count_files(dir: &Path) -> usize {
let Ok(entries) = fs::read_dir(dir) else {
return 0;
};
entries
.flatten()
.map(|e| {
let p = e.path();
if p.is_dir() { count_files(&p) } else { 1 }
})
.sum()
}
pub fn sha1_hex(bytes: &[u8]) -> String {
use sha1::Digest;
sha1::Sha1::digest(bytes)
.iter()
.map(|b| format!("{b:02x}"))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn reads_an_mrpack_index() {
let index = json!({"name": "Small", "dependencies": {"minecraft": "1.21.1", "neoforge": "21.1.233"},
"files": [{"path": "mods/create.jar", "hashes": {"sha1": "ab"}, "env": {"client": "required", "server": "required"},
"downloads": ["https://cdn.modrinth.com/create.jar"]},
{"path": "resourcepacks/look.zip", "hashes": {"sha1": "cd"}}]});
let names = [
"overrides/config/a.toml",
"overrides/config/b/c.toml",
"client-overrides/kubejs/x.js",
"overrides/",
]
.map(String::from);
let p = mrpack(&index, &names);
assert_eq!(
(
p.minecraft.as_str(),
p.loader.as_str(),
p.loader_version.as_str()
),
("1.21.1", "neoforge", "21.1.233")
);
assert_eq!(p.files.len(), 2);
assert_eq!(
(p.files[0].kind(), p.files[0].file_name()),
("mods", "create.jar")
);
assert_eq!(p.files[0].sha1.as_deref(), Some("ab"));
assert_eq!(p.files[1].kind(), "resourcepacks");
assert_eq!(
p.overrides,
BTreeMap::from([("config".into(), 2), ("kubejs".into(), 1)])
);
}
#[test]
fn reads_a_curseforge_manifest() {
let manifest = json!({"name": "CF", "overrides": "overrides",
"minecraft": {"version": "1.21.1", "modLoaders": [{"id": "neoforge-21.1.259", "primary": true}]},
"files": [{"projectID": 238222, "fileID": 5101366, "required": true}]});
let p = curseforge(&manifest, &["overrides/kubejs/server_scripts/a.js".into()]);
assert_eq!(
(p.loader.as_str(), p.loader_version.as_str()),
("neoforge", "21.1.259")
);
assert_eq!(p.files[0].curseforge, Some((238222, 5101366)));
assert_eq!(p.overrides["kubejs"], 1);
}
}
+225
View File
@@ -0,0 +1,225 @@
//! Step 2, inventory: one record per pack file, from the jar (when the
//! user's instance has it) and Modrinth's metadata, classified and ordered.
use crate::{
classify::{self, Facts, Known},
import::Pack,
jar::{self, JarInfo},
modrinth::Cache,
};
use serde::{Deserialize, Serialize};
use std::{
collections::{BTreeMap, HashMap},
path::{Path, PathBuf},
};
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct Record {
/// The mod id from the jar, else the Modrinth slug, else the file name.
pub key: String,
pub name: String,
pub version: String,
pub path: String,
pub class: String,
pub tier: String,
/// `known` (reviewed table), `sides`, `categories`, `jar`, `kind`,
/// `guess` or `unknown`.
pub confidence: String,
/// `modrinth`, `curseforge` or `file`.
pub source: String,
pub license: String,
pub client: String,
pub server: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub categories: Vec<String>,
/// Required dependencies inside the pack, by key.
pub depends: Vec<String>,
/// Required dependencies the pack does not have.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub missing: Vec<String>,
pub layer: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub jar: Option<PathBuf>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub info: Option<JarInfo>,
}
/// The jar of a pack file: the instance's own, or one the user put in
/// `<work>/jars/`.
fn jar_of(pack_jar: Option<&PathBuf>, file_name: &str, work: &Path) -> Option<PathBuf> {
pack_jar
.cloned()
.or_else(|| Some(work.join("jars").join(file_name)))
.filter(|p| p.is_file())
}
pub fn build(pack: &Pack, cache: &Cache, work: &Path) -> Vec<Record> {
let known = Known::atm10();
let mut records = Vec::new();
// Modrinth project id -> record key, for dependencies.
let mut by_project: HashMap<String, String> = HashMap::new();
let mut project_deps: Vec<Vec<String>> = Vec::new();
for f in &pack.files {
let jar = jar_of(f.jar.as_ref(), f.file_name(), work);
let info = jar.as_deref().and_then(|j| jar::read(j).ok());
let sha1 = f.sha1.as_deref().unwrap_or("");
let project = cache.project_of(sha1);
let p = |k: &str| project.and_then(|p| p[k].as_str()).unwrap_or("").to_owned();
let categories: Vec<String> = project
.into_iter()
.flat_map(|p| {
["categories", "additional_categories"]
.iter()
.flat_map(|k| p[*k].as_array().cloned().unwrap_or_default())
})
.filter_map(|c| c.as_str().map(str::to_owned))
.collect();
let stem = f.file_name().trim_end_matches(".jar").to_owned();
let key = info
.as_ref()
.map(|i| i.mod_id.clone())
.filter(|k| !k.is_empty())
.or_else(|| Some(p("slug")).filter(|s| !s.is_empty()))
.unwrap_or(stem);
let (client, server) = match project {
Some(_) if f.client.is_empty() => (p("client_side"), p("server_side")),
_ => (f.client.clone(), f.server.clone()),
};
let census = info.as_ref().map(|i| &i.census);
let facts = Facts {
kind: f.kind(),
client: &client,
server: &server,
categories: &categories,
classes: census.map(|c| c.classes),
has_content: census.is_some_and(|c| c.blockstates + c.item_models + c.recipes > 0),
};
let name = info
.as_ref()
.map(|i| i.name.clone())
.filter(|n| !n.is_empty())
.unwrap_or_else(|| p("title"));
let (class, tier, confidence) =
match known.get(&[key.as_str(), p("slug").as_str(), name.as_str()]) {
Some((c, t)) => (c.clone(), t.clone(), "known".to_owned()),
None => {
let (c, t, conf) = classify::rules(&facts);
(c.into(), t.into(), conf.into())
}
};
let license = info
.as_ref()
.map(|i| i.license.clone())
.filter(|l| !l.is_empty())
.or_else(|| project.and_then(|p| p["license"]["id"].as_str().map(str::to_owned)))
.unwrap_or_else(|| "unknown".into());
if let Some(id) = project.and_then(|p| p["id"].as_str()) {
by_project.insert(id.to_owned(), key.clone());
}
project_deps.push(cache.required(sha1));
records.push(Record {
name,
version: info.as_ref().map(|i| i.version.clone()).unwrap_or_else(|| {
cache
.versions
.get(sha1)
.and_then(|v| v["version_number"].as_str())
.unwrap_or("")
.to_owned()
}),
path: f.path.clone(),
class,
tier,
confidence,
source: if project.is_some() {
"modrinth".into()
} else if f.curseforge.is_some() {
"curseforge".into()
} else {
"file".into()
},
license,
client,
server,
categories,
depends: info.as_ref().map(|i| i.depends.clone()).unwrap_or_default(),
jar,
info,
key,
..Record::default()
});
}
// Dependencies: the jar's mod ids, else Modrinth's required projects.
let keys: std::collections::HashSet<String> = records.iter().map(|r| r.key.clone()).collect();
for (r, deps) in records.iter_mut().zip(project_deps) {
if r.depends.is_empty() {
r.depends = deps
.iter()
.map(|p| {
by_project
.get(p)
.cloned()
.unwrap_or_else(|| format!("modrinth:{p}"))
})
.collect();
}
let (inside, missing): (Vec<String>, Vec<String>) =
r.depends.drain(..).partition(|d| keys.contains(d));
r.depends = inside;
r.missing = missing;
}
let graph: BTreeMap<String, Vec<String>> = records
.iter()
.map(|r| (r.key.clone(), r.depends.clone()))
.collect();
let layers = classify::layers(&graph);
for r in &mut records {
r.layer = layers.get(&r.key).copied().unwrap_or(0);
}
records.sort_by(|a, b| (a.layer, &a.key).cmp(&(b.layer, &b.key)));
records
}
/// Counts by a field, largest first.
pub fn counts<'a>(
records: &'a [Record],
field: impl Fn(&'a Record) -> &'a str,
) -> Vec<(&'a str, usize)> {
let mut m: BTreeMap<&str, usize> = BTreeMap::new();
for r in records {
*m.entry(field(r)).or_default() += 1;
}
let mut v: Vec<_> = m.into_iter().collect();
v.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0)));
v
}
/// `inventory.csv`: the columns of `docs/modpacks/atm10-mods.csv`, plus the layer.
pub fn csv(records: &[Record]) -> String {
let field = |s: &str| {
if s.contains([',', '"', '\n']) {
format!("\"{}\"", s.replace('"', "\"\""))
} else {
s.to_owned()
}
};
let mut out = String::from(
"mod_id,name,version,class,tier,confidence,source,license,client,server,layer\n",
);
for r in records {
let row = [
&r.key,
&r.name,
&r.version,
&r.class,
&r.tier,
&r.confidence,
&r.source,
&r.license,
&r.client,
&r.server,
];
let cells: Vec<String> = row.iter().map(|s| field(s)).collect();
out.push_str(&format!("{},{}\n", cells.join(","), r.layer));
}
out
}
+245
View File
@@ -0,0 +1,245 @@
//! What a mod jar holds, read without decompiling: its metadata
//! (`neoforge.mods.toml`, `mods.toml`, `fabric.mod.json`) and a census of
//! its data, assets and classes.
use serde::{Deserialize, Serialize};
use std::{fs, io::Read, path::Path};
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct JarInfo {
pub mod_id: String,
pub name: String,
pub version: String,
pub license: String,
/// Required dependencies by mod id (loaders and `minecraft` left out).
pub depends: Vec<String>,
pub census: Census,
}
#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
pub struct Census {
pub blockstates: usize,
pub item_models: usize,
pub recipes: usize,
pub loot_tables: usize,
pub tags: usize,
pub worldgen: usize,
pub structures: usize,
pub languages: usize,
pub textures: usize,
pub sounds: usize,
pub classes: usize,
pub mixin_configs: usize,
}
const NOT_MODS: [&str; 5] = ["minecraft", "neoforge", "forge", "fabricloader", "java"];
pub fn read(path: &Path) -> Result<JarInfo, String> {
let file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
let mut zip = zip::ZipArchive::new(file).map_err(|e| format!("{}: {e}", path.display()))?;
let text = |zip: &mut zip::ZipArchive<fs::File>, name: &str| -> Option<String> {
let mut s = String::new();
zip.by_name(name).ok()?.read_to_string(&mut s).ok()?;
Some(s)
};
let mut info = if let Some(t) = text(&mut zip, "META-INF/neoforge.mods.toml")
.or_else(|| text(&mut zip, "META-INF/mods.toml"))
{
mods_toml(&t)?
} else if let Some(t) = text(&mut zip, "fabric.mod.json") {
fabric(&t)?
} else {
JarInfo::default()
};
let names: Vec<String> = zip.file_names().map(str::to_owned).collect();
info.census = census(&names);
if info.mod_id.is_empty() {
info.mod_id = path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_default();
}
Ok(info)
}
/// `[[mods]]` (the first one is the jar's mod), the top-level `license`, and
/// `[[dependencies.<id>]]` with `type = "required"` (or the older
/// `mandatory = true`).
pub fn mods_toml(text: &str) -> Result<JarInfo, String> {
let doc: toml::Table = toml::from_str(text).map_err(|e| format!("mods.toml: {e}"))?;
let first = doc
.get("mods")
.and_then(|m| m.as_array())
.and_then(|m| m.first())
.and_then(|m| m.as_table());
let get = |t: Option<&toml::Table>, k: &str| {
t.and_then(|t| t.get(k))
.and_then(|v| v.as_str())
.unwrap_or("")
.to_owned()
};
let mod_id = get(first, "modId");
let mut depends = Vec::new();
if let Some(deps) = doc
.get("dependencies")
.and_then(|d| d.get(&mod_id))
.and_then(|d| d.as_array())
{
for d in deps.iter().filter_map(|d| d.as_table()) {
let required = d.get("type").and_then(|t| t.as_str()).map_or_else(
|| d.get("mandatory").and_then(|m| m.as_bool()) == Some(true),
|t| t.eq_ignore_ascii_case("required"),
);
let id = get(Some(d), "modId");
if required && !NOT_MODS.contains(&id.as_str()) && !id.is_empty() {
depends.push(id);
}
}
}
Ok(JarInfo {
name: get(first, "displayName"),
version: get(first, "version"),
license: get(Some(&doc), "license"),
mod_id,
depends,
census: Census::default(),
})
}
pub fn fabric(text: &str) -> Result<JarInfo, String> {
let v: serde_json::Value =
serde_json::from_str(text).map_err(|e| format!("fabric.mod.json: {e}"))?;
let s = |k: &str| v[k].as_str().unwrap_or("").to_owned();
let license = match &v["license"] {
serde_json::Value::Array(l) => l
.iter()
.filter_map(|x| x.as_str())
.collect::<Vec<_>>()
.join(" OR "),
l => l.as_str().unwrap_or("").to_owned(),
};
let depends = v["depends"]
.as_object()
.into_iter()
.flatten()
.map(|(id, _)| id.clone())
.filter(|id| !NOT_MODS.contains(&id.as_str()) && !id.starts_with("fabric"))
.collect();
Ok(JarInfo {
mod_id: s("id"),
name: s("name"),
version: s("version"),
license,
depends,
census: Census::default(),
})
}
pub fn census(names: &[String]) -> Census {
let mut c = Census::default();
for n in names {
let parts: Vec<&str> = n.split('/').collect();
if n.ends_with(".class") {
c.classes += 1;
} else if n.ends_with(".mixins.json") || (parts.len() == 1 && n.contains("mixins")) {
c.mixin_configs += 1;
}
// assets/<ns>/<kind>/... and data/<ns>/<kind>/...
let (Some(root), Some(kind)) = (parts.first(), parts.get(2)) else {
continue;
};
let json = n.ends_with(".json");
match (*root, *kind) {
("assets", "blockstates") if json => c.blockstates += 1,
("assets", "models") if json && parts.get(3) == Some(&"item") => c.item_models += 1,
("assets", "lang") => c.languages += 1,
("assets", "textures") if n.ends_with(".png") => c.textures += 1,
("assets", "sounds.json") => c.sounds += 1,
("data", "recipe" | "recipes") if json => c.recipes += 1,
("data", "loot_table" | "loot_tables") if json => c.loot_tables += 1,
("data", "tags") if json => c.tags += 1,
("data", "worldgen") if json => c.worldgen += 1,
("data", "structure" | "structures") if n.ends_with(".nbt") => c.structures += 1,
_ => {}
}
}
c
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_neoforge_metadata() {
let info = mods_toml(
r#"modLoader = "javafml"
license = "MIT"
[[mods]]
modId = "ruby"
version = "1.2.0"
displayName = "Ruby"
[[dependencies.ruby]]
modId = "neoforge"
type = "required"
[[dependencies.ruby]]
modId = "geckolib"
type = "required"
[[dependencies.ruby]]
modId = "jei"
type = "optional"
[[dependencies.ruby]]
modId = "curios"
mandatory = true
"#,
)
.unwrap();
assert_eq!(
(
info.mod_id.as_str(),
info.version.as_str(),
info.license.as_str()
),
("ruby", "1.2.0", "MIT")
);
assert_eq!(info.depends, ["geckolib", "curios"]);
}
#[test]
fn reads_fabric_metadata() {
let info = fabric(
r#"{"id": "gem", "version": "3", "name": "Gem", "license": ["MIT", "Apache-2.0"],
"depends": {"fabricloader": "*", "fabric-api": "*", "minecraft": "1.21.1", "cloth-config": "*"}}"#,
)
.unwrap();
assert_eq!(info.license, "MIT OR Apache-2.0");
assert_eq!(info.depends, ["cloth-config"]);
}
#[test]
fn counts_files_by_kind() {
let names = [
"assets/ruby/blockstates/ruby_block.json",
"assets/ruby/models/item/ruby.json",
"assets/ruby/models/block/ruby_block.json",
"assets/ruby/textures/item/ruby.png",
"assets/ruby/lang/en_us.json",
"data/ruby/recipe/ruby.json",
"data/ruby/loot_table/blocks/ruby_block.json",
"data/minecraft/tags/block/mineable/pickaxe.json",
"data/ruby/structure/hut.nbt",
"com/example/Ruby.class",
"ruby.mixins.json",
]
.map(String::from);
let c = census(&names);
assert_eq!(
(c.blockstates, c.item_models, c.textures, c.languages),
(1, 1, 1, 1)
);
assert_eq!(
(c.recipes, c.loot_tables, c.tags, c.structures),
(1, 1, 1, 1)
);
assert_eq!((c.classes, c.mixin_configs), (1, 1));
}
}
+104
View File
@@ -0,0 +1,104 @@
//! `horizon-port-pack`: port a Java modpack to Horizon (`docs/modpacks.md`).
//!
//! - `import <pack> <work>`: a `.mrpack`, CurseForge zip or instance folder
//! to `<work>/pack.json`.
//! - `inventory <work> [--offline]`: Modrinth metadata (cached in
//! `<work>/modrinth.json`), the jars in the instance or `<work>/jars/`,
//! classes, tiers and dependency layers in `<work>/inventory.json` and
//! `inventory.csv`.
//!
//! The work folder belongs to the user: jars stay there and are never
//! uploaded or committed.
mod classify;
mod import;
mod inventory;
mod jar;
mod modrinth;
use std::{fs, path::Path, process::ExitCode};
const USAGE: &str = "usage: horizon-port-pack import <pack> <work> | inventory <work> [--offline]";
fn main() -> ExitCode {
let args: Vec<String> = std::env::args().skip(1).collect();
let path = |i: usize| args.get(i).map(Path::new).ok_or(USAGE.to_owned());
let result = match args.first().map(String::as_str) {
Some("import") => path(1)
.and_then(|p| Ok((p, path(2)?)))
.and_then(|(p, w)| import(p, w)),
Some("inventory") => {
path(1).and_then(|w| inventory(w, args.iter().any(|a| a == "--offline")))
}
_ => Err(USAGE.to_owned()),
};
match result {
Ok(()) => ExitCode::SUCCESS,
Err(e) => {
eprintln!("error: {e}");
ExitCode::FAILURE
}
}
}
fn write_json(path: &Path, value: &impl serde::Serialize) -> Result<(), String> {
let text = serde_json::to_string_pretty(value).map_err(|e| e.to_string())?;
fs::write(path, text).map_err(|e| format!("{}: {e}", path.display()))
}
fn read_json<T: serde::de::DeserializeOwned>(path: &Path) -> Result<T, String> {
let text = fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?;
serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))
}
fn import(pack: &Path, work: &Path) -> Result<(), String> {
let pack = import::import(pack)?;
fs::create_dir_all(work).map_err(|e| format!("{}: {e}", work.display()))?;
write_json(&work.join("pack.json"), &pack)?;
let mut kinds = std::collections::BTreeMap::<&str, usize>::new();
for f in &pack.files {
*kinds.entry(f.kind()).or_default() += 1;
}
println!(
"{} ({}): Minecraft {}, {} {}; files {kinds:?}; overrides {:?}",
pack.name, pack.format, pack.minecraft, pack.loader, pack.loader_version, pack.overrides
);
Ok(())
}
fn inventory(work: &Path, offline: bool) -> Result<(), String> {
let pack: import::Pack = read_json(&work.join("pack.json"))?;
let cache_path = work.join("modrinth.json");
let mut cache = modrinth::Cache::load(&cache_path);
if !offline {
let hashes: Vec<String> = pack.files.iter().filter_map(|f| f.sha1.clone()).collect();
cache.fetch(&hashes)?;
write_json(&cache_path, &cache)?;
}
let records = inventory::build(&pack, &cache, work);
write_json(&work.join("inventory.json"), &records)?;
fs::write(work.join("inventory.csv"), inventory::csv(&records))
.map_err(|e| format!("inventory.csv: {e}"))?;
let jars = records.iter().filter(|r| r.jar.is_some()).count();
println!("{} files, {jars} jars read", records.len());
for (title, field) in [
(
"class",
(|r: &inventory::Record| r.class.as_str()) as fn(&inventory::Record) -> &str,
),
("tier", |r| r.tier.as_str()),
("confidence", |r| r.confidence.as_str()),
("source", |r| r.source.as_str()),
] {
println!("{title}: {:?}", inventory::counts(&records, field));
}
let layers = records.iter().map(|r| r.layer).max().map_or(0, |m| m + 1);
let per_layer: Vec<usize> = (0..layers)
.map(|l| records.iter().filter(|r| r.layer == l).count())
.collect();
println!("layers: {per_layer:?}");
let missing: usize = records.iter().map(|r| r.missing.len()).sum();
if missing > 0 {
println!("{missing} required dependencies are not in the pack (see inventory.json)");
}
Ok(())
}
+113
View File
@@ -0,0 +1,113 @@
//! Modrinth metadata for the pack's files: the version of each file by
//! SHA-1, and its project (categories, licence, sides). Only metadata: no
//! jar is downloaded. Answers are cached in `<work>/modrinth.json`, so a
//! rerun works offline.
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::{collections::BTreeMap, path::Path};
const API: &str = "https://api.modrinth.com/v2";
const AGENT: &str = "ludovic111/horizon horizon-port-pack/0.1 (docs/modpacks.md)";
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct Cache {
/// Version objects by file SHA-1.
pub versions: BTreeMap<String, Value>,
/// Project objects by project id.
pub projects: BTreeMap<String, Value>,
}
impl Cache {
pub fn load(path: &Path) -> Self {
std::fs::read_to_string(path)
.ok()
.and_then(|t| serde_json::from_str(&t).ok())
.unwrap_or_default()
}
/// Ask Modrinth about the hashes and projects not cached yet.
pub fn fetch(&mut self, hashes: &[String]) -> Result<(), String> {
let missing: Vec<&String> = hashes
.iter()
.filter(|h| !self.versions.contains_key(*h))
.collect();
for chunk in missing.chunks(500) {
let body = serde_json::json!({"hashes": chunk, "algorithm": "sha1"});
let found = post(&format!("{API}/version_files"), &body)?;
for (hash, version) in found.as_object().into_iter().flatten() {
self.versions.insert(hash.clone(), version.clone());
}
}
let mut ids: Vec<String> = self
.versions
.values()
.filter_map(|v| v["project_id"].as_str().map(str::to_owned))
.filter(|id| !self.projects.contains_key(id))
.collect();
ids.sort();
ids.dedup();
for chunk in ids.chunks(100) {
let list = serde_json::to_string(chunk).unwrap_or_default();
let found = get(&format!("{API}/projects?ids={}", encode(&list)))?;
for project in found.as_array().into_iter().flatten() {
if let Some(id) = project["id"].as_str() {
self.projects.insert(id.to_owned(), project.clone());
}
}
}
Ok(())
}
pub fn project_of(&self, sha1: &str) -> Option<&Value> {
let id = self.versions.get(sha1)?["project_id"].as_str()?;
self.projects.get(id)
}
/// Required dependencies of a file's version, as project ids.
pub fn required(&self, sha1: &str) -> Vec<String> {
let Some(v) = self.versions.get(sha1) else {
return Vec::new();
};
v["dependencies"]
.as_array()
.into_iter()
.flatten()
.filter(|d| d["dependency_type"] == "required")
.filter_map(|d| d["project_id"].as_str().map(str::to_owned))
.collect()
}
}
fn post(url: &str, body: &Value) -> Result<Value, String> {
let response = ureq::post(url)
.set("User-Agent", AGENT)
.set("Content-Type", "application/json")
.send_string(&body.to_string())
.map_err(|e| format!("{url}: {e}"))?;
read(url, response)
}
fn get(url: &str) -> Result<Value, String> {
let response = ureq::get(url)
.set("User-Agent", AGENT)
.call()
.map_err(|e| format!("{url}: {e}"))?;
read(url, response)
}
fn read(url: &str, response: ureq::Response) -> Result<Value, String> {
let text = response.into_string().map_err(|e| format!("{url}: {e}"))?;
serde_json::from_str(&text).map_err(|e| format!("{url}: {e}"))
}
/// Percent-encoding for a query value.
fn encode(s: &str) -> String {
s.bytes()
.map(|b| match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
(b as char).to_string()
}
_ => format!("%{b:02X}"),
})
.collect()
}
+14
View File
@@ -262,6 +262,20 @@ inspected:
└── coverage.json, coverage.md
```
**Status.** `import` and `inventory` run today, `carry` follows (see below):
```
cargo run -p horizon-port-pack -- import ~/Downloads/pack.mrpack ~/.horizon/ports/pack
cargo run -p horizon-port-pack -- inventory ~/.horizon/ports/pack # --offline reuses modrinth.json
```
`inventory` reads metadata only: Modrinth answers by file hash, and the jars only if the user's
instance has them (an imported instance folder, or jars put in `<work>/jars/`). Without jars,
classes come from the reviewed table, Modrinth categories and sides; with jars, from their
`mods.toml` / `fabric.mod.json` and file census too. On Create+ 6.0.0 alpha f (155 files, metadata
only): content 54, library 33, client-ui 26, performance 25, other 14, unknown 2, server-only 1;
40 rows matched the reviewed ATM10 table; three dependency layers (93 / 57 / 5).
### 1. Import
| Format | Read | Jars come from |