Merge pull request #648 from ludovic111/feat/modpack-bench

Content mods: refuse more block states than chunks hold; the pack against NeoForge, measured

crates/horizon-client/src/mod_content.rs

@@ -314,6 +314,22 @@ impl<'a> Plan<'a> {
                 });
             }
         }
+        if next > MAX_STATES {
+            let mut by_mod: BTreeMap<&str, u32> = BTreeMap::new();
+            for b in &blocks {
+                *by_mod
+                    .entry(&b.decl.id[..b.decl.id.find(':').unwrap_or(0)])
+                    .or_default() += b.rename.states.len() as u32;
+            }
+            let list: Vec<String> = by_mod.iter().map(|(m, n)| format!("{m} {n}")).collect();
+            return Err(format!(
+                "{next} block states, more than the {MAX_STATES} chunks can hold ({} in the game; \
+                 the mods: {})",
+                names.len(),
+                list.join(", ")
+            )
+            .into());
+        }
         // Block items copy the template block's item.
         let gameplay = read_json(&base.join("gameplay-facts.json"))?;
         let rows = match gameplay.get("states") {
@@ -628,6 +644,9 @@ const NBT_TABLES: [&str; 6] = [
 /// In a root built here: the content mods' ids, the namespaces loaders
 /// read besides `minecraft`.
 const MARKER: &str = "horizon-mod-content.json";
+/// Block states a world can hold: chunk sections store a state in 16 bits
+/// (crate::chunk). Mods past it are refused, not wrapped onto other blocks.
+pub const MAX_STATES: u32 = 1 << 16;
 
 /// The content mods' namespaces in a root built by [`build`]; none in any
 /// other root, so vanilla's loaders read exactly what they did.

crates/horizon-client/src/mod_content_tests.rs

@@ -220,6 +220,32 @@ fn refuses_unknown_templates_and_vanilla_ids() {
     fs::remove_dir_all(&scratch).unwrap();
 }
 
+#[test]
+fn refuses_more_states_than_chunks_hold() {
+    let Some(base) = prepared() else {
+        return;
+    };
+    let scratch = std::env::temp_dir().join(format!("horizon-mod-states-{}", std::process::id()));
+    let _ = fs::remove_dir_all(&scratch);
+    let mut m = ruby_mod(&scratch.join("ruby"));
+    // Redstone wire has 1296 states: 40 copies pass any game's 65536.
+    let mut wire = m.content.blocks[0].clone();
+    wire.template = "minecraft:redstone_wire".into();
+    for i in 0..40 {
+        let mut b = wire.clone();
+        b.id = format!("ruby:wire_{i}");
+        m.content.blocks.push(b);
+    }
+    let e = build(&base, std::slice::from_ref(&m), &scratch.join("out")).unwrap_err();
+    assert!(
+        e.to_string()
+            .contains("more than the 65536 chunks can hold"),
+        "{e}"
+    );
+    assert!(e.to_string().contains("the mods: ruby "), "{e}");
+    fs::remove_dir_all(&scratch).unwrap();
+}
+
 #[test]
 fn no_content_keeps_the_base_root() {
     let base = Path::new("/nonexistent/base");

docs/modpacks.md

@@ -237,6 +237,11 @@ How the game loads them, without touching vanilla's path:
 - Loaders that read only `data/minecraft/` (recipes, loot, tags) read every namespace, as
   vanilla does. With no mods there is only `minecraft`, so nothing changes.
 - The overlay is cached like the resource-pack overlay and rebuilt when a mod file changes.
+- **Limits.** Chunk sections store a block state in 16 bits: vanilla 26.3 has 35,723 states,
+  so mods may add 29,813. Past that the content root is refused with the states per mod
+  (Macaw's Doors, Fences, Paths and Trapdoors alone add 36,154: not all four fit). Block
+  sprites fill texture arrays of 2048 layers, four at most (8,192; vanilla takes 1,250,
+  identical textures share a layer). Wider state ids are the next step for big packs.
 - Not in 0.2: multiplayer registry sync (single player and the integrated server first), worlds
   opened without a mod they used (vanilla turns unknown blocks into air; we will ask first),
   vanilla ids redefined by a mod (refused).
@@ -494,10 +499,12 @@ published.
 ## Measuring "10x"
 
 `tools/perf_bench.py` already measures Horizon against vanilla and vanilla with performance
-mods on the same machine, world and camera path ([benchmark.md](benchmark.md)). It gains
-`--game pack --pack <file or instance>`: the pack on its own loader (NeoForge 21.1.259 for
-ATM10), launched with the pack's recommended JVM arguments, driven by the same dev-only bench
-mod, ported to that loader.
+mods on the same machine, world and camera path ([benchmark.md](benchmark.md)). With
+`--game neoforge --mod-jars <dir> --deps <dir>` it runs the mods on their own loader (NeoForge
+21.1.259 for ATM10, installed under `.local/neoforge-oracle`, `-Xmx2G` like the launcher),
+driven by the same steps through a dev-only NeoForge port of the bench mod
+(`mods/horizon/src/neoforge-bench`). `--game horizon --port-work <work>` loads a work folder's
+carried-over mods into Horizon. A whole pack (`--pack`, its recommended JVM arguments) is next.
 
 | Metric | How | Horizon side |
 |---|---|---|
@@ -520,6 +527,37 @@ confidence interval. Other ratios are reported as they come out. Caveat (in ever
 the pack runs Minecraft 1.21.1 and Horizon runs 26.3, so the vanilla parts differ by a few
 versions.
 
+### Measured: six ported mods against the same mods on NeoForge
+
+Cloud Glass, Factory Blocks, Glassential, Macaw's Fences, Paths and Trapdoors (Doors left out:
+with Trapdoors it passes the block states chunks hold, see Limits), carried over and loaded as
+dev mods in Horizon, against their jars plus Architectury and Fusion on NeoForge 21.1.259.
+Mac mini M4 Pro, 1280x720, seed 1234, the four flight scenes, 2 games per row, 2026-10-17.
+Means; FPS is the mean of the four scenes. "Title" is start to title screen, "world" start to
+a fresh world drawn around the spawn, "scene load" a teleport to every chunk drawn.
+
+| RD | game | title s | world s | scene load s | FPS | RSS steady / peak MB | load avg |
+|---|---|---:|---:|---:|---:|---:|---:|
+| 8 | Horizon + 6 mods | 9.8 | 16.2 | 4.0 | 122 | 2,644 / 2,771 | 32 |
+| 8 | NeoForge + 6 mods | 19.5 | 50.0 | 30.2 | 72 | 2,325 / 2,592 | 56 |
+| 8 | Horizon | 5.0 | 11.0 | 4.0 | 124 | 1,559 / 1,629 | 51 |
+| 8 | NeoForge | 12.8 | 36.1 | 21.9 | 104 | 2,334 / 2,558 | 30 |
+| 16 | Horizon + 6 mods | 10.8 | 27.0 | 11.3 | 90 | 2,925 / 3,032 | 40 |
+| 16 | NeoForge + 6 mods | 26.4 | 82.5 | 64.3 | 25 | 3,016 / 3,519 | 45 |
+| 16 | Horizon | 3.5 | 10.0 | 7.4 | 119 | 1,732 / 1,871 | 56 |
+| 16 | NeoForge | 16.9 | 73.7 | 43.2 | 36 | 2,748 / 3,425 | 48 |
+
+Same content, Horizon against NeoForge: the world is ready 3.1x sooner at 8 chunks and 3.1x at
+16, the title screen 2.0x and 2.4x sooner, and the flights run 1.7x and 3.6x the frames. RAM is
+not lower: Horizon's steady RSS grows 1.1 GB with the mods (61,816 block states against
+vanilla's 35,723; the per-state tables are the next thing to measure), while the JVM stays
+inside its 2 GB heap. Caveats, so none of this is a "10x" claim yet: other sessions and CI
+runners kept the machine's load average between 30 and 56 on 14 threads, so the confidence
+intervals are as wide as the means (`perf_bench.py --table`); Horizon's frames are paced near
+the display's 144 Hz at 8 chunks; Horizon's title time includes building the content overlay on
+a first start (about 5 s, cached afterwards); and the pack runs 1.21.1 against Horizon's 26.3.
+A quiet rerun with more games is the next measurement.
+
 ## The working version for 2026-10-23
 
 In scope (each is a PR or a small stack):
@@ -545,7 +583,11 @@ In scope (each is a PR or a small stack):
 6. **Measured:** load time, RAM and FPS of that pack on NeoForge 1.21.1 against Horizon, with
    the full ATM10 on NeoForge as the reference row.
 
-Out of scope for that date, and next in order: worldgen carry-over (the 30 or so structure and biome mods),
+Out of scope for that date, and next in order: worldgen carry-over (the 30 or so structure and biome mods;
+the first one looked at, Moog's Voyager Structures, declares all 130 structures and 116 structure
+sets with MoogsStructureLib's own `generic_jigsaw_structure` and `advanced_random_spread` types and
+NeoForge's `c:` biome tags, so its carry needs a mapping to vanilla's jigsaw and random spread
+types plus the oracle's biome tags, not only bundle merging),
 behaviour callbacks and server events (0.3), block entities, menus, capabilities and net
 (0.4), the first agent-ported logic-tier mod as the pilot of step 5, then machines. Porting all
 of ATM10 is a campaign. Each step publishes its coverage number; we make no claim beyond what

mods/horizon/src/neoforge-bench/java/dev/horizonlab/neoforgebench/NeoForgeBench.java

@@ -0,0 +1,275 @@
+package dev.horizonlab.neoforgebench;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.mojang.blaze3d.platform.GlUtil;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.List;
+import net.minecraft.client.Minecraft;
+import net.minecraft.client.gui.screens.TitleScreen;
+import net.minecraft.client.player.LocalPlayer;
+import net.minecraft.world.Difficulty;
+import net.minecraft.world.level.GameRules;
+import net.minecraft.world.level.GameType;
+import net.minecraft.world.level.LevelSettings;
+import net.minecraft.world.level.WorldDataConfiguration;
+import net.minecraft.world.level.levelgen.WorldOptions;
+import net.minecraft.world.level.levelgen.presets.WorldPresets;
+import net.minecraft.world.phys.Vec3;
+import net.neoforged.api.distmarker.Dist;
+import net.neoforged.fml.common.Mod;
+import net.neoforged.neoforge.client.event.RenderFrameEvent;
+import net.neoforged.neoforge.common.NeoForge;
+
+/**
+ * Dev-only benchmark driver for a NeoForge 1.21.1 client (tools/perf_bench.py
+ * --game neoforge): the steps of mods/horizon/src/bench's VanillaBench, on
+ * NeoForge's events. From the title screen it creates a fresh creative world
+ * with the script's seed, waits for the spawn area, then per scene teleports,
+ * waits for the chunks, settles and flies a straight line at constant speed,
+ * recording every frame time. Events go to horizon.bench.out as JSON lines;
+ * the game quits after the last scene. Never shipped.
+ */
+@Mod(value = "horizon_bench", dist = Dist.CLIENT)
+public final class NeoForgeBench {
+    private enum Phase { TITLE, WORLD, TELEPORT, LOAD, SETTLE, FLY, DONE }
+
+    private final JsonObject script;
+    private final Path out;
+    private final List<JsonObject> scenes = new ArrayList<>();
+    private Phase phase = Phase.TITLE;
+    private int scene = -1;
+    private long phaseStart;
+    private long lastFrame;
+    private long readyCheck;
+    private int lastLoaded = -1;
+    private long lastChange;
+    private final List<Float> frames = new ArrayList<>();
+
+    public NeoForgeBench() {
+        String scriptPath = System.getProperty("horizon.bench.script");
+        String outPath = System.getProperty("horizon.bench.out");
+        if (scriptPath == null || outPath == null) {
+            script = null;
+            out = null;
+            return;
+        }
+        try {
+            script = JsonParser.parseString(Files.readString(Path.of(scriptPath))).getAsJsonObject();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        out = Path.of(outPath);
+        for (var element : script.getAsJsonArray("scenes")) scenes.add(element.getAsJsonObject());
+        NeoForge.EVENT_BUS.addListener(RenderFrameEvent.Pre.class, e -> step(Minecraft.getInstance()));
+        event("init");
+    }
+
+    private void step(Minecraft mc) {
+        long now = System.nanoTime();
+        float frameMs = lastFrame == 0 ? 0 : (now - lastFrame) / 1e6f;
+        lastFrame = now;
+        double elapsed = (now - phaseStart) / 1e9;
+        if (phase != Phase.TITLE && phase != Phase.DONE && elapsed > script.get("timeout").getAsDouble()
+                && phase != Phase.FLY && phase != Phase.SETTLE) {
+            JsonObject e = new JsonObject();
+            e.addProperty("phase", phase.name());
+            event("timeout", e);
+            finish(mc);
+            return;
+        }
+        switch (phase) {
+            case TITLE -> {
+                if (mc.screen instanceof TitleScreen && mc.getOverlay() == null) {
+                    event("title", renderer(mc));
+                    LevelSettings settings = new LevelSettings(script.get("world").getAsString(), GameType.CREATIVE,
+                        false, Difficulty.NORMAL, true, new GameRules(), WorldDataConfiguration.DEFAULT);
+                    WorldOptions options = new WorldOptions(script.get("seed").getAsLong(), true, false);
+                    enter(Phase.WORLD, now);
+                    mc.createWorldOpenFlows().createFreshLevel(script.get("world").getAsString(), settings, options,
+                        WorldPresets::createNormalWorldDimensions, mc.screen);
+                }
+            }
+            case WORLD -> {
+                if (ready(mc, null, now)) {
+                    JsonObject e = new JsonObject();
+                    e.addProperty("ready_epoch_ms", epochMs(lastChange));
+                    e.addProperty("chunks", lastLoaded);
+                    event("world", e);
+                    mc.player.connection.sendCommand("weather clear");
+                    next(mc, now);
+                }
+            }
+            case TELEPORT -> {
+                JsonObject s = scenes.get(scene);
+                Vec3 from = vec(s.getAsJsonArray("from"));
+                LocalPlayer player = mc.player;
+                player.connection.sendCommand("time set 6000");
+                player.connection.sendCommand(String.format(java.util.Locale.ROOT, "tp @s %.3f %.3f %.3f %.3f %.3f",
+                    from.x, from.y, from.z, yaw(s), s.get("pitch").getAsFloat()));
+                event("teleport", named(s));
+                enter(Phase.LOAD, now);
+            }
+            case LOAD -> {
+                JsonObject s = scenes.get(scene);
+                Vec3 from = vec(s.getAsJsonArray("from"));
+                if (mc.player != null && mc.player.position().distanceToSqr(from) < 1024.0) place(mc, s, 0);
+                if (ready(mc, from, now)) {
+                    JsonObject e = named(s);
+                    e.addProperty("load_ms", (lastChange - phaseStart) / 1e6);
+                    e.addProperty("chunks", lastLoaded);
+                    event("loaded", e);
+                    enter(Phase.SETTLE, now);
+                }
+            }
+            case SETTLE -> {
+                place(mc, scenes.get(scene), 0);
+                if (elapsed >= script.get("settle").getAsDouble()) {
+                    frames.clear();
+                    event("fly", named(scenes.get(scene)));
+                    enter(Phase.FLY, now);
+                }
+            }
+            case FLY -> {
+                frames.add(frameMs);
+                double t = Math.min(1.0, elapsed / script.get("seconds").getAsDouble());
+                place(mc, scenes.get(scene), t);
+                if (t >= 1.0) {
+                    JsonObject e = named(scenes.get(scene));
+                    JsonArray times = new JsonArray();
+                    frames.forEach(times::add);
+                    e.add("frame_ms", times);
+                    e.addProperty("chunks", mc.level.getChunkSource().getLoadedChunksCount());
+                    event("flown", e);
+                    next(mc, now);
+                }
+            }
+            case DONE -> { }
+        }
+    }
+
+    private void next(Minecraft mc, long now) {
+        scene++;
+        if (scene >= scenes.size()) {
+            finish(mc);
+        } else {
+            enter(Phase.TELEPORT, now);
+        }
+    }
+
+    private void finish(Minecraft mc) {
+        phase = Phase.DONE;
+        event("done");
+        mc.stop();
+    }
+
+    private void enter(Phase next, long now) {
+        phase = next;
+        phaseStart = now;
+        lastLoaded = -1;
+        lastChange = now;
+    }
+
+    /** As VanillaBench.ready: spawned (near {@code at}), no screen, every chunk around, nothing to compile, 1 s stable. */
+    private boolean ready(Minecraft mc, Vec3 at, long now) {
+        if (now - readyCheck < 10_000_000L) return false;
+        readyCheck = now;
+        if (mc.level == null || mc.player == null || mc.screen != null || mc.getOverlay() != null
+                || (at != null && mc.player.position().distanceToSqr(at) > 4.0)) {
+            lastChange = now;
+            lastLoaded = -1;
+            return false;
+        }
+        int loaded = mc.level.getChunkSource().getLoadedChunksCount();
+        if (loaded != lastLoaded || !mc.levelRenderer.hasRenderedAllSections() || !chunksAround(mc)) {
+            lastLoaded = loaded;
+            lastChange = now;
+            return false;
+        }
+        return now - lastChange >= 1_000_000_000L;
+    }
+
+    private static boolean chunksAround(Minecraft mc) {
+        int r = Math.max(1, mc.options.getEffectiveRenderDistance() - 1);
+        var center = mc.player.chunkPosition();
+        var chunks = mc.level.getChunkSource();
+        for (int dx = -r; dx <= r; dx++) {
+            for (int dz = -r; dz <= r; dz++) {
+                if (dx * dx + dz * dz <= r * r && !chunks.hasChunk(center.x + dx, center.z + dz)) return false;
+            }
+        }
+        return true;
+    }
+
+    private static void place(Minecraft mc, JsonObject s, double t) {
+        LocalPlayer player = mc.player;
+        if (player == null) return;
+        Vec3 a = vec(s.getAsJsonArray("from")), b = vec(s.getAsJsonArray("to"));
+        Vec3 at = a.add(b.subtract(a).scale(t));
+        player.setPos(at.x, at.y, at.z);
+        player.setYRot(yaw(s));
+        player.setXRot(s.get("pitch").getAsFloat());
+        player.setOldPosAndRot();
+        player.setDeltaMovement(Vec3.ZERO);
+        player.fallDistance = 0;
+        if (!player.getAbilities().flying) {
+            player.getAbilities().flying = true;
+            player.onUpdateAbilities();
+        }
+    }
+
+    private static float yaw(JsonObject s) {
+        Vec3 a = vec(s.getAsJsonArray("from")), b = vec(s.getAsJsonArray("to"));
+        return (float) Math.toDegrees(Math.atan2(-(b.x - a.x), b.z - a.z));
+    }
+
+    private static Vec3 vec(JsonArray a) {
+        return new Vec3(a.get(0).getAsDouble(), a.get(1).getAsDouble(), a.get(2).getAsDouble());
+    }
+
+    private static JsonObject named(JsonObject s) {
+        JsonObject e = new JsonObject();
+        e.addProperty("scene", s.get("name").getAsString());
+        return e;
+    }
+
+    private static JsonObject renderer(Minecraft mc) {
+        JsonObject e = new JsonObject();
+        e.addProperty("gpu", GlUtil.getRenderer());
+        e.addProperty("vendor", GlUtil.getVendor());
+        e.addProperty("backend", "OpenGL");
+        e.addProperty("driver", GlUtil.getOpenGLVersion());
+        JsonArray size = new JsonArray();
+        size.add(mc.getWindow().getWidth());
+        size.add(mc.getWindow().getHeight());
+        e.add("size", size);
+        e.addProperty("render_distance", mc.options.renderDistance().get());
+        e.addProperty("simulation_distance", mc.options.simulationDistance().get());
+        return e;
+    }
+
+    private static long epochMs(long nanos) {
+        return System.currentTimeMillis() - (System.nanoTime() - nanos) / 1_000_000L;
+    }
+
+    private void event(String name) {
+        event(name, new JsonObject());
+    }
+
+    private void event(String name, JsonObject e) {
+        e.addProperty("event", name);
+        e.addProperty("epoch_ms", System.currentTimeMillis());
+        try {
+            Files.writeString(out, e + "\n", StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+        } catch (IOException error) {
+            throw new UncheckedIOException(error);
+        }
+    }
+}

mods/horizon/src/neoforge-bench/resources/META-INF/neoforge.mods.toml

@@ -0,0 +1,11 @@
+# Dev-only: the benchmark driver for a NeoForge client (tools/perf_bench.py --game neoforge).
+# Built and run by tools/perf_bench.py; never shipped.
+modLoader = "javafml"
+loaderVersion = "[4,)"
+license = "MIT"
+
+[[mods]]
+modId = "horizon_bench"
+version = "1"
+displayName = "Horizon benchmark driver (dev only)"
+description = "Flies the client along tools/perf_bench.py's paths and records frame times."

tools/mod_port_harness.py

@@ -17,7 +17,8 @@ and what the game's loaders make of it. For every block and item the mod registe
 - recipes: each recipe's result, count and ingredients;
 - items: present, and their stack size.
 
-Writes WORK/mods/<id>/harness/result.json per mod and prints a table. Jars, facts and results stay
+Each mod is probed on its own, against vanilla. Writes WORK/mods/<id>/harness/{horizon,result}.json
+per mod and prints a table. Jars, facts and results stay
 in WORK, outside the repository.
 """
 import argparse
@@ -185,18 +186,23 @@ def main():
     metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--no-deps', '--format-version', '1'],
                                                   cwd=ROOT, text=True))
     probe = Path(metadata['target_directory']) / 'release/examples/mod_port_probe'
-    out = work / 'horizon.json'
-    subprocess.run([str(probe), args.resources, str(work), str(out), *args.mods], check=True)
     oracle = json.loads(facts.read_text())
-    horizon = json.loads(out.read_text())
-    upgrades = sound_upgrades(oracle, horizon)
     old_items = set(oracle.get('vanilla_items', []))
     mods = args.mods or sorted(p.name for p in (work / 'mods').iterdir() if (p / 'carry').is_dir())
-    print(f'{"mod":<16} {"result":<7} ' + ' '.join(f'{c:>12}' for c in CHECKS))
+    rows = []
     for mod in mods:
+        # One mod at a time: each against vanilla, as a player adding it would load it (and a pack's
+        # sum may pass the block states a world holds, which the probe would refuse).
+        out = work / 'mods' / mod / 'harness' / 'horizon.json'
+        out.parent.mkdir(parents=True, exist_ok=True)
+        subprocess.run([str(probe), args.resources, str(work), str(out), mod], check=True)
+        horizon = json.loads(out.read_text())
         carry_report = work / 'mods' / mod / 'carry-report.json'
         carry = json.loads(carry_report.read_text()) if carry_report.exists() else {}
-        result = compare_mod(mod, oracle, horizon, carry, upgrades, old_items)
+        rows.append(compare_mod(mod, oracle, horizon, carry, sound_upgrades(oracle, horizon), old_items))
+    print(f'{"mod":<16} {"result":<7} ' + ' '.join(f'{c:>12}' for c in CHECKS))
+    for result in rows:
+        mod = result['mod']
         (work / 'mods' / mod / 'harness').mkdir(parents=True, exist_ok=True)
         (work / 'mods' / mod / 'harness' / 'result.json').write_text(json.dumps(result, indent=1))
         cols = []

tools/neoforge_oracle.py

@@ -45,6 +45,84 @@ def install():
     return home
 
 
+def client():
+    """The NeoForge client (tools/perf_bench.py --game neoforge), installed once per version: the installer's
+    own files, then the vanilla libraries its version inherits, fetched as the launcher does (sha1-checked)."""
+    sys.path.insert(0, str(ROOT / 'tools'))
+    import hashlib
+    import lab
+    home = LOCAL / f'client-{NEOFORGE}'
+    profile = home / 'versions' / f'neoforge-{NEOFORGE}' / f'neoforge-{NEOFORGE}.json'
+    if not profile.exists():
+        install()  # the installer jar
+        home.mkdir(parents=True, exist_ok=True)
+        (home / 'launcher_profiles.json').write_text('{"profiles":{}}')
+        with (LOCAL / f'install-client-{NEOFORGE}.log').open('w') as log:
+            subprocess.run(['java', '-jar', str(LOCAL / f'neoforge-{NEOFORGE}-installer.jar'), '--installClient',
+                            str(home)], cwd=LOCAL, stdout=log, stderr=subprocess.STDOUT, check=True)
+    neo = json.loads(profile.read_text())
+    base = json.loads((home / 'versions' / neo['inheritsFrom'] / f'{neo["inheritsFrom"]}.json').read_text())
+    jars = []
+    for manifest in (neo, base):
+        names = {library['name'] for library in manifest['libraries']}
+        for library in manifest['libraries']:
+            artifact = library.get('downloads', {}).get('artifact')
+            if not artifact or not lab.library_allowed(library) or not lab.native_library_matches(library, None, names):
+                continue
+            path = home / 'libraries' / artifact['path']
+            if not path.exists():
+                with urllib.request.urlopen(artifact['url'], timeout=120) as r:
+                    data = r.read()
+                if hashlib.sha1(data).hexdigest() != artifact['sha1']:
+                    sys.exit(f'{artifact["url"]}: sha1 mismatch')
+                path.parent.mkdir(parents=True, exist_ok=True)
+                path.write_bytes(data)
+            if path not in jars:
+                jars.append(path)
+    return home, neo, base, jars
+
+
+def client_command(java, heap, game_dir, natives, assets, jvm=(), game=()):
+    """The command the launcher would run for the NeoForge client profile, with no account."""
+    home, neo, base, jars = client()
+    import lab
+    values = {'library_directory': str(home / 'libraries'), 'classpath_separator': os.pathsep,
+              'version_name': f'neoforge-{NEOFORGE}', 'natives_directory': str(natives),
+              'launcher_name': 'horizon-bench', 'launcher_version': '1'}
+    expand = lambda s: s.replace('${', '{').format(**values)
+    command = [str(java), f'-Xmx{heap}']
+    for argument in base['arguments']['jvm']:
+        if isinstance(argument, dict):
+            if lab.library_allowed(argument):
+                command += [argument['value']] if isinstance(argument['value'], str) else argument['value']
+        elif argument == '${classpath}':
+            # Every library; not the vanilla jar, which NeoForge's patched client replaces.
+            command.append(os.pathsep.join(map(str, jars)))
+        else:
+            command.append(expand(argument))
+    command += [expand(a) for a in neo['arguments']['jvm']]
+    command += [*jvm, neo['mainClass']]
+    command += ['--username', 'Bench', '--uuid', '00000000000000000000000000000000', '--accessToken', '0',
+                '--version', f'neoforge-{NEOFORGE}', '--gameDir', str(game_dir), '--assetsDir', str(assets),
+                '--assetIndex', base['assetIndex']['id'], '--userType', 'msa', '--versionType', 'release',
+                *neo['arguments']['game'], *game]
+    return home, command
+
+
+def build_bench(home):
+    """The NeoForge benchmark driver (mods/horizon/src/neoforge-bench), against the client."""
+    return build(home, ROOT / 'mods/horizon/src/neoforge-bench', 'horizon-bench-neoforge.jar', client_classpath(home))
+
+
+def client_classpath(home):
+    libs = home / 'libraries'
+    first = [libs / f'net/neoforged/neoforge/{NEOFORGE}/neoforge-{NEOFORGE}-client.jar',
+             libs / f'net/neoforged/neoforge/{NEOFORGE}/neoforge-{NEOFORGE}-universal.jar']
+    rest = sorted(p for p in libs.rglob('*.jar') if p not in first and 'slim' not in p.name
+                  and 'extra' not in p.name and not p.name.endswith('-unpacked.jar'))
+    return os.pathsep.join(map(str, first + rest))
+
+
 def classpath(home):
     libs = home / 'libraries'
     first = [libs / f'net/neoforged/neoforge/{NEOFORGE}/neoforge-{NEOFORGE}-server.jar',
@@ -54,27 +132,27 @@ def classpath(home):
     return os.pathsep.join(map(str, first + rest))
 
 
-def build(home):
-    """The exporter mod jar, rebuilt when its sources change."""
+def build(home, source=SOURCE, name='horizon-oracle.jar', cp=None):
+    """A dev-only mod jar (by default the exporter), rebuilt when its sources change."""
     out = LOCAL / 'build'
-    jar = out / 'horizon-oracle.jar'
-    sources = sorted((SOURCE / 'java').rglob('*.java'))
-    resources = sorted(p for p in (SOURCE / 'resources').rglob('*') if p.is_file())
+    jar = out / name
+    sources = sorted((source / 'java').rglob('*.java'))
+    resources = sorted(p for p in (source / 'resources').rglob('*') if p.is_file())
     newest = max(p.stat().st_mtime for p in sources + resources)
     if jar.exists() and jar.stat().st_mtime > newest:
         return jar
-    classes = out / 'classes'
+    classes = out / f'classes-{Path(name).stem}'
     shutil.rmtree(classes, ignore_errors=True)
     classes.mkdir(parents=True)
-    javac = subprocess.run(['javac', '--release', '21', '-proc:none', '-nowarn', '-cp', classpath(home),
+    javac = subprocess.run(['javac', '--release', '21', '-proc:none', '-nowarn', '-cp', cp or classpath(home),
                             '-d', str(classes), *map(str, sources)], capture_output=True, text=True)
     if javac.returncode:
-        sys.exit(f'the exporter does not compile:\n{javac.stdout}{javac.stderr}')
+        sys.exit(f'{name} does not compile:\n{javac.stdout}{javac.stderr}')
     with zipfile.ZipFile(jar, 'w', zipfile.ZIP_DEFLATED) as z:
         for p in sorted(classes.rglob('*.class')):
             z.write(p, p.relative_to(classes).as_posix())
         for p in resources:
-            z.write(p, p.relative_to(SOURCE / 'resources').as_posix())
+            z.write(p, p.relative_to(source / 'resources').as_posix())
     return jar
 
 

tools/perf_bench.py

@@ -13,6 +13,11 @@ for every game. RSS is sampled from outside the process for every game too.
     python3 tools/perf_bench.py --game vanilla-mods --label mods-m4
     python3 tools/perf_bench.py --table horizon-m4 vanilla-m4 mods-m4
 
+A ported mod pack against the same mods on their own loader (docs/modpacks.md, "Measured"):
+
+    python3 tools/perf_bench.py --game horizon --port-work WORK --label pack-horizon
+    python3 tools/perf_bench.py --game neoforge --mod-jars JARS --deps DEPS --label pack-neoforge
+
 Games:
   horizon       target/release/horizon-client, driven over its control socket
   vanilla       your own 26.3 client.jar + Fabric Loader + a dev-only driver
@@ -20,6 +25,13 @@ Games:
                 is redistributed
   vanilla-mods  the same + Sodium, Lithium, FerriteCore and ImmediatelyFast
                 (pinned in packs/mods.lock.json)
+  neoforge      Minecraft 1.21.1 on NeoForge (dev-only, tools/neoforge_oracle.py
+                installs it under .local/neoforge-oracle) + a driver mod
+                (mods/horizon/src/neoforge-bench) + the jars of --mod-jars and
+                --deps, the user's own; 26.3 options it does not know are ignored
+
+--port-work loads the mods a horizon-port-pack work folder carried over into
+Horizon as dev mods.
 
 Results: benchmarks/perf/<label>-rd<N>.json, one benchmark report per render
 distance (docs/benchmark-report.md), over --repeats games. Games run with no
@@ -105,6 +117,8 @@ OPTIONS = {
 LAUNCHER_JVM = ["-XX:+UnlockExperimentalVMOptions", "-XX:+UseG1GC", "-XX:G1NewSizePercent=20",
                 "-XX:G1ReservePercent=20", "-XX:MaxGCPauseMillis=50", "-XX:G1HeapRegionSize=32M"]
 PERF_MODS = ["sodium", "lithium", "ferrite-core", "immediatelyfast"]
+# Minecraft 1.21.1's data version, for the NeoForge client's options.txt (it ignores 26.3's newer keys).
+NEOFORGE_DATA_VERSION = "3955"
 
 
 # The benchmark report format, version 1 (docs/benchmark-report.md). crates/horizon-bench is the
@@ -211,9 +225,10 @@ def nvidia_memory_mb(pid):
     return None
 
 
-def write_options(game_dir, distance):
+def write_options(game_dir, distance, version=None):
     game_dir.mkdir(parents=True, exist_ok=True)
-    text = "".join(f"{k}:{v.format(distance=distance)}\n" for k, v in OPTIONS.items())
+    options = dict(OPTIONS, version=version) if version else OPTIONS
+    text = "".join(f"{k}:{v.format(distance=distance)}\n" for k, v in options.items())
     (game_dir / "options.txt").write_text(text)
     return text
 
@@ -324,6 +339,8 @@ def run_horizon(args, distance):
                "--control-socket", str(sock), "--no-pause", "--weather", "clear", "--time", "6000"]
     if args.offscreen:
         command.append("--offscreen")
+    for carry in dev_mods(args):
+        command += ["--dev-mod", str(carry)]
     env = dict(os.environ, HORIZON_NO_AUDIO="1")
     log = (WORK / f"horizon-rd{distance}.log").open("w")
     started = time.time()
@@ -376,6 +393,13 @@ def run_horizon(args, distance):
     return result
 
 
+def dev_mods(args):
+    """The carried-over mods of a horizon-port-pack work folder (docs/modpacks.md), loaded as dev mods."""
+    if not args.port_work:
+        return []
+    return sorted(p for p in (Path(args.port_work) / "mods").glob("*/carry") if (p / "horizon-mod.toml").exists())
+
+
 # Vanilla -------------------------------------------------------------------
 
 def vanilla_manifest():
@@ -491,6 +515,11 @@ def run_vanilla(args, distance):
         libs = [p for p in (os.environ.get("NIX_LD_LIBRARY_PATH", ""), "/run/opengl-driver/lib") if p]
         env["LD_LIBRARY_PATH"] = ":".join(libs + [env.get("LD_LIBRARY_PATH", "")]).strip(":")
     log = (WORK / f"{args.game}-rd{distance}.log").open("w")
+    return driven(args, distance, command, game_dir, env, log, events, options, loaded_mods)
+
+
+def driven(args, distance, command, game_dir, env, log, events, options, loaded_mods):
+    """Runs a Java client whose driver mod writes `events`, and reads its results."""
     started = time.time()
     proc = subprocess.Popen(command, cwd=game_dir, env=env, stdout=log, stderr=subprocess.STDOUT)
     sampler = Sampler(proc.pid)
@@ -508,7 +537,7 @@ def run_vanilla(args, distance):
     lines = [json.loads(l) for l in events.read_text().splitlines()] if events.exists() else []
     by = lambda name: [e for e in lines if e["event"] == name]
     if not by("done") or by("timeout"):
-        raise RuntimeError(f"vanilla run failed: {lines[-1:] if lines else 'no events'}; see {log.name}")
+        raise RuntimeError(f"{args.game} run failed: {lines[-1:] if lines else 'no events'}; see {log.name}")
     title, world = by("title")[0], by("world")[0]
     result = {
         "options_txt": options, "mods": loaded_mods, "heap": args.heap,
@@ -532,6 +561,55 @@ def run_vanilla(args, distance):
     return result
 
 
+# NeoForge ------------------------------------------------------------------
+
+def run_neoforge(args, distance):
+    """A NeoForge 1.21.1 client (dev-only, tools/neoforge_oracle.py) with the driver mod and --mod-jars:
+    the mods a ported pack replaces, on the loader the pack runs on."""
+    import neoforge_oracle
+    home = neoforge_oracle.client()[0]
+    mod = neoforge_oracle.build_bench(home)
+    game_dir = WORK / args.game
+    shutil.rmtree(game_dir, ignore_errors=True)
+    options = write_options(game_dir, distance, version=NEOFORGE_DATA_VERSION)
+    mods = game_dir / "mods"
+    mods.mkdir()
+    shutil.copy(mod, mods)
+    loaded_mods = ["horizon-bench"]
+    for folder in filter(None, (args.mod_jars, args.deps)):
+        for jar in sorted(Path(folder).glob("*.jar")):
+            shutil.copy(jar, mods)
+            loaded_mods.append(jar.name)
+    natives = WORK / "natives-neoforge"
+    script = WORK / "script.json"
+    script.write_text(json.dumps({
+        "world": WORLD, "seed": SEED, "settle": args.settle, "seconds": args.seconds, "timeout": args.timeout,
+        "scenes": [{"name": n, "from": a, "to": b, "pitch": p} for n, a, b, p in scene_list(args)],
+    }))
+    events = WORK / f"{args.game}-rd{distance}.events.jsonl"
+    events.unlink(missing_ok=True)
+    assets = WORK / "assets"
+    (assets / "indexes").mkdir(parents=True, exist_ok=True)
+    java = Path(os.environ["JAVA_HOME"]) / "bin/java" if os.environ.get("JAVA_HOME") else "java"
+    _, command = neoforge_oracle.client_command(
+        java, args.heap, game_dir, natives, assets,
+        jvm=[*LAUNCHER_JVM, f"-Dhorizon.bench.script={script}", f"-Dhorizon.bench.out={events}"],
+        game=["--width", str(args.width), "--height", str(args.height)])
+    natives.mkdir(parents=True, exist_ok=True)
+    extract_natives(neoforge_oracle.client()[3], natives)
+    env = dict(os.environ, ALSOFT_DRIVERS="null")
+    for attempt in (1, 2):
+        log = (WORK / f"{args.game}-rd{distance}.log").open("w")
+        try:
+            return driven(args, distance, command, game_dir, env, log, events, options, loaded_mods)
+        except RuntimeError:
+            # FML sometimes reads config/fml.toml while creating it on a first start, and crashes before
+            # the title screen; a crash once the driver started is a real failure.
+            if attempt == 2 or events.exists():
+                raise
+            shutil.rmtree(game_dir / "crash-reports", ignore_errors=True)
+
+
 def stop(proc):
     try:
         proc.wait(timeout=60)
@@ -654,7 +732,7 @@ def table(labels):
 
 def main():
     p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    p.add_argument("--game", choices=["horizon", "vanilla", "vanilla-mods"], default="horizon")
+    p.add_argument("--game", choices=["horizon", "vanilla", "vanilla-mods", "neoforge"], default="horizon")
     p.add_argument("--label", help="result name (default <game>-<host>)")
     p.add_argument("--distances", default="8,16,32")
     p.add_argument("--scenes", help="comma-separated subset of " + ",".join(s[0] for s in SCENES))
@@ -669,6 +747,9 @@ def main():
     p.add_argument("--resources", default=str(ROOT / ".local/horizon-client/26.3"))
     p.add_argument("--build", default=str(ROOT / "target/release"))
     p.add_argument("--repeats", type=int, default=3, help="games per render distance; the report gives means and 95%% CIs")
+    p.add_argument("--port-work", help="horizon: also load the mods carried over in this horizon-port-pack work folder")
+    p.add_argument("--mod-jars", help="neoforge: the mod jars to load (the user's own; never committed)")
+    p.add_argument("--deps", help="neoforge: their required libraries")
     p.add_argument("--table", nargs="+", metavar="LABEL", help="print markdown tables of saved runs, and exit")
     args = p.parse_args()
     if args.table:
@@ -676,7 +757,7 @@ def main():
         return
     args.label = args.label or f"{args.game}-{platform.node().split('.')[0]}"
     WORK.mkdir(parents=True, exist_ok=True)
-    run = run_horizon if args.game == "horizon" else run_vanilla
+    run = {"horizon": run_horizon, "neoforge": run_neoforge}.get(args.game, run_vanilla)
     for d in [int(x) for x in args.distances.split(",")]:
         runs = []
         for _ in range(args.repeats):
@@ -696,7 +777,13 @@ def app(args):
                                 capture_output=True, text=True).stdout.strip() or None
         manifest = (ROOT / "crates/horizon-client/Cargo.toml").read_text()
         version = next(l.split('"')[1] for l in manifest.splitlines() if l.startswith("version"))
-        return {"name": "horizon-client", "version": version, "commit": commit}
+        mods = [{"name": p.parent.name, "version": "carried"} for p in dev_mods(args)]
+        return {"name": "horizon-client", "version": version, "commit": commit, **({"mods": mods} if mods else {})}
+    if args.game == "neoforge":
+        import neoforge_oracle
+        jars = [j.name for f in filter(None, (args.mod_jars, args.deps)) for j in sorted(Path(f).glob("*.jar"))]
+        return {"name": "minecraft", "version": "1.21.1",
+                "mods": [{"name": "neoforge", "version": neoforge_oracle.NEOFORGE}] + [{"name": j} for j in jars]}
     lock = json.loads((ROOT / "packs/mods.lock.json").read_text())
     mods = [{"name": "fabric-loader", "version": lock["fabric_loader"]}]
     if args.game == "vanilla-mods":