Content mods: refuse more block states than chunks hold; the pack against NeoForge, measured
Loading the eight carried-over mods in the game (not only the probe) showed two limits real packs reach. Block textures past 2048 layers are another PR; this one handles block states and measures a pack. - Chunk sections store a block state in 16 bits. Vanilla 26.3 has 35,723 states; Macaw's four mods add 36,154, the eight 41,389. Past 65,536 ids wrapped onto other blocks (a trapdoor became a hanging sign). mod_content::build now refuses a set of mods past MAX_STATES and names the states per mod; the game then starts without content mods, as for any refused content. - tools/mod_port_harness.py probes each mod on its own against vanilla, as a player adding it would load it. - tools/perf_bench.py --game neoforge: Minecraft 1.21.1 on NeoForge 21.1.259 with --mod-jars and --deps, driven by a dev-only NeoForge port of the bench mod (mods/horizon/src/neoforge-bench); --port-work loads a work folder's carried-over mods into Horizon. tools/neoforge_oracle.py installs the client once under .local/neoforge-oracle (the installer, then the vanilla libraries it inherits, sha1-checked) and builds the driver. A NeoForge start that crashes before the driver loads (FML reading fml.toml while creating it) is retried once. Measured (docs/modpacks.md): six ported mods in Horizon against the same mods on NeoForge, Mac mini M4 Pro, 2 games per row on a loaded machine. The world is ready 3.1x sooner at 8 and 16 chunks, the flights run 1.7x and 3.6x the frames, and Horizon's RSS is not lower (+1.1 GB with the mods). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
@@ -314,6 +314,22 @@ impl<'a> Plan<'a> {
|
||||
});
|
||||
}
|
||||
}
|
||||
if next > MAX_STATES {
|
||||
let mut by_mod: BTreeMap<&str, u32> = BTreeMap::new();
|
||||
for b in &blocks {
|
||||
*by_mod
|
||||
.entry(&b.decl.id[..b.decl.id.find(':').unwrap_or(0)])
|
||||
.or_default() += b.rename.states.len() as u32;
|
||||
}
|
||||
let list: Vec<String> = by_mod.iter().map(|(m, n)| format!("{m} {n}")).collect();
|
||||
return Err(format!(
|
||||
"{next} block states, more than the {MAX_STATES} chunks can hold ({} in the game; \
|
||||
the mods: {})",
|
||||
names.len(),
|
||||
list.join(", ")
|
||||
)
|
||||
.into());
|
||||
}
|
||||
// Block items copy the template block's item.
|
||||
let gameplay = read_json(&base.join("gameplay-facts.json"))?;
|
||||
let rows = match gameplay.get("states") {
|
||||
@@ -628,6 +644,9 @@ const NBT_TABLES: [&str; 6] = [
|
||||
/// In a root built here: the content mods' ids, the namespaces loaders
|
||||
/// read besides `minecraft`.
|
||||
const MARKER: &str = "horizon-mod-content.json";
|
||||
/// Block states a world can hold: chunk sections store a state in 16 bits
|
||||
/// (crate::chunk). Mods past it are refused, not wrapped onto other blocks.
|
||||
pub const MAX_STATES: u32 = 1 << 16;
|
||||
|
||||
/// The content mods' namespaces in a root built by [`build`]; none in any
|
||||
/// other root, so vanilla's loaders read exactly what they did.
|
||||
|
||||
@@ -220,6 +220,32 @@ fn refuses_unknown_templates_and_vanilla_ids() {
|
||||
fs::remove_dir_all(&scratch).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_more_states_than_chunks_hold() {
|
||||
let Some(base) = prepared() else {
|
||||
return;
|
||||
};
|
||||
let scratch = std::env::temp_dir().join(format!("horizon-mod-states-{}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&scratch);
|
||||
let mut m = ruby_mod(&scratch.join("ruby"));
|
||||
// Redstone wire has 1296 states: 40 copies pass any game's 65536.
|
||||
let mut wire = m.content.blocks[0].clone();
|
||||
wire.template = "minecraft:redstone_wire".into();
|
||||
for i in 0..40 {
|
||||
let mut b = wire.clone();
|
||||
b.id = format!("ruby:wire_{i}");
|
||||
m.content.blocks.push(b);
|
||||
}
|
||||
let e = build(&base, std::slice::from_ref(&m), &scratch.join("out")).unwrap_err();
|
||||
assert!(
|
||||
e.to_string()
|
||||
.contains("more than the 65536 chunks can hold"),
|
||||
"{e}"
|
||||
);
|
||||
assert!(e.to_string().contains("the mods: ruby "), "{e}");
|
||||
fs::remove_dir_all(&scratch).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_content_keeps_the_base_root() {
|
||||
let base = Path::new("/nonexistent/base");
|
||||
|
||||
+47
-5
@@ -237,6 +237,11 @@ How the game loads them, without touching vanilla's path:
|
||||
- Loaders that read only `data/minecraft/` (recipes, loot, tags) read every namespace, as
|
||||
vanilla does. With no mods there is only `minecraft`, so nothing changes.
|
||||
- The overlay is cached like the resource-pack overlay and rebuilt when a mod file changes.
|
||||
- **Limits.** Chunk sections store a block state in 16 bits: vanilla 26.3 has 35,723 states,
|
||||
so mods may add 29,813. Past that the content root is refused with the states per mod
|
||||
(Macaw's Doors, Fences, Paths and Trapdoors alone add 36,154: not all four fit). Block
|
||||
sprites fill texture arrays of 2048 layers, four at most (8,192; vanilla takes 1,250,
|
||||
identical textures share a layer). Wider state ids are the next step for big packs.
|
||||
- Not in 0.2: multiplayer registry sync (single player and the integrated server first), worlds
|
||||
opened without a mod they used (vanilla turns unknown blocks into air; we will ask first),
|
||||
vanilla ids redefined by a mod (refused).
|
||||
@@ -494,10 +499,12 @@ published.
|
||||
## Measuring "10x"
|
||||
|
||||
`tools/perf_bench.py` already measures Horizon against vanilla and vanilla with performance
|
||||
mods on the same machine, world and camera path ([benchmark.md](benchmark.md)). It gains
|
||||
`--game pack --pack <file or instance>`: the pack on its own loader (NeoForge 21.1.259 for
|
||||
ATM10), launched with the pack's recommended JVM arguments, driven by the same dev-only bench
|
||||
mod, ported to that loader.
|
||||
mods on the same machine, world and camera path ([benchmark.md](benchmark.md)). With
|
||||
`--game neoforge --mod-jars <dir> --deps <dir>` it runs the mods on their own loader (NeoForge
|
||||
21.1.259 for ATM10, installed under `.local/neoforge-oracle`, `-Xmx2G` like the launcher),
|
||||
driven by the same steps through a dev-only NeoForge port of the bench mod
|
||||
(`mods/horizon/src/neoforge-bench`). `--game horizon --port-work <work>` loads a work folder's
|
||||
carried-over mods into Horizon. A whole pack (`--pack`, its recommended JVM arguments) is next.
|
||||
|
||||
| Metric | How | Horizon side |
|
||||
|---|---|---|
|
||||
@@ -520,6 +527,37 @@ confidence interval. Other ratios are reported as they come out. Caveat (in ever
|
||||
the pack runs Minecraft 1.21.1 and Horizon runs 26.3, so the vanilla parts differ by a few
|
||||
versions.
|
||||
|
||||
### Measured: six ported mods against the same mods on NeoForge
|
||||
|
||||
Cloud Glass, Factory Blocks, Glassential, Macaw's Fences, Paths and Trapdoors (Doors left out:
|
||||
with Trapdoors it passes the block states chunks hold, see Limits), carried over and loaded as
|
||||
dev mods in Horizon, against their jars plus Architectury and Fusion on NeoForge 21.1.259.
|
||||
Mac mini M4 Pro, 1280x720, seed 1234, the four flight scenes, 2 games per row, 2026-10-17.
|
||||
Means; FPS is the mean of the four scenes. "Title" is start to title screen, "world" start to
|
||||
a fresh world drawn around the spawn, "scene load" a teleport to every chunk drawn.
|
||||
|
||||
| RD | game | title s | world s | scene load s | FPS | RSS steady / peak MB | load avg |
|
||||
|---|---|---:|---:|---:|---:|---:|---:|
|
||||
| 8 | Horizon + 6 mods | 9.8 | 16.2 | 4.0 | 122 | 2,644 / 2,771 | 32 |
|
||||
| 8 | NeoForge + 6 mods | 19.5 | 50.0 | 30.2 | 72 | 2,325 / 2,592 | 56 |
|
||||
| 8 | Horizon | 5.0 | 11.0 | 4.0 | 124 | 1,559 / 1,629 | 51 |
|
||||
| 8 | NeoForge | 12.8 | 36.1 | 21.9 | 104 | 2,334 / 2,558 | 30 |
|
||||
| 16 | Horizon + 6 mods | 10.8 | 27.0 | 11.3 | 90 | 2,925 / 3,032 | 40 |
|
||||
| 16 | NeoForge + 6 mods | 26.4 | 82.5 | 64.3 | 25 | 3,016 / 3,519 | 45 |
|
||||
| 16 | Horizon | 3.5 | 10.0 | 7.4 | 119 | 1,732 / 1,871 | 56 |
|
||||
| 16 | NeoForge | 16.9 | 73.7 | 43.2 | 36 | 2,748 / 3,425 | 48 |
|
||||
|
||||
Same content, Horizon against NeoForge: the world is ready 3.1x sooner at 8 chunks and 3.1x at
|
||||
16, the title screen 2.0x and 2.4x sooner, and the flights run 1.7x and 3.6x the frames. RAM is
|
||||
not lower: Horizon's steady RSS grows 1.1 GB with the mods (61,816 block states against
|
||||
vanilla's 35,723; the per-state tables are the next thing to measure), while the JVM stays
|
||||
inside its 2 GB heap. Caveats, so none of this is a "10x" claim yet: other sessions and CI
|
||||
runners kept the machine's load average between 30 and 56 on 14 threads, so the confidence
|
||||
intervals are as wide as the means (`perf_bench.py --table`); Horizon's frames are paced near
|
||||
the display's 144 Hz at 8 chunks; Horizon's title time includes building the content overlay on
|
||||
a first start (about 5 s, cached afterwards); and the pack runs 1.21.1 against Horizon's 26.3.
|
||||
A quiet rerun with more games is the next measurement.
|
||||
|
||||
## The working version for 2026-10-23
|
||||
|
||||
In scope (each is a PR or a small stack):
|
||||
@@ -545,7 +583,11 @@ In scope (each is a PR or a small stack):
|
||||
6. **Measured:** load time, RAM and FPS of that pack on NeoForge 1.21.1 against Horizon, with
|
||||
the full ATM10 on NeoForge as the reference row.
|
||||
|
||||
Out of scope for that date, and next in order: worldgen carry-over (the 30 or so structure and biome mods),
|
||||
Out of scope for that date, and next in order: worldgen carry-over (the 30 or so structure and biome mods;
|
||||
the first one looked at, Moog's Voyager Structures, declares all 130 structures and 116 structure
|
||||
sets with MoogsStructureLib's own `generic_jigsaw_structure` and `advanced_random_spread` types and
|
||||
NeoForge's `c:` biome tags, so its carry needs a mapping to vanilla's jigsaw and random spread
|
||||
types plus the oracle's biome tags, not only bundle merging),
|
||||
behaviour callbacks and server events (0.3), block entities, menus, capabilities and net
|
||||
(0.4), the first agent-ported logic-tier mod as the pilot of step 5, then machines. Porting all
|
||||
of ATM10 is a campaign. Each step publishes its coverage number; we make no claim beyond what
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
package dev.horizonlab.neoforgebench;
|
||||
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import com.mojang.blaze3d.platform.GlUtil;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import net.minecraft.client.Minecraft;
|
||||
import net.minecraft.client.gui.screens.TitleScreen;
|
||||
import net.minecraft.client.player.LocalPlayer;
|
||||
import net.minecraft.world.Difficulty;
|
||||
import net.minecraft.world.level.GameRules;
|
||||
import net.minecraft.world.level.GameType;
|
||||
import net.minecraft.world.level.LevelSettings;
|
||||
import net.minecraft.world.level.WorldDataConfiguration;
|
||||
import net.minecraft.world.level.levelgen.WorldOptions;
|
||||
import net.minecraft.world.level.levelgen.presets.WorldPresets;
|
||||
import net.minecraft.world.phys.Vec3;
|
||||
import net.neoforged.api.distmarker.Dist;
|
||||
import net.neoforged.fml.common.Mod;
|
||||
import net.neoforged.neoforge.client.event.RenderFrameEvent;
|
||||
import net.neoforged.neoforge.common.NeoForge;
|
||||
|
||||
/**
|
||||
* Dev-only benchmark driver for a NeoForge 1.21.1 client (tools/perf_bench.py
|
||||
* --game neoforge): the steps of mods/horizon/src/bench's VanillaBench, on
|
||||
* NeoForge's events. From the title screen it creates a fresh creative world
|
||||
* with the script's seed, waits for the spawn area, then per scene teleports,
|
||||
* waits for the chunks, settles and flies a straight line at constant speed,
|
||||
* recording every frame time. Events go to horizon.bench.out as JSON lines;
|
||||
* the game quits after the last scene. Never shipped.
|
||||
*/
|
||||
@Mod(value = "horizon_bench", dist = Dist.CLIENT)
|
||||
public final class NeoForgeBench {
|
||||
private enum Phase { TITLE, WORLD, TELEPORT, LOAD, SETTLE, FLY, DONE }
|
||||
|
||||
private final JsonObject script;
|
||||
private final Path out;
|
||||
private final List<JsonObject> scenes = new ArrayList<>();
|
||||
private Phase phase = Phase.TITLE;
|
||||
private int scene = -1;
|
||||
private long phaseStart;
|
||||
private long lastFrame;
|
||||
private long readyCheck;
|
||||
private int lastLoaded = -1;
|
||||
private long lastChange;
|
||||
private final List<Float> frames = new ArrayList<>();
|
||||
|
||||
public NeoForgeBench() {
|
||||
String scriptPath = System.getProperty("horizon.bench.script");
|
||||
String outPath = System.getProperty("horizon.bench.out");
|
||||
if (scriptPath == null || outPath == null) {
|
||||
script = null;
|
||||
out = null;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
script = JsonParser.parseString(Files.readString(Path.of(scriptPath))).getAsJsonObject();
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
out = Path.of(outPath);
|
||||
for (var element : script.getAsJsonArray("scenes")) scenes.add(element.getAsJsonObject());
|
||||
NeoForge.EVENT_BUS.addListener(RenderFrameEvent.Pre.class, e -> step(Minecraft.getInstance()));
|
||||
event("init");
|
||||
}
|
||||
|
||||
private void step(Minecraft mc) {
|
||||
long now = System.nanoTime();
|
||||
float frameMs = lastFrame == 0 ? 0 : (now - lastFrame) / 1e6f;
|
||||
lastFrame = now;
|
||||
double elapsed = (now - phaseStart) / 1e9;
|
||||
if (phase != Phase.TITLE && phase != Phase.DONE && elapsed > script.get("timeout").getAsDouble()
|
||||
&& phase != Phase.FLY && phase != Phase.SETTLE) {
|
||||
JsonObject e = new JsonObject();
|
||||
e.addProperty("phase", phase.name());
|
||||
event("timeout", e);
|
||||
finish(mc);
|
||||
return;
|
||||
}
|
||||
switch (phase) {
|
||||
case TITLE -> {
|
||||
if (mc.screen instanceof TitleScreen && mc.getOverlay() == null) {
|
||||
event("title", renderer(mc));
|
||||
LevelSettings settings = new LevelSettings(script.get("world").getAsString(), GameType.CREATIVE,
|
||||
false, Difficulty.NORMAL, true, new GameRules(), WorldDataConfiguration.DEFAULT);
|
||||
WorldOptions options = new WorldOptions(script.get("seed").getAsLong(), true, false);
|
||||
enter(Phase.WORLD, now);
|
||||
mc.createWorldOpenFlows().createFreshLevel(script.get("world").getAsString(), settings, options,
|
||||
WorldPresets::createNormalWorldDimensions, mc.screen);
|
||||
}
|
||||
}
|
||||
case WORLD -> {
|
||||
if (ready(mc, null, now)) {
|
||||
JsonObject e = new JsonObject();
|
||||
e.addProperty("ready_epoch_ms", epochMs(lastChange));
|
||||
e.addProperty("chunks", lastLoaded);
|
||||
event("world", e);
|
||||
mc.player.connection.sendCommand("weather clear");
|
||||
next(mc, now);
|
||||
}
|
||||
}
|
||||
case TELEPORT -> {
|
||||
JsonObject s = scenes.get(scene);
|
||||
Vec3 from = vec(s.getAsJsonArray("from"));
|
||||
LocalPlayer player = mc.player;
|
||||
player.connection.sendCommand("time set 6000");
|
||||
player.connection.sendCommand(String.format(java.util.Locale.ROOT, "tp @s %.3f %.3f %.3f %.3f %.3f",
|
||||
from.x, from.y, from.z, yaw(s), s.get("pitch").getAsFloat()));
|
||||
event("teleport", named(s));
|
||||
enter(Phase.LOAD, now);
|
||||
}
|
||||
case LOAD -> {
|
||||
JsonObject s = scenes.get(scene);
|
||||
Vec3 from = vec(s.getAsJsonArray("from"));
|
||||
if (mc.player != null && mc.player.position().distanceToSqr(from) < 1024.0) place(mc, s, 0);
|
||||
if (ready(mc, from, now)) {
|
||||
JsonObject e = named(s);
|
||||
e.addProperty("load_ms", (lastChange - phaseStart) / 1e6);
|
||||
e.addProperty("chunks", lastLoaded);
|
||||
event("loaded", e);
|
||||
enter(Phase.SETTLE, now);
|
||||
}
|
||||
}
|
||||
case SETTLE -> {
|
||||
place(mc, scenes.get(scene), 0);
|
||||
if (elapsed >= script.get("settle").getAsDouble()) {
|
||||
frames.clear();
|
||||
event("fly", named(scenes.get(scene)));
|
||||
enter(Phase.FLY, now);
|
||||
}
|
||||
}
|
||||
case FLY -> {
|
||||
frames.add(frameMs);
|
||||
double t = Math.min(1.0, elapsed / script.get("seconds").getAsDouble());
|
||||
place(mc, scenes.get(scene), t);
|
||||
if (t >= 1.0) {
|
||||
JsonObject e = named(scenes.get(scene));
|
||||
JsonArray times = new JsonArray();
|
||||
frames.forEach(times::add);
|
||||
e.add("frame_ms", times);
|
||||
e.addProperty("chunks", mc.level.getChunkSource().getLoadedChunksCount());
|
||||
event("flown", e);
|
||||
next(mc, now);
|
||||
}
|
||||
}
|
||||
case DONE -> { }
|
||||
}
|
||||
}
|
||||
|
||||
private void next(Minecraft mc, long now) {
|
||||
scene++;
|
||||
if (scene >= scenes.size()) {
|
||||
finish(mc);
|
||||
} else {
|
||||
enter(Phase.TELEPORT, now);
|
||||
}
|
||||
}
|
||||
|
||||
private void finish(Minecraft mc) {
|
||||
phase = Phase.DONE;
|
||||
event("done");
|
||||
mc.stop();
|
||||
}
|
||||
|
||||
private void enter(Phase next, long now) {
|
||||
phase = next;
|
||||
phaseStart = now;
|
||||
lastLoaded = -1;
|
||||
lastChange = now;
|
||||
}
|
||||
|
||||
/** As VanillaBench.ready: spawned (near {@code at}), no screen, every chunk around, nothing to compile, 1 s stable. */
|
||||
private boolean ready(Minecraft mc, Vec3 at, long now) {
|
||||
if (now - readyCheck < 10_000_000L) return false;
|
||||
readyCheck = now;
|
||||
if (mc.level == null || mc.player == null || mc.screen != null || mc.getOverlay() != null
|
||||
|| (at != null && mc.player.position().distanceToSqr(at) > 4.0)) {
|
||||
lastChange = now;
|
||||
lastLoaded = -1;
|
||||
return false;
|
||||
}
|
||||
int loaded = mc.level.getChunkSource().getLoadedChunksCount();
|
||||
if (loaded != lastLoaded || !mc.levelRenderer.hasRenderedAllSections() || !chunksAround(mc)) {
|
||||
lastLoaded = loaded;
|
||||
lastChange = now;
|
||||
return false;
|
||||
}
|
||||
return now - lastChange >= 1_000_000_000L;
|
||||
}
|
||||
|
||||
private static boolean chunksAround(Minecraft mc) {
|
||||
int r = Math.max(1, mc.options.getEffectiveRenderDistance() - 1);
|
||||
var center = mc.player.chunkPosition();
|
||||
var chunks = mc.level.getChunkSource();
|
||||
for (int dx = -r; dx <= r; dx++) {
|
||||
for (int dz = -r; dz <= r; dz++) {
|
||||
if (dx * dx + dz * dz <= r * r && !chunks.hasChunk(center.x + dx, center.z + dz)) return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static void place(Minecraft mc, JsonObject s, double t) {
|
||||
LocalPlayer player = mc.player;
|
||||
if (player == null) return;
|
||||
Vec3 a = vec(s.getAsJsonArray("from")), b = vec(s.getAsJsonArray("to"));
|
||||
Vec3 at = a.add(b.subtract(a).scale(t));
|
||||
player.setPos(at.x, at.y, at.z);
|
||||
player.setYRot(yaw(s));
|
||||
player.setXRot(s.get("pitch").getAsFloat());
|
||||
player.setOldPosAndRot();
|
||||
player.setDeltaMovement(Vec3.ZERO);
|
||||
player.fallDistance = 0;
|
||||
if (!player.getAbilities().flying) {
|
||||
player.getAbilities().flying = true;
|
||||
player.onUpdateAbilities();
|
||||
}
|
||||
}
|
||||
|
||||
private static float yaw(JsonObject s) {
|
||||
Vec3 a = vec(s.getAsJsonArray("from")), b = vec(s.getAsJsonArray("to"));
|
||||
return (float) Math.toDegrees(Math.atan2(-(b.x - a.x), b.z - a.z));
|
||||
}
|
||||
|
||||
private static Vec3 vec(JsonArray a) {
|
||||
return new Vec3(a.get(0).getAsDouble(), a.get(1).getAsDouble(), a.get(2).getAsDouble());
|
||||
}
|
||||
|
||||
private static JsonObject named(JsonObject s) {
|
||||
JsonObject e = new JsonObject();
|
||||
e.addProperty("scene", s.get("name").getAsString());
|
||||
return e;
|
||||
}
|
||||
|
||||
private static JsonObject renderer(Minecraft mc) {
|
||||
JsonObject e = new JsonObject();
|
||||
e.addProperty("gpu", GlUtil.getRenderer());
|
||||
e.addProperty("vendor", GlUtil.getVendor());
|
||||
e.addProperty("backend", "OpenGL");
|
||||
e.addProperty("driver", GlUtil.getOpenGLVersion());
|
||||
JsonArray size = new JsonArray();
|
||||
size.add(mc.getWindow().getWidth());
|
||||
size.add(mc.getWindow().getHeight());
|
||||
e.add("size", size);
|
||||
e.addProperty("render_distance", mc.options.renderDistance().get());
|
||||
e.addProperty("simulation_distance", mc.options.simulationDistance().get());
|
||||
return e;
|
||||
}
|
||||
|
||||
private static long epochMs(long nanos) {
|
||||
return System.currentTimeMillis() - (System.nanoTime() - nanos) / 1_000_000L;
|
||||
}
|
||||
|
||||
private void event(String name) {
|
||||
event(name, new JsonObject());
|
||||
}
|
||||
|
||||
private void event(String name, JsonObject e) {
|
||||
e.addProperty("event", name);
|
||||
e.addProperty("epoch_ms", System.currentTimeMillis());
|
||||
try {
|
||||
Files.writeString(out, e + "\n", StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
|
||||
} catch (IOException error) {
|
||||
throw new UncheckedIOException(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
# Dev-only: the benchmark driver for a NeoForge client (tools/perf_bench.py --game neoforge).
|
||||
# Built and run by tools/perf_bench.py; never shipped.
|
||||
modLoader = "javafml"
|
||||
loaderVersion = "[4,)"
|
||||
license = "MIT"
|
||||
|
||||
[[mods]]
|
||||
modId = "horizon_bench"
|
||||
version = "1"
|
||||
displayName = "Horizon benchmark driver (dev only)"
|
||||
description = "Flies the client along tools/perf_bench.py's paths and records frame times."
|
||||
@@ -17,7 +17,8 @@ and what the game's loaders make of it. For every block and item the mod registe
|
||||
- recipes: each recipe's result, count and ingredients;
|
||||
- items: present, and their stack size.
|
||||
|
||||
Writes WORK/mods/<id>/harness/result.json per mod and prints a table. Jars, facts and results stay
|
||||
Each mod is probed on its own, against vanilla. Writes WORK/mods/<id>/harness/{horizon,result}.json
|
||||
per mod and prints a table. Jars, facts and results stay
|
||||
in WORK, outside the repository.
|
||||
"""
|
||||
import argparse
|
||||
@@ -185,18 +186,23 @@ def main():
|
||||
metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--no-deps', '--format-version', '1'],
|
||||
cwd=ROOT, text=True))
|
||||
probe = Path(metadata['target_directory']) / 'release/examples/mod_port_probe'
|
||||
out = work / 'horizon.json'
|
||||
subprocess.run([str(probe), args.resources, str(work), str(out), *args.mods], check=True)
|
||||
oracle = json.loads(facts.read_text())
|
||||
horizon = json.loads(out.read_text())
|
||||
upgrades = sound_upgrades(oracle, horizon)
|
||||
old_items = set(oracle.get('vanilla_items', []))
|
||||
mods = args.mods or sorted(p.name for p in (work / 'mods').iterdir() if (p / 'carry').is_dir())
|
||||
print(f'{"mod":<16} {"result":<7} ' + ' '.join(f'{c:>12}' for c in CHECKS))
|
||||
rows = []
|
||||
for mod in mods:
|
||||
# One mod at a time: each against vanilla, as a player adding it would load it (and a pack's
|
||||
# sum may pass the block states a world holds, which the probe would refuse).
|
||||
out = work / 'mods' / mod / 'harness' / 'horizon.json'
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run([str(probe), args.resources, str(work), str(out), mod], check=True)
|
||||
horizon = json.loads(out.read_text())
|
||||
carry_report = work / 'mods' / mod / 'carry-report.json'
|
||||
carry = json.loads(carry_report.read_text()) if carry_report.exists() else {}
|
||||
result = compare_mod(mod, oracle, horizon, carry, upgrades, old_items)
|
||||
rows.append(compare_mod(mod, oracle, horizon, carry, sound_upgrades(oracle, horizon), old_items))
|
||||
print(f'{"mod":<16} {"result":<7} ' + ' '.join(f'{c:>12}' for c in CHECKS))
|
||||
for result in rows:
|
||||
mod = result['mod']
|
||||
(work / 'mods' / mod / 'harness').mkdir(parents=True, exist_ok=True)
|
||||
(work / 'mods' / mod / 'harness' / 'result.json').write_text(json.dumps(result, indent=1))
|
||||
cols = []
|
||||
|
||||
@@ -45,6 +45,84 @@ def install():
|
||||
return home
|
||||
|
||||
|
||||
def client():
|
||||
"""The NeoForge client (tools/perf_bench.py --game neoforge), installed once per version: the installer's
|
||||
own files, then the vanilla libraries its version inherits, fetched as the launcher does (sha1-checked)."""
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import hashlib
|
||||
import lab
|
||||
home = LOCAL / f'client-{NEOFORGE}'
|
||||
profile = home / 'versions' / f'neoforge-{NEOFORGE}' / f'neoforge-{NEOFORGE}.json'
|
||||
if not profile.exists():
|
||||
install() # the installer jar
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
(home / 'launcher_profiles.json').write_text('{"profiles":{}}')
|
||||
with (LOCAL / f'install-client-{NEOFORGE}.log').open('w') as log:
|
||||
subprocess.run(['java', '-jar', str(LOCAL / f'neoforge-{NEOFORGE}-installer.jar'), '--installClient',
|
||||
str(home)], cwd=LOCAL, stdout=log, stderr=subprocess.STDOUT, check=True)
|
||||
neo = json.loads(profile.read_text())
|
||||
base = json.loads((home / 'versions' / neo['inheritsFrom'] / f'{neo["inheritsFrom"]}.json').read_text())
|
||||
jars = []
|
||||
for manifest in (neo, base):
|
||||
names = {library['name'] for library in manifest['libraries']}
|
||||
for library in manifest['libraries']:
|
||||
artifact = library.get('downloads', {}).get('artifact')
|
||||
if not artifact or not lab.library_allowed(library) or not lab.native_library_matches(library, None, names):
|
||||
continue
|
||||
path = home / 'libraries' / artifact['path']
|
||||
if not path.exists():
|
||||
with urllib.request.urlopen(artifact['url'], timeout=120) as r:
|
||||
data = r.read()
|
||||
if hashlib.sha1(data).hexdigest() != artifact['sha1']:
|
||||
sys.exit(f'{artifact["url"]}: sha1 mismatch')
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
if path not in jars:
|
||||
jars.append(path)
|
||||
return home, neo, base, jars
|
||||
|
||||
|
||||
def client_command(java, heap, game_dir, natives, assets, jvm=(), game=()):
|
||||
"""The command the launcher would run for the NeoForge client profile, with no account."""
|
||||
home, neo, base, jars = client()
|
||||
import lab
|
||||
values = {'library_directory': str(home / 'libraries'), 'classpath_separator': os.pathsep,
|
||||
'version_name': f'neoforge-{NEOFORGE}', 'natives_directory': str(natives),
|
||||
'launcher_name': 'horizon-bench', 'launcher_version': '1'}
|
||||
expand = lambda s: s.replace('${', '{').format(**values)
|
||||
command = [str(java), f'-Xmx{heap}']
|
||||
for argument in base['arguments']['jvm']:
|
||||
if isinstance(argument, dict):
|
||||
if lab.library_allowed(argument):
|
||||
command += [argument['value']] if isinstance(argument['value'], str) else argument['value']
|
||||
elif argument == '${classpath}':
|
||||
# Every library; not the vanilla jar, which NeoForge's patched client replaces.
|
||||
command.append(os.pathsep.join(map(str, jars)))
|
||||
else:
|
||||
command.append(expand(argument))
|
||||
command += [expand(a) for a in neo['arguments']['jvm']]
|
||||
command += [*jvm, neo['mainClass']]
|
||||
command += ['--username', 'Bench', '--uuid', '00000000000000000000000000000000', '--accessToken', '0',
|
||||
'--version', f'neoforge-{NEOFORGE}', '--gameDir', str(game_dir), '--assetsDir', str(assets),
|
||||
'--assetIndex', base['assetIndex']['id'], '--userType', 'msa', '--versionType', 'release',
|
||||
*neo['arguments']['game'], *game]
|
||||
return home, command
|
||||
|
||||
|
||||
def build_bench(home):
|
||||
"""The NeoForge benchmark driver (mods/horizon/src/neoforge-bench), against the client."""
|
||||
return build(home, ROOT / 'mods/horizon/src/neoforge-bench', 'horizon-bench-neoforge.jar', client_classpath(home))
|
||||
|
||||
|
||||
def client_classpath(home):
|
||||
libs = home / 'libraries'
|
||||
first = [libs / f'net/neoforged/neoforge/{NEOFORGE}/neoforge-{NEOFORGE}-client.jar',
|
||||
libs / f'net/neoforged/neoforge/{NEOFORGE}/neoforge-{NEOFORGE}-universal.jar']
|
||||
rest = sorted(p for p in libs.rglob('*.jar') if p not in first and 'slim' not in p.name
|
||||
and 'extra' not in p.name and not p.name.endswith('-unpacked.jar'))
|
||||
return os.pathsep.join(map(str, first + rest))
|
||||
|
||||
|
||||
def classpath(home):
|
||||
libs = home / 'libraries'
|
||||
first = [libs / f'net/neoforged/neoforge/{NEOFORGE}/neoforge-{NEOFORGE}-server.jar',
|
||||
@@ -54,27 +132,27 @@ def classpath(home):
|
||||
return os.pathsep.join(map(str, first + rest))
|
||||
|
||||
|
||||
def build(home):
|
||||
"""The exporter mod jar, rebuilt when its sources change."""
|
||||
def build(home, source=SOURCE, name='horizon-oracle.jar', cp=None):
|
||||
"""A dev-only mod jar (by default the exporter), rebuilt when its sources change."""
|
||||
out = LOCAL / 'build'
|
||||
jar = out / 'horizon-oracle.jar'
|
||||
sources = sorted((SOURCE / 'java').rglob('*.java'))
|
||||
resources = sorted(p for p in (SOURCE / 'resources').rglob('*') if p.is_file())
|
||||
jar = out / name
|
||||
sources = sorted((source / 'java').rglob('*.java'))
|
||||
resources = sorted(p for p in (source / 'resources').rglob('*') if p.is_file())
|
||||
newest = max(p.stat().st_mtime for p in sources + resources)
|
||||
if jar.exists() and jar.stat().st_mtime > newest:
|
||||
return jar
|
||||
classes = out / 'classes'
|
||||
classes = out / f'classes-{Path(name).stem}'
|
||||
shutil.rmtree(classes, ignore_errors=True)
|
||||
classes.mkdir(parents=True)
|
||||
javac = subprocess.run(['javac', '--release', '21', '-proc:none', '-nowarn', '-cp', classpath(home),
|
||||
javac = subprocess.run(['javac', '--release', '21', '-proc:none', '-nowarn', '-cp', cp or classpath(home),
|
||||
'-d', str(classes), *map(str, sources)], capture_output=True, text=True)
|
||||
if javac.returncode:
|
||||
sys.exit(f'the exporter does not compile:\n{javac.stdout}{javac.stderr}')
|
||||
sys.exit(f'{name} does not compile:\n{javac.stdout}{javac.stderr}')
|
||||
with zipfile.ZipFile(jar, 'w', zipfile.ZIP_DEFLATED) as z:
|
||||
for p in sorted(classes.rglob('*.class')):
|
||||
z.write(p, p.relative_to(classes).as_posix())
|
||||
for p in resources:
|
||||
z.write(p, p.relative_to(SOURCE / 'resources').as_posix())
|
||||
z.write(p, p.relative_to(source / 'resources').as_posix())
|
||||
return jar
|
||||
|
||||
|
||||
|
||||
+93
-6
@@ -13,6 +13,11 @@ for every game. RSS is sampled from outside the process for every game too.
|
||||
python3 tools/perf_bench.py --game vanilla-mods --label mods-m4
|
||||
python3 tools/perf_bench.py --table horizon-m4 vanilla-m4 mods-m4
|
||||
|
||||
A ported mod pack against the same mods on their own loader (docs/modpacks.md, "Measured"):
|
||||
|
||||
python3 tools/perf_bench.py --game horizon --port-work WORK --label pack-horizon
|
||||
python3 tools/perf_bench.py --game neoforge --mod-jars JARS --deps DEPS --label pack-neoforge
|
||||
|
||||
Games:
|
||||
horizon target/release/horizon-client, driven over its control socket
|
||||
vanilla your own 26.3 client.jar + Fabric Loader + a dev-only driver
|
||||
@@ -20,6 +25,13 @@ Games:
|
||||
is redistributed
|
||||
vanilla-mods the same + Sodium, Lithium, FerriteCore and ImmediatelyFast
|
||||
(pinned in packs/mods.lock.json)
|
||||
neoforge Minecraft 1.21.1 on NeoForge (dev-only, tools/neoforge_oracle.py
|
||||
installs it under .local/neoforge-oracle) + a driver mod
|
||||
(mods/horizon/src/neoforge-bench) + the jars of --mod-jars and
|
||||
--deps, the user's own; 26.3 options it does not know are ignored
|
||||
|
||||
--port-work loads the mods a horizon-port-pack work folder carried over into
|
||||
Horizon as dev mods.
|
||||
|
||||
Results: benchmarks/perf/<label>-rd<N>.json, one benchmark report per render
|
||||
distance (docs/benchmark-report.md), over --repeats games. Games run with no
|
||||
@@ -105,6 +117,8 @@ OPTIONS = {
|
||||
LAUNCHER_JVM = ["-XX:+UnlockExperimentalVMOptions", "-XX:+UseG1GC", "-XX:G1NewSizePercent=20",
|
||||
"-XX:G1ReservePercent=20", "-XX:MaxGCPauseMillis=50", "-XX:G1HeapRegionSize=32M"]
|
||||
PERF_MODS = ["sodium", "lithium", "ferrite-core", "immediatelyfast"]
|
||||
# Minecraft 1.21.1's data version, for the NeoForge client's options.txt (it ignores 26.3's newer keys).
|
||||
NEOFORGE_DATA_VERSION = "3955"
|
||||
|
||||
|
||||
# The benchmark report format, version 1 (docs/benchmark-report.md). crates/horizon-bench is the
|
||||
@@ -211,9 +225,10 @@ def nvidia_memory_mb(pid):
|
||||
return None
|
||||
|
||||
|
||||
def write_options(game_dir, distance):
|
||||
def write_options(game_dir, distance, version=None):
|
||||
game_dir.mkdir(parents=True, exist_ok=True)
|
||||
text = "".join(f"{k}:{v.format(distance=distance)}\n" for k, v in OPTIONS.items())
|
||||
options = dict(OPTIONS, version=version) if version else OPTIONS
|
||||
text = "".join(f"{k}:{v.format(distance=distance)}\n" for k, v in options.items())
|
||||
(game_dir / "options.txt").write_text(text)
|
||||
return text
|
||||
|
||||
@@ -324,6 +339,8 @@ def run_horizon(args, distance):
|
||||
"--control-socket", str(sock), "--no-pause", "--weather", "clear", "--time", "6000"]
|
||||
if args.offscreen:
|
||||
command.append("--offscreen")
|
||||
for carry in dev_mods(args):
|
||||
command += ["--dev-mod", str(carry)]
|
||||
env = dict(os.environ, HORIZON_NO_AUDIO="1")
|
||||
log = (WORK / f"horizon-rd{distance}.log").open("w")
|
||||
started = time.time()
|
||||
@@ -376,6 +393,13 @@ def run_horizon(args, distance):
|
||||
return result
|
||||
|
||||
|
||||
def dev_mods(args):
|
||||
"""The carried-over mods of a horizon-port-pack work folder (docs/modpacks.md), loaded as dev mods."""
|
||||
if not args.port_work:
|
||||
return []
|
||||
return sorted(p for p in (Path(args.port_work) / "mods").glob("*/carry") if (p / "horizon-mod.toml").exists())
|
||||
|
||||
|
||||
# Vanilla -------------------------------------------------------------------
|
||||
|
||||
def vanilla_manifest():
|
||||
@@ -491,6 +515,11 @@ def run_vanilla(args, distance):
|
||||
libs = [p for p in (os.environ.get("NIX_LD_LIBRARY_PATH", ""), "/run/opengl-driver/lib") if p]
|
||||
env["LD_LIBRARY_PATH"] = ":".join(libs + [env.get("LD_LIBRARY_PATH", "")]).strip(":")
|
||||
log = (WORK / f"{args.game}-rd{distance}.log").open("w")
|
||||
return driven(args, distance, command, game_dir, env, log, events, options, loaded_mods)
|
||||
|
||||
|
||||
def driven(args, distance, command, game_dir, env, log, events, options, loaded_mods):
|
||||
"""Runs a Java client whose driver mod writes `events`, and reads its results."""
|
||||
started = time.time()
|
||||
proc = subprocess.Popen(command, cwd=game_dir, env=env, stdout=log, stderr=subprocess.STDOUT)
|
||||
sampler = Sampler(proc.pid)
|
||||
@@ -508,7 +537,7 @@ def run_vanilla(args, distance):
|
||||
lines = [json.loads(l) for l in events.read_text().splitlines()] if events.exists() else []
|
||||
by = lambda name: [e for e in lines if e["event"] == name]
|
||||
if not by("done") or by("timeout"):
|
||||
raise RuntimeError(f"vanilla run failed: {lines[-1:] if lines else 'no events'}; see {log.name}")
|
||||
raise RuntimeError(f"{args.game} run failed: {lines[-1:] if lines else 'no events'}; see {log.name}")
|
||||
title, world = by("title")[0], by("world")[0]
|
||||
result = {
|
||||
"options_txt": options, "mods": loaded_mods, "heap": args.heap,
|
||||
@@ -532,6 +561,55 @@ def run_vanilla(args, distance):
|
||||
return result
|
||||
|
||||
|
||||
# NeoForge ------------------------------------------------------------------
|
||||
|
||||
def run_neoforge(args, distance):
|
||||
"""A NeoForge 1.21.1 client (dev-only, tools/neoforge_oracle.py) with the driver mod and --mod-jars:
|
||||
the mods a ported pack replaces, on the loader the pack runs on."""
|
||||
import neoforge_oracle
|
||||
home = neoforge_oracle.client()[0]
|
||||
mod = neoforge_oracle.build_bench(home)
|
||||
game_dir = WORK / args.game
|
||||
shutil.rmtree(game_dir, ignore_errors=True)
|
||||
options = write_options(game_dir, distance, version=NEOFORGE_DATA_VERSION)
|
||||
mods = game_dir / "mods"
|
||||
mods.mkdir()
|
||||
shutil.copy(mod, mods)
|
||||
loaded_mods = ["horizon-bench"]
|
||||
for folder in filter(None, (args.mod_jars, args.deps)):
|
||||
for jar in sorted(Path(folder).glob("*.jar")):
|
||||
shutil.copy(jar, mods)
|
||||
loaded_mods.append(jar.name)
|
||||
natives = WORK / "natives-neoforge"
|
||||
script = WORK / "script.json"
|
||||
script.write_text(json.dumps({
|
||||
"world": WORLD, "seed": SEED, "settle": args.settle, "seconds": args.seconds, "timeout": args.timeout,
|
||||
"scenes": [{"name": n, "from": a, "to": b, "pitch": p} for n, a, b, p in scene_list(args)],
|
||||
}))
|
||||
events = WORK / f"{args.game}-rd{distance}.events.jsonl"
|
||||
events.unlink(missing_ok=True)
|
||||
assets = WORK / "assets"
|
||||
(assets / "indexes").mkdir(parents=True, exist_ok=True)
|
||||
java = Path(os.environ["JAVA_HOME"]) / "bin/java" if os.environ.get("JAVA_HOME") else "java"
|
||||
_, command = neoforge_oracle.client_command(
|
||||
java, args.heap, game_dir, natives, assets,
|
||||
jvm=[*LAUNCHER_JVM, f"-Dhorizon.bench.script={script}", f"-Dhorizon.bench.out={events}"],
|
||||
game=["--width", str(args.width), "--height", str(args.height)])
|
||||
natives.mkdir(parents=True, exist_ok=True)
|
||||
extract_natives(neoforge_oracle.client()[3], natives)
|
||||
env = dict(os.environ, ALSOFT_DRIVERS="null")
|
||||
for attempt in (1, 2):
|
||||
log = (WORK / f"{args.game}-rd{distance}.log").open("w")
|
||||
try:
|
||||
return driven(args, distance, command, game_dir, env, log, events, options, loaded_mods)
|
||||
except RuntimeError:
|
||||
# FML sometimes reads config/fml.toml while creating it on a first start, and crashes before
|
||||
# the title screen; a crash once the driver started is a real failure.
|
||||
if attempt == 2 or events.exists():
|
||||
raise
|
||||
shutil.rmtree(game_dir / "crash-reports", ignore_errors=True)
|
||||
|
||||
|
||||
def stop(proc):
|
||||
try:
|
||||
proc.wait(timeout=60)
|
||||
@@ -654,7 +732,7 @@ def table(labels):
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--game", choices=["horizon", "vanilla", "vanilla-mods"], default="horizon")
|
||||
p.add_argument("--game", choices=["horizon", "vanilla", "vanilla-mods", "neoforge"], default="horizon")
|
||||
p.add_argument("--label", help="result name (default <game>-<host>)")
|
||||
p.add_argument("--distances", default="8,16,32")
|
||||
p.add_argument("--scenes", help="comma-separated subset of " + ",".join(s[0] for s in SCENES))
|
||||
@@ -669,6 +747,9 @@ def main():
|
||||
p.add_argument("--resources", default=str(ROOT / ".local/horizon-client/26.3"))
|
||||
p.add_argument("--build", default=str(ROOT / "target/release"))
|
||||
p.add_argument("--repeats", type=int, default=3, help="games per render distance; the report gives means and 95%% CIs")
|
||||
p.add_argument("--port-work", help="horizon: also load the mods carried over in this horizon-port-pack work folder")
|
||||
p.add_argument("--mod-jars", help="neoforge: the mod jars to load (the user's own; never committed)")
|
||||
p.add_argument("--deps", help="neoforge: their required libraries")
|
||||
p.add_argument("--table", nargs="+", metavar="LABEL", help="print markdown tables of saved runs, and exit")
|
||||
args = p.parse_args()
|
||||
if args.table:
|
||||
@@ -676,7 +757,7 @@ def main():
|
||||
return
|
||||
args.label = args.label or f"{args.game}-{platform.node().split('.')[0]}"
|
||||
WORK.mkdir(parents=True, exist_ok=True)
|
||||
run = run_horizon if args.game == "horizon" else run_vanilla
|
||||
run = {"horizon": run_horizon, "neoforge": run_neoforge}.get(args.game, run_vanilla)
|
||||
for d in [int(x) for x in args.distances.split(",")]:
|
||||
runs = []
|
||||
for _ in range(args.repeats):
|
||||
@@ -696,7 +777,13 @@ def app(args):
|
||||
capture_output=True, text=True).stdout.strip() or None
|
||||
manifest = (ROOT / "crates/horizon-client/Cargo.toml").read_text()
|
||||
version = next(l.split('"')[1] for l in manifest.splitlines() if l.startswith("version"))
|
||||
return {"name": "horizon-client", "version": version, "commit": commit}
|
||||
mods = [{"name": p.parent.name, "version": "carried"} for p in dev_mods(args)]
|
||||
return {"name": "horizon-client", "version": version, "commit": commit, **({"mods": mods} if mods else {})}
|
||||
if args.game == "neoforge":
|
||||
import neoforge_oracle
|
||||
jars = [j.name for f in filter(None, (args.mod_jars, args.deps)) for j in sorted(Path(f).glob("*.jar"))]
|
||||
return {"name": "minecraft", "version": "1.21.1",
|
||||
"mods": [{"name": "neoforge", "version": neoforge_oracle.NEOFORGE}] + [{"name": j} for j in jars]}
|
||||
lock = json.loads((ROOT / "packs/mods.lock.json").read_text())
|
||||
mods = [{"name": "fabric-loader", "version": lock["fabric_loader"]}]
|
||||
if args.game == "vanilla-mods":
|
||||
|
||||
Reference in New Issue
Block a user