7eded537d6
Right after a teleport, and at spawn, Horizon v2026.10.22.04 reports nothing queued and steady chunk counts for over a second before it starts loading. wait_loaded() took that for "loaded" and stopped the clock with 0 to 3 sections meshed (load 0.1 s), so the flights ran over empty terrain and the frame rates meant nothing. Now the counts must change once since the wait began (or stay at the full view for 5 s), and the resident chunks must reach 80% of the view circle (pi x RD^2), before a steady second counts as loaded. Measured with it on the Mac mini and the ThinkPad for the website's Benchmarks page (#1274). At RD 32 a 4-thread laptop needs --timeout above the default 300 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
854 lines
40 KiB
Python
854 lines
40 KiB
Python
#!/usr/bin/env python3
|
||
"""Fair Horizon vs vanilla 26.3 benchmark (methodology: docs/benchmark.md).
|
||
|
||
Runs one game per render distance, each from a fresh world (seed 1234,
|
||
creative), through the same steps: start to the title screen, create the
|
||
world, wait for the spawn area, then per scene teleport, wait for the chunks,
|
||
settle and fly a fixed straight path at constant speed. Every frame time of
|
||
every flight is kept and the statistics are computed here, by one function,
|
||
for every game. RSS is sampled from outside the process for every game too.
|
||
|
||
python3 tools/perf_bench.py --game horizon --label horizon-m4
|
||
python3 tools/perf_bench.py --game vanilla --label vanilla-m4
|
||
python3 tools/perf_bench.py --game vanilla-mods --label mods-m4
|
||
python3 tools/perf_bench.py --table horizon-m4 vanilla-m4 mods-m4
|
||
|
||
A ported mod pack against the same mods on their own loader (docs/modpacks.md, "Measured"):
|
||
|
||
python3 tools/perf_bench.py --game horizon --port-work WORK --label pack-horizon
|
||
python3 tools/perf_bench.py --game neoforge --mod-jars JARS --deps DEPS --label pack-neoforge
|
||
|
||
Games:
|
||
horizon target/release/horizon-client, driven over its control socket
|
||
vanilla your own 26.3 client.jar + Fabric Loader + a dev-only driver
|
||
mod (mods/horizon/src/bench), built here; nothing of the game
|
||
is redistributed
|
||
vanilla-mods the same + Sodium, Lithium, FerriteCore and ImmediatelyFast
|
||
(pinned in packs/mods.lock.json)
|
||
neoforge Minecraft 1.21.1 on NeoForge (dev-only, tools/neoforge_oracle.py
|
||
installs it under .local/neoforge-oracle) + a driver mod
|
||
(mods/horizon/src/neoforge-bench) + the jars of --mod-jars and
|
||
--deps, the user's own; 26.3 options it does not know are ignored
|
||
|
||
--port-work loads the mods a horizon-port-pack work folder carried over into
|
||
Horizon as dev mods.
|
||
|
||
Results: benchmarks/perf/<label>-rd<N>.json, one benchmark report per render
|
||
distance (docs/benchmark-report.md), over --repeats games. Games run with no
|
||
audio, from a scratch directory under .local/perf-bench.
|
||
"""
|
||
import argparse
|
||
import bisect
|
||
import datetime
|
||
import json
|
||
import math
|
||
import os
|
||
import platform
|
||
import re
|
||
import shutil
|
||
import socket
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import zipfile
|
||
from pathlib import Path
|
||
|
||
try:
|
||
from . import lab, runtime
|
||
except ImportError:
|
||
import lab
|
||
import runtime
|
||
|
||
ROOT = lab.ROOT
|
||
WORK = ROOT / ".local/perf-bench"
|
||
VANILLA = lab.GAME
|
||
SEED = 1234
|
||
WORLD = "PerfBench"
|
||
# (name, from xyz, to xyz, pitch): straight 192-block flights over fixed
|
||
# scenery of seed 1234 (the same paths as tools/graphics_bench.py).
|
||
SCENES = [
|
||
("village", (-960.0, 125.0, 80.0), (-960.0, 125.0, 272.0), 35.0),
|
||
("forest", (-1100.0, 120.0, -560.0), (-1100.0, 120.0, -368.0), 25.0),
|
||
("ocean", (-300.0, 85.0, -420.0), (-300.0, 85.0, -228.0), 10.0),
|
||
("mountain", (560.0, 260.0, 860.0), (752.0, 260.0, 860.0), 30.0),
|
||
]
|
||
# One options.txt for every game: vanilla's own keys, which Horizon reads too.
|
||
# The graphics settings are those of vanilla's default preset (Fancy) with the
|
||
# preset set to Custom, so the render distance is the one asked for (a preset
|
||
# would replace it on load). maxFps 260 is "Unlimited"; inactivityFpsLimit
|
||
# minimized turns off the 30 FPS limit both games apply after a minute without
|
||
# input. Everything not listed stays at vanilla's default.
|
||
OPTIONS = {
|
||
"version": "{data_version}",
|
||
"graphicsPreset": '"custom"',
|
||
"renderDistance": "{distance}",
|
||
"simulationDistance": "12",
|
||
"biomeBlendRadius": "2",
|
||
"prioritizeChunkUpdates": "1",
|
||
"ao": "true",
|
||
"renderClouds": '"true"',
|
||
"cloudRange": "64",
|
||
"particles": "0",
|
||
"mipmapLevels": "4",
|
||
"entityShadows": "true",
|
||
"entityDistanceScaling": "1.0",
|
||
"menuBackgroundBlurriness": "5",
|
||
"cutoutLeaves": "true",
|
||
"improvedTransparency": "false",
|
||
"weatherRadius": "10",
|
||
"maxAnisotropyBit": "1",
|
||
"textureFiltering": "1",
|
||
"maxFps": "260",
|
||
"enableVsync": "false",
|
||
"inactivityFpsLimit": '"minimized"',
|
||
"fov": "0.0",
|
||
"guiScale": "0",
|
||
"fullscreen": "false",
|
||
"pauseOnLostFocus": "false",
|
||
"onboardAccessibility": "false",
|
||
"skipMultiplayerWarning": "true",
|
||
"tutorialStep": "none",
|
||
"narrator": "0",
|
||
"soundCategory_master": "0.0",
|
||
"realmsNotifications": "false",
|
||
}
|
||
# The official launcher's default JVM arguments for 26.3: what players run.
|
||
LAUNCHER_JVM = ["-XX:+UnlockExperimentalVMOptions", "-XX:+UseG1GC", "-XX:G1NewSizePercent=20",
|
||
"-XX:G1ReservePercent=20", "-XX:MaxGCPauseMillis=50", "-XX:G1HeapRegionSize=32M"]
|
||
PERF_MODS = ["sodium", "lithium", "ferrite-core", "immediatelyfast"]
|
||
# Minecraft 1.21.1's data version, for the NeoForge client's options.txt (it ignores 26.3's newer keys).
|
||
NEOFORGE_DATA_VERSION = "3955"
|
||
|
||
|
||
# The benchmark report format, version 1 (docs/benchmark-report.md). crates/horizon-bench is the
|
||
# reference: these functions compute the same figures, with the same rounding, for every game.
|
||
REPORT_FORMAT, REPORT_VERSION = "horizon-benchmark-report", 1
|
||
HISTOGRAM_EDGES_MS = [0, 1, 2, 3, 4.17, 5, 6, 7, 8.33, 10, 12, 14, 16.67, 20, 25, 33.33, 40,
|
||
50, 66.67, 100, 150, 250, 500, 1000]
|
||
# Two-sided 95% critical values of Student's t for 1..30 degrees of freedom.
|
||
T95 = [12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160,
|
||
2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056,
|
||
2.052, 2.048, 2.045, 2.042]
|
||
|
||
|
||
def rnd(v):
|
||
"""Three decimals, halves away from zero (Rust's f64::round)."""
|
||
return math.copysign(math.floor(abs(v) * 1000 + 0.5), v) / 1000
|
||
|
||
|
||
def frames(phase, repeat, frame_ms, cpu_ms=(), gpu_ms=()):
|
||
"""Every figure of a flight, from its frame times in milliseconds (horizon_bench::Frames::of).
|
||
|
||
The x% low is the average frame time of the slowest x% of frames, as
|
||
frames per second (so a single 2-second freeze shows in the 0.1% low of a
|
||
10,000-frame flight)."""
|
||
times = sorted(t for t in frame_ms if math.isfinite(t))
|
||
n = len(times)
|
||
if n == 0:
|
||
return None
|
||
total = sum(times)
|
||
pct = lambda values, p: values[min(len(values), max(1, math.ceil(len(values) * p))) - 1]
|
||
|
||
def low(share):
|
||
slowest = times[n - max(1, math.ceil(n * share)):]
|
||
return 1000.0 * len(slowest) / sum(slowest)
|
||
|
||
histogram = [0] * len(HISTOGRAM_EDGES_MS)
|
||
for t in times:
|
||
histogram[max(1, bisect.bisect_right(HISTOGRAM_EDGES_MS, t)) - 1] += 1
|
||
entry = {
|
||
"phase": phase, "repeat": repeat, "frames": n, "seconds": rnd(total / 1000.0),
|
||
"avg_fps": rnd(n * 1000.0 / total), "low_1pct_fps": rnd(low(0.01)), "low_01pct_fps": rnd(low(0.001)),
|
||
"p50_ms": rnd(pct(times, 0.5)), "p95_ms": rnd(pct(times, 0.95)), "p99_ms": rnd(pct(times, 0.99)),
|
||
"p999_ms": rnd(pct(times, 0.999)), "max_ms": rnd(times[-1]),
|
||
}
|
||
for key, values in (("cpu_p50_ms", cpu_ms), ("gpu_p50_ms", gpu_ms)):
|
||
if values:
|
||
entry[key] = rnd(pct(sorted(values), 0.5))
|
||
entry["histogram"] = histogram
|
||
return entry
|
||
|
||
|
||
def summary(values):
|
||
"""A measurement over repeats (horizon_bench::Summary::of); None for no values."""
|
||
values = [v for v in values if v is not None]
|
||
n = len(values)
|
||
if n == 0:
|
||
return None
|
||
mean = sum(values) / n
|
||
sd = ci = None
|
||
if n > 1:
|
||
sd = math.sqrt(sum((v - mean) ** 2 for v in values) / (n - 1))
|
||
ci = rnd((T95[n - 2] if n - 1 <= len(T95) else 1.96) * sd / math.sqrt(n))
|
||
sd = rnd(sd)
|
||
return {"n": n, "mean": rnd(mean), "stdev": sd, "min": rnd(min(values)), "max": rnd(max(values)), "ci95": ci}
|
||
|
||
|
||
def phys_footprint(pid):
|
||
"""macOS: the bytes the kernel charges the process (`footprint`), without
|
||
the freed pages the allocator marked reusable, which `ps` still counts as
|
||
resident until memory runs short. None elsewhere."""
|
||
if sys.platform != "darwin":
|
||
return None
|
||
out = subprocess.run(["footprint", "-f", "bytes", str(pid)], capture_output=True, text=True)
|
||
m = re.search(r"phys_footprint:\s*(\d+)", out.stdout)
|
||
return int(m.group(1)) if m else None
|
||
|
||
|
||
class Sampler(threading.Thread):
|
||
"""Samples a process's resident memory every quarter second (ps, so the
|
||
same reading for every game and OS) and, on macOS, its physical footprint
|
||
every two seconds."""
|
||
|
||
def __init__(self, pid):
|
||
super().__init__(daemon=True)
|
||
self.pid, self.samples, self.footprints, self.stop = pid, [], [], threading.Event()
|
||
|
||
def run(self):
|
||
tick = 0
|
||
while not self.stop.is_set():
|
||
out = subprocess.run(["ps", "-o", "rss=", "-p", str(self.pid)], capture_output=True, text=True)
|
||
if out.stdout.strip():
|
||
self.samples.append((time.time(), int(out.stdout.split()[0]) * 1024))
|
||
if tick % 8 == 0 and (fp := phys_footprint(self.pid)) is not None:
|
||
self.footprints.append((time.time(), fp))
|
||
tick += 1
|
||
self.stop.wait(0.25)
|
||
|
||
def summary(self, windows):
|
||
"""Peak over the whole run; steady = median over the flight windows."""
|
||
if not self.samples:
|
||
return {}
|
||
during = [rss for t, rss in self.samples if any(a <= t <= b for a, b in windows)]
|
||
out = {
|
||
"rss_peak_mb": max(rss for _, rss in self.samples) / 2**20,
|
||
"rss_steady_mb": statistics.median(during) / 2**20 if during else None,
|
||
"rss_samples": len(self.samples),
|
||
}
|
||
if self.footprints:
|
||
steady = [fp for t, fp in self.footprints if any(a <= t <= b for a, b in windows)]
|
||
out["footprint_peak_mb"] = max(fp for _, fp in self.footprints) / 2**20
|
||
out["footprint_steady_mb"] = statistics.median(steady) / 2**20 if steady else None
|
||
return out
|
||
|
||
|
||
def nvidia_memory_mb(pid):
|
||
"""Video memory nvidia-smi attributes to the process, if it can."""
|
||
if not shutil.which("nvidia-smi"):
|
||
return None
|
||
out = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_memory", "--format=csv,noheader,nounits"],
|
||
capture_output=True, text=True)
|
||
for line in out.stdout.splitlines():
|
||
p, _, mem = line.partition(",")
|
||
if p.strip() == str(pid):
|
||
return float(mem)
|
||
return None
|
||
|
||
|
||
def write_options(game_dir, distance, version=None):
|
||
game_dir.mkdir(parents=True, exist_ok=True)
|
||
data_version = lab.version_info()["world_version"]
|
||
options = dict(OPTIONS, version=version) if version else OPTIONS
|
||
text = "".join(f"{k}:{v.format(distance=distance, data_version=data_version)}\n" for k, v in options.items())
|
||
(game_dir / "options.txt").write_text(text)
|
||
return text
|
||
|
||
|
||
def scene_list(args):
|
||
picked = args.scenes.split(",") if args.scenes else [s[0] for s in SCENES]
|
||
return [s for s in SCENES if s[0] in picked]
|
||
|
||
|
||
def yaw(start, end):
|
||
return math.degrees(math.atan2(-(end[0] - start[0]), end[2] - start[2]))
|
||
|
||
|
||
# Horizon -------------------------------------------------------------------
|
||
|
||
class Control:
|
||
def __init__(self, path):
|
||
self.path = path
|
||
|
||
def __call__(self, line, timeout=600):
|
||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
|
||
s.settimeout(timeout)
|
||
s.connect(str(self.path))
|
||
s.sendall((line + "\n").encode())
|
||
data = b""
|
||
while not data.endswith(b"\n"):
|
||
chunk = s.recv(1 << 20)
|
||
if not chunk:
|
||
break
|
||
data += chunk
|
||
reply = json.loads(data)
|
||
if isinstance(reply, dict) and "error" in reply:
|
||
raise RuntimeError(f"{line}: {reply['error']}")
|
||
return reply
|
||
|
||
|
||
def poll(control, check, limit, interval=0.05):
|
||
"""Calls control('status') until check(status); returns (status, time)."""
|
||
deadline = time.time() + limit
|
||
while time.time() < deadline:
|
||
try:
|
||
status = control("status", timeout=10)
|
||
if check(status):
|
||
return status, time.time()
|
||
except (OSError, ValueError, RuntimeError):
|
||
pass
|
||
time.sleep(interval)
|
||
raise TimeoutError(f"not ready after {limit} s")
|
||
|
||
|
||
# Fewer section meshes than this in a second is the game remeshing blocks that change (random
|
||
# ticks), not loading.
|
||
QUIET_MESHES_PER_S = 100
|
||
|
||
|
||
def view_chunks(distance):
|
||
"""80% of the view circle (pi x RD^2): Horizon keeps 637 columns at RD 12 and 3725 at RD 32."""
|
||
return int(0.8 * math.pi * distance ** 2)
|
||
|
||
|
||
def wait_loaded(control, near, limit, min_chunks=0):
|
||
"""Waits until the player is spawned (near(position)), nothing is queued, the
|
||
resident and generated chunk counts have not changed for a second, as the
|
||
vanilla driver does, and the last sections are drawn; returns when the last
|
||
of these happened. Builds that report `status.streaming` are drawn once no
|
||
stage (generation, decoration, light, meshing) has held work for half a
|
||
second; blocks that change after loading (fluids settling, random ticks)
|
||
only remesh in short bursts. Older builds fall back to meshing going quiet
|
||
(fewer than QUIET_MESHES_PER_S section meshes in a second), which also
|
||
waits through that tail. (pending_chunks alone misses work the threads have
|
||
already taken, and sections waiting for their neighbours' light.)"""
|
||
deadline = time.time() + limit
|
||
last, changed, previous, meshing = None, time.time(), None, time.time()
|
||
# Right after a teleport (and at spawn) Horizon reports nothing queued and steady counts for over a second
|
||
# before it starts loading: the counts must change once, or stay full for 5 s, and the resident chunks reach
|
||
# min_chunks, before "steady" means loaded.
|
||
first, moved, began = None, False, time.time()
|
||
while time.time() < deadline:
|
||
try:
|
||
s = control("status", timeout=10)
|
||
except (OSError, ValueError, RuntimeError):
|
||
s = {}
|
||
player = s.get("player")
|
||
now = time.time()
|
||
if not player or not near(player["position"]) or s.get("pending_chunks") != 0 or s.get("screen") == "title":
|
||
last, changed, previous, meshing = None, now, None, now
|
||
else:
|
||
counts = (s["chunks"], s["generated_chunks"])
|
||
first = first or counts
|
||
moved = moved or counts != first or (s["chunks"] >= min_chunks > 0 and now - began > 5)
|
||
if not moved or s["chunks"] < min_chunks:
|
||
last, changed, previous, meshing = None, now, None, now
|
||
time.sleep(0.05)
|
||
continue
|
||
if counts != last:
|
||
last, changed = counts, now
|
||
stages = s.get("streaming")
|
||
sample = (now, s["meshed_sections"])
|
||
if stages is not None:
|
||
# Protos outside the view distance wait for neighbours that are never generated.
|
||
if any(v for k, v in stages.items() if k != "protos"):
|
||
meshing = now
|
||
quiet = 0.5
|
||
else:
|
||
if previous and sample[1] - previous[1] > QUIET_MESHES_PER_S * (now - previous[0]):
|
||
meshing = now
|
||
quiet = 1.0
|
||
previous = sample
|
||
if now - changed >= 1.0 and now - meshing >= quiet:
|
||
return max(changed, meshing)
|
||
time.sleep(0.05)
|
||
raise TimeoutError(f"not loaded after {limit} s")
|
||
|
||
|
||
def run_horizon(args, distance):
|
||
game_dir = WORK / "horizon"
|
||
shutil.rmtree(game_dir, ignore_errors=True)
|
||
options = write_options(game_dir, distance)
|
||
sock = Path("/tmp") / f"horizon-bench-{os.getpid()}.sock"
|
||
sock.unlink(missing_ok=True)
|
||
client = Path(args.build).resolve() / "horizon-client"
|
||
command = [str(client), "--resources", str(Path(args.resources).resolve()),
|
||
"--control-socket", str(sock), "--no-pause", "--weather", "clear", "--time", "6000"]
|
||
if args.offscreen:
|
||
command.append("--offscreen")
|
||
for carry in dev_mods(args):
|
||
command += ["--dev-mod", str(carry)]
|
||
env = dict(os.environ, HORIZON_NO_AUDIO="1")
|
||
if args.warm:
|
||
# A first start to the title screen builds what later starts reuse (the content-mod
|
||
# root, the block behaviour snapshot). It is not measured: players see it once.
|
||
with (WORK / f"horizon-rd{distance}-warm.log").open("w") as log:
|
||
proc = subprocess.Popen(command, cwd=game_dir, env=env, stdout=log, stderr=subprocess.STDOUT)
|
||
try:
|
||
poll(Control(sock), lambda s: s.get("screen") == "title", 600)
|
||
try:
|
||
Control(sock)("quit", timeout=60)
|
||
except (OSError, ValueError, RuntimeError):
|
||
pass
|
||
finally:
|
||
stop(proc)
|
||
sock.unlink(missing_ok=True)
|
||
log = (WORK / f"horizon-rd{distance}.log").open("w")
|
||
started = time.time()
|
||
proc = subprocess.Popen(command, cwd=game_dir, env=env, stdout=log, stderr=subprocess.STDOUT)
|
||
sampler = Sampler(proc.pid)
|
||
sampler.start()
|
||
control = Control(sock)
|
||
result = {"options_txt": options, "scenes": {}}
|
||
windows = []
|
||
try:
|
||
title, at = poll(control, lambda s: s.get("screen") == "title" and s.get("renderer"), 300)
|
||
result["renderer"] = title["renderer"]
|
||
result["startup"] = {"title_s": at - started}
|
||
control(f"open {WORLD} {SEED} creative")
|
||
result["startup"]["world_s"] = wait_loaded(control, lambda p: True, args.timeout, view_chunks(distance)) - started
|
||
for name, start, end, pitch in scene_list(args):
|
||
control("time 6000")
|
||
meshed = control("status")["meshed_sections"]
|
||
control("tp {} {} {}".format(*start))
|
||
tp = time.time()
|
||
# A zero-length flight holds the player at the start, flying.
|
||
control("fly {0} {1} {2} {0} {1} {2} 0.1 {3}".format(*start, pitch))
|
||
control(f"look {yaw(start, end):.3f} {pitch}")
|
||
near = lambda p: sum((a - b) ** 2 for a, b in zip(p, start)) < 4.0
|
||
scene = {"load_s": wait_loaded(control, near, args.timeout, view_chunks(distance)) - tp}
|
||
# Section meshes built while loading and while flying (Horizon only).
|
||
loaded_meshes = control("status")["meshed_sections"]
|
||
scene["meshes_load"] = loaded_meshes - meshed
|
||
time.sleep(args.settle)
|
||
loaded_meshes = control("status")["meshed_sections"]
|
||
begin = time.time()
|
||
summary = control("fly {} {} {} {} {} {} {} {} raw".format(*start, *end, args.seconds, pitch))
|
||
windows.append((begin, time.time()))
|
||
scene.update({k: summary.get(k) or [] for k in ("frame_ms", "cpu_ms", "gpu_ms")})
|
||
status = control("status")
|
||
scene["meshes_flight"] = status["meshed_sections"] - loaded_meshes
|
||
renderer = status.get("renderer") or {}
|
||
scene["gpu_mb"] = nvidia_memory_mb(proc.pid) or (renderer.get("gpu_allocated_bytes") or 0) / 2**20 or None
|
||
result["scenes"][name] = scene
|
||
print_scene(args.game, distance, name, scene)
|
||
finally:
|
||
try:
|
||
control("quit", timeout=60)
|
||
except (OSError, ValueError, RuntimeError):
|
||
pass
|
||
stop(proc)
|
||
sampler.stop.set()
|
||
sock.unlink(missing_ok=True)
|
||
result["memory"] = sampler.summary(windows)
|
||
return result
|
||
|
||
|
||
def dev_mods(args):
|
||
"""The carried-over mods of a horizon-port-pack work folder (docs/modpacks.md), loaded as dev mods."""
|
||
if not args.port_work:
|
||
return []
|
||
return sorted(p for p in (Path(args.port_work) / "mods").glob("*/carry") if (p / "horizon-mod.toml").exists())
|
||
|
||
|
||
# Vanilla -------------------------------------------------------------------
|
||
|
||
def vanilla_manifest():
|
||
if not (VANILLA / "manifest.json").exists():
|
||
sys.exit(f"{VANILLA} missing: prepare Horizon first (tools/prepare_native_client.py)")
|
||
return json.loads((VANILLA / "manifest.json").read_text())
|
||
|
||
|
||
def vanilla_classpath(manifest):
|
||
"""The official launcher's client libraries for this OS, from your own prepared copy."""
|
||
jars = [VANILLA / "jars/client.jar"]
|
||
for artifact in lab.client_library_artifacts(manifest):
|
||
jars.append(VANILLA / "libraries/client" / artifact["path"])
|
||
return jars
|
||
|
||
|
||
def extract_natives(jars, target):
|
||
target.mkdir(parents=True, exist_ok=True)
|
||
for jar in jars:
|
||
if "natives-" not in jar.name:
|
||
continue
|
||
with zipfile.ZipFile(jar) as archive:
|
||
for name in archive.namelist():
|
||
if name.endswith((".so", ".dylib", ".jnilib", ".dll")):
|
||
out = target / Path(name).name
|
||
if not out.exists():
|
||
out.write_bytes(archive.read(name))
|
||
|
||
|
||
def build_bench_mod(jars):
|
||
"""Compiles the dev-only driver mod against your own client.jar."""
|
||
source = ROOT / "mods/horizon/src/bench"
|
||
build = WORK / "mod"
|
||
classes = build / "classes"
|
||
shutil.rmtree(classes, ignore_errors=True)
|
||
classes.mkdir(parents=True)
|
||
sources = sorted(map(str, (source / "java").rglob("*.java")))
|
||
subprocess.run(["javac", "--release", "25", "-proc:none", "-encoding", "UTF-8", "-d", str(classes),
|
||
"-classpath", os.pathsep.join(map(str, jars)), *sources], check=True)
|
||
output = build / "horizon-bench.jar"
|
||
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||
for path in sorted(classes.rglob("*.class")):
|
||
archive.write(path, path.relative_to(classes).as_posix())
|
||
for path in sorted((source / "resources").rglob("*")):
|
||
if path.is_file():
|
||
archive.write(path, path.relative_to(source / "resources").as_posix())
|
||
return output
|
||
|
||
|
||
def perf_mod_jars():
|
||
"""Sodium, Lithium, FerriteCore and ImmediatelyFast for 26.3 (downloaded from
|
||
Modrinth, checked against packs/mods.lock.json, never committed), plus
|
||
Fabric API when one of them needs it."""
|
||
lock = json.loads((ROOT / "packs/mods.lock.json").read_text())
|
||
jars, needs_api = [], False
|
||
for name in PERF_MODS:
|
||
project = lock["projects"][name]
|
||
jar = runtime.fetch(project["url"], ROOT / ".local/pack-cache" / project["filename"],
|
||
project["hashes"]["sha512"], "sha512")
|
||
with zipfile.ZipFile(jar) as archive:
|
||
depends = json.loads(archive.read("fabric.mod.json")).get("depends", {})
|
||
needs_api |= any(d == "fabric-api" or d.startswith("fabric-") and d != "fabricloader" for d in depends)
|
||
jars.append(jar)
|
||
if needs_api:
|
||
api = lock["projects"]["fabric-api"]
|
||
jars.append(runtime.fetch(api["url"], ROOT / ".local/pack-cache" / api["filename"],
|
||
api["hashes"]["sha512"], "sha512"))
|
||
return jars
|
||
|
||
|
||
def run_vanilla(args, distance):
|
||
manifest = vanilla_manifest()
|
||
game_jars = vanilla_classpath(manifest)
|
||
fabric = runtime.fabric_libraries()
|
||
mod = build_bench_mod(game_jars + fabric)
|
||
game_dir = WORK / args.game
|
||
shutil.rmtree(game_dir, ignore_errors=True)
|
||
options = write_options(game_dir, distance)
|
||
mods = game_dir / "mods"
|
||
mods.mkdir()
|
||
shutil.copy(mod, mods)
|
||
loaded_mods = ["horizon-bench"]
|
||
if args.game == "vanilla-mods":
|
||
for jar in perf_mod_jars():
|
||
shutil.copy(jar, mods)
|
||
loaded_mods.append(jar.name)
|
||
natives = WORK / "natives"
|
||
extract_natives(game_jars, natives)
|
||
script = WORK / "script.json"
|
||
script.write_text(json.dumps({
|
||
"world": WORLD, "seed": SEED, "settle": args.settle, "seconds": args.seconds, "timeout": args.timeout,
|
||
"scenes": [{"name": n, "from": a, "to": b, "pitch": p} for n, a, b, p in scene_list(args)],
|
||
}))
|
||
events = WORK / f"{args.game}-rd{distance}.events.jsonl"
|
||
events.unlink(missing_ok=True)
|
||
assets = WORK / "assets"
|
||
(assets / "indexes").mkdir(parents=True, exist_ok=True)
|
||
java = Path(os.environ["JAVA_HOME"]) / "bin/java" if os.environ.get("JAVA_HOME") else "java"
|
||
command = [str(java), f"-Xmx{args.heap}", *LAUNCHER_JVM,
|
||
*runtime.conditional_client_jvm_options(manifest),
|
||
"--enable-native-access=ALL-UNNAMED",
|
||
f"-Djava.library.path={natives}", f"-Dorg.lwjgl.system.SharedLibraryExtractPath={natives}",
|
||
f"-Dhorizon.bench.script={script}", f"-Dhorizon.bench.out={events}",
|
||
"-Dfabric.log.disableAnsi=true",
|
||
"-cp", os.pathsep.join(map(str, game_jars + fabric)),
|
||
"net.fabricmc.loader.impl.launch.knot.KnotClient",
|
||
"--username", "Bench", "--uuid", "00000000000000000000000000000000", "--accessToken", "0",
|
||
"--offlineDeveloperMode", "--version", lab.VERSION, "--gameDir", str(game_dir),
|
||
"--assetsDir", str(assets), "--assetIndex", manifest["assetIndex"]["id"],
|
||
"--width", str(args.width), "--height", str(args.height)]
|
||
env = dict(os.environ, ALSOFT_DRIVERS="null")
|
||
if sys.platform.startswith("linux"):
|
||
libs = [p for p in (os.environ.get("NIX_LD_LIBRARY_PATH", ""), "/run/opengl-driver/lib") if p]
|
||
env["LD_LIBRARY_PATH"] = ":".join(libs + [env.get("LD_LIBRARY_PATH", "")]).strip(":")
|
||
log = (WORK / f"{args.game}-rd{distance}.log").open("w")
|
||
return driven(args, distance, command, game_dir, env, log, events, options, loaded_mods)
|
||
|
||
|
||
def driven(args, distance, command, game_dir, env, log, events, options, loaded_mods):
|
||
"""Runs a Java client whose driver mod writes `events`, and reads its results."""
|
||
started = time.time()
|
||
proc = subprocess.Popen(command, cwd=game_dir, env=env, stdout=log, stderr=subprocess.STDOUT)
|
||
sampler = Sampler(proc.pid)
|
||
sampler.start()
|
||
nvidia = None
|
||
try:
|
||
deadline = started + args.timeout * (len(scene_list(args)) + 2)
|
||
while proc.poll() is None and time.time() < deadline:
|
||
time.sleep(0.5)
|
||
if nvidia is None:
|
||
nvidia = nvidia_memory_mb(proc.pid)
|
||
finally:
|
||
stop(proc)
|
||
sampler.stop.set()
|
||
lines = [json.loads(l) for l in events.read_text().splitlines()] if events.exists() else []
|
||
by = lambda name: [e for e in lines if e["event"] == name]
|
||
if not by("done") or by("timeout"):
|
||
raise RuntimeError(f"{args.game} run failed: {lines[-1:] if lines else 'no events'}; see {log.name}")
|
||
title, world = by("title")[0], by("world")[0]
|
||
result = {
|
||
"options_txt": options, "mods": loaded_mods, "heap": args.heap,
|
||
"renderer": {k: title[k] for k in ("gpu", "vendor", "backend", "driver", "size",
|
||
"render_distance", "simulation_distance")},
|
||
"startup": {"title_s": title["epoch_ms"] / 1000 - started,
|
||
"world_s": world["ready_epoch_ms"] / 1000 - started},
|
||
"scenes": {},
|
||
}
|
||
windows = []
|
||
loaded = {e["scene"]: e for e in by("loaded")}
|
||
flights = {e["scene"]: e for e in by("fly")}
|
||
for flown in by("flown"):
|
||
name = flown["scene"]
|
||
windows.append((flights[name]["epoch_ms"] / 1000, flown["epoch_ms"] / 1000))
|
||
scene = {"load_s": loaded[name]["load_ms"] / 1000, "frame_ms": flown["frame_ms"],
|
||
"chunks": flown["chunks"], "gpu_mb": nvidia}
|
||
result["scenes"][name] = scene
|
||
print_scene(args.game, distance, name, scene)
|
||
result["memory"] = sampler.summary(windows)
|
||
return result
|
||
|
||
|
||
# NeoForge ------------------------------------------------------------------
|
||
|
||
def run_neoforge(args, distance):
|
||
"""A NeoForge 1.21.1 client (dev-only, tools/neoforge_oracle.py) with the driver mod and --mod-jars:
|
||
the mods a ported pack replaces, on the loader the pack runs on."""
|
||
import neoforge_oracle
|
||
home = neoforge_oracle.client()[0]
|
||
mod = neoforge_oracle.build_bench(home)
|
||
game_dir = WORK / args.game
|
||
shutil.rmtree(game_dir, ignore_errors=True)
|
||
options = write_options(game_dir, distance, version=NEOFORGE_DATA_VERSION)
|
||
mods = game_dir / "mods"
|
||
mods.mkdir()
|
||
shutil.copy(mod, mods)
|
||
loaded_mods = ["horizon-bench"]
|
||
for folder in filter(None, (args.mod_jars, args.deps)):
|
||
for jar in sorted(Path(folder).glob("*.jar")):
|
||
shutil.copy(jar, mods)
|
||
loaded_mods.append(jar.name)
|
||
natives = WORK / "natives-neoforge"
|
||
script = WORK / "script.json"
|
||
script.write_text(json.dumps({
|
||
"world": WORLD, "seed": SEED, "settle": args.settle, "seconds": args.seconds, "timeout": args.timeout,
|
||
"scenes": [{"name": n, "from": a, "to": b, "pitch": p} for n, a, b, p in scene_list(args)],
|
||
}))
|
||
events = WORK / f"{args.game}-rd{distance}.events.jsonl"
|
||
events.unlink(missing_ok=True)
|
||
assets = WORK / "assets"
|
||
(assets / "indexes").mkdir(parents=True, exist_ok=True)
|
||
java = Path(os.environ["JAVA_HOME"]) / "bin/java" if os.environ.get("JAVA_HOME") else "java"
|
||
_, command = neoforge_oracle.client_command(
|
||
java, args.heap, game_dir, natives, assets,
|
||
jvm=[*LAUNCHER_JVM, f"-Dhorizon.bench.script={script}", f"-Dhorizon.bench.out={events}"],
|
||
game=["--width", str(args.width), "--height", str(args.height)])
|
||
natives.mkdir(parents=True, exist_ok=True)
|
||
extract_natives(neoforge_oracle.client()[3], natives)
|
||
env = dict(os.environ, ALSOFT_DRIVERS="null")
|
||
for attempt in (1, 2):
|
||
log = (WORK / f"{args.game}-rd{distance}.log").open("w")
|
||
try:
|
||
return driven(args, distance, command, game_dir, env, log, events, options, loaded_mods)
|
||
except RuntimeError:
|
||
# FML sometimes reads config/fml.toml while creating it on a first start, and crashes before
|
||
# the title screen; a crash once the driver started is a real failure.
|
||
if attempt == 2 or events.exists():
|
||
raise
|
||
shutil.rmtree(game_dir / "crash-reports", ignore_errors=True)
|
||
|
||
|
||
def stop(proc):
|
||
try:
|
||
proc.wait(timeout=60)
|
||
except subprocess.TimeoutExpired:
|
||
proc.terminate()
|
||
try:
|
||
proc.wait(timeout=20)
|
||
except subprocess.TimeoutExpired:
|
||
proc.kill()
|
||
|
||
|
||
def print_scene(game, distance, name, scene):
|
||
s = frames(name, 0, scene["frame_ms"]) or {}
|
||
print(f" {game:<12} rd {distance:>2} {name:<9} load {scene['load_s']:6.1f} s {s.get('avg_fps', 0):7.1f} fps"
|
||
f" 1% {s.get('low_1pct_fps', 0):6.1f} 0.1% {s.get('low_01pct_fps', 0):6.1f}"
|
||
f" p50 {s.get('p50_ms', 0):6.2f} p99 {s.get('p99_ms', 0):7.2f} ms", flush=True)
|
||
|
||
|
||
# Report --------------------------------------------------------------------
|
||
|
||
def hardware():
|
||
"""horizon_bench::Hardware::detect: the machine, without its name."""
|
||
info = {"os": {"darwin": "macos", "win32": "windows"}.get(sys.platform, sys.platform),
|
||
"arch": {"arm64": "aarch64", "AMD64": "x86_64"}.get(platform.machine(), platform.machine()),
|
||
"cpu_threads": os.cpu_count()}
|
||
if sys.platform == "darwin":
|
||
q = lambda *cmd: subprocess.run(cmd, capture_output=True, text=True).stdout.strip()
|
||
info.update(os_version=q("sw_vers", "-productVersion"), cpu=q("sysctl", "-n", "machdep.cpu.brand_string"),
|
||
ram_mb=int(q("sysctl", "-n", "hw.memsize") or 0) >> 20)
|
||
else:
|
||
release = dict(l.split("=", 1) for l in Path("/etc/os-release").read_text().splitlines() if "=" in l)
|
||
info["os_version"] = f"{release.get('PRETTY_NAME', '').strip(chr(34))} (kernel {platform.release()})"
|
||
for line in Path("/proc/cpuinfo").read_text().splitlines():
|
||
if line.startswith("model name"):
|
||
info["cpu"] = line.split(":", 1)[1].strip()
|
||
break
|
||
info["ram_mb"] = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") >> 20
|
||
return info
|
||
|
||
|
||
def report(args, distance, runs, app):
|
||
"""One v1 report for a game at a render distance, over its repeats."""
|
||
first = runs[0]
|
||
renderer = first.get("renderer") or {}
|
||
hw = hardware()
|
||
if renderer.get("gpu"):
|
||
hw["gpu"] = {"name": renderer["gpu"], "vendor": str(renderer.get("vendor") or ""),
|
||
"driver": str(renderer.get("driver") or ""), "backend": str(renderer.get("backend") or "").lower(),
|
||
"kind": renderer.get("device_type") or ""}
|
||
size = renderer.get("size")
|
||
conditions = {"scenario": "perf_bench:" + ",".join(first["scenes"]), "seed": str(SEED),
|
||
"render_distance": distance, "simulation_distance": 12, "vsync": False,
|
||
"offscreen": bool(args.offscreen), "repeats": len(runs), "warmup_s": args.settle,
|
||
"load_avg_1m": rnd(first["load_before"][0])}
|
||
if size:
|
||
conditions["window"] = list(size)
|
||
metrics, entries = {}, []
|
||
|
||
def metric(name, values):
|
||
if (m := summary(values)) is not None:
|
||
metrics[name] = m
|
||
|
||
metric("startup.title_ms", [r["startup"]["title_s"] * 1000 for r in runs])
|
||
metric("startup.world_ms", [r["startup"]["world_s"] * 1000 for r in runs])
|
||
for name in first["scenes"]:
|
||
scenes = [r["scenes"][name] for r in runs if name in r["scenes"]]
|
||
flights = [frames(name, i, s["frame_ms"], s.get("cpu_ms", ()), s.get("gpu_ms", ())) for i, s in enumerate(scenes)]
|
||
entries += [f for f in flights if f]
|
||
metric(f"chunks.{name}.load_ms", [s["load_s"] * 1000 for s in scenes])
|
||
metric(f"chunks.{name}.meshed_count", [s.get("meshes_load") for s in scenes])
|
||
for key in ("avg_fps", "low_1pct_fps", "low_01pct_fps"):
|
||
metric(f"fps.{name}.{key}", [f[key] for f in flights if f])
|
||
metric(f"memory.{name}.gpu_mb", [s.get("gpu_mb") for s in scenes])
|
||
metric("memory.rss_peak_mb", [r["memory"].get("rss_peak_mb") for r in runs])
|
||
metric("memory.rss_steady_mb", [r["memory"].get("rss_steady_mb") for r in runs])
|
||
metric("memory.footprint_peak_mb", [r["memory"].get("footprint_peak_mb") for r in runs])
|
||
metric("memory.footprint_steady_mb", [r["memory"].get("footprint_steady_mb") for r in runs])
|
||
return {"format": REPORT_FORMAT, "version": REPORT_VERSION, "kind": "game",
|
||
"created": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||
"app": app, "hardware": hw, "conditions": conditions, "metrics": metrics, "frames": entries,
|
||
"label": args.label}
|
||
|
||
|
||
def table(labels):
|
||
"""Markdown tables comparing saved runs (benchmarks/perf/<label>-rd<N>.json): means and 95% CIs."""
|
||
runs = {}
|
||
for label in labels:
|
||
for path in sorted((ROOT / "benchmarks/perf").glob(f"{label}-rd*.json"), key=lambda p: int(p.stem.rsplit("rd", 1)[1])):
|
||
r = json.loads(path.read_text())
|
||
runs.setdefault(label, {})[r["conditions"]["render_distance"]] = r
|
||
distances = sorted({d for by in runs.values() for d in by})
|
||
cell = lambda m, digits=0: "–" if not m else f"{m['mean']:.{digits}f}" + (f" ±{m['ci95']:.{digits}f}" if m["ci95"] else "")
|
||
print("| RD | scene | " + " | ".join(f"{l}: avg / 1% / 0.1% low FPS" for l in labels) + " |")
|
||
print("| --- | --- |" + " --- |" * len(labels))
|
||
for d in distances:
|
||
scenes = sorted({k.split(".")[1] for by in runs.values() if d in by for k in by[d]["metrics"] if k.startswith("fps.")})
|
||
for scene in scenes:
|
||
cells = []
|
||
for l in labels:
|
||
m = runs.get(l, {}).get(d, {}).get("metrics", {})
|
||
cells.append(" / ".join(cell(m.get(f"fps.{scene}.{k}")) for k in ("avg_fps", "low_1pct_fps", "low_01pct_fps")))
|
||
print(f"| {d} | {scene} | " + " | ".join(cells) + " |")
|
||
print()
|
||
print("| RD | " + " | ".join(f"{l}: title / world s, mean scene load s, RSS steady / peak MB, footprint steady / peak MB, load avg" for l in labels) + " |")
|
||
print("| --- |" + " --- |" * len(labels))
|
||
for d in distances:
|
||
cells = []
|
||
for l in labels:
|
||
r = runs.get(l, {}).get(d)
|
||
if not r:
|
||
cells.append("–")
|
||
continue
|
||
m = r["metrics"]
|
||
loads = [v["mean"] for k, v in m.items() if k.endswith(".load_ms")]
|
||
sec = lambda k: f"{m[k]['mean'] / 1000:.1f}" if k in m else "–"
|
||
mb = lambda k: f"{m[k]['mean']:.0f}" if k in m else "–"
|
||
cells.append(f"{sec('startup.title_ms')} / {sec('startup.world_ms')}, "
|
||
f"{statistics.mean(loads) / 1000 if loads else 0:.1f}, "
|
||
f"{mb('memory.rss_steady_mb')} / {mb('memory.rss_peak_mb')}, "
|
||
f"{mb('memory.footprint_steady_mb')} / {mb('memory.footprint_peak_mb')}, "
|
||
f"{r['conditions'].get('load_avg_1m')}")
|
||
print(f"| {d} | " + " | ".join(cells) + " |")
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
p.add_argument("--game", choices=["horizon", "vanilla", "vanilla-mods", "neoforge"], default="horizon")
|
||
p.add_argument("--label", help="result name (default <game>-<host>)")
|
||
p.add_argument("--distances", default="8,16,32")
|
||
p.add_argument("--scenes", help="comma-separated subset of " + ",".join(s[0] for s in SCENES))
|
||
p.add_argument("--seconds", type=float, default=12.0, help="flight duration per scene")
|
||
p.add_argument("--settle", type=float, default=2.0, help="seconds held at the start before each flight")
|
||
p.add_argument("--timeout", type=float, default=300.0, help="seconds allowed for each load")
|
||
p.add_argument("--width", type=int, default=1280, help="vanilla window width (Horizon's is 1280x720)")
|
||
p.add_argument("--height", type=int, default=720)
|
||
p.add_argument("--heap", default="2G", help="vanilla -Xmx (the official launcher's default is 2G)")
|
||
p.add_argument("--offscreen", action="store_true",
|
||
help="Horizon renders into a texture and does not wait for the window (locked macOS sessions)")
|
||
p.add_argument("--resources", default=str(lab.RESOURCES))
|
||
p.add_argument("--build", default=str(ROOT / "target/release"))
|
||
p.add_argument("--repeats", type=int, default=3, help="games per render distance; the report gives means and 95%% CIs")
|
||
p.add_argument("--port-work", help="horizon: also load the mods carried over in this horizon-port-pack work folder")
|
||
p.add_argument("--warm", action="store_true",
|
||
help="horizon: measure a later start, after one start to the title screen built the caches")
|
||
p.add_argument("--mod-jars", help="neoforge: the mod jars to load (the user's own; never committed)")
|
||
p.add_argument("--deps", help="neoforge: their required libraries")
|
||
p.add_argument("--table", nargs="+", metavar="LABEL", help="print markdown tables of saved runs, and exit")
|
||
args = p.parse_args()
|
||
if args.table:
|
||
table(args.table)
|
||
return
|
||
args.label = args.label or f"{args.game}-{platform.node().split('.')[0]}"
|
||
WORK.mkdir(parents=True, exist_ok=True)
|
||
run = {"horizon": run_horizon, "neoforge": run_neoforge}.get(args.game, run_vanilla)
|
||
for d in [int(x) for x in args.distances.split(",")]:
|
||
runs = []
|
||
for _ in range(args.repeats):
|
||
load_before = os.getloadavg()
|
||
result = run(args, d)
|
||
result["load_before"], result["load_after"] = load_before, os.getloadavg()
|
||
runs.append(result)
|
||
out = ROOT / "benchmarks/perf" / f"{args.label}-rd{d}.json"
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
out.write_text(json.dumps(report(args, d, runs, app(args)), indent=1) + "\n")
|
||
print(out)
|
||
|
||
|
||
def app(args):
|
||
if args.game == "horizon":
|
||
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT,
|
||
capture_output=True, text=True).stdout.strip() or None
|
||
manifest = (ROOT / "crates/horizon-client/Cargo.toml").read_text()
|
||
version = next(l.split('"')[1] for l in manifest.splitlines() if l.startswith("version"))
|
||
mods = [{"name": p.parent.name, "version": "carried"} for p in dev_mods(args)]
|
||
return {"name": "horizon-client", "version": version, "commit": commit, **({"mods": mods} if mods else {})}
|
||
if args.game == "neoforge":
|
||
import neoforge_oracle
|
||
jars = [j.name for f in filter(None, (args.mod_jars, args.deps)) for j in sorted(Path(f).glob("*.jar"))]
|
||
return {"name": "minecraft", "version": "1.21.1",
|
||
"mods": [{"name": "neoforge", "version": neoforge_oracle.NEOFORGE}] + [{"name": j} for j in jars]}
|
||
lock = json.loads((ROOT / "packs/mods.lock.json").read_text())
|
||
mods = [{"name": "fabric-loader", "version": lock["fabric_loader"]}]
|
||
if args.game == "vanilla-mods":
|
||
mods += [{"name": n, "version": lock["projects"][n]["version"]} for n in PERF_MODS]
|
||
return {"name": "minecraft", "version": lab.VERSION, "mods": mods}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|