Merge pull request #264 from ludovic111/feat/perf-bench-report-v1

perf_bench.py: repeats with 95% CIs, the shared report format, loads end when meshing is done
This commit is contained in:
Ludovic Marie
2026-10-15 07:20:42 +02:00
committed by GitHub
3 changed files with 275 additions and 112 deletions
+3 -3
View File
@@ -26,11 +26,11 @@ One game is started per render distance (8, 16 and 32 chunks by default), each f
## Known caveats
- **Fabric Loader is present in the vanilla runs**, to load the driver mod. The only patch it applies is the driver's frame hook, so its cost after startup is negligible, but its mod discovery and class transformation add to vanilla's startup time (the modded runs use the same loader).
- **Readiness is detected differently.** Horizon knows exactly how many chunks are pending; for vanilla the driver waits until the loaded chunk count is unchanged for a second with nothing left to compile, and takes the time of the last change. Both stop the clock at the last chunk arriving, within a frame or two.
- **Readiness is detected differently.** For Horizon the driver waits until nothing is queued, the chunk counts are unchanged for a second and meshing has gone quiet (fewer than 100 section meshes a second: blocks that change keep being remeshed), and takes the time of the last of these; sections wait for their neighbours' light before they are meshed, so the chunk counts alone stop the clock before the last sections are drawn. For vanilla it waits until the loaded chunk count is unchanged for a second with nothing left to compile, and takes the time of the last change. Both stop the clock within a frame or two of the last section drawn.
- **Locked or headless screens.** A window nobody can see is not composited. On a Mac whose session is locked, Horizon draws only with `--offscreen` (it renders into a texture and does not wait for the window), and vanilla keeps rendering into its window without it being shown. Such runs measure rendering without presentation and are marked `offscreen`; numbers for players come from runs with the window on a real screen.
- **Different graphics APIs.** Horizon renders with wgpu (Vulkan on Linux and Windows, Metal on macOS); vanilla 26.3 uses OpenGL (or its Vulkan backend when chosen). The backend each game used is stored with the results. This is part of what is being compared.
- **World generation is the same algorithm but not the same threads.** Both generate the same terrain (the parity harnesses check it), with their own threading; the work per chunk differs, which is also part of what is being compared.
- **Variance.** Flights are short (12 seconds); run each configuration more than once and compare medians before drawing conclusions from small differences. Lows are the noisiest figures.
- **Variance.** Flights are short (12 seconds), so each configuration runs `--repeats` times (3 by default), each from a fresh world, and every figure is reported as the mean over the repeats with its standard deviation and 95% confidence interval (Student's t). Differences inside the intervals are noise. Lows are the noisiest figures.
## Reproduce it
@@ -43,7 +43,7 @@ python3 tools/perf_bench.py --game vanilla-mods --label mods-mine
python3 tools/perf_bench.py --table horizon-mine vanilla-mine mods-mine
```
Options: `--distances 8,16`, `--scenes village,ocean`, `--seconds 20`, `--heap 4G` (vanilla), `--offscreen` (Horizon, for locked or headless sessions). Results go to `benchmarks/perf/<label>.json`; each game's log, and the vanilla driver's event log, stay in `.local/perf-bench/`. Keep the machine otherwise idle, plugged in, and the window on a screen.
Options: `--distances 8,16`, `--scenes village,ocean`, `--seconds 20`, `--repeats 5`, `--heap 4G` (vanilla), `--offscreen` (Horizon, for locked or headless sessions). Results go to `benchmarks/perf/<label>-rd<N>.json`, one [benchmark report](benchmark-report.md) per render distance (the same format the game's `--benchmark` and the launcher write), with the load average before each game; each game's log, and the vanilla driver's event log, stay in `.local/perf-bench/`. Keep the machine otherwise idle, plugged in, and the window on a screen.
## Results
+68 -15
View File
@@ -1,30 +1,42 @@
import json
import unittest
from tools.perf_bench import OPTIONS, stats
from tools.perf_bench import HISTOGRAM_EDGES_MS, OPTIONS, frames, rnd, summary
class PerfBenchStatsTests(unittest.TestCase):
def test_lows_average_the_slowest_frames(self):
times = [10.0] * 990 + [100.0] * 9 + [1000.0]
s = stats(times)
s = frames("village", 0, times)
self.assertEqual(s["frames"], 1000)
self.assertAlmostEqual(s["avg_fps"], 1000 * 1000 / (9900 + 900 + 1000))
self.assertEqual(s["avg_fps"], rnd(1000 * 1000 / (9900 + 900 + 1000)))
# Slowest 1%: 9 frames of 100 ms and one of 1000 ms.
self.assertAlmostEqual(s["low_1pct_fps"], 1000 * 10 / 1900)
self.assertEqual(s["low_1pct_fps"], rnd(1000 * 10 / 1900))
# Slowest 0.1%: the single 1000 ms frame.
self.assertAlmostEqual(s["low_01pct_fps"], 1.0)
self.assertEqual(s["p50_ms"], 10.0)
self.assertEqual(s["p99_ms"], 10.0)
self.assertEqual(s["p999_ms"], 100.0)
self.assertEqual(s["max_ms"], 1000.0)
self.assertAlmostEqual(sum(s["histogram"].values()), 1.0)
self.assertAlmostEqual(s["histogram"][">=100ms"], 0.01)
self.assertEqual(s["low_01pct_fps"], 1.0)
self.assertEqual((s["p50_ms"], s["p99_ms"], s["p999_ms"], s["max_ms"]), (10.0, 10.0, 100.0, 1000.0))
self.assertEqual(sum(s["histogram"]), 1000)
self.assertEqual(len(s["histogram"]), len(HISTOGRAM_EDGES_MS))
self.assertEqual(s["histogram"][HISTOGRAM_EDGES_MS.index(10)], 990)
self.assertEqual(s["histogram"][HISTOGRAM_EDGES_MS.index(100)], 9)
self.assertEqual(s["histogram"][-1], 1, "1000 ms and over")
def test_same_figures_as_horizon_bench(self):
# crates/horizon-bench's frame_figures_and_histogram test, value for value.
s = frames("flight", 0, [2.0] * 998 + [20.0, 1500.0])
self.assertEqual((s["seconds"], s["low_1pct_fps"], s["low_01pct_fps"]), (3.516, 6.51, 0.667))
self.assertEqual((s["histogram"][2], s["histogram"][13], s["histogram"][23]), (998, 1, 1))
self.assertNotIn("cpu_p50_ms", s)
r = summary([2.0, 4.0, 6.0])
self.assertEqual(r, {"n": 3, "mean": 4.0, "stdev": 2.0, "min": 2.0, "max": 6.0, "ci95": 4.969})
self.assertEqual(summary([5.0])["ci95"], None)
self.assertIsNone(summary([None]))
def test_a_short_flight_still_has_lows(self):
s = stats([5.0, 20.0])
self.assertAlmostEqual(s["low_1pct_fps"], 50.0)
self.assertAlmostEqual(s["low_01pct_fps"], 50.0)
self.assertEqual(stats([]), {"frames": 0})
s = frames("x", 0, [5.0, 20.0])
self.assertEqual((s["low_1pct_fps"], s["low_01pct_fps"]), (50.0, 50.0))
self.assertIsNone(frames("x", 0, []))
json.dumps(s)
def test_one_options_file_sets_the_render_distance(self):
self.assertEqual(OPTIONS["renderDistance"].format(distance=16), "16")
@@ -33,3 +45,44 @@ class PerfBenchStatsTests(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
class WaitLoadedTests(unittest.TestCase):
"""wait_loaded on a scripted game with a fake clock: one status per 50 ms poll."""
def run_script(self, statuses):
import tools.perf_bench as pb
clock = [0.0]
it = iter(statuses)
def control(_command, timeout=None):
return next(it)
real_time, real_sleep = pb.time.time, pb.time.sleep
pb.time.time = lambda: clock[0]
pb.time.sleep = lambda s: clock.__setitem__(0, clock[0] + s)
try:
return pb.wait_loaded(control, lambda p: True, 60)
finally:
pb.time.time, pb.time.sleep = real_time, real_sleep
@staticmethod
def status(chunks, meshed, stages=None):
s = {"player": {"position": [0, 0, 0]}, "pending_chunks": 0, "screen": "game",
"chunks": chunks, "generated_chunks": chunks, "meshed_sections": meshed}
if stages is not None:
s["streaming"] = {"queued": 0, "generating": 0, "protos": 296, "decorating": 0,
"finishing": 0, "lighting": 0, "meshing": stages}
return s
def test_stages_end_the_load_before_the_remesh_tail(self):
# 20 polls (1 s) of meshing work, then idle stages while blocks keep remeshing
# at 400 meshes/s, with a short burst of work every 2 s.
loading = [self.status(900, 1000 * i, stages=500) for i in range(20)]
tail = [self.status(901, 20000 + 20 * i, stages=40 if i % 40 == 0 else 0) for i in range(400)]
self.assertAlmostEqual(self.run_script(loading + tail), 1.0, places=6)
def test_old_builds_wait_for_meshing_to_go_quiet(self):
busy = [self.status(901, 1000 * i) for i in range(20)]
quiet = [self.status(901, 20000 + i) for i in range(40)]
self.assertAlmostEqual(self.run_script(busy + quiet), 1.0, places=6)
+204 -94
View File
@@ -21,10 +21,12 @@ Games:
vanilla-mods the same + Sodium, Lithium, FerriteCore and ImmediatelyFast
(pinned in packs/mods.lock.json)
Results: benchmarks/perf/<label>.json. Games run with no audio, from a
scratch directory under .local/perf-bench.
Results: benchmarks/perf/<label>-rd<N>.json, one benchmark report per render
distance (docs/benchmark-report.md), over --repeats games. Games run with no
audio, from a scratch directory under .local/perf-bench.
"""
import argparse
import bisect
import datetime
import json
import math
@@ -105,43 +107,68 @@ LAUNCHER_JVM = ["-XX:+UnlockExperimentalVMOptions", "-XX:+UseG1GC", "-XX:G1NewSi
PERF_MODS = ["sodium", "lithium", "ferrite-core", "immediatelyfast"]
def stats(frame_ms):
"""Every number of a flight, from its frame times in milliseconds.
# The benchmark report format, version 1 (docs/benchmark-report.md). crates/horizon-bench is the
# reference: these functions compute the same figures, with the same rounding, for every game.
REPORT_FORMAT, REPORT_VERSION = "horizon-benchmark-report", 1
HISTOGRAM_EDGES_MS = [0, 1, 2, 3, 4.17, 5, 6, 7, 8.33, 10, 12, 14, 16.67, 20, 25, 33.33, 40,
50, 66.67, 100, 150, 250, 500, 1000]
# Two-sided 95% critical values of Student's t for 1..30 degrees of freedom.
T95 = [12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160,
2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056,
2.052, 2.048, 2.045, 2.042]
def rnd(v):
"""Three decimals, halves away from zero (Rust's f64::round)."""
return math.copysign(math.floor(abs(v) * 1000 + 0.5), v) / 1000
def frames(phase, repeat, frame_ms, cpu_ms=(), gpu_ms=()):
"""Every figure of a flight, from its frame times in milliseconds (horizon_bench::Frames::of).
The x% low is the average frame time of the slowest x% of frames, as
frames per second (so a single 2-second freeze shows in the 0.1% low of a
10,000-frame flight)."""
times = sorted(frame_ms)
times = sorted(t for t in frame_ms if math.isfinite(t))
n = len(times)
if n == 0:
return {"frames": 0}
return None
total = sum(times)
pct = lambda p: times[min(n, max(1, math.ceil(n * p))) - 1]
pct = lambda values, p: values[min(len(values), max(1, math.ceil(len(values) * p))) - 1]
def low(share):
slowest = times[n - max(1, math.ceil(n * share)):]
return 1000.0 * len(slowest) / sum(slowest)
buckets = [4.17, 8.33, 16.67, 33.33, 50.0, 100.0]
hist = {}
lo = 0.0
for hi in buckets + [float("inf")]:
key = f"<{hi:g}ms" if hi != float("inf") else f">={buckets[-1]:g}ms"
hist[key] = sum(1 for t in times if lo <= t < hi) / n
lo = hi
return {
"frames": n,
"seconds": total / 1000.0,
"avg_fps": n * 1000.0 / total,
"low_1pct_fps": low(0.01),
"low_01pct_fps": low(0.001),
"p50_ms": pct(0.5),
"p95_ms": pct(0.95),
"p99_ms": pct(0.99),
"p999_ms": pct(0.999),
"max_ms": times[-1],
"histogram": hist,
histogram = [0] * len(HISTOGRAM_EDGES_MS)
for t in times:
histogram[max(1, bisect.bisect_right(HISTOGRAM_EDGES_MS, t)) - 1] += 1
entry = {
"phase": phase, "repeat": repeat, "frames": n, "seconds": rnd(total / 1000.0),
"avg_fps": rnd(n * 1000.0 / total), "low_1pct_fps": rnd(low(0.01)), "low_01pct_fps": rnd(low(0.001)),
"p50_ms": rnd(pct(times, 0.5)), "p95_ms": rnd(pct(times, 0.95)), "p99_ms": rnd(pct(times, 0.99)),
"p999_ms": rnd(pct(times, 0.999)), "max_ms": rnd(times[-1]),
}
for key, values in (("cpu_p50_ms", cpu_ms), ("gpu_p50_ms", gpu_ms)):
if values:
entry[key] = rnd(pct(sorted(values), 0.5))
entry["histogram"] = histogram
return entry
def summary(values):
"""A measurement over repeats (horizon_bench::Summary::of); None for no values."""
values = [v for v in values if v is not None]
n = len(values)
if n == 0:
return None
mean = sum(values) / n
sd = ci = None
if n > 1:
sd = math.sqrt(sum((v - mean) ** 2 for v in values) / (n - 1))
ci = rnd((T95[n - 2] if n - 1 <= len(T95) else 1.96) * sd / math.sqrt(n))
sd = rnd(sd)
return {"n": n, "mean": rnd(mean), "stdev": sd, "min": rnd(min(values)), "max": rnd(max(values)), "ci95": ci}
class Sampler(threading.Thread):
@@ -237,28 +264,51 @@ def poll(control, check, limit, interval=0.05):
raise TimeoutError(f"not ready after {limit} s")
# Fewer section meshes than this in a second is the game remeshing blocks that change (random
# ticks), not loading.
QUIET_MESHES_PER_S = 100
def wait_loaded(control, near, limit):
"""Waits until the player is spawned (near(position)), nothing is queued and
the resident and generated chunk counts have not changed for a second, as
the vanilla driver does; returns the time of the last change.
(pending_chunks alone misses work the threads have already taken; sections
are remeshed all the time as blocks change, so meshes are not counted.)"""
"""Waits until the player is spawned (near(position)), nothing is queued, the
resident and generated chunk counts have not changed for a second, as the
vanilla driver does, and the last sections are drawn; returns when the last
of these happened. Builds that report `status.streaming` are drawn once no
stage (generation, decoration, light, meshing) has held work for half a
second; blocks that change after loading (fluids settling, random ticks)
only remesh in short bursts. Older builds fall back to meshing going quiet
(fewer than QUIET_MESHES_PER_S section meshes in a second), which also
waits through that tail. (pending_chunks alone misses work the threads have
already taken, and sections waiting for their neighbours' light.)"""
deadline = time.time() + limit
last, changed = None, time.time()
last, changed, previous, meshing = None, time.time(), None, time.time()
while time.time() < deadline:
try:
s = control("status", timeout=10)
except (OSError, ValueError, RuntimeError):
s = {}
player = s.get("player")
now = time.time()
if not player or not near(player["position"]) or s.get("pending_chunks") != 0 or s.get("screen") == "title":
last, changed = None, time.time()
last, changed, previous, meshing = None, now, None, now
else:
counts = (s["chunks"], s["generated_chunks"])
if counts != last:
last, changed = counts, time.time()
elif time.time() - changed >= 1.0:
return changed
last, changed = counts, now
stages = s.get("streaming")
sample = (now, s["meshed_sections"])
if stages is not None:
# Protos outside the view distance wait for neighbours that are never generated.
if any(v for k, v in stages.items() if k != "protos"):
meshing = now
quiet = 0.5
else:
if previous and sample[1] - previous[1] > QUIET_MESHES_PER_S * (now - previous[0]):
meshing = now
quiet = 1.0
previous = sample
if now - changed >= 1.0 and now - meshing >= quiet:
return max(changed, meshing)
time.sleep(0.05)
raise TimeoutError(f"not loaded after {limit} s")
@@ -307,13 +357,11 @@ def run_horizon(args, distance):
begin = time.time()
summary = control("fly {} {} {} {} {} {} {} {} raw".format(*start, *end, args.seconds, pitch))
windows.append((begin, time.time()))
scene.update(stats(summary.pop("frame_ms")))
scene["horizon_summary"] = {k: summary.get(k) for k in ("cpu", "gpu", "phases_avg_ms", "slow_phases_avg_ms")}
scene.update({k: summary.get(k) or [] for k in ("frame_ms", "cpu_ms", "gpu_ms")})
status = control("status")
scene["meshes_flight"] = status["meshed_sections"] - loaded_meshes
renderer = status.get("renderer") or {}
scene["gpu_allocated_mb"] = (renderer.get("gpu_allocated_bytes") or 0) / 2**20 or None
scene["nvidia_mb"] = nvidia_memory_mb(proc.pid)
scene["gpu_mb"] = nvidia_memory_mb(proc.pid) or (renderer.get("gpu_allocated_bytes") or 0) / 2**20 or None
result["scenes"][name] = scene
print_scene(args.game, distance, name, scene)
finally:
@@ -476,8 +524,8 @@ def run_vanilla(args, distance):
for flown in by("flown"):
name = flown["scene"]
windows.append((flights[name]["epoch_ms"] / 1000, flown["epoch_ms"] / 1000))
scene = {"load_s": loaded[name]["load_ms"] / 1000, **stats(flown["frame_ms"]),
"chunks": flown["chunks"], "nvidia_mb": nvidia}
scene = {"load_s": loaded[name]["load_ms"] / 1000, "frame_ms": flown["frame_ms"],
"chunks": flown["chunks"], "gpu_mb": nvidia}
result["scenes"][name] = scene
print_scene(args.game, distance, name, scene)
result["memory"] = sampler.summary(windows)
@@ -495,58 +543,113 @@ def stop(proc):
proc.kill()
def print_scene(game, distance, name, s):
print(f" {game:<12} rd {distance:>2} {name:<9} load {s['load_s']:6.1f} s {s['avg_fps']:7.1f} fps"
f" 1% {s['low_1pct_fps']:6.1f} 0.1% {s['low_01pct_fps']:6.1f}"
f" p50 {s['p50_ms']:6.2f} p99 {s['p99_ms']:7.2f} ms", flush=True)
def print_scene(game, distance, name, scene):
s = frames(name, 0, scene["frame_ms"]) or {}
print(f" {game:<12} rd {distance:>2} {name:<9} load {scene['load_s']:6.1f} s {s.get('avg_fps', 0):7.1f} fps"
f" 1% {s.get('low_1pct_fps', 0):6.1f} 0.1% {s.get('low_01pct_fps', 0):6.1f}"
f" p50 {s.get('p50_ms', 0):6.2f} p99 {s.get('p99_ms', 0):7.2f} ms", flush=True)
# Report --------------------------------------------------------------------
def machine():
info = {"host": platform.node(), "os": f"{platform.system()} {platform.release()}",
"arch": platform.machine(), "python": platform.python_version(), "cpus": os.cpu_count()}
def hardware():
"""horizon_bench::Hardware::detect: the machine, without its name."""
info = {"os": {"darwin": "macos", "win32": "windows"}.get(sys.platform, sys.platform),
"arch": {"arm64": "aarch64", "AMD64": "x86_64"}.get(platform.machine(), platform.machine()),
"cpu_threads": os.cpu_count()}
if sys.platform == "darwin":
q = lambda key: subprocess.run(["sysctl", "-n", key], capture_output=True, text=True).stdout.strip()
info["cpu"], info["ram_gb"] = q("machdep.cpu.brand_string"), int(q("hw.memsize") or 0) / 2**30
q = lambda *cmd: subprocess.run(cmd, capture_output=True, text=True).stdout.strip()
info.update(os_version=q("sw_vers", "-productVersion"), cpu=q("sysctl", "-n", "machdep.cpu.brand_string"),
ram_mb=int(q("sysctl", "-n", "hw.memsize") or 0) >> 20)
else:
release = dict(l.split("=", 1) for l in Path("/etc/os-release").read_text().splitlines() if "=" in l)
info["os_version"] = f"{release.get('PRETTY_NAME', '').strip(chr(34))} (kernel {platform.release()})"
for line in Path("/proc/cpuinfo").read_text().splitlines():
if line.startswith("model name"):
info["cpu"] = line.split(":", 1)[1].strip()
break
info["ram_gb"] = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / 2**30
info["ram_mb"] = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") >> 20
return info
def report(args, distance, runs, app):
"""One v1 report for a game at a render distance, over its repeats."""
first = runs[0]
renderer = first.get("renderer") or {}
hw = hardware()
if renderer.get("gpu"):
hw["gpu"] = {"name": renderer["gpu"], "vendor": str(renderer.get("vendor") or ""),
"driver": str(renderer.get("driver") or ""), "backend": str(renderer.get("backend") or "").lower(),
"kind": renderer.get("device_type") or ""}
size = renderer.get("size")
conditions = {"scenario": "perf_bench:" + ",".join(first["scenes"]), "seed": str(SEED),
"render_distance": distance, "simulation_distance": 12, "vsync": False,
"offscreen": bool(args.offscreen), "repeats": len(runs), "warmup_s": args.settle,
"load_avg_1m": rnd(first["load_before"][0])}
if size:
conditions["window"] = list(size)
metrics, entries = {}, []
def metric(name, values):
if (m := summary(values)) is not None:
metrics[name] = m
metric("startup.title_ms", [r["startup"]["title_s"] * 1000 for r in runs])
metric("startup.world_ms", [r["startup"]["world_s"] * 1000 for r in runs])
for name in first["scenes"]:
scenes = [r["scenes"][name] for r in runs if name in r["scenes"]]
flights = [frames(name, i, s["frame_ms"], s.get("cpu_ms", ()), s.get("gpu_ms", ())) for i, s in enumerate(scenes)]
entries += [f for f in flights if f]
metric(f"chunks.{name}.load_ms", [s["load_s"] * 1000 for s in scenes])
metric(f"chunks.{name}.meshed_count", [s.get("meshes_load") for s in scenes])
for key in ("avg_fps", "low_1pct_fps", "low_01pct_fps"):
metric(f"fps.{name}.{key}", [f[key] for f in flights if f])
metric(f"memory.{name}.gpu_mb", [s.get("gpu_mb") for s in scenes])
metric("memory.rss_peak_mb", [r["memory"].get("rss_peak_mb") for r in runs])
metric("memory.rss_steady_mb", [r["memory"].get("rss_steady_mb") for r in runs])
return {"format": REPORT_FORMAT, "version": REPORT_VERSION, "kind": "game",
"created": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"app": app, "hardware": hw, "conditions": conditions, "metrics": metrics, "frames": entries,
"label": args.label}
def table(labels):
"""Markdown tables comparing saved runs (benchmarks/perf/<label>.json)."""
runs = [json.loads((ROOT / "benchmarks/perf" / f"{l}.json").read_text()) for l in labels]
names = [r["label"] for r in runs]
print("| RD | scene | " + " | ".join(f"{n} avg / 1% / 0.1% low FPS" for n in names) + " |")
print("| --- | --- |" + " --- |" * len(runs))
distances = list(runs[0]["distances"])
for rd in distances:
for scene in runs[0]["distances"][rd]["scenes"]:
"""Markdown tables comparing saved runs (benchmarks/perf/<label>-rd<N>.json): means and 95% CIs."""
runs = {}
for label in labels:
for path in sorted((ROOT / "benchmarks/perf").glob(f"{label}-rd*.json"), key=lambda p: int(p.stem.rsplit("rd", 1)[1])):
r = json.loads(path.read_text())
runs.setdefault(label, {})[r["conditions"]["render_distance"]] = r
distances = sorted({d for by in runs.values() for d in by})
cell = lambda m, digits=0: "–" if not m else f"{m['mean']:.{digits}f}" + (f" ±{m['ci95']:.{digits}f}" if m["ci95"] else "")
print("| RD | scene | " + " | ".join(f"{l}: avg / 1% / 0.1% low FPS" for l in labels) + " |")
print("| --- | --- |" + " --- |" * len(labels))
for d in distances:
scenes = sorted({k.split(".")[1] for by in runs.values() if d in by for k in by[d]["metrics"] if k.startswith("fps.")})
for scene in scenes:
cells = []
for r in runs:
s = r["distances"].get(rd, {}).get("scenes", {}).get(scene)
cells.append(f"{s['avg_fps']:.0f} / {s['low_1pct_fps']:.0f} / {s['low_01pct_fps']:.0f}" if s else "–")
print(f"| {rd} | {scene} | " + " | ".join(cells) + " |")
for l in labels:
m = runs.get(l, {}).get(d, {}).get("metrics", {})
cells.append(" / ".join(cell(m.get(f"fps.{scene}.{k}")) for k in ("avg_fps", "low_1pct_fps", "low_01pct_fps")))
print(f"| {d} | {scene} | " + " | ".join(cells) + " |")
print()
print("| RD | " + " | ".join(f"{n}: title / world s, scene load s, RSS steady / peak MB" for n in names) + " |")
print("| --- |" + " --- |" * len(runs))
for rd in distances:
print("| RD | " + " | ".join(f"{l}: title / world s, mean scene load s, RSS steady / peak MB, load avg" for l in labels) + " |")
print("| --- |" + " --- |" * len(labels))
for d in distances:
cells = []
for r in runs:
d = r["distances"].get(rd)
if not d:
for l in labels:
r = runs.get(l, {}).get(d)
if not r:
cells.append("–")
continue
loads = [s["load_s"] for s in d["scenes"].values()]
m = d.get("memory", {})
cells.append(f"{d['startup']['title_s']:.1f} / {d['startup']['world_s']:.1f}, "
f"{statistics.mean(loads):.1f}, {m.get('rss_steady_mb') or 0:.0f} / {m.get('rss_peak_mb') or 0:.0f}")
print(f"| {rd} | " + " | ".join(cells) + " |")
m = r["metrics"]
loads = [v["mean"] for k, v in m.items() if k.endswith(".load_ms")]
sec = lambda k: f"{m[k]['mean'] / 1000:.1f}" if k in m else "–"
mb = lambda k: f"{m[k]['mean']:.0f}" if k in m else "–"
cells.append(f"{sec('startup.title_ms')} / {sec('startup.world_ms')}, "
f"{statistics.mean(loads) / 1000 if loads else 0:.1f}, "
f"{mb('memory.rss_steady_mb')} / {mb('memory.rss_peak_mb')}, {r['conditions'].get('load_avg_1m')}")
print(f"| {d} | " + " | ".join(cells) + " |")
def main():
@@ -565,33 +668,40 @@ def main():
help="Horizon renders into a texture and does not wait for the window (locked macOS sessions)")
p.add_argument("--resources", default=str(ROOT / ".local/horizon-client/26.3"))
p.add_argument("--build", default=str(ROOT / "target/release"))
p.add_argument("--note", help="free text stored with the results")
p.add_argument("--repeats", type=int, default=3, help="games per render distance; the report gives means and 95%% CIs")
p.add_argument("--table", nargs="+", metavar="LABEL", help="print markdown tables of saved runs, and exit")
args = p.parse_args()
if args.table:
table(args.table)
return
label = args.label or f"{args.game}-{platform.node().split('.')[0]}"
args.label = args.label or f"{args.game}-{platform.node().split('.')[0]}"
WORK.mkdir(parents=True, exist_ok=True)
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT,
capture_output=True, text=True).stdout.strip() or None
report = {
"label": label, "game": args.game, "date": datetime.date.today().isoformat(), "commit": commit,
"machine": machine(), "offscreen": args.offscreen, "seed": SEED, "flight_seconds": args.seconds,
"settle_seconds": args.settle, "note": args.note,
"scenes": {n: {"from": a, "to": b, "pitch": c} for n, a, b, c in scene_list(args)},
"distances": {},
}
run = run_horizon if args.game == "horizon" else run_vanilla
for d in [int(x) for x in args.distances.split(",")]:
load_before = os.getloadavg()
result = run(args, d)
result["load_before"], result["load_after"] = load_before, os.getloadavg()
report["distances"][str(d)] = result
out = ROOT / "benchmarks/perf" / f"{label}.json"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report, indent=1) + "\n")
print(out)
runs = []
for _ in range(args.repeats):
load_before = os.getloadavg()
result = run(args, d)
result["load_before"], result["load_after"] = load_before, os.getloadavg()
runs.append(result)
out = ROOT / "benchmarks/perf" / f"{args.label}-rd{d}.json"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report(args, d, runs, app(args)), indent=1) + "\n")
print(out)
def app(args):
if args.game == "horizon":
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT,
capture_output=True, text=True).stdout.strip() or None
manifest = (ROOT / "crates/horizon-client/Cargo.toml").read_text()
version = next(l.split('"')[1] for l in manifest.splitlines() if l.startswith("version"))
return {"name": "horizon-client", "version": version, "commit": commit}
lock = json.loads((ROOT / "packs/mods.lock.json").read_text())
mods = [{"name": "fabric-loader", "version": lock["fabric_loader"]}]
if args.game == "vanilla-mods":
mods += [{"name": n, "version": lock["projects"][n]["version"]} for n in PERF_MODS]
return {"name": "minecraft", "version": "26.3", "mods": mods}
if __name__ == "__main__":