Merge pull request #25 from ludovic111/benchmark-tick-state

Disclose frozen and altered tick workloads in benchmark reports
This commit is contained in:
Ludovic Marie
2026-10-04 08:57:34 +02:00
committed by GitHub
16 changed files with 459 additions and 10 deletions
+2
View File
@@ -61,6 +61,8 @@ jobs:
run: python -m unittest discover -s tests -v
- name: Compile the private dispatch fixture
run: python -c "from tools.build_mod import build_dispatch_fixture; build_dispatch_fixture()"
- name: Compile the private tick-manager state fixture
run: python -c "from tools.build_mod import build_integration_fixture; build_integration_fixture('tick-state', 'dev.horizonlab.metrics.TickStateIntegration')"
- name: Compile the private terrain tick membership fixture
run: python -c "from tools.build_mod import build_tick_dedup_fixture; build_tick_dedup_fixture()"
- name: Compile the private culling whitelist fixture
+2 -2
View File
@@ -6,7 +6,7 @@ Client-first performance research for Minecraft Java **26.3**, preserving vanill
## Play on your Mac
Download **horizon-mac-starter-0.19.0.zip** from [Releases](https://github.com/ludovic111/minecraft-optimization-lab/releases), extract it and open `Mac-Start.command`. It verifies and installs official universal Prism Launcher in `~/Applications` if needed, then opens the balanced pack import. Sign into Microsoft **inside Prism yourself**. This project needs no credentials.
Download **horizon-mac-starter-0.20.0.zip** from [Releases](https://github.com/ludovic111/minecraft-optimization-lab/releases), extract it and open `Mac-Start.command`. It verifies and installs official universal Prism Launcher in `~/Applications` if needed, then opens the balanced pack import. Sign into Microsoft **inside Prism yourself**. This project needs no credentials.
Use native **Java 25 ARM64**, maximum heap **6144 MiB**, minimum **512 MiB** on your 16 GB M2 Pro. Prism can download Java automatically. Start at 1280×800 or 1920×1200; full Retina resolution is a larger GPU workload. [START-HERE.txt](packs/START-HERE.txt) has details. The same `.mrpack` files import into Prism on Windows and Linux.
@@ -107,7 +107,7 @@ Press **F8** during gameplay to start a 120-second recording, or again to stop e
Use separate identical world copies and the same camera route, resolution, render/simulation distance, heap, VSync and FPS limit. Warm up, then alternate at least three baseline/optimized runs. Test fresh chunk exploration separately. For everyday play cap FPS or enable VSync to reduce power usage.
The instance's `horizon-benchmarks/` folder receives JSON/CSV frame and integrated-server tick timings: average FPS, slowest-1%-average FPS, p95/p99 durations, stalls, JVM heap/GC, sampled process RSS when available, GPU/backend/driver and render settings. Frames include presentation/limiting waits and exclude menus/pauses/unfocused time. Tick `meanRate` is execution capacity, **not TPS**; observed tick counts are separate. Resident process memory is sampled on a shared background thread only during recordings; it includes native resident pages, while heap measures JVM objects. Neither is a complete measurement of unified GPU memory. [Memory measurement details](docs/process-memory.md).
The instance's `horizon-benchmarks/` folder receives JSON/CSV frame and integrated-server tick timings: average FPS, slowest-1%-average FPS, p95/p99 durations, stalls, JVM heap/GC, sampled process RSS when available, GPU/backend/driver and render settings. Frames include presentation/limiting waits and exclude menus/pauses/unfocused time. Tick `meanRate` is execution capacity, **not TPS**; observed tick counts are separate. Version 0.20 [captures sampled tick-manager state](docs/tick-state.md), including restored freeze/step/sprint/rate changes; this does not measure completed world work. Resident process memory is sampled on a shared background thread only during recordings; it includes native resident pages, while heap measures JVM objects. Neither is a complete measurement of unified GPU memory. [Memory measurement details](docs/process-memory.md).
```sh
python3 tools/compare_benchmarks.py baseline-frames.json optimized-frames.json --strict
@@ -0,0 +1,78 @@
{
"schema": 1,
"source_commit": "02d7a5189d82a1bd528ca64148e17fc74700da6f",
"minecraft": "26.3",
"original_mod_sha256": "e651ef49bc97486e8dacfd4b640465c442525cd645e2dba71ddea9fb64347dd8",
"observations": {
"java_vendor": "Ubuntu",
"java_runtime_version": "25.0.4.1+1-1-26.04.4-Ubuntu",
"elapsed_seconds": 10.027359028,
"observed_ticks_per_second": 28.422239515327746,
"tick_state_capture_status": "sampled_server_ticks",
"tick_state_samples": 285,
"tick_state_invalid_samples": 0,
"tick_state_first": {
"sprinting": false,
"configured_tick_rate": 20.0,
"frozen": false,
"runs_game_elements": true,
"stepping": false
},
"tick_state_last": {
"sprinting": false,
"configured_tick_rate": 20.0,
"frozen": false,
"runs_game_elements": true,
"stepping": false
},
"tick_state_changed": [
"configured_tick_rate",
"frozen",
"stepping",
"sprinting",
"runs_game_elements"
],
"tick_state_frozen_samples": 161,
"tick_state_stepping_samples": 9,
"tick_state_sprinting_samples": 5,
"tick_state_suppressed_samples": 151,
"tick_state_nonstandard_rate_samples": 161,
"dropped_samples": 0,
"peak_players": 1,
"java_version": "25.0.4.1",
"os": "Linux",
"os_version": "7.0.0-31-generic",
"architecture": "amd64",
"garbage_collectors": [
"G1 Concurrent GC",
"G1 Old Generation",
"G1 Young Generation"
],
"compact_object_headers": true,
"stats": {
"count": 285,
"meanMs": 15.065535726315789,
"p50Ms": 8.230476,
"p95Ms": 53.157093,
"p99Ms": 79.723861,
"maxMs": 148.427376,
"meanRate": 66.37666380845957,
"lowOnePercentRate": 9.564567089457089,
"over16Ms": 100,
"over33Ms": 38,
"over50Ms": 16,
"over100Ms": 1
}
},
"csv_count_and_mean_verified": true,
"launcher_exit_code": 0,
"notes": [
"Private functional fixture on a disposable demo-world copy; not a performance comparison.",
"Changes are scheduled from client callbacks and applied by server tasks; exact transition times depend on callback scheduling.",
"Normal initial state, custom 40 tick rate and freeze, ten steps, restore 20/unfrozen, five-tick sprint, normal final state.",
"First and last sampled states match; all five intermediate change fields remain recorded.",
"Stepping state is sampled after invocation; the final stepping tick has already decremented its remaining-step count.",
"Raw worlds, logs, JFR files and fixture binaries remain private.",
"No live Mac or hardware GPU result."
]
}
+26
View File
@@ -0,0 +1,26 @@
# Interpreting measured server ticks
Tick execution time omits scheduler sleep. `stats.meanRate` is the inverse average duration, not actual TPS. `observed_ticks_per_second` already counts accepted plus dropped measured invocations divided by wall-clock recording duration. Frozen tick invocations count; paused wall time stays in the denominator. Neither number measures completed simulation work.
Version 0.20 observes the server tick manager after each measured invocation on the server thread. Reports record the first and last sampled configured rate, frozen/stepping/sprinting flags and `runs_game_elements`, plus counters for samples with each nonstandard state. This is read-only instrumentation: it does not set a tick rate, freeze worlds or change game rules. Frame reports do not claim to sample server state; use their accompanying integrated-server tick reports.
The recorder retains which state fields changed between samples. A 20 → 40 → 20 rate change or freeze → unfreeze sequence stays visible even when first and last states match. Both comparison tools mark such recordings as changed workloads, mark differing sampled boundary states, and warn about frozen, stepping, sprinting, suppressed-game-element and nonstandard-rate samples. A consistently configured custom rate can match another report at the same rate, with an explicit warning about that workload. Older, absent or invalid state metadata remains unknown, rather than normal by assumption.
State and tick duration are accepted under the same recording lock, including buffer overflow. The observation path retains scalar values and allocates no maps or lists per invocation; report serialization creates them once. Non-finite or sub-minimum rates are counted as invalid observations without aborting duration capture. The state object is only allocated for tick recordings.
Sampling has limits. These are first/last sampled server states, not exact recording boundary states. Changes entirely between samples can be missed. An integrated server paused by the game menu does not call the measured superclass method, so state counters do not enumerate paused time. The manager flag also does not establish that every entity/chunk was simulated, or detect all other reasons a server skips work. A final stepping tick may already have decremented the remaining-step counter to zero while `runs_game_elements` is still true. Compare world copies, entity populations, completed work and real player workloads separately.
## Validation
Pure recording checks exercise restored intermediate changes, distinct frozen/suppressed states, late observations, absent/invalid manager state and wall-clock TPS calculation under G1, compact headers and ZGC. Offline comparison checks cover map ordering, changed/custom states and legacy/malformed input. The desktop and 390-pixel viewer checks also load the actual discarded paused capture (12 ticks over approximately 120 seconds), display its roughly 0.1 observed TPS and distinguish it from execution capacity. That capture remains excluded from performance evidence.
The private `TickStateIntegration` fixture runs only on a disposable demo-world copy. It starts a short recording, samples normal ticking, changes to 40 configured ticks/second and freezes, steps ten ticks, restores 20/unfrozen state, and requests a five-tick sprint. It checks that sampled boundary states match, all five intermediate state changes/counters survive, and accepted state/duration counts agree. The fixture is compiled in platform CI and excluded from the production JAR and downloadable packs. The [final local applied-hook observation](../benchmarks/tick-state-2026-10-04/live-observations.json) records 285 invocations, with 161 frozen, 9 stepping, 5 sprinting, 151 suppressed-game-element and 161 nonstandard-rate samples. All five change fields survive restored boundary states; CSV count/mean and zero invalid/drop observations are verified, and the client exits with code zero. This is functional evidence rather than a performance comparison. Compilation is not a live Mac result.
Compile the private fixture after building the original mod:
```sh
python3 tools/build_mod.py
python3 -c "from tools.build_mod import build_integration_fixture; build_integration_fixture('tick-state', 'dev.horizonlab.metrics.TickStateIntegration')"
```
Install `.local/mod-build/tick-state-fixture.jar` only into a disposable locally prepared balanced demo instance. Open its copied demo world and dismiss the demo screen. The fixture validates the report and closes the client after its sequence. Never add it to a normal play instance or starter package.
@@ -53,8 +53,9 @@ public final class HorizonLab implements ModInitializer {
public static void tick(long durationNs, MinecraftServer server) {
Recording recording = ticks;
if (recording == null) return;
recording.add(durationNs);
recording.players(server.getPlayerList().getPlayerCount());
var tickRate = server.tickRateManager();
recording.addTick(durationNs, server.getPlayerList().getPlayerCount(), tickRate.tickrate(), tickRate.isFrozen(), tickRate.isSteppingForward(),
tickRate.isSprinting(), tickRate.runsNormally());
if (System.nanoTime() >= recording.deadlineNs) finishTicks();
}
@@ -35,12 +35,14 @@ public final class Recording {
private int residentSamples;
private long residentSampleNs;
private boolean finished;
private final TickState tickState;
public Recording(String label, String kind, Path output, int seconds, Map<String, Object> context) {
if (seconds < 1 || seconds > 600) throw new IllegalArgumentException("Duration must be 1..600 seconds");
String safeLabel = label.replaceAll("[^A-Za-z0-9_-]", "_");
this.label = safeLabel.substring(0, Math.min(safeLabel.length(), 64));
this.kind = kind;
this.tickState = kind.equals("ticks") ? new TickState() : null;
this.output = output;
this.context = new LinkedHashMap<>(context);
this.startedNs = System.nanoTime();
@@ -63,6 +65,13 @@ public final class Recording {
public synchronized void exclude() { excluded++; }
public synchronized void players(int players) { peakPlayers = Math.max(players, peakPlayers); }
public synchronized void addTick(long ns, int players, float rate, boolean frozen, boolean stepping, boolean sprinting, boolean runsGameElements) {
if (ns <= 0 || finished || !kind.equals("ticks")) return;
tickState.observe(rate, frozen, stepping, sprinting, runsGameElements);
add(ns);
players(players);
}
public synchronized Snapshot finish() {
finished = true;
ProcessMemoryMonitor.untrack(this);
@@ -73,7 +82,10 @@ public final class Recording {
metadata.put("schema", 1);
double elapsed = (System.nanoTime() - startedNs) / 1e9;
metadata.put("elapsed_seconds", elapsed);
if (kind.equals("ticks")) metadata.put("observed_ticks_per_second", (count + (long) dropped) / elapsed);
if (kind.equals("ticks")) {
metadata.put("observed_ticks_per_second", (count + (long) dropped) / elapsed);
metadata.putAll(tickState.metadata());
}
metadata.put("dropped_samples", dropped);
metadata.put("excluded_frames", excluded);
metadata.put("peak_players", peakPlayers);
@@ -0,0 +1,76 @@
package dev.horizonlab.metrics;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.Map;
/** Scalar-only observations of the tick manager after each measured server invocation. */
public final class TickState {
private static final int FROZEN = 1;
private static final int STEPPING = 2;
private static final int SPRINTING = 4;
private static final int RUNS_GAME_ELEMENTS = 8;
private long count;
private float firstRate;
private float lastRate;
private int firstFlags;
private int lastFlags;
private int changedFlags;
private boolean rateChanged;
private long frozen;
private long stepping;
private long sprinting;
private long suppressed;
private long nonstandardRate;
private long invalid;
public void observe(float rate, boolean isFrozen, boolean isStepping, boolean isSprinting, boolean runsGameElements) {
if (!Float.isFinite(rate) || rate < 1) { invalid++; return; }
int flags = (isFrozen ? FROZEN : 0) | (isStepping ? STEPPING : 0)
| (isSprinting ? SPRINTING : 0) | (runsGameElements ? RUNS_GAME_ELEMENTS : 0);
if (count == 0) {
firstRate = rate;
firstFlags = flags;
} else {
rateChanged |= rate != lastRate;
changedFlags |= flags ^ lastFlags;
}
lastRate = rate;
lastFlags = flags;
count++;
if (isFrozen) frozen++;
if (isStepping) stepping++;
if (isSprinting) sprinting++;
if (!runsGameElements) suppressed++;
if (rate != 20) nonstandardRate++;
}
public Map<String, Object> metadata() {
Map<String, Object> result = new LinkedHashMap<>();
result.put("tick_state_capture_status", invalid > 0 ? "invalid_samples" : count == 0 ? "unavailable" : "sampled_server_ticks");
result.put("tick_state_samples", count);
result.put("tick_state_invalid_samples", invalid);
if (count == 0) return result;
result.put("tick_state_first", state(firstRate, firstFlags));
result.put("tick_state_last", state(lastRate, lastFlags));
var changed = new ArrayList<String>();
if (rateChanged) changed.add("configured_tick_rate");
if ((changedFlags & FROZEN) != 0) changed.add("frozen");
if ((changedFlags & STEPPING) != 0) changed.add("stepping");
if ((changedFlags & SPRINTING) != 0) changed.add("sprinting");
if ((changedFlags & RUNS_GAME_ELEMENTS) != 0) changed.add("runs_game_elements");
result.put("tick_state_changed", changed);
result.put("tick_state_frozen_samples", frozen);
result.put("tick_state_stepping_samples", stepping);
result.put("tick_state_sprinting_samples", sprinting);
result.put("tick_state_suppressed_samples", suppressed);
result.put("tick_state_nonstandard_rate_samples", nonstandardRate);
return result;
}
private static Map<String, Object> state(float rate, int flags) {
return Map.of("configured_tick_rate", rate, "frozen", (flags & FROZEN) != 0,
"stepping", (flags & STEPPING) != 0, "sprinting", (flags & SPRINTING) != 0,
"runs_game_elements", (flags & RUNS_GAME_ELEMENTS) != 0);
}
}
@@ -1,7 +1,7 @@
{
"schemaVersion": 1,
"id": "horizonlab",
"version": "0.19.0",
"version": "0.20.0",
"name": "Horizon Lab",
"description": "Client frame pacing and server tick measurements for reproducible optimization experiments.",
"authors": [
@@ -52,6 +52,37 @@ public final class MetricsTest {
check(residentSnapshot.metadata().get("process_rss_peak_sampled_bytes").equals(200L), "RSS peak can exceed last sample");
check(residentSnapshot.metadata().get("process_rss_samples").equals(3), "invalid memory samples excluded");
check(resident.finish().samples().length == 1, "late samples do not change a finished recording");
check(!snapshot.metadata().containsKey("tick_state_capture_status"), "frame reports do not pretend to capture server state");
Recording ticks = new Recording("ticks", "ticks", Path.of("unused"), 10, Map.of());
ticks.addTick(10_000_000, 1, 20, false, false, false, true);
ticks.addTick(5_000_000, 1, 40, true, false, false, false);
ticks.addTick(5_000_000, 1, 20, true, true, true, true);
ticks.addTick(10_000_000, 1, 20, false, false, false, true);
var tickSnapshot = ticks.finish();
ticks.addTick(10_000_000, 99, 40, true, false, true, false);
var tickMetadata = tickSnapshot.metadata();
check(tickMetadata.get("tick_state_first").equals(tickMetadata.get("tick_state_last")), "restored boundary states match");
check(tickMetadata.get("tick_state_changed").equals(java.util.List.of("configured_tick_rate", "frozen", "stepping", "sprinting", "runs_game_elements")),
"intermediate changes are retained after restoration");
check(tickMetadata.get("tick_state_samples").equals(4L), "state is sampled with accepted tick durations");
check(tickMetadata.get("tick_state_frozen_samples").equals(2L), "frozen includes stepping ticks");
check(tickMetadata.get("tick_state_suppressed_samples").equals(1L), "frozen and simulation suppression are distinct");
check(tickMetadata.get("tick_state_nonstandard_rate_samples").equals(1L), "custom rate observed between normal boundaries");
check((double) tickMetadata.get("observed_ticks_per_second") == 4 / (double) tickMetadata.get("elapsed_seconds"),
"observed TPS uses wall time rather than inverse execution duration");
check(ticks.finish().metadata().get("tick_state_samples").equals(4L), "late tick observations are ignored");
Recording unsampled = new Recording("unsampled", "ticks", Path.of("unused"), 10, Map.of());
unsampled.add(10_000_000);
check(unsampled.finish().metadata().get("tick_state_capture_status").equals("unavailable"), "missing state is not assumed normal");
Recording invalid = new Recording("invalid", "ticks", Path.of("unused"), 10, Map.of());
invalid.addTick(1_000_000, 1, Float.NaN, false, false, false, true);
invalid.addTick(1_000_000, 1, Float.POSITIVE_INFINITY, false, false, false, true);
invalid.addTick(1_000_000, 1, 0, false, false, false, true);
invalid.addTick(1_000_000, 1, 20, false, false, false, true);
var invalidSnapshot = invalid.finish();
check(invalidSnapshot.samples().length == 4, "invalid manager observations do not abort tick duration capture");
check(invalidSnapshot.metadata().get("tick_state_capture_status").equals("invalid_samples"), "invalid state is not silently assumed normal");
check(invalidSnapshot.metadata().get("tick_state_invalid_samples").equals(3L), "invalid observations counted");
System.out.println("Metrics tests passed.");
}
@@ -0,0 +1,70 @@
package dev.horizonlab.metrics;
import com.google.gson.JsonParser;
import dev.horizonlab.HorizonLab;
import java.nio.file.Files;
import java.nio.file.Path;
import net.fabricmc.api.ClientModInitializer;
import net.fabricmc.fabric.api.client.event.lifecycle.v1.ClientTickEvents;
/** Private fixture. Run only on a disposable demo world; never distribute this mod. */
public final class TickStateIntegration implements ClientModInitializer {
private int phase;
private long started;
@Override public void onInitializeClient() {
ClientTickEvents.END_CLIENT_TICK.register(client -> {
var server = client.getSingleplayerServer();
if (server == null || client.level == null || client.isPaused() || client.gui.screen() != null) return;
Path output = client.gameDirectory.toPath().resolve("horizon-benchmarks");
if (phase == 0) {
phase++;
started = System.nanoTime();
server.execute(() -> {
var manager = server.tickRateManager();
if (manager.tickrate() != 20 || manager.isFrozen() || manager.isSprinting())
throw new AssertionError("Fixture requires normal initial tick-manager state");
if (!HorizonLab.startTicks("tick-state-fixture", 10, output, "private-fixture"))
throw new AssertionError("Fixture recording was not started");
});
}
double elapsed = (System.nanoTime() - started) / 1e9;
if (phase == 1 && elapsed >= 2) {
phase++;
server.execute(() -> { server.tickRateManager().setTickRate(40); server.tickRateManager().setFrozen(true); });
} else if (phase == 2 && elapsed >= 4) {
phase++;
server.execute(() -> {
if (!server.tickRateManager().stepGameIfPaused(10)) throw new AssertionError("Fixture step failed");
});
} else if (phase == 3 && elapsed >= 6) {
phase++;
server.execute(() -> { server.tickRateManager().setFrozen(false); server.tickRateManager().setTickRate(20); });
} else if (phase == 4 && elapsed >= 8) {
phase++;
server.execute(() -> server.tickRateManager().requestGameToSprint(5));
} else if (phase == 5 && elapsed >= 12) {
try (var paths = Files.list(output)) {
var reports = paths.filter(p -> p.getFileName().toString().startsWith("tick-state-fixture-ticks-")
&& p.toString().endsWith(".json")).toList();
if (reports.size() != 1) throw new AssertionError("Expected one completed fixture report");
var report = JsonParser.parseString(Files.readString(reports.getFirst())).getAsJsonObject();
if (!report.get("tick_state_first").equals(report.get("tick_state_last")))
throw new AssertionError("Fixture did not restore its boundary state");
if (report.get("tick_state_samples").getAsLong() != report.getAsJsonObject("stats").get("count").getAsLong())
throw new AssertionError("Applied tick-state and duration counts differ");
for (String field : new String[]{"frozen", "stepping", "sprinting", "suppressed", "nonstandard_rate"})
if (report.get("tick_state_" + field + "_samples").getAsLong() < 1)
throw new AssertionError("Applied fixture did not observe " + field);
var result = new com.google.gson.JsonObject();
for (var entry : report.entrySet())
if (entry.getKey().startsWith("tick_state_") || entry.getKey().equals("observed_ticks_per_second"))
result.add(entry.getKey(), entry.getValue());
System.out.println("HORIZON_TICK_STATE_RESULT " + result);
phase++;
client.stop();
} catch (Exception error) { throw new AssertionError("Private applied tick-state fixture failed", error); }
}
});
}
}
+5
View File
@@ -148,6 +148,11 @@ warnings before drawing conclusions. Save comparison exports a summary JSON.
The viewer also shows Java implementation and loaded mod versions. Expand the
mod list to inspect it. Upgrades, additions and removals are flagged as experiment
variables; older reports with missing software identity show an unknown warning.
Tick reports also show sampled configured rate, freeze, step and sprint state.
Restored changes remain flagged; old or invalid state is unknown. Observed TPS
counts measured invocations, including frozen ticks, over recording wall time.
State is sampled after server invocations; it is not a completed-work or pause
trace. Compare the accompanying tick reports when judging integrated-server load.
DH changes clouds, chunk fade and transparency to blend its distant renderer;
assess visual differences as well as speed. For collector experiments test
+1 -1
View File
@@ -3,7 +3,7 @@
"minecraft": "26.3",
"java_major": 25,
"fabric_loader": "0.19.5",
"pack_version": "0.19.0",
"pack_version": "0.20.0",
"options_data_version": 5023,
"projects": {
"sodium": {
+49
View File
@@ -265,6 +265,55 @@ class ComparisonTests(unittest.TestCase):
self.assertEqual(result["metrics"][0]["change_percent"], 50)
self.assertIn("framebuffer_width", result["mismatches"][0])
def tick_report(self):
report = self.report("ticks")
state = dict(configured_tick_rate=20, frozen=False, stepping=False, sprinting=False, runs_game_elements=True)
report.update(tick_state_capture_status="sampled_server_ticks", tick_state_samples=1000,
tick_state_first=state, tick_state_last=copy.deepcopy(state), tick_state_changed=[],
observed_ticks_per_second=1000 / 120)
report.update({"tick_state_" + field + "_samples": 0
for field in ["frozen", "stepping", "sprinting", "suppressed", "nonstandard_rate"]})
return report
def test_restored_tick_changes_still_disqualify_unchanged_workload(self):
baseline = self.tick_report()
candidate = copy.deepcopy(baseline)
candidate.update(tick_state_changed=["configured_tick_rate", "frozen"],
tick_state_frozen_samples=50, tick_state_nonstandard_rate_samples=50)
self.assertEqual(baseline["tick_state_first"], candidate["tick_state_last"])
result = compare_benchmarks.compare(baseline, candidate)
self.assertIn("candidate tick-manager state changed during recording", result["mismatches"])
self.assertTrue(any("frozen, nonstandard_rate" in warning for warning in result["warnings"]))
observed = next(row for row in result["metrics"] if row["metric"] == "observed_ticks_per_second")
self.assertAlmostEqual(observed["baseline"], 8.333333333)
self.assertTrue(any("execution capacity is not TPS" in warning for warning in result["warnings"]))
def test_equal_tick_state_ignores_map_order_and_warns_on_custom_rate(self):
baseline = self.tick_report()
for boundary in ["tick_state_first", "tick_state_last"]:
baseline[boundary]["configured_tick_rate"] = 40
baseline["tick_state_nonstandard_rate_samples"] = 1000
candidate = copy.deepcopy(baseline)
candidate["tick_state_first"] = dict(reversed(list(candidate["tick_state_first"].items())))
result = compare_benchmarks.compare(baseline, candidate)
self.assertEqual(result["mismatches"], [])
self.assertTrue(any("nonstandard_rate" in warning for warning in result["warnings"]))
candidate["tick_state_last"]["frozen"] = True
self.assertTrue(any("tick_state_last.frozen" in mismatch
for mismatch in compare_benchmarks.compare(baseline, candidate)["mismatches"]))
def test_legacy_and_malformed_tick_state_are_unknown_not_normal(self):
baseline = self.tick_report()
for field, bad in [("tick_state_first", None), ("tick_state_samples", True),
("tick_state_samples", -1), ("tick_state_changed", ["invented"]),
("tick_state_frozen_samples", 1001)]:
candidate = copy.deepcopy(baseline)
candidate[field] = bad
self.assertFalse(compare_benchmarks.tick_state_available(candidate))
result = compare_benchmarks.compare(baseline, candidate)
self.assertTrue(any("candidate tick-manager state was not captured" in warning for warning in result["warnings"]))
self.assertFalse(compare_benchmarks.tick_state_available(self.report("ticks")))
def test_no_samples_and_mixed_kinds_are_rejected(self):
empty = self.report()
empty["stats"]["count"] = 0
+42 -2
View File
@@ -6,7 +6,7 @@
<title>Horizon Lab — Compare runs</title>
<style>
:root{color-scheme:light;--ink:#192920;--muted:#65726a;--line:#d8ded8;--paper:#f6f7f3;--base:#247854;--candidate:#bc642f}
*{box-sizing:border-box}body{margin:0;background:var(--paper);color:var(--ink);font:15px/1.5 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif}main{max-width:1100px;margin:auto;padding:40px 28px 70px}header{display:flex;align-items:center;justify-content:space-between;gap:20px;margin-bottom:26px}.brand{display:flex;align-items:center;gap:11px;font-weight:700}.mark{width:30px;height:30px;background:var(--base);box-shadow:inset 0 -10px #195a3c,inset -10px 0 #2a8c62}.badge{border:1px solid var(--line);padding:4px 10px;border-radius:20px;color:var(--muted);font-size:12px}h1{font-size:clamp(30px,5vw,44px);letter-spacing:-1.3px;line-height:1.15;margin:0 0 10px}h2{font-size:20px;letter-spacing:-.3px;margin:0}p{margin:8px 0;color:var(--muted)}.intro{max-width:750px;margin-bottom:26px}.uploads{display:grid;grid-template-columns:1fr 1fr;gap:18px}.card{background:white;border:1px solid var(--line);border-radius:12px;padding:22px;min-width:0}.card-top{display:flex;gap:9px;align-items:center;margin-bottom:6px}.dot{width:9px;height:9px;border-radius:50%;background:var(--base)}.candidate .dot{background:var(--candidate)}input[type=file]{width:100%;font:inherit;font-size:13px;color:var(--muted);margin-top:15px}input::file-selector-button,button{font:inherit;font-size:13px;background:white;color:var(--ink);border:1px solid var(--line);border-radius:6px;padding:9px 13px;cursor:pointer;margin-right:10px}button:disabled{opacity:.45;cursor:default}input:focus-visible,button:focus-visible{outline:3px solid #80b998;outline-offset:3px}.status{font-size:13px;overflow-wrap:anywhere;margin-top:14px;min-height:20px}.error{color:#a23624}.details{font-size:12px;color:var(--muted);margin:12px 0 0;display:grid;grid-template-columns:100px 1fr;gap:4px 12px;overflow-wrap:anywhere}.details dt,.details dd{margin:0}.details dt{color:var(--ink)}.empty{border:1px dashed var(--line);border-radius:12px;padding:34px;text-align:center;margin-top:22px}.section{margin-top:24px}.section-top{display:flex;justify-content:space-between;align-items:center;gap:12px;margin-bottom:14px}.notice{background:#fff8e8;border:1px solid #e7d9af;border-radius:9px;padding:14px 18px;margin-bottom:18px;font-size:13px}.notice p{color:#685c3f;margin:3px 0}.notice ul{padding-left:18px;margin:8px 0}.metrics{overflow:auto;border:1px solid var(--line);border-radius:10px;background:white}table{width:100%;border-collapse:collapse;white-space:nowrap;text-align:right;font-variant-numeric:tabular-nums}th,td{padding:12px 18px;border-bottom:1px solid #edf0ea}th{font-size:12px;font-weight:600;color:var(--muted)}th:first-child,td:first-child{text-align:left}tr:last-child td{border-bottom:0}td:first-child{font-weight:500}td:last-child{font-family:ui-monospace,SFMono-Regular,Consolas,monospace;font-size:13px}.chart{padding:20px;background:white;border:1px solid var(--line);border-radius:10px}.chart svg{display:block;width:100%;height:auto}.chart-tools{display:flex;align-items:center;justify-content:space-between;gap:12px;margin-bottom:12px}.chart-tools label{font-size:13px;color:var(--muted)}.chart-info{min-height:22px;font-size:13px;color:var(--muted);font-variant-numeric:tabular-nums}.footnote{font-size:12px;margin-top:15px;max-width:850px}.hidden{display:none!important}footer{margin-top:30px;border-top:1px solid var(--line);padding-top:16px;color:var(--muted);font-size:12px}.legend{display:flex;gap:18px;font-size:12px}.legend span{display:flex;gap:6px;align-items:center}@media(max-width:650px){main{padding:24px 16px 40px}.uploads{grid-template-columns:1fr}.card{padding:18px}header{align-items:flex-start}.section-top{align-items:flex-start}th,td{padding:10px 12px}.badge{white-space:nowrap;font-size:11px}.chart{padding:12px}}
*{box-sizing:border-box}body{margin:0;background:var(--paper);color:var(--ink);font:15px/1.5 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif}main{max-width:1100px;margin:auto;padding:40px 28px 70px}header{display:flex;align-items:center;justify-content:space-between;gap:20px;margin-bottom:26px}.brand{display:flex;align-items:center;gap:11px;font-weight:700}.mark{width:30px;height:30px;background:var(--base);box-shadow:inset 0 -10px #195a3c,inset -10px 0 #2a8c62}.badge{border:1px solid var(--line);padding:4px 10px;border-radius:20px;color:var(--muted);font-size:12px}h1{font-size:clamp(30px,5vw,44px);letter-spacing:-1.3px;line-height:1.15;margin:0 0 10px}h2{font-size:20px;letter-spacing:-.3px;margin:0}p{margin:8px 0;color:var(--muted)}.intro{max-width:750px;margin-bottom:26px}.uploads{display:grid;grid-template-columns:1fr 1fr;gap:18px}.card{background:white;border:1px solid var(--line);border-radius:12px;padding:22px;min-width:0}.card-top{display:flex;gap:9px;align-items:center;margin-bottom:6px}.dot{width:9px;height:9px;border-radius:50%;background:var(--base)}.candidate .dot{background:var(--candidate)}input[type=file]{width:100%;font:inherit;font-size:13px;color:var(--muted);margin-top:15px}input::file-selector-button,button{font:inherit;font-size:13px;background:white;color:var(--ink);border:1px solid var(--line);border-radius:6px;padding:9px 13px;cursor:pointer;margin-right:10px}button:disabled{opacity:.45;cursor:default}input:focus-visible,button:focus-visible{outline:3px solid #80b998;outline-offset:3px}.status{font-size:13px;overflow-wrap:anywhere;margin-top:14px;min-height:20px}.error{color:#a23624}.details{font-size:12px;color:var(--muted);margin:12px 0 0;display:grid;grid-template-columns:100px 1fr;gap:4px 12px;overflow-wrap:anywhere}.details dt,.details dd{margin:0}.details dt{color:var(--ink)}.empty{border:1px dashed var(--line);border-radius:12px;padding:34px;text-align:center;margin-top:22px}.section{margin-top:24px}.section-top{display:flex;justify-content:space-between;align-items:center;gap:12px;margin-bottom:14px}.notice{overflow-wrap:anywhere;background:#fff8e8;border:1px solid #e7d9af;border-radius:9px;padding:14px 18px;margin-bottom:18px;font-size:13px}.notice p{color:#685c3f;margin:3px 0}.notice ul{padding-left:18px;margin:8px 0}.metrics{overflow:auto;border:1px solid var(--line);border-radius:10px;background:white}table{width:100%;border-collapse:collapse;white-space:nowrap;text-align:right;font-variant-numeric:tabular-nums}th,td{padding:12px 18px;border-bottom:1px solid #edf0ea}th{font-size:12px;font-weight:600;color:var(--muted)}th:first-child,td:first-child{text-align:left}tr:last-child td{border-bottom:0}td:first-child{font-weight:500}td:last-child{font-family:ui-monospace,SFMono-Regular,Consolas,monospace;font-size:13px}.chart{padding:20px;background:white;border:1px solid var(--line);border-radius:10px}.chart svg{display:block;width:100%;height:auto}.chart-tools{display:flex;align-items:center;justify-content:space-between;gap:12px;margin-bottom:12px}.chart-tools label{font-size:13px;color:var(--muted)}.chart-info{min-height:22px;font-size:13px;color:var(--muted);font-variant-numeric:tabular-nums}.footnote{font-size:12px;margin-top:15px;max-width:850px}.hidden{display:none!important}footer{margin-top:30px;border-top:1px solid var(--line);padding-top:16px;color:var(--muted);font-size:12px}.legend{display:flex;gap:18px;font-size:12px}.legend span{display:flex;gap:6px;align-items:center}@media(max-width:650px){main{padding:24px 16px 40px}.uploads{grid-template-columns:1fr}.card{padding:18px}header{align-items:flex-start}.section-top{align-items:flex-start}th,td{padding:10px 12px}.badge{white-space:nowrap;font-size:11px}.chart{padding:12px}}
</style>
<main>
<header><div class="brand"><span class="mark" aria-hidden="true"></span>Horizon Lab</div><span class="badge">Offline · files stay on your device</span></header>
@@ -73,12 +73,44 @@ function softwareWarnings(a,b){
}
return warnings;
}
const TICK_STATE_FIELDS=['configured_tick_rate','frozen','stepping','sprinting','runs_game_elements'];
function tickStateAvailable(r){
if(r.tick_state_capture_status!=='sampled_server_ticks'||!Number.isSafeInteger(r.tick_state_samples)||r.tick_state_samples<=0)return false;
for(const boundary of ['tick_state_first','tick_state_last']){
const s=r[boundary];
if(!s||!number(s.configured_tick_rate)||s.configured_tick_rate<1||TICK_STATE_FIELDS.slice(1).some(key=>typeof s[key]!=='boolean'))return false;
}
if(!Array.isArray(r.tick_state_changed)||r.tick_state_changed.some(key=>!TICK_STATE_FIELDS.includes(key)))return false;
return ['frozen','stepping','sprinting','suppressed','nonstandard_rate'].every(key=>{
const v=r['tick_state_'+key+'_samples'];return Number.isSafeInteger(v)&&v>=0&&v<=r.tick_state_samples;
});
}
function compareTickStates(a,b){
const differences=[],warnings=[],available=[tickStateAvailable(a),tickStateAvailable(b)];
for(const [index,label,r] of [[0,'Baseline',a],[1,'Candidate',b]]){
if(!available[index]){warnings.push(label+' tick-manager state was not captured or is invalid; simulation workload equivalence is unknown.');continue;}
if(r.tick_state_changed.length){
differences.push({setting:label+' tick-manager state changed during recording',baseline:label==='Baseline'?r.tick_state_changed:null,candidate:label==='Candidate'?r.tick_state_changed:null});
warnings.push(label+' sampled tick-manager changes: '+r.tick_state_changed.join(', ')+'.');
}
const abnormal=['frozen','stepping','sprinting','suppressed','nonstandard_rate'].filter(key=>r['tick_state_'+key+'_samples']>0);
if(abnormal.length)warnings.push(label+' includes tick-manager samples with '+abnormal.join(', ')+'; interpret execution times as this workload.');
}
if(available.every(Boolean))for(const boundary of ['tick_state_first','tick_state_last'])for(const key of TICK_STATE_FIELDS){
if(a[boundary][key]!==b[boundary][key])differences.push({setting:boundary+'.'+key,baseline:a[boundary][key],candidate:b[boundary][key]});
}
warnings.push('Tick-manager state is sampled after measured server invocations. Changes entirely between samples, paused intervals and completed world work are not verified.');
return{differences,warnings};
}
function compareReports(a,b){
if(a.kind!==b.kind)throw Error('Compare frame runs with frame runs, or tick runs with tick runs.');
const differences=SETTINGS.filter(key=>JSON.stringify(a[key])!==JSON.stringify(b[key])).map(key=>({setting:key,baseline:a[key]??null,candidate:b[key]??null}));
if(a.kind==='ticks'&&a.peak_players!==b.peak_players)differences.push({setting:'peak_players',baseline:a.peak_players??null,candidate:b.peak_players??null});
const warnings=['World, camera route, shaders and terrain-cache warm-up still need to match. These are not verified by the captured settings.'];
warnings.push(...softwareWarnings(a,b));
if(a.kind==='ticks'){
const tick=compareTickStates(a,b);differences.push(...tick.differences);warnings.push(...tick.warnings);
}
for(const key of ['compact_object_headers','garbage_collectors']){
if(!(key in a)||!(key in b))warnings.push(key+' was not captured in both reports; JVM configuration equivalence is unknown.');
else if(JSON.stringify(a[key])!==JSON.stringify(b[key]))warnings.push('JVM '+key+' differs. Treat it as an explicit experiment variable.');
@@ -111,7 +143,7 @@ function compareReports(a,b){
add('Average FPS','meanRate','FPS',1,true);add('Slowest 1% average','lowOnePercentRate','FPS',1,true);
}else{
add('Average tick duration','meanMs','ms',1,true);add('Observed tick rate','observed_ticks_per_second','TPS');
warnings.push('Tick durations exclude scheduler sleep. Observed TPS includes recording time spent paused; tick execution capacity is not TPS.');
warnings.push('Tick durations exclude scheduler sleep. Observed TPS counts measured invocations, including frozen ticks, and includes recording time spent paused; tick execution capacity is not TPS.');
}
add('95th percentile duration','p95Ms','ms',1,true);add('99th percentile duration','p99Ms','ms',1,true);add('Longest duration','maxMs','ms',1,true);
add('Sampled peak process RSS','process_rss_peak_sampled_bytes','MiB',1048576);add('Last sampled process RSS','process_rss_last_sampled_bytes','MiB',1048576);
@@ -133,6 +165,14 @@ function renderDetails(side){
if(r.kind==='frames')fields.push(['Graphics',[r.gpu,r.graphics_backend].filter(Boolean).join(' · ')||'Not captured'],['Window',number(r.framebuffer_width)?r.framebuffer_width+' × '+r.framebuffer_height:'Not captured'],['Experiments','Adaptive: '+(r.adaptive_budget_enabled?'on':'off')+' · precise pacing: '+(r.precise_worker_pacing?'on':'off')+' · queue wait: '+(r.debounce_queue_wait?'on':'off')]);
fields.push(['JVM', 'Compact headers: '+(r.compact_object_headers===true?'on':r.compact_object_headers===false?'off':'not captured')+' · GC: '+(Array.isArray(r.garbage_collectors)?r.garbage_collectors.join(', '):'not captured')]);
fields.push(['Java',[r.java_vendor,r.java_vm_name,r.java_runtime_version||r.java_version].filter(Boolean).join(' · ')||'Not captured']);
if(r.kind==='ticks'){
if(tickStateAvailable(r)){
const first=r.tick_state_first,last=r.tick_state_last;
fields.push(['Configured ticks',format(first.configured_tick_rate)+' / s at first sample · '+format(last.configured_tick_rate)+' / s at last sample']);
fields.push(['Tick-manager samples',r.tick_state_samples+' · frozen '+r.tick_state_frozen_samples+' · stepping '+r.tick_state_stepping_samples+' · sprinting '+r.tick_state_sprinting_samples+' · suppressed '+r.tick_state_suppressed_samples]);
fields.push(['Tick-manager changes',r.tick_state_changed.length?r.tick_state_changed.join(', '):'None observed']);
}else fields.push(['Tick-manager state','Not captured or invalid']);
}
if(r.kind==='frames'){
fields.push(['Distant terrain',r.dh_capture_status==='not_installed'?'Not installed':r.dh_capture_status==='available'?(r.dh_rendering_enabled?'Enabled':'Disabled')+' · '+r.dh_lod_radius_chunks+' chunk radius · '+r.dh_horizontal_quality+'/'+r.dh_vertical_quality+' quality · '+r.dh_max_horizontal_resolution+' resolution':'Not captured']);
if(number(r.dh_thread_count))fields.push(['Terrain workers',r.dh_thread_count+' · runtime ratio at start '+format(r.dh_runtime_ratio)+(number(r.visual_settings_end?.dh_runtime_ratio)?' / end '+format(r.visual_settings_end.dh_runtime_ratio):'')]);
+1 -1
View File
@@ -25,7 +25,7 @@ def test_metrics():
destination = BUILD / "metric-tests"
destination.mkdir(parents=True, exist_ok=True)
sources = [SOURCE / "main/java/dev/horizonlab/metrics" / name
for name in ["SampleStats.java", "Recording.java", "ProcessMemoryMonitor.java"]]
for name in ["SampleStats.java", "Recording.java", "ProcessMemoryMonitor.java", "TickState.java"]]
sources.append(SOURCE / "test/java/dev/horizonlab/metrics/MetricsTest.java")
subprocess.run(["javac", "--release", "25", "-d", str(destination), *map(str, sources)], check=True)
subprocess.run(["java", "-cp", str(destination), "dev.horizonlab.metrics.MetricsTest"], check=True)
+59
View File
@@ -3,6 +3,7 @@
import argparse
import json
import math
from pathlib import Path
MATCH_FIELDS = ("minecraft", "kind", "os", "os_version", "architecture", "gpu", "graphics_backend", "graphics_driver",
@@ -14,6 +15,59 @@ MATCH_FIELDS = ("minecraft", "kind", "os", "os_version", "architecture", "gpu",
"dh_transparency", "dh_lod_shading", "dh_lod_only_mode", "dh_camera_quality_dropoff",
"dh_ambient_occlusion", "iris_capture_status", "iris_shaders_enabled", "iris_shader_pack")
TICK_STATE_FIELDS = ("configured_tick_rate", "frozen", "stepping", "sprinting", "runs_game_elements")
def tick_state_available(report):
def number(value):
if not isinstance(value, (int, float)) or isinstance(value, bool):
return False
try:
return math.isfinite(value)
except OverflowError:
return False
if report.get("tick_state_capture_status") != "sampled_server_ticks":
return False
count = report.get("tick_state_samples")
if not number(count) or count <= 0 or count > 9_007_199_254_740_991 or count != int(count):
return False
for boundary in ["tick_state_first", "tick_state_last"]:
state = report.get(boundary)
if not isinstance(state, dict) or not number(state.get("configured_tick_rate")) or state["configured_tick_rate"] < 1:
return False
if any(not isinstance(state.get(field), bool) for field in TICK_STATE_FIELDS[1:]):
return False
changed = report.get("tick_state_changed")
if not isinstance(changed, list) or any(field not in TICK_STATE_FIELDS for field in changed):
return False
return all(number(report.get("tick_state_" + field + "_samples"))
and 0 <= report["tick_state_" + field + "_samples"] <= count
and report["tick_state_" + field + "_samples"] == int(report["tick_state_" + field + "_samples"])
for field in ["frozen", "stepping", "sprinting", "suppressed", "nonstandard_rate"])
def compare_tick_states(baseline, candidate):
mismatches, warnings = [], []
available = [tick_state_available(report) for report in [baseline, candidate]]
for name, report, captured in zip(["baseline", "candidate"], [baseline, candidate], available):
if not captured:
warnings.append(f"{name} tick-manager state was not captured or is invalid; simulation workload equivalence is unknown.")
continue
if report["tick_state_changed"]:
mismatches.append(f"{name} tick-manager state changed during recording")
warnings.append(f"{name} sampled tick-manager changes: " + ", ".join(report["tick_state_changed"]) + ".")
abnormal = [field for field in ["frozen", "stepping", "sprinting", "suppressed", "nonstandard_rate"]
if report["tick_state_" + field + "_samples"] > 0]
if abnormal:
warnings.append(f"{name} includes tick-manager samples with " + ", ".join(abnormal) + "; interpret execution times as this workload.")
if all(available):
for boundary in ["tick_state_first", "tick_state_last"]:
for field in TICK_STATE_FIELDS:
if baseline[boundary][field] != candidate[boundary][field]:
mismatches.append(f"{boundary}.{field}: {baseline[boundary][field]!r} vs {candidate[boundary][field]!r}")
warnings.append("Tick-manager state is sampled after measured server invocations. Changes entirely between samples, paused intervals and completed world work are not verified.")
return mismatches, warnings
def software_warnings(baseline, candidate):
warnings = []
@@ -50,6 +104,11 @@ def compare(baseline, candidate):
if baseline.get(key) != candidate.get(key)]
warnings = ["World, route, warm-up, shader state and LOD cache must also match; these are not verified by this tool."]
warnings.extend(software_warnings(baseline, candidate))
if baseline["kind"] == "ticks":
tick_mismatches, tick_warnings = compare_tick_states(baseline, candidate)
mismatches.extend(tick_mismatches)
warnings.extend(tick_warnings)
warnings.append("Tick durations exclude scheduler sleep. Observed TPS counts measured invocations, including frozen ticks, and includes recording time spent paused; tick execution capacity is not TPS.")
if baseline["kind"] == "frames":
for field in ["adaptive_budget_enabled", "adaptive_target_fps", "precise_worker_pacing", "debounce_queue_wait",
"wakeable_idle_wait", "dispatch_retry_backoff", "tick_membership_dedup", "empty_culling_whitelist", "culling_ray_scratch", "terrain_position_arithmetic", "terrain_quad_reuse", "interruptible_terrain_read", "dh_thread_count", "dh_runtime_ratio"]: