Merge pull request #18 from ludovic111/benchmark-software-provenance

Show Java and mod version changes in benchmark comparisons
This commit is contained in:
Ludovic Marie
2026-10-04 05:40:52 +02:00
committed by GitHub
11 changed files with 278 additions and 7 deletions
+9 -1
View File
@@ -6,7 +6,7 @@ Client-first performance research for Minecraft Java **26.3**, preserving vanill
## Play on your Mac
Download **horizon-mac-starter-0.16.0.zip** from [Releases](https://github.com/ludovic111/minecraft-optimization-lab/releases), extract it and open `Mac-Start.command`. It verifies and installs official universal Prism Launcher in `~/Applications` if needed, then opens the balanced pack import. Sign into Microsoft **inside Prism yourself**. This project needs no credentials.
Download **horizon-mac-starter-0.17.0.zip** from [Releases](https://github.com/ludovic111/minecraft-optimization-lab/releases), extract it and open `Mac-Start.command`. It verifies and installs official universal Prism Launcher in `~/Applications` if needed, then opens the balanced pack import. Sign into Microsoft **inside Prism yourself**. This project needs no credentials.
Use native **Java 25 ARM64**, maximum heap **6144 MiB**, minimum **512 MiB** on your 16 GB M2 Pro. Prism can download Java automatically. Start at 1280×800 or 1920×1200; full Retina resolution is a larger GPU workload. [START-HERE.txt](packs/START-HERE.txt) has details. The same `.mrpack` files import into Prism on Windows and Linux.
@@ -104,6 +104,14 @@ python3 tools/compare_benchmarks.py baseline-frames.json optimized-frames.json -
```
Open `Compare-Benchmarks.html` from the starter ZIP or [the source file](tools/Compare-Benchmarks.html) in your browser to compare without Python. Select each JSON and optionally its matching CSV. It exports summary JSON, flags settings and duration differences, and charts frames by recorded active time or ticks by sample number. It is self-contained and makes no network requests.
Version 0.17 also records loaded mod versions and the Java vendor, VM and runtime
build in client and server reports. Both comparison tools flag software changes
as explicit experiment variables; older reports remain usable with an unknown
software warning. The viewer can expand the complete loaded mod list.
[Live software identity check](benchmarks/software-identity-2026-10-04/live-observations.json)
records the metadata and file-count validation; its short capture is not a
performance comparison.
The comparison flags captured setting differences, but cannot verify route, world, shader state or cache warm-up. See [validation evidence](docs/playable-validation.md). Software-renderer test FPS is not evidence of a Mac/GPU speedup.
@@ -0,0 +1,160 @@
{
"schema": 1,
"purpose": "Functional software identity capture, not a performance comparison",
"minecraft": "26.3",
"platform": "Linux x86_64 software OpenGL",
"source_commit": "110c75d712ed4827ae5e72c613f834bcc4e97da2",
"original_mod_sha256": "98f8db7ca8962a71625eef835f57d25c8afb802f2cf1b7c4c832fc070757e011",
"results": [
{
"kind": "frames",
"elapsed_seconds": 15.335704769,
"java_vendor": "Ubuntu",
"java_vm_name": "OpenJDK 64-Bit Server VM",
"java_runtime_version": "25.0.4.1+1-1-26.04.4-Ubuntu",
"mods": [
"distanthorizons@3.3.4",
"entityculling@1.11.2",
"fabric-advancement-api-v1@1.0.0+38c7a2d55d",
"fabric-api-base@2.0.6+fcdff87f02",
"fabric-api-lookup-api-v1@2.0.24+3434d6d902",
"fabric-api@0.161.0+26.3",
"fabric-biome-api-v1@20.0.9+74ed1ea502",
"fabric-block-api-v1@3.1.0+0ae6f91102",
"fabric-block-getter-api-v2@2.0.9+3434d6d902",
"fabric-client-gametest-api-v1@6.0.7+4be74c3f02",
"fabric-command-api-v2@3.1.2+fcdff87f02",
"fabric-content-registries-v0@15.0.4+74ed1ea55d",
"fabric-convention-tags-v2@4.10.3+6b5b47fd5d",
"fabric-crash-report-info-v1@1.0.7+fcdff87f02",
"fabric-creative-tab-api-v1@5.0.21+fcdff87f02",
"fabric-data-attachment-api-v1@2.2.30+3434d6d902",
"fabric-data-generation-api-v1@27.2.4+427eab975d",
"fabric-debug-api-v1@1.0.4+3434d6d902",
"fabric-dimensions-v1@5.1.19+47f74c9802",
"fabric-entity-events-v1@6.0.4+3434d6d902",
"fabric-events-interaction-v0@5.3.6+3434d6d902",
"fabric-game-rule-api-v1@4.0.10+3434d6d902",
"fabric-item-api-v1@14.7.0+cf6bc2db5d",
"fabric-key-mapping-api-v1@2.0.8+3434d6d902",
"fabric-lifecycle-events-v1@4.1.9+ffef5f6702",
"fabric-loot-api-v3@4.0.8+4068fd645d",
"fabric-menu-api-v1@2.0.27+ed66f0a35d",
"fabric-message-api-v1@7.0.10+3434d6d902",
"fabric-model-loading-api-v1@8.0.35+fcdff87f5d",
"fabric-networking-api-v1@6.3.8+fcdff87f02",
"fabric-object-builder-api-v1@24.1.9+3434d6d902",
"fabric-particles-v1@5.0.24+3434d6d902",
"fabric-permission-api-v1@1.0.8+fcdff87f02",
"fabric-recipe-api-v1@10.0.8+fcdff87f02",
"fabric-registry-sync-v0@8.0.1+fcdff87f02",
"fabric-renderer-api-v1@17.0.15+79385d0b5d",
"fabric-renderer-indigo@9.1.23+b9d63e335d",
"fabric-rendering-fluids-v1@6.0.6+3434d6d902",
"fabric-rendering-v1@27.0.14+901a437c02",
"fabric-resource-conditions-api-v1@6.1.5+3434d6d902",
"fabric-resource-loader-v0@3.3.26+4fc5413f02",
"fabric-resource-loader-v1@3.0.4+fcdff87f02",
"fabric-screen-api-v1@5.2.4+48607d035d",
"fabric-serialization-api-v1@2.0.7+74ed1ea502",
"fabric-sound-api-v1@2.0.7+3434d6d902",
"fabric-tag-api-v1@2.1.10+fcdff87f02",
"fabric-transfer-api-v1@8.0.25+fcdff87f02",
"fabric-transitive-access-wideners-v1@8.1.16+52268a0e02",
"fabricloader@0.19.5",
"ferritecore@9.0.0",
"horizonlab@0.17.0",
"immediatelyfast@1.17.1+26.3",
"java@25",
"lithium@0.26.2+mc26.3",
"minecraft@26.3",
"mixinextras@0.5.5",
"sodium@0.9.2+mc26.3",
"transition@1.0.25",
"trender@1.0.17"
],
"excluded_frames": 0,
"dropped_samples": 0,
"samples": 38,
"csv_count_matched": true
},
{
"kind": "ticks",
"elapsed_seconds": 15.009787535,
"java_vendor": "Ubuntu",
"java_vm_name": "OpenJDK 64-Bit Server VM",
"java_runtime_version": "25.0.4.1+1-1-26.04.4-Ubuntu",
"mods": [
"distanthorizons@3.3.4",
"entityculling@1.11.2",
"fabric-advancement-api-v1@1.0.0+38c7a2d55d",
"fabric-api-base@2.0.6+fcdff87f02",
"fabric-api-lookup-api-v1@2.0.24+3434d6d902",
"fabric-api@0.161.0+26.3",
"fabric-biome-api-v1@20.0.9+74ed1ea502",
"fabric-block-api-v1@3.1.0+0ae6f91102",
"fabric-block-getter-api-v2@2.0.9+3434d6d902",
"fabric-client-gametest-api-v1@6.0.7+4be74c3f02",
"fabric-command-api-v2@3.1.2+fcdff87f02",
"fabric-content-registries-v0@15.0.4+74ed1ea55d",
"fabric-convention-tags-v2@4.10.3+6b5b47fd5d",
"fabric-crash-report-info-v1@1.0.7+fcdff87f02",
"fabric-creative-tab-api-v1@5.0.21+fcdff87f02",
"fabric-data-attachment-api-v1@2.2.30+3434d6d902",
"fabric-data-generation-api-v1@27.2.4+427eab975d",
"fabric-debug-api-v1@1.0.4+3434d6d902",
"fabric-dimensions-v1@5.1.19+47f74c9802",
"fabric-entity-events-v1@6.0.4+3434d6d902",
"fabric-events-interaction-v0@5.3.6+3434d6d902",
"fabric-game-rule-api-v1@4.0.10+3434d6d902",
"fabric-item-api-v1@14.7.0+cf6bc2db5d",
"fabric-key-mapping-api-v1@2.0.8+3434d6d902",
"fabric-lifecycle-events-v1@4.1.9+ffef5f6702",
"fabric-loot-api-v3@4.0.8+4068fd645d",
"fabric-menu-api-v1@2.0.27+ed66f0a35d",
"fabric-message-api-v1@7.0.10+3434d6d902",
"fabric-model-loading-api-v1@8.0.35+fcdff87f5d",
"fabric-networking-api-v1@6.3.8+fcdff87f02",
"fabric-object-builder-api-v1@24.1.9+3434d6d902",
"fabric-particles-v1@5.0.24+3434d6d902",
"fabric-permission-api-v1@1.0.8+fcdff87f02",
"fabric-recipe-api-v1@10.0.8+fcdff87f02",
"fabric-registry-sync-v0@8.0.1+fcdff87f02",
"fabric-renderer-api-v1@17.0.15+79385d0b5d",
"fabric-renderer-indigo@9.1.23+b9d63e335d",
"fabric-rendering-fluids-v1@6.0.6+3434d6d902",
"fabric-rendering-v1@27.0.14+901a437c02",
"fabric-resource-conditions-api-v1@6.1.5+3434d6d902",
"fabric-resource-loader-v0@3.3.26+4fc5413f02",
"fabric-resource-loader-v1@3.0.4+fcdff87f02",
"fabric-screen-api-v1@5.2.4+48607d035d",
"fabric-serialization-api-v1@2.0.7+74ed1ea502",
"fabric-sound-api-v1@2.0.7+3434d6d902",
"fabric-tag-api-v1@2.1.10+fcdff87f02",
"fabric-transfer-api-v1@8.0.25+fcdff87f02",
"fabric-transitive-access-wideners-v1@8.1.16+52268a0e02",
"fabricloader@0.19.5",
"ferritecore@9.0.0",
"horizonlab@0.17.0",
"immediatelyfast@1.17.1+26.3",
"java@25",
"lithium@0.26.2+mc26.3",
"minecraft@26.3",
"mixinextras@0.5.5",
"sodium@0.9.2+mc26.3",
"transition@1.0.25",
"trender@1.0.17"
],
"excluded_frames": 0,
"dropped_samples": 0,
"samples": 301,
"csv_count_matched": true
}
],
"notes": [
"Actual frame and integrated-server tick reports contain identical Java identity and 59 loaded mod versions, captured before each recording starts.",
"The short 15-second recording validates metadata and file output only; it is not a valid performance comparison or a Mac result.",
"The actual old/new report pair loads in the offline viewer with matching CSV timelines, an explicit Horizon mod-version difference and unknown legacy Java implementation warnings.",
"Synthetic changed versions, reordered mod lists, missing legacy identity, escaped mod names and expanded lists at 390px pass offline checks."
]
}
@@ -43,7 +43,10 @@ public final class HorizonLab implements ModInitializer {
public static synchronized boolean startTicks(String label, int seconds, Path output, String profile) {
if (ticks != null) return false;
ticks = new Recording(label, "ticks", output, seconds, Map.of("profile", profile, "minecraft", "26.3"));
Map<String, Object> context = dev.horizonlab.metrics.SoftwareContext.capture();
context.put("profile", profile);
context.put("minecraft", "26.3");
ticks = new Recording(label, "ticks", output, seconds, context);
return true;
}
@@ -107,9 +107,7 @@ public final class HorizonClient implements ClientModInitializer {
context.put("gpu", device.name());
context.put("graphics_backend", device.backendName());
context.put("graphics_driver", device.driverInfo());
context.put("mods", FabricLoader.getInstance().getAllMods().stream()
.map(mod -> mod.getMetadata().getId() + "@" + mod.getMetadata().getVersion().getFriendlyString())
.sorted().toList());
context.putAll(dev.horizonlab.metrics.SoftwareContext.capture());
frames = new Recording(label, "frames", output, seconds, context);
previousStart = 0;
if (client.getSingleplayerServer() != null) HorizonLab.startTicks(label, seconds, output, profile);
@@ -0,0 +1,21 @@
package dev.horizonlab.metrics;
import java.util.LinkedHashMap;
import java.util.Map;
import net.fabricmc.loader.api.FabricLoader;
/** Software identity captured once before a client or server recording starts. */
public final class SoftwareContext {
private SoftwareContext() {}
public static Map<String, Object> capture() {
Map<String, Object> context = new LinkedHashMap<>();
context.put("java_vendor", System.getProperty("java.vendor"));
context.put("java_vm_name", System.getProperty("java.vm.name"));
context.put("java_runtime_version", System.getProperty("java.runtime.version"));
context.put("mods", FabricLoader.getInstance().getAllMods().stream()
.map(mod -> mod.getMetadata().getId() + "@" + mod.getMetadata().getVersion().getFriendlyString())
.sorted().toList());
return context;
}
}
@@ -1,7 +1,7 @@
{
"schemaVersion": 1,
"id": "horizonlab",
"version": "0.16.0",
"version": "0.17.0",
"name": "Horizon Lab",
"description": "Client frame pacing and server tick measurements for reproducible optimization experiments.",
"authors": [
+3
View File
@@ -132,6 +132,9 @@ select each JSON together with its matching CSV to see duration charts. Frame an
tick reports are compared separately. Files stay on your device; the viewer needs
no server, account, installation or network access. Check its settings/route/warm-up
warnings before drawing conclusions. Save comparison exports a summary JSON.
The viewer also shows Java implementation and loaded mod versions. Expand the
mod list to inspect it. Upgrades, additions and removals are flagged as experiment
variables; older reports with missing software identity show an unknown warning.
DH changes clouds, chunk fade and transparency to blend its distant renderer;
assess visual differences as well as speed. For collector experiments test
+1 -1
View File
@@ -3,7 +3,7 @@
"minecraft": "26.3",
"java_major": 25,
"fabric_loader": "0.19.5",
"pack_version": "0.16.0",
"pack_version": "0.17.0",
"options_data_version": 5023,
"projects": {
"sodium": {
+28
View File
@@ -209,6 +209,34 @@ class ComparisonTests(unittest.TestCase):
self.assertEqual(result["mismatches"], [])
self.assertTrue(any("terrain_position_arithmetic differs" in warning for warning in result["warnings"]))
def test_mod_comparison_detects_upgrades_and_ignores_order(self):
baseline = self.report()
baseline.update(mods=["minecraft@26.3", "horizonlab@0.16.0", "sodium@0.9.2"],
java_vendor="Eclipse Adoptium", java_vm_name="OpenJDK 64-Bit Server VM",
java_runtime_version="25.0.4.1+1")
candidate = copy.deepcopy(baseline)
candidate["mods"] = list(reversed(candidate["mods"]))
self.assertFalse(compare_benchmarks.software_warnings(baseline, candidate))
candidate["mods"].remove("horizonlab@0.16.0")
candidate["mods"].extend(["horizonlab@0.17.0", "iris@1.11.7"])
result = compare_benchmarks.compare(baseline, candidate)
self.assertEqual(result["mismatches"], [])
self.assertIn("Only baseline: horizonlab@0.16.0.", result["warnings"])
self.assertIn("Only candidate: horizonlab@0.17.0, iris@1.11.7.", result["warnings"])
for field in ["java_vendor", "java_vm_name", "java_runtime_version"]:
candidate[field] = "Different implementation"
self.assertTrue(any(f"Software {field} differs" in warning
for warning in compare_benchmarks.software_warnings(baseline, candidate)))
def test_legacy_or_invalid_software_identity_stays_unknown(self):
baseline = self.report("ticks")
for entries in [None, "sodium@0.9.2", [3], [""]]:
candidate = copy.deepcopy(baseline)
candidate["mods"] = entries
result = compare_benchmarks.compare(baseline, candidate)
self.assertTrue(any("Loaded mod versions were not captured" in warning for warning in result["warnings"]))
self.assertFalse(any("Loaded mod versions differ." in warning for warning in result["warnings"]))
def test_excluded_frames_disclose_that_process_counters_cover_inactive_time(self):
baseline = self.report()
candidate = copy.deepcopy(baseline)
+26
View File
@@ -56,11 +56,29 @@ function parseCsv(source,count){
return duration/1e6;
});
}
function softwareWarnings(a,b){
const warnings=[];
for(const key of ['java_vendor','java_vm_name','java_runtime_version']){
if(![a,b].every(r=>typeof r[key]==='string'&&r[key].length))warnings.push(key+' was not captured in both reports; Java implementation equivalence is unknown.');
else if(a[key]!==b[key])warnings.push('Software '+key+' differs. Treat it as an explicit experiment variable.');
}
const mods=[a,b].map(r=>Array.isArray(r.mods)&&r.mods.every(entry=>typeof entry==='string'&&entry.length)?new Set(r.mods):null);
if(mods.some(entries=>entries===null))warnings.push('Loaded mod versions were not captured in both reports; software equivalence is unknown.');
else{
const only=[...mods[0]].filter(entry=>!mods[1].has(entry)).sort(),other=[...mods[1]].filter(entry=>!mods[0].has(entry)).sort();
if(only.length||other.length){
warnings.push('Loaded mod versions differ. Treat added, removed or upgraded mods as explicit experiment variables.');
for(const [label,entries] of [['baseline',only],['candidate',other]])if(entries.length)warnings.push('Only '+label+': '+entries.slice(0,12).join(', ')+(entries.length>12?'; '+(entries.length-12)+' more':'')+'.');
}
}
return warnings;
}
function compareReports(a,b){
if(a.kind!==b.kind)throw Error('Compare frame runs with frame runs, or tick runs with tick runs.');
const differences=SETTINGS.filter(key=>JSON.stringify(a[key])!==JSON.stringify(b[key])).map(key=>({setting:key,baseline:a[key]??null,candidate:b[key]??null}));
if(a.kind==='ticks'&&a.peak_players!==b.peak_players)differences.push({setting:'peak_players',baseline:a.peak_players??null,candidate:b.peak_players??null});
const warnings=['World, camera route, shaders and terrain-cache warm-up still need to match. These are not verified by the captured settings.'];
warnings.push(...softwareWarnings(a,b));
for(const key of ['compact_object_headers','garbage_collectors']){
if(!(key in a)||!(key in b))warnings.push(key+' was not captured in both reports; JVM configuration equivalence is unknown.');
else if(JSON.stringify(a[key])!==JSON.stringify(b[key]))warnings.push('JVM '+key+' differs. Treat it as an explicit experiment variable.');
@@ -114,6 +132,7 @@ function renderDetails(side){
const r=entry.report,fields=[['Profile',r.profile||r.label||'Unlabeled'],['Recording',format(r.elapsed_seconds)+' s · '+format(r.stats.count)+' samples'],['Platform',[r.os,r.architecture].filter(Boolean).join(' ')||'Not captured']];
if(r.kind==='frames')fields.push(['Graphics',[r.gpu,r.graphics_backend].filter(Boolean).join(' · ')||'Not captured'],['Window',number(r.framebuffer_width)?r.framebuffer_width+' × '+r.framebuffer_height:'Not captured'],['Experiments','Adaptive: '+(r.adaptive_budget_enabled?'on':'off')+' · precise pacing: '+(r.precise_worker_pacing?'on':'off')+' · queue wait: '+(r.debounce_queue_wait?'on':'off')]);
fields.push(['JVM', 'Compact headers: '+(r.compact_object_headers===true?'on':r.compact_object_headers===false?'off':'not captured')+' · GC: '+(Array.isArray(r.garbage_collectors)?r.garbage_collectors.join(', '):'not captured')]);
fields.push(['Java',[r.java_vendor,r.java_vm_name,r.java_runtime_version||r.java_version].filter(Boolean).join(' · ')||'Not captured']);
if(r.kind==='frames'){
fields.push(['Distant terrain',r.dh_capture_status==='not_installed'?'Not installed':r.dh_capture_status==='available'?(r.dh_rendering_enabled?'Enabled':'Disabled')+' · '+r.dh_lod_radius_chunks+' chunk radius · '+r.dh_horizontal_quality+'/'+r.dh_vertical_quality+' quality · '+r.dh_max_horizontal_resolution+' resolution':'Not captured']);
if(number(r.dh_thread_count))fields.push(['Terrain workers',r.dh_thread_count+' · runtime ratio at start '+format(r.dh_runtime_ratio)+(number(r.visual_settings_end?.dh_runtime_ratio)?' / end '+format(r.visual_settings_end.dh_runtime_ratio):'')]);
@@ -131,6 +150,13 @@ function renderDetails(side){
if(r.kind==='frames')fields.push(['Empty culling whitelist',r.empty_culling_whitelist===true?'on':r.empty_culling_whitelist===false?'off':'not captured']);
if(r.kind==='frames')fields.push(['Interruptible terrain read',r.interruptible_terrain_read===true?'on':r.interruptible_terrain_read===false?'off':'not captured']);
for(const [label,value]of fields)target.append(text('dt',label),text('dd',value));
if(Array.isArray(r.mods)&&r.mods.every(entry=>typeof entry==='string'&&entry.length)){
const versions=[...new Set(r.mods)].sort(),list=text('details','');
list.append(text('summary',versions.length+' loaded mods'),text('div',versions.join('\n')));
list.lastChild.style.whiteSpace='pre-wrap';
list.lastChild.style.maxHeight='16rem';list.lastChild.style.overflow='auto';
const value=text('dd','');value.append(list);target.append(text('dt','Software'),value);
}else target.append(text('dt','Software'),text('dd','Loaded mod versions not captured'));
}
async function loadFiles(side,files){
const token=++generation[side];state[side]=null;render();
+24
View File
@@ -15,6 +15,29 @@ MATCH_FIELDS = ("minecraft", "kind", "os", "os_version", "architecture", "gpu",
"dh_ambient_occlusion", "iris_capture_status", "iris_shaders_enabled", "iris_shader_pack")
def software_warnings(baseline, candidate):
warnings = []
for field in ["java_vendor", "java_vm_name", "java_runtime_version"]:
if not all(isinstance(report.get(field), str) and report[field] for report in [baseline, candidate]):
warnings.append(f"{field} was not captured in both reports; Java implementation equivalence is unknown.")
elif baseline[field] != candidate[field]:
warnings.append(f"Software {field} differs: {baseline[field]!r} vs {candidate[field]!r}. Treat it as an explicit experiment variable.")
mods = []
for report in [baseline, candidate]:
entries = report.get("mods")
mods.append(set(entries) if isinstance(entries, list) and all(isinstance(entry, str) and entry for entry in entries) else None)
if any(entries is None for entries in mods):
warnings.append("Loaded mod versions were not captured in both reports; software equivalence is unknown.")
elif mods[0] != mods[1]:
warnings.append("Loaded mod versions differ. Treat added, removed or upgraded mods as explicit experiment variables.")
for label, entries in [("baseline", mods[0] - mods[1]), ("candidate", mods[1] - mods[0])]:
ordered = sorted(entries)
if ordered:
suffix = f"; {len(ordered) - 12} more" if len(ordered) > 12 else ""
warnings.append(f"Only {label}: " + ", ".join(ordered[:12]) + suffix + ".")
return warnings
def compare(baseline, candidate):
for report in [baseline, candidate]:
if report.get("schema") != 1 or report.get("kind") not in ["frames", "ticks"]:
@@ -26,6 +49,7 @@ def compare(baseline, candidate):
mismatches = [f"{key}: {baseline.get(key)!r} vs {candidate.get(key)!r}" for key in MATCH_FIELDS
if baseline.get(key) != candidate.get(key)]
warnings = ["World, route, warm-up, shader state and LOD cache must also match; these are not verified by this tool."]
warnings.extend(software_warnings(baseline, candidate))
if baseline["kind"] == "frames":
for field in ["adaptive_budget_enabled", "adaptive_target_fps", "precise_worker_pacing", "debounce_queue_wait",
"wakeable_idle_wait", "dispatch_retry_backoff", "tick_membership_dedup", "empty_culling_whitelist", "culling_ray_scratch", "terrain_position_arithmetic", "interruptible_terrain_read", "dh_thread_count", "dh_runtime_ratio"]: