diff --git a/README.md b/README.md index d6d02addc..852869d5e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Client-first performance research for Minecraft Java **26.3**, preserving vanill ## Play on your Mac -Download **horizon-mac-starter-0.16.0.zip** from [Releases](https://github.com/ludovic111/minecraft-optimization-lab/releases), extract it and open `Mac-Start.command`. It verifies and installs official universal Prism Launcher in `~/Applications` if needed, then opens the balanced pack import. Sign into Microsoft **inside Prism yourself**. This project needs no credentials. +Download **horizon-mac-starter-0.17.0.zip** from [Releases](https://github.com/ludovic111/minecraft-optimization-lab/releases), extract it and open `Mac-Start.command`. It verifies and installs official universal Prism Launcher in `~/Applications` if needed, then opens the balanced pack import. Sign into Microsoft **inside Prism yourself**. This project needs no credentials. Use native **Java 25 ARM64**, maximum heap **6144 MiB**, minimum **512 MiB** on your 16 GB M2 Pro. Prism can download Java automatically. Start at 1280×800 or 1920×1200; full Retina resolution is a larger GPU workload. [START-HERE.txt](packs/START-HERE.txt) has details. The same `.mrpack` files import into Prism on Windows and Linux. @@ -104,6 +104,14 @@ python3 tools/compare_benchmarks.py baseline-frames.json optimized-frames.json - ``` Open `Compare-Benchmarks.html` from the starter ZIP or [the source file](tools/Compare-Benchmarks.html) in your browser to compare without Python. Select each JSON and optionally its matching CSV. It exports summary JSON, flags settings and duration differences, and charts frames by recorded active time or ticks by sample number. It is self-contained and makes no network requests. +Version 0.17 also records loaded mod versions and the Java vendor, VM and runtime +build in client and server reports. Both comparison tools flag software changes +as explicit experiment variables; older reports remain usable with an unknown +software warning. The viewer can expand the complete loaded mod list. +[Live software identity check](benchmarks/software-identity-2026-10-04/live-observations.json) +records the metadata and file-count validation; its short capture is not a +performance comparison. + The comparison flags captured setting differences, but cannot verify route, world, shader state or cache warm-up. See [validation evidence](docs/playable-validation.md). Software-renderer test FPS is not evidence of a Mac/GPU speedup. diff --git a/benchmarks/software-identity-2026-10-04/live-observations.json b/benchmarks/software-identity-2026-10-04/live-observations.json new file mode 100644 index 000000000..dd266ee4a --- /dev/null +++ b/benchmarks/software-identity-2026-10-04/live-observations.json @@ -0,0 +1,160 @@ +{ + "schema": 1, + "purpose": "Functional software identity capture, not a performance comparison", + "minecraft": "26.3", + "platform": "Linux x86_64 software OpenGL", + "source_commit": "110c75d712ed4827ae5e72c613f834bcc4e97da2", + "original_mod_sha256": "98f8db7ca8962a71625eef835f57d25c8afb802f2cf1b7c4c832fc070757e011", + "results": [ + { + "kind": "frames", + "elapsed_seconds": 15.335704769, + "java_vendor": "Ubuntu", + "java_vm_name": "OpenJDK 64-Bit Server VM", + "java_runtime_version": "25.0.4.1+1-1-26.04.4-Ubuntu", + "mods": [ + "distanthorizons@3.3.4", + "entityculling@1.11.2", + "fabric-advancement-api-v1@1.0.0+38c7a2d55d", + "fabric-api-base@2.0.6+fcdff87f02", + "fabric-api-lookup-api-v1@2.0.24+3434d6d902", + "fabric-api@0.161.0+26.3", + "fabric-biome-api-v1@20.0.9+74ed1ea502", + "fabric-block-api-v1@3.1.0+0ae6f91102", + "fabric-block-getter-api-v2@2.0.9+3434d6d902", + "fabric-client-gametest-api-v1@6.0.7+4be74c3f02", + "fabric-command-api-v2@3.1.2+fcdff87f02", + "fabric-content-registries-v0@15.0.4+74ed1ea55d", + "fabric-convention-tags-v2@4.10.3+6b5b47fd5d", + "fabric-crash-report-info-v1@1.0.7+fcdff87f02", + "fabric-creative-tab-api-v1@5.0.21+fcdff87f02", + "fabric-data-attachment-api-v1@2.2.30+3434d6d902", + "fabric-data-generation-api-v1@27.2.4+427eab975d", + "fabric-debug-api-v1@1.0.4+3434d6d902", + "fabric-dimensions-v1@5.1.19+47f74c9802", + "fabric-entity-events-v1@6.0.4+3434d6d902", + "fabric-events-interaction-v0@5.3.6+3434d6d902", + "fabric-game-rule-api-v1@4.0.10+3434d6d902", + "fabric-item-api-v1@14.7.0+cf6bc2db5d", + "fabric-key-mapping-api-v1@2.0.8+3434d6d902", + "fabric-lifecycle-events-v1@4.1.9+ffef5f6702", + "fabric-loot-api-v3@4.0.8+4068fd645d", + "fabric-menu-api-v1@2.0.27+ed66f0a35d", + "fabric-message-api-v1@7.0.10+3434d6d902", + "fabric-model-loading-api-v1@8.0.35+fcdff87f5d", + "fabric-networking-api-v1@6.3.8+fcdff87f02", + "fabric-object-builder-api-v1@24.1.9+3434d6d902", + "fabric-particles-v1@5.0.24+3434d6d902", + "fabric-permission-api-v1@1.0.8+fcdff87f02", + "fabric-recipe-api-v1@10.0.8+fcdff87f02", + "fabric-registry-sync-v0@8.0.1+fcdff87f02", + "fabric-renderer-api-v1@17.0.15+79385d0b5d", + "fabric-renderer-indigo@9.1.23+b9d63e335d", + "fabric-rendering-fluids-v1@6.0.6+3434d6d902", + "fabric-rendering-v1@27.0.14+901a437c02", + "fabric-resource-conditions-api-v1@6.1.5+3434d6d902", + "fabric-resource-loader-v0@3.3.26+4fc5413f02", + "fabric-resource-loader-v1@3.0.4+fcdff87f02", + "fabric-screen-api-v1@5.2.4+48607d035d", + "fabric-serialization-api-v1@2.0.7+74ed1ea502", + "fabric-sound-api-v1@2.0.7+3434d6d902", + "fabric-tag-api-v1@2.1.10+fcdff87f02", + "fabric-transfer-api-v1@8.0.25+fcdff87f02", + "fabric-transitive-access-wideners-v1@8.1.16+52268a0e02", + "fabricloader@0.19.5", + "ferritecore@9.0.0", + "horizonlab@0.17.0", + "immediatelyfast@1.17.1+26.3", + "java@25", + "lithium@0.26.2+mc26.3", + "minecraft@26.3", + "mixinextras@0.5.5", + "sodium@0.9.2+mc26.3", + "transition@1.0.25", + "trender@1.0.17" + ], + "excluded_frames": 0, + "dropped_samples": 0, + "samples": 38, + "csv_count_matched": true + }, + { + "kind": "ticks", + "elapsed_seconds": 15.009787535, + "java_vendor": "Ubuntu", + "java_vm_name": "OpenJDK 64-Bit Server VM", + "java_runtime_version": "25.0.4.1+1-1-26.04.4-Ubuntu", + "mods": [ + "distanthorizons@3.3.4", + "entityculling@1.11.2", + "fabric-advancement-api-v1@1.0.0+38c7a2d55d", + "fabric-api-base@2.0.6+fcdff87f02", + "fabric-api-lookup-api-v1@2.0.24+3434d6d902", + "fabric-api@0.161.0+26.3", + "fabric-biome-api-v1@20.0.9+74ed1ea502", + "fabric-block-api-v1@3.1.0+0ae6f91102", + "fabric-block-getter-api-v2@2.0.9+3434d6d902", + "fabric-client-gametest-api-v1@6.0.7+4be74c3f02", + "fabric-command-api-v2@3.1.2+fcdff87f02", + "fabric-content-registries-v0@15.0.4+74ed1ea55d", + "fabric-convention-tags-v2@4.10.3+6b5b47fd5d", + "fabric-crash-report-info-v1@1.0.7+fcdff87f02", + "fabric-creative-tab-api-v1@5.0.21+fcdff87f02", + "fabric-data-attachment-api-v1@2.2.30+3434d6d902", + "fabric-data-generation-api-v1@27.2.4+427eab975d", + "fabric-debug-api-v1@1.0.4+3434d6d902", + "fabric-dimensions-v1@5.1.19+47f74c9802", + "fabric-entity-events-v1@6.0.4+3434d6d902", + "fabric-events-interaction-v0@5.3.6+3434d6d902", + "fabric-game-rule-api-v1@4.0.10+3434d6d902", + "fabric-item-api-v1@14.7.0+cf6bc2db5d", + "fabric-key-mapping-api-v1@2.0.8+3434d6d902", + "fabric-lifecycle-events-v1@4.1.9+ffef5f6702", + "fabric-loot-api-v3@4.0.8+4068fd645d", + "fabric-menu-api-v1@2.0.27+ed66f0a35d", + "fabric-message-api-v1@7.0.10+3434d6d902", + "fabric-model-loading-api-v1@8.0.35+fcdff87f5d", + "fabric-networking-api-v1@6.3.8+fcdff87f02", + "fabric-object-builder-api-v1@24.1.9+3434d6d902", + "fabric-particles-v1@5.0.24+3434d6d902", + "fabric-permission-api-v1@1.0.8+fcdff87f02", + "fabric-recipe-api-v1@10.0.8+fcdff87f02", + "fabric-registry-sync-v0@8.0.1+fcdff87f02", + "fabric-renderer-api-v1@17.0.15+79385d0b5d", + "fabric-renderer-indigo@9.1.23+b9d63e335d", + "fabric-rendering-fluids-v1@6.0.6+3434d6d902", + "fabric-rendering-v1@27.0.14+901a437c02", + "fabric-resource-conditions-api-v1@6.1.5+3434d6d902", + "fabric-resource-loader-v0@3.3.26+4fc5413f02", + "fabric-resource-loader-v1@3.0.4+fcdff87f02", + "fabric-screen-api-v1@5.2.4+48607d035d", + "fabric-serialization-api-v1@2.0.7+74ed1ea502", + "fabric-sound-api-v1@2.0.7+3434d6d902", + "fabric-tag-api-v1@2.1.10+fcdff87f02", + "fabric-transfer-api-v1@8.0.25+fcdff87f02", + "fabric-transitive-access-wideners-v1@8.1.16+52268a0e02", + "fabricloader@0.19.5", + "ferritecore@9.0.0", + "horizonlab@0.17.0", + "immediatelyfast@1.17.1+26.3", + "java@25", + "lithium@0.26.2+mc26.3", + "minecraft@26.3", + "mixinextras@0.5.5", + "sodium@0.9.2+mc26.3", + "transition@1.0.25", + "trender@1.0.17" + ], + "excluded_frames": 0, + "dropped_samples": 0, + "samples": 301, + "csv_count_matched": true + } + ], + "notes": [ + "Actual frame and integrated-server tick reports contain identical Java identity and 59 loaded mod versions, captured before each recording starts.", + "The short 15-second recording validates metadata and file output only; it is not a valid performance comparison or a Mac result.", + "The actual old/new report pair loads in the offline viewer with matching CSV timelines, an explicit Horizon mod-version difference and unknown legacy Java implementation warnings.", + "Synthetic changed versions, reordered mod lists, missing legacy identity, escaped mod names and expanded lists at 390px pass offline checks." + ] +} diff --git a/mods/horizon-lab/src/main/java/dev/horizonlab/HorizonLab.java b/mods/horizon-lab/src/main/java/dev/horizonlab/HorizonLab.java index ffd88ed31..50d018e73 100644 --- a/mods/horizon-lab/src/main/java/dev/horizonlab/HorizonLab.java +++ b/mods/horizon-lab/src/main/java/dev/horizonlab/HorizonLab.java @@ -43,7 +43,10 @@ public final class HorizonLab implements ModInitializer { public static synchronized boolean startTicks(String label, int seconds, Path output, String profile) { if (ticks != null) return false; - ticks = new Recording(label, "ticks", output, seconds, Map.of("profile", profile, "minecraft", "26.3")); + Map context = dev.horizonlab.metrics.SoftwareContext.capture(); + context.put("profile", profile); + context.put("minecraft", "26.3"); + ticks = new Recording(label, "ticks", output, seconds, context); return true; } diff --git a/mods/horizon-lab/src/main/java/dev/horizonlab/client/HorizonClient.java b/mods/horizon-lab/src/main/java/dev/horizonlab/client/HorizonClient.java index 7aa1d718e..3a0d51b14 100644 --- a/mods/horizon-lab/src/main/java/dev/horizonlab/client/HorizonClient.java +++ b/mods/horizon-lab/src/main/java/dev/horizonlab/client/HorizonClient.java @@ -107,9 +107,7 @@ public final class HorizonClient implements ClientModInitializer { context.put("gpu", device.name()); context.put("graphics_backend", device.backendName()); context.put("graphics_driver", device.driverInfo()); - context.put("mods", FabricLoader.getInstance().getAllMods().stream() - .map(mod -> mod.getMetadata().getId() + "@" + mod.getMetadata().getVersion().getFriendlyString()) - .sorted().toList()); + context.putAll(dev.horizonlab.metrics.SoftwareContext.capture()); frames = new Recording(label, "frames", output, seconds, context); previousStart = 0; if (client.getSingleplayerServer() != null) HorizonLab.startTicks(label, seconds, output, profile); diff --git a/mods/horizon-lab/src/main/java/dev/horizonlab/metrics/SoftwareContext.java b/mods/horizon-lab/src/main/java/dev/horizonlab/metrics/SoftwareContext.java new file mode 100644 index 000000000..688ca44f6 --- /dev/null +++ b/mods/horizon-lab/src/main/java/dev/horizonlab/metrics/SoftwareContext.java @@ -0,0 +1,21 @@ +package dev.horizonlab.metrics; + +import java.util.LinkedHashMap; +import java.util.Map; +import net.fabricmc.loader.api.FabricLoader; + +/** Software identity captured once before a client or server recording starts. */ +public final class SoftwareContext { + private SoftwareContext() {} + + public static Map capture() { + Map context = new LinkedHashMap<>(); + context.put("java_vendor", System.getProperty("java.vendor")); + context.put("java_vm_name", System.getProperty("java.vm.name")); + context.put("java_runtime_version", System.getProperty("java.runtime.version")); + context.put("mods", FabricLoader.getInstance().getAllMods().stream() + .map(mod -> mod.getMetadata().getId() + "@" + mod.getMetadata().getVersion().getFriendlyString()) + .sorted().toList()); + return context; + } +} diff --git a/mods/horizon-lab/src/main/resources/fabric.mod.json b/mods/horizon-lab/src/main/resources/fabric.mod.json index 74d5bce58..b24b419a8 100644 --- a/mods/horizon-lab/src/main/resources/fabric.mod.json +++ b/mods/horizon-lab/src/main/resources/fabric.mod.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, "id": "horizonlab", - "version": "0.16.0", + "version": "0.17.0", "name": "Horizon Lab", "description": "Client frame pacing and server tick measurements for reproducible optimization experiments.", "authors": [ diff --git a/packs/START-HERE.txt b/packs/START-HERE.txt index f9fdc4381..51640b0e3 100644 --- a/packs/START-HERE.txt +++ b/packs/START-HERE.txt @@ -132,6 +132,9 @@ select each JSON together with its matching CSV to see duration charts. Frame an tick reports are compared separately. Files stay on your device; the viewer needs no server, account, installation or network access. Check its settings/route/warm-up warnings before drawing conclusions. Save comparison exports a summary JSON. +The viewer also shows Java implementation and loaded mod versions. Expand the +mod list to inspect it. Upgrades, additions and removals are flagged as experiment +variables; older reports with missing software identity show an unknown warning. DH changes clouds, chunk fade and transparency to blend its distant renderer; assess visual differences as well as speed. For collector experiments test diff --git a/packs/mods.lock.json b/packs/mods.lock.json index cfee0e8f7..b610e0ed2 100644 --- a/packs/mods.lock.json +++ b/packs/mods.lock.json @@ -3,7 +3,7 @@ "minecraft": "26.3", "java_major": 25, "fabric_loader": "0.19.5", - "pack_version": "0.16.0", + "pack_version": "0.17.0", "options_data_version": 5023, "projects": { "sodium": { diff --git a/tests/test_packs.py b/tests/test_packs.py index ed381a768..a4d7a6b27 100644 --- a/tests/test_packs.py +++ b/tests/test_packs.py @@ -209,6 +209,34 @@ class ComparisonTests(unittest.TestCase): self.assertEqual(result["mismatches"], []) self.assertTrue(any("terrain_position_arithmetic differs" in warning for warning in result["warnings"])) + def test_mod_comparison_detects_upgrades_and_ignores_order(self): + baseline = self.report() + baseline.update(mods=["minecraft@26.3", "horizonlab@0.16.0", "sodium@0.9.2"], + java_vendor="Eclipse Adoptium", java_vm_name="OpenJDK 64-Bit Server VM", + java_runtime_version="25.0.4.1+1") + candidate = copy.deepcopy(baseline) + candidate["mods"] = list(reversed(candidate["mods"])) + self.assertFalse(compare_benchmarks.software_warnings(baseline, candidate)) + candidate["mods"].remove("horizonlab@0.16.0") + candidate["mods"].extend(["horizonlab@0.17.0", "iris@1.11.7"]) + result = compare_benchmarks.compare(baseline, candidate) + self.assertEqual(result["mismatches"], []) + self.assertIn("Only baseline: horizonlab@0.16.0.", result["warnings"]) + self.assertIn("Only candidate: horizonlab@0.17.0, iris@1.11.7.", result["warnings"]) + for field in ["java_vendor", "java_vm_name", "java_runtime_version"]: + candidate[field] = "Different implementation" + self.assertTrue(any(f"Software {field} differs" in warning + for warning in compare_benchmarks.software_warnings(baseline, candidate))) + + def test_legacy_or_invalid_software_identity_stays_unknown(self): + baseline = self.report("ticks") + for entries in [None, "sodium@0.9.2", [3], [""]]: + candidate = copy.deepcopy(baseline) + candidate["mods"] = entries + result = compare_benchmarks.compare(baseline, candidate) + self.assertTrue(any("Loaded mod versions were not captured" in warning for warning in result["warnings"])) + self.assertFalse(any("Loaded mod versions differ." in warning for warning in result["warnings"])) + def test_excluded_frames_disclose_that_process_counters_cover_inactive_time(self): baseline = self.report() candidate = copy.deepcopy(baseline) diff --git a/tools/Compare-Benchmarks.html b/tools/Compare-Benchmarks.html index 8e5953d67..45a031780 100644 --- a/tools/Compare-Benchmarks.html +++ b/tools/Compare-Benchmarks.html @@ -56,11 +56,29 @@ function parseCsv(source,count){ return duration/1e6; }); } +function softwareWarnings(a,b){ + const warnings=[]; + for(const key of ['java_vendor','java_vm_name','java_runtime_version']){ + if(![a,b].every(r=>typeof r[key]==='string'&&r[key].length))warnings.push(key+' was not captured in both reports; Java implementation equivalence is unknown.'); + else if(a[key]!==b[key])warnings.push('Software '+key+' differs. Treat it as an explicit experiment variable.'); + } + const mods=[a,b].map(r=>Array.isArray(r.mods)&&r.mods.every(entry=>typeof entry==='string'&&entry.length)?new Set(r.mods):null); + if(mods.some(entries=>entries===null))warnings.push('Loaded mod versions were not captured in both reports; software equivalence is unknown.'); + else{ + const only=[...mods[0]].filter(entry=>!mods[1].has(entry)).sort(),other=[...mods[1]].filter(entry=>!mods[0].has(entry)).sort(); + if(only.length||other.length){ + warnings.push('Loaded mod versions differ. Treat added, removed or upgraded mods as explicit experiment variables.'); + for(const [label,entries] of [['baseline',only],['candidate',other]])if(entries.length)warnings.push('Only '+label+': '+entries.slice(0,12).join(', ')+(entries.length>12?'; '+(entries.length-12)+' more':'')+'.'); + } + } + return warnings; +} function compareReports(a,b){ if(a.kind!==b.kind)throw Error('Compare frame runs with frame runs, or tick runs with tick runs.'); const differences=SETTINGS.filter(key=>JSON.stringify(a[key])!==JSON.stringify(b[key])).map(key=>({setting:key,baseline:a[key]??null,candidate:b[key]??null})); if(a.kind==='ticks'&&a.peak_players!==b.peak_players)differences.push({setting:'peak_players',baseline:a.peak_players??null,candidate:b.peak_players??null}); const warnings=['World, camera route, shaders and terrain-cache warm-up still need to match. These are not verified by the captured settings.']; + warnings.push(...softwareWarnings(a,b)); for(const key of ['compact_object_headers','garbage_collectors']){ if(!(key in a)||!(key in b))warnings.push(key+' was not captured in both reports; JVM configuration equivalence is unknown.'); else if(JSON.stringify(a[key])!==JSON.stringify(b[key]))warnings.push('JVM '+key+' differs. Treat it as an explicit experiment variable.'); @@ -114,6 +132,7 @@ function renderDetails(side){ const r=entry.report,fields=[['Profile',r.profile||r.label||'Unlabeled'],['Recording',format(r.elapsed_seconds)+' s · '+format(r.stats.count)+' samples'],['Platform',[r.os,r.architecture].filter(Boolean).join(' ')||'Not captured']]; if(r.kind==='frames')fields.push(['Graphics',[r.gpu,r.graphics_backend].filter(Boolean).join(' · ')||'Not captured'],['Window',number(r.framebuffer_width)?r.framebuffer_width+' × '+r.framebuffer_height:'Not captured'],['Experiments','Adaptive: '+(r.adaptive_budget_enabled?'on':'off')+' · precise pacing: '+(r.precise_worker_pacing?'on':'off')+' · queue wait: '+(r.debounce_queue_wait?'on':'off')]); fields.push(['JVM', 'Compact headers: '+(r.compact_object_headers===true?'on':r.compact_object_headers===false?'off':'not captured')+' · GC: '+(Array.isArray(r.garbage_collectors)?r.garbage_collectors.join(', '):'not captured')]); + fields.push(['Java',[r.java_vendor,r.java_vm_name,r.java_runtime_version||r.java_version].filter(Boolean).join(' · ')||'Not captured']); if(r.kind==='frames'){ fields.push(['Distant terrain',r.dh_capture_status==='not_installed'?'Not installed':r.dh_capture_status==='available'?(r.dh_rendering_enabled?'Enabled':'Disabled')+' · '+r.dh_lod_radius_chunks+' chunk radius · '+r.dh_horizontal_quality+'/'+r.dh_vertical_quality+' quality · '+r.dh_max_horizontal_resolution+' resolution':'Not captured']); if(number(r.dh_thread_count))fields.push(['Terrain workers',r.dh_thread_count+' · runtime ratio at start '+format(r.dh_runtime_ratio)+(number(r.visual_settings_end?.dh_runtime_ratio)?' / end '+format(r.visual_settings_end.dh_runtime_ratio):'')]); @@ -131,6 +150,13 @@ function renderDetails(side){ if(r.kind==='frames')fields.push(['Empty culling whitelist',r.empty_culling_whitelist===true?'on':r.empty_culling_whitelist===false?'off':'not captured']); if(r.kind==='frames')fields.push(['Interruptible terrain read',r.interruptible_terrain_read===true?'on':r.interruptible_terrain_read===false?'off':'not captured']); for(const [label,value]of fields)target.append(text('dt',label),text('dd',value)); + if(Array.isArray(r.mods)&&r.mods.every(entry=>typeof entry==='string'&&entry.length)){ + const versions=[...new Set(r.mods)].sort(),list=text('details',''); + list.append(text('summary',versions.length+' loaded mods'),text('div',versions.join('\n'))); + list.lastChild.style.whiteSpace='pre-wrap'; + list.lastChild.style.maxHeight='16rem';list.lastChild.style.overflow='auto'; + const value=text('dd','');value.append(list);target.append(text('dt','Software'),value); + }else target.append(text('dt','Software'),text('dd','Loaded mod versions not captured')); } async function loadFiles(side,files){ const token=++generation[side];state[side]=null;render(); diff --git a/tools/compare_benchmarks.py b/tools/compare_benchmarks.py index 3d215c43c..6a4db9413 100644 --- a/tools/compare_benchmarks.py +++ b/tools/compare_benchmarks.py @@ -15,6 +15,29 @@ MATCH_FIELDS = ("minecraft", "kind", "os", "os_version", "architecture", "gpu", "dh_ambient_occlusion", "iris_capture_status", "iris_shaders_enabled", "iris_shader_pack") +def software_warnings(baseline, candidate): + warnings = [] + for field in ["java_vendor", "java_vm_name", "java_runtime_version"]: + if not all(isinstance(report.get(field), str) and report[field] for report in [baseline, candidate]): + warnings.append(f"{field} was not captured in both reports; Java implementation equivalence is unknown.") + elif baseline[field] != candidate[field]: + warnings.append(f"Software {field} differs: {baseline[field]!r} vs {candidate[field]!r}. Treat it as an explicit experiment variable.") + mods = [] + for report in [baseline, candidate]: + entries = report.get("mods") + mods.append(set(entries) if isinstance(entries, list) and all(isinstance(entry, str) and entry for entry in entries) else None) + if any(entries is None for entries in mods): + warnings.append("Loaded mod versions were not captured in both reports; software equivalence is unknown.") + elif mods[0] != mods[1]: + warnings.append("Loaded mod versions differ. Treat added, removed or upgraded mods as explicit experiment variables.") + for label, entries in [("baseline", mods[0] - mods[1]), ("candidate", mods[1] - mods[0])]: + ordered = sorted(entries) + if ordered: + suffix = f"; {len(ordered) - 12} more" if len(ordered) > 12 else "" + warnings.append(f"Only {label}: " + ", ".join(ordered[:12]) + suffix + ".") + return warnings + + def compare(baseline, candidate): for report in [baseline, candidate]: if report.get("schema") != 1 or report.get("kind") not in ["frames", "ticks"]: @@ -26,6 +49,7 @@ def compare(baseline, candidate): mismatches = [f"{key}: {baseline.get(key)!r} vs {candidate.get(key)!r}" for key in MATCH_FIELDS if baseline.get(key) != candidate.get(key)] warnings = ["World, route, warm-up, shader state and LOD cache must also match; these are not verified by this tool."] + warnings.extend(software_warnings(baseline, candidate)) if baseline["kind"] == "frames": for field in ["adaptive_budget_enabled", "adaptive_target_fps", "precise_worker_pacing", "debounce_queue_wait", "wakeable_idle_wait", "dispatch_retry_backoff", "tick_membership_dedup", "empty_culling_whitelist", "culling_ray_scratch", "terrain_position_arithmetic", "interruptible_terrain_read", "dh_thread_count", "dh_runtime_ratio"]: