From e0ab3f8fccf9790656f85ff0983b70dfe8ffee9e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:19:24 +0200 Subject: [PATCH 1/3] Benchmark report format v1: docs/benchmark-report.md and crates/horizon-bench One versioned JSON report for every benchmark Horizon writes (the game's --benchmark, the launcher's measurements, tools/perf_bench.py): the local log a player can attach and, with consent, what the telemetry upload sends. Summaries over repeats (mean, stdev, 95% CI) and fixed-edge frame-time histograms, never raw frame streams; no host, user or path in uploads. Co-Authored-By: Claude Opus 5.5 --- Cargo.toml | 2 +- crates/horizon-bench/Cargo.toml | 10 + crates/horizon-bench/src/hardware.rs | 175 +++++++++++ crates/horizon-bench/src/lib.rs | 452 +++++++++++++++++++++++++++ docs/README.md | 1 + docs/benchmark-report.md | 121 +++++++ 6 files changed, 760 insertions(+), 1 deletion(-) create mode 100644 crates/horizon-bench/Cargo.toml create mode 100644 crates/horizon-bench/src/hardware.rs create mode 100644 crates/horizon-bench/src/lib.rs create mode 100644 docs/benchmark-report.md diff --git a/Cargo.toml b/Cargo.toml index eb10100f5..584a87e5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] exclude = ["launcher", "crates/horizon-telemetry-server"] -members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-friends", "crates/horizon-relay", "xtask"] +members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "xtask"] resolver = "2" [workspace.package] diff --git a/crates/horizon-bench/Cargo.toml b/crates/horizon-bench/Cargo.toml new file mode 100644 index 000000000..b945577b0 --- /dev/null +++ b/crates/horizon-bench/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "horizon-bench" +version = "0.1.0" +edition = "2021" +license = "MIT" +description = "The benchmark report format (docs/benchmark-report.md), shared by the game, the launcher and the upload" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/crates/horizon-bench/src/hardware.rs b/crates/horizon-bench/src/hardware.rs new file mode 100644 index 000000000..d1164e826 --- /dev/null +++ b/crates/horizon-bench/src/hardware.rs @@ -0,0 +1,175 @@ +//! The machine a report was measured on. CPU, RAM and OS are detected here; the GPU and the display +//! come from the program that rendered (the game knows its adapter, the launcher its window). + +use std::process::Command; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Hardware { + /// `macos`, `linux`, `windows`. + pub os: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub os_version: Option, + pub arch: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpu: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpu_threads: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ram_mb: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gpu: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Gpu { + pub name: String, + pub vendor: String, + pub driver: String, + /// `metal`, `vulkan`, `dx12`, `gl`. + pub backend: String, + /// `integrated`, `discrete`, `virtual`, `cpu`, `other`. + pub kind: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Display { + /// Physical pixels. + pub width: u32, + pub height: u32, + pub scale: f64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub refresh_hz: Option, +} + +impl Hardware { + pub fn detect() -> Hardware { + Hardware { + os: std::env::consts::OS.to_owned(), + os_version: os_version(), + arch: std::env::consts::ARCH.to_owned(), + cpu: cpu(), + cpu_threads: std::thread::available_parallelism().ok().map(|n| n.get()), + ram_mb: ram_bytes().map(|b| b >> 20), + gpu: None, + display: None, + } + } +} + +fn output(program: &str, args: &[&str]) -> Option { + let out = Command::new(program).args(args).output().ok()?; + let text = String::from_utf8(out.stdout).ok()?.trim().to_owned(); + (out.status.success() && !text.is_empty()).then_some(text) +} + +#[cfg(target_os = "macos")] +fn os_version() -> Option { + output("sw_vers", &["-productVersion"]) +} + +#[cfg(target_os = "macos")] +fn cpu() -> Option { + output("sysctl", &["-n", "machdep.cpu.brand_string"]) +} + +#[cfg(target_os = "macos")] +fn ram_bytes() -> Option { + output("sysctl", &["-n", "hw.memsize"])?.parse().ok() +} + +#[cfg(target_os = "linux")] +fn os_version() -> Option { + let release = std::fs::read_to_string("/etc/os-release").ok()?; + let name = key_value(&release, "PRETTY_NAME", '=')? + .trim_matches('"') + .to_owned(); + Some(match output("uname", &["-r"]) { + Some(kernel) => format!("{name} (kernel {kernel})"), + None => name, + }) +} + +#[cfg(target_os = "linux")] +fn cpu() -> Option { + let info = std::fs::read_to_string("/proc/cpuinfo").ok()?; + // x86 names the CPU per core; ARM boards name the machine once. + key_value(&info, "model name", ':') + .or_else(|| key_value(&info, "Model", ':')) + .map(str::to_owned) +} + +#[cfg(target_os = "linux")] +fn ram_bytes() -> Option { + let info = std::fs::read_to_string("/proc/meminfo").ok()?; + let kb = key_value(&info, "MemTotal", ':')? + .trim_end_matches("kB") + .trim(); + kb.parse::().ok().map(|kb| kb * 1024) +} + +#[cfg(target_os = "linux")] +fn key_value<'a>(text: &'a str, key: &str, sep: char) -> Option<&'a str> { + text.lines().find_map(|line| { + let (k, v) = line.split_once(sep)?; + (k.trim() == key).then(|| v.trim()) + }) +} + +#[cfg(target_os = "windows")] +fn os_version() -> Option { + output("cmd", &["/C", "ver"]) +} + +#[cfg(target_os = "windows")] +fn cpu() -> Option { + std::env::var("PROCESSOR_IDENTIFIER").ok() +} + +#[cfg(target_os = "windows")] +fn ram_bytes() -> Option { + let out = output( + "powershell", + &[ + "-NoProfile", + "-Command", + "(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory", + ], + )?; + out.parse().ok() +} + +#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] +fn os_version() -> Option { + None +} + +#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] +fn cpu() -> Option { + None +} + +#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] +fn ram_bytes() -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_this_machine() { + let hw = Hardware::detect(); + assert_eq!(hw.os, std::env::consts::OS); + assert!(hw.cpu_threads.unwrap() >= 1); + if cfg!(any(target_os = "macos", target_os = "linux")) { + assert!(hw.cpu.is_some(), "{hw:?}"); + assert!(hw.ram_mb.unwrap() > 256, "{hw:?}"); + assert!(hw.os_version.is_some(), "{hw:?}"); + } + } +} diff --git a/crates/horizon-bench/src/lib.rs b/crates/horizon-bench/src/lib.rs new file mode 100644 index 000000000..dd5bdec82 --- /dev/null +++ b/crates/horizon-bench/src/lib.rs @@ -0,0 +1,452 @@ +//! The benchmark report format, version 1 (docs/benchmark-report.md). +//! +//! One JSON file per run, the same for the game, the launcher and `tools/perf_bench.py`: the local log +//! a player can attach, and, with consent, what the telemetry upload sends ([`Report::for_upload`]). +//! It holds summaries and fixed-edge histogram counts, never raw frame streams. + +use std::collections::BTreeMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +mod hardware; + +pub use hardware::{Display, Gpu, Hardware}; + +pub const FORMAT: &str = "horizon-benchmark-report"; +pub const VERSION: u32 = 1; + +/// Lower edges of the frame-time histogram buckets, in milliseconds; the last bucket is 1000 ms and +/// over. Fixed by the format version so histograms from any number of reports add up. +pub const HISTOGRAM_EDGES_MS: [f64; 24] = [ + 0.0, 1.0, 2.0, 3.0, 4.17, 5.0, 6.0, 7.0, 8.33, 10.0, 12.0, 14.0, 16.67, 20.0, 25.0, 33.33, + 40.0, 50.0, 66.67, 100.0, 150.0, 250.0, 500.0, 1000.0, +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Kind { + Game, + Launcher, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct App { + pub name: String, + pub version: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub commit: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub mods: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Mod { + pub name: String, + pub version: String, +} + +/// What has to match for two reports to be compared. Every field is optional. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Conditions { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scenario: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub seed: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub render_distance: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub simulation_distance: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option<[u32; 2]>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vsync: Option, + /// Absent: unlimited. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fps_cap: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub offscreen: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repeats: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub warmup_s: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub power: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub load_avg_1m: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backend: Option, +} + +/// A measurement over a run's repeats. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Summary { + pub n: usize, + pub mean: f64, + /// Sample standard deviation; `None` for a single value. + pub stdev: Option, + pub min: f64, + pub max: f64, + /// Half-width of the 95% confidence interval of the mean (Student's t); `None` for a single value. + pub ci95: Option, +} + +impl Summary { + /// `None` for no values. + pub fn of(values: &[f64]) -> Option { + let n = values.len(); + if n == 0 { + return None; + } + let mean = values.iter().sum::() / n as f64; + let (stdev, ci95) = if n > 1 { + let var = values.iter().map(|v| (v - mean).powi(2)).sum::() / (n - 1) as f64; + let sd = var.sqrt(); + (Some(sd), Some(t95(n - 1) * sd / (n as f64).sqrt())) + } else { + (None, None) + }; + Some(Summary { + n, + mean: round(mean), + stdev: stdev.map(round), + min: round(values.iter().copied().fold(f64::INFINITY, f64::min)), + max: round(values.iter().copied().fold(f64::NEG_INFINITY, f64::max)), + ci95: ci95.map(round), + }) + } +} + +/// Two-sided 95% critical value of Student's t for `df` degrees of freedom. +fn t95(df: usize) -> f64 { + const TABLE: [f64; 30] = [ + 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160, + 2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056, + 2.052, 2.048, 2.045, 2.042, + ]; + TABLE.get(df.wrapping_sub(1)).copied().unwrap_or(1.96) +} + +/// One measured phase of one repeat. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Frames { + pub phase: String, + pub repeat: u32, + pub frames: usize, + pub seconds: f64, + pub avg_fps: f64, + pub low_1pct_fps: f64, + pub low_01pct_fps: f64, + pub p50_ms: f64, + pub p95_ms: f64, + pub p99_ms: f64, + pub p999_ms: f64, + pub max_ms: f64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpu_p50_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gpu_p50_ms: Option, + /// Frame counts per bucket of [`HISTOGRAM_EDGES_MS`]. + pub histogram: Vec, +} + +impl Frames { + /// The figures of a phase from its frame times in milliseconds (time between the starts of + /// consecutive frames); `None` for no frames. CPU and GPU times are per frame too, when measured. + pub fn of( + phase: &str, + repeat: u32, + frame_ms: &[f32], + cpu_ms: &[f32], + gpu_ms: &[f32], + ) -> Option { + let times = sorted(frame_ms); + let n = times.len(); + if n == 0 { + return None; + } + let total: f64 = times.iter().sum(); + // The average frame time of the slowest `share` of frames, as frames per second. + let low = |share: f64| { + let slowest = ×[n - ((n as f64 * share).ceil() as usize).max(1)..]; + 1000.0 * slowest.len() as f64 / slowest.iter().sum::() + }; + let mut histogram = vec![0; HISTOGRAM_EDGES_MS.len()]; + for &t in × { + histogram[HISTOGRAM_EDGES_MS.partition_point(|&edge| edge <= t).max(1) - 1] += 1; + } + Some(Frames { + phase: phase.to_owned(), + repeat, + frames: n, + seconds: round(total / 1000.0), + avg_fps: round(n as f64 * 1000.0 / total), + low_1pct_fps: round(low(0.01)), + low_01pct_fps: round(low(0.001)), + p50_ms: round(percentile(×, 0.5)), + p95_ms: round(percentile(×, 0.95)), + p99_ms: round(percentile(×, 0.99)), + p999_ms: round(percentile(×, 0.999)), + max_ms: round(times[n - 1]), + cpu_p50_ms: (!cpu_ms.is_empty()).then(|| round(percentile(&sorted(cpu_ms), 0.5))), + gpu_p50_ms: (!gpu_ms.is_empty()).then(|| round(percentile(&sorted(gpu_ms), 0.5))), + histogram, + }) + } +} + +fn sorted(ms: &[f32]) -> Vec { + let mut v: Vec = ms + .iter() + .map(|&m| f64::from(m)) + .filter(|m| m.is_finite()) + .collect(); + v.sort_by(f64::total_cmp); + v +} + +/// Three decimals: plenty for milliseconds and FPS, short in JSON, and parsed back exactly. +fn round(v: f64) -> f64 { + (v * 1000.0).round() / 1000.0 +} + +/// Nearest-rank percentile of sorted, non-empty values. +fn percentile(sorted: &[f64], p: f64) -> f64 { + let n = sorted.len(); + sorted[((n as f64 * p).ceil() as usize).clamp(1, n) - 1] +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Report { + pub format: String, + pub version: u32, + pub kind: Kind, + /// When the run ended, UTC, `YYYY-MM-DDTHH:MM:SSZ`. + pub created: String, + pub app: App, + pub hardware: Hardware, + #[serde(default)] + pub conditions: Conditions, + /// Scalar measurements by name; the unit is the name's suffix (`_ms`, `_mb`...). + #[serde(default)] + pub metrics: BTreeMap, + #[serde(default)] + pub frames: Vec, + /// Local only: a name the person running it chose. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub label: Option, + /// Local only: logs the run wrote next to the report. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub local: Vec, +} + +impl Report { + /// An empty report for this machine, created now. + pub fn new(kind: Kind, app: App) -> Report { + Report { + format: FORMAT.to_owned(), + version: VERSION, + kind, + created: utc_now(), + app, + hardware: Hardware::detect(), + conditions: Conditions::default(), + metrics: BTreeMap::new(), + frames: Vec::new(), + label: None, + local: Vec::new(), + } + } + + /// Records a measurement over the repeats; no values records nothing. + pub fn metric(&mut self, name: &str, values: &[f64]) { + if let Some(summary) = Summary::of(values) { + self.metrics.insert(name.to_owned(), summary); + } + } + + /// Reads a report, rejecting other formats and newer versions. + pub fn parse(json: &str) -> Result { + let report: Report = serde_json::from_str(json).map_err(|e| e.to_string())?; + if report.format != FORMAT { + return Err(format!( + "not a benchmark report: format {:?}", + report.format + )); + } + if report.version > VERSION { + return Err(format!( + "report version {} is newer than {VERSION}", + report.version + )); + } + Ok(report) + } + + /// The report without its local-only fields: what an upload sends. + pub fn for_upload(&self) -> Report { + Report { + label: None, + local: Vec::new(), + ..self.clone() + } + } + + pub fn to_json(&self) -> String { + serde_json::to_string_pretty(self).expect("a report always serializes") + } + + /// Writes `/--.json` and returns its path. + pub fn write_to(&self, dir: &Path) -> io::Result { + std::fs::create_dir_all(dir)?; + let kind = match self.kind { + Kind::Game => "game", + Kind::Launcher => "launcher", + }; + let stamp: String = self.created.chars().filter(char::is_ascii_digit).collect(); + let scenario: String = self + .conditions + .scenario + .as_deref() + .unwrap_or("run") + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' { + c + } else { + '_' + } + }) + .collect(); + let path = dir.join(format!("{kind}-{stamp}-{scenario}.json")); + std::fs::write(&path, self.to_json() + "\n")?; + Ok(path) + } +} + +/// The current time, UTC, `YYYY-MM-DDTHH:MM:SSZ`. +pub fn utc_now() -> String { + let secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + utc(secs) +} + +fn utc(secs: u64) -> String { + let (days, rem) = (secs / 86_400, secs % 86_400); + // Civil date from days since 1970-01-01 (Howard Hinnant's algorithm). + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe + era * 400 + i64::from(month <= 2); + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z", + rem / 3600, + rem / 60 % 60, + rem % 60 + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn utc_dates() { + assert_eq!(utc(0), "1970-01-01T00:00:00Z"); + assert_eq!(utc(951_782_400), "2000-02-29T00:00:00Z"); + assert_eq!(utc(1_791_983_007), "2026-10-14T13:03:27Z"); + } + + #[test] + fn summary_of_repeats() { + assert_eq!(Summary::of(&[]), None); + let one = Summary::of(&[5.0]).unwrap(); + assert_eq!((one.n, one.mean, one.stdev, one.ci95), (1, 5.0, None, None)); + let s = Summary::of(&[2.0, 4.0, 6.0]).unwrap(); + assert_eq!((s.mean, s.min, s.max), (4.0, 2.0, 6.0)); + assert_eq!(s.stdev, Some(2.0)); + assert_eq!(s.ci95, Some(4.969), "4.303 * 2 / sqrt(3)"); + } + + #[test] + fn frame_figures_and_histogram() { + // 998 frames of 2 ms, one of 20 ms, one of 1500 ms. + let mut ms = vec![2.0f32; 998]; + ms.extend([20.0, 1500.0]); + let f = Frames::of("flight", 0, &ms, &[], &[]).unwrap(); + assert_eq!(f.frames, 1000); + assert_eq!(f.seconds, 3.516); + assert_eq!((f.p50_ms, f.p999_ms, f.max_ms), (2.0, 20.0, 1500.0)); + // Slowest 1% = 10 frames: 8 of 2 ms, 20 ms, 1500 ms. + assert_eq!(f.low_1pct_fps, 6.51, "1000 * 10 / 1536"); + // Slowest 0.1% = the 1500 ms frame. + assert_eq!(f.low_01pct_fps, 0.667); + assert_eq!(f.histogram.len(), HISTOGRAM_EDGES_MS.len()); + assert_eq!(f.histogram[2], 998, "2 ms falls in [2, 3)"); + assert_eq!(f.histogram[13], 1, "20 ms falls in [20, 25)"); + assert_eq!(f.histogram[23], 1, "1500 ms falls in [1000, inf)"); + assert_eq!(f.histogram.iter().sum::(), 1000); + assert_eq!((f.cpu_p50_ms, f.gpu_p50_ms), (None, None)); + assert_eq!(Frames::of("x", 0, &[], &[], &[]), None); + } + + #[test] + fn round_trip_and_upload_strips_local_fields() { + let mut r = Report::new( + Kind::Game, + App { + name: "horizon-client".into(), + version: "0.1.0".into(), + ..App::default() + }, + ); + r.conditions.scenario = Some("flight".into()); + r.metric("startup.title_ms", &[800.0, 820.0]); + r.frames + .extend(Frames::of("village", 0, &[2.0, 3.0], &[1.0], &[])); + r.label = Some("my run".into()); + r.local.push("/home/someone/logs/latest.log".into()); + let back = Report::parse(&r.to_json()).unwrap(); + assert_eq!(back, r); + let upload = r.for_upload().to_json(); + assert!( + !upload.contains("my run") && !upload.contains("someone"), + "{upload}" + ); + } + + #[test] + fn rejects_other_formats_and_newer_versions() { + let r = Report::new(Kind::Launcher, App::default()); + let newer = r.to_json().replace("\"version\": 1", "\"version\": 2"); + assert!(Report::parse(&newer).unwrap_err().contains("newer")); + let other = r.to_json().replace(FORMAT, "something-else"); + assert!(Report::parse(&other).is_err()); + } + + #[test] + fn writes_into_a_folder() { + let dir = std::env::temp_dir().join(format!("horizon-bench-test-{}", std::process::id())); + let mut r = Report::new(Kind::Launcher, App::default()); + r.created = "2026-10-15T14:03:27Z".into(); + r.conditions.scenario = Some("cold start".into()); + let path = r.write_to(&dir).unwrap(); + assert_eq!( + path.file_name().unwrap(), + "launcher-20261015140327-cold_start.json" + ); + assert_eq!( + Report::parse(&std::fs::read_to_string(&path).unwrap()).unwrap(), + r + ); + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/docs/README.md b/docs/README.md index b30f331a8..275052798 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ To play, use [the launcher](../launcher/README.md); the project site is [playhor - [Extras](extras.md): optional zoom, minimap, FPS and keystroke overlays and auto-reconnect, all off by default. - [Launcher](../launcher/README.md): installs, updates and starts builds; its CLI and MCP server; releasing it. - [Opt-in data collection](telemetry.md): the consent, exactly what is sent, how it is stored and sent. +- [Benchmark report format](benchmark-report.md): the versioned JSON every benchmark writes (game, launcher, `tools/perf_bench.py`), the local log and, with consent, the upload. - Menus: the title, world list, create world, multiplayer, options and pause screens share the launcher's design tokens and fonts (`crates/horizon-theme`), drawn by `ui_theme` over a live frozen backdrop (`game/backdrop.rs`, blurred and graded by `post.rs`). Options → Interface → Menu style switches to vanilla's screens (Classic, kept in `horizon.txt`; `HORIZON_MENU_STYLE=horizon|classic` overrides it, and scripted runs default to Classic). [Title](media/menu-title.png) ([webm](media/menu-title.webm)), [worlds](media/menu-worlds.png), [options](media/menu-options.png), [pause](media/menu-pause.png). ## Parity diff --git a/docs/benchmark-report.md b/docs/benchmark-report.md new file mode 100644 index 000000000..5348a0882 --- /dev/null +++ b/docs/benchmark-report.md @@ -0,0 +1,121 @@ +# Benchmark report format (version 1) + +Every benchmark Horizon writes uses this one JSON format: `horizon-client --benchmark`, the launcher's own +measurements, and `tools/perf_bench.py` (which also writes it for vanilla and vanilla with mods). The same +file is the local log a player can attach to an issue and, only with the player's consent, what the +telemetry upload sends (minus the local-only fields below). The types, the statistics and the hardware +detection are in `crates/horizon-bench`; nothing else builds a report by hand. + +A report holds summaries and histogram counts, never raw frame streams: a 10-minute run is a few +kilobytes whatever the frame rate. Measured numbers are rounded to 3 decimals. + +## Example + +```json +{ + "format": "horizon-benchmark-report", + "version": 1, + "kind": "game", + "created": "2026-10-15T14:03:27Z", + "app": { "name": "horizon-client", "version": "0.21.0", "commit": "e96e1571" }, + "hardware": { + "os": "macos", "os_version": "27.0", "arch": "aarch64", + "cpu": "Apple M4 Pro", "cpu_threads": 14, "ram_mb": 65536, + "gpu": { "name": "Apple M4 Pro", "vendor": "apple", "driver": "", "backend": "metal", "kind": "integrated" }, + "display": { "width": 2560, "height": 1440, "scale": 2.0, "refresh_hz": 60 } + }, + "conditions": { + "scenario": "flight", "seed": "1234", "render_distance": 16, "simulation_distance": 12, + "window": [1280, 720], "vsync": false, "repeats": 3, "warmup_s": 2.0, "power": "ac", "load_avg_1m": 1.2 + }, + "metrics": { + "startup.title_ms": { "n": 3, "mean": 812.0, "stdev": 21.5, "min": 790.0, "max": 833.0, "ci95": 53.4 }, + "memory.rss_peak_mb": { "n": 3, "mean": 1210.0, "stdev": 4.0, "min": 1206.0, "max": 1214.0, "ci95": 9.9 } + }, + "frames": [ + { + "phase": "village", "repeat": 0, "frames": 4920, "seconds": 12.0, + "avg_fps": 410.0, "low_1pct_fps": 120.3, "low_01pct_fps": 61.0, + "p50_ms": 2.31, "p95_ms": 3.8, "p99_ms": 6.9, "p999_ms": 14.2, "max_ms": 21.0, + "cpu_p50_ms": 1.4, "gpu_p50_ms": 1.9, + "histogram": [0, 0, 3011, 1520, 301, 62, 15, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + ] +} +``` + +## Fields + +**`format`, `version`.** Always `"horizon-benchmark-report"` and `1`. A reader rejects other formats and +newer versions. Adding an optional field keeps the version; renaming, removing or changing the meaning of +one (histogram edges included) makes it 2. + +**`kind`.** `"game"` (a game run: Horizon, or vanilla from `tools/perf_bench.py`) or `"launcher"`. + +**`created`.** When the run ended, UTC, `YYYY-MM-DDTHH:MM:SSZ`. + +**`app`.** What was measured: `name` (`horizon-client`, `horizon-launcher`, `minecraft`), `version`, +optionally `commit` (Horizon builds) and `mods` (`[{ "name", "version" }]`, vanilla with mods). + +**`hardware`.** `os` (`macos`, `linux`, `windows`), `os_version`, `arch`, `cpu` (brand string), +`cpu_threads`, `ram_mb`; `gpu` (`name`, `vendor`, `driver`, `backend`: `metal`, `vulkan`, `dx12`, `gl`; +`kind`: `integrated`, `discrete`, `virtual`, `cpu`, `other`) and `display` (the screen the window was on: +`width`, `height` in physical pixels, `scale`, `refresh_hz`) when known. Any field can be missing. + +**`conditions`.** What has to match for two reports to be compared, all optional: `scenario`, `seed`, +`render_distance`, `simulation_distance`, `window` (`[width, height]` actually rendered), `vsync`, +`fps_cap` (absent: unlimited), `offscreen` (rendered without presenting: not a player number), `repeats`, +`warmup_s`, `power` (`ac`, `battery`), `load_avg_1m` (before the run; above about 1 the machine was busy), +`backend` (when forced). Readers compare reports only with equal `scenario`, distances, `window`, `vsync`, +`fps_cap` and `offscreen`. + +**`metrics`.** Scalar measurements, by name; each is a summary over the run's repeats: `n`, `mean`, +`stdev` (sample standard deviation, null when `n` is 1), `min`, `max`, `ci95` (half-width of the 95% +confidence interval of the mean, Student's t, null when `n` is 1). The unit is the name's suffix: `_ms`, +`_s`, `_mb`, `_bytes`, `_fps`, `_pct`, `_mspt`, `_w`, `_c` (degrees Celsius), `_count`. Names in use: + +| Name | What | +| --- | --- | +| `startup.title_ms` | process start to the first frame of the title screen | +| `startup.world_ms` | process start to in-world, every chunk around the spawn loaded and drawn | +| `chunks..load_ms` | teleport to every chunk in view distance loaded and drawn | +| `chunks.generated_per_s`, `chunks.meshed_per_s` | throughput while filling the view distance | +| `chunks.meshes_per_section_count` | meshes built per section (re-meshing; 1 is ideal) | +| `fps..avg_fps`, `fps..low_1pct_fps`, `fps..low_01pct_fps` | per-phase figures across repeats | +| `tick.mspt`, `tick.tps` | integrated server tick time and rate | +| `memory.rss_peak_mb`, `memory.rss_steady_mb` | resident memory: peak over the run, median while flying | +| `memory.gpu_mb` | video memory the game allocated, where readable | +| `power.package_w`, `power.battery_drain_pct` | power draw, battery used per hour (laptops) | +| `thermal.cpu_c`, `thermal.throttled_pct` | temperature, share of the run spent throttled | +| `launcher.cold_start_ms`, `launcher.warm_start_ms` | launcher process start to its first frame | +| `launcher.rss_idle_mb`, `launcher.rss_peak_mb`, `launcher.cpu_idle_pct` | launcher memory and CPU | +| `launcher.release_list_ms` | fetching the release list | +| `launcher.download_ms`, `launcher.install_ms`, `launcher.prepare_ms` | getting a build ready, with `conditions.scenario` `first-install`, `update` or `reuse` | +| `launcher.play_to_game_frame_ms` | Play pressed to the game's first frame | +| `launcher.binary_bytes`, `game.binary_bytes` | size of the executables | + +New names can be added without a new version; the table lists the ones the tools write. + +**`frames`.** One entry per measured phase and repeat (a scene's flight, the launcher's page switching): +`phase`, `repeat`, `frames`, `seconds`, `avg_fps`, `low_1pct_fps` and `low_01pct_fps` (the average frame +time of the slowest 1% or 0.1% of frames, as FPS), `p50_ms`, `p95_ms`, `p99_ms`, `p999_ms`, `max_ms`, +`cpu_p50_ms` and `gpu_p50_ms` when measured, and `histogram`: frame counts per bucket. The bucket edges +are fixed by the version, so histograms from any number of reports add up: + + 0, 1, 2, 3, 4.17, 5, 6, 7, 8.33, 10, 12, 14, 16.67, 20, 25, 33.33, 40, 50, 66.67, 100, 150, 250, 500, 1000 ms + +Bucket *i* counts frames from edge *i* (included) to edge *i* + 1; the last bucket is 1000 ms and over +(24 buckets). 4.17, 8.33, 16.67 and 33.33 ms are 240, 120, 60 and 30 FPS. + +## Local only + +`label` (a name the person running it chose) and `local` (paths of the logs the run wrote next to the +report) stay on the machine: the upload removes them (`Report::for_upload`). Nothing in a report +identifies the player, the machine's name or the world: no host name, user name, path or address. + +## Where reports go + +- The game: `benchmarks/--.json` in the game folder. +- The launcher: `benchmarks/` in the launcher's data folder. +- `tools/perf_bench.py`: one report per game and render distance, `benchmarks/perf/