Merge pull request #247 from ludovic111/feat/bench-report
Benchmark report format v1 (docs/benchmark-report.md, crates/horizon-bench)
This commit is contained in:
Generated
+8
@@ -993,6 +993,14 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "horizon-bench"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "horizon-client"
|
||||
version = "0.1.0"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[workspace]
|
||||
exclude = ["launcher", "crates/horizon-telemetry-server"]
|
||||
members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-friends", "crates/horizon-relay", "xtask"]
|
||||
members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "xtask"]
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
[package]
|
||||
name = "horizon-bench"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "MIT"
|
||||
description = "The benchmark report format (docs/benchmark-report.md), shared by the game, the launcher and the upload"
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
@@ -0,0 +1,259 @@
|
||||
//! The machine a report was measured on. CPU, RAM and OS are detected here; the GPU and the display
|
||||
//! come from the program that rendered (the game knows its adapter, the launcher its window).
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Hardware {
|
||||
/// `macos`, `linux`, `windows`.
|
||||
pub os: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub os_version: Option<String>,
|
||||
pub arch: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cpu: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cpu_threads: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub ram_mb: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub gpu: Option<Gpu>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub display: Option<Display>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Gpu {
|
||||
pub name: String,
|
||||
pub vendor: String,
|
||||
pub driver: String,
|
||||
/// `metal`, `vulkan`, `dx12`, `gl`.
|
||||
pub backend: String,
|
||||
/// `integrated`, `discrete`, `virtual`, `cpu`, `other`.
|
||||
pub kind: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Display {
|
||||
/// Physical pixels.
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub scale: f64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub refresh_hz: Option<u32>,
|
||||
}
|
||||
|
||||
impl Hardware {
|
||||
pub fn detect() -> Hardware {
|
||||
Hardware {
|
||||
os: std::env::consts::OS.to_owned(),
|
||||
os_version: os_version(),
|
||||
arch: std::env::consts::ARCH.to_owned(),
|
||||
cpu: cpu(),
|
||||
cpu_threads: std::thread::available_parallelism().ok().map(|n| n.get()),
|
||||
ram_mb: ram_bytes().map(|b| b >> 20),
|
||||
gpu: None,
|
||||
display: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The one-minute load average: above about the number of threads the machine was busy.
|
||||
pub fn load_average() -> Option<f64> {
|
||||
if cfg!(target_os = "macos") {
|
||||
// "{ 1.23 1.10 1.00 }"
|
||||
output("sysctl", &["-n", "vm.loadavg"])?
|
||||
.trim_start_matches('{')
|
||||
.split_whitespace()
|
||||
.next()?
|
||||
.parse()
|
||||
.ok()
|
||||
} else {
|
||||
std::fs::read_to_string("/proc/loadavg")
|
||||
.ok()?
|
||||
.split_whitespace()
|
||||
.next()?
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// `ac` or `battery`; `None` where it can't be read (and on machines without a battery on Linux).
|
||||
pub fn power_source() -> Option<&'static str> {
|
||||
if cfg!(target_os = "macos") {
|
||||
let batt = output("pmset", &["-g", "batt"])?;
|
||||
return Some(if batt.contains("'Battery Power'") {
|
||||
"battery"
|
||||
} else {
|
||||
"ac"
|
||||
});
|
||||
}
|
||||
let mut on_battery = None;
|
||||
for entry in std::fs::read_dir("/sys/class/power_supply").ok()?.flatten() {
|
||||
let read =
|
||||
|name: &str| std::fs::read_to_string(entry.path().join(name)).unwrap_or_default();
|
||||
match read("type").trim() {
|
||||
"Mains" if read("online").trim() == "1" => return Some("ac"),
|
||||
"Battery" => on_battery = Some("battery"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
on_battery
|
||||
}
|
||||
|
||||
/// Resident memory of a process, in bytes (macOS, Linux).
|
||||
pub fn rss_bytes(pid: u32) -> Option<u64> {
|
||||
if cfg!(target_os = "macos") {
|
||||
return output("ps", &["-o", "rss=", "-p", &pid.to_string()])?
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.map(|kb| kb * 1024);
|
||||
}
|
||||
let status = std::fs::read_to_string(format!("/proc/{pid}/status")).ok()?;
|
||||
let line = status.lines().find(|l| l.starts_with("VmRSS:"))?;
|
||||
line.split_whitespace()
|
||||
.nth(1)?
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.map(|kb| kb * 1024)
|
||||
}
|
||||
|
||||
/// CPU time a process has used, user and system, in seconds (macOS, Linux).
|
||||
pub fn cpu_seconds(pid: u32) -> Option<f64> {
|
||||
if cfg!(target_os = "macos") {
|
||||
// "[[dd-]hh:]mm:ss.cc"
|
||||
let time = output("ps", &["-o", "time=", "-p", &pid.to_string()])?;
|
||||
let (rest, days) = match time.split_once('-') {
|
||||
Some((d, rest)) => (rest.to_owned(), d.parse::<f64>().ok()?),
|
||||
None => (time, 0.0),
|
||||
};
|
||||
let seconds = rest.split(':').try_fold(0.0, |acc, part| {
|
||||
part.parse::<f64>().ok().map(|v| acc * 60.0 + v)
|
||||
})?;
|
||||
return Some(days * 86_400.0 + seconds);
|
||||
}
|
||||
// utime and stime, fields 14 and 15, in clock ticks (100 per second on Linux).
|
||||
let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
|
||||
let fields: Vec<&str> = stat.rsplit_once(')')?.1.split_whitespace().collect();
|
||||
let ticks: f64 = fields.get(11)?.parse::<f64>().ok()? + fields.get(12)?.parse::<f64>().ok()?;
|
||||
Some(ticks / 100.0)
|
||||
}
|
||||
|
||||
fn output(program: &str, args: &[&str]) -> Option<String> {
|
||||
let out = Command::new(program).args(args).output().ok()?;
|
||||
let text = String::from_utf8(out.stdout).ok()?.trim().to_owned();
|
||||
(out.status.success() && !text.is_empty()).then_some(text)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn os_version() -> Option<String> {
|
||||
output("sw_vers", &["-productVersion"])
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn cpu() -> Option<String> {
|
||||
output("sysctl", &["-n", "machdep.cpu.brand_string"])
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn ram_bytes() -> Option<u64> {
|
||||
output("sysctl", &["-n", "hw.memsize"])?.parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn os_version() -> Option<String> {
|
||||
let release = std::fs::read_to_string("/etc/os-release").ok()?;
|
||||
let name = key_value(&release, "PRETTY_NAME", '=')?
|
||||
.trim_matches('"')
|
||||
.to_owned();
|
||||
Some(match output("uname", &["-r"]) {
|
||||
Some(kernel) => format!("{name} (kernel {kernel})"),
|
||||
None => name,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn cpu() -> Option<String> {
|
||||
let info = std::fs::read_to_string("/proc/cpuinfo").ok()?;
|
||||
// x86 names the CPU per core; ARM boards name the machine once.
|
||||
key_value(&info, "model name", ':')
|
||||
.or_else(|| key_value(&info, "Model", ':'))
|
||||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn ram_bytes() -> Option<u64> {
|
||||
let info = std::fs::read_to_string("/proc/meminfo").ok()?;
|
||||
let kb = key_value(&info, "MemTotal", ':')?
|
||||
.trim_end_matches("kB")
|
||||
.trim();
|
||||
kb.parse::<u64>().ok().map(|kb| kb * 1024)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn key_value<'a>(text: &'a str, key: &str, sep: char) -> Option<&'a str> {
|
||||
text.lines().find_map(|line| {
|
||||
let (k, v) = line.split_once(sep)?;
|
||||
(k.trim() == key).then(|| v.trim())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn os_version() -> Option<String> {
|
||||
output("cmd", &["/C", "ver"])
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn cpu() -> Option<String> {
|
||||
std::env::var("PROCESSOR_IDENTIFIER").ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn ram_bytes() -> Option<u64> {
|
||||
let out = output(
|
||||
"powershell",
|
||||
&[
|
||||
"-NoProfile",
|
||||
"-Command",
|
||||
"(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory",
|
||||
],
|
||||
)?;
|
||||
out.parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
|
||||
fn os_version() -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
|
||||
fn cpu() -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
|
||||
fn ram_bytes() -> Option<u64> {
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn detects_this_machine() {
|
||||
let hw = Hardware::detect();
|
||||
assert_eq!(hw.os, std::env::consts::OS);
|
||||
assert!(hw.cpu_threads.unwrap() >= 1);
|
||||
if cfg!(any(target_os = "macos", target_os = "linux")) {
|
||||
assert!(hw.cpu.is_some(), "{hw:?}");
|
||||
assert!(hw.ram_mb.unwrap() > 256, "{hw:?}");
|
||||
assert!(hw.os_version.is_some(), "{hw:?}");
|
||||
assert!(load_average().is_some());
|
||||
assert!(rss_bytes(std::process::id()).unwrap() > 1 << 20);
|
||||
assert!(cpu_seconds(std::process::id()).unwrap() >= 0.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
//! The benchmark report format, version 1 (docs/benchmark-report.md).
|
||||
//!
|
||||
//! One JSON file per run, the same for the game, the launcher and `tools/perf_bench.py`: the local log
|
||||
//! a player can attach, and, with consent, what the telemetry upload sends ([`Report::for_upload`]).
|
||||
//! It holds summaries and fixed-edge histogram counts, never raw frame streams.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
mod hardware;
|
||||
|
||||
pub use hardware::{cpu_seconds, load_average, power_source, rss_bytes, Display, Gpu, Hardware};
|
||||
|
||||
pub const FORMAT: &str = "horizon-benchmark-report";
|
||||
pub const VERSION: u32 = 1;
|
||||
|
||||
/// Lower edges of the frame-time histogram buckets, in milliseconds; the last bucket is 1000 ms and
|
||||
/// over. Fixed by the format version so histograms from any number of reports add up.
|
||||
pub const HISTOGRAM_EDGES_MS: [f64; 24] = [
|
||||
0.0, 1.0, 2.0, 3.0, 4.17, 5.0, 6.0, 7.0, 8.33, 10.0, 12.0, 14.0, 16.67, 20.0, 25.0, 33.33,
|
||||
40.0, 50.0, 66.67, 100.0, 150.0, 250.0, 500.0, 1000.0,
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Kind {
|
||||
Game,
|
||||
Launcher,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct App {
|
||||
pub name: String,
|
||||
pub version: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub commit: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub mods: Vec<Mod>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Mod {
|
||||
pub name: String,
|
||||
pub version: String,
|
||||
}
|
||||
|
||||
/// What has to match for two reports to be compared. Every field is optional.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Conditions {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scenario: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub render_distance: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub simulation_distance: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub window: Option<[u32; 2]>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub vsync: Option<bool>,
|
||||
/// Absent: unlimited.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub fps_cap: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub offscreen: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repeats: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub warmup_s: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub power: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub load_avg_1m: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub backend: Option<String>,
|
||||
}
|
||||
|
||||
/// A measurement over a run's repeats.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Summary {
|
||||
pub n: usize,
|
||||
pub mean: f64,
|
||||
/// Sample standard deviation; `None` for a single value.
|
||||
pub stdev: Option<f64>,
|
||||
pub min: f64,
|
||||
pub max: f64,
|
||||
/// Half-width of the 95% confidence interval of the mean (Student's t); `None` for a single value.
|
||||
pub ci95: Option<f64>,
|
||||
}
|
||||
|
||||
impl Summary {
|
||||
/// `None` for no values.
|
||||
pub fn of(values: &[f64]) -> Option<Summary> {
|
||||
let n = values.len();
|
||||
if n == 0 {
|
||||
return None;
|
||||
}
|
||||
let mean = values.iter().sum::<f64>() / n as f64;
|
||||
let (stdev, ci95) = if n > 1 {
|
||||
let var = values.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / (n - 1) as f64;
|
||||
let sd = var.sqrt();
|
||||
(Some(sd), Some(t95(n - 1) * sd / (n as f64).sqrt()))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
Some(Summary {
|
||||
n,
|
||||
mean: round(mean),
|
||||
stdev: stdev.map(round),
|
||||
min: round(values.iter().copied().fold(f64::INFINITY, f64::min)),
|
||||
max: round(values.iter().copied().fold(f64::NEG_INFINITY, f64::max)),
|
||||
ci95: ci95.map(round),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Two-sided 95% critical value of Student's t for `df` degrees of freedom.
|
||||
fn t95(df: usize) -> f64 {
|
||||
const TABLE: [f64; 30] = [
|
||||
12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160,
|
||||
2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056,
|
||||
2.052, 2.048, 2.045, 2.042,
|
||||
];
|
||||
TABLE.get(df.wrapping_sub(1)).copied().unwrap_or(1.96)
|
||||
}
|
||||
|
||||
/// One measured phase of one repeat.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Frames {
|
||||
pub phase: String,
|
||||
pub repeat: u32,
|
||||
pub frames: usize,
|
||||
pub seconds: f64,
|
||||
pub avg_fps: f64,
|
||||
pub low_1pct_fps: f64,
|
||||
pub low_01pct_fps: f64,
|
||||
pub p50_ms: f64,
|
||||
pub p95_ms: f64,
|
||||
pub p99_ms: f64,
|
||||
pub p999_ms: f64,
|
||||
pub max_ms: f64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cpu_p50_ms: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub gpu_p50_ms: Option<f64>,
|
||||
/// Frame counts per bucket of [`HISTOGRAM_EDGES_MS`].
|
||||
pub histogram: Vec<u64>,
|
||||
}
|
||||
|
||||
impl Frames {
|
||||
/// The figures of a phase from its frame times in milliseconds (time between the starts of
|
||||
/// consecutive frames); `None` for no frames. CPU and GPU times are per frame too, when measured.
|
||||
pub fn of(
|
||||
phase: &str,
|
||||
repeat: u32,
|
||||
frame_ms: &[f32],
|
||||
cpu_ms: &[f32],
|
||||
gpu_ms: &[f32],
|
||||
) -> Option<Frames> {
|
||||
let times = sorted(frame_ms);
|
||||
let n = times.len();
|
||||
if n == 0 {
|
||||
return None;
|
||||
}
|
||||
let total: f64 = times.iter().sum();
|
||||
// The average frame time of the slowest `share` of frames, as frames per second.
|
||||
let low = |share: f64| {
|
||||
let slowest = ×[n - ((n as f64 * share).ceil() as usize).max(1)..];
|
||||
1000.0 * slowest.len() as f64 / slowest.iter().sum::<f64>()
|
||||
};
|
||||
let mut histogram = vec![0; HISTOGRAM_EDGES_MS.len()];
|
||||
for &t in × {
|
||||
histogram[HISTOGRAM_EDGES_MS.partition_point(|&edge| edge <= t).max(1) - 1] += 1;
|
||||
}
|
||||
Some(Frames {
|
||||
phase: phase.to_owned(),
|
||||
repeat,
|
||||
frames: n,
|
||||
seconds: round(total / 1000.0),
|
||||
avg_fps: round(n as f64 * 1000.0 / total),
|
||||
low_1pct_fps: round(low(0.01)),
|
||||
low_01pct_fps: round(low(0.001)),
|
||||
p50_ms: round(percentile(×, 0.5)),
|
||||
p95_ms: round(percentile(×, 0.95)),
|
||||
p99_ms: round(percentile(×, 0.99)),
|
||||
p999_ms: round(percentile(×, 0.999)),
|
||||
max_ms: round(times[n - 1]),
|
||||
cpu_p50_ms: (!cpu_ms.is_empty()).then(|| round(percentile(&sorted(cpu_ms), 0.5))),
|
||||
gpu_p50_ms: (!gpu_ms.is_empty()).then(|| round(percentile(&sorted(gpu_ms), 0.5))),
|
||||
histogram,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn sorted(ms: &[f32]) -> Vec<f64> {
|
||||
let mut v: Vec<f64> = ms
|
||||
.iter()
|
||||
.map(|&m| f64::from(m))
|
||||
.filter(|m| m.is_finite())
|
||||
.collect();
|
||||
v.sort_by(f64::total_cmp);
|
||||
v
|
||||
}
|
||||
|
||||
/// Three decimals: plenty for milliseconds and FPS, short in JSON, and parsed back exactly.
|
||||
fn round(v: f64) -> f64 {
|
||||
(v * 1000.0).round() / 1000.0
|
||||
}
|
||||
|
||||
/// Nearest-rank percentile of sorted, non-empty values.
|
||||
fn percentile(sorted: &[f64], p: f64) -> f64 {
|
||||
let n = sorted.len();
|
||||
sorted[((n as f64 * p).ceil() as usize).clamp(1, n) - 1]
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Report {
|
||||
pub format: String,
|
||||
pub version: u32,
|
||||
pub kind: Kind,
|
||||
/// When the run ended, UTC, `YYYY-MM-DDTHH:MM:SSZ`.
|
||||
pub created: String,
|
||||
pub app: App,
|
||||
pub hardware: Hardware,
|
||||
#[serde(default)]
|
||||
pub conditions: Conditions,
|
||||
/// Scalar measurements by name; the unit is the name's suffix (`_ms`, `_mb`...).
|
||||
#[serde(default)]
|
||||
pub metrics: BTreeMap<String, Summary>,
|
||||
#[serde(default)]
|
||||
pub frames: Vec<Frames>,
|
||||
/// Local only: a name the person running it chose.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub label: Option<String>,
|
||||
/// Local only: logs the run wrote next to the report.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub local: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
impl Report {
|
||||
/// An empty report for this machine, created now.
|
||||
pub fn new(kind: Kind, app: App) -> Report {
|
||||
Report {
|
||||
format: FORMAT.to_owned(),
|
||||
version: VERSION,
|
||||
kind,
|
||||
created: utc_now(),
|
||||
app,
|
||||
hardware: Hardware::detect(),
|
||||
conditions: Conditions::default(),
|
||||
metrics: BTreeMap::new(),
|
||||
frames: Vec::new(),
|
||||
label: None,
|
||||
local: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a measurement over the repeats; no values records nothing.
|
||||
pub fn metric(&mut self, name: &str, values: &[f64]) {
|
||||
if let Some(summary) = Summary::of(values) {
|
||||
self.metrics.insert(name.to_owned(), summary);
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads a report, rejecting other formats and newer versions.
|
||||
pub fn parse(json: &str) -> Result<Report, String> {
|
||||
let report: Report = serde_json::from_str(json).map_err(|e| e.to_string())?;
|
||||
if report.format != FORMAT {
|
||||
return Err(format!(
|
||||
"not a benchmark report: format {:?}",
|
||||
report.format
|
||||
));
|
||||
}
|
||||
if report.version > VERSION {
|
||||
return Err(format!(
|
||||
"report version {} is newer than {VERSION}",
|
||||
report.version
|
||||
));
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// The report without its local-only fields, dated to the day: what an upload sends.
|
||||
pub fn for_upload(&self) -> Report {
|
||||
let day = self.created.get(..10).unwrap_or_default();
|
||||
Report {
|
||||
label: None,
|
||||
local: Vec::new(),
|
||||
created: format!("{day}T00:00:00Z"),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_json(&self) -> String {
|
||||
serde_json::to_string_pretty(self).expect("a report always serializes")
|
||||
}
|
||||
|
||||
/// Writes `<dir>/<kind>-<created>-<scenario>.json` and returns its path.
|
||||
pub fn write_to(&self, dir: &Path) -> io::Result<PathBuf> {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
let kind = match self.kind {
|
||||
Kind::Game => "game",
|
||||
Kind::Launcher => "launcher",
|
||||
};
|
||||
let stamp: String = self.created.chars().filter(char::is_ascii_digit).collect();
|
||||
let scenario: String = self
|
||||
.conditions
|
||||
.scenario
|
||||
.as_deref()
|
||||
.unwrap_or("run")
|
||||
.chars()
|
||||
.map(|c| {
|
||||
if c.is_ascii_alphanumeric() || c == '-' {
|
||||
c
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let path = dir.join(format!("{kind}-{stamp}-{scenario}.json"));
|
||||
std::fs::write(&path, self.to_json() + "\n")?;
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
|
||||
/// The current time, UTC, `YYYY-MM-DDTHH:MM:SSZ`.
|
||||
pub fn utc_now() -> String {
|
||||
let secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_secs());
|
||||
utc(secs)
|
||||
}
|
||||
|
||||
fn utc(secs: u64) -> String {
|
||||
let (days, rem) = (secs / 86_400, secs % 86_400);
|
||||
// Civil date from days since 1970-01-01 (Howard Hinnant's algorithm).
|
||||
let z = days as i64 + 719_468;
|
||||
let era = z.div_euclid(146_097);
|
||||
let doe = z - era * 146_097;
|
||||
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let day = doy - (153 * mp + 2) / 5 + 1;
|
||||
let month = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let year = yoe + era * 400 + i64::from(month <= 2);
|
||||
format!(
|
||||
"{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z",
|
||||
rem / 3600,
|
||||
rem / 60 % 60,
|
||||
rem % 60
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn utc_dates() {
|
||||
assert_eq!(utc(0), "1970-01-01T00:00:00Z");
|
||||
assert_eq!(utc(951_782_400), "2000-02-29T00:00:00Z");
|
||||
assert_eq!(utc(1_791_983_007), "2026-10-14T13:03:27Z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summary_of_repeats() {
|
||||
assert_eq!(Summary::of(&[]), None);
|
||||
let one = Summary::of(&[5.0]).unwrap();
|
||||
assert_eq!((one.n, one.mean, one.stdev, one.ci95), (1, 5.0, None, None));
|
||||
let s = Summary::of(&[2.0, 4.0, 6.0]).unwrap();
|
||||
assert_eq!((s.mean, s.min, s.max), (4.0, 2.0, 6.0));
|
||||
assert_eq!(s.stdev, Some(2.0));
|
||||
assert_eq!(s.ci95, Some(4.969), "4.303 * 2 / sqrt(3)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_figures_and_histogram() {
|
||||
// 998 frames of 2 ms, one of 20 ms, one of 1500 ms.
|
||||
let mut ms = vec![2.0f32; 998];
|
||||
ms.extend([20.0, 1500.0]);
|
||||
let f = Frames::of("flight", 0, &ms, &[], &[]).unwrap();
|
||||
assert_eq!(f.frames, 1000);
|
||||
assert_eq!(f.seconds, 3.516);
|
||||
assert_eq!((f.p50_ms, f.p999_ms, f.max_ms), (2.0, 20.0, 1500.0));
|
||||
// Slowest 1% = 10 frames: 8 of 2 ms, 20 ms, 1500 ms.
|
||||
assert_eq!(f.low_1pct_fps, 6.51, "1000 * 10 / 1536");
|
||||
// Slowest 0.1% = the 1500 ms frame.
|
||||
assert_eq!(f.low_01pct_fps, 0.667);
|
||||
assert_eq!(f.histogram.len(), HISTOGRAM_EDGES_MS.len());
|
||||
assert_eq!(f.histogram[2], 998, "2 ms falls in [2, 3)");
|
||||
assert_eq!(f.histogram[13], 1, "20 ms falls in [20, 25)");
|
||||
assert_eq!(f.histogram[23], 1, "1500 ms falls in [1000, inf)");
|
||||
assert_eq!(f.histogram.iter().sum::<u64>(), 1000);
|
||||
assert_eq!((f.cpu_p50_ms, f.gpu_p50_ms), (None, None));
|
||||
assert_eq!(Frames::of("x", 0, &[], &[], &[]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_and_upload_strips_local_fields() {
|
||||
let mut r = Report::new(
|
||||
Kind::Game,
|
||||
App {
|
||||
name: "horizon-client".into(),
|
||||
version: "0.1.0".into(),
|
||||
..App::default()
|
||||
},
|
||||
);
|
||||
r.conditions.scenario = Some("flight".into());
|
||||
r.metric("startup.title_ms", &[800.0, 820.0]);
|
||||
r.frames
|
||||
.extend(Frames::of("village", 0, &[2.0, 3.0], &[1.0], &[]));
|
||||
r.label = Some("my run".into());
|
||||
r.local.push("/home/someone/logs/latest.log".into());
|
||||
let back = Report::parse(&r.to_json()).unwrap();
|
||||
assert_eq!(back, r);
|
||||
r.created = "2026-10-15T14:03:27Z".into();
|
||||
let upload = r.for_upload();
|
||||
assert_eq!(upload.created, "2026-10-15T00:00:00Z", "the day only");
|
||||
let upload = upload.to_json();
|
||||
assert!(
|
||||
!upload.contains("my run") && !upload.contains("someone"),
|
||||
"{upload}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_other_formats_and_newer_versions() {
|
||||
let r = Report::new(Kind::Launcher, App::default());
|
||||
let newer = r.to_json().replace("\"version\": 1", "\"version\": 2");
|
||||
assert!(Report::parse(&newer).unwrap_err().contains("newer"));
|
||||
let other = r.to_json().replace(FORMAT, "something-else");
|
||||
assert!(Report::parse(&other).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writes_into_a_folder() {
|
||||
let dir = std::env::temp_dir().join(format!("horizon-bench-test-{}", std::process::id()));
|
||||
let mut r = Report::new(Kind::Launcher, App::default());
|
||||
r.created = "2026-10-15T14:03:27Z".into();
|
||||
r.conditions.scenario = Some("cold start".into());
|
||||
let path = r.write_to(&dir).unwrap();
|
||||
assert_eq!(
|
||||
path.file_name().unwrap(),
|
||||
"launcher-20261015140327-cold_start.json"
|
||||
);
|
||||
assert_eq!(
|
||||
Report::parse(&std::fs::read_to_string(&path).unwrap()).unwrap(),
|
||||
r
|
||||
);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ To play, use [the launcher](../launcher/README.md); the project site is [playhor
|
||||
- [Extras](extras.md): optional zoom, minimap, FPS and keystroke overlays and auto-reconnect, all off by default.
|
||||
- [Launcher](../launcher/README.md): installs, updates and starts builds; its CLI and MCP server; releasing it.
|
||||
- [Opt-in data collection](telemetry.md): the consent, exactly what is sent, how it is stored and sent.
|
||||
- [Benchmark report format](benchmark-report.md): the versioned JSON every benchmark writes (game, launcher, `tools/perf_bench.py`), the local log and, with consent, the upload.
|
||||
- Menus: the title, world list, create world, multiplayer, options and pause screens share the launcher's design tokens and fonts (`crates/horizon-theme`), drawn by `ui_theme` over a live frozen backdrop (`game/backdrop.rs`, blurred and graded by `post.rs`). Options → Interface → Menu style switches to vanilla's screens (Classic, kept in `horizon.txt`; `HORIZON_MENU_STYLE=horizon|classic` overrides it, and scripted runs default to Classic). [Title](media/menu-title.png) ([webm](media/menu-title.webm)), [worlds](media/menu-worlds.png), [options](media/menu-options.png), [pause](media/menu-pause.png).
|
||||
|
||||
## Parity
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# Benchmark report format (version 1)
|
||||
|
||||
Every benchmark Horizon writes uses this one JSON format: `horizon-client --benchmark`, the launcher's own
|
||||
measurements, and `tools/perf_bench.py` (which also writes it for vanilla and vanilla with mods). The same
|
||||
file is the local log a player can attach to an issue and, only with the player's consent, what the
|
||||
telemetry upload sends (minus the local-only fields below). The types, the statistics and the hardware
|
||||
detection are in `crates/horizon-bench`; nothing else builds a report by hand.
|
||||
|
||||
A report holds summaries and histogram counts, never raw frame streams: a 10-minute run is a few
|
||||
kilobytes whatever the frame rate. Measured numbers are rounded to 3 decimals.
|
||||
|
||||
## Example
|
||||
|
||||
```json
|
||||
{
|
||||
"format": "horizon-benchmark-report",
|
||||
"version": 1,
|
||||
"kind": "game",
|
||||
"created": "2026-10-15T14:03:27Z",
|
||||
"app": { "name": "horizon-client", "version": "0.21.0", "commit": "e96e1571" },
|
||||
"hardware": {
|
||||
"os": "macos", "os_version": "27.0", "arch": "aarch64",
|
||||
"cpu": "Apple M4 Pro", "cpu_threads": 14, "ram_mb": 65536,
|
||||
"gpu": { "name": "Apple M4 Pro", "vendor": "apple", "driver": "", "backend": "metal", "kind": "integrated" },
|
||||
"display": { "width": 2560, "height": 1440, "scale": 2.0, "refresh_hz": 60 }
|
||||
},
|
||||
"conditions": {
|
||||
"scenario": "flight", "seed": "1234", "render_distance": 16, "simulation_distance": 12,
|
||||
"window": [1280, 720], "vsync": false, "repeats": 3, "warmup_s": 2.0, "power": "ac", "load_avg_1m": 1.2
|
||||
},
|
||||
"metrics": {
|
||||
"startup.title_ms": { "n": 3, "mean": 812.0, "stdev": 21.5, "min": 790.0, "max": 833.0, "ci95": 53.4 },
|
||||
"memory.rss_peak_mb": { "n": 3, "mean": 1210.0, "stdev": 4.0, "min": 1206.0, "max": 1214.0, "ci95": 9.9 }
|
||||
},
|
||||
"frames": [
|
||||
{
|
||||
"phase": "village", "repeat": 0, "frames": 4920, "seconds": 12.0,
|
||||
"avg_fps": 410.0, "low_1pct_fps": 120.3, "low_01pct_fps": 61.0,
|
||||
"p50_ms": 2.31, "p95_ms": 3.8, "p99_ms": 6.9, "p999_ms": 14.2, "max_ms": 21.0,
|
||||
"cpu_p50_ms": 1.4, "gpu_p50_ms": 1.9,
|
||||
"histogram": [0, 0, 3011, 1520, 301, 62, 15, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Fields
|
||||
|
||||
**`format`, `version`.** Always `"horizon-benchmark-report"` and `1`. A reader rejects other formats and
|
||||
newer versions. Adding an optional field keeps the version; renaming, removing or changing the meaning of
|
||||
one (histogram edges included) makes it 2.
|
||||
|
||||
**`kind`.** `"game"` (a game run: Horizon, or vanilla from `tools/perf_bench.py`) or `"launcher"`.
|
||||
|
||||
**`created`.** When the run ended, UTC, `YYYY-MM-DDTHH:MM:SSZ`.
|
||||
|
||||
**`app`.** What was measured: `name` (`horizon-client`, `horizon-launcher`, `minecraft`), `version`,
|
||||
optionally `commit` (Horizon builds) and `mods` (`[{ "name", "version" }]`, vanilla with mods).
|
||||
|
||||
**`hardware`.** `os` (`macos`, `linux`, `windows`), `os_version`, `arch`, `cpu` (brand string),
|
||||
`cpu_threads`, `ram_mb`; `gpu` (`name`, `vendor`, `driver`, `backend`: `metal`, `vulkan`, `dx12`, `gl`;
|
||||
`kind`: `integrated`, `discrete`, `virtual`, `cpu`, `other`) and `display` (the screen the window was on:
|
||||
`width`, `height` in physical pixels, `scale`, `refresh_hz`) when known. Any field can be missing.
|
||||
|
||||
**`conditions`.** What has to match for two reports to be compared, all optional: `scenario`, `seed`,
|
||||
`render_distance`, `simulation_distance`, `window` (`[width, height]` actually rendered), `vsync`,
|
||||
`fps_cap` (absent: unlimited), `offscreen` (rendered without presenting: not a player number), `repeats`,
|
||||
`warmup_s`, `power` (`ac`, `battery`), `load_avg_1m` (before the run; above about 1 the machine was busy),
|
||||
`backend` (when forced). Readers compare reports only with equal `scenario`, distances, `window`, `vsync`,
|
||||
`fps_cap` and `offscreen`.
|
||||
|
||||
**`metrics`.** Scalar measurements, by name; each is a summary over the run's repeats: `n`, `mean`,
|
||||
`stdev` (sample standard deviation, null when `n` is 1), `min`, `max`, `ci95` (half-width of the 95%
|
||||
confidence interval of the mean, Student's t, null when `n` is 1). The unit is the name's suffix: `_ms`,
|
||||
`_s`, `_mb`, `_bytes`, `_fps`, `_pct`, `_mspt`, `_w`, `_c` (degrees Celsius), `_count`. Names in use:
|
||||
|
||||
| Name | What |
|
||||
| --- | --- |
|
||||
| `startup.title_ms` | process start to the first frame of the title screen |
|
||||
| `startup.world_ms` | process start to in-world, every chunk around the spawn loaded and drawn |
|
||||
| `chunks.<phase>.load_ms` | teleport to every chunk in view distance loaded and drawn |
|
||||
| `chunks.<phase>.generated_count`, `chunks.<phase>.meshed_count`, `chunks.<phase>.generated_per_s` | chunks generated and section meshes built while loading, and the rate |
|
||||
| `fps.<phase>.avg_fps`, `fps.<phase>.low_1pct_fps`, `fps.<phase>.low_01pct_fps` | per-phase figures across repeats |
|
||||
| `tick.mspt`, `tick.tps` | integrated server tick time and rate (planned) |
|
||||
| `memory.rss_peak_mb`, `memory.rss_steady_mb` | resident memory: peak over the run, median while flying |
|
||||
| `memory.gpu_mb`, `memory.<phase>.gpu_mb` | video memory the game allocated (or NVIDIA attributes to it), where readable |
|
||||
| `power.package_w`, `power.battery_drain_pct` | power draw, battery used per hour (laptops; planned) |
|
||||
| `thermal.cpu_c`, `thermal.throttled_pct` | temperature, share of the run spent throttled (planned) |
|
||||
| `launcher.start_ms` | launcher process start to its first frame (a session) |
|
||||
| `launcher.cold_start_ms`, `launcher.warm_start_ms` | the same in `horizon-launcher benchmark`: its first start, then the others |
|
||||
| `launcher.rss_mb`, `launcher.cpu_idle_pct` | launcher resident memory (sampled; `max` is the peak), CPU used while idle |
|
||||
| `launcher.release_list_ms` | fetching the release list |
|
||||
| `launcher.install_ms`, `launcher.update_ms` | downloading and unpacking a version: a new one, an update |
|
||||
| `launcher.prepare_ms` | preparing a version's resources |
|
||||
| `launcher.play_to_game_frame_ms` | Play pressed to the game's first frame |
|
||||
| `launcher.binary_bytes`, `game.binary_bytes` | size of the executables |
|
||||
|
||||
New names can be added without a new version; the table lists the ones the tools write.
|
||||
|
||||
**`frames`.** One entry per measured phase and repeat (a scene's flight, the launcher's page switching):
|
||||
`phase`, `repeat`, `frames`, `seconds`, `avg_fps`, `low_1pct_fps` and `low_01pct_fps` (the average frame
|
||||
time of the slowest 1% or 0.1% of frames, as FPS), `p50_ms`, `p95_ms`, `p99_ms`, `p999_ms`, `max_ms`,
|
||||
`cpu_p50_ms` and `gpu_p50_ms` when measured, and `histogram`: frame counts per bucket. The bucket edges
|
||||
are fixed by the version, so histograms from any number of reports add up:
|
||||
|
||||
0, 1, 2, 3, 4.17, 5, 6, 7, 8.33, 10, 12, 14, 16.67, 20, 25, 33.33, 40, 50, 66.67, 100, 150, 250, 500, 1000 ms
|
||||
|
||||
Bucket *i* counts frames from edge *i* (included) to edge *i* + 1; the last bucket is 1000 ms and over
|
||||
(24 buckets). 4.17, 8.33, 16.67 and 33.33 ms are 240, 120, 60 and 30 FPS.
|
||||
|
||||
## Local only
|
||||
|
||||
`label` (a name the person running it chose) and `local` (paths of the logs the run wrote next to the
|
||||
report) stay on the machine: the upload removes them and dates the report to the day only, as every
|
||||
telemetry item is (`Report::for_upload`). The upload is `horizon-telemetry`'s `benchmark` item: this
|
||||
report, as it is, only with the player's consent to benchmark uploads (docs/telemetry.md). Nothing in a report
|
||||
identifies the player, the machine's name or the world: no host name, user name, path or address.
|
||||
|
||||
## Where reports go
|
||||
|
||||
- The game: `benchmarks/<kind>-<created>-<scenario>.json` in the game folder.
|
||||
- The launcher: `benchmarks/` in the launcher's data folder.
|
||||
- `tools/perf_bench.py`: one report per game and render distance, `benchmarks/perf/<label>-rd<N>.json` in the
|
||||
repository checkout.
|
||||
Reference in New Issue
Block a user