Files
horizon-airdef/crates/horizon-telemetry/src/payload.rs
T
Ludovic Marie 43138159b2 Opt-in data collection (1/5): consent, payloads, outbox and a gated uploader
horizon-telemetry, shared by the launcher, the game and the ingest server:
- consent.json in the game folder: four switches, all off until the player
  answers, a consent version that re-asks when what is collected changes, the
  answer's time, and a random install ID made at the first yes.
- Payloads: perf summaries (percentiles + 14 frame-time buckets from a
  fixed-size Timings recorder), crash and bug reports, and benchmark reports
  carried as docs/benchmark-report.md defines them; size caps per kind.
- A scrubber for user folders, user names, e-mail and IP addresses, UUIDs and
  caller-known private terms.
- The outbox: only allowed kinds get in, daily caps, 200 items / 2 MiB.
- zstd batches with bounded decoding; the uploader re-reads the consent and
  drops withdrawn items before connecting, backs off and caps batches a day.
- tests/consent_gating.rs: a local mock server sees zero connections with the
  consent missing, outdated or withdrawn.

No server is set (DEFAULT_ENDPOINT is None): nothing is sent yet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 02:21:13 +02:00

422 lines
13 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! What an item holds. Summaries only: a session's frame times become a few percentiles and
//! [`BUCKET_EDGES_MS`] counts; benchmark reports are the documented report
//! (`docs/benchmark-report.md`) as it is.
use serde::{Deserialize, Serialize};
/// The four kinds of data a player can allow, one switch each.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Kind {
Perf,
Crash,
Bug,
Benchmark,
}
impl Kind {
pub const ALL: [Kind; 4] = [Kind::Perf, Kind::Crash, Kind::Bug, Kind::Benchmark];
pub fn name(self) -> &'static str {
match self {
Kind::Perf => "perf",
Kind::Crash => "crash",
Kind::Bug => "bug",
Kind::Benchmark => "benchmark",
}
}
pub fn from_name(name: &str) -> Option<Kind> {
Kind::ALL.into_iter().find(|k| k.name() == name)
}
/// The largest an item of this kind may be, as JSON.
pub fn max_bytes(self) -> usize {
match self {
Kind::Perf => 4 * 1024,
Kind::Crash => 32 * 1024,
Kind::Bug => 80 * 1024,
Kind::Benchmark => 128 * 1024,
}
}
/// How many items of this kind one install keeps a day; the rest are dropped.
pub fn daily_cap(self) -> u32 {
match self {
Kind::Perf => 12,
Kind::Crash => 5,
Kind::Bug => 5,
Kind::Benchmark => 10,
}
}
}
/// What made the item.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Source {
Launcher,
Game,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Item {
/// Days since 1970 (UTC): no finer time is kept.
pub day: u32,
pub source: Source,
/// The Horizon release (`v0.4.2`), or the launcher's version.
pub version: String,
#[serde(flatten)]
pub payload: Payload,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", content = "data", rename_all = "snake_case")]
pub enum Payload {
Perf(Perf),
Crash(Crash),
Bug(Bug),
/// A benchmark report as `docs/benchmark-report.md` defines it, with its own schema field.
Benchmark(serde_json::Value),
}
impl Item {
pub fn new(source: Source, version: impl Into<String>, payload: Payload) -> Self {
Self {
day: crate::today(),
source,
version: version.into(),
payload,
}
}
pub fn kind(&self) -> Kind {
match self.payload {
Payload::Perf(_) => Kind::Perf,
Payload::Crash(_) => Kind::Crash,
Payload::Bug(_) => Kind::Bug,
Payload::Benchmark(_) => Kind::Benchmark,
}
}
/// Checks what the client and the server both refuse: an item over its kind's size, an
/// overlong text, a benchmark report that is not an object.
pub fn validate(&self) -> Result<(), String> {
let size = serde_json::to_vec(self).map_or(usize::MAX, |j| j.len());
let kind = self.kind();
if size > kind.max_bytes() {
return Err(format!(
"a {} item of {size} bytes is over its {} bytes",
kind.name(),
kind.max_bytes()
));
}
if self.version.len() > 64 {
return Err("the version is too long".into());
}
match &self.payload {
Payload::Bug(bug) if bug.text.trim().is_empty() => {
Err("a bug report needs text".into())
}
Payload::Bug(bug) if bug.text.chars().count() > Bug::MAX_TEXT => Err(format!(
"a bug report is at most {} characters",
Bug::MAX_TEXT
)),
Payload::Benchmark(report) if !report.is_object() => {
Err("a benchmark report is a JSON object".into())
}
_ => Ok(()),
}
}
}
/// The machine, as coarse as is useful: no serial numbers, names or exact memory.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct System {
/// `linux`, `macos` or `windows`.
pub os: String,
/// The distribution or system version (`macOS 26.0`, `Fedora Linux 43`).
pub os_version: String,
pub arch: String,
pub cpu: String,
pub threads: u16,
/// Installed memory, rounded to the GiB.
pub ram_gb: u16,
pub gpu: String,
pub driver: String,
/// `Metal`, `Vulkan`, `Dx12`…
pub backend: String,
}
/// The settings that change performance.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct Settings {
pub render_distance: u8,
pub simulation_distance: u8,
/// 0: unlimited.
pub max_fps: u16,
pub vsync: bool,
pub fullscreen: bool,
/// `fast`, `fancy` or `fabulous`.
pub graphics: String,
}
/// One play session's performance.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct Perf {
pub system: System,
pub settings: Settings,
/// Seconds of play measured (menus and loading screens excluded).
pub seconds: u32,
pub frames: Summary,
/// The integrated server's milliseconds per tick, in single player.
#[serde(skip_serializing_if = "Option::is_none")]
pub ticks: Option<Summary>,
/// The game's peak resident memory.
pub peak_ram_mb: u32,
/// From start to the title screen.
#[serde(skip_serializing_if = "Option::is_none")]
pub to_title_ms: Option<u32>,
/// From choosing a world to playing in it, the first time this session.
#[serde(skip_serializing_if = "Option::is_none")]
pub to_world_ms: Option<u32>,
}
/// A crash, scrubbed: the error and its stack, the thread, and the details that say what the
/// game was doing (graphics adapter, settings), never its log, world or server.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct Crash {
pub system: System,
pub error: String,
/// One frame a line, `function (file:line:col)`.
pub stack: String,
pub thread: String,
pub details: Vec<(String, String)>,
}
/// A bug report the player wrote, with the log only if they chose to add it after reading it.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct Bug {
pub system: System,
pub text: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub log: Option<String>,
}
impl Bug {
pub const MAX_TEXT: usize = 4000;
}
/// Durations at a glance: percentiles and how many fell in each of [`BUCKET_EDGES_MS`].
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct Summary {
pub count: u64,
pub p50_ms: f32,
pub p95_ms: f32,
pub p99_ms: f32,
pub max_ms: f32,
/// One count a bucket: under 4 ms, 4–6 ms, … 100–250 ms, 250 ms and more.
pub buckets: Vec<u32>,
}
/// The upper edges of [`Summary::buckets`] in milliseconds, around the usual refresh rates
/// (240, 165, 144, 120, 60, 30 Hz…); the last bucket has no edge.
pub const BUCKET_EDGES_MS: [f32; 13] = [
4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.7, 20.0, 25.0, 33.3, 50.0, 100.0, 250.0,
];
/// Records durations in tenths of a millisecond up to [`Timings::FINE_MS`] (beyond, they count
/// at the top) for percentiles finer than the buckets; 20 KB whatever the session's length.
#[derive(Clone, Debug)]
pub struct Timings {
tenths: Vec<u32>,
count: u64,
max_ms: f32,
}
impl Default for Timings {
fn default() -> Self {
Self::new()
}
}
impl Timings {
pub const FINE_MS: usize = 500;
pub fn new() -> Self {
Self {
tenths: vec![0; Self::FINE_MS * 10 + 1],
count: 0,
max_ms: 0.0,
}
}
pub fn record(&mut self, ms: f32) {
if !ms.is_finite() || ms < 0.0 {
return;
}
let bin = ((ms * 10.0) as usize).min(self.tenths.len() - 1);
self.tenths[bin] = self.tenths[bin].saturating_add(1);
self.count += 1;
self.max_ms = self.max_ms.max(ms);
}
pub fn count(&self) -> u64 {
self.count
}
/// The duration under which a fraction `p` of them fell, to the tenth of a millisecond.
fn percentile(&self, p: f64) -> f32 {
let rank = ((self.count as f64 * p).ceil() as u64).max(1);
let mut seen = 0u64;
for (bin, &n) in self.tenths.iter().enumerate() {
seen += u64::from(n);
if seen >= rank {
return if bin + 1 == self.tenths.len() {
self.max_ms
} else {
((bin + 1) as f32 / 10.0).min(self.max_ms)
};
}
}
self.max_ms
}
pub fn summary(&self) -> Summary {
let mut buckets = vec![0u32; BUCKET_EDGES_MS.len() + 1];
for (bin, &n) in self.tenths.iter().enumerate() {
let ms = bin as f32 / 10.0;
let bucket = BUCKET_EDGES_MS
.iter()
.position(|&edge| ms < edge - 0.01)
.unwrap_or(BUCKET_EDGES_MS.len());
buckets[bucket] = buckets[bucket].saturating_add(n);
}
let round = |ms: f32| (ms * 10.0).round() / 10.0;
Summary {
count: self.count,
p50_ms: round(self.percentile(0.50)),
p95_ms: round(self.percentile(0.95)),
p99_ms: round(self.percentile(0.99)),
max_ms: round(self.max_ms),
buckets,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn items_are_tagged_by_kind_and_read_back() {
let item = Item::new(
Source::Game,
"v0.4.2",
Payload::Bug(Bug {
text: "Water flickers".into(),
..Bug::default()
}),
);
let json = serde_json::to_value(&item).unwrap();
assert_eq!(json["kind"], "bug");
assert_eq!(json["source"], "game");
assert_eq!(json["data"]["text"], "Water flickers");
assert_eq!(serde_json::from_value::<Item>(json).unwrap(), item);
}
#[test]
fn a_benchmark_report_is_carried_as_it_is() {
let report = serde_json::json!({"schema": "horizon-benchmark-report/1", "scenes": []});
let item = Item::new(
Source::Launcher,
"0.1.5",
Payload::Benchmark(report.clone()),
);
let json = serde_json::to_value(&item).unwrap();
assert_eq!(json["data"], report);
assert!(item.validate().is_ok());
let not_object = Item::new(Source::Launcher, "0.1.5", Payload::Benchmark(1.into()));
assert!(not_object.validate().is_err());
}
#[test]
fn oversized_items_are_refused() {
let bug = |text: String| {
Item::new(
Source::Game,
"v1",
Payload::Bug(Bug {
text,
..Bug::default()
}),
)
};
assert!(bug("x".repeat(Bug::MAX_TEXT)).validate().is_ok());
assert!(bug("x".repeat(Bug::MAX_TEXT + 1)).validate().is_err());
assert!(bug(" ".into()).validate().is_err());
let huge = Item::new(
Source::Game,
"v1",
Payload::Crash(Crash {
stack: "frame\n".repeat(10_000),
..Crash::default()
}),
);
assert!(huge.validate().unwrap_err().contains("over its"));
}
#[test]
fn timings_give_percentiles_and_buckets() {
let mut t = Timings::new();
// 90 frames at 16.6 ms (60 Hz), 9 at 30 ms, one 400 ms hitch.
(0..90).for_each(|_| t.record(16.6));
(0..9).for_each(|_| t.record(30.0));
t.record(400.0);
t.record(f32::NAN);
let s = t.summary();
assert_eq!(s.count, 100);
assert_eq!(s.p50_ms, 16.7);
assert_eq!(s.p95_ms, 30.1);
assert_eq!(s.p99_ms, 30.1);
assert_eq!(s.max_ms, 400.0);
assert_eq!(s.buckets.len(), 14);
assert_eq!(s.buckets[6], 90, "14–16.7 ms");
assert_eq!(s.buckets[9], 9, "25–33.3 ms");
assert_eq!(s.buckets[13], 1, "250 ms and more");
assert_eq!(s.buckets.iter().sum::<u32>(), 100);
}
#[test]
fn hitches_beyond_the_fine_range_keep_their_maximum() {
let mut t = Timings::new();
t.record(2_000.0);
let s = t.summary();
assert_eq!((s.p50_ms, s.max_ms), (2_000.0, 2_000.0));
assert_eq!(s.buckets[13], 1);
}
#[test]
fn a_session_summary_is_small() {
let mut t = Timings::new();
(0..200_000).for_each(|i| t.record(5.0 + (i % 300) as f32 / 10.0));
let perf = Item::new(
Source::Game,
"v0.4.2",
Payload::Perf(Perf {
frames: t.summary(),
ticks: Some(t.summary()),
..Perf::default()
}),
);
let size = serde_json::to_vec(&perf).unwrap().len();
assert!(size < 1024, "{size} bytes");
assert!(perf.validate().is_ok());
}
}