Merge pull request #251 from ludovic111/feat/telemetry-server

Opt-in data collection (4/5): the ingest server (SQLite, rollups, retention, aggregate endpoint)
This commit is contained in:
Ludovic Marie
2026-10-15 03:04:31 +02:00
committed by GitHub
12 changed files with 2413 additions and 2 deletions
+34
View File
@@ -0,0 +1,34 @@
name: telemetry-server
# The ingest server is its own workspace (crates/horizon-telemetry-server), so the checks job doesn't build
# it: this one does, when it or the crate it shares with the launcher and the game changes.
on:
push:
branches: [main]
paths:
- 'crates/horizon-telemetry/**'
- 'crates/horizon-telemetry-server/**'
- '.github/workflows/telemetry-server.yml'
pull_request:
paths:
- 'crates/horizon-telemetry/**'
- 'crates/horizon-telemetry-server/**'
- '.github/workflows/telemetry-server.yml'
permissions:
contents: read
jobs:
check:
runs-on: ubuntu-latest
timeout-minutes: 20
defaults:
run:
working-directory: crates/horizon-telemetry-server
steps:
- uses: actions/checkout@v4
- run: rustup toolchain install stable --profile minimal --component rustfmt --component clippy
- uses: Swatinem/rust-cache@v2
with:
workspaces: crates/horizon-telemetry-server
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo +stable fmt --check
- run: cargo +stable clippy --all-targets --locked -- -D warnings
- run: cargo +stable test --locked
+1 -1
View File
@@ -1,5 +1,5 @@
[workspace]
exclude = ["launcher"]
exclude = ["launcher", "crates/horizon-telemetry-server"]
members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "xtask"]
resolver = "2"
@@ -0,0 +1 @@
/target/
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,31 @@
[package]
name = "horizon-telemetry-server"
version = "0.1.0"
edition = "2024"
license = "MIT"
rust-version = "1.90"
publish = false
description = "Receives opt-in data collection batches, keeps them compactly in SQLite and serves anonymous totals"
# Its own workspace, as the launcher's: the game's workspace never builds a web server.
[workspace]
[dependencies]
horizon-telemetry = { path = "../horizon-telemetry", default-features = false }
axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json", "query"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "signal", "time"] }
# SQLite built in: one file on a small volume is the whole database.
rusqlite = { version = "0.37", features = ["bundled"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
ruzstd = "0.9"
[dev-dependencies]
# The uploader the launcher uses, against this server, end to end.
horizon-telemetry = { path = "../horizon-telemetry" }
tower = { version = "0.5", features = ["util"] }
http-body-util = "0.1"
[profile.release]
strip = true
lto = "thin"
@@ -0,0 +1,13 @@
# Built from the repository root: docker build -f crates/horizon-telemetry-server/Dockerfile .
FROM rust:1-slim AS build
WORKDIR /src
# The root manifest only for the shared crate's workspace settings (edition, lints).
COPY Cargo.toml Cargo.toml
COPY crates/horizon-telemetry crates/horizon-telemetry
COPY crates/horizon-telemetry-server crates/horizon-telemetry-server
RUN cargo build --release --locked --manifest-path crates/horizon-telemetry-server/Cargo.toml
FROM debian:stable-slim
COPY --from=build /src/crates/horizon-telemetry-server/target/release/horizon-telemetry-server /usr/local/bin/
ENV HORIZON_TELEMETRY_DB=/data/telemetry.sqlite3 HORIZON_TELEMETRY_TRUST_PROXY=1
CMD ["horizon-telemetry-server"]
+58
View File
@@ -0,0 +1,58 @@
# horizon-telemetry-server
This server receives what players choose to send (performance summaries, crash and bug reports, benchmark uploads: [docs/telemetry.md](../../docs/telemetry.md)) and stores it in one SQLite file. It serves anonymous totals to the website. It is one small Rust process, about 7 MB of memory when idle.
cargo run --release # in this folder: its own workspace, like the launcher's
PORT=8080 HORIZON_TELEMETRY_DB=/data/telemetry.sqlite3 horizon-telemetry-server
The launcher sends to it when `HORIZON_TELEMETRY_URL` is set (and to `DEFAULT_ENDPOINT` in `crates/horizon-telemetry` once there is one).
## Endpoints
| | |
|---|---|
| `POST /v1/batch` | A zstd batch (`Content-Encoding: zstd`, at most 256 KiB, 1 MiB inflated, 64 items). Returns 204. Returns 400 for a bad batch, 413 for one too large, 415 if it is not zstd, 429 when over a limit. |
| `DELETE /v1/installs/<id>` | Deletes everything that install sent. Its share of the daily totals is rebuilt at once. |
| `GET /v1/aggregate?days=30` | The totals the website reads. Cached 10 minutes, with `Access-Control-Allow-Origin` set to the website. |
| `GET /healthz` | `ok` |
The totals are grouped per version, OS, GPU vendor and render distance. Each group has its sessions, install-days, hours played, the 14 frame-time buckets summed, and the mean p50 and p99 frame times. Groups with fewer than 5 install-days are left out. The totals also list crash counts per version and the newest 50 benchmark uploads.
## Storage
- `systems`: each distinct machine description once. Sessions point at it.
- `perf`: one row per session, in columns, its frame and tick summaries as 80-byte blobs.
- `reports`: crash and bug reports and benchmarks, their data compressed with zstd.
- `perf_daily`: the totals per day, rebuilt from `perf` when a day changes (new sessions, a deletion).
Measured: **about 190 bytes per play session** on disk, indexes included (`tests/server.rs`, 10 000 sessions). 1 000 players playing twice a day for 180 days is about 70 MB.
The day stored is the day the server received an item. The sender's IP address is never written down. The per-address rate limit keeps addresses in memory for an hour at most.
Raw rows are deleted after `HORIZON_TELEMETRY_RETENTION_DAYS` (180). The daily totals stay; they hold no install ID.
## Settings
All are environment variables:
- `PORT`
- `HORIZON_TELEMETRY_DB`
- `HORIZON_TELEMETRY_RETENTION_DAYS` (180)
- `HORIZON_TELEMETRY_MIN_INSTALL_DAYS` (5)
- `HORIZON_TELEMETRY_REQUESTS_PER_HOUR` per address (60, bursts of 6)
- `HORIZON_TELEMETRY_ITEMS_PER_DAY` per install (100)
- `HORIZON_TELEMETRY_TRUST_PROXY=1` behind a proxy (the client's address is then the first of `X-Forwarded-For`)
- `HORIZON_TELEMETRY_ALLOW_ORIGIN` (`https://playhorizon.dev`)
## Hosting
`Dockerfile` builds it from the repository root (`docker build -f crates/horizon-telemetry-server/Dockerfile .`). The database goes on a volume mounted at `/data`.
At Railway's prices (RAM $10/GB-month, CPU $20/vCPU-month, volumes $0.15/GB-month, egress $0.05/GB), one more service with a 1 GB volume uses about $0.35 a month:
- RAM about 15 MB: $0.15
- CPU nearly idle: under $0.05
- the volume: $0.15
- egress, the cached totals: cents
That fits inside the Hobby plan's included $5 of usage. Railway Postgres would cost a few dollars a month for its memory alone and buys nothing at this size. A bucket isn't needed: crash reports are a few KB compressed and live in the same file.
+307
View File
@@ -0,0 +1,307 @@
//! The ingest server for opt-in data collection (docs/telemetry.md).
//!
//! - `POST /v1/batch`: a zstd batch from the launcher, checked by `horizon_telemetry::batch`.
//! - `DELETE /v1/installs/<id>`: everything an install sent goes.
//! - `GET /v1/aggregate?days=30`: the anonymous totals the website reads, cached.
//! - `GET /healthz`.
//!
//! Abuse limits: a token bucket per IP address (kept in memory for an hour, never stored), a
//! daily item cap per install, and the batch size limits. Raw rows expire after
//! `retention_days`; the daily totals stay. One small process and one SQLite file: see
//! `README.md` for running it.
use std::collections::HashMap;
use std::net::{IpAddr, SocketAddr};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use axum::Router;
use axum::extract::{ConnectInfo, Path, Query, Request, State};
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::{delete, get, post};
use horizon_telemetry::batch;
pub mod store;
use store::Store;
pub struct Config {
/// Days raw rows are kept.
pub retention_days: u32,
/// Groups in the public totals need at least this many install-days.
pub min_install_days: u64,
/// Requests an IP address may make an hour (with a burst of a tenth of it).
pub requests_per_hour: f64,
/// Items one install may send a day.
pub items_per_day: u32,
/// Behind a proxy (Railway, Fly…): the client's address is the first of X-Forwarded-For.
pub trust_proxy: bool,
/// The website allowed to read the totals from a browser.
pub allow_origin: String,
}
impl Default for Config {
fn default() -> Self {
Self {
retention_days: 180,
min_install_days: 5,
requests_per_hour: 60.0,
items_per_day: 100,
trust_proxy: false,
allow_origin: "https://playhorizon.dev".into(),
}
}
}
impl Config {
/// The defaults, with `HORIZON_TELEMETRY_RETENTION_DAYS`, `…_MIN_INSTALL_DAYS`,
/// `…_REQUESTS_PER_HOUR`, `…_ITEMS_PER_DAY`, `…_TRUST_PROXY` (1) and `…_ALLOW_ORIGIN`.
pub fn from_env() -> Self {
let var = |k: &str| std::env::var(format!("HORIZON_TELEMETRY_{k}")).ok();
let d = Self::default();
Self {
retention_days: var("RETENTION_DAYS")
.and_then(|v| v.parse().ok())
.unwrap_or(d.retention_days),
min_install_days: var("MIN_INSTALL_DAYS")
.and_then(|v| v.parse().ok())
.unwrap_or(d.min_install_days),
requests_per_hour: var("REQUESTS_PER_HOUR")
.and_then(|v| v.parse().ok())
.unwrap_or(d.requests_per_hour),
items_per_day: var("ITEMS_PER_DAY")
.and_then(|v| v.parse().ok())
.unwrap_or(d.items_per_day),
trust_proxy: var("TRUST_PROXY").is_some_and(|v| v == "1"),
allow_origin: var("ALLOW_ORIGIN").unwrap_or(d.allow_origin),
}
}
}
pub struct App {
store: Mutex<Store>,
config: Config,
/// Tokens left and when they were counted, per client address.
buckets: Mutex<HashMap<IpAddr, (f64, Instant)>>,
/// `/v1/aggregate` answers by `days`, with when they were made.
cache: Mutex<HashMap<u32, (Instant, String)>>,
/// Today, in days since 1970 (a test moves it).
today: Box<dyn Fn() -> u32 + Send + Sync>,
}
const CACHE_FOR: Duration = Duration::from_secs(600);
fn status(code: StatusCode, message: &str) -> Response {
(code, message.to_owned()).into_response()
}
impl App {
pub fn new(store: Store, config: Config) -> Arc<Self> {
Self::with_clock(store, config, Box::new(horizon_telemetry::today))
}
pub fn with_clock(
store: Store,
config: Config,
today: Box<dyn Fn() -> u32 + Send + Sync>,
) -> Arc<Self> {
Arc::new(Self {
store: Mutex::new(store),
config,
buckets: Mutex::default(),
cache: Mutex::default(),
today,
})
}
fn store(&self) -> std::sync::MutexGuard<'_, Store> {
self.store.lock().unwrap_or_else(|e| e.into_inner())
}
/// Takes a token for `ip`; false when it has none left.
fn allow(&self, ip: IpAddr) -> bool {
let rate = self.config.requests_per_hour / 3600.0;
let burst = (self.config.requests_per_hour / 10.0).max(1.0);
let mut buckets = self.buckets.lock().unwrap_or_else(|e| e.into_inner());
let now = Instant::now();
let (tokens, at) = buckets.entry(ip).or_insert((burst, now));
*tokens = (*tokens + now.duration_since(*at).as_secs_f64() * rate).min(burst);
*at = now;
if *tokens < 1.0 {
return false;
}
*tokens -= 1.0;
true
}
fn client(&self, headers: &HeaderMap, peer: Option<SocketAddr>) -> IpAddr {
let forwarded = self
.config
.trust_proxy
.then(|| {
headers
.get("x-forwarded-for")?
.to_str()
.ok()?
.split(',')
.next()?
.trim()
.parse()
.ok()
})
.flatten();
forwarded
.or(peer.map(|p| p.ip()))
.unwrap_or(IpAddr::from([0, 0, 0, 0]))
}
/// The day an install's newest row came in, if it has any left.
pub fn newest_day(&self, install: &[u8]) -> Option<u32> {
self.store().newest_day(install).ok().flatten()
}
/// Daily upkeep: totals rebuilt, expired rows removed, idle rate-limit entries forgotten.
pub fn maintain(&self) -> rusqlite::Result<usize> {
let oldest = (self.today)().saturating_sub(self.config.retention_days);
let removed = self.store().expire(oldest)?;
self.buckets
.lock()
.unwrap_or_else(|e| e.into_inner())
.retain(|_, (_, at)| at.elapsed() < Duration::from_secs(3600));
if removed > 0 {
self.cache.lock().unwrap_or_else(|e| e.into_inner()).clear();
}
Ok(removed)
}
}
fn peer(req: &Request) -> Option<SocketAddr> {
req.extensions()
.get::<ConnectInfo<SocketAddr>>()
.map(|c| c.0)
}
async fn receive(State(app): State<Arc<App>>, req: Request) -> Response {
let ip = app.client(req.headers(), peer(&req));
if !app.allow(ip) {
return status(StatusCode::TOO_MANY_REQUESTS, "too many requests");
}
let zstd = req
.headers()
.get(header::CONTENT_ENCODING)
.is_some_and(|v| v.as_bytes().eq_ignore_ascii_case(b"zstd"));
if !zstd {
return status(
StatusCode::UNSUPPORTED_MEDIA_TYPE,
"send zstd (Content-Encoding: zstd)",
);
}
let Ok(body) = axum::body::to_bytes(req.into_body(), batch::MAX_COMPRESSED).await else {
return status(StatusCode::PAYLOAD_TOO_LARGE, "a batch is at most 256 KiB");
};
let batch = match batch::decode(&body) {
Ok(batch) => batch,
Err(e) => return status(StatusCode::BAD_REQUEST, &e),
};
let today = (app.today)();
let install = store::install_key(&batch.install_id).unwrap_or_default();
let mut store = app.store();
let sent = store.count_on(&install, today).unwrap_or(u32::MAX);
if sent.saturating_add(batch.items.len() as u32) > app.config.items_per_day {
return status(
StatusCode::TOO_MANY_REQUESTS,
"this install's daily limit is reached",
);
}
match store.insert(&batch, today) {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(e) => {
eprintln!("horizon-telemetry-server: could not store a batch: {e}");
status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
}
}
}
async fn forget(State(app): State<Arc<App>>, Path(id): Path<String>, req: Request) -> Response {
if !app.allow(app.client(req.headers(), peer(&req))) {
return status(StatusCode::TOO_MANY_REQUESTS, "too many requests");
}
let Some(install) = store::install_key(&id) else {
return status(StatusCode::BAD_REQUEST, "not an install ID");
};
match app.store().delete_install(&install) {
Ok(_) => {
app.cache.lock().unwrap_or_else(|e| e.into_inner()).clear();
StatusCode::NO_CONTENT.into_response()
}
Err(e) => {
eprintln!("horizon-telemetry-server: could not delete an install: {e}");
status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
}
}
}
#[derive(serde::Deserialize)]
struct Window {
days: Option<u32>,
}
async fn aggregate(State(app): State<Arc<App>>, Query(window): Query<Window>) -> Response {
let days = window.days.unwrap_or(30).clamp(1, 366);
let cached = app
.cache
.lock()
.unwrap_or_else(|e| e.into_inner())
.get(&days)
.filter(|(at, _)| at.elapsed() < CACHE_FOR)
.map(|(_, json)| json.clone());
let json = match cached {
Some(json) => json,
None => {
let to = (app.today)();
let made =
app.store()
.aggregate(to + 1 - days.min(to + 1), to, app.config.min_install_days);
match made {
Ok(value) => {
let json = value.to_string();
app.cache
.lock()
.unwrap_or_else(|e| e.into_inner())
.insert(days, (Instant::now(), json.clone()));
json
}
Err(e) => {
eprintln!("horizon-telemetry-server: could not build the totals: {e}");
return status(StatusCode::SERVICE_UNAVAILABLE, "try again later");
}
}
}
};
let origin =
HeaderValue::from_str(&app.config.allow_origin).unwrap_or(HeaderValue::from_static("*"));
(
[
(
header::CONTENT_TYPE,
HeaderValue::from_static("application/json"),
),
(
header::CACHE_CONTROL,
HeaderValue::from_static("public, max-age=600"),
),
(header::ACCESS_CONTROL_ALLOW_ORIGIN, origin),
],
json,
)
.into_response()
}
pub fn router(app: Arc<App>) -> Router {
Router::new()
.route("/v1/batch", post(receive))
.route("/v1/installs/{id}", delete(forget))
.route("/v1/aggregate", get(aggregate))
.route("/healthz", get(|| async { "ok" }))
.with_state(app)
}
@@ -0,0 +1,40 @@
//! `horizon-telemetry-server`: serves on `PORT` (default 8080), keeps its data in
//! `HORIZON_TELEMETRY_DB` (default `telemetry.sqlite3`), and does its upkeep every ten minutes.
use std::net::SocketAddr;
use std::time::Duration;
use horizon_telemetry_server::{App, Config, router, store::Store};
#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
async fn main() {
let path = std::env::var("HORIZON_TELEMETRY_DB").unwrap_or_else(|_| "telemetry.sqlite3".into());
let port: u16 = std::env::var("PORT")
.ok()
.and_then(|p| p.parse().ok())
.unwrap_or(8080);
let store = Store::open(&path).unwrap_or_else(|e| panic!("{path}: {e}"));
let app = App::new(store, Config::from_env());
let upkeep = app.clone();
tokio::spawn(async move {
let mut every = tokio::time::interval(Duration::from_secs(600));
loop {
every.tick().await;
if let Err(e) = upkeep.maintain() {
eprintln!("horizon-telemetry-server: upkeep failed: {e}");
}
}
});
let listener = tokio::net::TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], port)))
.await
.unwrap_or_else(|e| panic!("port {port}: {e}"));
eprintln!("horizon-telemetry-server: listening on {port}, data in {path}");
axum::serve(
listener,
router(app).into_make_service_with_connect_info::<SocketAddr>(),
)
.with_graceful_shutdown(async {
let _ = tokio::signal::ctrl_c().await;
})
.await
.unwrap_or_else(|e| panic!("{e}"));
}
@@ -0,0 +1,428 @@
//! The database: one SQLite file, kept small.
//!
//! - `systems`: each distinct machine description once. Rows point at it, so a CPU and GPU
//! name isn't stored again with every session.
//! - `perf`: one row per play session, in columns. A frame or tick summary is an 80-byte blob
//! ([`pack`]), not JSON.
//! - `reports`: crash and bug reports and benchmark uploads, their `data` zstd-compressed.
//! - `perf_daily`: the anonymous totals per day, version, OS, GPU vendor and render distance
//! that `/v1/aggregate` reads. Rebuilt from `perf` when a day changes (new sessions, a
//! deletion), and kept after the raw rows expire.
//!
//! Dates are the day the server received an item (days since 1970, UTC). The sender's IP
//! address is never stored.
use std::collections::{HashMap, HashSet};
use horizon_telemetry::Kind;
use horizon_telemetry::batch::Batch;
use horizon_telemetry::payload::{BUCKET_EDGES_MS, Payload, Summary, System};
use rusqlite::{Connection, OptionalExtension, params};
const BUCKETS: usize = BUCKET_EDGES_MS.len() + 1;
const SCHEMA: &str = "
PRAGMA journal_mode = WAL;
PRAGMA synchronous = NORMAL;
CREATE TABLE IF NOT EXISTS systems (
id INTEGER PRIMARY KEY,
os TEXT NOT NULL, os_version TEXT NOT NULL, arch TEXT NOT NULL, cpu TEXT NOT NULL,
threads INTEGER NOT NULL, ram_gb INTEGER NOT NULL,
gpu TEXT NOT NULL, driver TEXT NOT NULL, backend TEXT NOT NULL,
UNIQUE (os, os_version, arch, cpu, threads, ram_gb, gpu, driver, backend)
);
CREATE TABLE IF NOT EXISTS perf (
install BLOB NOT NULL, day INTEGER NOT NULL, version TEXT NOT NULL, system INTEGER NOT NULL,
render_distance INTEGER NOT NULL, simulation_distance INTEGER NOT NULL, max_fps INTEGER NOT NULL,
vsync INTEGER NOT NULL, fullscreen INTEGER NOT NULL, graphics TEXT NOT NULL,
seconds INTEGER NOT NULL, frames BLOB NOT NULL, ticks BLOB,
peak_ram_mb INTEGER NOT NULL, to_title_ms INTEGER, to_world_ms INTEGER
);
CREATE INDEX IF NOT EXISTS perf_install ON perf (install);
CREATE INDEX IF NOT EXISTS perf_day ON perf (day);
CREATE TABLE IF NOT EXISTS reports (
install BLOB NOT NULL, day INTEGER NOT NULL, kind INTEGER NOT NULL, version TEXT NOT NULL,
system INTEGER, body BLOB NOT NULL
);
CREATE INDEX IF NOT EXISTS reports_install ON reports (install);
CREATE INDEX IF NOT EXISTS reports_kind_day ON reports (kind, day);
CREATE TABLE IF NOT EXISTS perf_daily (
day INTEGER NOT NULL, version TEXT NOT NULL, os TEXT NOT NULL, gpu_vendor TEXT NOT NULL,
render_distance INTEGER NOT NULL,
sessions INTEGER NOT NULL, installs INTEGER NOT NULL, seconds INTEGER NOT NULL,
buckets TEXT NOT NULL, p50_sum REAL NOT NULL, p99_sum REAL NOT NULL,
PRIMARY KEY (day, version, os, gpu_vendor, render_distance)
) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS dirty (day INTEGER PRIMARY KEY) WITHOUT ROWID;
";
fn kind_code(kind: Kind) -> i64 {
match kind {
Kind::Perf => 0,
Kind::Crash => 1,
Kind::Bug => 2,
Kind::Benchmark => 3,
}
}
/// An install ID as 16 bytes.
pub fn install_key(id: &str) -> Option<Vec<u8>> {
horizon_telemetry::consent::is_install_id(id).then(|| {
(0..16)
.map(|i| u8::from_str_radix(&id[2 * i..2 * i + 2], 16).unwrap_or(0))
.collect()
})
}
/// A summary as 80 bytes: the count, four percentiles, then the 14 buckets, little-endian.
pub fn pack(s: &Summary) -> Vec<u8> {
let mut out = Vec::with_capacity(8 + 16 + 4 * BUCKETS);
out.extend(s.count.to_le_bytes());
for ms in [s.p50_ms, s.p95_ms, s.p99_ms, s.max_ms] {
out.extend(ms.to_le_bytes());
}
for i in 0..BUCKETS {
out.extend(s.buckets.get(i).copied().unwrap_or(0).to_le_bytes());
}
out
}
pub fn unpack(bytes: &[u8]) -> Option<Summary> {
if bytes.len() != 8 + 16 + 4 * BUCKETS {
return None;
}
let u32_at = |i: usize| u32::from_le_bytes(bytes[i..i + 4].try_into().unwrap_or_default());
let f32_at = |i: usize| f32::from_bits(u32_at(i));
Some(Summary {
count: u64::from_le_bytes(bytes[..8].try_into().ok()?),
p50_ms: f32_at(8),
p95_ms: f32_at(12),
p99_ms: f32_at(16),
max_ms: f32_at(20),
buckets: (0..BUCKETS).map(|i| u32_at(24 + 4 * i)).collect(),
})
}
/// The GPU's maker, the one GPU detail the public totals group by.
pub fn gpu_vendor(gpu: &str) -> &'static str {
let gpu = gpu.to_ascii_lowercase();
[
("apple", "apple"),
("nvidia", "nvidia"),
("geforce", "nvidia"),
("radeon", "amd"),
("amd", "amd"),
("intel", "intel"),
("adreno", "qualcomm"),
("llvmpipe", "software"),
]
.into_iter()
.find(|(needle, _)| gpu.contains(needle))
.map_or(if gpu.is_empty() { "unknown" } else { "other" }, |(_, v)| v)
}
pub struct Store {
db: Connection,
}
impl Store {
pub fn open(path: &str) -> rusqlite::Result<Self> {
let db = Connection::open(path)?;
db.execute_batch(SCHEMA)?;
Ok(Self { db })
}
/// How many items an install has sent on `day`.
pub fn count_on(&self, install: &[u8], day: u32) -> rusqlite::Result<u32> {
self.db.query_row(
"SELECT (SELECT count(*) FROM perf WHERE install = ?1 AND day = ?2)
+ (SELECT count(*) FROM reports WHERE install = ?1 AND day = ?2)",
params![install, day],
|r| r.get(0),
)
}
/// Keeps a decoded batch, received on `day`.
pub fn insert(&mut self, batch: &Batch, day: u32) -> rusqlite::Result<()> {
let install = install_key(&batch.install_id).unwrap_or_default();
let tx = self.db.transaction()?;
for item in &batch.items {
match &item.payload {
Payload::Perf(p) => {
let system = system_id(&tx, &p.system)?;
tx.execute(
"INSERT INTO perf (install, day, version, system, render_distance, simulation_distance,
max_fps, vsync, fullscreen, graphics, seconds, frames, ticks, peak_ram_mb,
to_title_ms, to_world_ms) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
params![
install, day, item.version, system,
p.settings.render_distance, p.settings.simulation_distance, p.settings.max_fps,
p.settings.vsync, p.settings.fullscreen, p.settings.graphics,
p.seconds, pack(&p.frames), p.ticks.as_ref().map(pack), p.peak_ram_mb,
p.to_title_ms, p.to_world_ms,
],
)?;
}
payload => {
let system = match payload {
Payload::Crash(c) => Some(system_id(&tx, &c.system)?),
Payload::Bug(b) => Some(system_id(&tx, &b.system)?),
_ => None,
};
let data = match payload {
Payload::Crash(c) => serde_json::to_vec(c),
Payload::Bug(b) => serde_json::to_vec(b),
Payload::Benchmark(r) => serde_json::to_vec(r),
Payload::Perf(_) => unreachable!("handled above"),
}
.unwrap_or_default();
let body = ruzstd::encoding::compress_to_vec(
data.as_slice(),
ruzstd::encoding::CompressionLevel::Fastest,
);
tx.execute(
"INSERT INTO reports (install, day, kind, version, system, body) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![install, day, kind_code(item.kind()), item.version, system, body],
)?;
}
}
}
tx.execute("INSERT OR IGNORE INTO dirty (day) VALUES (?1)", [day])?;
tx.commit()
}
/// Deletes everything an install sent; the days it touched get their totals rebuilt.
pub fn delete_install(&mut self, install: &[u8]) -> rusqlite::Result<usize> {
let tx = self.db.transaction()?;
tx.execute(
"INSERT OR IGNORE INTO dirty (day) SELECT DISTINCT day FROM perf WHERE install = ?1",
[install],
)?;
let mut deleted = tx.execute("DELETE FROM perf WHERE install = ?1", [install])?;
deleted += tx.execute("DELETE FROM reports WHERE install = ?1", [install])?;
tx.commit()?;
self.rebuild_dirty()?;
Ok(deleted)
}
/// Removes raw rows received before `oldest_day`; the daily totals stay.
pub fn expire(&mut self, oldest_day: u32) -> rusqlite::Result<usize> {
// A day's totals are final before its rows go.
self.rebuild_dirty()?;
let mut n = self
.db
.execute("DELETE FROM perf WHERE day < ?1", [oldest_day])?;
n += self
.db
.execute("DELETE FROM reports WHERE day < ?1", [oldest_day])?;
self.db.execute(
"DELETE FROM systems WHERE id NOT IN (SELECT system FROM perf)
AND id NOT IN (SELECT system FROM reports WHERE system IS NOT NULL)",
[],
)?;
Ok(n)
}
/// Rebuilds the daily totals of every day marked changed.
pub fn rebuild_dirty(&mut self) -> rusqlite::Result<()> {
let days: Vec<u32> = {
let mut q = self.db.prepare("SELECT day FROM dirty")?;
q.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?
};
for day in days {
self.rebuild_day(day)?;
}
Ok(())
}
fn rebuild_day(&mut self, day: u32) -> rusqlite::Result<()> {
#[derive(Default)]
struct Group {
sessions: u64,
installs: HashSet<Vec<u8>>,
seconds: u64,
buckets: [u64; BUCKETS],
p50_sum: f64,
p99_sum: f64,
}
let tx = self.db.transaction()?;
let mut groups: HashMap<(String, String, &'static str, u8), Group> = HashMap::new();
{
let mut q = tx.prepare(
"SELECT perf.install, perf.version, systems.os, systems.gpu, perf.render_distance,
perf.seconds, perf.frames FROM perf JOIN systems ON systems.id = perf.system
WHERE perf.day = ?1",
)?;
let mut rows = q.query([day])?;
while let Some(r) = rows.next()? {
let gpu: String = r.get(3)?;
let key = (r.get(1)?, r.get(2)?, gpu_vendor(&gpu), r.get(4)?);
let Some(frames) = unpack(&r.get::<_, Vec<u8>>(6)?) else {
continue;
};
let g = groups.entry(key).or_default();
g.sessions += 1;
g.installs.insert(r.get(0)?);
g.seconds += r.get::<_, u64>(5)?;
for (sum, n) in g.buckets.iter_mut().zip(&frames.buckets) {
*sum += u64::from(*n);
}
g.p50_sum += f64::from(frames.p50_ms);
g.p99_sum += f64::from(frames.p99_ms);
}
}
tx.execute("DELETE FROM perf_daily WHERE day = ?1", [day])?;
for ((version, os, vendor, distance), g) in groups {
tx.execute(
"INSERT INTO perf_daily VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
params![
day,
version,
os,
vendor,
distance,
g.sessions,
g.installs.len(),
g.seconds,
serde_json::to_string(&g.buckets.to_vec()).unwrap_or_default(),
g.p50_sum,
g.p99_sum,
],
)?;
}
tx.execute("DELETE FROM dirty WHERE day = ?1", [day])?;
tx.commit()
}
/// The anonymous totals for the days `from..=to`, groups with fewer than `k` install-days
/// left out, and the newest benchmark uploads.
pub fn aggregate(&mut self, from: u32, to: u32, k: u64) -> rusqlite::Result<serde_json::Value> {
self.rebuild_dirty()?;
/// One group's sums over the window.
#[derive(Default)]
struct Totals {
sessions: u64,
install_days: u64,
seconds: u64,
buckets: [u64; BUCKETS],
p50_sum: f64,
p99_sum: f64,
}
let mut groups: HashMap<(String, String, String, u8), Totals> = HashMap::new();
{
let mut q = self.db.prepare(
"SELECT version, os, gpu_vendor, render_distance, sessions, installs, seconds, buckets,
p50_sum, p99_sum FROM perf_daily WHERE day BETWEEN ?1 AND ?2",
)?;
let mut rows = q.query([from, to])?;
while let Some(r) = rows.next()? {
let g = groups
.entry((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))
.or_default();
g.sessions += r.get::<_, u64>(4)?;
g.install_days += r.get::<_, u64>(5)?;
g.seconds += r.get::<_, u64>(6)?;
let buckets: Vec<u64> =
serde_json::from_str(&r.get::<_, String>(7)?).unwrap_or_default();
for (sum, n) in g.buckets.iter_mut().zip(buckets) {
*sum += n;
}
g.p50_sum += r.get::<_, f64>(8)?;
g.p99_sum += r.get::<_, f64>(9)?;
}
}
let (mut sessions, mut hours) = (0, 0.0);
let mut out: Vec<serde_json::Value> = groups
.into_iter()
.filter(|(_, g)| g.install_days >= k)
.map(|((version, os, vendor, distance), g)| {
sessions += g.sessions;
hours += g.seconds as f64 / 3600.0;
let avg = |sum: f64| (sum / g.sessions as f64 * 10.0).round() / 10.0;
serde_json::json!({
"version": version, "os": os, "gpu_vendor": vendor, "render_distance": distance,
"sessions": g.sessions, "install_days": g.install_days,
"hours": (g.seconds as f64 / 360.0).round() / 10.0,
"frame_buckets": g.buckets.to_vec(), "p50_ms": avg(g.p50_sum), "p99_ms": avg(g.p99_sum),
})
})
.collect();
out.sort_by(|a, b| b["sessions"].as_u64().cmp(&a["sessions"].as_u64()));
let crashes: Vec<serde_json::Value> = {
let mut q = self.db.prepare(
"SELECT version, count(*) FROM reports WHERE kind = ?1 AND day BETWEEN ?2 AND ?3
GROUP BY version ORDER BY 2 DESC LIMIT 20",
)?;
q.query_map(params![kind_code(Kind::Crash), from, to], |r| {
Ok(serde_json::json!({ "version": r.get::<_, String>(0)?, "crashes": r.get::<_, u64>(1)? }))
})?
.collect::<Result<_, _>>()?
};
let benchmarks: Vec<serde_json::Value> = {
let mut q = self.db.prepare(
"SELECT day, version, body FROM reports WHERE kind = ?1 AND day BETWEEN ?2 AND ?3
ORDER BY rowid DESC LIMIT 50",
)?;
q.query_map(params![kind_code(Kind::Benchmark), from, to], |r| {
let body: Vec<u8> = r.get(2)?;
Ok(serde_json::json!({
"day": r.get::<_, u32>(0)?, "version": r.get::<_, String>(1)?, "report": inflate(&body),
}))
})?
.collect::<Result<_, _>>()?
};
Ok(serde_json::json!({
"from_day": from, "to_day": to, "min_install_days": k,
"bucket_edges_ms": BUCKET_EDGES_MS.map(|e| (f64::from(e) * 10.0).round() / 10.0),
"totals": { "sessions": sessions, "hours": (hours * 10.0).round() / 10.0 },
"groups": out, "crashes_by_version": crashes, "benchmarks": benchmarks,
}))
}
/// The day an install's newest row came in, if any (tests, and checking a deletion).
pub fn newest_day(&self, install: &[u8]) -> rusqlite::Result<Option<u32>> {
self.db
.query_row(
"SELECT max(day) FROM (SELECT day FROM perf WHERE install = ?1
UNION ALL SELECT day FROM reports WHERE install = ?1)",
[install],
|r| r.get(0),
)
.optional()
.map(Option::flatten)
}
}
fn system_id(tx: &rusqlite::Transaction, s: &System) -> rusqlite::Result<i64> {
tx.execute(
"INSERT OR IGNORE INTO systems (os, os_version, arch, cpu, threads, ram_gb, gpu, driver, backend)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
params![s.os, s.os_version, s.arch, s.cpu, s.threads, s.ram_gb, s.gpu, s.driver, s.backend],
)?;
tx.query_row(
"SELECT id FROM systems WHERE os = ?1 AND os_version = ?2 AND arch = ?3 AND cpu = ?4
AND threads = ?5 AND ram_gb = ?6 AND gpu = ?7 AND driver = ?8 AND backend = ?9",
params![
s.os,
s.os_version,
s.arch,
s.cpu,
s.threads,
s.ram_gb,
s.gpu,
s.driver,
s.backend
],
|r| r.get(0),
)
}
/// A stored body back as JSON.
pub fn inflate(body: &[u8]) -> serde_json::Value {
use std::io::Read;
let mut json = Vec::new();
ruzstd::decoding::StreamingDecoder::new(body)
.ok()
.and_then(|d| d.take(1 << 20).read_to_end(&mut json).ok())
.and_then(|_| serde_json::from_slice(&json).ok())
.unwrap_or(serde_json::Value::Null)
}
@@ -0,0 +1,393 @@
//! The server through its HTTP interface, and end to end with the launcher's uploader.
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
use axum::body::Body;
use axum::http::{Request, StatusCode};
use horizon_telemetry::batch::{self, Batch};
use horizon_telemetry::payload::{Crash, Payload, Perf, Settings, Source, System, Timings};
use horizon_telemetry::{Item, outbox};
use horizon_telemetry_server::store::{self, Store};
use horizon_telemetry_server::{App, Config, router};
use http_body_util::BodyExt;
use tower::ServiceExt;
const DAY: u32 = 20_741;
fn install(n: u32) -> String {
format!("{n:032x}")
}
fn perf(gpu: &str, distance: u8) -> Item {
let mut t = Timings::new();
(0..30_000).for_each(|i| t.record(8.0 + (i % 50) as f32 / 10.0));
let mut item = Item::new(
Source::Game,
"v2026.10.15",
Payload::Perf(Perf {
system: System {
os: "linux".into(),
os_version: "Fedora Linux 43".into(),
cpu: "AMD Ryzen 9 7950X".into(),
threads: 32,
ram_gb: 32,
gpu: gpu.into(),
driver: "570.86".into(),
backend: "Vulkan".into(),
..System::default()
},
settings: Settings {
render_distance: distance,
simulation_distance: 12,
graphics: "fancy".into(),
..Settings::default()
},
seconds: 1800,
frames: t.summary(),
peak_ram_mb: 2300,
..Perf::default()
}),
);
item.day = DAY;
item
}
struct Server {
app: Arc<App>,
day: Arc<AtomicU32>,
}
fn server(config: Config) -> Server {
let day = Arc::new(AtomicU32::new(DAY));
let clock = day.clone();
let app = App::with_clock(
Store::open(":memory:").unwrap(),
config,
Box::new(move || clock.load(Ordering::SeqCst)),
);
Server { app, day }
}
impl Server {
async fn send(&self, req: Request<Body>) -> (StatusCode, String) {
let res = router(self.app.clone()).oneshot(req).await.unwrap();
let code = res.status();
let body = res.into_body().collect().await.unwrap().to_bytes();
(code, String::from_utf8_lossy(&body).into_owned())
}
async fn post(&self, id: &str, items: Vec<Item>) -> StatusCode {
let body = batch::encode(&Batch::new(id, items));
self.post_raw(body, Some("zstd")).await
}
async fn post_raw(&self, body: Vec<u8>, encoding: Option<&str>) -> StatusCode {
let mut req = Request::post("/v1/batch").header("x-forwarded-for", "203.0.113.9");
if let Some(e) = encoding {
req = req.header("content-encoding", e);
}
self.send(req.body(Body::from(body)).unwrap()).await.0
}
async fn aggregate(&self) -> serde_json::Value {
let (code, body) = self
.send(
Request::get("/v1/aggregate?days=30")
.body(Body::empty())
.unwrap(),
)
.await;
assert_eq!(code, StatusCode::OK);
serde_json::from_str(&body).unwrap()
}
}
fn generous() -> Config {
Config {
requests_per_hour: 100_000.0,
..Config::default()
}
}
#[tokio::test]
async fn totals_show_only_groups_with_enough_installs() {
let s = server(generous());
for n in 0..6 {
assert_eq!(
s.post(&install(n), vec![perf("NVIDIA GeForce RTX 4070", 12); 2])
.await,
StatusCode::NO_CONTENT
);
}
for n in 10..12 {
assert_eq!(
s.post(&install(n), vec![perf("Intel(R) Arc A770", 8)])
.await,
StatusCode::NO_CONTENT
);
}
let totals = s.aggregate().await;
let groups = totals["groups"].as_array().unwrap();
assert_eq!(
groups.len(),
1,
"two installs are too few to show: {totals}"
);
let g = &groups[0];
assert_eq!(g["gpu_vendor"], "nvidia");
assert_eq!(
(g["sessions"].as_u64(), g["install_days"].as_u64()),
(Some(12), Some(6))
);
assert_eq!(g["hours"], 6.0);
assert_eq!(
g["frame_buckets"]
.as_array()
.unwrap()
.iter()
.map(|b| b.as_u64().unwrap())
.sum::<u64>(),
12 * 30_000
);
assert_eq!(totals["bucket_edges_ms"].as_array().unwrap().len(), 13);
let text = totals.to_string();
assert!(
!text.contains(&install(0)) && !text.contains("Ryzen"),
"no install or machine in the totals"
);
}
#[tokio::test]
async fn bad_batches_are_refused() {
let s = server(generous());
let good = batch::encode(&Batch::new(&install(1), vec![perf("x", 8)]));
assert_eq!(
s.post_raw(good.clone(), None).await,
StatusCode::UNSUPPORTED_MEDIA_TYPE
);
assert_eq!(
s.post_raw(b"garbage".to_vec(), Some("zstd")).await,
StatusCode::BAD_REQUEST
);
assert_eq!(
s.post_raw(vec![0; batch::MAX_COMPRESSED + 1], Some("zstd"))
.await,
StatusCode::PAYLOAD_TOO_LARGE
);
assert_eq!(
s.post("not-an-id", vec![perf("x", 8)]).await,
StatusCode::BAD_REQUEST
);
assert_eq!(s.post_raw(good, Some("zstd")).await, StatusCode::NO_CONTENT);
}
#[tokio::test]
async fn an_address_is_rate_limited() {
let s = server(Config {
requests_per_hour: 30.0,
trust_proxy: true,
..Config::default()
});
for n in 0..3 {
assert_eq!(
s.post(&install(n), vec![perf("x", 8)]).await,
StatusCode::NO_CONTENT
);
}
assert_eq!(
s.post(&install(9), vec![perf("x", 8)]).await,
StatusCode::TOO_MANY_REQUESTS
);
// Another address has its own allowance.
let body = batch::encode(&Batch::new(&install(9), vec![perf("x", 8)]));
let req = Request::post("/v1/batch")
.header("x-forwarded-for", "198.51.100.7, 10.0.0.1")
.header("content-encoding", "zstd")
.body(Body::from(body))
.unwrap();
assert_eq!(s.send(req).await.0, StatusCode::NO_CONTENT);
}
#[tokio::test]
async fn an_install_has_a_daily_cap() {
let s = server(Config {
items_per_day: 3,
..generous()
});
assert_eq!(
s.post(&install(1), vec![perf("x", 8); 2]).await,
StatusCode::NO_CONTENT
);
assert_eq!(
s.post(&install(1), vec![perf("x", 8); 2]).await,
StatusCode::TOO_MANY_REQUESTS
);
assert_eq!(
s.post(&install(1), vec![perf("x", 8)]).await,
StatusCode::NO_CONTENT
);
s.day.store(DAY + 1, Ordering::SeqCst);
assert_eq!(
s.post(&install(1), vec![perf("x", 8); 3]).await,
StatusCode::NO_CONTENT
);
}
#[tokio::test]
async fn deleting_an_install_removes_its_rows_and_its_share_of_the_totals() {
let s = server(Config {
min_install_days: 1,
..generous()
});
let crash = Item::new(Source::Game, "v1", Payload::Crash(Crash::default()));
assert_eq!(
s.post(&install(1), vec![perf("Apple M4 Pro", 12), crash])
.await,
StatusCode::NO_CONTENT
);
assert_eq!(
s.post(&install(2), vec![perf("Apple M4 Pro", 12)]).await,
StatusCode::NO_CONTENT
);
assert_eq!(s.aggregate().await["groups"][0]["sessions"], 2);
let (code, _) = s
.send(
Request::delete(format!("/v1/installs/{}", install(1)))
.body(Body::empty())
.unwrap(),
)
.await;
assert_eq!(code, StatusCode::NO_CONTENT);
let key = store::install_key(&install(1)).unwrap();
assert_eq!(s.app.newest_day(&key), None);
let totals = s.aggregate().await;
assert_eq!(
totals["groups"][0]["sessions"], 1,
"the cache was dropped and the day rebuilt"
);
assert_eq!(totals["crashes_by_version"].as_array().unwrap().len(), 0);
let (code, _) = s
.send(
Request::delete("/v1/installs/nope")
.body(Body::empty())
.unwrap(),
)
.await;
assert_eq!(code, StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn raw_rows_expire_and_the_totals_stay() {
let s = server(Config {
min_install_days: 1,
retention_days: 180,
..generous()
});
assert_eq!(
s.post(&install(1), vec![perf("Apple M4 Pro", 12)]).await,
StatusCode::NO_CONTENT
);
s.day.store(DAY + 179, Ordering::SeqCst);
assert_eq!(s.app.maintain().unwrap(), 0, "still within retention");
s.day.store(DAY + 181, Ordering::SeqCst);
assert_eq!(s.app.maintain().unwrap(), 1);
assert_eq!(
s.app.newest_day(&store::install_key(&install(1)).unwrap()),
None
);
let (_, body) = s
.send(
Request::get("/v1/aggregate?days=366")
.body(Body::empty())
.unwrap(),
)
.await;
let totals: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(totals["groups"][0]["sessions"], 1);
}
#[test]
fn a_session_costs_little_storage() {
let dir = std::env::temp_dir().join(format!(
"horizon-telemetry-server-size-{}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("t.sqlite3");
let mut store = Store::open(path.to_str().unwrap()).unwrap();
let gpus = [
"Apple M4 Pro",
"NVIDIA GeForce RTX 4070",
"AMD Radeon RX 7800 XT",
"Intel(R) Arc A770",
];
let sessions = 10_000;
for n in 0..sessions / 10 {
let items = (0..10)
.map(|i| perf(gpus[(n + i) as usize % 4], 8 + (i % 3) as u8 * 4))
.collect();
store
.insert(&Batch::new(&install(n), items), DAY + n % 30)
.unwrap();
}
store.rebuild_dirty().unwrap();
drop(store);
let bytes: u64 = std::fs::read_dir(&dir)
.unwrap()
.flatten()
.map(|e| e.metadata().unwrap().len())
.sum();
let per = bytes / u64::from(sessions);
eprintln!("{sessions} sessions: {bytes} bytes on disk, {per} bytes a session");
assert!(per < 250, "{per} bytes a session");
}
#[tokio::test]
async fn the_launchers_uploader_reaches_it_end_to_end() {
let s = server(Config {
min_install_days: 1,
..generous()
});
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let service =
router(s.app.clone()).into_make_service_with_connect_info::<std::net::SocketAddr>();
tokio::spawn(async move { axum::serve(listener, service).await.unwrap() });
let home = std::env::temp_dir().join(format!(
"horizon-telemetry-server-e2e-{}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&home);
let mut consent = horizon_telemetry::Consent::load(&home);
consent.answer(horizon_telemetry::Choices::ALL);
consent.save(&home).unwrap();
for _ in 0..3 {
assert!(outbox::push(&home, &perf("Apple M4 Pro", 12)).unwrap());
}
let (h, u) = (home.clone(), url.clone());
let sent = tokio::task::spawn_blocking(move || horizon_telemetry::upload::flush(&h, &u))
.await
.unwrap();
assert_eq!(
sent,
horizon_telemetry::upload::Flush::Sent {
items: 3,
batches: 1
}
);
assert_eq!(s.aggregate().await["groups"][0]["sessions"], 3);
let id = consent.install_id.clone();
let deleted =
tokio::task::spawn_blocking(move || horizon_telemetry::upload::delete_remote(&url, &id))
.await
.unwrap();
assert_eq!(deleted, Ok(()));
assert_eq!(
s.app
.newest_day(&store::install_key(&consent.install_id).unwrap()),
None
);
}
+1 -1
View File
@@ -52,4 +52,4 @@ Never sent: account names or UUIDs, world names, server or LAN addresses, chat,
- `DELETE /v1/installs/<install_id>`: deletes everything sent under that ID.
- `GET /v1/aggregate`: the anonymous totals the website's results page reads.
Retention, rate limits and storage are described with the server.
`crates/horizon-telemetry-server` is the server. Its [README](../crates/horizon-telemetry-server/README.md) covers storage (about 190 bytes per play session), retention (180 days of raw rows, anonymous daily totals kept), rate limits and hosting.