Files
horizon-airdef/crates/horizon-telemetry-server/src/lib.rs
T
Ludovic Marie be16401f90 Telemetry: send to telemetry.playhorizon.dev, a benchmarks summary, the Railway config
- DEFAULT_ENDPOINT is https://telemetry.playhorizon.dev. The heartbeat, the launcher's live count and the
  outbox upload all go there; HORIZON_TELEMETRY_URL still names another server, and HORIZON_TELEMETRY_URL=off
  sends nothing at all. CI workflows and the harnesses that start the game or the launcher set off. Consent
  is unchanged: the four switches stay opt-in, only the anonymous ping is always on.
- GET /v1/benchmarks/summary: players' benchmark uploads grouped by GPU, CPU, render distance and game, groups
  of 3+ installs only, medians per install then across installs, no install ID, day, OS version, driver,
  memory or screen. Cached 10 minutes, CORS for the website. Same rules as website/scripts/benchmarks.mjs,
  tested on the same fixtures; a deleted install leaves its groups at once.
- /v1/aggregate no longer lists the newest 50 raw benchmark reports (hardware, driver, day, one install each):
  uploads are published only through the summary.
- Behind Railway's proxy the rate limit counts X-Real-IP, else the last X-Forwarded-For entry (the one the
  proxy adds; a client could choose the first). Still in memory only, never stored or logged.
- crates/horizon-telemetry-server/railway.json: Dockerfile build from the repository root, health check
  /healthz, one replica in europe-west4 (one SQLite file on the /data volume).
- Docs: docs/telemetry.md, the server README (the summary's shape, Railway setup), the privacy page (the ping
  is on, where the data lives, how benchmark uploads are shown).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 21:46:32 +02:00

487 lines
18 KiB
Rust

//! The ingest server for opt-in data collection (docs/telemetry.md).
//!
//! - `POST /v1/batch`: a zstd batch from the launcher, checked by `horizon_telemetry::batch`.
//! - `DELETE /v1/installs/<id>`: everything an install sent goes.
//! - `GET /v1/aggregate?days=30`: the anonymous totals the website reads, cached.
//! - `GET /v1/benchmarks/summary`: players' benchmark results grouped by GPU and CPU, groups of at
//! least 3 installs only (`summary`), cached; the website's Benchmarks page reads it.
//! - `POST /v1/live`: a game's anonymous "I'm playing" ping (`live`, in memory only).
//! - `GET /v1/live`: how many play now, rounded, cached briefly.
//! - `GET /healthz`.
//!
//! Abuse limits: a token bucket per IP address (kept in memory for an hour, never stored), a
//! daily item cap per install, and the batch size limits. The live count has its own, larger
//! bucket per address (several players can share one) and a cap on the sessions it holds. Raw
//! rows expire after `retention_days`; the daily totals stay. One small process and one SQLite
//! file: see `README.md` for running it.
use std::collections::HashMap;
use std::net::{IpAddr, SocketAddr};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use axum::Router;
use axum::extract::{ConnectInfo, Path, Query, Request, State};
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::{delete, get, post};
use horizon_telemetry::batch;
pub mod live;
pub mod store;
pub mod summary;
use horizon_telemetry::live::{MAX_PING_BYTES, Ping};
use live::Live;
use store::Store;
pub struct Config {
/// Days raw rows are kept.
pub retention_days: u32,
/// Groups in the public totals need at least this many install-days.
pub min_install_days: u64,
/// Requests an IP address may make an hour (with a burst of a tenth of it).
pub requests_per_hour: f64,
/// Items one install may send a day.
pub items_per_day: u32,
/// Behind a proxy (Railway, Fly…): the client's address is the proxy's `X-Real-IP`, else the
/// last `X-Forwarded-For` entry.
pub trust_proxy: bool,
/// The website allowed to read the totals from a browser.
pub allow_origin: String,
/// Live count requests (pings and reads) an IP address may make an hour, with a burst of a
/// tenth of it. A game pings once a minute, so the default lets ten play behind one address.
pub live_per_hour: f64,
/// How long a session counts after its last ping.
pub live_window: Duration,
/// Sessions the live count holds at most; a new one past that is refused.
pub live_max_sessions: usize,
}
impl Default for Config {
fn default() -> Self {
Self {
retention_days: 180,
min_install_days: 5,
requests_per_hour: 60.0,
items_per_day: 100,
trust_proxy: false,
allow_origin: "https://playhorizon.dev".into(),
live_per_hour: 600.0,
live_window: horizon_telemetry::live::WINDOW,
live_max_sessions: 200_000,
}
}
}
impl Config {
/// The defaults, with `HORIZON_TELEMETRY_RETENTION_DAYS`, `…_MIN_INSTALL_DAYS`,
/// `…_REQUESTS_PER_HOUR`, `…_ITEMS_PER_DAY`, `…_TRUST_PROXY` (1), `…_ALLOW_ORIGIN`,
/// `…_LIVE_PER_HOUR`, `…_LIVE_WINDOW_SECONDS` and `…_LIVE_MAX_SESSIONS`.
pub fn from_env() -> Self {
let var = |k: &str| std::env::var(format!("HORIZON_TELEMETRY_{k}")).ok();
let d = Self::default();
Self {
retention_days: var("RETENTION_DAYS")
.and_then(|v| v.parse().ok())
.unwrap_or(d.retention_days),
min_install_days: var("MIN_INSTALL_DAYS")
.and_then(|v| v.parse().ok())
.unwrap_or(d.min_install_days),
requests_per_hour: var("REQUESTS_PER_HOUR")
.and_then(|v| v.parse().ok())
.unwrap_or(d.requests_per_hour),
items_per_day: var("ITEMS_PER_DAY")
.and_then(|v| v.parse().ok())
.unwrap_or(d.items_per_day),
trust_proxy: var("TRUST_PROXY").is_some_and(|v| v == "1"),
allow_origin: var("ALLOW_ORIGIN").unwrap_or(d.allow_origin),
live_per_hour: var("LIVE_PER_HOUR")
.and_then(|v| v.parse().ok())
.unwrap_or(d.live_per_hour),
live_window: var("LIVE_WINDOW_SECONDS")
.and_then(|v| v.parse().ok())
.map_or(d.live_window, Duration::from_secs),
live_max_sessions: var("LIVE_MAX_SESSIONS")
.and_then(|v| v.parse().ok())
.unwrap_or(d.live_max_sessions),
}
}
}
pub struct App {
store: Mutex<Store>,
config: Config,
/// Tokens left and when they were counted, per client address.
buckets: Mutex<Buckets>,
/// The same for the live count's requests.
live_buckets: Mutex<Buckets>,
/// Who is playing now (`live`).
live: Mutex<Live>,
/// `GET /v1/live`'s answer and when it was made.
live_cache: Mutex<Option<(Instant, String)>>,
/// `/v1/aggregate` answers by `days`, with when they were made.
cache: Mutex<HashMap<u32, (Instant, String)>>,
/// `/v1/benchmarks/summary`'s answer and when it was made.
summary_cache: Mutex<Option<(Instant, String)>>,
/// Today, in days since 1970 (a test moves it).
today: Box<dyn Fn() -> u32 + Send + Sync>,
}
type Buckets = HashMap<IpAddr, (f64, Instant)>;
const CACHE_FOR: Duration = Duration::from_secs(600);
fn status(code: StatusCode, message: &str) -> Response {
(code, message.to_owned()).into_response()
}
impl App {
pub fn new(store: Store, config: Config) -> Arc<Self> {
Self::with_clock(store, config, Box::new(horizon_telemetry::today))
}
pub fn with_clock(
store: Store,
config: Config,
today: Box<dyn Fn() -> u32 + Send + Sync>,
) -> Arc<Self> {
let live = Live::new(config.live_window, config.live_max_sessions);
Arc::new(Self {
store: Mutex::new(store),
config,
buckets: Mutex::default(),
live_buckets: Mutex::default(),
live: Mutex::new(live),
live_cache: Mutex::default(),
cache: Mutex::default(),
summary_cache: Mutex::default(),
today,
})
}
fn store(&self) -> std::sync::MutexGuard<'_, Store> {
self.store.lock().unwrap_or_else(|e| e.into_inner())
}
/// Takes a token for `ip`; false when it has none left.
fn allow(&self, ip: IpAddr) -> bool {
take(&self.buckets, self.config.requests_per_hour, ip)
}
/// [`Self::allow`] for the live count's requests.
fn allow_live(&self, ip: IpAddr) -> bool {
take(&self.live_buckets, self.config.live_per_hour, ip)
}
fn live(&self) -> std::sync::MutexGuard<'_, Live> {
self.live.lock().unwrap_or_else(|e| e.into_inner())
}
/// The address requests are counted under, for the rate limits only: kept in memory, never
/// stored or logged. Behind a proxy (Railway) the TCP peer is the proxy, so it is the proxy's
/// `X-Real-IP`, else the last `X-Forwarded-For` entry: the one the proxy added, which a client
/// can't choose (the first one it can).
fn client(&self, headers: &HeaderMap, peer: Option<SocketAddr>) -> IpAddr {
let header = |name: &str| headers.get(name)?.to_str().ok();
let forwarded = self
.config
.trust_proxy
.then(|| {
header("x-real-ip")
.and_then(|ip| ip.trim().parse().ok())
.or_else(|| {
header("x-forwarded-for")?
.rsplit(',')
.next()?
.trim()
.parse()
.ok()
})
})
.flatten();
forwarded
.or(peer.map(|p| p.ip()))
.unwrap_or(IpAddr::from([0, 0, 0, 0]))
}
/// The day an install's newest row came in, if it has any left.
pub fn newest_day(&self, install: &[u8]) -> Option<u32> {
self.store().newest_day(install).ok().flatten()
}
/// Drops the cached public answers (after a deletion or expiry).
fn forget_cached(&self) {
self.cache.lock().unwrap_or_else(|e| e.into_inner()).clear();
*self.summary_cache.lock().unwrap_or_else(|e| e.into_inner()) = None;
}
/// Daily upkeep: totals rebuilt, expired rows removed, idle rate-limit entries and live
/// sessions forgotten.
pub fn maintain(&self) -> rusqlite::Result<usize> {
let oldest = (self.today)().saturating_sub(self.config.retention_days);
let removed = self.store().expire(oldest)?;
for buckets in [&self.buckets, &self.live_buckets] {
buckets
.lock()
.unwrap_or_else(|e| e.into_inner())
.retain(|_, (_, at)| at.elapsed() < Duration::from_secs(3600));
}
self.live().forget_expired(Instant::now());
if removed > 0 {
self.forget_cached();
}
Ok(removed)
}
}
/// Takes a token for `ip` from its bucket in `buckets` (`per_hour`, bursts of a tenth of it);
/// false when it has none left.
fn take(buckets: &Mutex<Buckets>, per_hour: f64, ip: IpAddr) -> bool {
let rate = per_hour / 3600.0;
let burst = (per_hour / 10.0).max(1.0);
let mut buckets = buckets.lock().unwrap_or_else(|e| e.into_inner());
let now = Instant::now();
let (tokens, at) = buckets.entry(ip).or_insert((burst, now));
*tokens = (*tokens + now.duration_since(*at).as_secs_f64() * rate).min(burst);
*at = now;
if *tokens < 1.0 {
return false;
}
*tokens -= 1.0;
true
}
fn peer(req: &Request) -> Option<SocketAddr> {
req.extensions()
.get::<ConnectInfo<SocketAddr>>()
.map(|c| c.0)
}
async fn receive(State(app): State<Arc<App>>, req: Request) -> Response {
let ip = app.client(req.headers(), peer(&req));
if !app.allow(ip) {
return status(StatusCode::TOO_MANY_REQUESTS, "too many requests");
}
let zstd = req
.headers()
.get(header::CONTENT_ENCODING)
.is_some_and(|v| v.as_bytes().eq_ignore_ascii_case(b"zstd"));
if !zstd {
return status(
StatusCode::UNSUPPORTED_MEDIA_TYPE,
"send zstd (Content-Encoding: zstd)",
);
}
let Ok(body) = axum::body::to_bytes(req.into_body(), batch::MAX_COMPRESSED).await else {
return status(StatusCode::PAYLOAD_TOO_LARGE, "a batch is at most 256 KiB");
};
let batch = match batch::decode(&body) {
Ok(batch) => batch,
Err(e) => return status(StatusCode::BAD_REQUEST, &e),
};
let today = (app.today)();
let install = store::install_key(&batch.install_id).unwrap_or_default();
let mut store = app.store();
let sent = store.count_on(&install, today).unwrap_or(u32::MAX);
if sent.saturating_add(batch.items.len() as u32) > app.config.items_per_day {
return status(
StatusCode::TOO_MANY_REQUESTS,
"this install's daily limit is reached",
);
}
match store.insert(&batch, today) {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(e) => {
eprintln!("horizon-telemetry-server: could not store a batch: {e}");
status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
}
}
}
async fn forget(State(app): State<Arc<App>>, Path(id): Path<String>, req: Request) -> Response {
if !app.allow(app.client(req.headers(), peer(&req))) {
return status(StatusCode::TOO_MANY_REQUESTS, "too many requests");
}
let Some(install) = store::install_key(&id) else {
return status(StatusCode::BAD_REQUEST, "not an install ID");
};
match app.store().delete_install(&install) {
Ok(_) => {
app.forget_cached();
StatusCode::NO_CONTENT.into_response()
}
Err(e) => {
eprintln!("horizon-telemetry-server: could not delete an install: {e}");
status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
}
}
}
#[derive(serde::Deserialize)]
struct Window {
days: Option<u32>,
}
async fn aggregate(State(app): State<Arc<App>>, Query(window): Query<Window>) -> Response {
let days = window.days.unwrap_or(30).clamp(1, 366);
let cached = app
.cache
.lock()
.unwrap_or_else(|e| e.into_inner())
.get(&days)
.filter(|(at, _)| at.elapsed() < CACHE_FOR)
.map(|(_, json)| json.clone());
let json = match cached {
Some(json) => json,
None => {
let to = (app.today)();
let made =
app.store()
.aggregate(to + 1 - days.min(to + 1), to, app.config.min_install_days);
match made {
Ok(value) => {
let json = value.to_string();
app.cache
.lock()
.unwrap_or_else(|e| e.into_inner())
.insert(days, (Instant::now(), json.clone()));
json
}
Err(e) => {
eprintln!("horizon-telemetry-server: could not build the totals: {e}");
return status(StatusCode::SERVICE_UNAVAILABLE, "try again later");
}
}
}
};
json_response(&app, json, CACHE_FOR.as_secs())
}
/// `GET /v1/benchmarks/summary`: players' benchmark groups (`summary`), made again at most every
/// ten minutes.
async fn benchmarks(State(app): State<Arc<App>>) -> Response {
let cached = app
.summary_cache
.lock()
.unwrap_or_else(|e| e.into_inner())
.as_ref()
.filter(|(at, _)| at.elapsed() < CACHE_FOR)
.map(|(_, json)| json.clone());
let json = match cached {
Some(json) => json,
None => {
let oldest = (app.today)().saturating_sub(app.config.retention_days);
let uploads = match app.store().benchmark_uploads(oldest) {
Ok(uploads) => uploads,
Err(e) => {
eprintln!("horizon-telemetry-server: could not read the benchmarks: {e}");
return status(StatusCode::SERVICE_UNAVAILABLE, "try again later");
}
};
let json = summary::summary(&uploads, summary::MIN_INSTALLS, &utc_now()).to_string();
*app.summary_cache.lock().unwrap_or_else(|e| e.into_inner()) =
Some((Instant::now(), json.clone()));
json
}
};
json_response(&app, json, CACHE_FOR.as_secs())
}
/// Now as `YYYY-MM-DDTHH:MM:SSZ`.
fn utc_now() -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
let (days, rest) = (secs / 86_400, secs % 86_400);
// Days since 1970 to a civil date (Howard Hinnant's algorithm).
let z = days as i64 + 719_468;
let era = z.div_euclid(146_097);
let doe = z - era * 146_097;
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let day = doy - (153 * mp + 2) / 5 + 1;
let month = if mp < 10 { mp + 3 } else { mp - 9 };
let year = yoe + era * 400 + i64::from(month <= 2);
format!(
"{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z",
rest / 3600,
rest / 60 % 60,
rest % 60
)
}
/// `POST /v1/live`: one "I'm playing". 204, or 400 for a bad ping, 413 for one too large, 429
/// over the address's limit, 503 when the table is full.
async fn ping(State(app): State<Arc<App>>, req: Request) -> Response {
if !app.allow_live(app.client(req.headers(), peer(&req))) {
return status(StatusCode::TOO_MANY_REQUESTS, "too many requests");
}
let Ok(body) = axum::body::to_bytes(req.into_body(), MAX_PING_BYTES).await else {
return status(StatusCode::PAYLOAD_TOO_LARGE, "a ping is at most 512 bytes");
};
let ping: Ping = match serde_json::from_slice(&body) {
Ok(ping) => ping,
Err(e) => return status(StatusCode::BAD_REQUEST, &e.to_string()),
};
if let Err(e) = ping.check() {
return status(StatusCode::BAD_REQUEST, e);
}
match app.live().ping(&ping, Instant::now()) {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(live::Full) => status(StatusCode::SERVICE_UNAVAILABLE, "try again later"),
}
}
/// `GET /v1/live`: the rounded live count (`horizon_telemetry::live::Count`), made again at
/// most every eighth of the window.
async fn playing(State(app): State<Arc<App>>, req: Request) -> Response {
if !app.allow_live(app.client(req.headers(), peer(&req))) {
return status(StatusCode::TOO_MANY_REQUESTS, "too many requests");
}
let fresh_for = app.config.live_window / 8;
let json = {
let mut cache = app.live_cache.lock().unwrap_or_else(|e| e.into_inner());
match &*cache {
Some((at, json)) if at.elapsed() < fresh_for => json.clone(),
_ => {
let json =
serde_json::to_string(&app.live().count(Instant::now())).unwrap_or_default();
*cache = Some((Instant::now(), json.clone()));
json
}
}
};
json_response(&app, json, fresh_for.as_secs())
}
fn json_response(app: &App, json: String, max_age: u64) -> Response {
let origin =
HeaderValue::from_str(&app.config.allow_origin).unwrap_or(HeaderValue::from_static("*"));
let cache = HeaderValue::from_str(&format!("public, max-age={max_age}"))
.unwrap_or(HeaderValue::from_static("no-cache"));
(
[
(
header::CONTENT_TYPE,
HeaderValue::from_static("application/json"),
),
(header::CACHE_CONTROL, cache),
(header::ACCESS_CONTROL_ALLOW_ORIGIN, origin),
],
json,
)
.into_response()
}
pub fn router(app: Arc<App>) -> Router {
Router::new()
.route("/v1/batch", post(receive))
.route("/v1/live", post(ping).get(playing))
.route("/v1/installs/{id}", delete(forget))
.route("/v1/aggregate", get(aggregate))
.route("/v1/benchmarks/summary", get(benchmarks))
.route("/healthz", get(|| async { "ok" }))
.with_state(app)
}