Merge pull request #251 from ludovic111/feat/telemetry-server
Opt-in data collection (4/5): the ingest server (SQLite, rollups, retention, aggregate endpoint)
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
name: telemetry-server
|
||||
# The ingest server is its own workspace (crates/horizon-telemetry-server), so the checks job doesn't build
|
||||
# it: this one does, when it or the crate it shares with the launcher and the game changes.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'crates/horizon-telemetry/**'
|
||||
- 'crates/horizon-telemetry-server/**'
|
||||
- '.github/workflows/telemetry-server.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'crates/horizon-telemetry/**'
|
||||
- 'crates/horizon-telemetry-server/**'
|
||||
- '.github/workflows/telemetry-server.yml'
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
defaults:
|
||||
run:
|
||||
working-directory: crates/horizon-telemetry-server
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: rustup toolchain install stable --profile minimal --component rustfmt --component clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: crates/horizon-telemetry-server
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
- run: cargo +stable fmt --check
|
||||
- run: cargo +stable clippy --all-targets --locked -- -D warnings
|
||||
- run: cargo +stable test --locked
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
[workspace]
|
||||
exclude = ["launcher"]
|
||||
exclude = ["launcher", "crates/horizon-telemetry-server"]
|
||||
members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "xtask"]
|
||||
resolver = "2"
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
/target/
|
||||
+1106
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,31 @@
|
||||
[package]
|
||||
name = "horizon-telemetry-server"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
rust-version = "1.90"
|
||||
publish = false
|
||||
description = "Receives opt-in data collection batches, keeps them compactly in SQLite and serves anonymous totals"
|
||||
|
||||
# Its own workspace, as the launcher's: the game's workspace never builds a web server.
|
||||
[workspace]
|
||||
|
||||
[dependencies]
|
||||
horizon-telemetry = { path = "../horizon-telemetry", default-features = false }
|
||||
axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json", "query"] }
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "signal", "time"] }
|
||||
# SQLite built in: one file on a small volume is the whole database.
|
||||
rusqlite = { version = "0.37", features = ["bundled"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
ruzstd = "0.9"
|
||||
|
||||
[dev-dependencies]
|
||||
# The uploader the launcher uses, against this server, end to end.
|
||||
horizon-telemetry = { path = "../horizon-telemetry" }
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
http-body-util = "0.1"
|
||||
|
||||
[profile.release]
|
||||
strip = true
|
||||
lto = "thin"
|
||||
@@ -0,0 +1,13 @@
|
||||
# Built from the repository root: docker build -f crates/horizon-telemetry-server/Dockerfile .
|
||||
FROM rust:1-slim AS build
|
||||
WORKDIR /src
|
||||
# The root manifest only for the shared crate's workspace settings (edition, lints).
|
||||
COPY Cargo.toml Cargo.toml
|
||||
COPY crates/horizon-telemetry crates/horizon-telemetry
|
||||
COPY crates/horizon-telemetry-server crates/horizon-telemetry-server
|
||||
RUN cargo build --release --locked --manifest-path crates/horizon-telemetry-server/Cargo.toml
|
||||
|
||||
FROM debian:stable-slim
|
||||
COPY --from=build /src/crates/horizon-telemetry-server/target/release/horizon-telemetry-server /usr/local/bin/
|
||||
ENV HORIZON_TELEMETRY_DB=/data/telemetry.sqlite3 HORIZON_TELEMETRY_TRUST_PROXY=1
|
||||
CMD ["horizon-telemetry-server"]
|
||||
@@ -0,0 +1,58 @@
|
||||
# horizon-telemetry-server
|
||||
|
||||
This server receives what players choose to send (performance summaries, crash and bug reports, benchmark uploads: [docs/telemetry.md](../../docs/telemetry.md)) and stores it in one SQLite file. It serves anonymous totals to the website. It is one small Rust process, about 7 MB of memory when idle.
|
||||
|
||||
cargo run --release # in this folder: its own workspace, like the launcher's
|
||||
PORT=8080 HORIZON_TELEMETRY_DB=/data/telemetry.sqlite3 horizon-telemetry-server
|
||||
|
||||
The launcher sends to it when `HORIZON_TELEMETRY_URL` is set (and to `DEFAULT_ENDPOINT` in `crates/horizon-telemetry` once there is one).
|
||||
|
||||
## Endpoints
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `POST /v1/batch` | A zstd batch (`Content-Encoding: zstd`, at most 256 KiB, 1 MiB inflated, 64 items). Returns 204. Returns 400 for a bad batch, 413 for one too large, 415 if it is not zstd, 429 when over a limit. |
|
||||
| `DELETE /v1/installs/<id>` | Deletes everything that install sent. Its share of the daily totals is rebuilt at once. |
|
||||
| `GET /v1/aggregate?days=30` | The totals the website reads. Cached 10 minutes, with `Access-Control-Allow-Origin` set to the website. |
|
||||
| `GET /healthz` | `ok` |
|
||||
|
||||
The totals are grouped per version, OS, GPU vendor and render distance. Each group has its sessions, install-days, hours played, the 14 frame-time buckets summed, and the mean p50 and p99 frame times. Groups with fewer than 5 install-days are left out. The totals also list crash counts per version and the newest 50 benchmark uploads.
|
||||
|
||||
## Storage
|
||||
|
||||
- `systems`: each distinct machine description once. Sessions point at it.
|
||||
- `perf`: one row per session, in columns, its frame and tick summaries as 80-byte blobs.
|
||||
- `reports`: crash and bug reports and benchmarks, their data compressed with zstd.
|
||||
- `perf_daily`: the totals per day, rebuilt from `perf` when a day changes (new sessions, a deletion).
|
||||
|
||||
Measured: **about 190 bytes per play session** on disk, indexes included (`tests/server.rs`, 10 000 sessions). 1 000 players playing twice a day for 180 days is about 70 MB.
|
||||
|
||||
The day stored is the day the server received an item. The sender's IP address is never written down. The per-address rate limit keeps addresses in memory for an hour at most.
|
||||
|
||||
Raw rows are deleted after `HORIZON_TELEMETRY_RETENTION_DAYS` (180). The daily totals stay; they hold no install ID.
|
||||
|
||||
## Settings
|
||||
|
||||
All are environment variables:
|
||||
|
||||
- `PORT`
|
||||
- `HORIZON_TELEMETRY_DB`
|
||||
- `HORIZON_TELEMETRY_RETENTION_DAYS` (180)
|
||||
- `HORIZON_TELEMETRY_MIN_INSTALL_DAYS` (5)
|
||||
- `HORIZON_TELEMETRY_REQUESTS_PER_HOUR` per address (60, bursts of 6)
|
||||
- `HORIZON_TELEMETRY_ITEMS_PER_DAY` per install (100)
|
||||
- `HORIZON_TELEMETRY_TRUST_PROXY=1` behind a proxy (the client's address is then the first of `X-Forwarded-For`)
|
||||
- `HORIZON_TELEMETRY_ALLOW_ORIGIN` (`https://playhorizon.dev`)
|
||||
|
||||
## Hosting
|
||||
|
||||
`Dockerfile` builds it from the repository root (`docker build -f crates/horizon-telemetry-server/Dockerfile .`). The database goes on a volume mounted at `/data`.
|
||||
|
||||
At Railway's prices (RAM $10/GB-month, CPU $20/vCPU-month, volumes $0.15/GB-month, egress $0.05/GB), one more service with a 1 GB volume uses about $0.35 a month:
|
||||
|
||||
- RAM about 15 MB: $0.15
|
||||
- CPU nearly idle: under $0.05
|
||||
- the volume: $0.15
|
||||
- egress, the cached totals: cents
|
||||
|
||||
That fits inside the Hobby plan's included $5 of usage. Railway Postgres would cost a few dollars a month for its memory alone and buys nothing at this size. A bucket isn't needed: crash reports are a few KB compressed and live in the same file.
|
||||
@@ -0,0 +1,307 @@
|
||||
//! The ingest server for opt-in data collection (docs/telemetry.md).
|
||||
//!
|
||||
//! - `POST /v1/batch`: a zstd batch from the launcher, checked by `horizon_telemetry::batch`.
|
||||
//! - `DELETE /v1/installs/<id>`: everything an install sent goes.
|
||||
//! - `GET /v1/aggregate?days=30`: the anonymous totals the website reads, cached.
|
||||
//! - `GET /healthz`.
|
||||
//!
|
||||
//! Abuse limits: a token bucket per IP address (kept in memory for an hour, never stored), a
|
||||
//! daily item cap per install, and the batch size limits. Raw rows expire after
|
||||
//! `retention_days`; the daily totals stay. One small process and one SQLite file: see
|
||||
//! `README.md` for running it.
|
||||
use std::collections::HashMap;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use axum::Router;
|
||||
use axum::extract::{ConnectInfo, Path, Query, Request, State};
|
||||
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{delete, get, post};
|
||||
use horizon_telemetry::batch;
|
||||
|
||||
pub mod store;
|
||||
|
||||
use store::Store;
|
||||
|
||||
pub struct Config {
|
||||
/// Days raw rows are kept.
|
||||
pub retention_days: u32,
|
||||
/// Groups in the public totals need at least this many install-days.
|
||||
pub min_install_days: u64,
|
||||
/// Requests an IP address may make an hour (with a burst of a tenth of it).
|
||||
pub requests_per_hour: f64,
|
||||
/// Items one install may send a day.
|
||||
pub items_per_day: u32,
|
||||
/// Behind a proxy (Railway, Fly…): the client's address is the first of X-Forwarded-For.
|
||||
pub trust_proxy: bool,
|
||||
/// The website allowed to read the totals from a browser.
|
||||
pub allow_origin: String,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
retention_days: 180,
|
||||
min_install_days: 5,
|
||||
requests_per_hour: 60.0,
|
||||
items_per_day: 100,
|
||||
trust_proxy: false,
|
||||
allow_origin: "https://playhorizon.dev".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// The defaults, with `HORIZON_TELEMETRY_RETENTION_DAYS`, `…_MIN_INSTALL_DAYS`,
|
||||
/// `…_REQUESTS_PER_HOUR`, `…_ITEMS_PER_DAY`, `…_TRUST_PROXY` (1) and `…_ALLOW_ORIGIN`.
|
||||
pub fn from_env() -> Self {
|
||||
let var = |k: &str| std::env::var(format!("HORIZON_TELEMETRY_{k}")).ok();
|
||||
let d = Self::default();
|
||||
Self {
|
||||
retention_days: var("RETENTION_DAYS")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(d.retention_days),
|
||||
min_install_days: var("MIN_INSTALL_DAYS")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(d.min_install_days),
|
||||
requests_per_hour: var("REQUESTS_PER_HOUR")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(d.requests_per_hour),
|
||||
items_per_day: var("ITEMS_PER_DAY")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(d.items_per_day),
|
||||
trust_proxy: var("TRUST_PROXY").is_some_and(|v| v == "1"),
|
||||
allow_origin: var("ALLOW_ORIGIN").unwrap_or(d.allow_origin),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct App {
|
||||
store: Mutex<Store>,
|
||||
config: Config,
|
||||
/// Tokens left and when they were counted, per client address.
|
||||
buckets: Mutex<HashMap<IpAddr, (f64, Instant)>>,
|
||||
/// `/v1/aggregate` answers by `days`, with when they were made.
|
||||
cache: Mutex<HashMap<u32, (Instant, String)>>,
|
||||
/// Today, in days since 1970 (a test moves it).
|
||||
today: Box<dyn Fn() -> u32 + Send + Sync>,
|
||||
}
|
||||
|
||||
const CACHE_FOR: Duration = Duration::from_secs(600);
|
||||
|
||||
fn status(code: StatusCode, message: &str) -> Response {
|
||||
(code, message.to_owned()).into_response()
|
||||
}
|
||||
|
||||
impl App {
|
||||
pub fn new(store: Store, config: Config) -> Arc<Self> {
|
||||
Self::with_clock(store, config, Box::new(horizon_telemetry::today))
|
||||
}
|
||||
|
||||
pub fn with_clock(
|
||||
store: Store,
|
||||
config: Config,
|
||||
today: Box<dyn Fn() -> u32 + Send + Sync>,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
store: Mutex::new(store),
|
||||
config,
|
||||
buckets: Mutex::default(),
|
||||
cache: Mutex::default(),
|
||||
today,
|
||||
})
|
||||
}
|
||||
|
||||
fn store(&self) -> std::sync::MutexGuard<'_, Store> {
|
||||
self.store.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
/// Takes a token for `ip`; false when it has none left.
|
||||
fn allow(&self, ip: IpAddr) -> bool {
|
||||
let rate = self.config.requests_per_hour / 3600.0;
|
||||
let burst = (self.config.requests_per_hour / 10.0).max(1.0);
|
||||
let mut buckets = self.buckets.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let now = Instant::now();
|
||||
let (tokens, at) = buckets.entry(ip).or_insert((burst, now));
|
||||
*tokens = (*tokens + now.duration_since(*at).as_secs_f64() * rate).min(burst);
|
||||
*at = now;
|
||||
if *tokens < 1.0 {
|
||||
return false;
|
||||
}
|
||||
*tokens -= 1.0;
|
||||
true
|
||||
}
|
||||
|
||||
fn client(&self, headers: &HeaderMap, peer: Option<SocketAddr>) -> IpAddr {
|
||||
let forwarded = self
|
||||
.config
|
||||
.trust_proxy
|
||||
.then(|| {
|
||||
headers
|
||||
.get("x-forwarded-for")?
|
||||
.to_str()
|
||||
.ok()?
|
||||
.split(',')
|
||||
.next()?
|
||||
.trim()
|
||||
.parse()
|
||||
.ok()
|
||||
})
|
||||
.flatten();
|
||||
forwarded
|
||||
.or(peer.map(|p| p.ip()))
|
||||
.unwrap_or(IpAddr::from([0, 0, 0, 0]))
|
||||
}
|
||||
|
||||
/// The day an install's newest row came in, if it has any left.
|
||||
pub fn newest_day(&self, install: &[u8]) -> Option<u32> {
|
||||
self.store().newest_day(install).ok().flatten()
|
||||
}
|
||||
|
||||
/// Daily upkeep: totals rebuilt, expired rows removed, idle rate-limit entries forgotten.
|
||||
pub fn maintain(&self) -> rusqlite::Result<usize> {
|
||||
let oldest = (self.today)().saturating_sub(self.config.retention_days);
|
||||
let removed = self.store().expire(oldest)?;
|
||||
self.buckets
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.retain(|_, (_, at)| at.elapsed() < Duration::from_secs(3600));
|
||||
if removed > 0 {
|
||||
self.cache.lock().unwrap_or_else(|e| e.into_inner()).clear();
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
}
|
||||
|
||||
fn peer(req: &Request) -> Option<SocketAddr> {
|
||||
req.extensions()
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|c| c.0)
|
||||
}
|
||||
|
||||
async fn receive(State(app): State<Arc<App>>, req: Request) -> Response {
|
||||
let ip = app.client(req.headers(), peer(&req));
|
||||
if !app.allow(ip) {
|
||||
return status(StatusCode::TOO_MANY_REQUESTS, "too many requests");
|
||||
}
|
||||
let zstd = req
|
||||
.headers()
|
||||
.get(header::CONTENT_ENCODING)
|
||||
.is_some_and(|v| v.as_bytes().eq_ignore_ascii_case(b"zstd"));
|
||||
if !zstd {
|
||||
return status(
|
||||
StatusCode::UNSUPPORTED_MEDIA_TYPE,
|
||||
"send zstd (Content-Encoding: zstd)",
|
||||
);
|
||||
}
|
||||
let Ok(body) = axum::body::to_bytes(req.into_body(), batch::MAX_COMPRESSED).await else {
|
||||
return status(StatusCode::PAYLOAD_TOO_LARGE, "a batch is at most 256 KiB");
|
||||
};
|
||||
let batch = match batch::decode(&body) {
|
||||
Ok(batch) => batch,
|
||||
Err(e) => return status(StatusCode::BAD_REQUEST, &e),
|
||||
};
|
||||
let today = (app.today)();
|
||||
let install = store::install_key(&batch.install_id).unwrap_or_default();
|
||||
let mut store = app.store();
|
||||
let sent = store.count_on(&install, today).unwrap_or(u32::MAX);
|
||||
if sent.saturating_add(batch.items.len() as u32) > app.config.items_per_day {
|
||||
return status(
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"this install's daily limit is reached",
|
||||
);
|
||||
}
|
||||
match store.insert(&batch, today) {
|
||||
Ok(()) => StatusCode::NO_CONTENT.into_response(),
|
||||
Err(e) => {
|
||||
eprintln!("horizon-telemetry-server: could not store a batch: {e}");
|
||||
status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn forget(State(app): State<Arc<App>>, Path(id): Path<String>, req: Request) -> Response {
|
||||
if !app.allow(app.client(req.headers(), peer(&req))) {
|
||||
return status(StatusCode::TOO_MANY_REQUESTS, "too many requests");
|
||||
}
|
||||
let Some(install) = store::install_key(&id) else {
|
||||
return status(StatusCode::BAD_REQUEST, "not an install ID");
|
||||
};
|
||||
match app.store().delete_install(&install) {
|
||||
Ok(_) => {
|
||||
app.cache.lock().unwrap_or_else(|e| e.into_inner()).clear();
|
||||
StatusCode::NO_CONTENT.into_response()
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("horizon-telemetry-server: could not delete an install: {e}");
|
||||
status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Window {
|
||||
days: Option<u32>,
|
||||
}
|
||||
|
||||
async fn aggregate(State(app): State<Arc<App>>, Query(window): Query<Window>) -> Response {
|
||||
let days = window.days.unwrap_or(30).clamp(1, 366);
|
||||
let cached = app
|
||||
.cache
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.get(&days)
|
||||
.filter(|(at, _)| at.elapsed() < CACHE_FOR)
|
||||
.map(|(_, json)| json.clone());
|
||||
let json = match cached {
|
||||
Some(json) => json,
|
||||
None => {
|
||||
let to = (app.today)();
|
||||
let made =
|
||||
app.store()
|
||||
.aggregate(to + 1 - days.min(to + 1), to, app.config.min_install_days);
|
||||
match made {
|
||||
Ok(value) => {
|
||||
let json = value.to_string();
|
||||
app.cache
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.insert(days, (Instant::now(), json.clone()));
|
||||
json
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("horizon-telemetry-server: could not build the totals: {e}");
|
||||
return status(StatusCode::SERVICE_UNAVAILABLE, "try again later");
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let origin =
|
||||
HeaderValue::from_str(&app.config.allow_origin).unwrap_or(HeaderValue::from_static("*"));
|
||||
(
|
||||
[
|
||||
(
|
||||
header::CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/json"),
|
||||
),
|
||||
(
|
||||
header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("public, max-age=600"),
|
||||
),
|
||||
(header::ACCESS_CONTROL_ALLOW_ORIGIN, origin),
|
||||
],
|
||||
json,
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub fn router(app: Arc<App>) -> Router {
|
||||
Router::new()
|
||||
.route("/v1/batch", post(receive))
|
||||
.route("/v1/installs/{id}", delete(forget))
|
||||
.route("/v1/aggregate", get(aggregate))
|
||||
.route("/healthz", get(|| async { "ok" }))
|
||||
.with_state(app)
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
//! `horizon-telemetry-server`: serves on `PORT` (default 8080), keeps its data in
|
||||
//! `HORIZON_TELEMETRY_DB` (default `telemetry.sqlite3`), and does its upkeep every ten minutes.
|
||||
use std::net::SocketAddr;
|
||||
use std::time::Duration;
|
||||
|
||||
use horizon_telemetry_server::{App, Config, router, store::Store};
|
||||
|
||||
#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn main() {
|
||||
let path = std::env::var("HORIZON_TELEMETRY_DB").unwrap_or_else(|_| "telemetry.sqlite3".into());
|
||||
let port: u16 = std::env::var("PORT")
|
||||
.ok()
|
||||
.and_then(|p| p.parse().ok())
|
||||
.unwrap_or(8080);
|
||||
let store = Store::open(&path).unwrap_or_else(|e| panic!("{path}: {e}"));
|
||||
let app = App::new(store, Config::from_env());
|
||||
let upkeep = app.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut every = tokio::time::interval(Duration::from_secs(600));
|
||||
loop {
|
||||
every.tick().await;
|
||||
if let Err(e) = upkeep.maintain() {
|
||||
eprintln!("horizon-telemetry-server: upkeep failed: {e}");
|
||||
}
|
||||
}
|
||||
});
|
||||
let listener = tokio::net::TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], port)))
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("port {port}: {e}"));
|
||||
eprintln!("horizon-telemetry-server: listening on {port}, data in {path}");
|
||||
axum::serve(
|
||||
listener,
|
||||
router(app).into_make_service_with_connect_info::<SocketAddr>(),
|
||||
)
|
||||
.with_graceful_shutdown(async {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("{e}"));
|
||||
}
|
||||
@@ -0,0 +1,428 @@
|
||||
//! The database: one SQLite file, kept small.
|
||||
//!
|
||||
//! - `systems`: each distinct machine description once. Rows point at it, so a CPU and GPU
|
||||
//! name isn't stored again with every session.
|
||||
//! - `perf`: one row per play session, in columns. A frame or tick summary is an 80-byte blob
|
||||
//! ([`pack`]), not JSON.
|
||||
//! - `reports`: crash and bug reports and benchmark uploads, their `data` zstd-compressed.
|
||||
//! - `perf_daily`: the anonymous totals per day, version, OS, GPU vendor and render distance
|
||||
//! that `/v1/aggregate` reads. Rebuilt from `perf` when a day changes (new sessions, a
|
||||
//! deletion), and kept after the raw rows expire.
|
||||
//!
|
||||
//! Dates are the day the server received an item (days since 1970, UTC). The sender's IP
|
||||
//! address is never stored.
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use horizon_telemetry::Kind;
|
||||
use horizon_telemetry::batch::Batch;
|
||||
use horizon_telemetry::payload::{BUCKET_EDGES_MS, Payload, Summary, System};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
const BUCKETS: usize = BUCKET_EDGES_MS.len() + 1;
|
||||
|
||||
const SCHEMA: &str = "
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
CREATE TABLE IF NOT EXISTS systems (
|
||||
id INTEGER PRIMARY KEY,
|
||||
os TEXT NOT NULL, os_version TEXT NOT NULL, arch TEXT NOT NULL, cpu TEXT NOT NULL,
|
||||
threads INTEGER NOT NULL, ram_gb INTEGER NOT NULL,
|
||||
gpu TEXT NOT NULL, driver TEXT NOT NULL, backend TEXT NOT NULL,
|
||||
UNIQUE (os, os_version, arch, cpu, threads, ram_gb, gpu, driver, backend)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS perf (
|
||||
install BLOB NOT NULL, day INTEGER NOT NULL, version TEXT NOT NULL, system INTEGER NOT NULL,
|
||||
render_distance INTEGER NOT NULL, simulation_distance INTEGER NOT NULL, max_fps INTEGER NOT NULL,
|
||||
vsync INTEGER NOT NULL, fullscreen INTEGER NOT NULL, graphics TEXT NOT NULL,
|
||||
seconds INTEGER NOT NULL, frames BLOB NOT NULL, ticks BLOB,
|
||||
peak_ram_mb INTEGER NOT NULL, to_title_ms INTEGER, to_world_ms INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS perf_install ON perf (install);
|
||||
CREATE INDEX IF NOT EXISTS perf_day ON perf (day);
|
||||
CREATE TABLE IF NOT EXISTS reports (
|
||||
install BLOB NOT NULL, day INTEGER NOT NULL, kind INTEGER NOT NULL, version TEXT NOT NULL,
|
||||
system INTEGER, body BLOB NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS reports_install ON reports (install);
|
||||
CREATE INDEX IF NOT EXISTS reports_kind_day ON reports (kind, day);
|
||||
CREATE TABLE IF NOT EXISTS perf_daily (
|
||||
day INTEGER NOT NULL, version TEXT NOT NULL, os TEXT NOT NULL, gpu_vendor TEXT NOT NULL,
|
||||
render_distance INTEGER NOT NULL,
|
||||
sessions INTEGER NOT NULL, installs INTEGER NOT NULL, seconds INTEGER NOT NULL,
|
||||
buckets TEXT NOT NULL, p50_sum REAL NOT NULL, p99_sum REAL NOT NULL,
|
||||
PRIMARY KEY (day, version, os, gpu_vendor, render_distance)
|
||||
) WITHOUT ROWID;
|
||||
CREATE TABLE IF NOT EXISTS dirty (day INTEGER PRIMARY KEY) WITHOUT ROWID;
|
||||
";
|
||||
|
||||
fn kind_code(kind: Kind) -> i64 {
|
||||
match kind {
|
||||
Kind::Perf => 0,
|
||||
Kind::Crash => 1,
|
||||
Kind::Bug => 2,
|
||||
Kind::Benchmark => 3,
|
||||
}
|
||||
}
|
||||
|
||||
/// An install ID as 16 bytes.
|
||||
pub fn install_key(id: &str) -> Option<Vec<u8>> {
|
||||
horizon_telemetry::consent::is_install_id(id).then(|| {
|
||||
(0..16)
|
||||
.map(|i| u8::from_str_radix(&id[2 * i..2 * i + 2], 16).unwrap_or(0))
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
|
||||
/// A summary as 80 bytes: the count, four percentiles, then the 14 buckets, little-endian.
|
||||
pub fn pack(s: &Summary) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(8 + 16 + 4 * BUCKETS);
|
||||
out.extend(s.count.to_le_bytes());
|
||||
for ms in [s.p50_ms, s.p95_ms, s.p99_ms, s.max_ms] {
|
||||
out.extend(ms.to_le_bytes());
|
||||
}
|
||||
for i in 0..BUCKETS {
|
||||
out.extend(s.buckets.get(i).copied().unwrap_or(0).to_le_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn unpack(bytes: &[u8]) -> Option<Summary> {
|
||||
if bytes.len() != 8 + 16 + 4 * BUCKETS {
|
||||
return None;
|
||||
}
|
||||
let u32_at = |i: usize| u32::from_le_bytes(bytes[i..i + 4].try_into().unwrap_or_default());
|
||||
let f32_at = |i: usize| f32::from_bits(u32_at(i));
|
||||
Some(Summary {
|
||||
count: u64::from_le_bytes(bytes[..8].try_into().ok()?),
|
||||
p50_ms: f32_at(8),
|
||||
p95_ms: f32_at(12),
|
||||
p99_ms: f32_at(16),
|
||||
max_ms: f32_at(20),
|
||||
buckets: (0..BUCKETS).map(|i| u32_at(24 + 4 * i)).collect(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The GPU's maker, the one GPU detail the public totals group by.
|
||||
pub fn gpu_vendor(gpu: &str) -> &'static str {
|
||||
let gpu = gpu.to_ascii_lowercase();
|
||||
[
|
||||
("apple", "apple"),
|
||||
("nvidia", "nvidia"),
|
||||
("geforce", "nvidia"),
|
||||
("radeon", "amd"),
|
||||
("amd", "amd"),
|
||||
("intel", "intel"),
|
||||
("adreno", "qualcomm"),
|
||||
("llvmpipe", "software"),
|
||||
]
|
||||
.into_iter()
|
||||
.find(|(needle, _)| gpu.contains(needle))
|
||||
.map_or(if gpu.is_empty() { "unknown" } else { "other" }, |(_, v)| v)
|
||||
}
|
||||
|
||||
pub struct Store {
|
||||
db: Connection,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
pub fn open(path: &str) -> rusqlite::Result<Self> {
|
||||
let db = Connection::open(path)?;
|
||||
db.execute_batch(SCHEMA)?;
|
||||
Ok(Self { db })
|
||||
}
|
||||
|
||||
/// How many items an install has sent on `day`.
|
||||
pub fn count_on(&self, install: &[u8], day: u32) -> rusqlite::Result<u32> {
|
||||
self.db.query_row(
|
||||
"SELECT (SELECT count(*) FROM perf WHERE install = ?1 AND day = ?2)
|
||||
+ (SELECT count(*) FROM reports WHERE install = ?1 AND day = ?2)",
|
||||
params![install, day],
|
||||
|r| r.get(0),
|
||||
)
|
||||
}
|
||||
|
||||
/// Keeps a decoded batch, received on `day`.
|
||||
pub fn insert(&mut self, batch: &Batch, day: u32) -> rusqlite::Result<()> {
|
||||
let install = install_key(&batch.install_id).unwrap_or_default();
|
||||
let tx = self.db.transaction()?;
|
||||
for item in &batch.items {
|
||||
match &item.payload {
|
||||
Payload::Perf(p) => {
|
||||
let system = system_id(&tx, &p.system)?;
|
||||
tx.execute(
|
||||
"INSERT INTO perf (install, day, version, system, render_distance, simulation_distance,
|
||||
max_fps, vsync, fullscreen, graphics, seconds, frames, ticks, peak_ram_mb,
|
||||
to_title_ms, to_world_ms) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
|
||||
params![
|
||||
install, day, item.version, system,
|
||||
p.settings.render_distance, p.settings.simulation_distance, p.settings.max_fps,
|
||||
p.settings.vsync, p.settings.fullscreen, p.settings.graphics,
|
||||
p.seconds, pack(&p.frames), p.ticks.as_ref().map(pack), p.peak_ram_mb,
|
||||
p.to_title_ms, p.to_world_ms,
|
||||
],
|
||||
)?;
|
||||
}
|
||||
payload => {
|
||||
let system = match payload {
|
||||
Payload::Crash(c) => Some(system_id(&tx, &c.system)?),
|
||||
Payload::Bug(b) => Some(system_id(&tx, &b.system)?),
|
||||
_ => None,
|
||||
};
|
||||
let data = match payload {
|
||||
Payload::Crash(c) => serde_json::to_vec(c),
|
||||
Payload::Bug(b) => serde_json::to_vec(b),
|
||||
Payload::Benchmark(r) => serde_json::to_vec(r),
|
||||
Payload::Perf(_) => unreachable!("handled above"),
|
||||
}
|
||||
.unwrap_or_default();
|
||||
let body = ruzstd::encoding::compress_to_vec(
|
||||
data.as_slice(),
|
||||
ruzstd::encoding::CompressionLevel::Fastest,
|
||||
);
|
||||
tx.execute(
|
||||
"INSERT INTO reports (install, day, kind, version, system, body) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![install, day, kind_code(item.kind()), item.version, system, body],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.execute("INSERT OR IGNORE INTO dirty (day) VALUES (?1)", [day])?;
|
||||
tx.commit()
|
||||
}
|
||||
|
||||
/// Deletes everything an install sent; the days it touched get their totals rebuilt.
|
||||
pub fn delete_install(&mut self, install: &[u8]) -> rusqlite::Result<usize> {
|
||||
let tx = self.db.transaction()?;
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO dirty (day) SELECT DISTINCT day FROM perf WHERE install = ?1",
|
||||
[install],
|
||||
)?;
|
||||
let mut deleted = tx.execute("DELETE FROM perf WHERE install = ?1", [install])?;
|
||||
deleted += tx.execute("DELETE FROM reports WHERE install = ?1", [install])?;
|
||||
tx.commit()?;
|
||||
self.rebuild_dirty()?;
|
||||
Ok(deleted)
|
||||
}
|
||||
|
||||
/// Removes raw rows received before `oldest_day`; the daily totals stay.
|
||||
pub fn expire(&mut self, oldest_day: u32) -> rusqlite::Result<usize> {
|
||||
// A day's totals are final before its rows go.
|
||||
self.rebuild_dirty()?;
|
||||
let mut n = self
|
||||
.db
|
||||
.execute("DELETE FROM perf WHERE day < ?1", [oldest_day])?;
|
||||
n += self
|
||||
.db
|
||||
.execute("DELETE FROM reports WHERE day < ?1", [oldest_day])?;
|
||||
self.db.execute(
|
||||
"DELETE FROM systems WHERE id NOT IN (SELECT system FROM perf)
|
||||
AND id NOT IN (SELECT system FROM reports WHERE system IS NOT NULL)",
|
||||
[],
|
||||
)?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Rebuilds the daily totals of every day marked changed.
|
||||
pub fn rebuild_dirty(&mut self) -> rusqlite::Result<()> {
|
||||
let days: Vec<u32> = {
|
||||
let mut q = self.db.prepare("SELECT day FROM dirty")?;
|
||||
q.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?
|
||||
};
|
||||
for day in days {
|
||||
self.rebuild_day(day)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn rebuild_day(&mut self, day: u32) -> rusqlite::Result<()> {
|
||||
#[derive(Default)]
|
||||
struct Group {
|
||||
sessions: u64,
|
||||
installs: HashSet<Vec<u8>>,
|
||||
seconds: u64,
|
||||
buckets: [u64; BUCKETS],
|
||||
p50_sum: f64,
|
||||
p99_sum: f64,
|
||||
}
|
||||
let tx = self.db.transaction()?;
|
||||
let mut groups: HashMap<(String, String, &'static str, u8), Group> = HashMap::new();
|
||||
{
|
||||
let mut q = tx.prepare(
|
||||
"SELECT perf.install, perf.version, systems.os, systems.gpu, perf.render_distance,
|
||||
perf.seconds, perf.frames FROM perf JOIN systems ON systems.id = perf.system
|
||||
WHERE perf.day = ?1",
|
||||
)?;
|
||||
let mut rows = q.query([day])?;
|
||||
while let Some(r) = rows.next()? {
|
||||
let gpu: String = r.get(3)?;
|
||||
let key = (r.get(1)?, r.get(2)?, gpu_vendor(&gpu), r.get(4)?);
|
||||
let Some(frames) = unpack(&r.get::<_, Vec<u8>>(6)?) else {
|
||||
continue;
|
||||
};
|
||||
let g = groups.entry(key).or_default();
|
||||
g.sessions += 1;
|
||||
g.installs.insert(r.get(0)?);
|
||||
g.seconds += r.get::<_, u64>(5)?;
|
||||
for (sum, n) in g.buckets.iter_mut().zip(&frames.buckets) {
|
||||
*sum += u64::from(*n);
|
||||
}
|
||||
g.p50_sum += f64::from(frames.p50_ms);
|
||||
g.p99_sum += f64::from(frames.p99_ms);
|
||||
}
|
||||
}
|
||||
tx.execute("DELETE FROM perf_daily WHERE day = ?1", [day])?;
|
||||
for ((version, os, vendor, distance), g) in groups {
|
||||
tx.execute(
|
||||
"INSERT INTO perf_daily VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
|
||||
params![
|
||||
day,
|
||||
version,
|
||||
os,
|
||||
vendor,
|
||||
distance,
|
||||
g.sessions,
|
||||
g.installs.len(),
|
||||
g.seconds,
|
||||
serde_json::to_string(&g.buckets.to_vec()).unwrap_or_default(),
|
||||
g.p50_sum,
|
||||
g.p99_sum,
|
||||
],
|
||||
)?;
|
||||
}
|
||||
tx.execute("DELETE FROM dirty WHERE day = ?1", [day])?;
|
||||
tx.commit()
|
||||
}
|
||||
|
||||
/// The anonymous totals for the days `from..=to`, groups with fewer than `k` install-days
|
||||
/// left out, and the newest benchmark uploads.
|
||||
pub fn aggregate(&mut self, from: u32, to: u32, k: u64) -> rusqlite::Result<serde_json::Value> {
|
||||
self.rebuild_dirty()?;
|
||||
/// One group's sums over the window.
|
||||
#[derive(Default)]
|
||||
struct Totals {
|
||||
sessions: u64,
|
||||
install_days: u64,
|
||||
seconds: u64,
|
||||
buckets: [u64; BUCKETS],
|
||||
p50_sum: f64,
|
||||
p99_sum: f64,
|
||||
}
|
||||
let mut groups: HashMap<(String, String, String, u8), Totals> = HashMap::new();
|
||||
{
|
||||
let mut q = self.db.prepare(
|
||||
"SELECT version, os, gpu_vendor, render_distance, sessions, installs, seconds, buckets,
|
||||
p50_sum, p99_sum FROM perf_daily WHERE day BETWEEN ?1 AND ?2",
|
||||
)?;
|
||||
let mut rows = q.query([from, to])?;
|
||||
while let Some(r) = rows.next()? {
|
||||
let g = groups
|
||||
.entry((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))
|
||||
.or_default();
|
||||
g.sessions += r.get::<_, u64>(4)?;
|
||||
g.install_days += r.get::<_, u64>(5)?;
|
||||
g.seconds += r.get::<_, u64>(6)?;
|
||||
let buckets: Vec<u64> =
|
||||
serde_json::from_str(&r.get::<_, String>(7)?).unwrap_or_default();
|
||||
for (sum, n) in g.buckets.iter_mut().zip(buckets) {
|
||||
*sum += n;
|
||||
}
|
||||
g.p50_sum += r.get::<_, f64>(8)?;
|
||||
g.p99_sum += r.get::<_, f64>(9)?;
|
||||
}
|
||||
}
|
||||
let (mut sessions, mut hours) = (0, 0.0);
|
||||
let mut out: Vec<serde_json::Value> = groups
|
||||
.into_iter()
|
||||
.filter(|(_, g)| g.install_days >= k)
|
||||
.map(|((version, os, vendor, distance), g)| {
|
||||
sessions += g.sessions;
|
||||
hours += g.seconds as f64 / 3600.0;
|
||||
let avg = |sum: f64| (sum / g.sessions as f64 * 10.0).round() / 10.0;
|
||||
serde_json::json!({
|
||||
"version": version, "os": os, "gpu_vendor": vendor, "render_distance": distance,
|
||||
"sessions": g.sessions, "install_days": g.install_days,
|
||||
"hours": (g.seconds as f64 / 360.0).round() / 10.0,
|
||||
"frame_buckets": g.buckets.to_vec(), "p50_ms": avg(g.p50_sum), "p99_ms": avg(g.p99_sum),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
out.sort_by(|a, b| b["sessions"].as_u64().cmp(&a["sessions"].as_u64()));
|
||||
let crashes: Vec<serde_json::Value> = {
|
||||
let mut q = self.db.prepare(
|
||||
"SELECT version, count(*) FROM reports WHERE kind = ?1 AND day BETWEEN ?2 AND ?3
|
||||
GROUP BY version ORDER BY 2 DESC LIMIT 20",
|
||||
)?;
|
||||
q.query_map(params![kind_code(Kind::Crash), from, to], |r| {
|
||||
Ok(serde_json::json!({ "version": r.get::<_, String>(0)?, "crashes": r.get::<_, u64>(1)? }))
|
||||
})?
|
||||
.collect::<Result<_, _>>()?
|
||||
};
|
||||
let benchmarks: Vec<serde_json::Value> = {
|
||||
let mut q = self.db.prepare(
|
||||
"SELECT day, version, body FROM reports WHERE kind = ?1 AND day BETWEEN ?2 AND ?3
|
||||
ORDER BY rowid DESC LIMIT 50",
|
||||
)?;
|
||||
q.query_map(params![kind_code(Kind::Benchmark), from, to], |r| {
|
||||
let body: Vec<u8> = r.get(2)?;
|
||||
Ok(serde_json::json!({
|
||||
"day": r.get::<_, u32>(0)?, "version": r.get::<_, String>(1)?, "report": inflate(&body),
|
||||
}))
|
||||
})?
|
||||
.collect::<Result<_, _>>()?
|
||||
};
|
||||
Ok(serde_json::json!({
|
||||
"from_day": from, "to_day": to, "min_install_days": k,
|
||||
"bucket_edges_ms": BUCKET_EDGES_MS.map(|e| (f64::from(e) * 10.0).round() / 10.0),
|
||||
"totals": { "sessions": sessions, "hours": (hours * 10.0).round() / 10.0 },
|
||||
"groups": out, "crashes_by_version": crashes, "benchmarks": benchmarks,
|
||||
}))
|
||||
}
|
||||
|
||||
/// The day an install's newest row came in, if any (tests, and checking a deletion).
|
||||
pub fn newest_day(&self, install: &[u8]) -> rusqlite::Result<Option<u32>> {
|
||||
self.db
|
||||
.query_row(
|
||||
"SELECT max(day) FROM (SELECT day FROM perf WHERE install = ?1
|
||||
UNION ALL SELECT day FROM reports WHERE install = ?1)",
|
||||
[install],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map(Option::flatten)
|
||||
}
|
||||
}
|
||||
|
||||
fn system_id(tx: &rusqlite::Transaction, s: &System) -> rusqlite::Result<i64> {
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO systems (os, os_version, arch, cpu, threads, ram_gb, gpu, driver, backend)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
|
||||
params![s.os, s.os_version, s.arch, s.cpu, s.threads, s.ram_gb, s.gpu, s.driver, s.backend],
|
||||
)?;
|
||||
tx.query_row(
|
||||
"SELECT id FROM systems WHERE os = ?1 AND os_version = ?2 AND arch = ?3 AND cpu = ?4
|
||||
AND threads = ?5 AND ram_gb = ?6 AND gpu = ?7 AND driver = ?8 AND backend = ?9",
|
||||
params![
|
||||
s.os,
|
||||
s.os_version,
|
||||
s.arch,
|
||||
s.cpu,
|
||||
s.threads,
|
||||
s.ram_gb,
|
||||
s.gpu,
|
||||
s.driver,
|
||||
s.backend
|
||||
],
|
||||
|r| r.get(0),
|
||||
)
|
||||
}
|
||||
|
||||
/// A stored body back as JSON.
|
||||
pub fn inflate(body: &[u8]) -> serde_json::Value {
|
||||
use std::io::Read;
|
||||
let mut json = Vec::new();
|
||||
ruzstd::decoding::StreamingDecoder::new(body)
|
||||
.ok()
|
||||
.and_then(|d| d.take(1 << 20).read_to_end(&mut json).ok())
|
||||
.and_then(|_| serde_json::from_slice(&json).ok())
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
//! The server through its HTTP interface, and end to end with the launcher's uploader.
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::http::{Request, StatusCode};
|
||||
use horizon_telemetry::batch::{self, Batch};
|
||||
use horizon_telemetry::payload::{Crash, Payload, Perf, Settings, Source, System, Timings};
|
||||
use horizon_telemetry::{Item, outbox};
|
||||
use horizon_telemetry_server::store::{self, Store};
|
||||
use horizon_telemetry_server::{App, Config, router};
|
||||
use http_body_util::BodyExt;
|
||||
use tower::ServiceExt;
|
||||
|
||||
const DAY: u32 = 20_741;
|
||||
|
||||
fn install(n: u32) -> String {
|
||||
format!("{n:032x}")
|
||||
}
|
||||
|
||||
fn perf(gpu: &str, distance: u8) -> Item {
|
||||
let mut t = Timings::new();
|
||||
(0..30_000).for_each(|i| t.record(8.0 + (i % 50) as f32 / 10.0));
|
||||
let mut item = Item::new(
|
||||
Source::Game,
|
||||
"v2026.10.15",
|
||||
Payload::Perf(Perf {
|
||||
system: System {
|
||||
os: "linux".into(),
|
||||
os_version: "Fedora Linux 43".into(),
|
||||
cpu: "AMD Ryzen 9 7950X".into(),
|
||||
threads: 32,
|
||||
ram_gb: 32,
|
||||
gpu: gpu.into(),
|
||||
driver: "570.86".into(),
|
||||
backend: "Vulkan".into(),
|
||||
..System::default()
|
||||
},
|
||||
settings: Settings {
|
||||
render_distance: distance,
|
||||
simulation_distance: 12,
|
||||
graphics: "fancy".into(),
|
||||
..Settings::default()
|
||||
},
|
||||
seconds: 1800,
|
||||
frames: t.summary(),
|
||||
peak_ram_mb: 2300,
|
||||
..Perf::default()
|
||||
}),
|
||||
);
|
||||
item.day = DAY;
|
||||
item
|
||||
}
|
||||
|
||||
struct Server {
|
||||
app: Arc<App>,
|
||||
day: Arc<AtomicU32>,
|
||||
}
|
||||
|
||||
fn server(config: Config) -> Server {
|
||||
let day = Arc::new(AtomicU32::new(DAY));
|
||||
let clock = day.clone();
|
||||
let app = App::with_clock(
|
||||
Store::open(":memory:").unwrap(),
|
||||
config,
|
||||
Box::new(move || clock.load(Ordering::SeqCst)),
|
||||
);
|
||||
Server { app, day }
|
||||
}
|
||||
|
||||
impl Server {
|
||||
async fn send(&self, req: Request<Body>) -> (StatusCode, String) {
|
||||
let res = router(self.app.clone()).oneshot(req).await.unwrap();
|
||||
let code = res.status();
|
||||
let body = res.into_body().collect().await.unwrap().to_bytes();
|
||||
(code, String::from_utf8_lossy(&body).into_owned())
|
||||
}
|
||||
|
||||
async fn post(&self, id: &str, items: Vec<Item>) -> StatusCode {
|
||||
let body = batch::encode(&Batch::new(id, items));
|
||||
self.post_raw(body, Some("zstd")).await
|
||||
}
|
||||
|
||||
async fn post_raw(&self, body: Vec<u8>, encoding: Option<&str>) -> StatusCode {
|
||||
let mut req = Request::post("/v1/batch").header("x-forwarded-for", "203.0.113.9");
|
||||
if let Some(e) = encoding {
|
||||
req = req.header("content-encoding", e);
|
||||
}
|
||||
self.send(req.body(Body::from(body)).unwrap()).await.0
|
||||
}
|
||||
|
||||
async fn aggregate(&self) -> serde_json::Value {
|
||||
let (code, body) = self
|
||||
.send(
|
||||
Request::get("/v1/aggregate?days=30")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code, StatusCode::OK);
|
||||
serde_json::from_str(&body).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
fn generous() -> Config {
|
||||
Config {
|
||||
requests_per_hour: 100_000.0,
|
||||
..Config::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn totals_show_only_groups_with_enough_installs() {
|
||||
let s = server(generous());
|
||||
for n in 0..6 {
|
||||
assert_eq!(
|
||||
s.post(&install(n), vec![perf("NVIDIA GeForce RTX 4070", 12); 2])
|
||||
.await,
|
||||
StatusCode::NO_CONTENT
|
||||
);
|
||||
}
|
||||
for n in 10..12 {
|
||||
assert_eq!(
|
||||
s.post(&install(n), vec![perf("Intel(R) Arc A770", 8)])
|
||||
.await,
|
||||
StatusCode::NO_CONTENT
|
||||
);
|
||||
}
|
||||
let totals = s.aggregate().await;
|
||||
let groups = totals["groups"].as_array().unwrap();
|
||||
assert_eq!(
|
||||
groups.len(),
|
||||
1,
|
||||
"two installs are too few to show: {totals}"
|
||||
);
|
||||
let g = &groups[0];
|
||||
assert_eq!(g["gpu_vendor"], "nvidia");
|
||||
assert_eq!(
|
||||
(g["sessions"].as_u64(), g["install_days"].as_u64()),
|
||||
(Some(12), Some(6))
|
||||
);
|
||||
assert_eq!(g["hours"], 6.0);
|
||||
assert_eq!(
|
||||
g["frame_buckets"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|b| b.as_u64().unwrap())
|
||||
.sum::<u64>(),
|
||||
12 * 30_000
|
||||
);
|
||||
assert_eq!(totals["bucket_edges_ms"].as_array().unwrap().len(), 13);
|
||||
let text = totals.to_string();
|
||||
assert!(
|
||||
!text.contains(&install(0)) && !text.contains("Ryzen"),
|
||||
"no install or machine in the totals"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bad_batches_are_refused() {
|
||||
let s = server(generous());
|
||||
let good = batch::encode(&Batch::new(&install(1), vec![perf("x", 8)]));
|
||||
assert_eq!(
|
||||
s.post_raw(good.clone(), None).await,
|
||||
StatusCode::UNSUPPORTED_MEDIA_TYPE
|
||||
);
|
||||
assert_eq!(
|
||||
s.post_raw(b"garbage".to_vec(), Some("zstd")).await,
|
||||
StatusCode::BAD_REQUEST
|
||||
);
|
||||
assert_eq!(
|
||||
s.post_raw(vec![0; batch::MAX_COMPRESSED + 1], Some("zstd"))
|
||||
.await,
|
||||
StatusCode::PAYLOAD_TOO_LARGE
|
||||
);
|
||||
assert_eq!(
|
||||
s.post("not-an-id", vec![perf("x", 8)]).await,
|
||||
StatusCode::BAD_REQUEST
|
||||
);
|
||||
assert_eq!(s.post_raw(good, Some("zstd")).await, StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_address_is_rate_limited() {
|
||||
let s = server(Config {
|
||||
requests_per_hour: 30.0,
|
||||
trust_proxy: true,
|
||||
..Config::default()
|
||||
});
|
||||
for n in 0..3 {
|
||||
assert_eq!(
|
||||
s.post(&install(n), vec![perf("x", 8)]).await,
|
||||
StatusCode::NO_CONTENT
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
s.post(&install(9), vec![perf("x", 8)]).await,
|
||||
StatusCode::TOO_MANY_REQUESTS
|
||||
);
|
||||
// Another address has its own allowance.
|
||||
let body = batch::encode(&Batch::new(&install(9), vec![perf("x", 8)]));
|
||||
let req = Request::post("/v1/batch")
|
||||
.header("x-forwarded-for", "198.51.100.7, 10.0.0.1")
|
||||
.header("content-encoding", "zstd")
|
||||
.body(Body::from(body))
|
||||
.unwrap();
|
||||
assert_eq!(s.send(req).await.0, StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_install_has_a_daily_cap() {
|
||||
let s = server(Config {
|
||||
items_per_day: 3,
|
||||
..generous()
|
||||
});
|
||||
assert_eq!(
|
||||
s.post(&install(1), vec![perf("x", 8); 2]).await,
|
||||
StatusCode::NO_CONTENT
|
||||
);
|
||||
assert_eq!(
|
||||
s.post(&install(1), vec![perf("x", 8); 2]).await,
|
||||
StatusCode::TOO_MANY_REQUESTS
|
||||
);
|
||||
assert_eq!(
|
||||
s.post(&install(1), vec![perf("x", 8)]).await,
|
||||
StatusCode::NO_CONTENT
|
||||
);
|
||||
s.day.store(DAY + 1, Ordering::SeqCst);
|
||||
assert_eq!(
|
||||
s.post(&install(1), vec![perf("x", 8); 3]).await,
|
||||
StatusCode::NO_CONTENT
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deleting_an_install_removes_its_rows_and_its_share_of_the_totals() {
|
||||
let s = server(Config {
|
||||
min_install_days: 1,
|
||||
..generous()
|
||||
});
|
||||
let crash = Item::new(Source::Game, "v1", Payload::Crash(Crash::default()));
|
||||
assert_eq!(
|
||||
s.post(&install(1), vec![perf("Apple M4 Pro", 12), crash])
|
||||
.await,
|
||||
StatusCode::NO_CONTENT
|
||||
);
|
||||
assert_eq!(
|
||||
s.post(&install(2), vec![perf("Apple M4 Pro", 12)]).await,
|
||||
StatusCode::NO_CONTENT
|
||||
);
|
||||
assert_eq!(s.aggregate().await["groups"][0]["sessions"], 2);
|
||||
let (code, _) = s
|
||||
.send(
|
||||
Request::delete(format!("/v1/installs/{}", install(1)))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code, StatusCode::NO_CONTENT);
|
||||
let key = store::install_key(&install(1)).unwrap();
|
||||
assert_eq!(s.app.newest_day(&key), None);
|
||||
let totals = s.aggregate().await;
|
||||
assert_eq!(
|
||||
totals["groups"][0]["sessions"], 1,
|
||||
"the cache was dropped and the day rebuilt"
|
||||
);
|
||||
assert_eq!(totals["crashes_by_version"].as_array().unwrap().len(), 0);
|
||||
let (code, _) = s
|
||||
.send(
|
||||
Request::delete("/v1/installs/nope")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code, StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raw_rows_expire_and_the_totals_stay() {
|
||||
let s = server(Config {
|
||||
min_install_days: 1,
|
||||
retention_days: 180,
|
||||
..generous()
|
||||
});
|
||||
assert_eq!(
|
||||
s.post(&install(1), vec![perf("Apple M4 Pro", 12)]).await,
|
||||
StatusCode::NO_CONTENT
|
||||
);
|
||||
s.day.store(DAY + 179, Ordering::SeqCst);
|
||||
assert_eq!(s.app.maintain().unwrap(), 0, "still within retention");
|
||||
s.day.store(DAY + 181, Ordering::SeqCst);
|
||||
assert_eq!(s.app.maintain().unwrap(), 1);
|
||||
assert_eq!(
|
||||
s.app.newest_day(&store::install_key(&install(1)).unwrap()),
|
||||
None
|
||||
);
|
||||
let (_, body) = s
|
||||
.send(
|
||||
Request::get("/v1/aggregate?days=366")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
let totals: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||
assert_eq!(totals["groups"][0]["sessions"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_session_costs_little_storage() {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"horizon-telemetry-server-size-{}",
|
||||
std::process::id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("t.sqlite3");
|
||||
let mut store = Store::open(path.to_str().unwrap()).unwrap();
|
||||
let gpus = [
|
||||
"Apple M4 Pro",
|
||||
"NVIDIA GeForce RTX 4070",
|
||||
"AMD Radeon RX 7800 XT",
|
||||
"Intel(R) Arc A770",
|
||||
];
|
||||
let sessions = 10_000;
|
||||
for n in 0..sessions / 10 {
|
||||
let items = (0..10)
|
||||
.map(|i| perf(gpus[(n + i) as usize % 4], 8 + (i % 3) as u8 * 4))
|
||||
.collect();
|
||||
store
|
||||
.insert(&Batch::new(&install(n), items), DAY + n % 30)
|
||||
.unwrap();
|
||||
}
|
||||
store.rebuild_dirty().unwrap();
|
||||
drop(store);
|
||||
let bytes: u64 = std::fs::read_dir(&dir)
|
||||
.unwrap()
|
||||
.flatten()
|
||||
.map(|e| e.metadata().unwrap().len())
|
||||
.sum();
|
||||
let per = bytes / u64::from(sessions);
|
||||
eprintln!("{sessions} sessions: {bytes} bytes on disk, {per} bytes a session");
|
||||
assert!(per < 250, "{per} bytes a session");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_launchers_uploader_reaches_it_end_to_end() {
|
||||
let s = server(Config {
|
||||
min_install_days: 1,
|
||||
..generous()
|
||||
});
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let service =
|
||||
router(s.app.clone()).into_make_service_with_connect_info::<std::net::SocketAddr>();
|
||||
tokio::spawn(async move { axum::serve(listener, service).await.unwrap() });
|
||||
|
||||
let home = std::env::temp_dir().join(format!(
|
||||
"horizon-telemetry-server-e2e-{}",
|
||||
std::process::id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&home);
|
||||
let mut consent = horizon_telemetry::Consent::load(&home);
|
||||
consent.answer(horizon_telemetry::Choices::ALL);
|
||||
consent.save(&home).unwrap();
|
||||
for _ in 0..3 {
|
||||
assert!(outbox::push(&home, &perf("Apple M4 Pro", 12)).unwrap());
|
||||
}
|
||||
let (h, u) = (home.clone(), url.clone());
|
||||
let sent = tokio::task::spawn_blocking(move || horizon_telemetry::upload::flush(&h, &u))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
sent,
|
||||
horizon_telemetry::upload::Flush::Sent {
|
||||
items: 3,
|
||||
batches: 1
|
||||
}
|
||||
);
|
||||
assert_eq!(s.aggregate().await["groups"][0]["sessions"], 3);
|
||||
|
||||
let id = consent.install_id.clone();
|
||||
let deleted =
|
||||
tokio::task::spawn_blocking(move || horizon_telemetry::upload::delete_remote(&url, &id))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(deleted, Ok(()));
|
||||
assert_eq!(
|
||||
s.app
|
||||
.newest_day(&store::install_key(&consent.install_id).unwrap()),
|
||||
None
|
||||
);
|
||||
}
|
||||
+1
-1
@@ -52,4 +52,4 @@ Never sent: account names or UUIDs, world names, server or LAN addresses, chat,
|
||||
- `DELETE /v1/installs/<install_id>`: deletes everything sent under that ID.
|
||||
- `GET /v1/aggregate`: the anonymous totals the website's results page reads.
|
||||
|
||||
Retention, rate limits and storage are described with the server.
|
||||
`crates/horizon-telemetry-server` is the server. Its [README](../crates/horizon-telemetry-server/README.md) covers storage (about 190 bytes per play session), retention (180 days of raw rows, anonymous daily totals kept), rate limits and hosting.
|
||||
|
||||
Reference in New Issue
Block a user