Merge pull request #1839 from ludovic111/feat/hub-intake
🤖 Agent: merged after verify passed (hub: website tests 30/30, roadmap tests + sync check, telemetry server 35 tests; local end-to-end run by roadmap-1).
This commit is contained in:
+52
@@ -2,6 +2,12 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "atomic-waker"
|
||||
version = "1.1.2"
|
||||
@@ -92,6 +98,15 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.7"
|
||||
@@ -131,6 +146,17 @@ version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.1.5"
|
||||
@@ -235,6 +261,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tower",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -473,6 +500,16 @@ version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.2.4"
|
||||
@@ -701,6 +738,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
@@ -851,10 +894,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"flate2",
|
||||
"log",
|
||||
"once_cell",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"url",
|
||||
"webpki-roots 0.26.11",
|
||||
]
|
||||
@@ -1099,6 +1145,12 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
|
||||
@@ -19,6 +19,8 @@ rusqlite = { version = "0.37", features = ["bundled"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
ruzstd = "0.9"
|
||||
# Opens the hub's bug reports as GitHub issues (blocking, run off the async workers; rustls).
|
||||
ureq = { version = "2.12", features = ["json"] }
|
||||
|
||||
[dev-dependencies]
|
||||
# The uploader the launcher uses, against this server, end to end.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# horizon-telemetry-server
|
||||
|
||||
This server receives what players choose to send (performance summaries, crash and bug reports, benchmark uploads: [docs/telemetry.md](../../docs/telemetry.md)) and stores it in one SQLite file. It serves anonymous totals to the website. It also keeps the live player count from the games' anonymous "I'm playing" pings, in memory only, and the 👍 players give roadmap items on the hub (playhorizon.dev/hub). It is one small Rust process, about 7 MB of memory when idle.
|
||||
This server receives what players choose to send (performance summaries, crash and bug reports, benchmark uploads: [docs/telemetry.md](../../docs/telemetry.md)) and stores it in one SQLite file. It serves anonymous totals to the website. It also keeps the live player count from the games' anonymous "I'm playing" pings, in memory only, and takes what players send from the hub (playhorizon.dev/hub): votes, bug reports and suggestions. It is one small Rust process, about 7 MB of memory when idle.
|
||||
|
||||
cargo run --release # in this folder: its own workspace, like the launcher's
|
||||
PORT=8080 HORIZON_TELEMETRY_DB=/data/telemetry.sqlite3 horizon-telemetry-server
|
||||
@@ -17,10 +17,15 @@ It runs at **https://telemetry.playhorizon.dev** (`DEFAULT_ENDPOINT` in `crates/
|
||||
| `GET /v1/benchmarks/summary` | Players' benchmark uploads grouped by GPU, CPU, render distance and game, for the website's Benchmarks page (below). Cached 10 minutes, same `Access-Control-Allow-Origin`. |
|
||||
| `POST /v1/live` | A game's "I'm playing" ping (JSON, at most 512 bytes). Returns 204. Returns 400 for a bad ping, 413 for one too large, 429 over the address's limit, 503 when it holds `LIVE_MAX_SESSIONS` already. |
|
||||
| `GET /v1/live` | How many play now, rounded: `playing`, `lan`, `versions`, `platforms`. Made again at most every 15 seconds, with the same `Access-Control-Allow-Origin`. |
|
||||
| `POST /hub/vote` | `{"item": "<roadmap id>", "voter": "<the browser's random id>", "up": true}`: a 👍 on a roadmap item, one per voter; `"up": false` takes it back. Returns `{"item", "votes"}`. 400 for a bad id, 429 over the address's limit. `OPTIONS` answers the website's preflight. |
|
||||
| `POST /hub/vote` | `{"item": "<roadmap id>", "voter": "<the browser's random id>", "up": true}`: a 👍 on a roadmap item, one per voter; `"up": false` takes it back. Returns `{"item", "votes"}`. 400 for a bad id, 429 over the address's limit. |
|
||||
| `GET /hub/votes` | `{"votes": {"<item>": n}}`. Made again at most every minute. |
|
||||
| `POST /hub/report` | A bug report from the hub's form: `{"title", "did", "happened", "expected", "version"?, "system", "name"?}`. Opens a GitHub issue at once (labels `bug` and `from-website`, the `#horizon-bugs` template, "Reported on playhorizon.dev") and returns 201 `{"id", "issue": "<url>"}`. If GitHub fails: 202 `{"id", "issue": null}`, and the upkeep tries again for a week. The same report twice in a day returns the first one's issue. 400 for an empty or too long field, 429 over the address's limit, 503 past the day's cap or without `GITHUB_TOKEN`. |
|
||||
| `POST /hub/suggest` | `{"text", "item"?, "name"?}` into the suggestion queue. Returns 202 `{"id"}`. Never published: Kira relays it to the maintainers' private channel. |
|
||||
| `GET /hub/suggestions?after=<id>&limit=100` | The queue, oldest first, for Kira: `{"suggestions": [{"id", "text", "item"?, "name"?, "created_at"}]}`. Needs `Authorization: Bearer <HORIZON_TELEMETRY_HUB_TOKEN>`: 401 without it, 404 when the server has none. |
|
||||
| `GET /healthz` | `ok` |
|
||||
|
||||
The hub's posts take JSON from the website (`OPTIONS` preflights answer with its origin). A form's hidden `website` field filled in (a bot) gets 202 and is dropped.
|
||||
|
||||
The totals are grouped per version, OS, GPU vendor and render distance. Each group has its sessions, install-days, hours played, the 14 frame-time buckets summed, and the mean p50 and p99 frame times. Groups with fewer than 5 install-days are left out. The totals also list crash counts per version. Benchmark uploads are published only through the summary below, never one by one.
|
||||
|
||||
## Benchmarks summary
|
||||
@@ -58,7 +63,7 @@ The live count is a table in memory: a session ID, when it last pinged, its vers
|
||||
|
||||
The day stored is the day the server received an item. The sender's IP address is never written down. The per-address rate limit keeps addresses in memory for an hour at most.
|
||||
|
||||
A hub vote is stored as an item ID and the browser's random ID. The hub's rate limits key their buckets by a hash of the address with a key drawn at each start, in memory.
|
||||
The hub keeps votes as an item ID and the browser's random ID, and reports and suggestions as their text, for `HORIZON_TELEMETRY_HUB_KEEP_DAYS` (90). Its rate limits key their buckets by a hash of the address with a key drawn at each start, in memory.
|
||||
|
||||
Raw rows are deleted after `HORIZON_TELEMETRY_RETENTION_DAYS` (180). The daily totals stay; they hold no install ID.
|
||||
|
||||
@@ -78,6 +83,13 @@ All are environment variables:
|
||||
- `HORIZON_TELEMETRY_LIVE_WINDOW_SECONDS` (120)
|
||||
- `HORIZON_TELEMETRY_LIVE_MAX_SESSIONS` (200 000)
|
||||
- `HORIZON_TELEMETRY_HUB_PER_HOUR` votes and vote reads per address (600, bursts of 60: a page load is one read, and several players can share an address)
|
||||
- `HORIZON_TELEMETRY_HUB_POSTS_PER_HOUR` reports and suggestions per address (20, bursts of 2)
|
||||
- `HORIZON_TELEMETRY_HUB_REPORTS_PER_DAY` from everyone (30): each opens an issue
|
||||
- `HORIZON_TELEMETRY_HUB_SUGGESTIONS_PER_DAY` from everyone (200)
|
||||
- `HORIZON_TELEMETRY_HUB_KEEP_DAYS` (90)
|
||||
- `HORIZON_TELEMETRY_HUB_TOKEN`: the token Kira reads the suggestions with (unset: `GET /hub/suggestions` is off)
|
||||
- `HORIZON_TELEMETRY_GITHUB_TOKEN`: a fine-grained token with Issues read and write on the repository only (unset: `POST /hub/report` answers 503 and the form points to GitHub and Discord)
|
||||
- `HORIZON_TELEMETRY_GITHUB_REPO` (`ludovic111/horizon`)
|
||||
|
||||
## Hosting
|
||||
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
//! A website bug report as a GitHub issue: the text made inert (no mentions, no cross-references, no HTML),
|
||||
//! written as the template Kira's `/report` and `#horizon-bugs` use, and the call that opens it.
|
||||
use std::time::Duration;
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
/// Opens an issue; its URL, or why not. The server's is [`Filer::github`]; tests give their own.
|
||||
pub type Filer = dyn Fn(&Issue) -> Result<String, String> + Send + Sync;
|
||||
|
||||
pub struct Issue {
|
||||
pub title: String,
|
||||
pub body: String,
|
||||
pub labels: Vec<String>,
|
||||
}
|
||||
|
||||
/// Labels every website report gets.
|
||||
pub const LABELS: [&str; 2] = ["bug", "from-website"];
|
||||
|
||||
/// Text people wrote, safe in an issue: no HTML, no `@` mentions, no `#12` or issue-URL cross-references,
|
||||
/// no closing keywords, no code spans to break out of. The same rules as Kira's `neutralise`.
|
||||
pub fn neutralise(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let chars: Vec<char> = text.chars().collect();
|
||||
let mut i = 0;
|
||||
while i < chars.len() {
|
||||
let c = chars[i];
|
||||
let next = chars.get(i + 1).copied();
|
||||
let word_start = i == 0 || !chars[i - 1].is_alphanumeric();
|
||||
match c {
|
||||
'<' => out.push_str("<"),
|
||||
'>' => out.push_str(">"),
|
||||
'`' => out.push('\''),
|
||||
// A zero-width space after `@` or `#` keeps the text and drops the mention or the link.
|
||||
'@' | '#' if next.is_some_and(|n| n.is_alphanumeric()) => {
|
||||
out.push(c);
|
||||
out.push('\u{200b}');
|
||||
}
|
||||
_ if word_start && closing_keyword(&chars[i..]) => {
|
||||
out.push(c);
|
||||
out.push('\u{200b}');
|
||||
}
|
||||
_ => out.push(c),
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
// github.com/owner/repo/issues/12 links cross-reference the issue: shown, not linked.
|
||||
out.replace("github.com/", "github.com\u{200b}/")
|
||||
}
|
||||
|
||||
fn closing_keyword(rest: &[char]) -> bool {
|
||||
let word: String = rest
|
||||
.iter()
|
||||
.take_while(|c| c.is_alphabetic())
|
||||
.flat_map(|c| c.to_lowercase())
|
||||
.collect();
|
||||
[
|
||||
"close", "closes", "closed", "fix", "fixes", "fixed", "resolve", "resolves", "resolved",
|
||||
]
|
||||
.contains(&word.as_str())
|
||||
}
|
||||
|
||||
/// Text as a markdown quote, every line prefixed.
|
||||
fn quote(text: &str) -> String {
|
||||
text.lines()
|
||||
.map(|l| format!("> {l}").trim_end().to_owned())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
/// The issue for a report's fields (`hub::REPORT`'s names, already checked).
|
||||
pub fn issue(fields: &Map<String, Value>) -> Issue {
|
||||
let field = |name: &str| {
|
||||
fields
|
||||
.get(name)
|
||||
.and_then(Value::as_str)
|
||||
.map(neutralise)
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| "not given".into())
|
||||
};
|
||||
let title = neutralise(&field("title").replace(['\n', '\r'], " "));
|
||||
let text = [
|
||||
format!("**What I did:**\n{}", field("did")),
|
||||
format!("**What happened:**\n{}", field("happened")),
|
||||
format!("**What I expected:**\n{}", field("expected")),
|
||||
format!("**Horizon version:** {}", field("version")),
|
||||
format!("**OS, CPU, GPU:** {}", field("system")),
|
||||
]
|
||||
.join("\n");
|
||||
let by = fields
|
||||
.get("name")
|
||||
.and_then(Value::as_str)
|
||||
.map(|n| neutralise(n).replace(['*', '_', '~', '|', '[', ']'], ""))
|
||||
.filter(|n| !n.is_empty())
|
||||
.map(|n| format!(" by **{n}**"))
|
||||
.unwrap_or_default();
|
||||
let body = format!(
|
||||
"{}\n\nReported on playhorizon.dev{by}\n\n<sub>Opened by the bug form on playhorizon.dev/hub. The reporter has this issue's link; they don't get GitHub notifications.</sub>",
|
||||
quote(&text)
|
||||
);
|
||||
Issue {
|
||||
title,
|
||||
body,
|
||||
labels: LABELS.iter().map(|l| (*l).to_owned()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens issues in `repo` (`owner/name`) with `token` (a fine-grained token with Issues: write on that repo
|
||||
/// only). Errors never carry the token or GitHub's answer body.
|
||||
pub fn github(repo: String, token: String) -> Box<Filer> {
|
||||
Box::new(move |issue: &Issue| {
|
||||
let response = ureq::post(&format!("https://api.github.com/repos/{repo}/issues"))
|
||||
.set("Authorization", &format!("Bearer {token}"))
|
||||
.set("Accept", "application/vnd.github+json")
|
||||
.set("X-GitHub-Api-Version", "2022-11-28")
|
||||
.set("User-Agent", "horizon-telemetry-server")
|
||||
.timeout(Duration::from_secs(20))
|
||||
.send_json(json!({ "title": issue.title, "body": issue.body, "labels": issue.labels }));
|
||||
match response {
|
||||
Ok(response) => response
|
||||
.into_json::<Value>()
|
||||
.ok()
|
||||
.and_then(|v| v["html_url"].as_str().map(str::to_owned))
|
||||
.ok_or_else(|| "GitHub's answer had no issue URL".into()),
|
||||
Err(ureq::Error::Status(code, _)) => Err(format!("GitHub answered {code}")),
|
||||
Err(ureq::Error::Transport(e)) => Err(format!("could not reach GitHub: {}", e.kind())),
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
//! The hub's votes: a 👍 on a roadmap item at playhorizon.dev/hub. roadmap/README.md documents them for Kira and the website.
|
||||
//! The hub's votes: a 👍 on a roadmap item at playhorizon.dev/hub, and the helpers the hub's other
|
||||
//! endpoints (`intake`) share. roadmap/README.md documents them for Kira and the website.
|
||||
//!
|
||||
//! - `POST /hub/vote`: a 👍 on a roadmap item, one per browser (`voter`, a random id the page makes and
|
||||
//! keeps); `"up": false` takes it back. Answers the item's count.
|
||||
@@ -114,10 +115,12 @@ impl App {
|
||||
|
||||
/// The hub's share of `maintain`: idle buckets are forgotten.
|
||||
pub(crate) fn forget_hub_buckets(&self) {
|
||||
self.hub_buckets
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.retain(|_, (_, at)| at.elapsed() < Duration::from_secs(3600));
|
||||
for buckets in [&self.hub_buckets, &self.hub_post_buckets] {
|
||||
buckets
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.retain(|_, (_, at)| at.elapsed() < Duration::from_secs(3600));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
//! What players send from the hub (playhorizon.dev/hub) for the maintainers: bug reports and suggestions.
|
||||
//! roadmap/README.md documents it for Kira and the website.
|
||||
//!
|
||||
//! - `POST /hub/report`: a bug report, opened at once as a GitHub issue (labels `bug`, `from-website`) in
|
||||
//! the template Kira's `/report` uses (`github`). Kept and tried again by the upkeep if GitHub fails.
|
||||
//! - `POST /hub/suggest`: a suggestion, into the queue. Never published: Kira relays it to the
|
||||
//! maintainers' private channel.
|
||||
//! - `GET /hub/suggestions?after=<id>`: the queue for Kira, with the `HORIZON_TELEMETRY_HUB_TOKEN` bearer.
|
||||
//!
|
||||
//! Stored: reports and suggestions as their text, for `hub_keep_days`. Never an IP address.
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::{Query, Request, State};
|
||||
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use rusqlite::params;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::hub::{answer, item_id, json_body, refuse};
|
||||
use crate::store::Store;
|
||||
use crate::{App, github, take, utc_now};
|
||||
|
||||
pub const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS hub_queue (
|
||||
id INTEGER PRIMARY KEY, kind TEXT NOT NULL, day INTEGER NOT NULL, created TEXT NOT NULL,
|
||||
state TEXT NOT NULL DEFAULT 'new', issue TEXT NOT NULL DEFAULT '', body TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS hub_queue_kind ON hub_queue (kind, state, id);
|
||||
";
|
||||
|
||||
/// Days the upkeep keeps trying to open a report's issue.
|
||||
const RETRY_DAYS: u32 = 7;
|
||||
|
||||
/// A field's rules: its name, whether it is needed, its length in characters, whether it may hold lines.
|
||||
struct Field(&'static str, bool, std::ops::RangeInclusive<usize>, bool);
|
||||
|
||||
/// A bug report: the `#horizon-bugs` template's fields.
|
||||
const REPORT: &[Field] = &[
|
||||
Field("title", true, 5..=120, false),
|
||||
Field("did", true, 3..=2000, true),
|
||||
Field("happened", true, 3..=2000, true),
|
||||
Field("expected", true, 3..=2000, true),
|
||||
Field("version", false, 0..=60, false),
|
||||
Field("system", true, 3..=200, false),
|
||||
Field("name", false, 0..=64, false),
|
||||
];
|
||||
const SUGGESTION: &[Field] = &[
|
||||
Field("text", true, 10..=1000, true),
|
||||
Field("item", false, 0..=48, false),
|
||||
Field("name", false, 0..=64, false),
|
||||
];
|
||||
|
||||
/// Text as stored: no control characters but line breaks (where allowed), trimmed.
|
||||
fn clean(text: &str, lines: bool) -> String {
|
||||
let text = text.replace("\r\n", "\n");
|
||||
let kept: String = text
|
||||
.chars()
|
||||
.map(|c| if !lines && c == '\n' { ' ' } else { c })
|
||||
.filter(|c| !c.is_control() || (lines && *c == '\n'))
|
||||
.collect();
|
||||
kept.trim().to_owned()
|
||||
}
|
||||
|
||||
/// The request's fields checked against `rules`: the cleaned object, or what is wrong.
|
||||
fn fields(body: &Value, rules: &[Field]) -> Result<Map<String, Value>, String> {
|
||||
let object = body.as_object().ok_or("send a JSON object")?;
|
||||
let mut out = Map::new();
|
||||
for Field(name, needed, length, lines) in rules {
|
||||
let text = match object.get(*name) {
|
||||
None | Some(Value::Null) => String::new(),
|
||||
Some(Value::String(s)) => clean(s, *lines),
|
||||
Some(_) => return Err(format!("{name} is text")),
|
||||
};
|
||||
if text.is_empty() && !needed {
|
||||
continue;
|
||||
}
|
||||
if !length.contains(&text.chars().count()) {
|
||||
return Err(format!(
|
||||
"{name}: {} to {} characters",
|
||||
length.start(),
|
||||
length.end()
|
||||
));
|
||||
}
|
||||
out.insert((*name).to_owned(), Value::String(text));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// How many entries of `kind` came in on `day`.
|
||||
pub fn queued_on(&self, kind: &str, day: u32) -> rusqlite::Result<u32> {
|
||||
self.db.query_row(
|
||||
"SELECT count(*) FROM hub_queue WHERE kind = ?1 AND day = ?2",
|
||||
params![kind, day],
|
||||
|r| r.get(0),
|
||||
)
|
||||
}
|
||||
|
||||
/// Queues an entry: its id, and its issue when the same text already came in since `since` and was
|
||||
/// opened (a double click, a form sent twice).
|
||||
pub fn enqueue(
|
||||
&mut self,
|
||||
kind: &str,
|
||||
day: u32,
|
||||
since: u32,
|
||||
body: &Map<String, Value>,
|
||||
) -> rusqlite::Result<(i64, Option<String>)> {
|
||||
let body = Value::Object(body.clone()).to_string();
|
||||
let same = self
|
||||
.db
|
||||
.prepare(
|
||||
"SELECT id, issue FROM hub_queue WHERE kind = ?1 AND body = ?2 AND day >= ?3 LIMIT 1",
|
||||
)?
|
||||
.query_row(params![kind, body, since], |r| {
|
||||
Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))
|
||||
});
|
||||
match same {
|
||||
Ok((id, issue)) => return Ok((id, Some(issue).filter(|i| !i.is_empty()))),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
self.db.execute(
|
||||
"INSERT INTO hub_queue (kind, day, created, body) VALUES (?1, ?2, ?3, ?4)",
|
||||
params![kind, day, utc_now(), body],
|
||||
)?;
|
||||
Ok((self.db.last_insert_rowid(), None))
|
||||
}
|
||||
|
||||
/// Suggestions after `after`, oldest first: `{id, text, item?, name?, created_at}`.
|
||||
pub fn suggestions(&self, after: i64, limit: u32) -> rusqlite::Result<Vec<Value>> {
|
||||
let mut statement = self.db.prepare(
|
||||
"SELECT id, created, body FROM hub_queue WHERE kind = 'suggestion' AND id > ?1 ORDER BY id LIMIT ?2",
|
||||
)?;
|
||||
let rows = statement.query_map(params![after, limit], |r| {
|
||||
let mut entry: Map<String, Value> =
|
||||
serde_json::from_str(&r.get::<_, String>(2)?).unwrap_or_default();
|
||||
entry.insert("id".into(), r.get::<_, i64>(0)?.into());
|
||||
entry.insert("created_at".into(), r.get::<_, String>(1)?.into());
|
||||
Ok(Value::Object(entry))
|
||||
})?;
|
||||
rows.collect()
|
||||
}
|
||||
|
||||
/// Reports whose issue isn't open yet, from `since` on: id and fields.
|
||||
pub fn unfiled(&self, since: u32) -> rusqlite::Result<Vec<(i64, Map<String, Value>)>> {
|
||||
let mut statement = self.db.prepare(
|
||||
"SELECT id, body FROM hub_queue WHERE kind = 'report' AND state = 'new' AND day >= ?1 ORDER BY id",
|
||||
)?;
|
||||
let rows = statement.query_map([since], |r| {
|
||||
Ok((
|
||||
r.get::<_, i64>(0)?,
|
||||
serde_json::from_str(&r.get::<_, String>(1)?).unwrap_or_default(),
|
||||
))
|
||||
})?;
|
||||
rows.collect()
|
||||
}
|
||||
|
||||
/// Marks a report as opened: `issue` is its URL.
|
||||
pub fn filed(&mut self, id: i64, issue: &str) -> rusqlite::Result<()> {
|
||||
self.db.execute(
|
||||
"UPDATE hub_queue SET state = 'filed', issue = ?2 WHERE id = ?1",
|
||||
params![id, issue],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reports from before `retry_from` that never got an issue are given up; entries before `oldest_day`
|
||||
/// go.
|
||||
pub fn expire_queue(&mut self, oldest_day: u32, retry_from: u32) -> rusqlite::Result<usize> {
|
||||
self.db.execute(
|
||||
"UPDATE hub_queue SET state = 'failed' WHERE kind = 'report' AND state = 'new' AND day < ?1",
|
||||
[retry_from],
|
||||
)?;
|
||||
self.db
|
||||
.execute("DELETE FROM hub_queue WHERE day < ?1", [oldest_day])
|
||||
}
|
||||
}
|
||||
|
||||
impl App {
|
||||
/// The same for reports and suggestions, a much smaller bucket.
|
||||
fn allow_post(&self, req: &Request) -> bool {
|
||||
take(
|
||||
&self.hub_post_buckets,
|
||||
self.config.hub_posts_per_hour,
|
||||
self.hashed(req),
|
||||
)
|
||||
}
|
||||
|
||||
/// Opens the reports whose issue isn't open yet (the upkeep calls it off the async workers; it blocks).
|
||||
/// Stops at the first failure: GitHub is down or the token is wrong, the next round tries again.
|
||||
pub fn file_pending(&self) {
|
||||
let Some(filer) = &self.filer else { return };
|
||||
let since = (self.today)().saturating_sub(RETRY_DAYS);
|
||||
let pending = self.store().unfiled(since).unwrap_or_default();
|
||||
for (id, fields) in pending {
|
||||
match filer(&github::issue(&fields)) {
|
||||
Ok(url) => self.mark_filed(id, &url),
|
||||
Err(e) => {
|
||||
eprintln!("horizon-telemetry-server: report {id} still not opened: {e}");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_filed(&self, id: i64, url: &str) {
|
||||
let marked = self.store().filed(id, url);
|
||||
if let Err(e) = marked {
|
||||
eprintln!("horizon-telemetry-server: report {id} opened as {url}, not marked: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The intake's share of `maintain`: entries past `hub_keep_days` go, reports never opened after
|
||||
/// [`RETRY_DAYS`] are given up.
|
||||
pub(crate) fn expire_intake(&self) -> rusqlite::Result<usize> {
|
||||
let today = (self.today)();
|
||||
self.store().expire_queue(
|
||||
today.saturating_sub(self.config.hub_keep_days),
|
||||
today.saturating_sub(RETRY_DAYS),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// What `receive` keeps: the entry's id, its issue if already opened, its fields.
|
||||
type Received = (i64, Option<String>, Map<String, Value>);
|
||||
|
||||
/// Checks a report or a suggestion and queues it.
|
||||
async fn receive(
|
||||
app: &App,
|
||||
req: Request,
|
||||
kind: &str,
|
||||
rules: &[Field],
|
||||
per_day: u32,
|
||||
) -> Result<Received, Box<Response>> {
|
||||
let no = |code: StatusCode, message: &str| Box::new(refuse(app, code, message));
|
||||
if !app.allow_post(&req) {
|
||||
return Err(no(StatusCode::TOO_MANY_REQUESTS, "too many requests"));
|
||||
}
|
||||
let body = json_body(req)
|
||||
.await
|
||||
.map_err(|e| no(StatusCode::BAD_REQUEST, e))?;
|
||||
// The form's hidden field, which people leave empty and form-filling bots don't: thanked, dropped.
|
||||
if body["website"].as_str().is_some_and(|s| !s.is_empty()) {
|
||||
let thanks = answer(app, StatusCode::ACCEPTED, json!({ "id": 0 }));
|
||||
return Err(Box::new(thanks));
|
||||
}
|
||||
let entry = fields(&body, rules).map_err(|e| no(StatusCode::BAD_REQUEST, &e))?;
|
||||
if let Some(item) = entry.get("item").and_then(Value::as_str)
|
||||
&& !item_id(item)
|
||||
{
|
||||
return Err(no(StatusCode::BAD_REQUEST, "item: a roadmap item id"));
|
||||
}
|
||||
let today = (app.today)();
|
||||
let mut store = app.store();
|
||||
if store.queued_on(kind, today).unwrap_or(u32::MAX) >= per_day {
|
||||
let full = "too many today: please try again tomorrow";
|
||||
return Err(no(StatusCode::SERVICE_UNAVAILABLE, full));
|
||||
}
|
||||
match store.enqueue(kind, today, today.saturating_sub(1), &entry) {
|
||||
Ok((id, issue)) => Ok((id, issue, entry)),
|
||||
Err(e) => {
|
||||
eprintln!("horizon-telemetry-server: could not queue a {kind}: {e}");
|
||||
Err(no(StatusCode::SERVICE_UNAVAILABLE, "try again later"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /hub/report` `{"title", "did", "happened", "expected", "version"?, "system", "name"?}`: 201
|
||||
/// `{"id", "issue"}` once the issue is open, 202 `{"id", "issue": null}` when GitHub failed and the upkeep
|
||||
/// will try again; 503 when reports are off (no GitHub token).
|
||||
pub async fn report(State(app): State<Arc<App>>, req: Request) -> Response {
|
||||
let Some(filer) = app.filer.clone() else {
|
||||
return refuse(
|
||||
&app,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"reports are closed here: use GitHub or Discord",
|
||||
);
|
||||
};
|
||||
let per_day = app.config.hub_reports_per_day;
|
||||
let (id, issue, fields) = match receive(&app, req, "report", REPORT, per_day).await {
|
||||
Ok(received) => received,
|
||||
Err(response) => return *response,
|
||||
};
|
||||
if let Some(issue) = issue {
|
||||
return answer(
|
||||
&app,
|
||||
StatusCode::CREATED,
|
||||
json!({ "id": id, "issue": issue }),
|
||||
);
|
||||
}
|
||||
let opened = tokio::task::spawn_blocking(move || filer(&github::issue(&fields)))
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|opened| opened);
|
||||
match opened {
|
||||
Ok(url) => {
|
||||
app.mark_filed(id, &url);
|
||||
answer(&app, StatusCode::CREATED, json!({ "id": id, "issue": url }))
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("horizon-telemetry-server: report {id} not opened yet: {e}");
|
||||
answer(
|
||||
&app,
|
||||
StatusCode::ACCEPTED,
|
||||
json!({ "id": id, "issue": null }),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /hub/suggest` `{"text", "item"?, "name"?}`: 202 `{"id"}`.
|
||||
pub async fn suggest(State(app): State<Arc<App>>, req: Request) -> Response {
|
||||
let per_day = app.config.hub_suggestions_per_day;
|
||||
match receive(&app, req, "suggestion", SUGGESTION, per_day).await {
|
||||
Ok((id, ..)) => answer(&app, StatusCode::ACCEPTED, json!({ "id": id })),
|
||||
Err(response) => *response,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct After {
|
||||
after: Option<i64>,
|
||||
limit: Option<u32>,
|
||||
}
|
||||
|
||||
/// `GET /hub/suggestions?after=<id>&limit=100` with `Authorization: Bearer <HORIZON_TELEMETRY_HUB_TOKEN>`:
|
||||
/// `{"suggestions": [{"id", "text", "item"?, "name"?, "created_at"}]}`, oldest first. 404 when no token is
|
||||
/// set, 401 for a wrong one.
|
||||
pub async fn suggestions(
|
||||
State(app): State<Arc<App>>,
|
||||
headers: HeaderMap,
|
||||
Query(after): Query<After>,
|
||||
) -> Response {
|
||||
let Some(token) = app.config.hub_token.as_deref() else {
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
};
|
||||
let sent = headers
|
||||
.get(header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))
|
||||
.unwrap_or("");
|
||||
if !same(sent.as_bytes(), token.as_bytes()) {
|
||||
return StatusCode::UNAUTHORIZED.into_response();
|
||||
}
|
||||
let limit = after.limit.unwrap_or(100).clamp(1, 500);
|
||||
let listed = app.store().suggestions(after.after.unwrap_or(0), limit);
|
||||
match listed {
|
||||
Ok(list) => (
|
||||
[
|
||||
(
|
||||
header::CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/json"),
|
||||
),
|
||||
(header::CACHE_CONTROL, HeaderValue::from_static("no-store")),
|
||||
],
|
||||
json!({ "suggestions": list }).to_string(),
|
||||
)
|
||||
.into_response(),
|
||||
Err(e) => {
|
||||
eprintln!("horizon-telemetry-server: could not read the suggestions: {e}");
|
||||
StatusCode::SERVICE_UNAVAILABLE.into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Equal, in a time that doesn't depend on where they differ.
|
||||
fn same(a: &[u8], b: &[u8]) -> bool {
|
||||
a.len() == b.len() && a.iter().zip(b).fold(0, |diff, (x, y)| diff | (x ^ y)) == 0
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
//! least 3 installs only (`summary`), cached; the website's Benchmarks page reads it.
|
||||
//! - `POST /v1/live`: a game's anonymous "I'm playing" ping (`live`, in memory only).
|
||||
//! - `GET /v1/live`: how many play now, rounded, cached briefly.
|
||||
//! - `POST /hub/vote`, `GET /hub/votes`: the hub's 👍 on roadmap items (`hub`).
|
||||
//! - `/hub/…`: the hub's votes (`hub`), bug reports and suggestions (`intake`).
|
||||
//! - `GET /healthz`.
|
||||
//!
|
||||
//! Abuse limits: a token bucket per IP address (kept in memory for an hour, never stored), a
|
||||
@@ -29,11 +29,14 @@ use axum::routing::{delete, get, post};
|
||||
use horizon_telemetry::batch;
|
||||
|
||||
pub mod classes;
|
||||
pub mod github;
|
||||
pub mod hub;
|
||||
pub mod intake;
|
||||
pub mod live;
|
||||
pub mod store;
|
||||
pub mod summary;
|
||||
|
||||
use github::Filer;
|
||||
use horizon_telemetry::live::{MAX_PING_BYTES, Ping};
|
||||
use live::Live;
|
||||
use store::Store;
|
||||
@@ -61,6 +64,20 @@ pub struct Config {
|
||||
pub live_max_sessions: usize,
|
||||
/// The hub's votes and reads of their counts an IP address may make an hour.
|
||||
pub hub_per_hour: f64,
|
||||
/// Bug reports and suggestions together an IP address may send an hour.
|
||||
pub hub_posts_per_hour: f64,
|
||||
/// Bug reports taken a day, from everyone: each one opens a GitHub issue.
|
||||
pub hub_reports_per_day: u32,
|
||||
/// Suggestions taken a day, from everyone.
|
||||
pub hub_suggestions_per_day: u32,
|
||||
/// Days reports and suggestions are kept (Kira has relayed them long before).
|
||||
pub hub_keep_days: u32,
|
||||
/// The bearer token `GET /hub/suggestions` wants (Kira's); unset, it answers 404.
|
||||
pub hub_token: Option<String>,
|
||||
/// Opens the website's bug reports as issues; unset, `POST /hub/report` answers 503.
|
||||
pub github_token: Option<String>,
|
||||
/// Where those issues go, `owner/name`.
|
||||
pub github_repo: String,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
@@ -76,6 +93,13 @@ impl Default for Config {
|
||||
live_window: horizon_telemetry::live::WINDOW,
|
||||
live_max_sessions: 200_000,
|
||||
hub_per_hour: 600.0,
|
||||
hub_posts_per_hour: 20.0,
|
||||
hub_reports_per_day: 30,
|
||||
hub_suggestions_per_day: 200,
|
||||
hub_keep_days: 90,
|
||||
hub_token: None,
|
||||
github_token: None,
|
||||
github_repo: "ludovic111/horizon".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -83,9 +107,16 @@ impl Default for Config {
|
||||
impl Config {
|
||||
/// The defaults, with `HORIZON_TELEMETRY_RETENTION_DAYS`, `…_MIN_INSTALL_DAYS`,
|
||||
/// `…_REQUESTS_PER_HOUR`, `…_ITEMS_PER_DAY`, `…_TRUST_PROXY` (1), `…_ALLOW_ORIGIN`,
|
||||
/// `…_LIVE_PER_HOUR`, `…_LIVE_WINDOW_SECONDS`, `…_LIVE_MAX_SESSIONS` and `…_HUB_PER_HOUR`.
|
||||
/// `…_LIVE_PER_HOUR`, `…_LIVE_WINDOW_SECONDS`, `…_LIVE_MAX_SESSIONS`, `…_HUB_PER_HOUR`,
|
||||
/// `…_HUB_POSTS_PER_HOUR`, `…_HUB_REPORTS_PER_DAY`, `…_HUB_SUGGESTIONS_PER_DAY`, `…_HUB_KEEP_DAYS`,
|
||||
/// `…_HUB_TOKEN`, `…_GITHUB_TOKEN` and `…_GITHUB_REPO`. Empty tokens count as unset.
|
||||
pub fn from_env() -> Self {
|
||||
let var = |k: &str| std::env::var(format!("HORIZON_TELEMETRY_{k}")).ok();
|
||||
let secret = |k: &str| {
|
||||
var(k)
|
||||
.map(|v| v.trim().to_owned())
|
||||
.filter(|v| !v.is_empty())
|
||||
};
|
||||
let d = Self::default();
|
||||
Self {
|
||||
retention_days: var("RETENTION_DAYS")
|
||||
@@ -114,6 +145,21 @@ impl Config {
|
||||
hub_per_hour: var("HUB_PER_HOUR")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(d.hub_per_hour),
|
||||
hub_posts_per_hour: var("HUB_POSTS_PER_HOUR")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(d.hub_posts_per_hour),
|
||||
hub_reports_per_day: var("HUB_REPORTS_PER_DAY")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(d.hub_reports_per_day),
|
||||
hub_suggestions_per_day: var("HUB_SUGGESTIONS_PER_DAY")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(d.hub_suggestions_per_day),
|
||||
hub_keep_days: var("HUB_KEEP_DAYS")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(d.hub_keep_days),
|
||||
hub_token: secret("HUB_TOKEN"),
|
||||
github_token: secret("GITHUB_TOKEN"),
|
||||
github_repo: var("GITHUB_REPO").unwrap_or(d.github_repo),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -139,8 +185,12 @@ pub struct App {
|
||||
hub_key: RandomState,
|
||||
/// The hub's votes and vote counts, per hashed address.
|
||||
hub_buckets: Mutex<hub::HashedBuckets>,
|
||||
/// The hub's reports and suggestions, per hashed address.
|
||||
hub_post_buckets: Mutex<hub::HashedBuckets>,
|
||||
/// `GET /hub/votes`'s answer and when it was made.
|
||||
votes_cache: Mutex<Option<(Instant, String)>>,
|
||||
/// Opens bug reports as GitHub issues; None without a token.
|
||||
filer: Option<Arc<Filer>>,
|
||||
}
|
||||
|
||||
type Buckets = HashMap<IpAddr, (f64, Instant)>;
|
||||
@@ -160,6 +210,20 @@ impl App {
|
||||
store: Store,
|
||||
config: Config,
|
||||
today: Box<dyn Fn() -> u32 + Send + Sync>,
|
||||
) -> Arc<Self> {
|
||||
let filer = config
|
||||
.github_token
|
||||
.clone()
|
||||
.map(|token| Arc::from(github::github(config.github_repo.clone(), token)));
|
||||
Self::with_filer(store, config, today, filer)
|
||||
}
|
||||
|
||||
/// [`Self::with_clock`] with the hub's bug reports opened by `filer` (tests give their own).
|
||||
pub fn with_filer(
|
||||
store: Store,
|
||||
config: Config,
|
||||
today: Box<dyn Fn() -> u32 + Send + Sync>,
|
||||
filer: Option<Arc<Filer>>,
|
||||
) -> Arc<Self> {
|
||||
let live = Live::new(config.live_window, config.live_max_sessions);
|
||||
Arc::new(Self {
|
||||
@@ -174,7 +238,9 @@ impl App {
|
||||
today,
|
||||
hub_key: RandomState::new(),
|
||||
hub_buckets: Mutex::default(),
|
||||
hub_post_buckets: Mutex::default(),
|
||||
votes_cache: Mutex::default(),
|
||||
filer,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -235,10 +301,12 @@ impl App {
|
||||
}
|
||||
|
||||
/// Daily upkeep: totals rebuilt, expired rows removed, idle rate-limit entries and live
|
||||
/// sessions forgotten.
|
||||
/// sessions forgotten, the hub's old reports and suggestions removed. Reports still waiting for
|
||||
/// their issue are [`Self::file_pending`]'s.
|
||||
pub fn maintain(&self) -> rusqlite::Result<usize> {
|
||||
let oldest = (self.today)().saturating_sub(self.config.retention_days);
|
||||
let removed = self.store().expire(oldest)?;
|
||||
self.expire_intake()?;
|
||||
self.forget_hub_buckets();
|
||||
for buckets in [&self.buckets, &self.live_buckets] {
|
||||
buckets
|
||||
@@ -503,6 +571,12 @@ pub fn router(app: Arc<App>) -> Router {
|
||||
.route("/v1/benchmarks/summary", get(benchmarks))
|
||||
.route("/hub/vote", post(hub::vote).options(hub::preflight))
|
||||
.route("/hub/votes", get(hub::votes))
|
||||
.route("/hub/report", post(intake::report).options(hub::preflight))
|
||||
.route(
|
||||
"/hub/suggest",
|
||||
post(intake::suggest).options(hub::preflight),
|
||||
)
|
||||
.route("/hub/suggestions", get(intake::suggestions))
|
||||
.route("/healthz", get(|| async { "ok" }))
|
||||
.with_state(app)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,9 @@ async fn main() {
|
||||
if let Err(e) = upkeep.maintain() {
|
||||
eprintln!("horizon-telemetry-server: upkeep failed: {e}");
|
||||
}
|
||||
// The hub's reports GitHub refused before: their issues, off the async workers.
|
||||
let filing = upkeep.clone();
|
||||
let _ = tokio::task::spawn_blocking(move || filing.file_pending()).await;
|
||||
}
|
||||
});
|
||||
let listener = tokio::net::TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], port)))
|
||||
|
||||
@@ -131,6 +131,7 @@ impl Store {
|
||||
let mut db = Connection::open(path)?;
|
||||
db.execute_batch(SCHEMA)?;
|
||||
db.execute_batch(crate::hub::SCHEMA)?;
|
||||
db.execute_batch(crate::intake::SCHEMA)?;
|
||||
migrate(&mut db)?;
|
||||
Ok(Self { db })
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
//! A website bug report as the issue GitHub gets: the template, and people's text made inert.
|
||||
use horizon_telemetry_server::github::{issue, neutralise};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
#[test]
|
||||
fn people_text_mentions_and_references_nothing() {
|
||||
let text =
|
||||
neutralise("@octocat see #12 and https://github.com/a/b/issues/3 <img src=x> `x` Fixes it");
|
||||
assert!(!text.contains("@o") && !text.contains("#1") && !text.contains("github.com/"));
|
||||
assert!(!text.contains('<') && !text.contains('`'));
|
||||
assert!(text.contains("F\u{200b}ixes"));
|
||||
// Words that only start like a keyword stay as they are.
|
||||
assert_eq!(neutralise("closet fixture"), "closet fixture");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_report_reads_as_the_template() {
|
||||
let fields: Map<String, Value> = serde_json::from_value(json!({
|
||||
"title": "Water is black\non AMD", "did": "Opened a world", "happened": "Black water",
|
||||
"expected": "Blue water", "system": "Windows 11, RX 6600", "name": "steve_"
|
||||
}))
|
||||
.unwrap();
|
||||
let issue = issue(&fields);
|
||||
assert_eq!(issue.title, "Water is black on AMD");
|
||||
assert!(
|
||||
issue
|
||||
.body
|
||||
.starts_with("> **What I did:**\n> Opened a world\n> **What happened:**")
|
||||
);
|
||||
assert!(
|
||||
issue
|
||||
.body
|
||||
.contains("> **Horizon version:** not given\n> **OS, CPU, GPU:** Windows 11, RX 6600")
|
||||
);
|
||||
assert!(
|
||||
issue
|
||||
.body
|
||||
.contains("Reported on playhorizon.dev by **steve**")
|
||||
);
|
||||
assert_eq!(issue.labels, ["bug", "from-website"]);
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
//! The hub's votes through the HTTP interface.
|
||||
use std::sync::Arc;
|
||||
//! The hub's intake through its HTTP interface: votes, bug reports (opened as issues by a fake GitHub),
|
||||
//! suggestions and the queue Kira reads.
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::http::{Request, StatusCode};
|
||||
use horizon_telemetry_server::github::{Filer, Issue};
|
||||
use horizon_telemetry_server::store::Store;
|
||||
use horizon_telemetry_server::{App, Config, router};
|
||||
use http_body_util::BodyExt;
|
||||
@@ -12,17 +15,54 @@ use tower::ServiceExt;
|
||||
const DAY: u32 = 20_741;
|
||||
const VOTER: &str = "0b7c3c9e-3f0a-4d8e-9a39-4c1f5a6b7d21";
|
||||
|
||||
/// The issues the fake GitHub opened, and whether it is up.
|
||||
#[derive(Default)]
|
||||
struct GitHub {
|
||||
opened: Mutex<Vec<Issue>>,
|
||||
down: Mutex<bool>,
|
||||
}
|
||||
|
||||
struct Hub {
|
||||
app: Arc<App>,
|
||||
github: Arc<GitHub>,
|
||||
day: Arc<AtomicU32>,
|
||||
}
|
||||
|
||||
fn hub(config: Config) -> Hub {
|
||||
let app = App::with_clock(Store::open(":memory:").unwrap(), config, Box::new(|| DAY));
|
||||
Hub { app }
|
||||
let github = Arc::new(GitHub::default());
|
||||
let fake = github.clone();
|
||||
let filer: Arc<Filer> = Arc::new(move |issue: &Issue| {
|
||||
if *fake.down.lock().unwrap() {
|
||||
return Err("GitHub answered 502".into());
|
||||
}
|
||||
let mut opened = fake.opened.lock().unwrap();
|
||||
opened.push(Issue {
|
||||
title: issue.title.clone(),
|
||||
body: issue.body.clone(),
|
||||
labels: issue.labels.clone(),
|
||||
});
|
||||
Ok(format!(
|
||||
"https://github.com/ludovic111/horizon/issues/{}",
|
||||
1000 + opened.len()
|
||||
))
|
||||
});
|
||||
let day = Arc::new(AtomicU32::new(DAY));
|
||||
let clock = day.clone();
|
||||
let app = App::with_filer(
|
||||
Store::open(":memory:").unwrap(),
|
||||
config,
|
||||
Box::new(move || clock.load(Ordering::SeqCst)),
|
||||
Some(filer),
|
||||
);
|
||||
Hub { app, github, day }
|
||||
}
|
||||
|
||||
fn config() -> Config {
|
||||
Config::default()
|
||||
Config {
|
||||
hub_token: Some("kira-secret".into()),
|
||||
hub_posts_per_hour: 1000.0,
|
||||
..Config::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Hub {
|
||||
@@ -43,12 +83,27 @@ impl Hub {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get(&self, path: &str) -> (StatusCode, Value) {
|
||||
self.send(Request::get(path).body(Body::empty()).unwrap())
|
||||
.await
|
||||
async fn get(&self, path: &str, token: Option<&str>) -> (StatusCode, Value) {
|
||||
let mut req = Request::get(path);
|
||||
if let Some(token) = token {
|
||||
req = req.header("authorization", format!("Bearer {token}"));
|
||||
}
|
||||
self.send(req.body(Body::empty()).unwrap()).await
|
||||
}
|
||||
}
|
||||
|
||||
fn report(title: &str) -> Value {
|
||||
json!({
|
||||
"title": title,
|
||||
"did": "Opened a new world\nWalked to the sea",
|
||||
"happened": "The water is black",
|
||||
"expected": "Blue water, as in vanilla",
|
||||
"version": "v0.2.57",
|
||||
"system": "Windows 11, Ryzen 5 5600, RX 6600",
|
||||
"website": "",
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn votes_count_once_per_browser_and_can_be_taken_back() {
|
||||
let hub = hub(config());
|
||||
@@ -71,7 +126,7 @@ async fn votes_count_once_per_browser_and_can_be_taken_back() {
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
hub.get("/hub/votes").await,
|
||||
hub.get("/hub/votes", None).await,
|
||||
(StatusCode::OK, json!({ "votes": { "smooth-frames": 2 } }))
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -79,7 +134,7 @@ async fn votes_count_once_per_browser_and_can_be_taken_back() {
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
hub.get("/hub/votes").await.1["votes"]["smooth-frames"],
|
||||
hub.get("/hub/votes", None).await.1["votes"]["smooth-frames"],
|
||||
1,
|
||||
"a vote drops the cached counts"
|
||||
);
|
||||
@@ -92,12 +147,239 @@ async fn votes_count_once_per_browser_and_can_be_taken_back() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_report_becomes_an_issue_in_the_bug_template() {
|
||||
let hub = hub(config());
|
||||
let (code, body) = hub
|
||||
.post("/hub/report", report("Water is black on AMD"))
|
||||
.await;
|
||||
assert_eq!(code, StatusCode::CREATED);
|
||||
assert_eq!(
|
||||
body["issue"],
|
||||
"https://github.com/ludovic111/horizon/issues/1001"
|
||||
);
|
||||
let issue = hub.github.opened.lock().unwrap().remove(0);
|
||||
assert_eq!(issue.title, "Water is black on AMD");
|
||||
assert_eq!(issue.labels, ["bug", "from-website"]);
|
||||
for line in [
|
||||
"> **What I did:**\n> Opened a new world\n> Walked to the sea",
|
||||
"> **What happened:**\n> The water is black",
|
||||
"> **What I expected:**\n> Blue water, as in vanilla",
|
||||
"> **Horizon version:** v0.2.57",
|
||||
"> **OS, CPU, GPU:** Windows 11, Ryzen 5 5600, RX 6600",
|
||||
"Reported on playhorizon.dev",
|
||||
] {
|
||||
assert!(issue.body.contains(line), "{line}\n---\n{}", issue.body);
|
||||
}
|
||||
// The same report again (a double click) answers the same issue and opens nothing.
|
||||
let (code, again) = hub
|
||||
.post("/hub/report", report("Water is black on AMD"))
|
||||
.await;
|
||||
assert_eq!(
|
||||
(code, &again["issue"]),
|
||||
(StatusCode::CREATED, &body["issue"])
|
||||
);
|
||||
assert!(hub.github.opened.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reports_need_every_template_field_and_skip_bots() {
|
||||
let hub = hub(config());
|
||||
for empty in ["title", "did", "happened", "expected", "system"] {
|
||||
let mut body = report("Water is black on AMD");
|
||||
body[empty] = json!(" ");
|
||||
let (code, answer) = hub.post("/hub/report", body).await;
|
||||
assert_eq!(code, StatusCode::BAD_REQUEST, "{empty}");
|
||||
assert!(answer["error"].as_str().unwrap().starts_with(empty));
|
||||
}
|
||||
let mut bot = report("Cheap watches");
|
||||
bot["website"] = json!("https://spam.example");
|
||||
assert_eq!(
|
||||
hub.post("/hub/report", bot).await,
|
||||
(StatusCode::ACCEPTED, json!({ "id": 0 }))
|
||||
);
|
||||
assert!(hub.github.opened.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_report_github_refused_is_opened_by_the_upkeep() {
|
||||
let hub = hub(config());
|
||||
*hub.github.down.lock().unwrap() = true;
|
||||
let (code, body) = hub
|
||||
.post("/hub/report", report("Crash when opening a chest"))
|
||||
.await;
|
||||
assert_eq!((code, &body["issue"]), (StatusCode::ACCEPTED, &Value::Null));
|
||||
hub.app.file_pending();
|
||||
assert_eq!(hub.github.opened.lock().unwrap().len(), 0);
|
||||
*hub.github.down.lock().unwrap() = false;
|
||||
hub.app.file_pending();
|
||||
assert_eq!(hub.github.opened.lock().unwrap().len(), 1);
|
||||
hub.app.file_pending();
|
||||
assert_eq!(hub.github.opened.lock().unwrap().len(), 1, "opened once");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reports_are_closed_without_a_github_token() {
|
||||
let app = App::with_clock(
|
||||
Store::open(":memory:").unwrap(),
|
||||
Config::default(),
|
||||
Box::new(|| DAY),
|
||||
);
|
||||
let res = router(app)
|
||||
.oneshot(
|
||||
Request::post("/hub/report")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(report("Water is black").to_string()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn suggestions_wait_in_the_queue_for_kira() {
|
||||
let hub = hub(config());
|
||||
let first = json!({ "text": " Add a waypoint mod\r\nto the\u{7} browser ", "item": "mod-registry", "name": "alex", "extra": 1 });
|
||||
let (code, body) = hub.post("/hub/suggest", first).await;
|
||||
assert_eq!(code, StatusCode::ACCEPTED);
|
||||
let first_id = body["id"].as_i64().unwrap();
|
||||
hub.post(
|
||||
"/hub/suggest",
|
||||
json!({ "text": "Bigger render distance on the Steam Deck" }),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
hub.post("/hub/suggest", json!({ "text": "too short" }))
|
||||
.await
|
||||
.0,
|
||||
StatusCode::BAD_REQUEST
|
||||
);
|
||||
assert_eq!(
|
||||
hub.post(
|
||||
"/hub/suggest",
|
||||
json!({ "text": "A long enough idea", "item": "Bad Id" })
|
||||
)
|
||||
.await
|
||||
.0,
|
||||
StatusCode::BAD_REQUEST
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
hub.get("/hub/suggestions", None).await.0,
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
assert_eq!(
|
||||
hub.get("/hub/suggestions", Some("wrong-secret")).await.0,
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
let (code, all) = hub.get("/hub/suggestions", Some("kira-secret")).await;
|
||||
assert_eq!(code, StatusCode::OK);
|
||||
let list = all["suggestions"].as_array().unwrap();
|
||||
assert_eq!(list.len(), 2);
|
||||
assert_eq!(
|
||||
list[0]["text"], "Add a waypoint mod\nto the browser",
|
||||
"trimmed, no control characters"
|
||||
);
|
||||
assert!(list[0].get("extra").is_none());
|
||||
assert_eq!(list[0]["item"], "mod-registry");
|
||||
assert_eq!(list[0]["name"], "alex");
|
||||
assert!(list[0]["created_at"].as_str().unwrap().ends_with('Z'));
|
||||
assert!(list[1].get("name").is_none());
|
||||
let (_, after) = hub
|
||||
.get(
|
||||
&format!("/hub/suggestions?after={first_id}"),
|
||||
Some("kira-secret"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(after["suggestions"].as_array().unwrap().len(), 1);
|
||||
|
||||
let closed = hub_without_token();
|
||||
assert_eq!(
|
||||
closed.get("/hub/suggestions", Some("anything")).await.0,
|
||||
StatusCode::NOT_FOUND
|
||||
);
|
||||
}
|
||||
|
||||
fn hub_without_token() -> Hub {
|
||||
hub(Config::default())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn posts_are_rate_limited_per_address_and_capped_per_day() {
|
||||
let hub = hub(Config {
|
||||
hub_posts_per_hour: 20.0,
|
||||
hub_suggestions_per_day: 3,
|
||||
..config()
|
||||
});
|
||||
// A burst of a tenth of the hourly rate: two, then refused.
|
||||
let idea = |n: u32| json!({ "text": format!("Idea number {n}, long enough") });
|
||||
assert_eq!(
|
||||
hub.post("/hub/suggest", idea(1)).await.0,
|
||||
StatusCode::ACCEPTED
|
||||
);
|
||||
assert_eq!(
|
||||
hub.post("/hub/suggest", idea(2)).await.0,
|
||||
StatusCode::ACCEPTED
|
||||
);
|
||||
assert_eq!(
|
||||
hub.post("/hub/suggest", idea(3)).await.0,
|
||||
StatusCode::TOO_MANY_REQUESTS
|
||||
);
|
||||
|
||||
let busy = self::hub(Config {
|
||||
hub_posts_per_hour: 1000.0,
|
||||
hub_suggestions_per_day: 3,
|
||||
..config()
|
||||
});
|
||||
for n in 0..3 {
|
||||
assert_eq!(
|
||||
busy.post("/hub/suggest", idea(n)).await.0,
|
||||
StatusCode::ACCEPTED
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
busy.post("/hub/suggest", idea(9)).await.0,
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
);
|
||||
busy.day.store(DAY + 1, Ordering::SeqCst);
|
||||
assert_eq!(
|
||||
busy.post("/hub/suggest", idea(9)).await.0,
|
||||
StatusCode::ACCEPTED,
|
||||
"a new day"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn old_entries_go_and_reports_never_opened_are_given_up() {
|
||||
let hub = hub(config());
|
||||
*hub.github.down.lock().unwrap() = true;
|
||||
let (_, body) = hub
|
||||
.post("/hub/report", report("Crash when opening a chest"))
|
||||
.await;
|
||||
assert!(body["id"].as_i64().unwrap() > 0);
|
||||
hub.post("/hub/suggest", json!({ "text": "An idea from long ago" }))
|
||||
.await;
|
||||
hub.day.store(DAY + 8, Ordering::SeqCst);
|
||||
hub.app.maintain().unwrap();
|
||||
*hub.github.down.lock().unwrap() = false;
|
||||
hub.app.file_pending();
|
||||
assert!(
|
||||
hub.github.opened.lock().unwrap().is_empty(),
|
||||
"past a week a report is given up"
|
||||
);
|
||||
hub.day.store(DAY + 91, Ordering::SeqCst);
|
||||
hub.app.maintain().unwrap();
|
||||
let (_, left) = hub.get("/hub/suggestions", Some("kira-secret")).await;
|
||||
assert!(left["suggestions"].as_array().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_website_may_post_from_the_browser() {
|
||||
let hub = hub(config());
|
||||
let res = router(hub.app.clone())
|
||||
.oneshot(
|
||||
Request::options("/hub/vote")
|
||||
Request::options("/hub/report")
|
||||
.header("origin", "https://playhorizon.dev")
|
||||
.header("access-control-request-method", "POST")
|
||||
.body(Body::empty())
|
||||
|
||||
Reference in New Issue
Block a user