Files
horizon-airdef/crates/horizon-server/src/supervisor.rs
T
Ludovic Marie 7f77772e15 horizon-server: the web panel
One page embedded in the binary (crates/horizon-server/panel: HTML, CSS and JS, no framework or CDN): status,
live TPS, MSPT, RAM and CPU graphs over the last 10 minutes, players with op and kick, the console, and
backups with restore, plus restart and stop.

It listens on [panel] bind, 127.0.0.1:8130 by default; a tailnet address reaches it from the owner's
devices, and any other address is refused unless allow_public is set. `horizon-server panel` prints a
one-time login link (10 minutes) that becomes an HttpOnly, SameSite=Strict session cookie. The Host header
must name the panel, every POST needs X-Horizon: 1, failed logins are rate-limited, and the page is served
with a strict Content-Security-Policy. The control API gains GET /api/v1/metrics and, with the token
only, POST /api/v1/panel/login.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 22:18:46 +02:00

462 lines
17 KiB
Rust

//! `horizon-server run`: writes `server.properties` from `server.toml`, runs the engine with the console on
//! stdin, serves the control API, stops the game cleanly on Ctrl-C, SIGTERM or `stop`, and restarts it when
//! it crashes (docs/server.md, sections 2 and 7).
use crate::{
api::{self, Phase, Shared},
backup, config,
config::Config,
engine, signals,
};
use horizon_server_files::java_properties::now_millis;
use serde_json::json;
use std::{
fs::File,
io::{BufRead, BufReader, Read, Write},
path::Path,
process::{Child, ExitStatus},
sync::{Arc, atomic::Ordering, mpsc},
thread,
time::{Duration, Instant},
};
/// After this many crashes within [`CRASH_WINDOW`] the supervisor stops restarting the game.
const MAX_CRASHES: usize = 5;
const CRASH_WINDOW: Duration = Duration::from_secs(600);
/// The first wait before a restart; it doubles after each crash, up to [`MAX_BACKOFF`].
const FIRST_BACKOFF: Duration = Duration::from_secs(5);
const MAX_BACKOFF: Duration = Duration::from_secs(300);
/// How long `stop` may take before the game is killed.
const STOP_TIMEOUT: Duration = Duration::from_secs(120);
/// A line from the supervisor, in the game's log format (`[HH:mm:ss] [thread/LEVEL]: message`, UTC), on
/// stdout and in the log the API serves.
fn note(shared: &Shared, level: &str, message: &str) {
let secs = now_millis().div_euclid(1000).rem_euclid(86_400);
let line = format!(
"[{:02}:{:02}:{:02}] [Supervisor/{level}]: {message}",
secs / 3600,
secs / 60 % 60,
secs % 60
);
println!("{line}");
shared.log.lock().unwrap().push(line);
}
/// The folder's lock: one supervisor per server folder. Released when the process ends, even by a crash.
pub fn lock(dir: &Path) -> Result<File, String> {
let state = dir.join(".horizon");
std::fs::create_dir_all(&state).map_err(|e| format!("{}: {e}", state.display()))?;
let file = File::options()
.create(true)
.truncate(false)
.write(true)
.open(state.join("lock"))
.map_err(|e| e.to_string())?;
match file.try_lock() {
Ok(()) => Ok(file),
Err(std::fs::TryLockError::WouldBlock) => {
Err(format!("a server is already running in {}", dir.display()))
}
Err(std::fs::TryLockError::Error(e)) => Err(e.to_string()),
}
}
enum Exit {
/// Asked to stop, or the game ended by itself without an error (`stop` typed in its console, the EULA
/// not agreed, the port taken): not restarted.
Stopped,
/// Asked to restart (`horizon-server restart`).
Restart,
Crashed(ExitStatus),
}
pub fn run(dir: &Path) -> Result<(), String> {
signals::install();
let config = Config::load(dir)?;
let program = engine::program(&config, dir)?;
let resources = engine::resources(&config, dir, Some(&program))?;
let _lock = lock(dir)?;
let shared = Shared::new(dir);
let _endpoint = api::start(shared.clone())?;
match config.panel.address() {
Ok(Some(address)) => match crate::panel::start(shared.clone(), address) {
Ok(()) => note(
&shared,
"INFO",
&format!(
"Web panel on http://{address} (`horizon-server panel` prints a login link)"
),
),
Err(e) => note(
&shared,
"WARN",
&format!("The web panel can't listen on {address}: {e}"),
),
},
Ok(None) => {}
Err(e) => note(&shared, "WARN", &e),
}
sample(shared.clone());
schedule_backups(shared.clone());
let console = console_lines();
let mut crashes: Vec<Instant> = Vec::new();
loop {
// Read again before every start: edits to server.toml apply on the next restart.
let config = Config::load(dir)?;
let written = config
.write_properties(dir)
.map_err(|e| format!("server.properties: {e}"))?;
note(
&shared,
"INFO",
&format!(
"Starting {} with the game files in {}",
program.display(),
resources.display()
),
);
shared.set_phase(Phase::Starting);
let mut child = engine::spawn(&program, &resources, dir)
.map_err(|e| format!("{}: {e}", program.display()))?;
{
let mut state = shared.state.lock().unwrap();
state.engine_pid = Some(child.id());
state.engine_started = Some(Instant::now());
}
let exit = supervise(&shared, &mut child, &console);
{
let mut state = shared.state.lock().unwrap();
state.engine_pid = None;
state.engine_started = None;
}
match config::sync_back(dir, &written) {
Ok(keys) if !keys.is_empty() => note(
&shared,
"INFO",
&format!(
"Copied into server.toml what the game changed: {}",
keys.join(", ")
),
),
Ok(_) => {}
Err(e) => note(
&shared,
"WARN",
&format!("Could not update server.toml: {e}"),
),
}
let status = match exit {
Exit::Stopped => return Ok(()),
Exit::Restart => {
let restore = shared.restore.lock().unwrap().take();
if let Some(name) = restore {
match backup::restore(dir, &config, &name) {
Ok(Some(old)) => note(
&shared,
"INFO",
&format!(
"Restored {name}; the world it replaced is in {}",
old.display()
),
),
Ok(None) => note(&shared, "INFO", &format!("Restored {name}")),
Err(e) => note(&shared, "ERROR", &format!("Could not restore {name}: {e}")),
}
}
continue;
}
Exit::Crashed(status) => status,
};
{
let mut state = shared.state.lock().unwrap();
state.crashes += 1;
state.last_crash = Some(status.to_string());
}
if !config.engine.restart_on_crash {
return Err(format!(
"the game stopped unexpectedly ({status}); engine.restart_on_crash is off"
));
}
let now = Instant::now();
crashes.retain(|t| now.duration_since(*t) < CRASH_WINDOW);
crashes.push(now);
if crashes.len() >= MAX_CRASHES {
return Err(format!(
"the game stopped unexpectedly {MAX_CRASHES} times in {} minutes: not restarting it. \
See logs/latest.log and crash-reports/",
CRASH_WINDOW.as_secs() / 60
));
}
let wait = backoff(crashes.len());
note(
&shared,
"WARN",
&format!(
"The game stopped unexpectedly ({status}); restarting in {} s",
wait.as_secs()
),
);
shared.set_phase(Phase::Restarting);
let until = Instant::now() + wait;
while Instant::now() < until {
if signals::requests() > 0 || shared.stop.load(Ordering::SeqCst) {
return Ok(());
}
thread::sleep(Duration::from_millis(100));
}
}
}
/// The wait before the restart that follows the `crashes`th recent crash.
fn backoff(crashes: usize) -> Duration {
let doublings = u32::try_from(crashes.saturating_sub(1))
.unwrap_or(u32::MAX)
.min(16);
(FIRST_BACKOFF * 2u32.pow(doublings)).min(MAX_BACKOFF)
}
/// The supervisor's own stdin, line by line (nothing when it has none, as under a service manager).
fn console_lines() -> mpsc::Receiver<String> {
let (tx, rx) = mpsc::channel();
let _ = thread::Builder::new()
.name("console".into())
.spawn(move || {
for line in std::io::stdin().lock().lines() {
let Ok(line) = line else { break };
if tx.send(line).is_err() {
break;
}
}
});
rx
}
/// Copies the game's output to ours and to the API's log, line by line; the game's `Done` line means it
/// is up.
fn pump(shared: &Arc<Shared>, from: impl Read + Send + 'static, to_stderr: bool) {
let shared = shared.clone();
let _ = thread::Builder::new()
.name("engine-log".into())
.spawn(move || {
for line in BufReader::new(from).lines() {
let Ok(line) = line else { break };
if to_stderr {
eprintln!("{line}");
} else {
println!("{line}");
if line.contains("/INFO]: Done (") {
let mut state = shared.state.lock().unwrap();
if state.phase == Phase::Starting {
state.phase = Phase::Running;
}
}
}
shared.log.lock().unwrap().push(line);
}
});
}
/// Every second, the game's RAM, CPU and status, for `status` and the panel's graphs.
fn sample(shared: Arc<Shared>) {
let _ = thread::Builder::new()
.name("sampler".into())
.spawn(move || {
let mut control: Option<engine::Control> = None;
let mut previous: Option<(u32, Instant, f64)> = None;
loop {
thread::sleep(Duration::from_secs(1));
let (pid, running) = {
let state = shared.state.lock().unwrap();
(state.engine_pid, state.phase == Phase::Running)
};
let Some(pid) = pid else {
control = None;
previous = None;
shared.state.lock().unwrap().sample = serde_json::Value::Null;
continue;
};
let now = Instant::now();
let cpu = horizon_bench::cpu_seconds(pid);
let cpu_percent = match (previous, cpu) {
(Some((p, at, before)), Some(cpu)) if p == pid => {
Some((cpu - before) / now.duration_since(at).as_secs_f64() * 100.0)
}
_ => None,
};
previous = cpu.map(|c| (pid, now, c));
if !running {
control = None;
} else if control.is_none() {
control = engine::Control::connect(&shared.dir).ok();
}
let game = control
.as_mut()
.map(|c| c.ask("server-status"))
.transpose()
.unwrap_or_else(|_| {
control = None;
None
});
let rss = horizon_bench::rss_bytes(pid);
shared.record(json!({
"t": std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs()),
"tps": game.as_ref().and_then(|g| g["tps"].as_f64()),
"mspt": game.as_ref().and_then(|g| g["mspt"]["average"].as_f64()),
"mspt_p95": game.as_ref().and_then(|g| g["mspt"]["p95"].as_f64()),
"rss_bytes": rss,
"cpu_percent": cpu_percent,
"players": game.as_ref().and_then(|g| g["players"].as_array().map(Vec::len)),
}));
shared.state.lock().unwrap().sample = json!({
"rss_bytes": rss,
"cpu_percent": cpu_percent,
"game": game,
});
}
});
}
/// Automatic backups every `backups.every` while the game runs, keeping the last `backups.keep`.
fn schedule_backups(shared: Arc<Shared>) {
let _ = thread::Builder::new()
.name("backups".into())
.spawn(move || {
let mut last = Instant::now();
loop {
thread::sleep(Duration::from_secs(30));
let Ok(config) = Config::load(&shared.dir) else {
continue;
};
let Ok(Some(every)) = backup::interval(&config.backups.every) else {
continue;
};
if last.elapsed() < every || shared.state.lock().unwrap().phase != Phase::Running {
continue;
}
last = Instant::now();
match shared.back_up(&backup::auto_name(std::time::SystemTime::now())) {
Ok(path) => {
note(&shared, "INFO", &format!("Backed up to {}", path.display()));
let pruned = backup::prune(&shared.dir, config.backups.keep as usize);
if !pruned.is_empty() {
note(
&shared,
"INFO",
&format!("Deleted older backups: {}", pruned.join(", ")),
);
}
}
Err(e) => note(
&shared,
"WARN",
&format!("The automatic backup failed: {e}"),
),
}
}
});
}
/// Runs one engine process to its end: console lines in, the log out, and a clean `stop` when asked.
fn supervise(shared: &Arc<Shared>, child: &mut Child, console: &mpsc::Receiver<String>) -> Exit {
if let Some(out) = child.stdout.take() {
pump(shared, out, false);
}
if let Some(err) = child.stderr.take() {
pump(shared, err, true);
}
let mut stdin = child.stdin.take();
let mut stopping: Option<Instant> = None;
let mut restart = false;
let mut handled = signals::requests();
loop {
while let Ok(line) = console.try_recv() {
let command = line.trim().trim_start_matches('/');
if command == "stop" && stopping.is_none() {
stopping = Some(Instant::now());
shared.set_phase(Phase::Stopping);
}
if let Some(input) = stdin.as_mut()
&& writeln!(input, "{line}")
.and_then(|()| input.flush())
.is_err()
{
stdin = None;
}
}
let requests = signals::requests();
let asked = requests > handled || shared.stop.swap(false, Ordering::SeqCst);
handled = requests;
if shared.restart.swap(false, Ordering::SeqCst) && stopping.is_none() {
restart = true;
note(shared, "INFO", "Restarting the server");
stopping = Some(Instant::now());
shared.set_phase(Phase::Restarting);
send_stop(&mut stdin);
}
if asked {
restart = false;
if stopping.is_none() {
note(shared, "INFO", "Stopping the server");
stopping = Some(Instant::now());
shared.set_phase(Phase::Stopping);
send_stop(&mut stdin);
} else {
note(shared, "WARN", "Stopping the game at once, without saving");
let _ = child.kill();
}
}
if stopping.is_some_and(|t| t.elapsed() > STOP_TIMEOUT) {
note(
shared,
"WARN",
&format!(
"The game did not stop within {} s: killing it",
STOP_TIMEOUT.as_secs()
),
);
let _ = child.kill();
stopping = Some(Instant::now());
}
match child.try_wait() {
Ok(Some(_)) if restart => return Exit::Restart,
Ok(Some(status)) if stopping.is_some() || status.success() => return Exit::Stopped,
Ok(Some(status)) => return Exit::Crashed(status),
Ok(None) => thread::sleep(Duration::from_millis(50)),
Err(e) => {
note(shared, "ERROR", &format!("Lost track of the game: {e}"));
return Exit::Stopped;
}
}
}
}
fn send_stop(stdin: &mut Option<std::process::ChildStdin>) {
if let Some(input) = stdin.as_mut() {
let _ = writeln!(input, "stop").and_then(|()| input.flush());
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn backoff_doubles_up_to_five_minutes() {
let waits: Vec<u64> = (1..=8).map(|n| backoff(n).as_secs()).collect();
assert_eq!(waits, [5, 10, 20, 40, 80, 160, 300, 300]);
}
#[test]
fn one_supervisor_per_folder() {
let dir = std::env::temp_dir().join(format!("horizon-server-lock-{}", std::process::id()));
let first = lock(&dir).unwrap();
assert!(lock(&dir).unwrap_err().contains("already running"));
drop(first);
lock(&dir).unwrap();
std::fs::remove_dir_all(&dir).unwrap();
}
}