From e8832190d1017de6d23eb66c0b8dc75ca35cc95f Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:06:24 +0200 Subject: [PATCH] horizon-server: start, stop, status, cmd and logs through a local control API The supervisor serves a small HTTP API on 127.0.0.1 (a token in .horizon/api.json, mode 0600, and a Host check): status, logs, console commands, stop and restart. `start` runs the server in the background and returns once the game is up; `restart` and `cmd` wait for it too. A sampler reads the game's tick rate, tick times, players and chunks every second over the existing control endpoint (`--control-socket`), and RAM and CPU from the system. The dedicated loop now records its tick times in the game clock, so `/tick query` reports real numbers on a dedicated server, as it does in single player. The changes to game/dedicated.rs are additive: the log capture for command replies, the tick counter and the control requests (game/dedicated/status.rs). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + crates/horizon-client/src/game/dedicated.rs | 30 ++ .../src/game/dedicated/status.rs | 100 +++++ crates/horizon-server/Cargo.toml | 3 + crates/horizon-server/src/api.rs | 378 ++++++++++++++++++ crates/horizon-server/src/cli.rs | 329 +++++++++++++++ crates/horizon-server/src/engine.rs | 57 ++- crates/horizon-server/src/http.rs | 219 ++++++++++ crates/horizon-server/src/lists.rs | 30 +- crates/horizon-server/src/main.rs | 39 +- crates/horizon-server/src/supervisor.rs | 162 +++++++- 11 files changed, 1309 insertions(+), 40 deletions(-) create mode 100644 crates/horizon-client/src/game/dedicated/status.rs create mode 100644 crates/horizon-server/src/api.rs create mode 100644 crates/horizon-server/src/cli.rs create mode 100644 crates/horizon-server/src/http.rs diff --git a/Cargo.lock b/Cargo.lock index a54f2627b..3d5583167 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1532,9 +1532,11 @@ dependencies = [ name = "horizon-server" version = "0.1.0" dependencies = [ + "horizon-bench", "horizon-server-files", "libc", "serde", + "serde_json", "toml", "toml_edit", "windows-sys 0.61.2", diff --git a/crates/horizon-client/src/game/dedicated.rs b/crates/horizon-client/src/game/dedicated.rs index cd7744d87..48ee966de 100644 --- a/crates/horizon-client/src/game/dedicated.rs +++ b/crates/horizon-client/src/game/dedicated.rs @@ -29,6 +29,8 @@ use std::{ time::{Duration, Instant}, }; +mod status; + /// Double.toString for the magnitudes positions have (no exponent form /// between 1e-3 and 1e7). pub(super) fn java_double(v: f64) -> String { @@ -64,6 +66,11 @@ pub(super) fn log(level: &str, message: &str) { _ => ::log::Level::Info, }; horizon_client::logging::write_only(file_level, message); + CAPTURE.with_borrow_mut(|lines| { + if let Some(lines) = lines { + lines.push(message.to_owned()); + } + }); let secs = now_millis().div_euclid(1000).rem_euclid(86_400); println!( "[{:02}:{:02}:{:02}] [Server thread/{level}]: {message}", @@ -73,6 +80,18 @@ pub(super) fn log(level: &str, message: &str) { ); } +thread_local! { + /// The lines logged while a control request runs a console command. + static CAPTURE: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; +} + +/// Runs `f` and returns the lines it logged. +fn capture_log(f: impl FnOnce()) -> Vec { + CAPTURE.set(Some(Vec::new())); + f(); + CAPTURE.take().unwrap_or_default() +} + /// Main.main up to the server start: arguments, server.properties (stored /// back at once), eula.txt. False when the process should stop here. pub(super) fn preflight(options: &mut Options) -> Result { @@ -289,6 +308,9 @@ struct ServerState { stop: bool, autosave: bool, empty_ticks: i32, + /// When the last passes of the loop ended, for the tick rate horizon-server shows. + ticks: std::collections::VecDeque, + paused: bool, } impl Game<'_> { @@ -469,14 +491,18 @@ impl Game<'_> { stop: false, autosave: true, empty_ticks: 0, + ticks: Default::default(), + paused: false, }; let pause_after = props.pause_when_empty_seconds * 20; let idle_timeout = props.player_idle_timeout; let mut next = Instant::now(); while !state.stop { + let tick_start = Instant::now(); while let Ok(line) = state.console.try_recv() { self.console_command(&mut state, &line); } + self.poll_server_control(&mut state); let players = self.lan.as_ref().map_or(0, |h| h.hub.guests.len()); if pause_after > 0 { if players == 0 { @@ -486,6 +512,7 @@ impl Game<'_> { } } let paused = pause_after > 0 && state.empty_ticks >= pause_after; + state.paused = paused; if paused && state.empty_ticks == pause_after { log( "INFO", @@ -511,6 +538,9 @@ impl Game<'_> { if idle_timeout > 0 { self.kick_idle(idle_timeout); } + // MinecraftServer.tickServer records each tick's time, for /tick query (and the status). + self.clock.record(tick_start.elapsed()); + state.ticked(Instant::now()); let rate = self.clock.rate().max(1.0); next += Duration::from_secs_f64(1.0 / f64::from(rate)); let now = Instant::now(); diff --git a/crates/horizon-client/src/game/dedicated/status.rs b/crates/horizon-client/src/game/dedicated/status.rs new file mode 100644 index 000000000..a2ec4c1ae --- /dev/null +++ b/crates/horizon-client/src/game/dedicated/status.rs @@ -0,0 +1,100 @@ +//! What `horizon-server` asks the dedicated server on its control endpoint (`--control-socket`, docs/server.md +//! section 2): `server-status` answers with the tick rate, the tick times `/tick query` reports, the players +//! and the loaded chunks; `console ` runs a console command and answers with the lines it logged. +use super::{ServerState, capture_log}; +use crate::game::Game; +use serde_json::{Value, json}; +use std::{ + sync::atomic::Ordering, + time::{Duration, Instant}, +}; + +/// The window the tick rate is measured over. +const TPS_WINDOW: Duration = Duration::from_secs(5); + +impl ServerState { + /// One pass of the server loop ended. + pub(super) fn ticked(&mut self, at: Instant) { + self.ticks.push_back(at); + while self + .ticks + .front() + .is_some_and(|t| at.duration_since(*t) > TPS_WINDOW) + { + self.ticks.pop_front(); + } + } + /// Ticks per second over the last few seconds. + fn tps(&self) -> f64 { + match (self.ticks.front(), self.ticks.back()) { + (Some(first), Some(last)) if self.ticks.len() > 1 => { + (self.ticks.len() - 1) as f64 / last.duration_since(*first).as_secs_f64().max(1e-9) + } + _ => 0.0, + } + } +} + +impl Game<'_> { + /// Answers the requests waiting on the control endpoint. + pub(super) fn poll_server_control(&mut self, state: &mut ServerState) { + while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) { + let reply = match request.line.trim().split_once(' ') { + _ if request.line.trim() == "server-status" => self.server_status(state), + Some(("console", line)) => { + // Command feedback is logged as system chat; the reply carries the text alone. + let output: Vec = capture_log(|| self.console_command(state, line)) + .into_iter() + .map(|l| { + l.strip_prefix("System chat: ") + .map_or(l.clone(), str::to_owned) + }) + .collect(); + json!({ "output": output }) + } + _ => json!({ "error": format!("unknown request {:?}", request.line) }), + }; + let _ = request.reply.send(reply.to_string()); + } + } + + fn server_status(&self, state: &ServerState) -> Value { + let millis = |nanos: u64| nanos as f64 / 1e6; + let samples = self.clock.sorted_samples(); + // The percentiles `/tick query` prints (TickCommand.percentileToString). + let at = |q: f64| { + let i = ((samples.len() as f64 * q) as usize).min(samples.len().saturating_sub(1)); + samples.get(i).copied().map_or(0.0, millis) + }; + let (players, max_players) = self.lan.as_ref().map_or((Vec::new(), 0), |h| { + let players = h + .hub + .guests + .iter() + .map(|g| { + json!({ + "name": g.name, + "uuid": horizon_protocol::crypto::uuid_string(g.uuid), + "address": g.address.ip().to_string(), + "ping_ms": g.latency.load(Ordering::Relaxed), + }) + }) + .collect(); + (players, h.hub.lan.shared.max_players()) + }); + json!({ + "tps": state.tps(), + "target_tps": self.clock.rate(), + "mspt": { + "average": millis(self.clock.average_nanos()), + "p50": at(0.5), + "p95": at(0.95), + "p99": at(0.99), + }, + "players": players, + "max_players": max_players, + "chunks": self.world.chunks.len(), + "paused": state.paused, + }) + } +} diff --git a/crates/horizon-server/Cargo.toml b/crates/horizon-server/Cargo.toml index 919001968..deab442ae 100644 --- a/crates/horizon-server/Cargo.toml +++ b/crates/horizon-server/Cargo.toml @@ -8,7 +8,10 @@ description = "Host a Horizon server: server.toml, the CLI and the supervisor th [dependencies] horizon-server-files = { path = "../horizon-server-files" } +# The game process's RAM and CPU (the same probes the benchmark reports use). +horizon-bench = { path = "../horizon-bench" } serde = { version = "1", features = ["derive"] } +serde_json = "1" toml = "1.1" # Edits server.toml in place, keeping the owner's comments and order. toml_edit = "0.25" diff --git a/crates/horizon-server/src/api.rs b/crates/horizon-server/src/api.rs new file mode 100644 index 000000000..dbca00215 --- /dev/null +++ b/crates/horizon-server/src/api.rs @@ -0,0 +1,378 @@ +//! The control API (docs/server.md, section 8): HTTP and JSON on 127.0.0.1, with a random token in +//! `.horizon/api.json` that only the server's own user can read. The CLI manages a running server through +//! it; the web panel and the MCP server will too. +//! +//! - `GET /api/v1/status`: the supervisor's state, the game's status (TPS, MSPT, players), RAM and CPU. +//! - `GET /api/v1/logs?since=N`: the log lines after the Nth. +//! - `POST /api/v1/command {"line": "..."}`: a console command; the lines it logged. +//! - `POST /api/v1/stop`, `POST /api/v1/restart`. + +use crate::{ + engine, + http::{self, Request, Response}, +}; +use serde_json::{Value, json}; +use std::{ + collections::VecDeque, + io::Write, + net::{SocketAddr, TcpListener}, + path::{Path, PathBuf}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; + +const ENDPOINT: &str = ".horizon/api.json"; +/// The log lines kept for `logs` and the panel's console. +const LOG_LINES: usize = 2000; + +/// What the supervisor is doing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Phase { + Starting, + Running, + Stopping, + /// Stopping the game to start it again (`restart`), or waiting to after a crash. + Restarting, +} + +impl Phase { + fn name(self) -> &'static str { + match self { + Self::Starting => "starting", + Self::Running => "running", + Self::Stopping => "stopping", + Self::Restarting => "restarting", + } + } +} + +pub struct State { + pub phase: Phase, + pub engine_pid: Option, + pub engine_started: Option, + pub crashes: u32, + pub last_crash: Option, + /// The latest sample (`sample`): the game's status, RAM and CPU. + pub sample: Value, +} + +/// The log, numbered from the first line since the supervisor started. +#[derive(Default)] +pub struct Log { + lines: VecDeque, + first: u64, +} + +impl Log { + pub fn push(&mut self, line: String) { + if self.lines.len() == LOG_LINES { + self.lines.pop_front(); + self.first += 1; + } + self.lines.push_back(line); + } + /// The lines after the `since`th, and the number to ask from next. + pub fn since(&self, since: u64) -> (u64, Vec) { + let skip = since.saturating_sub(self.first) as usize; + let next = self.first + self.lines.len() as u64; + (next, self.lines.iter().skip(skip).cloned().collect()) + } +} + +/// What the supervisor and the API threads share. +pub struct Shared { + pub dir: PathBuf, + pub state: Mutex, + pub log: Mutex, + pub stop: AtomicBool, + pub restart: AtomicBool, + pub started: Instant, +} + +impl Shared { + pub fn new(dir: &Path) -> Arc { + Arc::new(Self { + dir: dir.to_owned(), + state: Mutex::new(State { + phase: Phase::Starting, + engine_pid: None, + engine_started: None, + crashes: 0, + last_crash: None, + sample: Value::Null, + }), + log: Mutex::new(Log::default()), + stop: AtomicBool::new(false), + restart: AtomicBool::new(false), + started: Instant::now(), + }) + } + pub fn set_phase(&self, phase: Phase) { + self.state.lock().unwrap().phase = phase; + } +} + +fn status(shared: &Shared) -> Value { + let state = shared.state.lock().unwrap(); + json!({ + "phase": state.phase.name(), + "version": env!("CARGO_PKG_VERSION"), + "pid": std::process::id(), + "uptime_seconds": shared.started.elapsed().as_secs(), + "engine_pid": state.engine_pid, + "engine_uptime_seconds": state.engine_started.map(|t| t.elapsed().as_secs()), + "crashes": state.crashes, + "last_crash": state.last_crash, + "sample": state.sample, + }) +} + +/// A random token: two 64-bit hashes keyed by the OS's random hasher seeds, the time and the process. +fn random_token() -> String { + use std::hash::{BuildHasher, Hasher}; + (0..2u64) + .map(|i| { + let mut h = std::collections::hash_map::RandomState::new().build_hasher(); + h.write_u64(i); + h.write_u128( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()), + ); + h.write_u32(std::process::id()); + format!("{:016x}", h.finish()) + }) + .collect() +} + +fn same_token(a: &str, b: &str) -> bool { + a.len() == b.len() && a.bytes().zip(b.bytes()).fold(0u8, |d, (x, y)| d | (x ^ y)) == 0 +} + +/// Writes a file only its owner can read. +pub fn write_private(path: &Path, text: &str) -> std::io::Result<()> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + options.open(path)?.write_all(text.as_bytes()) +} + +/// Removes the endpoint file when the supervisor ends. +pub struct Endpoint(PathBuf); +impl Drop for Endpoint { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// Starts the API on a free port of 127.0.0.1 and writes where it is and its token. +pub fn start(shared: Arc) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").map_err(|e| format!("control API: {e}"))?; + let address = listener.local_addr().map_err(|e| e.to_string())?; + let token = random_token(); + let path = shared.dir.join(ENDPOINT); + write_private( + &path, + &json!({ "address": address.to_string(), "token": token, "pid": std::process::id() }) + .to_string(), + ) + .map_err(|e| format!("{}: {e}", path.display()))?; + let hosts = [address.to_string(), format!("localhost:{}", address.port())]; + http::serve(listener, move |request| { + // A page in a browser can't send the token, and a rebound DNS name can't pass as this address. + let authorised = request + .header("authorization") + .and_then(|h| h.strip_prefix("Bearer ")) + .is_some_and(|t| same_token(t.trim(), &token)); + if !authorised { + return Response::error(401, "missing or wrong token (see .horizon/api.json)"); + } + if !request + .header("host") + .is_some_and(|h| hosts.iter().any(|a| a == h)) + { + return Response::error(403, "wrong Host"); + } + route(&shared, request) + }) + .map_err(|e| e.to_string())?; + Ok(Endpoint(path)) +} + +fn route(shared: &Shared, request: &Request) -> Response { + match (request.method.as_str(), request.path.as_str()) { + ("GET", "/api/v1/status") => Response::json(200, &status(shared)), + ("GET", "/api/v1/logs") => { + let since = request + .param("since") + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + let (next, lines) = shared.log.lock().unwrap().since(since); + Response::json(200, &json!({ "next": next, "lines": lines })) + } + ("POST", "/api/v1/command") => { + let line = serde_json::from_slice::(&request.body) + .ok() + .and_then(|v| v["line"].as_str().map(str::to_owned)); + let Some(line) = line.filter(|l| !l.trim().is_empty() && !l.contains('\n')) else { + return Response::error(400, "the body must be {\"line\": \"\"}"); + }; + if shared.state.lock().unwrap().phase != Phase::Running { + return Response::error(409, "the game isn't running right now"); + } + match engine::Control::connect(&shared.dir) + .and_then(|mut c| c.ask(&format!("console {}", line.trim_start_matches('/')))) + { + Ok(reply) => Response::json(200, &reply), + Err(e) => Response::error(409, &format!("the game didn't answer: {e}")), + } + } + ("POST", "/api/v1/stop") => { + shared.stop.store(true, Ordering::SeqCst); + Response::json(200, &json!({ "ok": true })) + } + ("POST", "/api/v1/restart") => { + shared.restart.store(true, Ordering::SeqCst); + Response::json(200, &json!({ "ok": true })) + } + ( + _, + "/api/v1/status" | "/api/v1/logs" | "/api/v1/command" | "/api/v1/stop" + | "/api/v1/restart", + ) => Response::error(405, "wrong method"), + _ => Response::error(404, "no such endpoint"), + } +} + +/// The API of the server running in a folder, for the CLI. +pub struct Client { + address: SocketAddr, + token: String, +} + +impl Client { + /// The running server's API; `None` when no server runs in `dir`. + pub fn open(dir: &Path) -> Result, String> { + if crate::supervisor::lock(dir).is_ok() { + return Ok(None); + } + // The supervisor writes the endpoint just after taking the lock: give it a moment. + let path = dir.join(ENDPOINT); + let deadline = Instant::now() + Duration::from_secs(5); + let text = loop { + match std::fs::read_to_string(&path) { + Ok(text) => break text, + Err(_) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(100)) + } + Err(e) => return Err(format!("{}: {e}", path.display())), + } + }; + let v: Value = + serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))?; + let address = v["address"] + .as_str() + .and_then(|a| a.parse().ok()) + .ok_or_else(|| format!("{}: no address", path.display()))?; + let token = v["token"].as_str().unwrap_or_default().to_owned(); + Ok(Some(Self { address, token })) + } + + pub fn call(&self, method: &str, path: &str, body: Option<&Value>) -> Result { + let (status, text) = http::request( + self.address, + &self.token, + method, + path, + body, + Duration::from_secs(60), + ) + .map_err(|e| format!("the server didn't answer: {e}"))?; + let value: Value = serde_json::from_str(&text).unwrap_or(Value::Null); + if status == 200 { + Ok(value) + } else { + Err(value["error"].as_str().unwrap_or(&text).to_owned()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn log_keeps_the_last_lines_numbered() { + let mut log = Log::default(); + for i in 0..LOG_LINES + 10 { + log.push(i.to_string()); + } + let (next, lines) = log.since(0); + assert_eq!(next, (LOG_LINES + 10) as u64); + assert_eq!(lines.len(), LOG_LINES); + assert_eq!(lines[0], "10"); + let (next2, tail) = log.since(next - 2); + assert_eq!( + (next2, tail), + ( + next, + vec![(LOG_LINES + 8).to_string(), (LOG_LINES + 9).to_string()] + ) + ); + } + + #[test] + fn the_api_wants_its_token() { + let dir = std::env::temp_dir().join(format!("horizon-server-api-{}", std::process::id())); + std::fs::create_dir_all(dir.join(".horizon")).unwrap(); + let shared = Shared::new(&dir); + let _endpoint = start(shared.clone()).unwrap(); + let v: Value = + serde_json::from_str(&std::fs::read_to_string(dir.join(ENDPOINT)).unwrap()).unwrap(); + let address: SocketAddr = v["address"].as_str().unwrap().parse().unwrap(); + let token = v["token"].as_str().unwrap(); + let get = |token: &str| { + http::request( + address, + token, + "GET", + "/api/v1/status", + None, + Duration::from_secs(5), + ) + .unwrap() + }; + assert_eq!(get("nope").0, 401); + let (status, body) = get(token); + assert_eq!(status, 200); + let status: Value = serde_json::from_str(&body).unwrap(); + assert_eq!(status["phase"], "starting"); + let (code, _) = http::request( + address, + token, + "POST", + "/api/v1/stop", + None, + Duration::from_secs(5), + ) + .unwrap(); + assert_eq!(code, 200); + assert!(shared.stop.load(Ordering::SeqCst)); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(dir.join(ENDPOINT)) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600); + } + drop(_endpoint); + assert!(!dir.join(ENDPOINT).exists()); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/horizon-server/src/cli.rs b/crates/horizon-server/src/cli.rs new file mode 100644 index 000000000..b05292f5f --- /dev/null +++ b/crates/horizon-server/src/cli.rs @@ -0,0 +1,329 @@ +//! The commands that manage a server in the background (docs/server.md, section 5): `start`, `stop`, +//! `restart`, `status`, `cmd` and `logs`. They talk to the running supervisor through the control API. + +use crate::{api::Client, signals, supervisor}; +use serde_json::{Value, json}; +use std::{ + path::Path, + process::{Command, Stdio}, + time::{Duration, Instant}, +}; + +/// How long `start` waits for the game's `Done`, and `stop` for the server to end. +const START_TIMEOUT: Duration = Duration::from_secs(300); +const STOP_TIMEOUT: Duration = Duration::from_secs(180); + +fn running(dir: &Path) -> Result { + Client::open(dir)?.ok_or_else(|| format!("no server is running in {}", dir.display())) +} + +/// `start`: `run` in the background, its output in logs/supervisor.log; returns once the game is up. +pub fn start(dir: &Path) -> Result<(), String> { + if Client::open(dir)?.is_some() { + return Err(format!("a server is already running in {}", dir.display())); + } + crate::config::Config::load(dir)?; + // Left by a supervisor that was killed: the new one writes its own. + let endpoint = dir.join(".horizon/api.json"); + let _ = std::fs::remove_file(&endpoint); + let logs = dir.join("logs"); + std::fs::create_dir_all(&logs).map_err(|e| e.to_string())?; + let log_path = logs.join("supervisor.log"); + let log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .map_err(|e| format!("{}: {e}", log_path.display()))?; + let exe = std::env::current_exe().map_err(|e| e.to_string())?; + let mut command = Command::new(exe); + command + .arg("--dir") + .arg(dir) + .arg("run") + .stdin(Stdio::null()) + .stdout(log.try_clone().map_err(|e| e.to_string())?) + .stderr(log); + // Its own process group: closing this terminal doesn't stop the server. + #[cfg(unix)] + std::os::unix::process::CommandExt::process_group(&mut command, 0); + #[cfg(windows)] + std::os::windows::process::CommandExt::creation_flags( + &mut command, + windows_sys::Win32::System::Threading::DETACHED_PROCESS + | windows_sys::Win32::System::Threading::CREATE_NEW_PROCESS_GROUP, + ); + let mut child = command.spawn().map_err(|e| e.to_string())?; + let deadline = Instant::now() + START_TIMEOUT; + print!("Starting"); + loop { + if let Ok(Some(status)) = child.try_wait() { + println!(); + return Err(format!( + "the server stopped ({status}); see {}", + log_path.display() + )); + } + // Only once the new supervisor holds the folder (its endpoint is there) may we look: probing the + // lock any earlier could take it from under it. + if endpoint.exists() + && let Ok(Some(client)) = Client::open(dir) + && let Ok(status) = client.call("GET", "/api/v1/status", None) + && status["phase"] == "running" + { + println!( + "\nThe server is running (pid {}). `horizon-server status` shows how it does, \ + `horizon-server stop` stops it.", + child.id() + ); + return Ok(()); + } + if Instant::now() > deadline { + println!(); + return Err(format!( + "the game isn't up after {} s; it may still be starting: see {}", + START_TIMEOUT.as_secs(), + log_path.display() + )); + } + print!("."); + let _ = std::io::Write::flush(&mut std::io::stdout()); + std::thread::sleep(Duration::from_millis(500)); + } +} + +/// `stop` and `restart`. +pub fn stop(dir: &Path, restart: bool) -> Result<(), String> { + let client = running(dir)?; + if restart { + let before = client.call("GET", "/api/v1/status", None)?["engine_pid"].as_u64(); + client.call("POST", "/api/v1/restart", None)?; + print!("Restarting"); + wait_running(&client, before)?; + println!("\nThe game is running again."); + return Ok(()); + } + client.call("POST", "/api/v1/stop", None)?; + let deadline = Instant::now() + STOP_TIMEOUT; + while supervisor::lock(dir).is_err() { + if Instant::now() > deadline { + return Err("the server is still stopping".into()); + } + std::thread::sleep(Duration::from_millis(200)); + } + println!("Stopped."); + Ok(()) +} + +/// Waits until the game is up: a new one when `replaced` is the engine's pid before a restart. +fn wait_running(client: &Client, replaced: Option) -> Result<(), String> { + let deadline = Instant::now() + START_TIMEOUT; + loop { + let status = client.call("GET", "/api/v1/status", None)?; + if status["phase"] == "running" && status["engine_pid"].as_u64() != replaced { + return Ok(()); + } + if status["phase"] == "stopped" || status["phase"] == "stopping" { + return Err("the server is stopping".into()); + } + if Instant::now() > deadline { + return Err(format!( + "the game isn't up after {} s; `horizon-server logs` shows why", + START_TIMEOUT.as_secs() + )); + } + if replaced.is_some() { + print!("."); + let _ = std::io::Write::flush(&mut std::io::stdout()); + } + std::thread::sleep(Duration::from_millis(500)); + } +} + +/// `cmd`: one console command, and what it printed. +pub fn command(dir: &Path, words: &[String]) -> Result<(), String> { + if words.is_empty() { + return Err("cmd needs a console command, for example: horizon-server cmd list".into()); + } + let client = running(dir)?; + // A game that is starting or restarting takes the command once it is up. + wait_running(&client, None)?; + let reply = client.call( + "POST", + "/api/v1/command", + Some(&json!({ "line": words.join(" ") })), + )?; + for line in reply["output"].as_array().into_iter().flatten() { + println!("{}", line.as_str().unwrap_or_default()); + } + Ok(()) +} + +/// `status [--json]`. +pub fn status(dir: &Path, as_json: bool) -> Result<(), String> { + let Some(client) = Client::open(dir)? else { + if as_json { + println!("{}", json!({ "phase": "stopped" })); + } else { + println!("Stopped."); + } + return Ok(()); + }; + let status = client.call("GET", "/api/v1/status", None)?; + if as_json { + println!("{status}"); + } else { + print!("{}", describe(&status)); + } + Ok(()) +} + +/// The status for people: three short lines. +fn describe(status: &Value) -> String { + let mut out = String::new(); + let phase = status["phase"].as_str().unwrap_or("?"); + let uptime = status["engine_uptime_seconds"] + .as_u64() + .or(status["uptime_seconds"].as_u64()) + .map_or_else(String::new, |s| format!(", up {}", duration(s))); + let crashes = match status["crashes"].as_u64() { + Some(0) | None => String::new(), + Some(n) => format!( + ", {n} crash{} (last: {})", + if n == 1 { "" } else { "es" }, + status["last_crash"].as_str().unwrap_or("?") + ), + }; + out += &format!("{}{uptime}{crashes}\n", capitalised(phase)); + let sample = &status["sample"]; + let game = &sample["game"]; + if let Some(tps) = game["tps"].as_f64() { + let mspt = &game["mspt"]; + out += &format!( + "TPS {tps:.1}, MSPT {:.1} ms average, {:.1} ms P95{}\n", + mspt["average"].as_f64().unwrap_or(0.0), + mspt["p95"].as_f64().unwrap_or(0.0), + if game["paused"] == true { + " (paused: nobody online)" + } else { + "" + } + ); + let names: Vec<&str> = game["players"] + .as_array() + .into_iter() + .flatten() + .filter_map(|p| p["name"].as_str()) + .collect(); + out += &format!( + "{} of {} players{}{}, {} chunks loaded\n", + names.len(), + game["max_players"].as_u64().unwrap_or(0), + if names.is_empty() { "" } else { ": " }, + names.join(", "), + game["chunks"].as_u64().unwrap_or(0) + ); + } + let ram = sample["rss_bytes"] + .as_f64() + .map(|b| format!("RAM {:.2} GB", b / 1e9)); + let cpu = sample["cpu_percent"] + .as_f64() + .map(|c| format!("CPU {c:.0} % of a core")); + let usage: Vec = ram.into_iter().chain(cpu).collect(); + if !usage.is_empty() { + out += &format!("{}\n", usage.join(", ")); + } + out +} + +fn capitalised(s: &str) -> String { + let mut c = s.chars(); + c.next() + .map(|f| f.to_uppercase().chain(c).collect()) + .unwrap_or_default() +} + +fn duration(secs: u64) -> String { + match secs { + 0..60 => format!("{secs} s"), + 60..3600 => format!("{} min", secs / 60), + 3600..86_400 => format!("{} h {:02} min", secs / 3600, secs / 60 % 60), + _ => format!("{} d {} h", secs / 86_400, secs / 3600 % 24), + } +} + +/// `logs [-f] [-n N]`: the last lines, from the running server or else from logs/latest.log. +pub fn logs(dir: &Path, follow: bool, count: usize) -> Result<(), String> { + let Some(client) = Client::open(dir)? else { + if follow { + return Err("no server is running to follow".into()); + } + let path = dir.join("logs/latest.log"); + let text = + std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?; + let lines: Vec<&str> = text.lines().collect(); + for line in &lines[lines.len().saturating_sub(count)..] { + println!("{line}"); + } + return Ok(()); + }; + let reply = client.call("GET", "/api/v1/logs?since=0", None)?; + let lines = reply["lines"].as_array().cloned().unwrap_or_default(); + for line in &lines[lines.len().saturating_sub(count)..] { + println!("{}", line.as_str().unwrap_or_default()); + } + if !follow { + return Ok(()); + } + signals::install(); + let mut next = reply["next"].as_u64().unwrap_or(0); + while signals::requests() == 0 { + std::thread::sleep(Duration::from_millis(500)); + let Ok(reply) = client.call("GET", &format!("/api/v1/logs?since={next}"), None) else { + break; + }; + for line in reply["lines"].as_array().into_iter().flatten() { + println!("{}", line.as_str().unwrap_or_default()); + } + next = reply["next"].as_u64().unwrap_or(next); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_reads_well() { + let status = json!({ + "phase": "running", + "engine_uptime_seconds": 7380, + "crashes": 1, + "last_crash": "signal: 9 (SIGKILL)", + "sample": { + "rss_bytes": 1_420_000_000u64, + "cpu_percent": 12.4, + "game": { + "tps": 20.0, + "mspt": { "average": 3.12, "p95": 7.9 }, + "players": [{ "name": "Alex" }, { "name": "Steve" }], + "max_players": 20, + "chunks": 1234, + "paused": false + } + } + }); + assert_eq!( + describe(&status), + "Running, up 2 h 03 min, 1 crash (last: signal: 9 (SIGKILL))\n\ + TPS 20.0, MSPT 3.1 ms average, 7.9 ms P95\n\ + 2 of 20 players: Alex, Steve, 1234 chunks loaded\n\ + RAM 1.42 GB, CPU 12 % of a core\n" + ); + assert_eq!( + describe(&json!({ "phase": "starting", "uptime_seconds": 4 })), + "Starting, up 4 s\n" + ); + } +} diff --git a/crates/horizon-server/src/engine.rs b/crates/horizon-server/src/engine.rs index 41f71f433..c0043e1d3 100644 --- a/crates/horizon-server/src/engine.rs +++ b/crates/horizon-server/src/engine.rs @@ -4,8 +4,11 @@ use crate::config::Config; use std::{ + io::{BufRead, BufReader, Write}, + net::{Ipv4Addr, TcpStream}, path::{Path, PathBuf}, process::{Child, Command, Stdio}, + time::Duration, }; /// The game's program names, most specific first: a headless engine once there is one, else the game. @@ -83,12 +86,15 @@ pub fn resources(config: &Config, dir: &Path, program: Option<&Path>) -> Result< }) } -/// Starts the engine in the server folder, with its console and log piped to the supervisor. +/// Starts the engine in the server folder, with its console and log piped to the supervisor and its status +/// on the control endpoint. pub fn spawn(program: &Path, resources: &Path, dir: &Path) -> std::io::Result { let mut command = Command::new(program); command .arg("--resources") .arg(resources) + .arg("--control-socket") + .arg(control_path(dir)) .args(["--dedicated", "--nogui"]) .current_dir(dir) .env("HORIZON_NO_AUDIO", "1") @@ -106,6 +112,55 @@ pub fn spawn(program: &Path, resources: &Path, dir: &Path) -> std::io::Result PathBuf { + dir.join(".horizon/engine.json") +} + +/// A connection to the game's control endpoint: one JSON line asked, one answered. +pub struct Control { + writer: TcpStream, + reader: BufReader, +} + +impl Control { + pub fn connect(dir: &Path) -> std::io::Result { + let text = std::fs::read_to_string(control_path(dir))?; + let bad = || std::io::Error::new(std::io::ErrorKind::InvalidData, "not a control endpoint"); + let v: serde_json::Value = serde_json::from_str(&text).map_err(|_| bad())?; + let port = v["port"] + .as_u64() + .and_then(|p| u16::try_from(p).ok()) + .ok_or_else(bad)?; + let token = v["token"].as_str().ok_or_else(bad)?; + let writer = TcpStream::connect((Ipv4Addr::LOCALHOST, port))?; + writer.set_read_timeout(Some(Duration::from_secs(30)))?; + let mut control = Self { + reader: BufReader::new(writer.try_clone()?), + writer, + }; + let reply = control.ask(&format!("auth {token}"))?; + if reply["ok"] != true { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "the game refused the token", + )); + } + Ok(control) + } + + pub fn ask(&mut self, line: &str) -> std::io::Result { + writeln!(self.writer, "{line}")?; + self.writer.flush()?; + let mut reply = String::new(); + if self.reader.read_line(&mut reply)? == 0 { + return Err(std::io::ErrorKind::UnexpectedEof.into()); + } + serde_json::from_str(&reply) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/horizon-server/src/http.rs b/crates/horizon-server/src/http.rs new file mode 100644 index 000000000..3d1507194 --- /dev/null +++ b/crates/horizon-server/src/http.rs @@ -0,0 +1,219 @@ +//! Just enough HTTP/1.1 for the control API on 127.0.0.1 (docs/server.md, section 8): one request per +//! connection, JSON bodies with a Content-Length. No async runtime and no dependency: a thread per +//! connection is plenty for a CLI, a panel tab and an MCP client. + +use std::{ + io::{BufRead, BufReader, Read, Write}, + net::{SocketAddr, TcpListener, TcpStream}, + time::Duration, +}; + +/// The largest request body the API takes (a console command, a settings change). +const MAX_BODY: usize = 1 << 20; + +pub struct Request { + pub method: String, + pub path: String, + pub query: String, + pub headers: Vec<(String, String)>, + pub body: Vec, +} + +impl Request { + pub fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.as_str()) + } + /// One `key=value` from the query string (no percent-decoding: the API's values are numbers). + pub fn param(&self, key: &str) -> Option<&str> { + self.query + .split('&') + .filter_map(|p| p.split_once('=')) + .find(|(k, _)| *k == key) + .map(|(_, v)| v) + } +} + +pub struct Response { + pub status: u16, + pub body: String, +} + +impl Response { + pub fn json(status: u16, value: &serde_json::Value) -> Self { + Self { + status, + body: value.to_string(), + } + } + pub fn error(status: u16, message: &str) -> Self { + Self::json(status, &serde_json::json!({ "error": message })) + } +} + +fn reason(status: u16) -> &'static str { + match status { + 200 => "OK", + 400 => "Bad Request", + 401 => "Unauthorized", + 403 => "Forbidden", + 404 => "Not Found", + 405 => "Method Not Allowed", + 409 => "Conflict", + 413 => "Payload Too Large", + _ => "Error", + } +} + +fn read_request(stream: &TcpStream) -> Result { + let bad = |m: &str| Response::error(400, m); + let mut reader = BufReader::new(stream); + let mut line = String::new(); + reader.read_line(&mut line).map_err(|_| bad("no request"))?; + let mut parts = line.split_whitespace(); + let (Some(method), Some(target)) = (parts.next(), parts.next()) else { + return Err(bad("malformed request line")); + }; + let (path, query) = target.split_once('?').unwrap_or((target, "")); + let mut headers = Vec::new(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).map_err(|_| bad("headers"))? == 0 || line.trim().is_empty() { + break; + } + if headers.len() > 64 { + return Err(bad("too many headers")); + } + if let Some((k, v)) = line.split_once(':') { + headers.push((k.trim().to_owned(), v.trim().to_owned())); + } + } + let length = headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("content-length")) + .map_or(Ok(0), |(_, v)| v.parse::()) + .map_err(|_| bad("bad Content-Length"))?; + if length > MAX_BODY { + return Err(Response::error(413, "the body is too large")); + } + let mut body = vec![0; length]; + reader + .read_exact(&mut body) + .map_err(|_| bad("the body is shorter than its Content-Length"))?; + Ok(Request { + method: method.to_owned(), + path: path.to_owned(), + query: query.to_owned(), + headers, + body, + }) +} + +fn write_response(mut stream: &TcpStream, response: &Response) { + let _ = write!( + stream, + "HTTP/1.1 {} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n{}", + response.status, + reason(response.status), + response.body.len(), + response.body + ); + let _ = stream.flush(); +} + +/// Serves `handle` on `listener` until the process ends, a thread per connection. +pub fn serve( + listener: TcpListener, + handle: impl Fn(&Request) -> Response + Send + Sync + 'static, +) -> std::io::Result<()> { + let handle = std::sync::Arc::new(handle); + std::thread::Builder::new() + .name("api".into()) + .spawn(move || { + for stream in listener.incoming().flatten() { + let handle = handle.clone(); + let _ = std::thread::Builder::new() + .name("api-client".into()) + .spawn(move || { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let response = match read_request(&stream) { + Ok(request) => handle(&request), + Err(response) => response, + }; + write_response(&stream, &response); + }); + } + })?; + Ok(()) +} + +/// One request to `address`, with a bearer token; the status and the body. +pub fn request( + address: SocketAddr, + token: &str, + method: &str, + path: &str, + body: Option<&serde_json::Value>, + timeout: Duration, +) -> std::io::Result<(u16, String)> { + let mut stream = TcpStream::connect_timeout(&address, Duration::from_secs(5))?; + stream.set_read_timeout(Some(timeout))?; + let body = body.map(ToString::to_string).unwrap_or_default(); + write!( + stream, + "{method} {path} HTTP/1.1\r\nHost: {address}\r\nAuthorization: Bearer {token}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + )?; + stream.flush()?; + let mut text = String::new(); + stream.read_to_string(&mut text)?; + let (head, body) = text.split_once("\r\n\r\n").unwrap_or((&text, "")); + let status = head + .split_whitespace() + .nth(1) + .and_then(|s| s.parse().ok()) + .ok_or_else(|| std::io::Error::other("not an HTTP response"))?; + Ok((status, body.to_owned())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + serve(listener, |r| { + Response::json( + 200, + &serde_json::json!({ + "method": r.method, + "path": r.path, + "since": r.param("since"), + "auth": r.header("authorization"), + "body": String::from_utf8_lossy(&r.body), + }), + ) + }) + .unwrap(); + let (status, body) = request( + address, + "t0k", + "POST", + "/api/v1/command?since=7", + Some(&serde_json::json!({ "line": "list" })), + Duration::from_secs(5), + ) + .unwrap(); + assert_eq!(status, 200); + let v: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(v["method"], "POST"); + assert_eq!(v["path"], "/api/v1/command"); + assert_eq!(v["since"], "7"); + assert_eq!(v["auth"], "Bearer t0k"); + assert_eq!(v["body"], r#"{"line":"list"}"#); + } +} diff --git a/crates/horizon-server/src/lists.rs b/crates/horizon-server/src/lists.rs index b0dc5a375..398ccfe10 100644 --- a/crates/horizon-server/src/lists.rs +++ b/crates/horizon-server/src/lists.rs @@ -1,6 +1,6 @@ -//! `op`, `deop` and `whitelist` while the server is stopped: they edit vanilla's files with the code the game -//! reads them with (horizon-server-files) and answer with vanilla's console messages. While it runs, the game -//! owns the lists, so the same commands are typed in its console instead. +//! `op`, `deop` and `whitelist`. While the server runs, the game owns the lists: the command goes to its +//! console. While it is stopped, they edit vanilla's files with the code the game reads them with +//! (horizon-server-files) and answer with vanilla's console messages. use crate::{config, supervisor}; use horizon_server_files::user_lists::{NameAndId, OpEntry, OpList, WhiteEntry, WhiteList}; @@ -9,12 +9,11 @@ use std::path::Path; pub fn run(dir: &Path, command: &str, args: &[String]) -> Result<(), String> { let config = config::Config::load(dir)?; // Holding the folder's lock keeps the server from starting halfway through. - let _lock = supervisor::lock(dir).map_err(|_| { - format!( - "the server is running: type `{command} {}` in its console", - args.join(" ") - ) - })?; + let Ok(_lock) = supervisor::lock(dir) else { + let mut words = vec![command.to_owned()]; + words.extend_from_slice(args); + return crate::cli::command(dir, &words); + }; let args: Vec<&str> = args.iter().map(String::as_str).collect(); let ops_path = dir.join("ops.json"); let white_path = dir.join("whitelist.json"); @@ -140,14 +139,15 @@ mod tests { .unwrap() .contains("Alex") ); - // A running server owns the lists. + // A running server owns the lists: without its API, nothing changes. let lock = supervisor::lock(&dir).unwrap(); - assert!( - run(&dir, "op", &s(&["Alex"])) - .unwrap_err() - .contains("running") - ); + assert!(run(&dir, "op", &s(&["Alex"])).is_err()); drop(lock); + assert!( + !std::fs::read_to_string(dir.join("ops.json")) + .unwrap() + .contains("Alex") + ); std::fs::remove_dir_all(&dir).unwrap(); } } diff --git a/crates/horizon-server/src/main.rs b/crates/horizon-server/src/main.rs index 43d08589b..94fcc5a69 100644 --- a/crates/horizon-server/src/main.rs +++ b/crates/horizon-server/src/main.rs @@ -1,8 +1,11 @@ //! `horizon-server`: host a Horizon server (docs/server.md). It keeps the server's settings in `server.toml`, //! runs the game headless as the engine, and manages the vanilla files around it. +mod api; +mod cli; mod config; mod engine; +mod http; mod init; mod lists; mod signals; @@ -20,11 +23,18 @@ Usage: horizon-server [--dir DIR] --yes) run Run the server in the foreground; type console commands, Ctrl-C stops it cleanly. It restarts the game if it crashes. + start Run the server in the background (its output goes to logs/supervisor.log) + stop, restart + status [--json] + Up or not, TPS, MSPT, players, RAM and CPU + cmd COMMAND... + Run a console command on the running server and print what it answers + logs [-f] [-n N] + The last N lines of the log (default 50); -f follows it import FILE Write server.toml from a vanilla server.properties op NAME, deop NAME whitelist on|off|list, whitelist add|remove NAME - Change who may join while the server is stopped (while it runs, type the same command - in its console) + Who may join (on the running server, or in the files when it is stopped) --dir DIR The server folder (default: the current folder) @@ -53,6 +63,12 @@ fn main() -> ExitCode { } Some("init") => init_answers(&args[1..]).and_then(|a| init::run(&dir, a)), Some("run") => supervisor::run(&dir), + Some("start") => cli::start(&dir), + Some("stop") => cli::stop(&dir, false), + Some("restart") => cli::stop(&dir, true), + Some("status") => cli::status(&dir, args[1..].iter().any(|a| a == "--json")), + Some("cmd") => cli::command(&dir, &args[1..]), + Some("logs") => logs(&dir, &args[1..]), Some("import") => import(&dir, &args[1..]), Some(command @ ("op" | "deop" | "whitelist")) => lists::run(&dir, command, &args[1..]), Some(other) => Err(format!( @@ -68,6 +84,25 @@ fn main() -> ExitCode { } } +fn logs(dir: &std::path::Path, args: &[String]) -> Result<(), String> { + let mut follow = false; + let mut count = 50; + let mut it = args.iter(); + while let Some(arg) = it.next() { + match arg.as_str() { + "-f" | "--follow" => follow = true, + "-n" => { + count = it + .next() + .and_then(|n| n.parse().ok()) + .ok_or("-n needs a number of lines")?; + } + other => return Err(format!("logs: unknown option {other}")), + } + } + cli::logs(dir, follow, count) +} + fn init_answers(args: &[String]) -> Result { let mut answers = init::Answers::default(); let mut it = args.iter(); diff --git a/crates/horizon-server/src/supervisor.rs b/crates/horizon-server/src/supervisor.rs index 376c4835e..bd35624f1 100644 --- a/crates/horizon-server/src/supervisor.rs +++ b/crates/horizon-server/src/supervisor.rs @@ -1,14 +1,21 @@ //! `horizon-server run`: writes `server.properties` from `server.toml`, runs the engine with the console on -//! stdin, stops it cleanly on Ctrl-C or SIGTERM, and restarts it when it crashes (docs/server.md, section 7). +//! stdin, serves the control API, stops the game cleanly on Ctrl-C, SIGTERM or `stop`, and restarts it when +//! it crashes (docs/server.md, sections 2 and 7). -use crate::{config, config::Config, engine, signals}; +use crate::{ + api::{self, Phase, Shared}, + config, + config::Config, + engine, signals, +}; use horizon_server_files::java_properties::now_millis; +use serde_json::json; use std::{ fs::File, io::{BufRead, BufReader, Read, Write}, path::Path, process::{Child, ExitStatus}, - sync::mpsc, + sync::{Arc, atomic::Ordering, mpsc}, thread, time::{Duration, Instant}, }; @@ -22,15 +29,18 @@ const MAX_BACKOFF: Duration = Duration::from_secs(300); /// How long `stop` may take before the game is killed. const STOP_TIMEOUT: Duration = Duration::from_secs(120); -/// A line from the supervisor, in the game's log format (`[HH:mm:ss] [thread/LEVEL]: message`, UTC). -pub fn note(level: &str, message: &str) { +/// A line from the supervisor, in the game's log format (`[HH:mm:ss] [thread/LEVEL]: message`, UTC), on +/// stdout and in the log the API serves. +fn note(shared: &Shared, level: &str, message: &str) { let secs = now_millis().div_euclid(1000).rem_euclid(86_400); - println!( + let line = format!( "[{:02}:{:02}:{:02}] [Supervisor/{level}]: {message}", secs / 3600, secs / 60 % 60, secs % 60 ); + println!("{line}"); + shared.log.lock().unwrap().push(line); } /// The folder's lock: one supervisor per server folder. Released when the process ends, even by a crash. @@ -56,6 +66,8 @@ enum Exit { /// Asked to stop, or the game ended by itself without an error (`stop` typed in its console, the EULA /// not agreed, the port taken): not restarted. Stopped, + /// Asked to restart (`horizon-server restart`). + Restart, Crashed(ExitStatus), } @@ -65,6 +77,9 @@ pub fn run(dir: &Path) -> Result<(), String> { let program = engine::program(&config, dir)?; let resources = engine::resources(&config, dir, Some(&program))?; let _lock = lock(dir)?; + let shared = Shared::new(dir); + let _endpoint = api::start(shared.clone())?; + sample(shared.clone()); let console = console_lines(); let mut crashes: Vec = Vec::new(); loop { @@ -74,6 +89,7 @@ pub fn run(dir: &Path) -> Result<(), String> { .write_properties(dir) .map_err(|e| format!("server.properties: {e}"))?; note( + &shared, "INFO", &format!( "Starting {} with the game files in {}", @@ -81,11 +97,23 @@ pub fn run(dir: &Path) -> Result<(), String> { resources.display() ), ); + shared.set_phase(Phase::Starting); let mut child = engine::spawn(&program, &resources, dir) .map_err(|e| format!("{}: {e}", program.display()))?; - let exit = supervise(&mut child, &console); + { + let mut state = shared.state.lock().unwrap(); + state.engine_pid = Some(child.id()); + state.engine_started = Some(Instant::now()); + } + let exit = supervise(&shared, &mut child, &console); + { + let mut state = shared.state.lock().unwrap(); + state.engine_pid = None; + state.engine_started = None; + } match config::sync_back(dir, &written) { Ok(keys) if !keys.is_empty() => note( + &shared, "INFO", &format!( "Copied into server.toml what the game changed: {}", @@ -93,12 +121,22 @@ pub fn run(dir: &Path) -> Result<(), String> { ), ), Ok(_) => {} - Err(e) => note("WARN", &format!("Could not update server.toml: {e}")), + Err(e) => note( + &shared, + "WARN", + &format!("Could not update server.toml: {e}"), + ), } let status = match exit { Exit::Stopped => return Ok(()), + Exit::Restart => continue, Exit::Crashed(status) => status, }; + { + let mut state = shared.state.lock().unwrap(); + state.crashes += 1; + state.last_crash = Some(status.to_string()); + } if !config.engine.restart_on_crash { return Err(format!( "the game stopped unexpectedly ({status}); engine.restart_on_crash is off" @@ -116,15 +154,17 @@ pub fn run(dir: &Path) -> Result<(), String> { } let wait = backoff(crashes.len()); note( + &shared, "WARN", &format!( "The game stopped unexpectedly ({status}); restarting in {} s", wait.as_secs() ), ); + shared.set_phase(Phase::Restarting); let until = Instant::now() + wait; while Instant::now() < until { - if signals::requests() > 0 { + if signals::requests() > 0 || shared.stop.load(Ordering::SeqCst) { return Ok(()); } thread::sleep(Duration::from_millis(100)); @@ -156,8 +196,10 @@ fn console_lines() -> mpsc::Receiver { rx } -/// Copies the game's output to ours, line by line. -fn pump(from: impl Read + Send + 'static, to_stderr: bool) { +/// Copies the game's output to ours and to the API's log, line by line; the game's `Done` line means it +/// is up. +fn pump(shared: &Arc, from: impl Read + Send + 'static, to_stderr: bool) { + let shared = shared.clone(); let _ = thread::Builder::new() .name("engine-log".into()) .spawn(move || { @@ -167,27 +209,86 @@ fn pump(from: impl Read + Send + 'static, to_stderr: bool) { eprintln!("{line}"); } else { println!("{line}"); + if line.contains("/INFO]: Done (") { + let mut state = shared.state.lock().unwrap(); + if state.phase == Phase::Starting { + state.phase = Phase::Running; + } + } } + shared.log.lock().unwrap().push(line); + } + }); +} + +/// Every second, the game's RAM, CPU and status, for `status` and (later) the panel's graphs. +fn sample(shared: Arc) { + let _ = thread::Builder::new() + .name("sampler".into()) + .spawn(move || { + let mut control: Option = None; + let mut previous: Option<(u32, Instant, f64)> = None; + loop { + thread::sleep(Duration::from_secs(1)); + let (pid, running) = { + let state = shared.state.lock().unwrap(); + (state.engine_pid, state.phase == Phase::Running) + }; + let Some(pid) = pid else { + control = None; + previous = None; + shared.state.lock().unwrap().sample = serde_json::Value::Null; + continue; + }; + let now = Instant::now(); + let cpu = horizon_bench::cpu_seconds(pid); + let cpu_percent = match (previous, cpu) { + (Some((p, at, before)), Some(cpu)) if p == pid => { + Some((cpu - before) / now.duration_since(at).as_secs_f64() * 100.0) + } + _ => None, + }; + previous = cpu.map(|c| (pid, now, c)); + if !running { + control = None; + } else if control.is_none() { + control = engine::Control::connect(&shared.dir).ok(); + } + let game = control + .as_mut() + .map(|c| c.ask("server-status")) + .transpose() + .unwrap_or_else(|_| { + control = None; + None + }); + shared.state.lock().unwrap().sample = json!({ + "rss_bytes": horizon_bench::rss_bytes(pid), + "cpu_percent": cpu_percent, + "game": game, + }); } }); } /// Runs one engine process to its end: console lines in, the log out, and a clean `stop` when asked. -fn supervise(child: &mut Child, console: &mpsc::Receiver) -> Exit { +fn supervise(shared: &Arc, child: &mut Child, console: &mpsc::Receiver) -> Exit { if let Some(out) = child.stdout.take() { - pump(out, false); + pump(shared, out, false); } if let Some(err) = child.stderr.take() { - pump(err, true); + pump(shared, err, true); } let mut stdin = child.stdin.take(); let mut stopping: Option = None; + let mut restart = false; let mut handled = signals::requests(); loop { while let Ok(line) = console.try_recv() { let command = line.trim().trim_start_matches('/'); if command == "stop" && stopping.is_none() { stopping = Some(Instant::now()); + shared.set_phase(Phase::Stopping); } if let Some(input) = stdin.as_mut() && writeln!(input, "{line}") @@ -198,21 +299,30 @@ fn supervise(child: &mut Child, console: &mpsc::Receiver) -> Exit { } } let requests = signals::requests(); - if requests > handled { - handled = requests; + let asked = requests > handled || shared.stop.swap(false, Ordering::SeqCst); + handled = requests; + if shared.restart.swap(false, Ordering::SeqCst) && stopping.is_none() { + restart = true; + note(shared, "INFO", "Restarting the server"); + stopping = Some(Instant::now()); + shared.set_phase(Phase::Restarting); + send_stop(&mut stdin); + } + if asked { + restart = false; if stopping.is_none() { - note("INFO", "Stopping the server"); + note(shared, "INFO", "Stopping the server"); stopping = Some(Instant::now()); - if let Some(input) = stdin.as_mut() { - let _ = writeln!(input, "stop").and_then(|()| input.flush()); - } + shared.set_phase(Phase::Stopping); + send_stop(&mut stdin); } else { - note("WARN", "Stopping the game at once, without saving"); + note(shared, "WARN", "Stopping the game at once, without saving"); let _ = child.kill(); } } if stopping.is_some_and(|t| t.elapsed() > STOP_TIMEOUT) { note( + shared, "WARN", &format!( "The game did not stop within {} s: killing it", @@ -220,19 +330,27 @@ fn supervise(child: &mut Child, console: &mpsc::Receiver) -> Exit { ), ); let _ = child.kill(); + stopping = Some(Instant::now()); } match child.try_wait() { + Ok(Some(_)) if restart => return Exit::Restart, Ok(Some(status)) if stopping.is_some() || status.success() => return Exit::Stopped, Ok(Some(status)) => return Exit::Crashed(status), Ok(None) => thread::sleep(Duration::from_millis(50)), Err(e) => { - note("ERROR", &format!("Lost track of the game: {e}")); + note(shared, "ERROR", &format!("Lost track of the game: {e}")); return Exit::Stopped; } } } } +fn send_stop(stdin: &mut Option) { + if let Some(input) = stdin.as_mut() { + let _ = writeln!(input, "stop").and_then(|()| input.flush()); + } +} + #[cfg(test)] mod tests { use super::*;