horizon-server: start, stop, status, cmd and logs through a local control API

The supervisor serves a small HTTP API on 127.0.0.1 (a token in .horizon/api.json, mode 0600, and a
Host check): status, logs, console commands, stop and restart. `start` runs the server in the
background and returns once the game is up; `restart` and `cmd` wait for it too. A sampler reads the
game's tick rate, tick times, players and chunks every second over the existing control endpoint
(`--control-socket`), and RAM and CPU from the system.

The dedicated loop now records its tick times in the game clock, so `/tick query` reports real
numbers on a dedicated server, as it does in single player. The changes to game/dedicated.rs are
additive: the log capture for command replies, the tick counter and the control requests
(game/dedicated/status.rs).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-16 20:06:24 +02:00
parent 9d21209ae8
commit e8832190d1
11 changed files with 1309 additions and 40 deletions
Generated
+2
View File
@@ -1532,9 +1532,11 @@ dependencies = [
name = "horizon-server"
version = "0.1.0"
dependencies = [
"horizon-bench",
"horizon-server-files",
"libc",
"serde",
"serde_json",
"toml",
"toml_edit",
"windows-sys 0.61.2",
@@ -29,6 +29,8 @@ use std::{
time::{Duration, Instant},
};
mod status;
/// Double.toString for the magnitudes positions have (no exponent form
/// between 1e-3 and 1e7).
pub(super) fn java_double(v: f64) -> String {
@@ -64,6 +66,11 @@ pub(super) fn log(level: &str, message: &str) {
_ => ::log::Level::Info,
};
horizon_client::logging::write_only(file_level, message);
CAPTURE.with_borrow_mut(|lines| {
if let Some(lines) = lines {
lines.push(message.to_owned());
}
});
let secs = now_millis().div_euclid(1000).rem_euclid(86_400);
println!(
"[{:02}:{:02}:{:02}] [Server thread/{level}]: {message}",
@@ -73,6 +80,18 @@ pub(super) fn log(level: &str, message: &str) {
);
}
thread_local! {
/// The lines logged while a control request runs a console command.
static CAPTURE: std::cell::RefCell<Option<Vec<String>>> = const { std::cell::RefCell::new(None) };
}
/// Runs `f` and returns the lines it logged.
fn capture_log(f: impl FnOnce()) -> Vec<String> {
CAPTURE.set(Some(Vec::new()));
f();
CAPTURE.take().unwrap_or_default()
}
/// Main.main up to the server start: arguments, server.properties (stored
/// back at once), eula.txt. False when the process should stop here.
pub(super) fn preflight(options: &mut Options) -> Result<bool> {
@@ -289,6 +308,9 @@ struct ServerState {
stop: bool,
autosave: bool,
empty_ticks: i32,
/// When the last passes of the loop ended, for the tick rate horizon-server shows.
ticks: std::collections::VecDeque<Instant>,
paused: bool,
}
impl Game<'_> {
@@ -469,14 +491,18 @@ impl Game<'_> {
stop: false,
autosave: true,
empty_ticks: 0,
ticks: Default::default(),
paused: false,
};
let pause_after = props.pause_when_empty_seconds * 20;
let idle_timeout = props.player_idle_timeout;
let mut next = Instant::now();
while !state.stop {
let tick_start = Instant::now();
while let Ok(line) = state.console.try_recv() {
self.console_command(&mut state, &line);
}
self.poll_server_control(&mut state);
let players = self.lan.as_ref().map_or(0, |h| h.hub.guests.len());
if pause_after > 0 {
if players == 0 {
@@ -486,6 +512,7 @@ impl Game<'_> {
}
}
let paused = pause_after > 0 && state.empty_ticks >= pause_after;
state.paused = paused;
if paused && state.empty_ticks == pause_after {
log(
"INFO",
@@ -511,6 +538,9 @@ impl Game<'_> {
if idle_timeout > 0 {
self.kick_idle(idle_timeout);
}
// MinecraftServer.tickServer records each tick's time, for /tick query (and the status).
self.clock.record(tick_start.elapsed());
state.ticked(Instant::now());
let rate = self.clock.rate().max(1.0);
next += Duration::from_secs_f64(1.0 / f64::from(rate));
let now = Instant::now();
@@ -0,0 +1,100 @@
//! What `horizon-server` asks the dedicated server on its control endpoint (`--control-socket`, docs/server.md
//! section 2): `server-status` answers with the tick rate, the tick times `/tick query` reports, the players
//! and the loaded chunks; `console <line>` runs a console command and answers with the lines it logged.
use super::{ServerState, capture_log};
use crate::game::Game;
use serde_json::{Value, json};
use std::{
sync::atomic::Ordering,
time::{Duration, Instant},
};
/// The window the tick rate is measured over.
const TPS_WINDOW: Duration = Duration::from_secs(5);
impl ServerState {
/// One pass of the server loop ended.
pub(super) fn ticked(&mut self, at: Instant) {
self.ticks.push_back(at);
while self
.ticks
.front()
.is_some_and(|t| at.duration_since(*t) > TPS_WINDOW)
{
self.ticks.pop_front();
}
}
/// Ticks per second over the last few seconds.
fn tps(&self) -> f64 {
match (self.ticks.front(), self.ticks.back()) {
(Some(first), Some(last)) if self.ticks.len() > 1 => {
(self.ticks.len() - 1) as f64 / last.duration_since(*first).as_secs_f64().max(1e-9)
}
_ => 0.0,
}
}
}
impl Game<'_> {
/// Answers the requests waiting on the control endpoint.
pub(super) fn poll_server_control(&mut self, state: &mut ServerState) {
while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) {
let reply = match request.line.trim().split_once(' ') {
_ if request.line.trim() == "server-status" => self.server_status(state),
Some(("console", line)) => {
// Command feedback is logged as system chat; the reply carries the text alone.
let output: Vec<String> = capture_log(|| self.console_command(state, line))
.into_iter()
.map(|l| {
l.strip_prefix("System chat: ")
.map_or(l.clone(), str::to_owned)
})
.collect();
json!({ "output": output })
}
_ => json!({ "error": format!("unknown request {:?}", request.line) }),
};
let _ = request.reply.send(reply.to_string());
}
}
fn server_status(&self, state: &ServerState) -> Value {
let millis = |nanos: u64| nanos as f64 / 1e6;
let samples = self.clock.sorted_samples();
// The percentiles `/tick query` prints (TickCommand.percentileToString).
let at = |q: f64| {
let i = ((samples.len() as f64 * q) as usize).min(samples.len().saturating_sub(1));
samples.get(i).copied().map_or(0.0, millis)
};
let (players, max_players) = self.lan.as_ref().map_or((Vec::new(), 0), |h| {
let players = h
.hub
.guests
.iter()
.map(|g| {
json!({
"name": g.name,
"uuid": horizon_protocol::crypto::uuid_string(g.uuid),
"address": g.address.ip().to_string(),
"ping_ms": g.latency.load(Ordering::Relaxed),
})
})
.collect();
(players, h.hub.lan.shared.max_players())
});
json!({
"tps": state.tps(),
"target_tps": self.clock.rate(),
"mspt": {
"average": millis(self.clock.average_nanos()),
"p50": at(0.5),
"p95": at(0.95),
"p99": at(0.99),
},
"players": players,
"max_players": max_players,
"chunks": self.world.chunks.len(),
"paused": state.paused,
})
}
}
+3
View File
@@ -8,7 +8,10 @@ description = "Host a Horizon server: server.toml, the CLI and the supervisor th
[dependencies]
horizon-server-files = { path = "../horizon-server-files" }
# The game process's RAM and CPU (the same probes the benchmark reports use).
horizon-bench = { path = "../horizon-bench" }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
toml = "1.1"
# Edits server.toml in place, keeping the owner's comments and order.
toml_edit = "0.25"
+378
View File
@@ -0,0 +1,378 @@
//! The control API (docs/server.md, section 8): HTTP and JSON on 127.0.0.1, with a random token in
//! `.horizon/api.json` that only the server's own user can read. The CLI manages a running server through
//! it; the web panel and the MCP server will too.
//!
//! - `GET /api/v1/status`: the supervisor's state, the game's status (TPS, MSPT, players), RAM and CPU.
//! - `GET /api/v1/logs?since=N`: the log lines after the Nth.
//! - `POST /api/v1/command {"line": "..."}`: a console command; the lines it logged.
//! - `POST /api/v1/stop`, `POST /api/v1/restart`.
use crate::{
engine,
http::{self, Request, Response},
};
use serde_json::{Value, json};
use std::{
collections::VecDeque,
io::Write,
net::{SocketAddr, TcpListener},
path::{Path, PathBuf},
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
},
time::{Duration, Instant},
};
const ENDPOINT: &str = ".horizon/api.json";
/// The log lines kept for `logs` and the panel's console.
const LOG_LINES: usize = 2000;
/// What the supervisor is doing.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Phase {
Starting,
Running,
Stopping,
/// Stopping the game to start it again (`restart`), or waiting to after a crash.
Restarting,
}
impl Phase {
fn name(self) -> &'static str {
match self {
Self::Starting => "starting",
Self::Running => "running",
Self::Stopping => "stopping",
Self::Restarting => "restarting",
}
}
}
pub struct State {
pub phase: Phase,
pub engine_pid: Option<u32>,
pub engine_started: Option<Instant>,
pub crashes: u32,
pub last_crash: Option<String>,
/// The latest sample (`sample`): the game's status, RAM and CPU.
pub sample: Value,
}
/// The log, numbered from the first line since the supervisor started.
#[derive(Default)]
pub struct Log {
lines: VecDeque<String>,
first: u64,
}
impl Log {
pub fn push(&mut self, line: String) {
if self.lines.len() == LOG_LINES {
self.lines.pop_front();
self.first += 1;
}
self.lines.push_back(line);
}
/// The lines after the `since`th, and the number to ask from next.
pub fn since(&self, since: u64) -> (u64, Vec<String>) {
let skip = since.saturating_sub(self.first) as usize;
let next = self.first + self.lines.len() as u64;
(next, self.lines.iter().skip(skip).cloned().collect())
}
}
/// What the supervisor and the API threads share.
pub struct Shared {
pub dir: PathBuf,
pub state: Mutex<State>,
pub log: Mutex<Log>,
pub stop: AtomicBool,
pub restart: AtomicBool,
pub started: Instant,
}
impl Shared {
pub fn new(dir: &Path) -> Arc<Self> {
Arc::new(Self {
dir: dir.to_owned(),
state: Mutex::new(State {
phase: Phase::Starting,
engine_pid: None,
engine_started: None,
crashes: 0,
last_crash: None,
sample: Value::Null,
}),
log: Mutex::new(Log::default()),
stop: AtomicBool::new(false),
restart: AtomicBool::new(false),
started: Instant::now(),
})
}
pub fn set_phase(&self, phase: Phase) {
self.state.lock().unwrap().phase = phase;
}
}
fn status(shared: &Shared) -> Value {
let state = shared.state.lock().unwrap();
json!({
"phase": state.phase.name(),
"version": env!("CARGO_PKG_VERSION"),
"pid": std::process::id(),
"uptime_seconds": shared.started.elapsed().as_secs(),
"engine_pid": state.engine_pid,
"engine_uptime_seconds": state.engine_started.map(|t| t.elapsed().as_secs()),
"crashes": state.crashes,
"last_crash": state.last_crash,
"sample": state.sample,
})
}
/// A random token: two 64-bit hashes keyed by the OS's random hasher seeds, the time and the process.
fn random_token() -> String {
use std::hash::{BuildHasher, Hasher};
(0..2u64)
.map(|i| {
let mut h = std::collections::hash_map::RandomState::new().build_hasher();
h.write_u64(i);
h.write_u128(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos()),
);
h.write_u32(std::process::id());
format!("{:016x}", h.finish())
})
.collect()
}
fn same_token(a: &str, b: &str) -> bool {
a.len() == b.len() && a.bytes().zip(b.bytes()).fold(0u8, |d, (x, y)| d | (x ^ y)) == 0
}
/// Writes a file only its owner can read.
pub fn write_private(path: &Path, text: &str) -> std::io::Result<()> {
let mut options = std::fs::OpenOptions::new();
options.write(true).create(true).truncate(true);
#[cfg(unix)]
std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
options.open(path)?.write_all(text.as_bytes())
}
/// Removes the endpoint file when the supervisor ends.
pub struct Endpoint(PathBuf);
impl Drop for Endpoint {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0);
}
}
/// Starts the API on a free port of 127.0.0.1 and writes where it is and its token.
pub fn start(shared: Arc<Shared>) -> Result<Endpoint, String> {
let listener = TcpListener::bind("127.0.0.1:0").map_err(|e| format!("control API: {e}"))?;
let address = listener.local_addr().map_err(|e| e.to_string())?;
let token = random_token();
let path = shared.dir.join(ENDPOINT);
write_private(
&path,
&json!({ "address": address.to_string(), "token": token, "pid": std::process::id() })
.to_string(),
)
.map_err(|e| format!("{}: {e}", path.display()))?;
let hosts = [address.to_string(), format!("localhost:{}", address.port())];
http::serve(listener, move |request| {
// A page in a browser can't send the token, and a rebound DNS name can't pass as this address.
let authorised = request
.header("authorization")
.and_then(|h| h.strip_prefix("Bearer "))
.is_some_and(|t| same_token(t.trim(), &token));
if !authorised {
return Response::error(401, "missing or wrong token (see .horizon/api.json)");
}
if !request
.header("host")
.is_some_and(|h| hosts.iter().any(|a| a == h))
{
return Response::error(403, "wrong Host");
}
route(&shared, request)
})
.map_err(|e| e.to_string())?;
Ok(Endpoint(path))
}
fn route(shared: &Shared, request: &Request) -> Response {
match (request.method.as_str(), request.path.as_str()) {
("GET", "/api/v1/status") => Response::json(200, &status(shared)),
("GET", "/api/v1/logs") => {
let since = request
.param("since")
.and_then(|s| s.parse().ok())
.unwrap_or(0);
let (next, lines) = shared.log.lock().unwrap().since(since);
Response::json(200, &json!({ "next": next, "lines": lines }))
}
("POST", "/api/v1/command") => {
let line = serde_json::from_slice::<Value>(&request.body)
.ok()
.and_then(|v| v["line"].as_str().map(str::to_owned));
let Some(line) = line.filter(|l| !l.trim().is_empty() && !l.contains('\n')) else {
return Response::error(400, "the body must be {\"line\": \"<one command>\"}");
};
if shared.state.lock().unwrap().phase != Phase::Running {
return Response::error(409, "the game isn't running right now");
}
match engine::Control::connect(&shared.dir)
.and_then(|mut c| c.ask(&format!("console {}", line.trim_start_matches('/'))))
{
Ok(reply) => Response::json(200, &reply),
Err(e) => Response::error(409, &format!("the game didn't answer: {e}")),
}
}
("POST", "/api/v1/stop") => {
shared.stop.store(true, Ordering::SeqCst);
Response::json(200, &json!({ "ok": true }))
}
("POST", "/api/v1/restart") => {
shared.restart.store(true, Ordering::SeqCst);
Response::json(200, &json!({ "ok": true }))
}
(
_,
"/api/v1/status" | "/api/v1/logs" | "/api/v1/command" | "/api/v1/stop"
| "/api/v1/restart",
) => Response::error(405, "wrong method"),
_ => Response::error(404, "no such endpoint"),
}
}
/// The API of the server running in a folder, for the CLI.
pub struct Client {
address: SocketAddr,
token: String,
}
impl Client {
/// The running server's API; `None` when no server runs in `dir`.
pub fn open(dir: &Path) -> Result<Option<Self>, String> {
if crate::supervisor::lock(dir).is_ok() {
return Ok(None);
}
// The supervisor writes the endpoint just after taking the lock: give it a moment.
let path = dir.join(ENDPOINT);
let deadline = Instant::now() + Duration::from_secs(5);
let text = loop {
match std::fs::read_to_string(&path) {
Ok(text) => break text,
Err(_) if Instant::now() < deadline => {
std::thread::sleep(Duration::from_millis(100))
}
Err(e) => return Err(format!("{}: {e}", path.display())),
}
};
let v: Value =
serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))?;
let address = v["address"]
.as_str()
.and_then(|a| a.parse().ok())
.ok_or_else(|| format!("{}: no address", path.display()))?;
let token = v["token"].as_str().unwrap_or_default().to_owned();
Ok(Some(Self { address, token }))
}
pub fn call(&self, method: &str, path: &str, body: Option<&Value>) -> Result<Value, String> {
let (status, text) = http::request(
self.address,
&self.token,
method,
path,
body,
Duration::from_secs(60),
)
.map_err(|e| format!("the server didn't answer: {e}"))?;
let value: Value = serde_json::from_str(&text).unwrap_or(Value::Null);
if status == 200 {
Ok(value)
} else {
Err(value["error"].as_str().unwrap_or(&text).to_owned())
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn log_keeps_the_last_lines_numbered() {
let mut log = Log::default();
for i in 0..LOG_LINES + 10 {
log.push(i.to_string());
}
let (next, lines) = log.since(0);
assert_eq!(next, (LOG_LINES + 10) as u64);
assert_eq!(lines.len(), LOG_LINES);
assert_eq!(lines[0], "10");
let (next2, tail) = log.since(next - 2);
assert_eq!(
(next2, tail),
(
next,
vec![(LOG_LINES + 8).to_string(), (LOG_LINES + 9).to_string()]
)
);
}
#[test]
fn the_api_wants_its_token() {
let dir = std::env::temp_dir().join(format!("horizon-server-api-{}", std::process::id()));
std::fs::create_dir_all(dir.join(".horizon")).unwrap();
let shared = Shared::new(&dir);
let _endpoint = start(shared.clone()).unwrap();
let v: Value =
serde_json::from_str(&std::fs::read_to_string(dir.join(ENDPOINT)).unwrap()).unwrap();
let address: SocketAddr = v["address"].as_str().unwrap().parse().unwrap();
let token = v["token"].as_str().unwrap();
let get = |token: &str| {
http::request(
address,
token,
"GET",
"/api/v1/status",
None,
Duration::from_secs(5),
)
.unwrap()
};
assert_eq!(get("nope").0, 401);
let (status, body) = get(token);
assert_eq!(status, 200);
let status: Value = serde_json::from_str(&body).unwrap();
assert_eq!(status["phase"], "starting");
let (code, _) = http::request(
address,
token,
"POST",
"/api/v1/stop",
None,
Duration::from_secs(5),
)
.unwrap();
assert_eq!(code, 200);
assert!(shared.stop.load(Ordering::SeqCst));
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = std::fs::metadata(dir.join(ENDPOINT))
.unwrap()
.permissions()
.mode();
assert_eq!(mode & 0o777, 0o600);
}
drop(_endpoint);
assert!(!dir.join(ENDPOINT).exists());
std::fs::remove_dir_all(&dir).unwrap();
}
}
+329
View File
@@ -0,0 +1,329 @@
//! The commands that manage a server in the background (docs/server.md, section 5): `start`, `stop`,
//! `restart`, `status`, `cmd` and `logs`. They talk to the running supervisor through the control API.
use crate::{api::Client, signals, supervisor};
use serde_json::{Value, json};
use std::{
path::Path,
process::{Command, Stdio},
time::{Duration, Instant},
};
/// How long `start` waits for the game's `Done`, and `stop` for the server to end.
const START_TIMEOUT: Duration = Duration::from_secs(300);
const STOP_TIMEOUT: Duration = Duration::from_secs(180);
fn running(dir: &Path) -> Result<Client, String> {
Client::open(dir)?.ok_or_else(|| format!("no server is running in {}", dir.display()))
}
/// `start`: `run` in the background, its output in logs/supervisor.log; returns once the game is up.
pub fn start(dir: &Path) -> Result<(), String> {
if Client::open(dir)?.is_some() {
return Err(format!("a server is already running in {}", dir.display()));
}
crate::config::Config::load(dir)?;
// Left by a supervisor that was killed: the new one writes its own.
let endpoint = dir.join(".horizon/api.json");
let _ = std::fs::remove_file(&endpoint);
let logs = dir.join("logs");
std::fs::create_dir_all(&logs).map_err(|e| e.to_string())?;
let log_path = logs.join("supervisor.log");
let log = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&log_path)
.map_err(|e| format!("{}: {e}", log_path.display()))?;
let exe = std::env::current_exe().map_err(|e| e.to_string())?;
let mut command = Command::new(exe);
command
.arg("--dir")
.arg(dir)
.arg("run")
.stdin(Stdio::null())
.stdout(log.try_clone().map_err(|e| e.to_string())?)
.stderr(log);
// Its own process group: closing this terminal doesn't stop the server.
#[cfg(unix)]
std::os::unix::process::CommandExt::process_group(&mut command, 0);
#[cfg(windows)]
std::os::windows::process::CommandExt::creation_flags(
&mut command,
windows_sys::Win32::System::Threading::DETACHED_PROCESS
| windows_sys::Win32::System::Threading::CREATE_NEW_PROCESS_GROUP,
);
let mut child = command.spawn().map_err(|e| e.to_string())?;
let deadline = Instant::now() + START_TIMEOUT;
print!("Starting");
loop {
if let Ok(Some(status)) = child.try_wait() {
println!();
return Err(format!(
"the server stopped ({status}); see {}",
log_path.display()
));
}
// Only once the new supervisor holds the folder (its endpoint is there) may we look: probing the
// lock any earlier could take it from under it.
if endpoint.exists()
&& let Ok(Some(client)) = Client::open(dir)
&& let Ok(status) = client.call("GET", "/api/v1/status", None)
&& status["phase"] == "running"
{
println!(
"\nThe server is running (pid {}). `horizon-server status` shows how it does, \
`horizon-server stop` stops it.",
child.id()
);
return Ok(());
}
if Instant::now() > deadline {
println!();
return Err(format!(
"the game isn't up after {} s; it may still be starting: see {}",
START_TIMEOUT.as_secs(),
log_path.display()
));
}
print!(".");
let _ = std::io::Write::flush(&mut std::io::stdout());
std::thread::sleep(Duration::from_millis(500));
}
}
/// `stop` and `restart`.
pub fn stop(dir: &Path, restart: bool) -> Result<(), String> {
let client = running(dir)?;
if restart {
let before = client.call("GET", "/api/v1/status", None)?["engine_pid"].as_u64();
client.call("POST", "/api/v1/restart", None)?;
print!("Restarting");
wait_running(&client, before)?;
println!("\nThe game is running again.");
return Ok(());
}
client.call("POST", "/api/v1/stop", None)?;
let deadline = Instant::now() + STOP_TIMEOUT;
while supervisor::lock(dir).is_err() {
if Instant::now() > deadline {
return Err("the server is still stopping".into());
}
std::thread::sleep(Duration::from_millis(200));
}
println!("Stopped.");
Ok(())
}
/// Waits until the game is up: a new one when `replaced` is the engine's pid before a restart.
fn wait_running(client: &Client, replaced: Option<u64>) -> Result<(), String> {
let deadline = Instant::now() + START_TIMEOUT;
loop {
let status = client.call("GET", "/api/v1/status", None)?;
if status["phase"] == "running" && status["engine_pid"].as_u64() != replaced {
return Ok(());
}
if status["phase"] == "stopped" || status["phase"] == "stopping" {
return Err("the server is stopping".into());
}
if Instant::now() > deadline {
return Err(format!(
"the game isn't up after {} s; `horizon-server logs` shows why",
START_TIMEOUT.as_secs()
));
}
if replaced.is_some() {
print!(".");
let _ = std::io::Write::flush(&mut std::io::stdout());
}
std::thread::sleep(Duration::from_millis(500));
}
}
/// `cmd`: one console command, and what it printed.
pub fn command(dir: &Path, words: &[String]) -> Result<(), String> {
if words.is_empty() {
return Err("cmd needs a console command, for example: horizon-server cmd list".into());
}
let client = running(dir)?;
// A game that is starting or restarting takes the command once it is up.
wait_running(&client, None)?;
let reply = client.call(
"POST",
"/api/v1/command",
Some(&json!({ "line": words.join(" ") })),
)?;
for line in reply["output"].as_array().into_iter().flatten() {
println!("{}", line.as_str().unwrap_or_default());
}
Ok(())
}
/// `status [--json]`.
pub fn status(dir: &Path, as_json: bool) -> Result<(), String> {
let Some(client) = Client::open(dir)? else {
if as_json {
println!("{}", json!({ "phase": "stopped" }));
} else {
println!("Stopped.");
}
return Ok(());
};
let status = client.call("GET", "/api/v1/status", None)?;
if as_json {
println!("{status}");
} else {
print!("{}", describe(&status));
}
Ok(())
}
/// The status for people: three short lines.
fn describe(status: &Value) -> String {
let mut out = String::new();
let phase = status["phase"].as_str().unwrap_or("?");
let uptime = status["engine_uptime_seconds"]
.as_u64()
.or(status["uptime_seconds"].as_u64())
.map_or_else(String::new, |s| format!(", up {}", duration(s)));
let crashes = match status["crashes"].as_u64() {
Some(0) | None => String::new(),
Some(n) => format!(
", {n} crash{} (last: {})",
if n == 1 { "" } else { "es" },
status["last_crash"].as_str().unwrap_or("?")
),
};
out += &format!("{}{uptime}{crashes}\n", capitalised(phase));
let sample = &status["sample"];
let game = &sample["game"];
if let Some(tps) = game["tps"].as_f64() {
let mspt = &game["mspt"];
out += &format!(
"TPS {tps:.1}, MSPT {:.1} ms average, {:.1} ms P95{}\n",
mspt["average"].as_f64().unwrap_or(0.0),
mspt["p95"].as_f64().unwrap_or(0.0),
if game["paused"] == true {
" (paused: nobody online)"
} else {
""
}
);
let names: Vec<&str> = game["players"]
.as_array()
.into_iter()
.flatten()
.filter_map(|p| p["name"].as_str())
.collect();
out += &format!(
"{} of {} players{}{}, {} chunks loaded\n",
names.len(),
game["max_players"].as_u64().unwrap_or(0),
if names.is_empty() { "" } else { ": " },
names.join(", "),
game["chunks"].as_u64().unwrap_or(0)
);
}
let ram = sample["rss_bytes"]
.as_f64()
.map(|b| format!("RAM {:.2} GB", b / 1e9));
let cpu = sample["cpu_percent"]
.as_f64()
.map(|c| format!("CPU {c:.0} % of a core"));
let usage: Vec<String> = ram.into_iter().chain(cpu).collect();
if !usage.is_empty() {
out += &format!("{}\n", usage.join(", "));
}
out
}
fn capitalised(s: &str) -> String {
let mut c = s.chars();
c.next()
.map(|f| f.to_uppercase().chain(c).collect())
.unwrap_or_default()
}
fn duration(secs: u64) -> String {
match secs {
0..60 => format!("{secs} s"),
60..3600 => format!("{} min", secs / 60),
3600..86_400 => format!("{} h {:02} min", secs / 3600, secs / 60 % 60),
_ => format!("{} d {} h", secs / 86_400, secs / 3600 % 24),
}
}
/// `logs [-f] [-n N]`: the last lines, from the running server or else from logs/latest.log.
pub fn logs(dir: &Path, follow: bool, count: usize) -> Result<(), String> {
let Some(client) = Client::open(dir)? else {
if follow {
return Err("no server is running to follow".into());
}
let path = dir.join("logs/latest.log");
let text =
std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?;
let lines: Vec<&str> = text.lines().collect();
for line in &lines[lines.len().saturating_sub(count)..] {
println!("{line}");
}
return Ok(());
};
let reply = client.call("GET", "/api/v1/logs?since=0", None)?;
let lines = reply["lines"].as_array().cloned().unwrap_or_default();
for line in &lines[lines.len().saturating_sub(count)..] {
println!("{}", line.as_str().unwrap_or_default());
}
if !follow {
return Ok(());
}
signals::install();
let mut next = reply["next"].as_u64().unwrap_or(0);
while signals::requests() == 0 {
std::thread::sleep(Duration::from_millis(500));
let Ok(reply) = client.call("GET", &format!("/api/v1/logs?since={next}"), None) else {
break;
};
for line in reply["lines"].as_array().into_iter().flatten() {
println!("{}", line.as_str().unwrap_or_default());
}
next = reply["next"].as_u64().unwrap_or(next);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn status_reads_well() {
let status = json!({
"phase": "running",
"engine_uptime_seconds": 7380,
"crashes": 1,
"last_crash": "signal: 9 (SIGKILL)",
"sample": {
"rss_bytes": 1_420_000_000u64,
"cpu_percent": 12.4,
"game": {
"tps": 20.0,
"mspt": { "average": 3.12, "p95": 7.9 },
"players": [{ "name": "Alex" }, { "name": "Steve" }],
"max_players": 20,
"chunks": 1234,
"paused": false
}
}
});
assert_eq!(
describe(&status),
"Running, up 2 h 03 min, 1 crash (last: signal: 9 (SIGKILL))\n\
TPS 20.0, MSPT 3.1 ms average, 7.9 ms P95\n\
2 of 20 players: Alex, Steve, 1234 chunks loaded\n\
RAM 1.42 GB, CPU 12 % of a core\n"
);
assert_eq!(
describe(&json!({ "phase": "starting", "uptime_seconds": 4 })),
"Starting, up 4 s\n"
);
}
}
+56 -1
View File
@@ -4,8 +4,11 @@
use crate::config::Config;
use std::{
io::{BufRead, BufReader, Write},
net::{Ipv4Addr, TcpStream},
path::{Path, PathBuf},
process::{Child, Command, Stdio},
time::Duration,
};
/// The game's program names, most specific first: a headless engine once there is one, else the game.
@@ -83,12 +86,15 @@ pub fn resources(config: &Config, dir: &Path, program: Option<&Path>) -> Result<
})
}
/// Starts the engine in the server folder, with its console and log piped to the supervisor.
/// Starts the engine in the server folder, with its console and log piped to the supervisor and its status
/// on the control endpoint.
pub fn spawn(program: &Path, resources: &Path, dir: &Path) -> std::io::Result<Child> {
let mut command = Command::new(program);
command
.arg("--resources")
.arg(resources)
.arg("--control-socket")
.arg(control_path(dir))
.args(["--dedicated", "--nogui"])
.current_dir(dir)
.env("HORIZON_NO_AUDIO", "1")
@@ -106,6 +112,55 @@ pub fn spawn(program: &Path, resources: &Path, dir: &Path) -> std::io::Result<Ch
command.spawn()
}
/// The game's control endpoint in a server folder (`--control-socket`): TCP on 127.0.0.1 with a token, on every OS.
pub fn control_path(dir: &Path) -> PathBuf {
dir.join(".horizon/engine.json")
}
/// A connection to the game's control endpoint: one JSON line asked, one answered.
pub struct Control {
writer: TcpStream,
reader: BufReader<TcpStream>,
}
impl Control {
pub fn connect(dir: &Path) -> std::io::Result<Self> {
let text = std::fs::read_to_string(control_path(dir))?;
let bad = || std::io::Error::new(std::io::ErrorKind::InvalidData, "not a control endpoint");
let v: serde_json::Value = serde_json::from_str(&text).map_err(|_| bad())?;
let port = v["port"]
.as_u64()
.and_then(|p| u16::try_from(p).ok())
.ok_or_else(bad)?;
let token = v["token"].as_str().ok_or_else(bad)?;
let writer = TcpStream::connect((Ipv4Addr::LOCALHOST, port))?;
writer.set_read_timeout(Some(Duration::from_secs(30)))?;
let mut control = Self {
reader: BufReader::new(writer.try_clone()?),
writer,
};
let reply = control.ask(&format!("auth {token}"))?;
if reply["ok"] != true {
return Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"the game refused the token",
));
}
Ok(control)
}
pub fn ask(&mut self, line: &str) -> std::io::Result<serde_json::Value> {
writeln!(self.writer, "{line}")?;
self.writer.flush()?;
let mut reply = String::new();
if self.reader.read_line(&mut reply)? == 0 {
return Err(std::io::ErrorKind::UnexpectedEof.into());
}
serde_json::from_str(&reply)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
}
}
#[cfg(test)]
mod tests {
use super::*;
+219
View File
@@ -0,0 +1,219 @@
//! Just enough HTTP/1.1 for the control API on 127.0.0.1 (docs/server.md, section 8): one request per
//! connection, JSON bodies with a Content-Length. No async runtime and no dependency: a thread per
//! connection is plenty for a CLI, a panel tab and an MCP client.
use std::{
io::{BufRead, BufReader, Read, Write},
net::{SocketAddr, TcpListener, TcpStream},
time::Duration,
};
/// The largest request body the API takes (a console command, a settings change).
const MAX_BODY: usize = 1 << 20;
pub struct Request {
pub method: String,
pub path: String,
pub query: String,
pub headers: Vec<(String, String)>,
pub body: Vec<u8>,
}
impl Request {
pub fn header(&self, name: &str) -> Option<&str> {
self.headers
.iter()
.find(|(k, _)| k.eq_ignore_ascii_case(name))
.map(|(_, v)| v.as_str())
}
/// One `key=value` from the query string (no percent-decoding: the API's values are numbers).
pub fn param(&self, key: &str) -> Option<&str> {
self.query
.split('&')
.filter_map(|p| p.split_once('='))
.find(|(k, _)| *k == key)
.map(|(_, v)| v)
}
}
pub struct Response {
pub status: u16,
pub body: String,
}
impl Response {
pub fn json(status: u16, value: &serde_json::Value) -> Self {
Self {
status,
body: value.to_string(),
}
}
pub fn error(status: u16, message: &str) -> Self {
Self::json(status, &serde_json::json!({ "error": message }))
}
}
fn reason(status: u16) -> &'static str {
match status {
200 => "OK",
400 => "Bad Request",
401 => "Unauthorized",
403 => "Forbidden",
404 => "Not Found",
405 => "Method Not Allowed",
409 => "Conflict",
413 => "Payload Too Large",
_ => "Error",
}
}
fn read_request(stream: &TcpStream) -> Result<Request, Response> {
let bad = |m: &str| Response::error(400, m);
let mut reader = BufReader::new(stream);
let mut line = String::new();
reader.read_line(&mut line).map_err(|_| bad("no request"))?;
let mut parts = line.split_whitespace();
let (Some(method), Some(target)) = (parts.next(), parts.next()) else {
return Err(bad("malformed request line"));
};
let (path, query) = target.split_once('?').unwrap_or((target, ""));
let mut headers = Vec::new();
loop {
let mut line = String::new();
if reader.read_line(&mut line).map_err(|_| bad("headers"))? == 0 || line.trim().is_empty() {
break;
}
if headers.len() > 64 {
return Err(bad("too many headers"));
}
if let Some((k, v)) = line.split_once(':') {
headers.push((k.trim().to_owned(), v.trim().to_owned()));
}
}
let length = headers
.iter()
.find(|(k, _)| k.eq_ignore_ascii_case("content-length"))
.map_or(Ok(0), |(_, v)| v.parse::<usize>())
.map_err(|_| bad("bad Content-Length"))?;
if length > MAX_BODY {
return Err(Response::error(413, "the body is too large"));
}
let mut body = vec![0; length];
reader
.read_exact(&mut body)
.map_err(|_| bad("the body is shorter than its Content-Length"))?;
Ok(Request {
method: method.to_owned(),
path: path.to_owned(),
query: query.to_owned(),
headers,
body,
})
}
fn write_response(mut stream: &TcpStream, response: &Response) {
let _ = write!(
stream,
"HTTP/1.1 {} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n{}",
response.status,
reason(response.status),
response.body.len(),
response.body
);
let _ = stream.flush();
}
/// Serves `handle` on `listener` until the process ends, a thread per connection.
pub fn serve(
listener: TcpListener,
handle: impl Fn(&Request) -> Response + Send + Sync + 'static,
) -> std::io::Result<()> {
let handle = std::sync::Arc::new(handle);
std::thread::Builder::new()
.name("api".into())
.spawn(move || {
for stream in listener.incoming().flatten() {
let handle = handle.clone();
let _ = std::thread::Builder::new()
.name("api-client".into())
.spawn(move || {
let _ = stream.set_read_timeout(Some(Duration::from_secs(10)));
let response = match read_request(&stream) {
Ok(request) => handle(&request),
Err(response) => response,
};
write_response(&stream, &response);
});
}
})?;
Ok(())
}
/// One request to `address`, with a bearer token; the status and the body.
pub fn request(
address: SocketAddr,
token: &str,
method: &str,
path: &str,
body: Option<&serde_json::Value>,
timeout: Duration,
) -> std::io::Result<(u16, String)> {
let mut stream = TcpStream::connect_timeout(&address, Duration::from_secs(5))?;
stream.set_read_timeout(Some(timeout))?;
let body = body.map(ToString::to_string).unwrap_or_default();
write!(
stream,
"{method} {path} HTTP/1.1\r\nHost: {address}\r\nAuthorization: Bearer {token}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
)?;
stream.flush()?;
let mut text = String::new();
stream.read_to_string(&mut text)?;
let (head, body) = text.split_once("\r\n\r\n").unwrap_or((&text, ""));
let status = head
.split_whitespace()
.nth(1)
.and_then(|s| s.parse().ok())
.ok_or_else(|| std::io::Error::other("not an HTTP response"))?;
Ok((status, body.to_owned()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn round_trip() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
serve(listener, |r| {
Response::json(
200,
&serde_json::json!({
"method": r.method,
"path": r.path,
"since": r.param("since"),
"auth": r.header("authorization"),
"body": String::from_utf8_lossy(&r.body),
}),
)
})
.unwrap();
let (status, body) = request(
address,
"t0k",
"POST",
"/api/v1/command?since=7",
Some(&serde_json::json!({ "line": "list" })),
Duration::from_secs(5),
)
.unwrap();
assert_eq!(status, 200);
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(v["method"], "POST");
assert_eq!(v["path"], "/api/v1/command");
assert_eq!(v["since"], "7");
assert_eq!(v["auth"], "Bearer t0k");
assert_eq!(v["body"], r#"{"line":"list"}"#);
}
}
+15 -15
View File
@@ -1,6 +1,6 @@
//! `op`, `deop` and `whitelist` while the server is stopped: they edit vanilla's files with the code the game
//! reads them with (horizon-server-files) and answer with vanilla's console messages. While it runs, the game
//! owns the lists, so the same commands are typed in its console instead.
//! `op`, `deop` and `whitelist`. While the server runs, the game owns the lists: the command goes to its
//! console. While it is stopped, they edit vanilla's files with the code the game reads them with
//! (horizon-server-files) and answer with vanilla's console messages.
use crate::{config, supervisor};
use horizon_server_files::user_lists::{NameAndId, OpEntry, OpList, WhiteEntry, WhiteList};
@@ -9,12 +9,11 @@ use std::path::Path;
pub fn run(dir: &Path, command: &str, args: &[String]) -> Result<(), String> {
let config = config::Config::load(dir)?;
// Holding the folder's lock keeps the server from starting halfway through.
let _lock = supervisor::lock(dir).map_err(|_| {
format!(
"the server is running: type `{command} {}` in its console",
args.join(" ")
)
})?;
let Ok(_lock) = supervisor::lock(dir) else {
let mut words = vec![command.to_owned()];
words.extend_from_slice(args);
return crate::cli::command(dir, &words);
};
let args: Vec<&str> = args.iter().map(String::as_str).collect();
let ops_path = dir.join("ops.json");
let white_path = dir.join("whitelist.json");
@@ -140,14 +139,15 @@ mod tests {
.unwrap()
.contains("Alex")
);
// A running server owns the lists.
// A running server owns the lists: without its API, nothing changes.
let lock = supervisor::lock(&dir).unwrap();
assert!(
run(&dir, "op", &s(&["Alex"]))
.unwrap_err()
.contains("running")
);
assert!(run(&dir, "op", &s(&["Alex"])).is_err());
drop(lock);
assert!(
!std::fs::read_to_string(dir.join("ops.json"))
.unwrap()
.contains("Alex")
);
std::fs::remove_dir_all(&dir).unwrap();
}
}
+37 -2
View File
@@ -1,8 +1,11 @@
//! `horizon-server`: host a Horizon server (docs/server.md). It keeps the server's settings in `server.toml`,
//! runs the game headless as the engine, and manages the vanilla files around it.
mod api;
mod cli;
mod config;
mod engine;
mod http;
mod init;
mod lists;
mod signals;
@@ -20,11 +23,18 @@ Usage: horizon-server [--dir DIR] <command>
--yes)
run Run the server in the foreground; type console commands, Ctrl-C stops it cleanly. It
restarts the game if it crashes.
start Run the server in the background (its output goes to logs/supervisor.log)
stop, restart
status [--json]
Up or not, TPS, MSPT, players, RAM and CPU
cmd COMMAND...
Run a console command on the running server and print what it answers
logs [-f] [-n N]
The last N lines of the log (default 50); -f follows it
import FILE Write server.toml from a vanilla server.properties
op NAME, deop NAME
whitelist on|off|list, whitelist add|remove NAME
Change who may join while the server is stopped (while it runs, type the same command
in its console)
Who may join (on the running server, or in the files when it is stopped)
--dir DIR The server folder (default: the current folder)
@@ -53,6 +63,12 @@ fn main() -> ExitCode {
}
Some("init") => init_answers(&args[1..]).and_then(|a| init::run(&dir, a)),
Some("run") => supervisor::run(&dir),
Some("start") => cli::start(&dir),
Some("stop") => cli::stop(&dir, false),
Some("restart") => cli::stop(&dir, true),
Some("status") => cli::status(&dir, args[1..].iter().any(|a| a == "--json")),
Some("cmd") => cli::command(&dir, &args[1..]),
Some("logs") => logs(&dir, &args[1..]),
Some("import") => import(&dir, &args[1..]),
Some(command @ ("op" | "deop" | "whitelist")) => lists::run(&dir, command, &args[1..]),
Some(other) => Err(format!(
@@ -68,6 +84,25 @@ fn main() -> ExitCode {
}
}
fn logs(dir: &std::path::Path, args: &[String]) -> Result<(), String> {
let mut follow = false;
let mut count = 50;
let mut it = args.iter();
while let Some(arg) = it.next() {
match arg.as_str() {
"-f" | "--follow" => follow = true,
"-n" => {
count = it
.next()
.and_then(|n| n.parse().ok())
.ok_or("-n needs a number of lines")?;
}
other => return Err(format!("logs: unknown option {other}")),
}
}
cli::logs(dir, follow, count)
}
fn init_answers(args: &[String]) -> Result<init::Answers, String> {
let mut answers = init::Answers::default();
let mut it = args.iter();
+140 -22
View File
@@ -1,14 +1,21 @@
//! `horizon-server run`: writes `server.properties` from `server.toml`, runs the engine with the console on
//! stdin, stops it cleanly on Ctrl-C or SIGTERM, and restarts it when it crashes (docs/server.md, section 7).
//! stdin, serves the control API, stops the game cleanly on Ctrl-C, SIGTERM or `stop`, and restarts it when
//! it crashes (docs/server.md, sections 2 and 7).
use crate::{config, config::Config, engine, signals};
use crate::{
api::{self, Phase, Shared},
config,
config::Config,
engine, signals,
};
use horizon_server_files::java_properties::now_millis;
use serde_json::json;
use std::{
fs::File,
io::{BufRead, BufReader, Read, Write},
path::Path,
process::{Child, ExitStatus},
sync::mpsc,
sync::{Arc, atomic::Ordering, mpsc},
thread,
time::{Duration, Instant},
};
@@ -22,15 +29,18 @@ const MAX_BACKOFF: Duration = Duration::from_secs(300);
/// How long `stop` may take before the game is killed.
const STOP_TIMEOUT: Duration = Duration::from_secs(120);
/// A line from the supervisor, in the game's log format (`[HH:mm:ss] [thread/LEVEL]: message`, UTC).
pub fn note(level: &str, message: &str) {
/// A line from the supervisor, in the game's log format (`[HH:mm:ss] [thread/LEVEL]: message`, UTC), on
/// stdout and in the log the API serves.
fn note(shared: &Shared, level: &str, message: &str) {
let secs = now_millis().div_euclid(1000).rem_euclid(86_400);
println!(
let line = format!(
"[{:02}:{:02}:{:02}] [Supervisor/{level}]: {message}",
secs / 3600,
secs / 60 % 60,
secs % 60
);
println!("{line}");
shared.log.lock().unwrap().push(line);
}
/// The folder's lock: one supervisor per server folder. Released when the process ends, even by a crash.
@@ -56,6 +66,8 @@ enum Exit {
/// Asked to stop, or the game ended by itself without an error (`stop` typed in its console, the EULA
/// not agreed, the port taken): not restarted.
Stopped,
/// Asked to restart (`horizon-server restart`).
Restart,
Crashed(ExitStatus),
}
@@ -65,6 +77,9 @@ pub fn run(dir: &Path) -> Result<(), String> {
let program = engine::program(&config, dir)?;
let resources = engine::resources(&config, dir, Some(&program))?;
let _lock = lock(dir)?;
let shared = Shared::new(dir);
let _endpoint = api::start(shared.clone())?;
sample(shared.clone());
let console = console_lines();
let mut crashes: Vec<Instant> = Vec::new();
loop {
@@ -74,6 +89,7 @@ pub fn run(dir: &Path) -> Result<(), String> {
.write_properties(dir)
.map_err(|e| format!("server.properties: {e}"))?;
note(
&shared,
"INFO",
&format!(
"Starting {} with the game files in {}",
@@ -81,11 +97,23 @@ pub fn run(dir: &Path) -> Result<(), String> {
resources.display()
),
);
shared.set_phase(Phase::Starting);
let mut child = engine::spawn(&program, &resources, dir)
.map_err(|e| format!("{}: {e}", program.display()))?;
let exit = supervise(&mut child, &console);
{
let mut state = shared.state.lock().unwrap();
state.engine_pid = Some(child.id());
state.engine_started = Some(Instant::now());
}
let exit = supervise(&shared, &mut child, &console);
{
let mut state = shared.state.lock().unwrap();
state.engine_pid = None;
state.engine_started = None;
}
match config::sync_back(dir, &written) {
Ok(keys) if !keys.is_empty() => note(
&shared,
"INFO",
&format!(
"Copied into server.toml what the game changed: {}",
@@ -93,12 +121,22 @@ pub fn run(dir: &Path) -> Result<(), String> {
),
),
Ok(_) => {}
Err(e) => note("WARN", &format!("Could not update server.toml: {e}")),
Err(e) => note(
&shared,
"WARN",
&format!("Could not update server.toml: {e}"),
),
}
let status = match exit {
Exit::Stopped => return Ok(()),
Exit::Restart => continue,
Exit::Crashed(status) => status,
};
{
let mut state = shared.state.lock().unwrap();
state.crashes += 1;
state.last_crash = Some(status.to_string());
}
if !config.engine.restart_on_crash {
return Err(format!(
"the game stopped unexpectedly ({status}); engine.restart_on_crash is off"
@@ -116,15 +154,17 @@ pub fn run(dir: &Path) -> Result<(), String> {
}
let wait = backoff(crashes.len());
note(
&shared,
"WARN",
&format!(
"The game stopped unexpectedly ({status}); restarting in {} s",
wait.as_secs()
),
);
shared.set_phase(Phase::Restarting);
let until = Instant::now() + wait;
while Instant::now() < until {
if signals::requests() > 0 {
if signals::requests() > 0 || shared.stop.load(Ordering::SeqCst) {
return Ok(());
}
thread::sleep(Duration::from_millis(100));
@@ -156,8 +196,10 @@ fn console_lines() -> mpsc::Receiver<String> {
rx
}
/// Copies the game's output to ours, line by line.
fn pump(from: impl Read + Send + 'static, to_stderr: bool) {
/// Copies the game's output to ours and to the API's log, line by line; the game's `Done` line means it
/// is up.
fn pump(shared: &Arc<Shared>, from: impl Read + Send + 'static, to_stderr: bool) {
let shared = shared.clone();
let _ = thread::Builder::new()
.name("engine-log".into())
.spawn(move || {
@@ -167,27 +209,86 @@ fn pump(from: impl Read + Send + 'static, to_stderr: bool) {
eprintln!("{line}");
} else {
println!("{line}");
if line.contains("/INFO]: Done (") {
let mut state = shared.state.lock().unwrap();
if state.phase == Phase::Starting {
state.phase = Phase::Running;
}
}
}
shared.log.lock().unwrap().push(line);
}
});
}
/// Every second, the game's RAM, CPU and status, for `status` and (later) the panel's graphs.
fn sample(shared: Arc<Shared>) {
let _ = thread::Builder::new()
.name("sampler".into())
.spawn(move || {
let mut control: Option<engine::Control> = None;
let mut previous: Option<(u32, Instant, f64)> = None;
loop {
thread::sleep(Duration::from_secs(1));
let (pid, running) = {
let state = shared.state.lock().unwrap();
(state.engine_pid, state.phase == Phase::Running)
};
let Some(pid) = pid else {
control = None;
previous = None;
shared.state.lock().unwrap().sample = serde_json::Value::Null;
continue;
};
let now = Instant::now();
let cpu = horizon_bench::cpu_seconds(pid);
let cpu_percent = match (previous, cpu) {
(Some((p, at, before)), Some(cpu)) if p == pid => {
Some((cpu - before) / now.duration_since(at).as_secs_f64() * 100.0)
}
_ => None,
};
previous = cpu.map(|c| (pid, now, c));
if !running {
control = None;
} else if control.is_none() {
control = engine::Control::connect(&shared.dir).ok();
}
let game = control
.as_mut()
.map(|c| c.ask("server-status"))
.transpose()
.unwrap_or_else(|_| {
control = None;
None
});
shared.state.lock().unwrap().sample = json!({
"rss_bytes": horizon_bench::rss_bytes(pid),
"cpu_percent": cpu_percent,
"game": game,
});
}
});
}
/// Runs one engine process to its end: console lines in, the log out, and a clean `stop` when asked.
fn supervise(child: &mut Child, console: &mpsc::Receiver<String>) -> Exit {
fn supervise(shared: &Arc<Shared>, child: &mut Child, console: &mpsc::Receiver<String>) -> Exit {
if let Some(out) = child.stdout.take() {
pump(out, false);
pump(shared, out, false);
}
if let Some(err) = child.stderr.take() {
pump(err, true);
pump(shared, err, true);
}
let mut stdin = child.stdin.take();
let mut stopping: Option<Instant> = None;
let mut restart = false;
let mut handled = signals::requests();
loop {
while let Ok(line) = console.try_recv() {
let command = line.trim().trim_start_matches('/');
if command == "stop" && stopping.is_none() {
stopping = Some(Instant::now());
shared.set_phase(Phase::Stopping);
}
if let Some(input) = stdin.as_mut()
&& writeln!(input, "{line}")
@@ -198,21 +299,30 @@ fn supervise(child: &mut Child, console: &mpsc::Receiver<String>) -> Exit {
}
}
let requests = signals::requests();
if requests > handled {
handled = requests;
let asked = requests > handled || shared.stop.swap(false, Ordering::SeqCst);
handled = requests;
if shared.restart.swap(false, Ordering::SeqCst) && stopping.is_none() {
restart = true;
note(shared, "INFO", "Restarting the server");
stopping = Some(Instant::now());
shared.set_phase(Phase::Restarting);
send_stop(&mut stdin);
}
if asked {
restart = false;
if stopping.is_none() {
note("INFO", "Stopping the server");
note(shared, "INFO", "Stopping the server");
stopping = Some(Instant::now());
if let Some(input) = stdin.as_mut() {
let _ = writeln!(input, "stop").and_then(|()| input.flush());
}
shared.set_phase(Phase::Stopping);
send_stop(&mut stdin);
} else {
note("WARN", "Stopping the game at once, without saving");
note(shared, "WARN", "Stopping the game at once, without saving");
let _ = child.kill();
}
}
if stopping.is_some_and(|t| t.elapsed() > STOP_TIMEOUT) {
note(
shared,
"WARN",
&format!(
"The game did not stop within {} s: killing it",
@@ -220,19 +330,27 @@ fn supervise(child: &mut Child, console: &mpsc::Receiver<String>) -> Exit {
),
);
let _ = child.kill();
stopping = Some(Instant::now());
}
match child.try_wait() {
Ok(Some(_)) if restart => return Exit::Restart,
Ok(Some(status)) if stopping.is_some() || status.success() => return Exit::Stopped,
Ok(Some(status)) => return Exit::Crashed(status),
Ok(None) => thread::sleep(Duration::from_millis(50)),
Err(e) => {
note("ERROR", &format!("Lost track of the game: {e}"));
note(shared, "ERROR", &format!("Lost track of the game: {e}"));
return Exit::Stopped;
}
}
}
}
fn send_stop(stdin: &mut Option<std::process::ChildStdin>) {
if let Some(input) = stdin.as_mut() {
let _ = writeln!(input, "stop").and_then(|()| input.flush());
}
}
#[cfg(test)]
mod tests {
use super::*;