e8832190d1
The supervisor serves a small HTTP API on 127.0.0.1 (a token in .horizon/api.json, mode 0600, and a Host check): status, logs, console commands, stop and restart. `start` runs the server in the background and returns once the game is up; `restart` and `cmd` wait for it too. A sampler reads the game's tick rate, tick times, players and chunks every second over the existing control endpoint (`--control-socket`), and RAM and CPU from the system. The dedicated loop now records its tick times in the game clock, so `/tick query` reports real numbers on a dedicated server, as it does in single player. The changes to game/dedicated.rs are additive: the log capture for command replies, the tick counter and the control requests (game/dedicated/status.rs). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
330 lines
12 KiB
Rust
330 lines
12 KiB
Rust
//! The commands that manage a server in the background (docs/server.md, section 5): `start`, `stop`,
|
|
//! `restart`, `status`, `cmd` and `logs`. They talk to the running supervisor through the control API.
|
|
|
|
use crate::{api::Client, signals, supervisor};
|
|
use serde_json::{Value, json};
|
|
use std::{
|
|
path::Path,
|
|
process::{Command, Stdio},
|
|
time::{Duration, Instant},
|
|
};
|
|
|
|
/// How long `start` waits for the game's `Done`, and `stop` for the server to end.
|
|
const START_TIMEOUT: Duration = Duration::from_secs(300);
|
|
const STOP_TIMEOUT: Duration = Duration::from_secs(180);
|
|
|
|
fn running(dir: &Path) -> Result<Client, String> {
|
|
Client::open(dir)?.ok_or_else(|| format!("no server is running in {}", dir.display()))
|
|
}
|
|
|
|
/// `start`: `run` in the background, its output in logs/supervisor.log; returns once the game is up.
|
|
pub fn start(dir: &Path) -> Result<(), String> {
|
|
if Client::open(dir)?.is_some() {
|
|
return Err(format!("a server is already running in {}", dir.display()));
|
|
}
|
|
crate::config::Config::load(dir)?;
|
|
// Left by a supervisor that was killed: the new one writes its own.
|
|
let endpoint = dir.join(".horizon/api.json");
|
|
let _ = std::fs::remove_file(&endpoint);
|
|
let logs = dir.join("logs");
|
|
std::fs::create_dir_all(&logs).map_err(|e| e.to_string())?;
|
|
let log_path = logs.join("supervisor.log");
|
|
let log = std::fs::OpenOptions::new()
|
|
.create(true)
|
|
.append(true)
|
|
.open(&log_path)
|
|
.map_err(|e| format!("{}: {e}", log_path.display()))?;
|
|
let exe = std::env::current_exe().map_err(|e| e.to_string())?;
|
|
let mut command = Command::new(exe);
|
|
command
|
|
.arg("--dir")
|
|
.arg(dir)
|
|
.arg("run")
|
|
.stdin(Stdio::null())
|
|
.stdout(log.try_clone().map_err(|e| e.to_string())?)
|
|
.stderr(log);
|
|
// Its own process group: closing this terminal doesn't stop the server.
|
|
#[cfg(unix)]
|
|
std::os::unix::process::CommandExt::process_group(&mut command, 0);
|
|
#[cfg(windows)]
|
|
std::os::windows::process::CommandExt::creation_flags(
|
|
&mut command,
|
|
windows_sys::Win32::System::Threading::DETACHED_PROCESS
|
|
| windows_sys::Win32::System::Threading::CREATE_NEW_PROCESS_GROUP,
|
|
);
|
|
let mut child = command.spawn().map_err(|e| e.to_string())?;
|
|
let deadline = Instant::now() + START_TIMEOUT;
|
|
print!("Starting");
|
|
loop {
|
|
if let Ok(Some(status)) = child.try_wait() {
|
|
println!();
|
|
return Err(format!(
|
|
"the server stopped ({status}); see {}",
|
|
log_path.display()
|
|
));
|
|
}
|
|
// Only once the new supervisor holds the folder (its endpoint is there) may we look: probing the
|
|
// lock any earlier could take it from under it.
|
|
if endpoint.exists()
|
|
&& let Ok(Some(client)) = Client::open(dir)
|
|
&& let Ok(status) = client.call("GET", "/api/v1/status", None)
|
|
&& status["phase"] == "running"
|
|
{
|
|
println!(
|
|
"\nThe server is running (pid {}). `horizon-server status` shows how it does, \
|
|
`horizon-server stop` stops it.",
|
|
child.id()
|
|
);
|
|
return Ok(());
|
|
}
|
|
if Instant::now() > deadline {
|
|
println!();
|
|
return Err(format!(
|
|
"the game isn't up after {} s; it may still be starting: see {}",
|
|
START_TIMEOUT.as_secs(),
|
|
log_path.display()
|
|
));
|
|
}
|
|
print!(".");
|
|
let _ = std::io::Write::flush(&mut std::io::stdout());
|
|
std::thread::sleep(Duration::from_millis(500));
|
|
}
|
|
}
|
|
|
|
/// `stop` and `restart`.
|
|
pub fn stop(dir: &Path, restart: bool) -> Result<(), String> {
|
|
let client = running(dir)?;
|
|
if restart {
|
|
let before = client.call("GET", "/api/v1/status", None)?["engine_pid"].as_u64();
|
|
client.call("POST", "/api/v1/restart", None)?;
|
|
print!("Restarting");
|
|
wait_running(&client, before)?;
|
|
println!("\nThe game is running again.");
|
|
return Ok(());
|
|
}
|
|
client.call("POST", "/api/v1/stop", None)?;
|
|
let deadline = Instant::now() + STOP_TIMEOUT;
|
|
while supervisor::lock(dir).is_err() {
|
|
if Instant::now() > deadline {
|
|
return Err("the server is still stopping".into());
|
|
}
|
|
std::thread::sleep(Duration::from_millis(200));
|
|
}
|
|
println!("Stopped.");
|
|
Ok(())
|
|
}
|
|
|
|
/// Waits until the game is up: a new one when `replaced` is the engine's pid before a restart.
|
|
fn wait_running(client: &Client, replaced: Option<u64>) -> Result<(), String> {
|
|
let deadline = Instant::now() + START_TIMEOUT;
|
|
loop {
|
|
let status = client.call("GET", "/api/v1/status", None)?;
|
|
if status["phase"] == "running" && status["engine_pid"].as_u64() != replaced {
|
|
return Ok(());
|
|
}
|
|
if status["phase"] == "stopped" || status["phase"] == "stopping" {
|
|
return Err("the server is stopping".into());
|
|
}
|
|
if Instant::now() > deadline {
|
|
return Err(format!(
|
|
"the game isn't up after {} s; `horizon-server logs` shows why",
|
|
START_TIMEOUT.as_secs()
|
|
));
|
|
}
|
|
if replaced.is_some() {
|
|
print!(".");
|
|
let _ = std::io::Write::flush(&mut std::io::stdout());
|
|
}
|
|
std::thread::sleep(Duration::from_millis(500));
|
|
}
|
|
}
|
|
|
|
/// `cmd`: one console command, and what it printed.
|
|
pub fn command(dir: &Path, words: &[String]) -> Result<(), String> {
|
|
if words.is_empty() {
|
|
return Err("cmd needs a console command, for example: horizon-server cmd list".into());
|
|
}
|
|
let client = running(dir)?;
|
|
// A game that is starting or restarting takes the command once it is up.
|
|
wait_running(&client, None)?;
|
|
let reply = client.call(
|
|
"POST",
|
|
"/api/v1/command",
|
|
Some(&json!({ "line": words.join(" ") })),
|
|
)?;
|
|
for line in reply["output"].as_array().into_iter().flatten() {
|
|
println!("{}", line.as_str().unwrap_or_default());
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// `status [--json]`.
|
|
pub fn status(dir: &Path, as_json: bool) -> Result<(), String> {
|
|
let Some(client) = Client::open(dir)? else {
|
|
if as_json {
|
|
println!("{}", json!({ "phase": "stopped" }));
|
|
} else {
|
|
println!("Stopped.");
|
|
}
|
|
return Ok(());
|
|
};
|
|
let status = client.call("GET", "/api/v1/status", None)?;
|
|
if as_json {
|
|
println!("{status}");
|
|
} else {
|
|
print!("{}", describe(&status));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// The status for people: three short lines.
|
|
fn describe(status: &Value) -> String {
|
|
let mut out = String::new();
|
|
let phase = status["phase"].as_str().unwrap_or("?");
|
|
let uptime = status["engine_uptime_seconds"]
|
|
.as_u64()
|
|
.or(status["uptime_seconds"].as_u64())
|
|
.map_or_else(String::new, |s| format!(", up {}", duration(s)));
|
|
let crashes = match status["crashes"].as_u64() {
|
|
Some(0) | None => String::new(),
|
|
Some(n) => format!(
|
|
", {n} crash{} (last: {})",
|
|
if n == 1 { "" } else { "es" },
|
|
status["last_crash"].as_str().unwrap_or("?")
|
|
),
|
|
};
|
|
out += &format!("{}{uptime}{crashes}\n", capitalised(phase));
|
|
let sample = &status["sample"];
|
|
let game = &sample["game"];
|
|
if let Some(tps) = game["tps"].as_f64() {
|
|
let mspt = &game["mspt"];
|
|
out += &format!(
|
|
"TPS {tps:.1}, MSPT {:.1} ms average, {:.1} ms P95{}\n",
|
|
mspt["average"].as_f64().unwrap_or(0.0),
|
|
mspt["p95"].as_f64().unwrap_or(0.0),
|
|
if game["paused"] == true {
|
|
" (paused: nobody online)"
|
|
} else {
|
|
""
|
|
}
|
|
);
|
|
let names: Vec<&str> = game["players"]
|
|
.as_array()
|
|
.into_iter()
|
|
.flatten()
|
|
.filter_map(|p| p["name"].as_str())
|
|
.collect();
|
|
out += &format!(
|
|
"{} of {} players{}{}, {} chunks loaded\n",
|
|
names.len(),
|
|
game["max_players"].as_u64().unwrap_or(0),
|
|
if names.is_empty() { "" } else { ": " },
|
|
names.join(", "),
|
|
game["chunks"].as_u64().unwrap_or(0)
|
|
);
|
|
}
|
|
let ram = sample["rss_bytes"]
|
|
.as_f64()
|
|
.map(|b| format!("RAM {:.2} GB", b / 1e9));
|
|
let cpu = sample["cpu_percent"]
|
|
.as_f64()
|
|
.map(|c| format!("CPU {c:.0} % of a core"));
|
|
let usage: Vec<String> = ram.into_iter().chain(cpu).collect();
|
|
if !usage.is_empty() {
|
|
out += &format!("{}\n", usage.join(", "));
|
|
}
|
|
out
|
|
}
|
|
|
|
fn capitalised(s: &str) -> String {
|
|
let mut c = s.chars();
|
|
c.next()
|
|
.map(|f| f.to_uppercase().chain(c).collect())
|
|
.unwrap_or_default()
|
|
}
|
|
|
|
fn duration(secs: u64) -> String {
|
|
match secs {
|
|
0..60 => format!("{secs} s"),
|
|
60..3600 => format!("{} min", secs / 60),
|
|
3600..86_400 => format!("{} h {:02} min", secs / 3600, secs / 60 % 60),
|
|
_ => format!("{} d {} h", secs / 86_400, secs / 3600 % 24),
|
|
}
|
|
}
|
|
|
|
/// `logs [-f] [-n N]`: the last lines, from the running server or else from logs/latest.log.
|
|
pub fn logs(dir: &Path, follow: bool, count: usize) -> Result<(), String> {
|
|
let Some(client) = Client::open(dir)? else {
|
|
if follow {
|
|
return Err("no server is running to follow".into());
|
|
}
|
|
let path = dir.join("logs/latest.log");
|
|
let text =
|
|
std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?;
|
|
let lines: Vec<&str> = text.lines().collect();
|
|
for line in &lines[lines.len().saturating_sub(count)..] {
|
|
println!("{line}");
|
|
}
|
|
return Ok(());
|
|
};
|
|
let reply = client.call("GET", "/api/v1/logs?since=0", None)?;
|
|
let lines = reply["lines"].as_array().cloned().unwrap_or_default();
|
|
for line in &lines[lines.len().saturating_sub(count)..] {
|
|
println!("{}", line.as_str().unwrap_or_default());
|
|
}
|
|
if !follow {
|
|
return Ok(());
|
|
}
|
|
signals::install();
|
|
let mut next = reply["next"].as_u64().unwrap_or(0);
|
|
while signals::requests() == 0 {
|
|
std::thread::sleep(Duration::from_millis(500));
|
|
let Ok(reply) = client.call("GET", &format!("/api/v1/logs?since={next}"), None) else {
|
|
break;
|
|
};
|
|
for line in reply["lines"].as_array().into_iter().flatten() {
|
|
println!("{}", line.as_str().unwrap_or_default());
|
|
}
|
|
next = reply["next"].as_u64().unwrap_or(next);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn status_reads_well() {
|
|
let status = json!({
|
|
"phase": "running",
|
|
"engine_uptime_seconds": 7380,
|
|
"crashes": 1,
|
|
"last_crash": "signal: 9 (SIGKILL)",
|
|
"sample": {
|
|
"rss_bytes": 1_420_000_000u64,
|
|
"cpu_percent": 12.4,
|
|
"game": {
|
|
"tps": 20.0,
|
|
"mspt": { "average": 3.12, "p95": 7.9 },
|
|
"players": [{ "name": "Alex" }, { "name": "Steve" }],
|
|
"max_players": 20,
|
|
"chunks": 1234,
|
|
"paused": false
|
|
}
|
|
}
|
|
});
|
|
assert_eq!(
|
|
describe(&status),
|
|
"Running, up 2 h 03 min, 1 crash (last: signal: 9 (SIGKILL))\n\
|
|
TPS 20.0, MSPT 3.1 ms average, 7.9 ms P95\n\
|
|
2 of 20 players: Alex, Steve, 1234 chunks loaded\n\
|
|
RAM 1.42 GB, CPU 12 % of a core\n"
|
|
);
|
|
assert_eq!(
|
|
describe(&json!({ "phase": "starting", "uptime_seconds": 4 })),
|
|
"Starting, up 4 s\n"
|
|
);
|
|
}
|
|
}
|