horizon-server: start, stop, status, cmd and logs through a local control API
The supervisor serves a small HTTP API on 127.0.0.1 (a token in .horizon/api.json, mode 0600, and a Host check): status, logs, console commands, stop and restart. `start` runs the server in the background and returns once the game is up; `restart` and `cmd` wait for it too. A sampler reads the game's tick rate, tick times, players and chunks every second over the existing control endpoint (`--control-socket`), and RAM and CPU from the system. The dedicated loop now records its tick times in the game clock, so `/tick query` reports real numbers on a dedicated server, as it does in single player. The changes to game/dedicated.rs are additive: the log capture for command replies, the tick counter and the control requests (game/dedicated/status.rs). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Generated
+2
@@ -1532,9 +1532,11 @@ dependencies = [
|
||||
name = "horizon-server"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"horizon-bench",
|
||||
"horizon-server-files",
|
||||
"libc",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"toml",
|
||||
"toml_edit",
|
||||
"windows-sys 0.61.2",
|
||||
|
||||
@@ -29,6 +29,8 @@ use std::{
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
mod status;
|
||||
|
||||
/// Double.toString for the magnitudes positions have (no exponent form
|
||||
/// between 1e-3 and 1e7).
|
||||
pub(super) fn java_double(v: f64) -> String {
|
||||
@@ -64,6 +66,11 @@ pub(super) fn log(level: &str, message: &str) {
|
||||
_ => ::log::Level::Info,
|
||||
};
|
||||
horizon_client::logging::write_only(file_level, message);
|
||||
CAPTURE.with_borrow_mut(|lines| {
|
||||
if let Some(lines) = lines {
|
||||
lines.push(message.to_owned());
|
||||
}
|
||||
});
|
||||
let secs = now_millis().div_euclid(1000).rem_euclid(86_400);
|
||||
println!(
|
||||
"[{:02}:{:02}:{:02}] [Server thread/{level}]: {message}",
|
||||
@@ -73,6 +80,18 @@ pub(super) fn log(level: &str, message: &str) {
|
||||
);
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
/// The lines logged while a control request runs a console command.
|
||||
static CAPTURE: std::cell::RefCell<Option<Vec<String>>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
/// Runs `f` and returns the lines it logged.
|
||||
fn capture_log(f: impl FnOnce()) -> Vec<String> {
|
||||
CAPTURE.set(Some(Vec::new()));
|
||||
f();
|
||||
CAPTURE.take().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Main.main up to the server start: arguments, server.properties (stored
|
||||
/// back at once), eula.txt. False when the process should stop here.
|
||||
pub(super) fn preflight(options: &mut Options) -> Result<bool> {
|
||||
@@ -289,6 +308,9 @@ struct ServerState {
|
||||
stop: bool,
|
||||
autosave: bool,
|
||||
empty_ticks: i32,
|
||||
/// When the last passes of the loop ended, for the tick rate horizon-server shows.
|
||||
ticks: std::collections::VecDeque<Instant>,
|
||||
paused: bool,
|
||||
}
|
||||
|
||||
impl Game<'_> {
|
||||
@@ -469,14 +491,18 @@ impl Game<'_> {
|
||||
stop: false,
|
||||
autosave: true,
|
||||
empty_ticks: 0,
|
||||
ticks: Default::default(),
|
||||
paused: false,
|
||||
};
|
||||
let pause_after = props.pause_when_empty_seconds * 20;
|
||||
let idle_timeout = props.player_idle_timeout;
|
||||
let mut next = Instant::now();
|
||||
while !state.stop {
|
||||
let tick_start = Instant::now();
|
||||
while let Ok(line) = state.console.try_recv() {
|
||||
self.console_command(&mut state, &line);
|
||||
}
|
||||
self.poll_server_control(&mut state);
|
||||
let players = self.lan.as_ref().map_or(0, |h| h.hub.guests.len());
|
||||
if pause_after > 0 {
|
||||
if players == 0 {
|
||||
@@ -486,6 +512,7 @@ impl Game<'_> {
|
||||
}
|
||||
}
|
||||
let paused = pause_after > 0 && state.empty_ticks >= pause_after;
|
||||
state.paused = paused;
|
||||
if paused && state.empty_ticks == pause_after {
|
||||
log(
|
||||
"INFO",
|
||||
@@ -511,6 +538,9 @@ impl Game<'_> {
|
||||
if idle_timeout > 0 {
|
||||
self.kick_idle(idle_timeout);
|
||||
}
|
||||
// MinecraftServer.tickServer records each tick's time, for /tick query (and the status).
|
||||
self.clock.record(tick_start.elapsed());
|
||||
state.ticked(Instant::now());
|
||||
let rate = self.clock.rate().max(1.0);
|
||||
next += Duration::from_secs_f64(1.0 / f64::from(rate));
|
||||
let now = Instant::now();
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
//! What `horizon-server` asks the dedicated server on its control endpoint (`--control-socket`, docs/server.md
|
||||
//! section 2): `server-status` answers with the tick rate, the tick times `/tick query` reports, the players
|
||||
//! and the loaded chunks; `console <line>` runs a console command and answers with the lines it logged.
|
||||
use super::{ServerState, capture_log};
|
||||
use crate::game::Game;
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
sync::atomic::Ordering,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
/// The window the tick rate is measured over.
|
||||
const TPS_WINDOW: Duration = Duration::from_secs(5);
|
||||
|
||||
impl ServerState {
|
||||
/// One pass of the server loop ended.
|
||||
pub(super) fn ticked(&mut self, at: Instant) {
|
||||
self.ticks.push_back(at);
|
||||
while self
|
||||
.ticks
|
||||
.front()
|
||||
.is_some_and(|t| at.duration_since(*t) > TPS_WINDOW)
|
||||
{
|
||||
self.ticks.pop_front();
|
||||
}
|
||||
}
|
||||
/// Ticks per second over the last few seconds.
|
||||
fn tps(&self) -> f64 {
|
||||
match (self.ticks.front(), self.ticks.back()) {
|
||||
(Some(first), Some(last)) if self.ticks.len() > 1 => {
|
||||
(self.ticks.len() - 1) as f64 / last.duration_since(*first).as_secs_f64().max(1e-9)
|
||||
}
|
||||
_ => 0.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Game<'_> {
|
||||
/// Answers the requests waiting on the control endpoint.
|
||||
pub(super) fn poll_server_control(&mut self, state: &mut ServerState) {
|
||||
while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) {
|
||||
let reply = match request.line.trim().split_once(' ') {
|
||||
_ if request.line.trim() == "server-status" => self.server_status(state),
|
||||
Some(("console", line)) => {
|
||||
// Command feedback is logged as system chat; the reply carries the text alone.
|
||||
let output: Vec<String> = capture_log(|| self.console_command(state, line))
|
||||
.into_iter()
|
||||
.map(|l| {
|
||||
l.strip_prefix("System chat: ")
|
||||
.map_or(l.clone(), str::to_owned)
|
||||
})
|
||||
.collect();
|
||||
json!({ "output": output })
|
||||
}
|
||||
_ => json!({ "error": format!("unknown request {:?}", request.line) }),
|
||||
};
|
||||
let _ = request.reply.send(reply.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
fn server_status(&self, state: &ServerState) -> Value {
|
||||
let millis = |nanos: u64| nanos as f64 / 1e6;
|
||||
let samples = self.clock.sorted_samples();
|
||||
// The percentiles `/tick query` prints (TickCommand.percentileToString).
|
||||
let at = |q: f64| {
|
||||
let i = ((samples.len() as f64 * q) as usize).min(samples.len().saturating_sub(1));
|
||||
samples.get(i).copied().map_or(0.0, millis)
|
||||
};
|
||||
let (players, max_players) = self.lan.as_ref().map_or((Vec::new(), 0), |h| {
|
||||
let players = h
|
||||
.hub
|
||||
.guests
|
||||
.iter()
|
||||
.map(|g| {
|
||||
json!({
|
||||
"name": g.name,
|
||||
"uuid": horizon_protocol::crypto::uuid_string(g.uuid),
|
||||
"address": g.address.ip().to_string(),
|
||||
"ping_ms": g.latency.load(Ordering::Relaxed),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
(players, h.hub.lan.shared.max_players())
|
||||
});
|
||||
json!({
|
||||
"tps": state.tps(),
|
||||
"target_tps": self.clock.rate(),
|
||||
"mspt": {
|
||||
"average": millis(self.clock.average_nanos()),
|
||||
"p50": at(0.5),
|
||||
"p95": at(0.95),
|
||||
"p99": at(0.99),
|
||||
},
|
||||
"players": players,
|
||||
"max_players": max_players,
|
||||
"chunks": self.world.chunks.len(),
|
||||
"paused": state.paused,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,10 @@ description = "Host a Horizon server: server.toml, the CLI and the supervisor th
|
||||
|
||||
[dependencies]
|
||||
horizon-server-files = { path = "../horizon-server-files" }
|
||||
# The game process's RAM and CPU (the same probes the benchmark reports use).
|
||||
horizon-bench = { path = "../horizon-bench" }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
toml = "1.1"
|
||||
# Edits server.toml in place, keeping the owner's comments and order.
|
||||
toml_edit = "0.25"
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
//! The control API (docs/server.md, section 8): HTTP and JSON on 127.0.0.1, with a random token in
|
||||
//! `.horizon/api.json` that only the server's own user can read. The CLI manages a running server through
|
||||
//! it; the web panel and the MCP server will too.
|
||||
//!
|
||||
//! - `GET /api/v1/status`: the supervisor's state, the game's status (TPS, MSPT, players), RAM and CPU.
|
||||
//! - `GET /api/v1/logs?since=N`: the log lines after the Nth.
|
||||
//! - `POST /api/v1/command {"line": "..."}`: a console command; the lines it logged.
|
||||
//! - `POST /api/v1/stop`, `POST /api/v1/restart`.
|
||||
|
||||
use crate::{
|
||||
engine,
|
||||
http::{self, Request, Response},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
io::Write,
|
||||
net::{SocketAddr, TcpListener},
|
||||
path::{Path, PathBuf},
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
const ENDPOINT: &str = ".horizon/api.json";
|
||||
/// The log lines kept for `logs` and the panel's console.
|
||||
const LOG_LINES: usize = 2000;
|
||||
|
||||
/// What the supervisor is doing.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Phase {
|
||||
Starting,
|
||||
Running,
|
||||
Stopping,
|
||||
/// Stopping the game to start it again (`restart`), or waiting to after a crash.
|
||||
Restarting,
|
||||
}
|
||||
|
||||
impl Phase {
|
||||
fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Starting => "starting",
|
||||
Self::Running => "running",
|
||||
Self::Stopping => "stopping",
|
||||
Self::Restarting => "restarting",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct State {
|
||||
pub phase: Phase,
|
||||
pub engine_pid: Option<u32>,
|
||||
pub engine_started: Option<Instant>,
|
||||
pub crashes: u32,
|
||||
pub last_crash: Option<String>,
|
||||
/// The latest sample (`sample`): the game's status, RAM and CPU.
|
||||
pub sample: Value,
|
||||
}
|
||||
|
||||
/// The log, numbered from the first line since the supervisor started.
|
||||
#[derive(Default)]
|
||||
pub struct Log {
|
||||
lines: VecDeque<String>,
|
||||
first: u64,
|
||||
}
|
||||
|
||||
impl Log {
|
||||
pub fn push(&mut self, line: String) {
|
||||
if self.lines.len() == LOG_LINES {
|
||||
self.lines.pop_front();
|
||||
self.first += 1;
|
||||
}
|
||||
self.lines.push_back(line);
|
||||
}
|
||||
/// The lines after the `since`th, and the number to ask from next.
|
||||
pub fn since(&self, since: u64) -> (u64, Vec<String>) {
|
||||
let skip = since.saturating_sub(self.first) as usize;
|
||||
let next = self.first + self.lines.len() as u64;
|
||||
(next, self.lines.iter().skip(skip).cloned().collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// What the supervisor and the API threads share.
|
||||
pub struct Shared {
|
||||
pub dir: PathBuf,
|
||||
pub state: Mutex<State>,
|
||||
pub log: Mutex<Log>,
|
||||
pub stop: AtomicBool,
|
||||
pub restart: AtomicBool,
|
||||
pub started: Instant,
|
||||
}
|
||||
|
||||
impl Shared {
|
||||
pub fn new(dir: &Path) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
dir: dir.to_owned(),
|
||||
state: Mutex::new(State {
|
||||
phase: Phase::Starting,
|
||||
engine_pid: None,
|
||||
engine_started: None,
|
||||
crashes: 0,
|
||||
last_crash: None,
|
||||
sample: Value::Null,
|
||||
}),
|
||||
log: Mutex::new(Log::default()),
|
||||
stop: AtomicBool::new(false),
|
||||
restart: AtomicBool::new(false),
|
||||
started: Instant::now(),
|
||||
})
|
||||
}
|
||||
pub fn set_phase(&self, phase: Phase) {
|
||||
self.state.lock().unwrap().phase = phase;
|
||||
}
|
||||
}
|
||||
|
||||
fn status(shared: &Shared) -> Value {
|
||||
let state = shared.state.lock().unwrap();
|
||||
json!({
|
||||
"phase": state.phase.name(),
|
||||
"version": env!("CARGO_PKG_VERSION"),
|
||||
"pid": std::process::id(),
|
||||
"uptime_seconds": shared.started.elapsed().as_secs(),
|
||||
"engine_pid": state.engine_pid,
|
||||
"engine_uptime_seconds": state.engine_started.map(|t| t.elapsed().as_secs()),
|
||||
"crashes": state.crashes,
|
||||
"last_crash": state.last_crash,
|
||||
"sample": state.sample,
|
||||
})
|
||||
}
|
||||
|
||||
/// A random token: two 64-bit hashes keyed by the OS's random hasher seeds, the time and the process.
|
||||
fn random_token() -> String {
|
||||
use std::hash::{BuildHasher, Hasher};
|
||||
(0..2u64)
|
||||
.map(|i| {
|
||||
let mut h = std::collections::hash_map::RandomState::new().build_hasher();
|
||||
h.write_u64(i);
|
||||
h.write_u128(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_nanos()),
|
||||
);
|
||||
h.write_u32(std::process::id());
|
||||
format!("{:016x}", h.finish())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn same_token(a: &str, b: &str) -> bool {
|
||||
a.len() == b.len() && a.bytes().zip(b.bytes()).fold(0u8, |d, (x, y)| d | (x ^ y)) == 0
|
||||
}
|
||||
|
||||
/// Writes a file only its owner can read.
|
||||
pub fn write_private(path: &Path, text: &str) -> std::io::Result<()> {
|
||||
let mut options = std::fs::OpenOptions::new();
|
||||
options.write(true).create(true).truncate(true);
|
||||
#[cfg(unix)]
|
||||
std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
|
||||
options.open(path)?.write_all(text.as_bytes())
|
||||
}
|
||||
|
||||
/// Removes the endpoint file when the supervisor ends.
|
||||
pub struct Endpoint(PathBuf);
|
||||
impl Drop for Endpoint {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts the API on a free port of 127.0.0.1 and writes where it is and its token.
|
||||
pub fn start(shared: Arc<Shared>) -> Result<Endpoint, String> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").map_err(|e| format!("control API: {e}"))?;
|
||||
let address = listener.local_addr().map_err(|e| e.to_string())?;
|
||||
let token = random_token();
|
||||
let path = shared.dir.join(ENDPOINT);
|
||||
write_private(
|
||||
&path,
|
||||
&json!({ "address": address.to_string(), "token": token, "pid": std::process::id() })
|
||||
.to_string(),
|
||||
)
|
||||
.map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
let hosts = [address.to_string(), format!("localhost:{}", address.port())];
|
||||
http::serve(listener, move |request| {
|
||||
// A page in a browser can't send the token, and a rebound DNS name can't pass as this address.
|
||||
let authorised = request
|
||||
.header("authorization")
|
||||
.and_then(|h| h.strip_prefix("Bearer "))
|
||||
.is_some_and(|t| same_token(t.trim(), &token));
|
||||
if !authorised {
|
||||
return Response::error(401, "missing or wrong token (see .horizon/api.json)");
|
||||
}
|
||||
if !request
|
||||
.header("host")
|
||||
.is_some_and(|h| hosts.iter().any(|a| a == h))
|
||||
{
|
||||
return Response::error(403, "wrong Host");
|
||||
}
|
||||
route(&shared, request)
|
||||
})
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(Endpoint(path))
|
||||
}
|
||||
|
||||
fn route(shared: &Shared, request: &Request) -> Response {
|
||||
match (request.method.as_str(), request.path.as_str()) {
|
||||
("GET", "/api/v1/status") => Response::json(200, &status(shared)),
|
||||
("GET", "/api/v1/logs") => {
|
||||
let since = request
|
||||
.param("since")
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0);
|
||||
let (next, lines) = shared.log.lock().unwrap().since(since);
|
||||
Response::json(200, &json!({ "next": next, "lines": lines }))
|
||||
}
|
||||
("POST", "/api/v1/command") => {
|
||||
let line = serde_json::from_slice::<Value>(&request.body)
|
||||
.ok()
|
||||
.and_then(|v| v["line"].as_str().map(str::to_owned));
|
||||
let Some(line) = line.filter(|l| !l.trim().is_empty() && !l.contains('\n')) else {
|
||||
return Response::error(400, "the body must be {\"line\": \"<one command>\"}");
|
||||
};
|
||||
if shared.state.lock().unwrap().phase != Phase::Running {
|
||||
return Response::error(409, "the game isn't running right now");
|
||||
}
|
||||
match engine::Control::connect(&shared.dir)
|
||||
.and_then(|mut c| c.ask(&format!("console {}", line.trim_start_matches('/'))))
|
||||
{
|
||||
Ok(reply) => Response::json(200, &reply),
|
||||
Err(e) => Response::error(409, &format!("the game didn't answer: {e}")),
|
||||
}
|
||||
}
|
||||
("POST", "/api/v1/stop") => {
|
||||
shared.stop.store(true, Ordering::SeqCst);
|
||||
Response::json(200, &json!({ "ok": true }))
|
||||
}
|
||||
("POST", "/api/v1/restart") => {
|
||||
shared.restart.store(true, Ordering::SeqCst);
|
||||
Response::json(200, &json!({ "ok": true }))
|
||||
}
|
||||
(
|
||||
_,
|
||||
"/api/v1/status" | "/api/v1/logs" | "/api/v1/command" | "/api/v1/stop"
|
||||
| "/api/v1/restart",
|
||||
) => Response::error(405, "wrong method"),
|
||||
_ => Response::error(404, "no such endpoint"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The API of the server running in a folder, for the CLI.
|
||||
pub struct Client {
|
||||
address: SocketAddr,
|
||||
token: String,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
/// The running server's API; `None` when no server runs in `dir`.
|
||||
pub fn open(dir: &Path) -> Result<Option<Self>, String> {
|
||||
if crate::supervisor::lock(dir).is_ok() {
|
||||
return Ok(None);
|
||||
}
|
||||
// The supervisor writes the endpoint just after taking the lock: give it a moment.
|
||||
let path = dir.join(ENDPOINT);
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let text = loop {
|
||||
match std::fs::read_to_string(&path) {
|
||||
Ok(text) => break text,
|
||||
Err(_) if Instant::now() < deadline => {
|
||||
std::thread::sleep(Duration::from_millis(100))
|
||||
}
|
||||
Err(e) => return Err(format!("{}: {e}", path.display())),
|
||||
}
|
||||
};
|
||||
let v: Value =
|
||||
serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
let address = v["address"]
|
||||
.as_str()
|
||||
.and_then(|a| a.parse().ok())
|
||||
.ok_or_else(|| format!("{}: no address", path.display()))?;
|
||||
let token = v["token"].as_str().unwrap_or_default().to_owned();
|
||||
Ok(Some(Self { address, token }))
|
||||
}
|
||||
|
||||
pub fn call(&self, method: &str, path: &str, body: Option<&Value>) -> Result<Value, String> {
|
||||
let (status, text) = http::request(
|
||||
self.address,
|
||||
&self.token,
|
||||
method,
|
||||
path,
|
||||
body,
|
||||
Duration::from_secs(60),
|
||||
)
|
||||
.map_err(|e| format!("the server didn't answer: {e}"))?;
|
||||
let value: Value = serde_json::from_str(&text).unwrap_or(Value::Null);
|
||||
if status == 200 {
|
||||
Ok(value)
|
||||
} else {
|
||||
Err(value["error"].as_str().unwrap_or(&text).to_owned())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn log_keeps_the_last_lines_numbered() {
|
||||
let mut log = Log::default();
|
||||
for i in 0..LOG_LINES + 10 {
|
||||
log.push(i.to_string());
|
||||
}
|
||||
let (next, lines) = log.since(0);
|
||||
assert_eq!(next, (LOG_LINES + 10) as u64);
|
||||
assert_eq!(lines.len(), LOG_LINES);
|
||||
assert_eq!(lines[0], "10");
|
||||
let (next2, tail) = log.since(next - 2);
|
||||
assert_eq!(
|
||||
(next2, tail),
|
||||
(
|
||||
next,
|
||||
vec![(LOG_LINES + 8).to_string(), (LOG_LINES + 9).to_string()]
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_api_wants_its_token() {
|
||||
let dir = std::env::temp_dir().join(format!("horizon-server-api-{}", std::process::id()));
|
||||
std::fs::create_dir_all(dir.join(".horizon")).unwrap();
|
||||
let shared = Shared::new(&dir);
|
||||
let _endpoint = start(shared.clone()).unwrap();
|
||||
let v: Value =
|
||||
serde_json::from_str(&std::fs::read_to_string(dir.join(ENDPOINT)).unwrap()).unwrap();
|
||||
let address: SocketAddr = v["address"].as_str().unwrap().parse().unwrap();
|
||||
let token = v["token"].as_str().unwrap();
|
||||
let get = |token: &str| {
|
||||
http::request(
|
||||
address,
|
||||
token,
|
||||
"GET",
|
||||
"/api/v1/status",
|
||||
None,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(get("nope").0, 401);
|
||||
let (status, body) = get(token);
|
||||
assert_eq!(status, 200);
|
||||
let status: Value = serde_json::from_str(&body).unwrap();
|
||||
assert_eq!(status["phase"], "starting");
|
||||
let (code, _) = http::request(
|
||||
address,
|
||||
token,
|
||||
"POST",
|
||||
"/api/v1/stop",
|
||||
None,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(code, 200);
|
||||
assert!(shared.stop.load(Ordering::SeqCst));
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mode = std::fs::metadata(dir.join(ENDPOINT))
|
||||
.unwrap()
|
||||
.permissions()
|
||||
.mode();
|
||||
assert_eq!(mode & 0o777, 0o600);
|
||||
}
|
||||
drop(_endpoint);
|
||||
assert!(!dir.join(ENDPOINT).exists());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
//! The commands that manage a server in the background (docs/server.md, section 5): `start`, `stop`,
|
||||
//! `restart`, `status`, `cmd` and `logs`. They talk to the running supervisor through the control API.
|
||||
|
||||
use crate::{api::Client, signals, supervisor};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
path::Path,
|
||||
process::{Command, Stdio},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
/// How long `start` waits for the game's `Done`, and `stop` for the server to end.
|
||||
const START_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const STOP_TIMEOUT: Duration = Duration::from_secs(180);
|
||||
|
||||
fn running(dir: &Path) -> Result<Client, String> {
|
||||
Client::open(dir)?.ok_or_else(|| format!("no server is running in {}", dir.display()))
|
||||
}
|
||||
|
||||
/// `start`: `run` in the background, its output in logs/supervisor.log; returns once the game is up.
|
||||
pub fn start(dir: &Path) -> Result<(), String> {
|
||||
if Client::open(dir)?.is_some() {
|
||||
return Err(format!("a server is already running in {}", dir.display()));
|
||||
}
|
||||
crate::config::Config::load(dir)?;
|
||||
// Left by a supervisor that was killed: the new one writes its own.
|
||||
let endpoint = dir.join(".horizon/api.json");
|
||||
let _ = std::fs::remove_file(&endpoint);
|
||||
let logs = dir.join("logs");
|
||||
std::fs::create_dir_all(&logs).map_err(|e| e.to_string())?;
|
||||
let log_path = logs.join("supervisor.log");
|
||||
let log = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&log_path)
|
||||
.map_err(|e| format!("{}: {e}", log_path.display()))?;
|
||||
let exe = std::env::current_exe().map_err(|e| e.to_string())?;
|
||||
let mut command = Command::new(exe);
|
||||
command
|
||||
.arg("--dir")
|
||||
.arg(dir)
|
||||
.arg("run")
|
||||
.stdin(Stdio::null())
|
||||
.stdout(log.try_clone().map_err(|e| e.to_string())?)
|
||||
.stderr(log);
|
||||
// Its own process group: closing this terminal doesn't stop the server.
|
||||
#[cfg(unix)]
|
||||
std::os::unix::process::CommandExt::process_group(&mut command, 0);
|
||||
#[cfg(windows)]
|
||||
std::os::windows::process::CommandExt::creation_flags(
|
||||
&mut command,
|
||||
windows_sys::Win32::System::Threading::DETACHED_PROCESS
|
||||
| windows_sys::Win32::System::Threading::CREATE_NEW_PROCESS_GROUP,
|
||||
);
|
||||
let mut child = command.spawn().map_err(|e| e.to_string())?;
|
||||
let deadline = Instant::now() + START_TIMEOUT;
|
||||
print!("Starting");
|
||||
loop {
|
||||
if let Ok(Some(status)) = child.try_wait() {
|
||||
println!();
|
||||
return Err(format!(
|
||||
"the server stopped ({status}); see {}",
|
||||
log_path.display()
|
||||
));
|
||||
}
|
||||
// Only once the new supervisor holds the folder (its endpoint is there) may we look: probing the
|
||||
// lock any earlier could take it from under it.
|
||||
if endpoint.exists()
|
||||
&& let Ok(Some(client)) = Client::open(dir)
|
||||
&& let Ok(status) = client.call("GET", "/api/v1/status", None)
|
||||
&& status["phase"] == "running"
|
||||
{
|
||||
println!(
|
||||
"\nThe server is running (pid {}). `horizon-server status` shows how it does, \
|
||||
`horizon-server stop` stops it.",
|
||||
child.id()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() > deadline {
|
||||
println!();
|
||||
return Err(format!(
|
||||
"the game isn't up after {} s; it may still be starting: see {}",
|
||||
START_TIMEOUT.as_secs(),
|
||||
log_path.display()
|
||||
));
|
||||
}
|
||||
print!(".");
|
||||
let _ = std::io::Write::flush(&mut std::io::stdout());
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
}
|
||||
}
|
||||
|
||||
/// `stop` and `restart`.
|
||||
pub fn stop(dir: &Path, restart: bool) -> Result<(), String> {
|
||||
let client = running(dir)?;
|
||||
if restart {
|
||||
let before = client.call("GET", "/api/v1/status", None)?["engine_pid"].as_u64();
|
||||
client.call("POST", "/api/v1/restart", None)?;
|
||||
print!("Restarting");
|
||||
wait_running(&client, before)?;
|
||||
println!("\nThe game is running again.");
|
||||
return Ok(());
|
||||
}
|
||||
client.call("POST", "/api/v1/stop", None)?;
|
||||
let deadline = Instant::now() + STOP_TIMEOUT;
|
||||
while supervisor::lock(dir).is_err() {
|
||||
if Instant::now() > deadline {
|
||||
return Err("the server is still stopping".into());
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
}
|
||||
println!("Stopped.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Waits until the game is up: a new one when `replaced` is the engine's pid before a restart.
|
||||
fn wait_running(client: &Client, replaced: Option<u64>) -> Result<(), String> {
|
||||
let deadline = Instant::now() + START_TIMEOUT;
|
||||
loop {
|
||||
let status = client.call("GET", "/api/v1/status", None)?;
|
||||
if status["phase"] == "running" && status["engine_pid"].as_u64() != replaced {
|
||||
return Ok(());
|
||||
}
|
||||
if status["phase"] == "stopped" || status["phase"] == "stopping" {
|
||||
return Err("the server is stopping".into());
|
||||
}
|
||||
if Instant::now() > deadline {
|
||||
return Err(format!(
|
||||
"the game isn't up after {} s; `horizon-server logs` shows why",
|
||||
START_TIMEOUT.as_secs()
|
||||
));
|
||||
}
|
||||
if replaced.is_some() {
|
||||
print!(".");
|
||||
let _ = std::io::Write::flush(&mut std::io::stdout());
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
}
|
||||
}
|
||||
|
||||
/// `cmd`: one console command, and what it printed.
|
||||
pub fn command(dir: &Path, words: &[String]) -> Result<(), String> {
|
||||
if words.is_empty() {
|
||||
return Err("cmd needs a console command, for example: horizon-server cmd list".into());
|
||||
}
|
||||
let client = running(dir)?;
|
||||
// A game that is starting or restarting takes the command once it is up.
|
||||
wait_running(&client, None)?;
|
||||
let reply = client.call(
|
||||
"POST",
|
||||
"/api/v1/command",
|
||||
Some(&json!({ "line": words.join(" ") })),
|
||||
)?;
|
||||
for line in reply["output"].as_array().into_iter().flatten() {
|
||||
println!("{}", line.as_str().unwrap_or_default());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `status [--json]`.
|
||||
pub fn status(dir: &Path, as_json: bool) -> Result<(), String> {
|
||||
let Some(client) = Client::open(dir)? else {
|
||||
if as_json {
|
||||
println!("{}", json!({ "phase": "stopped" }));
|
||||
} else {
|
||||
println!("Stopped.");
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
let status = client.call("GET", "/api/v1/status", None)?;
|
||||
if as_json {
|
||||
println!("{status}");
|
||||
} else {
|
||||
print!("{}", describe(&status));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The status for people: three short lines.
|
||||
fn describe(status: &Value) -> String {
|
||||
let mut out = String::new();
|
||||
let phase = status["phase"].as_str().unwrap_or("?");
|
||||
let uptime = status["engine_uptime_seconds"]
|
||||
.as_u64()
|
||||
.or(status["uptime_seconds"].as_u64())
|
||||
.map_or_else(String::new, |s| format!(", up {}", duration(s)));
|
||||
let crashes = match status["crashes"].as_u64() {
|
||||
Some(0) | None => String::new(),
|
||||
Some(n) => format!(
|
||||
", {n} crash{} (last: {})",
|
||||
if n == 1 { "" } else { "es" },
|
||||
status["last_crash"].as_str().unwrap_or("?")
|
||||
),
|
||||
};
|
||||
out += &format!("{}{uptime}{crashes}\n", capitalised(phase));
|
||||
let sample = &status["sample"];
|
||||
let game = &sample["game"];
|
||||
if let Some(tps) = game["tps"].as_f64() {
|
||||
let mspt = &game["mspt"];
|
||||
out += &format!(
|
||||
"TPS {tps:.1}, MSPT {:.1} ms average, {:.1} ms P95{}\n",
|
||||
mspt["average"].as_f64().unwrap_or(0.0),
|
||||
mspt["p95"].as_f64().unwrap_or(0.0),
|
||||
if game["paused"] == true {
|
||||
" (paused: nobody online)"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
);
|
||||
let names: Vec<&str> = game["players"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|p| p["name"].as_str())
|
||||
.collect();
|
||||
out += &format!(
|
||||
"{} of {} players{}{}, {} chunks loaded\n",
|
||||
names.len(),
|
||||
game["max_players"].as_u64().unwrap_or(0),
|
||||
if names.is_empty() { "" } else { ": " },
|
||||
names.join(", "),
|
||||
game["chunks"].as_u64().unwrap_or(0)
|
||||
);
|
||||
}
|
||||
let ram = sample["rss_bytes"]
|
||||
.as_f64()
|
||||
.map(|b| format!("RAM {:.2} GB", b / 1e9));
|
||||
let cpu = sample["cpu_percent"]
|
||||
.as_f64()
|
||||
.map(|c| format!("CPU {c:.0} % of a core"));
|
||||
let usage: Vec<String> = ram.into_iter().chain(cpu).collect();
|
||||
if !usage.is_empty() {
|
||||
out += &format!("{}\n", usage.join(", "));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn capitalised(s: &str) -> String {
|
||||
let mut c = s.chars();
|
||||
c.next()
|
||||
.map(|f| f.to_uppercase().chain(c).collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn duration(secs: u64) -> String {
|
||||
match secs {
|
||||
0..60 => format!("{secs} s"),
|
||||
60..3600 => format!("{} min", secs / 60),
|
||||
3600..86_400 => format!("{} h {:02} min", secs / 3600, secs / 60 % 60),
|
||||
_ => format!("{} d {} h", secs / 86_400, secs / 3600 % 24),
|
||||
}
|
||||
}
|
||||
|
||||
/// `logs [-f] [-n N]`: the last lines, from the running server or else from logs/latest.log.
|
||||
pub fn logs(dir: &Path, follow: bool, count: usize) -> Result<(), String> {
|
||||
let Some(client) = Client::open(dir)? else {
|
||||
if follow {
|
||||
return Err("no server is running to follow".into());
|
||||
}
|
||||
let path = dir.join("logs/latest.log");
|
||||
let text =
|
||||
std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
for line in &lines[lines.len().saturating_sub(count)..] {
|
||||
println!("{line}");
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
let reply = client.call("GET", "/api/v1/logs?since=0", None)?;
|
||||
let lines = reply["lines"].as_array().cloned().unwrap_or_default();
|
||||
for line in &lines[lines.len().saturating_sub(count)..] {
|
||||
println!("{}", line.as_str().unwrap_or_default());
|
||||
}
|
||||
if !follow {
|
||||
return Ok(());
|
||||
}
|
||||
signals::install();
|
||||
let mut next = reply["next"].as_u64().unwrap_or(0);
|
||||
while signals::requests() == 0 {
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
let Ok(reply) = client.call("GET", &format!("/api/v1/logs?since={next}"), None) else {
|
||||
break;
|
||||
};
|
||||
for line in reply["lines"].as_array().into_iter().flatten() {
|
||||
println!("{}", line.as_str().unwrap_or_default());
|
||||
}
|
||||
next = reply["next"].as_u64().unwrap_or(next);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn status_reads_well() {
|
||||
let status = json!({
|
||||
"phase": "running",
|
||||
"engine_uptime_seconds": 7380,
|
||||
"crashes": 1,
|
||||
"last_crash": "signal: 9 (SIGKILL)",
|
||||
"sample": {
|
||||
"rss_bytes": 1_420_000_000u64,
|
||||
"cpu_percent": 12.4,
|
||||
"game": {
|
||||
"tps": 20.0,
|
||||
"mspt": { "average": 3.12, "p95": 7.9 },
|
||||
"players": [{ "name": "Alex" }, { "name": "Steve" }],
|
||||
"max_players": 20,
|
||||
"chunks": 1234,
|
||||
"paused": false
|
||||
}
|
||||
}
|
||||
});
|
||||
assert_eq!(
|
||||
describe(&status),
|
||||
"Running, up 2 h 03 min, 1 crash (last: signal: 9 (SIGKILL))\n\
|
||||
TPS 20.0, MSPT 3.1 ms average, 7.9 ms P95\n\
|
||||
2 of 20 players: Alex, Steve, 1234 chunks loaded\n\
|
||||
RAM 1.42 GB, CPU 12 % of a core\n"
|
||||
);
|
||||
assert_eq!(
|
||||
describe(&json!({ "phase": "starting", "uptime_seconds": 4 })),
|
||||
"Starting, up 4 s\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,11 @@
|
||||
|
||||
use crate::config::Config;
|
||||
use std::{
|
||||
io::{BufRead, BufReader, Write},
|
||||
net::{Ipv4Addr, TcpStream},
|
||||
path::{Path, PathBuf},
|
||||
process::{Child, Command, Stdio},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
/// The game's program names, most specific first: a headless engine once there is one, else the game.
|
||||
@@ -83,12 +86,15 @@ pub fn resources(config: &Config, dir: &Path, program: Option<&Path>) -> Result<
|
||||
})
|
||||
}
|
||||
|
||||
/// Starts the engine in the server folder, with its console and log piped to the supervisor.
|
||||
/// Starts the engine in the server folder, with its console and log piped to the supervisor and its status
|
||||
/// on the control endpoint.
|
||||
pub fn spawn(program: &Path, resources: &Path, dir: &Path) -> std::io::Result<Child> {
|
||||
let mut command = Command::new(program);
|
||||
command
|
||||
.arg("--resources")
|
||||
.arg(resources)
|
||||
.arg("--control-socket")
|
||||
.arg(control_path(dir))
|
||||
.args(["--dedicated", "--nogui"])
|
||||
.current_dir(dir)
|
||||
.env("HORIZON_NO_AUDIO", "1")
|
||||
@@ -106,6 +112,55 @@ pub fn spawn(program: &Path, resources: &Path, dir: &Path) -> std::io::Result<Ch
|
||||
command.spawn()
|
||||
}
|
||||
|
||||
/// The game's control endpoint in a server folder (`--control-socket`): TCP on 127.0.0.1 with a token, on every OS.
|
||||
pub fn control_path(dir: &Path) -> PathBuf {
|
||||
dir.join(".horizon/engine.json")
|
||||
}
|
||||
|
||||
/// A connection to the game's control endpoint: one JSON line asked, one answered.
|
||||
pub struct Control {
|
||||
writer: TcpStream,
|
||||
reader: BufReader<TcpStream>,
|
||||
}
|
||||
|
||||
impl Control {
|
||||
pub fn connect(dir: &Path) -> std::io::Result<Self> {
|
||||
let text = std::fs::read_to_string(control_path(dir))?;
|
||||
let bad = || std::io::Error::new(std::io::ErrorKind::InvalidData, "not a control endpoint");
|
||||
let v: serde_json::Value = serde_json::from_str(&text).map_err(|_| bad())?;
|
||||
let port = v["port"]
|
||||
.as_u64()
|
||||
.and_then(|p| u16::try_from(p).ok())
|
||||
.ok_or_else(bad)?;
|
||||
let token = v["token"].as_str().ok_or_else(bad)?;
|
||||
let writer = TcpStream::connect((Ipv4Addr::LOCALHOST, port))?;
|
||||
writer.set_read_timeout(Some(Duration::from_secs(30)))?;
|
||||
let mut control = Self {
|
||||
reader: BufReader::new(writer.try_clone()?),
|
||||
writer,
|
||||
};
|
||||
let reply = control.ask(&format!("auth {token}"))?;
|
||||
if reply["ok"] != true {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
"the game refused the token",
|
||||
));
|
||||
}
|
||||
Ok(control)
|
||||
}
|
||||
|
||||
pub fn ask(&mut self, line: &str) -> std::io::Result<serde_json::Value> {
|
||||
writeln!(self.writer, "{line}")?;
|
||||
self.writer.flush()?;
|
||||
let mut reply = String::new();
|
||||
if self.reader.read_line(&mut reply)? == 0 {
|
||||
return Err(std::io::ErrorKind::UnexpectedEof.into());
|
||||
}
|
||||
serde_json::from_str(&reply)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
//! Just enough HTTP/1.1 for the control API on 127.0.0.1 (docs/server.md, section 8): one request per
|
||||
//! connection, JSON bodies with a Content-Length. No async runtime and no dependency: a thread per
|
||||
//! connection is plenty for a CLI, a panel tab and an MCP client.
|
||||
|
||||
use std::{
|
||||
io::{BufRead, BufReader, Read, Write},
|
||||
net::{SocketAddr, TcpListener, TcpStream},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
/// The largest request body the API takes (a console command, a settings change).
|
||||
const MAX_BODY: usize = 1 << 20;
|
||||
|
||||
pub struct Request {
|
||||
pub method: String,
|
||||
pub path: String,
|
||||
pub query: String,
|
||||
pub headers: Vec<(String, String)>,
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Request {
|
||||
pub fn header(&self, name: &str) -> Option<&str> {
|
||||
self.headers
|
||||
.iter()
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case(name))
|
||||
.map(|(_, v)| v.as_str())
|
||||
}
|
||||
/// One `key=value` from the query string (no percent-decoding: the API's values are numbers).
|
||||
pub fn param(&self, key: &str) -> Option<&str> {
|
||||
self.query
|
||||
.split('&')
|
||||
.filter_map(|p| p.split_once('='))
|
||||
.find(|(k, _)| *k == key)
|
||||
.map(|(_, v)| v)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Response {
|
||||
pub status: u16,
|
||||
pub body: String,
|
||||
}
|
||||
|
||||
impl Response {
|
||||
pub fn json(status: u16, value: &serde_json::Value) -> Self {
|
||||
Self {
|
||||
status,
|
||||
body: value.to_string(),
|
||||
}
|
||||
}
|
||||
pub fn error(status: u16, message: &str) -> Self {
|
||||
Self::json(status, &serde_json::json!({ "error": message }))
|
||||
}
|
||||
}
|
||||
|
||||
fn reason(status: u16) -> &'static str {
|
||||
match status {
|
||||
200 => "OK",
|
||||
400 => "Bad Request",
|
||||
401 => "Unauthorized",
|
||||
403 => "Forbidden",
|
||||
404 => "Not Found",
|
||||
405 => "Method Not Allowed",
|
||||
409 => "Conflict",
|
||||
413 => "Payload Too Large",
|
||||
_ => "Error",
|
||||
}
|
||||
}
|
||||
|
||||
fn read_request(stream: &TcpStream) -> Result<Request, Response> {
|
||||
let bad = |m: &str| Response::error(400, m);
|
||||
let mut reader = BufReader::new(stream);
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).map_err(|_| bad("no request"))?;
|
||||
let mut parts = line.split_whitespace();
|
||||
let (Some(method), Some(target)) = (parts.next(), parts.next()) else {
|
||||
return Err(bad("malformed request line"));
|
||||
};
|
||||
let (path, query) = target.split_once('?').unwrap_or((target, ""));
|
||||
let mut headers = Vec::new();
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
if reader.read_line(&mut line).map_err(|_| bad("headers"))? == 0 || line.trim().is_empty() {
|
||||
break;
|
||||
}
|
||||
if headers.len() > 64 {
|
||||
return Err(bad("too many headers"));
|
||||
}
|
||||
if let Some((k, v)) = line.split_once(':') {
|
||||
headers.push((k.trim().to_owned(), v.trim().to_owned()));
|
||||
}
|
||||
}
|
||||
let length = headers
|
||||
.iter()
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case("content-length"))
|
||||
.map_or(Ok(0), |(_, v)| v.parse::<usize>())
|
||||
.map_err(|_| bad("bad Content-Length"))?;
|
||||
if length > MAX_BODY {
|
||||
return Err(Response::error(413, "the body is too large"));
|
||||
}
|
||||
let mut body = vec![0; length];
|
||||
reader
|
||||
.read_exact(&mut body)
|
||||
.map_err(|_| bad("the body is shorter than its Content-Length"))?;
|
||||
Ok(Request {
|
||||
method: method.to_owned(),
|
||||
path: path.to_owned(),
|
||||
query: query.to_owned(),
|
||||
headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
|
||||
fn write_response(mut stream: &TcpStream, response: &Response) {
|
||||
let _ = write!(
|
||||
stream,
|
||||
"HTTP/1.1 {} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n{}",
|
||||
response.status,
|
||||
reason(response.status),
|
||||
response.body.len(),
|
||||
response.body
|
||||
);
|
||||
let _ = stream.flush();
|
||||
}
|
||||
|
||||
/// Serves `handle` on `listener` until the process ends, a thread per connection.
|
||||
pub fn serve(
|
||||
listener: TcpListener,
|
||||
handle: impl Fn(&Request) -> Response + Send + Sync + 'static,
|
||||
) -> std::io::Result<()> {
|
||||
let handle = std::sync::Arc::new(handle);
|
||||
std::thread::Builder::new()
|
||||
.name("api".into())
|
||||
.spawn(move || {
|
||||
for stream in listener.incoming().flatten() {
|
||||
let handle = handle.clone();
|
||||
let _ = std::thread::Builder::new()
|
||||
.name("api-client".into())
|
||||
.spawn(move || {
|
||||
let _ = stream.set_read_timeout(Some(Duration::from_secs(10)));
|
||||
let response = match read_request(&stream) {
|
||||
Ok(request) => handle(&request),
|
||||
Err(response) => response,
|
||||
};
|
||||
write_response(&stream, &response);
|
||||
});
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One request to `address`, with a bearer token; the status and the body.
|
||||
pub fn request(
|
||||
address: SocketAddr,
|
||||
token: &str,
|
||||
method: &str,
|
||||
path: &str,
|
||||
body: Option<&serde_json::Value>,
|
||||
timeout: Duration,
|
||||
) -> std::io::Result<(u16, String)> {
|
||||
let mut stream = TcpStream::connect_timeout(&address, Duration::from_secs(5))?;
|
||||
stream.set_read_timeout(Some(timeout))?;
|
||||
let body = body.map(ToString::to_string).unwrap_or_default();
|
||||
write!(
|
||||
stream,
|
||||
"{method} {path} HTTP/1.1\r\nHost: {address}\r\nAuthorization: Bearer {token}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)?;
|
||||
stream.flush()?;
|
||||
let mut text = String::new();
|
||||
stream.read_to_string(&mut text)?;
|
||||
let (head, body) = text.split_once("\r\n\r\n").unwrap_or((&text, ""));
|
||||
let status = head
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.and_then(|s| s.parse().ok())
|
||||
.ok_or_else(|| std::io::Error::other("not an HTTP response"))?;
|
||||
Ok((status, body.to_owned()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn round_trip() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
serve(listener, |r| {
|
||||
Response::json(
|
||||
200,
|
||||
&serde_json::json!({
|
||||
"method": r.method,
|
||||
"path": r.path,
|
||||
"since": r.param("since"),
|
||||
"auth": r.header("authorization"),
|
||||
"body": String::from_utf8_lossy(&r.body),
|
||||
}),
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
let (status, body) = request(
|
||||
address,
|
||||
"t0k",
|
||||
"POST",
|
||||
"/api/v1/command?since=7",
|
||||
Some(&serde_json::json!({ "line": "list" })),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(status, 200);
|
||||
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||
assert_eq!(v["method"], "POST");
|
||||
assert_eq!(v["path"], "/api/v1/command");
|
||||
assert_eq!(v["since"], "7");
|
||||
assert_eq!(v["auth"], "Bearer t0k");
|
||||
assert_eq!(v["body"], r#"{"line":"list"}"#);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
//! `op`, `deop` and `whitelist` while the server is stopped: they edit vanilla's files with the code the game
|
||||
//! reads them with (horizon-server-files) and answer with vanilla's console messages. While it runs, the game
|
||||
//! owns the lists, so the same commands are typed in its console instead.
|
||||
//! `op`, `deop` and `whitelist`. While the server runs, the game owns the lists: the command goes to its
|
||||
//! console. While it is stopped, they edit vanilla's files with the code the game reads them with
|
||||
//! (horizon-server-files) and answer with vanilla's console messages.
|
||||
|
||||
use crate::{config, supervisor};
|
||||
use horizon_server_files::user_lists::{NameAndId, OpEntry, OpList, WhiteEntry, WhiteList};
|
||||
@@ -9,12 +9,11 @@ use std::path::Path;
|
||||
pub fn run(dir: &Path, command: &str, args: &[String]) -> Result<(), String> {
|
||||
let config = config::Config::load(dir)?;
|
||||
// Holding the folder's lock keeps the server from starting halfway through.
|
||||
let _lock = supervisor::lock(dir).map_err(|_| {
|
||||
format!(
|
||||
"the server is running: type `{command} {}` in its console",
|
||||
args.join(" ")
|
||||
)
|
||||
})?;
|
||||
let Ok(_lock) = supervisor::lock(dir) else {
|
||||
let mut words = vec![command.to_owned()];
|
||||
words.extend_from_slice(args);
|
||||
return crate::cli::command(dir, &words);
|
||||
};
|
||||
let args: Vec<&str> = args.iter().map(String::as_str).collect();
|
||||
let ops_path = dir.join("ops.json");
|
||||
let white_path = dir.join("whitelist.json");
|
||||
@@ -140,14 +139,15 @@ mod tests {
|
||||
.unwrap()
|
||||
.contains("Alex")
|
||||
);
|
||||
// A running server owns the lists.
|
||||
// A running server owns the lists: without its API, nothing changes.
|
||||
let lock = supervisor::lock(&dir).unwrap();
|
||||
assert!(
|
||||
run(&dir, "op", &s(&["Alex"]))
|
||||
.unwrap_err()
|
||||
.contains("running")
|
||||
);
|
||||
assert!(run(&dir, "op", &s(&["Alex"])).is_err());
|
||||
drop(lock);
|
||||
assert!(
|
||||
!std::fs::read_to_string(dir.join("ops.json"))
|
||||
.unwrap()
|
||||
.contains("Alex")
|
||||
);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
//! `horizon-server`: host a Horizon server (docs/server.md). It keeps the server's settings in `server.toml`,
|
||||
//! runs the game headless as the engine, and manages the vanilla files around it.
|
||||
|
||||
mod api;
|
||||
mod cli;
|
||||
mod config;
|
||||
mod engine;
|
||||
mod http;
|
||||
mod init;
|
||||
mod lists;
|
||||
mod signals;
|
||||
@@ -20,11 +23,18 @@ Usage: horizon-server [--dir DIR] <command>
|
||||
--yes)
|
||||
run Run the server in the foreground; type console commands, Ctrl-C stops it cleanly. It
|
||||
restarts the game if it crashes.
|
||||
start Run the server in the background (its output goes to logs/supervisor.log)
|
||||
stop, restart
|
||||
status [--json]
|
||||
Up or not, TPS, MSPT, players, RAM and CPU
|
||||
cmd COMMAND...
|
||||
Run a console command on the running server and print what it answers
|
||||
logs [-f] [-n N]
|
||||
The last N lines of the log (default 50); -f follows it
|
||||
import FILE Write server.toml from a vanilla server.properties
|
||||
op NAME, deop NAME
|
||||
whitelist on|off|list, whitelist add|remove NAME
|
||||
Change who may join while the server is stopped (while it runs, type the same command
|
||||
in its console)
|
||||
Who may join (on the running server, or in the files when it is stopped)
|
||||
|
||||
--dir DIR The server folder (default: the current folder)
|
||||
|
||||
@@ -53,6 +63,12 @@ fn main() -> ExitCode {
|
||||
}
|
||||
Some("init") => init_answers(&args[1..]).and_then(|a| init::run(&dir, a)),
|
||||
Some("run") => supervisor::run(&dir),
|
||||
Some("start") => cli::start(&dir),
|
||||
Some("stop") => cli::stop(&dir, false),
|
||||
Some("restart") => cli::stop(&dir, true),
|
||||
Some("status") => cli::status(&dir, args[1..].iter().any(|a| a == "--json")),
|
||||
Some("cmd") => cli::command(&dir, &args[1..]),
|
||||
Some("logs") => logs(&dir, &args[1..]),
|
||||
Some("import") => import(&dir, &args[1..]),
|
||||
Some(command @ ("op" | "deop" | "whitelist")) => lists::run(&dir, command, &args[1..]),
|
||||
Some(other) => Err(format!(
|
||||
@@ -68,6 +84,25 @@ fn main() -> ExitCode {
|
||||
}
|
||||
}
|
||||
|
||||
fn logs(dir: &std::path::Path, args: &[String]) -> Result<(), String> {
|
||||
let mut follow = false;
|
||||
let mut count = 50;
|
||||
let mut it = args.iter();
|
||||
while let Some(arg) = it.next() {
|
||||
match arg.as_str() {
|
||||
"-f" | "--follow" => follow = true,
|
||||
"-n" => {
|
||||
count = it
|
||||
.next()
|
||||
.and_then(|n| n.parse().ok())
|
||||
.ok_or("-n needs a number of lines")?;
|
||||
}
|
||||
other => return Err(format!("logs: unknown option {other}")),
|
||||
}
|
||||
}
|
||||
cli::logs(dir, follow, count)
|
||||
}
|
||||
|
||||
fn init_answers(args: &[String]) -> Result<init::Answers, String> {
|
||||
let mut answers = init::Answers::default();
|
||||
let mut it = args.iter();
|
||||
|
||||
@@ -1,14 +1,21 @@
|
||||
//! `horizon-server run`: writes `server.properties` from `server.toml`, runs the engine with the console on
|
||||
//! stdin, stops it cleanly on Ctrl-C or SIGTERM, and restarts it when it crashes (docs/server.md, section 7).
|
||||
//! stdin, serves the control API, stops the game cleanly on Ctrl-C, SIGTERM or `stop`, and restarts it when
|
||||
//! it crashes (docs/server.md, sections 2 and 7).
|
||||
|
||||
use crate::{config, config::Config, engine, signals};
|
||||
use crate::{
|
||||
api::{self, Phase, Shared},
|
||||
config,
|
||||
config::Config,
|
||||
engine, signals,
|
||||
};
|
||||
use horizon_server_files::java_properties::now_millis;
|
||||
use serde_json::json;
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{BufRead, BufReader, Read, Write},
|
||||
path::Path,
|
||||
process::{Child, ExitStatus},
|
||||
sync::mpsc,
|
||||
sync::{Arc, atomic::Ordering, mpsc},
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
@@ -22,15 +29,18 @@ const MAX_BACKOFF: Duration = Duration::from_secs(300);
|
||||
/// How long `stop` may take before the game is killed.
|
||||
const STOP_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
|
||||
/// A line from the supervisor, in the game's log format (`[HH:mm:ss] [thread/LEVEL]: message`, UTC).
|
||||
pub fn note(level: &str, message: &str) {
|
||||
/// A line from the supervisor, in the game's log format (`[HH:mm:ss] [thread/LEVEL]: message`, UTC), on
|
||||
/// stdout and in the log the API serves.
|
||||
fn note(shared: &Shared, level: &str, message: &str) {
|
||||
let secs = now_millis().div_euclid(1000).rem_euclid(86_400);
|
||||
println!(
|
||||
let line = format!(
|
||||
"[{:02}:{:02}:{:02}] [Supervisor/{level}]: {message}",
|
||||
secs / 3600,
|
||||
secs / 60 % 60,
|
||||
secs % 60
|
||||
);
|
||||
println!("{line}");
|
||||
shared.log.lock().unwrap().push(line);
|
||||
}
|
||||
|
||||
/// The folder's lock: one supervisor per server folder. Released when the process ends, even by a crash.
|
||||
@@ -56,6 +66,8 @@ enum Exit {
|
||||
/// Asked to stop, or the game ended by itself without an error (`stop` typed in its console, the EULA
|
||||
/// not agreed, the port taken): not restarted.
|
||||
Stopped,
|
||||
/// Asked to restart (`horizon-server restart`).
|
||||
Restart,
|
||||
Crashed(ExitStatus),
|
||||
}
|
||||
|
||||
@@ -65,6 +77,9 @@ pub fn run(dir: &Path) -> Result<(), String> {
|
||||
let program = engine::program(&config, dir)?;
|
||||
let resources = engine::resources(&config, dir, Some(&program))?;
|
||||
let _lock = lock(dir)?;
|
||||
let shared = Shared::new(dir);
|
||||
let _endpoint = api::start(shared.clone())?;
|
||||
sample(shared.clone());
|
||||
let console = console_lines();
|
||||
let mut crashes: Vec<Instant> = Vec::new();
|
||||
loop {
|
||||
@@ -74,6 +89,7 @@ pub fn run(dir: &Path) -> Result<(), String> {
|
||||
.write_properties(dir)
|
||||
.map_err(|e| format!("server.properties: {e}"))?;
|
||||
note(
|
||||
&shared,
|
||||
"INFO",
|
||||
&format!(
|
||||
"Starting {} with the game files in {}",
|
||||
@@ -81,11 +97,23 @@ pub fn run(dir: &Path) -> Result<(), String> {
|
||||
resources.display()
|
||||
),
|
||||
);
|
||||
shared.set_phase(Phase::Starting);
|
||||
let mut child = engine::spawn(&program, &resources, dir)
|
||||
.map_err(|e| format!("{}: {e}", program.display()))?;
|
||||
let exit = supervise(&mut child, &console);
|
||||
{
|
||||
let mut state = shared.state.lock().unwrap();
|
||||
state.engine_pid = Some(child.id());
|
||||
state.engine_started = Some(Instant::now());
|
||||
}
|
||||
let exit = supervise(&shared, &mut child, &console);
|
||||
{
|
||||
let mut state = shared.state.lock().unwrap();
|
||||
state.engine_pid = None;
|
||||
state.engine_started = None;
|
||||
}
|
||||
match config::sync_back(dir, &written) {
|
||||
Ok(keys) if !keys.is_empty() => note(
|
||||
&shared,
|
||||
"INFO",
|
||||
&format!(
|
||||
"Copied into server.toml what the game changed: {}",
|
||||
@@ -93,12 +121,22 @@ pub fn run(dir: &Path) -> Result<(), String> {
|
||||
),
|
||||
),
|
||||
Ok(_) => {}
|
||||
Err(e) => note("WARN", &format!("Could not update server.toml: {e}")),
|
||||
Err(e) => note(
|
||||
&shared,
|
||||
"WARN",
|
||||
&format!("Could not update server.toml: {e}"),
|
||||
),
|
||||
}
|
||||
let status = match exit {
|
||||
Exit::Stopped => return Ok(()),
|
||||
Exit::Restart => continue,
|
||||
Exit::Crashed(status) => status,
|
||||
};
|
||||
{
|
||||
let mut state = shared.state.lock().unwrap();
|
||||
state.crashes += 1;
|
||||
state.last_crash = Some(status.to_string());
|
||||
}
|
||||
if !config.engine.restart_on_crash {
|
||||
return Err(format!(
|
||||
"the game stopped unexpectedly ({status}); engine.restart_on_crash is off"
|
||||
@@ -116,15 +154,17 @@ pub fn run(dir: &Path) -> Result<(), String> {
|
||||
}
|
||||
let wait = backoff(crashes.len());
|
||||
note(
|
||||
&shared,
|
||||
"WARN",
|
||||
&format!(
|
||||
"The game stopped unexpectedly ({status}); restarting in {} s",
|
||||
wait.as_secs()
|
||||
),
|
||||
);
|
||||
shared.set_phase(Phase::Restarting);
|
||||
let until = Instant::now() + wait;
|
||||
while Instant::now() < until {
|
||||
if signals::requests() > 0 {
|
||||
if signals::requests() > 0 || shared.stop.load(Ordering::SeqCst) {
|
||||
return Ok(());
|
||||
}
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
@@ -156,8 +196,10 @@ fn console_lines() -> mpsc::Receiver<String> {
|
||||
rx
|
||||
}
|
||||
|
||||
/// Copies the game's output to ours, line by line.
|
||||
fn pump(from: impl Read + Send + 'static, to_stderr: bool) {
|
||||
/// Copies the game's output to ours and to the API's log, line by line; the game's `Done` line means it
|
||||
/// is up.
|
||||
fn pump(shared: &Arc<Shared>, from: impl Read + Send + 'static, to_stderr: bool) {
|
||||
let shared = shared.clone();
|
||||
let _ = thread::Builder::new()
|
||||
.name("engine-log".into())
|
||||
.spawn(move || {
|
||||
@@ -167,27 +209,86 @@ fn pump(from: impl Read + Send + 'static, to_stderr: bool) {
|
||||
eprintln!("{line}");
|
||||
} else {
|
||||
println!("{line}");
|
||||
if line.contains("/INFO]: Done (") {
|
||||
let mut state = shared.state.lock().unwrap();
|
||||
if state.phase == Phase::Starting {
|
||||
state.phase = Phase::Running;
|
||||
}
|
||||
}
|
||||
}
|
||||
shared.log.lock().unwrap().push(line);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Every second, the game's RAM, CPU and status, for `status` and (later) the panel's graphs.
|
||||
fn sample(shared: Arc<Shared>) {
|
||||
let _ = thread::Builder::new()
|
||||
.name("sampler".into())
|
||||
.spawn(move || {
|
||||
let mut control: Option<engine::Control> = None;
|
||||
let mut previous: Option<(u32, Instant, f64)> = None;
|
||||
loop {
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
let (pid, running) = {
|
||||
let state = shared.state.lock().unwrap();
|
||||
(state.engine_pid, state.phase == Phase::Running)
|
||||
};
|
||||
let Some(pid) = pid else {
|
||||
control = None;
|
||||
previous = None;
|
||||
shared.state.lock().unwrap().sample = serde_json::Value::Null;
|
||||
continue;
|
||||
};
|
||||
let now = Instant::now();
|
||||
let cpu = horizon_bench::cpu_seconds(pid);
|
||||
let cpu_percent = match (previous, cpu) {
|
||||
(Some((p, at, before)), Some(cpu)) if p == pid => {
|
||||
Some((cpu - before) / now.duration_since(at).as_secs_f64() * 100.0)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
previous = cpu.map(|c| (pid, now, c));
|
||||
if !running {
|
||||
control = None;
|
||||
} else if control.is_none() {
|
||||
control = engine::Control::connect(&shared.dir).ok();
|
||||
}
|
||||
let game = control
|
||||
.as_mut()
|
||||
.map(|c| c.ask("server-status"))
|
||||
.transpose()
|
||||
.unwrap_or_else(|_| {
|
||||
control = None;
|
||||
None
|
||||
});
|
||||
shared.state.lock().unwrap().sample = json!({
|
||||
"rss_bytes": horizon_bench::rss_bytes(pid),
|
||||
"cpu_percent": cpu_percent,
|
||||
"game": game,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Runs one engine process to its end: console lines in, the log out, and a clean `stop` when asked.
|
||||
fn supervise(child: &mut Child, console: &mpsc::Receiver<String>) -> Exit {
|
||||
fn supervise(shared: &Arc<Shared>, child: &mut Child, console: &mpsc::Receiver<String>) -> Exit {
|
||||
if let Some(out) = child.stdout.take() {
|
||||
pump(out, false);
|
||||
pump(shared, out, false);
|
||||
}
|
||||
if let Some(err) = child.stderr.take() {
|
||||
pump(err, true);
|
||||
pump(shared, err, true);
|
||||
}
|
||||
let mut stdin = child.stdin.take();
|
||||
let mut stopping: Option<Instant> = None;
|
||||
let mut restart = false;
|
||||
let mut handled = signals::requests();
|
||||
loop {
|
||||
while let Ok(line) = console.try_recv() {
|
||||
let command = line.trim().trim_start_matches('/');
|
||||
if command == "stop" && stopping.is_none() {
|
||||
stopping = Some(Instant::now());
|
||||
shared.set_phase(Phase::Stopping);
|
||||
}
|
||||
if let Some(input) = stdin.as_mut()
|
||||
&& writeln!(input, "{line}")
|
||||
@@ -198,21 +299,30 @@ fn supervise(child: &mut Child, console: &mpsc::Receiver<String>) -> Exit {
|
||||
}
|
||||
}
|
||||
let requests = signals::requests();
|
||||
if requests > handled {
|
||||
handled = requests;
|
||||
let asked = requests > handled || shared.stop.swap(false, Ordering::SeqCst);
|
||||
handled = requests;
|
||||
if shared.restart.swap(false, Ordering::SeqCst) && stopping.is_none() {
|
||||
restart = true;
|
||||
note(shared, "INFO", "Restarting the server");
|
||||
stopping = Some(Instant::now());
|
||||
shared.set_phase(Phase::Restarting);
|
||||
send_stop(&mut stdin);
|
||||
}
|
||||
if asked {
|
||||
restart = false;
|
||||
if stopping.is_none() {
|
||||
note("INFO", "Stopping the server");
|
||||
note(shared, "INFO", "Stopping the server");
|
||||
stopping = Some(Instant::now());
|
||||
if let Some(input) = stdin.as_mut() {
|
||||
let _ = writeln!(input, "stop").and_then(|()| input.flush());
|
||||
}
|
||||
shared.set_phase(Phase::Stopping);
|
||||
send_stop(&mut stdin);
|
||||
} else {
|
||||
note("WARN", "Stopping the game at once, without saving");
|
||||
note(shared, "WARN", "Stopping the game at once, without saving");
|
||||
let _ = child.kill();
|
||||
}
|
||||
}
|
||||
if stopping.is_some_and(|t| t.elapsed() > STOP_TIMEOUT) {
|
||||
note(
|
||||
shared,
|
||||
"WARN",
|
||||
&format!(
|
||||
"The game did not stop within {} s: killing it",
|
||||
@@ -220,19 +330,27 @@ fn supervise(child: &mut Child, console: &mpsc::Receiver<String>) -> Exit {
|
||||
),
|
||||
);
|
||||
let _ = child.kill();
|
||||
stopping = Some(Instant::now());
|
||||
}
|
||||
match child.try_wait() {
|
||||
Ok(Some(_)) if restart => return Exit::Restart,
|
||||
Ok(Some(status)) if stopping.is_some() || status.success() => return Exit::Stopped,
|
||||
Ok(Some(status)) => return Exit::Crashed(status),
|
||||
Ok(None) => thread::sleep(Duration::from_millis(50)),
|
||||
Err(e) => {
|
||||
note("ERROR", &format!("Lost track of the game: {e}"));
|
||||
note(shared, "ERROR", &format!("Lost track of the game: {e}"));
|
||||
return Exit::Stopped;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn send_stop(stdin: &mut Option<std::process::ChildStdin>) {
|
||||
if let Some(input) = stdin.as_mut() {
|
||||
let _ = writeln!(input, "stop").and_then(|()| input.flush());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
Reference in New Issue
Block a user