Play with friends: bound handshakes and the relay's first line in time, cap relay connections
A read timeout restarts with every byte, so a peer trickling one byte at a time could hold the host's 8 handshake slots (or relay threads) for hours. The host's openings and the relay's first line now end after 10 s in all; the relay closes connections past RELAY_MAX_CONNECTIONS (4096). The docs say what happens to a router mapping when the game crashes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -37,6 +37,9 @@ pub const MAX_SESSIONS: usize = 8;
|
||||
pub const DEFAULT_PORT: u16 = 25580;
|
||||
/// Code handshakes in progress at once (scanners can't pile them up).
|
||||
const MAX_HANDSHAKES: usize = 8;
|
||||
/// The most an inbound stream's opening (hello, key exchange, confirmation)
|
||||
/// may take, however slowly its bytes come.
|
||||
const HANDSHAKE: Duration = Duration::from_secs(10);
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct HostConfig {
|
||||
@@ -296,6 +299,10 @@ fn route_inbound(listeners: Vec<TcpListener>, shared: Arc<Shared>) {
|
||||
.name("horizon-friends-inbound".into())
|
||||
.spawn(move || {
|
||||
let _ = s.set_nonblocking(false);
|
||||
let Ok(deadline) = crate::deadline(&s, HANDSHAKE) else {
|
||||
shared.handshakes.fetch_sub(1, Ordering::Relaxed);
|
||||
return;
|
||||
};
|
||||
match direct::read_hello(&mut s) {
|
||||
Ok(direct::Hello::Relayed(session)) => {
|
||||
shared.handshakes.fetch_sub(1, Ordering::Relaxed);
|
||||
@@ -310,7 +317,9 @@ fn route_inbound(listeners: Vec<TcpListener>, shared: Arc<Shared>) {
|
||||
let _ = tx.send(s);
|
||||
}
|
||||
}
|
||||
Ok(direct::Hello::Code(session)) => serve_code(&shared, s, session),
|
||||
Ok(direct::Hello::Code(session)) => {
|
||||
serve_code(&shared, s, session, deadline)
|
||||
}
|
||||
Err(_) => {
|
||||
shared.handshakes.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
@@ -324,8 +333,9 @@ fn route_inbound(listeners: Vec<TcpListener>, shared: Arc<Shared>) {
|
||||
}
|
||||
|
||||
/// A guest who dialled an endpoint from the code: agree on the key from
|
||||
/// the code's secret, then carry the session.
|
||||
fn serve_code(shared: &Arc<Shared>, mut s: TcpStream, session: u64) {
|
||||
/// the code's secret, then carry the session (`deadline` bounds the
|
||||
/// handshake, not the game).
|
||||
fn serve_code(shared: &Arc<Shared>, mut s: TcpStream, session: u64, deadline: crate::Deadline) {
|
||||
let secret = shared
|
||||
.invite
|
||||
.lock()
|
||||
@@ -338,6 +348,7 @@ fn serve_code(shared: &Arc<Shared>, mut s: TcpStream, session: u64) {
|
||||
let key = crypto::exchange(&mut s, &secret, "direct", session)?;
|
||||
session::secure(s, &key, Role::Host)
|
||||
})();
|
||||
drop(deadline);
|
||||
shared.handshakes.fetch_sub(1, Ordering::Relaxed);
|
||||
match secured {
|
||||
Ok(secure) => carry(shared, session, secure, true),
|
||||
|
||||
@@ -26,8 +26,10 @@ pub use host::{Host, HostConfig, HostEvent, HostState};
|
||||
|
||||
use std::{
|
||||
io,
|
||||
net::{SocketAddr, TcpStream, ToSocketAddrs},
|
||||
net::{Shutdown, SocketAddr, TcpStream, ToSocketAddrs},
|
||||
sync::mpsc::{RecvTimeoutError, Sender, channel},
|
||||
thread::JoinHandle,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
/// Fill `buf` from the OS random generator.
|
||||
@@ -35,6 +37,29 @@ pub(crate) fn fill_random(buf: &mut [u8]) -> io::Result<()> {
|
||||
getrandom::fill(buf).map_err(|e| io::Error::other(e.to_string()))
|
||||
}
|
||||
|
||||
/// Ends a stream's opening once its time is up; dropping it first lets the
|
||||
/// stream go on. A read timeout alone restarts with every byte, so a peer
|
||||
/// sending one byte at a time could hold a handshake (and its slot) for
|
||||
/// hours.
|
||||
pub struct Deadline {
|
||||
_cancel: Sender<()>,
|
||||
}
|
||||
|
||||
/// Shut `stream` down after `limit` unless the returned [`Deadline`] is
|
||||
/// dropped before.
|
||||
pub fn deadline(stream: &TcpStream, limit: Duration) -> io::Result<Deadline> {
|
||||
let stream = stream.try_clone()?;
|
||||
let (tx, rx) = channel::<()>();
|
||||
std::thread::Builder::new()
|
||||
.name("horizon-friends-deadline".into())
|
||||
.spawn(move || {
|
||||
if rx.recv_timeout(limit) == Err(RecvTimeoutError::Timeout) {
|
||||
let _ = stream.shutdown(Shutdown::Both);
|
||||
}
|
||||
})?;
|
||||
Ok(Deadline { _cancel: tx })
|
||||
}
|
||||
|
||||
/// Which paths a session may take (`HORIZON_FRIENDS_PATH`).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PathMode {
|
||||
@@ -127,3 +152,34 @@ pub fn pump(
|
||||
let _ = outbound.join();
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::{
|
||||
io::Read,
|
||||
net::TcpListener,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn a_deadline_ends_the_stream_unless_dropped() {
|
||||
let l = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let _kept = TcpStream::connect(l.local_addr().unwrap()).unwrap();
|
||||
let (mut kept, _) = l.accept().unwrap();
|
||||
let _ended = TcpStream::connect(l.local_addr().unwrap()).unwrap();
|
||||
let (mut ended, _) = l.accept().unwrap();
|
||||
drop(deadline(&kept, Duration::from_millis(100)).unwrap());
|
||||
let _running = deadline(&ended, Duration::from_millis(100)).unwrap();
|
||||
let started = Instant::now();
|
||||
assert_eq!(ended.read(&mut [0u8; 1]).unwrap(), 0);
|
||||
assert!(started.elapsed() < Duration::from_secs(2));
|
||||
kept.set_read_timeout(Some(Duration::from_millis(300)))
|
||||
.unwrap();
|
||||
let e = kept.read(&mut [0u8; 1]).unwrap_err();
|
||||
assert!(matches!(
|
||||
e.kind(),
|
||||
io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,8 @@ pub struct Config {
|
||||
pub max_hosts: usize,
|
||||
pub max_bridges: usize,
|
||||
pub max_pending_per_host: usize,
|
||||
/// Connections open at once, of every kind (each is a thread).
|
||||
pub max_connections: usize,
|
||||
/// Bytes a second each way per relayed stream.
|
||||
pub session_rate: f64,
|
||||
/// Bytes a second for everything relayed.
|
||||
@@ -55,6 +57,7 @@ impl Default for Config {
|
||||
max_hosts: 1000,
|
||||
max_bridges: 200,
|
||||
max_pending_per_host: 8,
|
||||
max_connections: 4096,
|
||||
session_rate: 1024.0 * 1024.0,
|
||||
total_rate: 100.0 * 1_000_000.0 / 8.0,
|
||||
rates: Rates::default(),
|
||||
@@ -87,6 +90,9 @@ impl Config {
|
||||
if let Some(v) = var("RELAY_MAX_PENDING_PER_HOST") {
|
||||
c.max_pending_per_host = v;
|
||||
}
|
||||
if let Some(v) = var("RELAY_MAX_CONNECTIONS") {
|
||||
c.max_connections = v;
|
||||
}
|
||||
if let Some(v) = var::<f64>("RELAY_SESSION_KBPS") {
|
||||
c.session_rate = v * 1024.0;
|
||||
}
|
||||
@@ -163,6 +169,7 @@ struct State {
|
||||
global_unknown: Mutex<Bucket>,
|
||||
total: Mutex<Bucket>,
|
||||
bridges: AtomicUsize,
|
||||
connections: AtomicUsize,
|
||||
registrations: AtomicU64,
|
||||
joins: AtomicU64,
|
||||
unknown: AtomicU64,
|
||||
@@ -202,6 +209,7 @@ impl Relay {
|
||||
config,
|
||||
inner: Mutex::new(Inner::default()),
|
||||
bridges: AtomicUsize::new(0),
|
||||
connections: AtomicUsize::new(0),
|
||||
registrations: AtomicU64::new(0),
|
||||
joins: AtomicU64::new(0),
|
||||
unknown: AtomicU64::new(0),
|
||||
@@ -232,14 +240,25 @@ impl Relay {
|
||||
while !self.state.stop.load(Ordering::Relaxed) {
|
||||
match listener.accept() {
|
||||
Ok((stream, peer)) => {
|
||||
// Beyond the cap the connection is closed unanswered.
|
||||
if self.state.connections.fetch_add(1, Ordering::Relaxed)
|
||||
>= self.state.config.max_connections
|
||||
{
|
||||
self.state.connections.fetch_sub(1, Ordering::Relaxed);
|
||||
continue;
|
||||
}
|
||||
let state = self.state.clone();
|
||||
let _ =
|
||||
let spawned =
|
||||
std::thread::Builder::new()
|
||||
.name("relay-conn".into())
|
||||
.spawn(move || {
|
||||
let _ = stream.set_nonblocking(false);
|
||||
let _ = handle(&state, stream, peer);
|
||||
state.connections.fetch_sub(1, Ordering::Relaxed);
|
||||
});
|
||||
if spawned.is_err() {
|
||||
self.state.connections.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
@@ -295,6 +314,8 @@ fn handle(state: &Arc<State>, mut stream: TcpStream, peer: SocketAddr) -> io::Re
|
||||
return Ok(());
|
||||
}
|
||||
let mut peer = unmap(peer);
|
||||
// The first line (and an HTTP request) within `first_line` in all.
|
||||
let first_line = horizon_friends::deadline(&stream, state.config.first_line)?;
|
||||
stream.set_read_timeout(Some(state.config.first_line))?;
|
||||
let _ = stream.set_nodelay(true);
|
||||
let mut line = wire::read_line(&mut stream)?.ok_or(io::ErrorKind::UnexpectedEof)?;
|
||||
@@ -307,6 +328,7 @@ fn handle(state: &Arc<State>, mut stream: TcpStream, peer: SocketAddr) -> io::Re
|
||||
if line.starts_with(b"GET ") {
|
||||
return http(state, &mut stream, &line, peer);
|
||||
}
|
||||
drop(first_line);
|
||||
let Ok(hello) = serde_json::from_slice::<ToRelay>(&line) else {
|
||||
return refuse(&mut stream, ErrorCode::BadRequest);
|
||||
};
|
||||
|
||||
@@ -432,3 +432,30 @@ fn health_and_whoami_answer_http() {
|
||||
assert!(get("/whoami").contains("\"address\":\"127.0.0.1:"));
|
||||
assert!(get("/").starts_with("HTTP/1.1 404"));
|
||||
}
|
||||
|
||||
/// A first line trickled a byte at a time still has to arrive within
|
||||
/// `first_line` in all; past `max_connections`, connections close unanswered.
|
||||
#[test]
|
||||
fn slow_first_lines_and_too_many_connections_are_closed() {
|
||||
let (_relay, address) = relay(Config {
|
||||
first_line: Duration::from_secs(1),
|
||||
max_connections: 2,
|
||||
..Config::default()
|
||||
});
|
||||
let started = Instant::now();
|
||||
let mut slow = TcpStream::connect(&address).unwrap();
|
||||
let held = TcpStream::connect(&address).unwrap();
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
let mut over = TcpStream::connect(&address).unwrap();
|
||||
over.set_read_timeout(Some(Duration::from_secs(5))).unwrap();
|
||||
assert_eq!(over.read(&mut [0u8; 1]).unwrap_or(0), 0);
|
||||
while slow.write_all(b"{").is_ok() && started.elapsed() < Duration::from_secs(5) {
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
}
|
||||
let closed = started.elapsed();
|
||||
assert!(
|
||||
closed < Duration::from_secs(3),
|
||||
"the slow first line held on for {closed:?}"
|
||||
);
|
||||
drop(held);
|
||||
}
|
||||
|
||||
@@ -32,7 +32,8 @@ LAN's defaults), then the host's game, on its own:
|
||||
3. asks the home router to open the direct port for an hour, renewed every half hour and deleted when
|
||||
sharing stops: **PCP** (RFC 6887), then **NAT-PMP** (RFC 6886) on the default gateway, then **UPnP
|
||||
IGD** (SSDP discovery, SOAP `AddPortMapping`). A router that only takes permanent UPnP mappings gets
|
||||
one, deleted on stop the same way;
|
||||
one, deleted on stop the same way (if the game crashes instead, a timed mapping expires within the
|
||||
hour; a permanent one stays until it is removed in the router's settings);
|
||||
4. asks public **STUN** servers for its public IPv4 address (only the address: STUN sees UDP, the game
|
||||
is TCP). Defaults: `stun.cloudflare.com:3478` and `stun.l.google.com:19302`, two independent
|
||||
operators of free STUN at a global scale, so one outage doesn't break discovery. `HORIZON_STUN_SERVERS`
|
||||
@@ -79,7 +80,8 @@ The code's secret never leaves the code. Every session runs a password-authentic
|
||||
per-direction ChaCha20-Poly1305 keys with HKDF-SHA256 from the exchange and two fresh randoms, with a
|
||||
key confirmation first. A wrong code, a changed byte, a replayed or reordered frame ends the stream.
|
||||
The same exchange runs over relayed sessions, where it matters most: the relay gets nothing to test
|
||||
guesses against offline.
|
||||
guesses against offline. Online guesses on the direct port face the 80-bit secret; at most 8 openings
|
||||
run at once there, and each must finish within 10 seconds however slowly its bytes come.
|
||||
|
||||
## Privacy: who sees what
|
||||
|
||||
@@ -155,6 +157,7 @@ Settings, all optional:
|
||||
| `RELAY_MAX_HOSTS` | 1000 | short codes at once |
|
||||
| `RELAY_MAX_BRIDGES` | 200 | relayed streams at once |
|
||||
| `RELAY_MAX_PENDING_PER_HOST` | 8 | guests joining one host at the same time |
|
||||
| `RELAY_MAX_CONNECTIONS` | 4096 | connections open at once, of every kind (more are closed unanswered) |
|
||||
| `RELAY_SESSION_KBPS` | 1024 | KiB/s each way per relayed stream |
|
||||
| `RELAY_TOTAL_MBPS` | 100 | Mbit/s for everything relayed |
|
||||
| `RELAY_GLOBAL_UNKNOWN_PER_MIN` | 120 | unknown short codes a minute, all clients together |
|
||||
@@ -163,7 +166,7 @@ Settings, all optional:
|
||||
|
||||
Fixed limits: per client address (IPv6 by /64), 6 host registrations a minute (burst 3), 12 joins a
|
||||
minute (burst 6), 5 unknown short codes in 10 minutes block joins for 10 minutes; a relayed stream idle
|
||||
for 120 s is closed (the game's keep-alive sends every 15 s); a first line must arrive within 10 s;
|
||||
for 120 s is closed (the game's keep-alive sends every 15 s); the first line must be complete within 10 s;
|
||||
hosts answer pings within 60 s; five wrong secrets on a short code and the host retires it for a new one.
|
||||
|
||||
### The relay protocol
|
||||
|
||||
Reference in New Issue
Block a user