diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..417a1867c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +# Docker builds (the relay) need the Rust sources only. +target/ +.local/ +**/node_modules/ +website/ +benchmarks/ +packs/ +mods/ diff --git a/Cargo.lock b/Cargo.lock index 0c37a850a..b72a657bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + [[package]] name = "ahash" version = "0.8.12" @@ -322,6 +332,30 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "chrono" version = "0.4.45" @@ -333,6 +367,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + [[package]] name = "clipboard-win" version = "5.4.1" @@ -493,6 +538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core", "typenum", ] @@ -502,6 +548,32 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "fiat-crypto", + "rand_core", + "rustc_version", + "subtle", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "dasp_sample" version = "0.11.0" @@ -527,6 +599,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -631,6 +704,12 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "find-msvc-tools" version = "0.1.14" @@ -886,6 +965,24 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "horizon-assets" version = "0.1.0" @@ -935,6 +1032,20 @@ dependencies = [ "libm", ] +[[package]] +name = "horizon-friends" +version = "0.1.0" +dependencies = [ + "chacha20poly1305", + "getrandom 0.3.4", + "hkdf", + "serde", + "serde_json", + "sha2", + "socket2", + "spake2", +] + [[package]] name = "horizon-prepare" version = "0.1.0" @@ -956,6 +1067,15 @@ dependencies = [ "sha1", ] +[[package]] +name = "horizon-relay" +version = "0.1.0" +dependencies = [ + "getrandom 0.3.4", + "horizon-friends", + "serde_json", +] + [[package]] name = "horizon-telemetry" version = "0.1.0" @@ -1141,6 +1261,15 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1869,6 +1998,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "orbclient" version = "0.3.55" @@ -2018,6 +2153,17 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -2102,6 +2248,15 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -2475,6 +2630,28 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spake2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5482afe85a0b6ce956c945401598dbc527593c77ba51d0a87a586938b1b893a" +dependencies = [ + "curve25519-dalek", + "hkdf", + "rand_core", + "sha2", +] + [[package]] name = "spirv" version = "0.4.0+sdk-1.4.341.0" @@ -2810,6 +2987,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 6c21e207d..eb10100f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] exclude = ["launcher", "crates/horizon-telemetry-server"] -members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "xtask"] +members = ["crates/horizon-core", "crates/horizon-world", "crates/voxel-kernels", "crates/horizon-client", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-friends", "crates/horizon-relay", "xtask"] resolver = "2" [workspace.package] diff --git a/crates/horizon-friends/Cargo.toml b/crates/horizon-friends/Cargo.toml new file mode 100644 index 000000000..2d08a0bdd --- /dev/null +++ b/crates/horizon-friends/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "horizon-friends" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "Play with friends: invite codes, the relay protocol, the end-to-end encrypted tunnel between a host's integrated server and its guests" + +[dependencies] +# The key exchange keyed by the code's secret, then the frames. +spake2 = { version = "0.4", features = ["getrandom"] } +chacha20poly1305 = "0.10" +hkdf = "0.12" +sha2 = "0.10" +getrandom = "0.3" +# SO_REUSEADDR/SO_REUSEPORT for the direct socket (TCP simultaneous open). +socket2 = { version = "0.5", features = ["all"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1.0.151" + +[lints] +workspace = true diff --git a/crates/horizon-friends/src/code.rs b/crates/horizon-friends/src/code.rs new file mode 100644 index 000000000..6ab06ff82 --- /dev/null +++ b/crates/horizon-friends/src/code.rs @@ -0,0 +1,159 @@ +//! Invite codes: eight Crockford base32 symbols shown as `K7QM-3XPD`. The +//! first four are the nameplate the relay finds the host by; the last four +//! are the secret the key exchange uses, which only the players see. +use std::fmt; + +/// Crockford base32: digits and letters without I, L, O and U. +pub const ALPHABET: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; +/// Symbols in each half of a code. +pub const HALF: usize = 4; + +/// The value of a symbol as a player may type it: any case, `O` for `0`, +/// `I` and `L` for `1`. +fn value(c: char) -> Option { + let c = match c.to_ascii_uppercase() { + 'O' => '0', + 'I' | 'L' => '1', + c => c, + }; + ALPHABET + .iter() + .position(|&a| char::from(a) == c) + .map(|i| i as u8) +} + +/// Four symbols (20 bits) as their canonical text. +fn symbols(bits: u32) -> String { + (0..HALF) + .rev() + .map(|i| char::from(ALPHABET[(bits >> (5 * i)) as usize & 31])) + .collect() +} + +/// A nameplate drawn from 20 random bits. +pub fn nameplate(bits: u32) -> String { + symbols(bits & 0xF_FFFF) +} + +/// A nameplate as a player or a client sent it, made canonical; None when it +/// isn't four symbols. +pub fn parse_nameplate(text: &str) -> Option { + let symbols: Option> = text.chars().map(value).collect(); + let symbols = symbols?; + (symbols.len() == HALF).then(|| { + symbols + .iter() + .map(|&v| char::from(ALPHABET[v as usize])) + .collect() + }) +} + +#[derive(Clone, PartialEq, Eq)] +pub struct Code { + pub nameplate: String, + pub secret: String, +} +impl Code { + /// A code for a nameplate the relay handed out, with a fresh secret from + /// the OS random generator. + pub fn new(nameplate: &str) -> std::io::Result { + let mut b = [0u8; 4]; + crate::fill_random(&mut b)?; + Ok(Self { + nameplate: nameplate.to_owned(), + secret: symbols(u32::from_le_bytes(b) & 0xF_FFFF), + }) + } + /// A code as a player typed it: spaces, dashes and case don't matter, `O` + /// reads as `0`, `I` and `L` as `1`. + pub fn parse(text: &str) -> Option { + let mut out = String::with_capacity(2 * HALF); + for c in text.chars() { + if c.is_whitespace() || c == '-' { + continue; + } + out.push(char::from(ALPHABET[value(c)? as usize])); + } + if out.len() != 2 * HALF { + return None; + } + let (nameplate, secret) = out.split_at(HALF); + Some(Self { + nameplate: nameplate.to_owned(), + secret: secret.to_owned(), + }) + } +} +impl fmt::Display for Code { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}-{}", self.nameplate, self.secret) + } +} +/// The secret stays out of debug output and logs. +impl fmt::Debug for Code { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Code({}-****)", self.nameplate) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn alphabet_has_no_ambiguous_letters() { + assert_eq!(ALPHABET.len(), 32); + for c in *b"ILOU" { + assert!(!ALPHABET.contains(&c)); + } + let mut sorted = ALPHABET.to_vec(); + sorted.dedup(); + assert_eq!(sorted.len(), 32); + } + + #[test] + fn nameplates_cover_twenty_bits() { + assert_eq!(nameplate(0), "0000"); + assert_eq!(nameplate(0xF_FFFF), "ZZZZ"); + assert_eq!(nameplate(0x8000 | 31), "100Z"); + // High bits beyond 20 are ignored. + assert_eq!(nameplate(0xFFF0_0001), "0001"); + let mut seen = std::collections::HashSet::new(); + for bits in (0..0x10_0000).step_by(997) { + assert!(seen.insert(nameplate(bits))); + } + } + + #[test] + fn parses_what_players_type() { + let code = Code::parse("k7qm-3xpd").unwrap(); + assert_eq!(code.nameplate, "K7QM"); + assert_eq!(code.secret, "3XPD"); + assert_eq!(code.to_string(), "K7QM-3XPD"); + assert_eq!(Code::parse(" K7QM 3XPD ").unwrap(), code); + assert_eq!(Code::parse("K7QM3XPD").unwrap(), code); + // O, I and L are the symbols they look like. + let code = Code::parse("OIL0-1lio").unwrap(); + assert_eq!(code.to_string(), "0110-1110"); + assert!(Code::parse("K7QM-3XP").is_none()); + assert!(Code::parse("K7QM-3XPDA").is_none()); + assert!(Code::parse("K7QM-3XPU").is_none()); + assert!(Code::parse("").is_none()); + } + + #[test] + fn fresh_secrets_are_canonical_and_differ() { + let a = Code::new("K7QM").unwrap(); + assert_eq!(Code::parse(&a.to_string()).unwrap(), a); + let different = (0..8).any(|_| Code::new("K7QM").unwrap().secret != a.secret); + assert!(different); + assert_eq!(format!("{a:?}"), "Code(K7QM-****)"); + } + + #[test] + fn nameplates_parse() { + assert_eq!(parse_nameplate("k7qm").as_deref(), Some("K7QM")); + assert_eq!(parse_nameplate("K7Q"), None); + assert_eq!(parse_nameplate("K7QU"), None); + } +} diff --git a/crates/horizon-friends/src/crypto.rs b/crates/horizon-friends/src/crypto.rs new file mode 100644 index 000000000..837377205 --- /dev/null +++ b/crates/horizon-friends/src/crypto.rs @@ -0,0 +1,392 @@ +//! End-to-end protection between a host and a guest, whatever carries the +//! bytes. SPAKE2 turns the code's short secret into a session key without +//! giving anyone on the path (the relay included) a way to test guesses +//! offline; every stream of the session then derives its own +//! ChaCha20-Poly1305 keys from it and two fresh randoms, and starts with a +//! key confirmation so a wrong code or a changed byte ends it. +use chacha20poly1305::{ + ChaCha20Poly1305, KeyInit, + aead::{Aead, Payload}, +}; +use hkdf::Hkdf; +use sha2::Sha256; +use spake2::{Ed25519Group, Identity, Password, Spake2}; +use std::io::{self, Read, Write}; + +/// The most game bytes in one frame. +pub const MAX_PAYLOAD: usize = 16 * 1024; +const TAG: usize = 16; +const CONFIRM: &[u8] = b"horizon-friends v1 confirm"; + +const DATA: u8 = 0; +const CONTROL: u8 = 1; +const CONFIRMATION: u8 = 2; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Role { + Host, + Guest, +} + +/// The session key both ends derive from the code. +#[derive(Clone)] +pub struct Key([u8; 32]); + +/// The error a mismatched code produces: the peer's confirmation does not +/// open with our key. +pub fn wrong_code() -> io::Error { + io::Error::new(io::ErrorKind::PermissionDenied, "the code doesn't match") +} +pub fn is_wrong_code(e: &io::Error) -> bool { + e.kind() == io::ErrorKind::PermissionDenied +} + +/// SPAKE2 over a stream: both ends send their message, then read the +/// other's. The identity binds the key to this nameplate and session. +pub fn exchange( + stream: &mut S, + secret: &str, + nameplate: &str, + session: u64, +) -> io::Result { + let identity = format!("horizon-friends v1/{nameplate}/{session}"); + let (state, outbound) = Spake2::::start_symmetric( + &Password::new(secret.as_bytes()), + &Identity::new(identity.as_bytes()), + ); + let len = u8::try_from(outbound.len()).map_err(io::Error::other)?; + let mut message = vec![len]; + message.extend_from_slice(&outbound); + stream.write_all(&message)?; + stream.flush()?; + let mut len = [0u8; 1]; + stream.read_exact(&mut len)?; + let mut inbound = vec![0u8; usize::from(len[0])]; + stream.read_exact(&mut inbound)?; + let key = state + .finish(&inbound) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{e:?}")))?; + let mut out = [0u8; 32]; + let hk = Hkdf::::new(None, &key); + hk.expand(b"horizon-friends v1 session", &mut out) + .expect("32 bytes is a valid HKDF output"); + Ok(Key(out)) +} + +fn nonce(counter: u64) -> [u8; 12] { + let mut n = [0u8; 12]; + n[4..].copy_from_slice(&counter.to_be_bytes()); + n +} + +fn cipher(key: &Key, salt: &[u8], info: &[u8]) -> ChaCha20Poly1305 { + let mut okm = [0u8; 32]; + Hkdf::::new(Some(salt), &key.0) + .expand(info, &mut okm) + .expect("32 bytes is a valid HKDF output"); + ChaCha20Poly1305::new(&okm.into()) +} + +pub enum Frame { + Data(Vec), + Control(Vec), +} + +pub struct SecureWriter { + inner: W, + cipher: ChaCha20Poly1305, + counter: u64, +} +impl SecureWriter { + fn frame(&mut self, kind: u8, payload: &[u8]) -> io::Result<()> { + debug_assert!(payload.len() <= MAX_PAYLOAD); + let mut plain = Vec::with_capacity(1 + payload.len()); + plain.push(kind); + plain.extend_from_slice(payload); + let len = (plain.len() + TAG) as u16; + let header = len.to_be_bytes(); + let sealed = self + .cipher + .encrypt( + &nonce(self.counter).into(), + Payload { + msg: &plain, + aad: &header, + }, + ) + .map_err(|_| io::Error::other("encryption failed"))?; + self.counter = self + .counter + .checked_add(1) + .ok_or_else(|| io::Error::other("frame counter exhausted"))?; + let mut out = Vec::with_capacity(2 + sealed.len()); + out.extend_from_slice(&header); + out.extend_from_slice(&sealed); + self.inner.write_all(&out) + } + /// Game bytes, split into frames. + pub fn write_data(&mut self, bytes: &[u8]) -> io::Result<()> { + for chunk in bytes.chunks(MAX_PAYLOAD) { + self.frame(DATA, chunk)?; + } + self.inner.flush() + } + /// A message between the two tunnel ends. + pub fn write_control(&mut self, message: &[u8]) -> io::Result<()> { + if message.len() > MAX_PAYLOAD { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "control message too long", + )); + } + self.frame(CONTROL, message)?; + self.inner.flush() + } + pub fn get_ref(&self) -> &W { + &self.inner + } +} + +pub struct SecureReader { + inner: R, + cipher: ChaCha20Poly1305, + counter: u64, +} +impl SecureReader { + /// The next frame's kind and payload; None at a clean end of stream. + fn open(&mut self) -> io::Result)>> { + let mut header = [0u8; 2]; + let mut got = 0; + while got < 2 { + match self.inner.read(&mut header[got..]) { + Ok(0) if got == 0 => return Ok(None), + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), + Ok(n) => got += n, + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + let len = usize::from(u16::from_be_bytes(header)); + if !(TAG + 1..=TAG + 1 + MAX_PAYLOAD).contains(&len) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "bad frame length", + )); + } + let mut sealed = vec![0u8; len]; + self.inner.read_exact(&mut sealed)?; + let plain = self + .cipher + .decrypt( + &nonce(self.counter).into(), + Payload { + msg: &sealed, + aad: &header, + }, + ) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "frame failed to open"))?; + self.counter += 1; + Ok(Some((plain[0], plain[1..].to_vec()))) + } + pub fn read_frame(&mut self) -> io::Result> { + match self.open()? { + None => Ok(None), + Some((DATA, p)) => Ok(Some(Frame::Data(p))), + Some((CONTROL, p)) => Ok(Some(Frame::Control(p))), + Some(_) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected frame", + )), + } + } + pub fn get_ref(&self) -> &R { + &self.inner + } +} + +/// Protect one stream of the session: swap randoms, derive the keys for +/// each direction, confirm. Fails with [`wrong_code`] when the peer used +/// another key. +pub fn secure( + mut read: R, + mut write: W, + key: &Key, + role: Role, +) -> io::Result<(SecureReader, SecureWriter)> { + let mut mine = [0u8; 16]; + crate::fill_random(&mut mine)?; + write.write_all(&mine)?; + write.flush()?; + let mut theirs = [0u8; 16]; + read.read_exact(&mut theirs)?; + let (guest, host) = match role { + Role::Guest => (mine, theirs), + Role::Host => (theirs, mine), + }; + let mut salt = [0u8; 32]; + salt[..16].copy_from_slice(&guest); + salt[16..].copy_from_slice(&host); + let to_guest = cipher(key, &salt, b"horizon-friends v1 host->guest"); + let to_host = cipher(key, &salt, b"horizon-friends v1 guest->host"); + let (send, recv) = match role { + Role::Host => (to_guest, to_host), + Role::Guest => (to_host, to_guest), + }; + let mut writer = SecureWriter { + inner: write, + cipher: send, + counter: 0, + }; + let mut reader = SecureReader { + inner: read, + cipher: recv, + counter: 0, + }; + writer.frame(CONFIRMATION, CONFIRM)?; + writer.inner.flush()?; + match reader.open() { + Ok(Some((CONFIRMATION, p))) if p == CONFIRM => Ok((reader, writer)), + Ok(_) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "no confirmation", + )), + Err(e) if e.kind() == io::ErrorKind::InvalidData => Err(wrong_code()), + Err(e) => Err(e), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{TcpListener, TcpStream}; + + fn pair() -> (TcpStream, TcpStream) { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let a = TcpStream::connect(l.local_addr().unwrap()).unwrap(); + let (b, _) = l.accept().unwrap(); + (a, b) + } + + fn both( + a: impl FnOnce() -> A + Send + 'static, + b: impl FnOnce() -> B + Send + 'static, + ) -> (A, B) { + let t = std::thread::spawn(a); + let rb = b(); + (t.join().unwrap(), rb) + } + + fn keys( + host_secret: &'static str, + guest_secret: &'static str, + ) -> (Key, Key, TcpStream, TcpStream) { + let (mut h, mut g) = pair(); + let (hk, gk) = both( + { + let mut h2 = h.try_clone().unwrap(); + move || exchange(&mut h2, host_secret, "K7QM", 9).unwrap() + }, + { + let mut g2 = g.try_clone().unwrap(); + move || exchange(&mut g2, guest_secret, "K7QM", 9).unwrap() + }, + ); + h.flush().unwrap(); + g.flush().unwrap(); + (hk, gk, h, g) + } + + #[test] + fn same_code_same_key_and_frames_flow() { + let (hk, gk, h, g) = keys("3XPD", "3XPD"); + assert_eq!(hk.0, gk.0); + let ((mut hr, mut hw), (mut gr, mut gw)) = both( + move || secure(h.try_clone().unwrap(), h, &hk, Role::Host).unwrap(), + move || secure(g.try_clone().unwrap(), g, &gk, Role::Guest).unwrap(), + ); + let big: Vec = (0..MAX_PAYLOAD * 2 + 5).map(|i| i as u8).collect(); + gw.write_data(&big).unwrap(); + gw.write_control(b"{\"use\":true}").unwrap(); + hw.write_data(b"pong").unwrap(); + let mut got = Vec::new(); + while got.len() < big.len() { + match hr.read_frame().unwrap().unwrap() { + Frame::Data(d) => got.extend(d), + Frame::Control(_) => panic!("control before data"), + } + } + assert_eq!(got, big); + assert!( + matches!(hr.read_frame().unwrap(), Some(Frame::Control(c)) if c == b"{\"use\":true}") + ); + assert!(matches!(gr.read_frame().unwrap(), Some(Frame::Data(d)) if d == b"pong")); + drop(gw); + drop(gr); + assert!(hr.read_frame().unwrap().is_none()); + } + + #[test] + fn a_wrong_secret_is_refused_on_both_ends() { + let (hk, gk, h, g) = keys("3XPD", "3XPE"); + assert_ne!(hk.0, gk.0); + let (a, b) = both( + move || secure(h.try_clone().unwrap(), h, &hk, Role::Host).err(), + move || secure(g.try_clone().unwrap(), g, &gk, Role::Guest).err(), + ); + assert!(is_wrong_code(&a.unwrap())); + assert!(is_wrong_code(&b.unwrap())); + } + + #[test] + fn the_session_is_part_of_the_key() { + let (mut h, mut g) = pair(); + let (hk, gk) = both( + move || exchange(&mut h, "3XPD", "K7QM", 1).unwrap(), + move || exchange(&mut g, "3XPD", "K7QM", 2).unwrap(), + ); + assert_ne!(hk.0, gk.0); + } + + /// A byte changed in flight, frames replayed or reordered: the reader + /// stops. Built on in-memory buffers to edit the bytes. + #[test] + fn tampering_is_detected() { + let key = Key([7; 32]); + let seal = |frames: &[&[u8]]| { + let mut w = SecureWriter { + inner: Vec::new(), + cipher: cipher(&key, &[1; 32], b"horizon-friends v1 guest->host"), + counter: 0, + }; + for f in frames { + w.write_data(f).unwrap(); + } + w.inner + }; + let reader = |bytes: Vec| SecureReader { + inner: io::Cursor::new(bytes), + cipher: cipher(&key, &[1; 32], b"horizon-friends v1 guest->host"), + counter: 0, + }; + let good = seal(&[b"hello", b"world"]); + let mut r = reader(good.clone()); + assert!(matches!(r.read_frame().unwrap(), Some(Frame::Data(d)) if d == b"hello")); + assert!(matches!(r.read_frame().unwrap(), Some(Frame::Data(d)) if d == b"world")); + assert!(r.read_frame().unwrap().is_none()); + // One flipped bit. + let mut bad = good.clone(); + bad[5] ^= 1; + assert!(reader(bad).read_frame().is_err()); + // Frames swapped: the counter no longer matches. + let first = 2 + usize::from(u16::from_be_bytes([good[0], good[1]])); + let mut swapped = good[first..].to_vec(); + swapped.extend_from_slice(&good[..first]); + assert!(reader(swapped).read_frame().is_err()); + // The first frame twice. + let mut r = reader([&good[..first], &good[..first]].concat()); + r.read_frame().unwrap(); + assert!(r.read_frame().is_err()); + // A header claiming more than a frame can hold. + assert!(reader(vec![0xFF, 0xFF, 0]).read_frame().is_err()); + } +} diff --git a/crates/horizon-friends/src/direct.rs b/crates/horizon-friends/src/direct.rs new file mode 100644 index 000000000..86e155858 --- /dev/null +++ b/crates/horizon-friends/src/direct.rs @@ -0,0 +1,317 @@ +//! Direct connections between a host and a guest: the addresses each end +//! offers, and the race that dials all of the other's at once from the port +//! it also listens on (so two NATs that map a port the same way for every +//! destination let the two SYNs cross: TCP simultaneous open). +use socket2::{Domain, Protocol, SockAddr, Socket, Type}; +use std::{ + io::{self, Read, Write}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener, TcpStream, UdpSocket}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc::Sender, + }, + time::{Duration, Instant}, +}; + +/// How long the ends try to reach each other directly. +pub const RACE: Duration = Duration::from_millis(3000); +const RETRY: Duration = Duration::from_millis(250); +const MAGIC: &[u8; 4] = b"HZF1"; + +fn socket_for(addr: &SocketAddr) -> io::Result { + let s = Socket::new( + Domain::for_address(*addr), + Type::STREAM, + Some(Protocol::TCP), + )?; + s.set_reuse_address(true)?; + #[cfg(all(unix, not(any(target_os = "solaris", target_os = "illumos"))))] + s.set_reuse_port(true)?; + if addr.is_ipv6() { + s.set_only_v6(true)?; + } + Ok(s) +} + +fn unspecified(v6: bool, port: u16) -> SocketAddr { + if v6 { + SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), port) + } else { + SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port) + } +} + +/// The direct socket's listeners: IPv4 on `port` (0 picks one), and IPv6 on +/// the same port when the system has it. +pub fn listen(port: u16) -> io::Result<(Vec, u16)> { + let bind = |addr: SocketAddr| -> io::Result { + let s = socket_for(&addr)?; + s.bind(&SockAddr::from(addr))?; + s.listen(16)?; + let l: TcpListener = s.into(); + l.set_nonblocking(true)?; + Ok(l) + }; + let v4 = bind(unspecified(false, port))?; + let port = v4.local_addr()?.port(); + let mut out = vec![v4]; + if let Ok(v6) = bind(unspecified(true, port)) { + out.push(v6); + } + Ok((out, port)) +} + +/// Connect to `to` from local `port` (shared with the listener), or from any +/// port when that one can't be used (the same pair already connected). +pub fn connect_from(port: Option, to: SocketAddr, timeout: Duration) -> io::Result { + if let Some(port) = port { + let s = socket_for(&to)?; + if s.bind(&SockAddr::from(unspecified(to.is_ipv6(), port))) + .is_ok() + { + match s.connect_timeout(&SockAddr::from(to), timeout) { + Ok(()) => return Ok(s.into()), + Err(e) + if e.kind() != io::ErrorKind::AddrInUse + && e.kind() != io::ErrorKind::AddrNotAvailable => + { + return Err(e); + } + Err(_) => {} + } + } + } + TcpStream::connect_timeout(&to, timeout) +} + +/// The source address the OS would use toward `to` (a UDP socket only picks +/// a route: no packet is sent). +fn route_to(to: SocketAddr) -> Option { + let bind = unspecified(to.is_ipv6(), 0); + let s = UdpSocket::bind(bind).ok()?; + s.connect(to).ok()?; + let ip = s.local_addr().ok()?.ip(); + (!ip.is_unspecified()).then_some(ip) +} + +/// The addresses this machine can be reached on, best first: toward the +/// relay (a LAN or tailnet address when the relay is near), then toward the +/// public IPv4 and IPv6 internet. Loopback only when the relay is local. +pub fn local_addresses(relay: Option) -> Vec { + let mut targets: Vec = relay.into_iter().collect(); + targets.push(SocketAddr::from(([1, 1, 1, 1], 53))); + targets.push(SocketAddr::from(( + [0x2606, 0x4700, 0x4700, 0, 0, 0, 0, 0x1111], + 53, + ))); + let local_relay = relay.is_some_and(|r| r.ip().is_loopback()); + let mut out = Vec::new(); + for t in targets { + if let Some(ip) = route_to(t) + && (local_relay || !ip.is_loopback()) + && !out.contains(&ip) + { + out.push(ip); + } + } + out +} + +/// This end's candidates: each local address with the direct port, then the +/// address the relay saw (when it is not one of them). +pub fn candidates( + port: u16, + relay: Option, + observed: Option, +) -> Vec { + let mut out: Vec = local_addresses(relay) + .into_iter() + .map(|ip| SocketAddr::new(ip, port)) + .collect(); + if let Some(o) = observed + && !o.ip().is_unspecified() + && !out.contains(&o) + { + out.push(o); + } + out +} + +/// Both ends open every direct stream the same way, whoever dialled: the +/// magic and the session, then the other's must match. +pub fn introduce(stream: &mut TcpStream, session: u64) -> io::Result<()> { + let mut hello = [0u8; 12]; + hello[..4].copy_from_slice(MAGIC); + hello[4..].copy_from_slice(&session.to_be_bytes()); + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + stream.write_all(&hello)?; + let mut theirs = [0u8; 12]; + stream.read_exact(&mut theirs)?; + if theirs != hello { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "not this session", + )); + } + Ok(()) +} + +/// The session an inbound direct stream belongs to (the host's listener +/// serves every session): reads the other end's introduction. +pub fn read_session(stream: &mut TcpStream) -> io::Result { + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let mut theirs = [0u8; 12]; + stream.read_exact(&mut theirs)?; + if &theirs[..4] != MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "not a direct stream", + )); + } + Ok(u64::from_be_bytes(theirs[4..].try_into().unwrap())) +} +/// Answer an inbound stream's introduction for `session`. +pub fn answer(stream: &mut TcpStream, session: u64) -> io::Result<()> { + let mut hello = [0u8; 12]; + hello[..4].copy_from_slice(MAGIC); + hello[4..].copy_from_slice(&session.to_be_bytes()); + stream.write_all(&hello) +} + +/// Dial each target from `port` until one connects, the deadline passes or +/// `stop` is set; streams that connect are introduced and sent on. +pub fn dial( + port: Option, + targets: Vec, + session: u64, + deadline: Instant, + stop: Arc, + found: Sender, +) { + for target in targets { + let (stop, found) = (stop.clone(), found.clone()); + let _ = std::thread::Builder::new() + .name("horizon-friends-dial".into()) + .spawn(move || { + while !stop.load(Ordering::Relaxed) { + let now = Instant::now(); + if now >= deadline { + return; + } + let tried = Instant::now(); + let wait = (deadline - now).min(Duration::from_millis(600)); + if let Ok(mut s) = connect_from(port, target, wait) + && introduce(&mut s, session).is_ok() + { + let _ = found.send(s); + return; + } + let spent = tried.elapsed(); + if spent < RETRY { + std::thread::sleep(RETRY - spent); + } + } + }); + } +} + +/// Accept on the listeners until the deadline or `stop`; inbound streams for +/// `session` are answered and sent on. +pub fn accept( + listeners: Vec, + session: u64, + deadline: Instant, + stop: Arc, + found: Sender, +) { + let _ = std::thread::Builder::new() + .name("horizon-friends-accept".into()) + .spawn(move || { + while !stop.load(Ordering::Relaxed) && Instant::now() < deadline { + let mut idle = true; + for l in &listeners { + if let Ok((mut s, _)) = l.accept() { + idle = false; + let _ = s.set_nonblocking(false); + if read_session(&mut s).ok() == Some(session) + && answer(&mut s, session).is_ok() + { + let _ = found.send(s); + } + } + } + if idle { + std::thread::sleep(Duration::from_millis(20)); + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::mpsc::channel; + + #[test] + fn candidates_have_the_port_and_the_observed_address() { + let relay = SocketAddr::from(([127, 0, 0, 1], 1)); + let observed = SocketAddr::from(([203, 0, 113, 5], 40000)); + let c = candidates(5555, Some(relay), Some(observed)); + assert!(c.contains(&SocketAddr::from(([127, 0, 0, 1], 5555)))); + assert_eq!(c.last(), Some(&observed)); + let far = candidates(5555, None, None); + assert!( + far.iter() + .all(|a| !a.ip().is_loopback() && a.port() == 5555) + ); + } + + /// Two direct sockets on loopback: each listens on its port and dials + /// the other's from it; the session's streams meet and wrong sessions + /// don't. + #[test] + fn the_race_connects_and_checks_the_session() { + let (la, pa) = listen(0).unwrap(); + let (lb, pb) = listen(0).unwrap(); + assert_ne!(pa, pb); + let stop = Arc::new(AtomicBool::new(false)); + let deadline = Instant::now() + Duration::from_secs(3); + let (ta, ra) = channel(); + let (tb, rb) = channel(); + accept(la, 42, deadline, stop.clone(), ta.clone()); + accept(lb, 42, deadline, stop.clone(), tb.clone()); + dial( + Some(pa), + vec![SocketAddr::from(([127, 0, 0, 1], pb))], + 42, + deadline, + stop.clone(), + ta, + ); + let mut a = ra.recv_timeout(Duration::from_secs(3)).unwrap(); + let mut b = rb.recv_timeout(Duration::from_secs(3)).unwrap(); + a.write_all(b"ping").unwrap(); + let mut got = [0u8; 4]; + b.read_exact(&mut got).unwrap(); + assert_eq!(&got, b"ping"); + stop.store(true, Ordering::Relaxed); + + let (lc, pc) = listen(0).unwrap(); + let (tc, rc) = channel(); + let deadline = Instant::now() + Duration::from_millis(800); + let stop = Arc::new(AtomicBool::new(false)); + accept(lc, 7, deadline, stop.clone(), tc.clone()); + let (td, rd) = channel(); + dial( + None, + vec![SocketAddr::from(([127, 0, 0, 1], pc))], + 8, + deadline, + stop, + td, + ); + assert!(rc.recv_timeout(Duration::from_secs(2)).is_err()); + assert!(rd.recv_timeout(Duration::from_millis(100)).is_err()); + } +} diff --git a/crates/horizon-friends/src/guest.rs b/crates/horizon-friends/src/guest.rs new file mode 100644 index 000000000..a40361acc --- /dev/null +++ b/crates/horizon-friends/src/guest.rs @@ -0,0 +1,233 @@ +//! The guest's end: find the host behind a code, agree on the key, settle +//! on a direct or relayed stream, then offer it on a loopback port the +//! game's client connects to like to any server. +use crate::{ + Code, PathMode, + crypto::{self, Role}, + direct, resolve_relay, + session::{self, Secure}, + wire::{self, ErrorCode, FromRelay, ToRelay}, +}; +use std::{ + fmt, io, + net::{Ipv4Addr, SocketAddr, TcpListener, TcpStream}, + thread::JoinHandle, + time::Duration, +}; + +pub struct GuestConfig { + pub relay: String, + pub code: Code, + /// The game's protocol number and version name (the relay refuses a + /// guest whose protocol differs from the host's). + pub protocol: i32, + pub game: String, + pub path: PathMode, +} + +/// Why joining failed, in words for the player. +#[derive(Debug)] +pub enum JoinError { + /// No relay is configured in this build. + NotAvailable, + Unreachable(String), + UnknownCode, + Ended, + VersionMismatch { + host: Option, + mine: String, + }, + HostUnreachable, + Full, + RateLimited, + WrongCode, + Failed(String), +} +impl fmt::Display for JoinError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotAvailable => write!(f, "Joining friends is not available yet in this build."), + Self::Unreachable(e) => write!(f, "Couldn't reach the invite service ({e})."), + Self::UnknownCode => write!( + f, + "No world uses this code. Check it, or ask your friend for a new one." + ), + Self::Ended => write!( + f, + "This invite has ended: your friend stopped sharing or left the world." + ), + Self::VersionMismatch { + host: Some(v), + mine, + } => { + write!(f, "Your friend plays Minecraft {v}; you have {mine}.") + } + Self::VersionMismatch { host: None, mine } => { + write!( + f, + "Your friend plays another version of Minecraft than {mine}." + ) + } + Self::HostUnreachable => write!(f, "Your friend's game didn't answer."), + Self::Full => write!( + f, + "Your friend's world is full, or the invite service is busy. Try again in a moment." + ), + Self::RateLimited => { + write!(f, "Too many attempts. Wait a few minutes, then try again.") + } + Self::WrongCode => write!(f, "The code doesn't match. Check the last four characters."), + Self::Failed(e) => write!(f, "Couldn't connect to your friend ({e})."), + } + } +} +impl std::error::Error for JoinError {} + +/// A settled session, ready to carry the game's connection. +pub struct Tunnel { + secure: Secure, + /// Whether the stream goes straight to the host (else through the relay). + pub direct: bool, +} + +/// Find the host, agree on the key and settle on a path. Blocks for up to a +/// few seconds (the direct race). +pub fn join(config: &GuestConfig) -> Result { + let relays = resolve_relay(&config.relay).map_err(|e| JoinError::Unreachable(e.to_string()))?; + let direct_allowed = config.path != PathMode::RelayOnly; + let (listeners, port) = if direct_allowed { + match direct::listen(0) { + Ok((l, p)) => (l, Some(p)), + Err(_) => (Vec::new(), None), + } + } else { + (Vec::new(), None) + }; + let mut last = io::Error::other("no address"); + let mut stream = None; + for relay in &relays { + match direct::connect_from(port, *relay, Duration::from_secs(5)) { + Ok(s) => { + stream = Some((s, *relay)); + break; + } + Err(e) => last = e, + } + } + let (mut stream, relay) = stream.ok_or_else(|| JoinError::Unreachable(last.to_string()))?; + let failed = |e: io::Error| JoinError::Failed(e.to_string()); + stream + .set_read_timeout(Some(Duration::from_secs(15))) + .map_err(failed)?; + let _ = stream.set_nodelay(true); + wire::send( + &mut stream, + &ToRelay::Join { + v: wire::VERSION, + nameplate: config.code.nameplate.clone(), + protocol: config.protocol, + game: config.game.clone(), + }, + ) + .map_err(failed)?; + let (session, observed) = match wire::recv::<_, FromRelay>(&mut stream).map_err(failed)? { + FromRelay::Bridged { session, observed } => (session, observed), + FromRelay::Error { code, game } => { + return Err(match code { + ErrorCode::UnknownCode => JoinError::UnknownCode, + ErrorCode::Ended => JoinError::Ended, + ErrorCode::VersionMismatch => JoinError::VersionMismatch { + host: game, + mine: config.game.clone(), + }, + ErrorCode::HostUnreachable => JoinError::HostUnreachable, + ErrorCode::Full => JoinError::Full, + ErrorCode::RateLimited => JoinError::RateLimited, + ErrorCode::BadRequest => { + JoinError::Failed("the invite service refused the request".into()) + } + }); + } + other => return Err(JoinError::Failed(format!("unexpected answer {other:?}"))), + }; + let key = crypto::exchange( + &mut stream, + &config.code.secret, + &config.code.nameplate, + session, + ) + .map_err(|_| JoinError::HostUnreachable)?; + let relayed = session::secure(stream, &key, Role::Guest).map_err(|e| { + if crypto::is_wrong_code(&e) { + JoinError::WrongCode + } else { + JoinError::HostUnreachable + } + })?; + let observed = observed.and_then(|o| o.parse::().ok()); + let candidates = port.map_or_else(Vec::new, |p| direct::candidates(p, Some(relay), observed)); + let settled = session::settle( + relayed, + &key, + Role::Guest, + config.path, + session, + port, + candidates, + Box::new(move |tx, deadline, stop| direct::accept(listeners, session, deadline, stop, tx)), + ) + .map_err(|e| JoinError::Failed(e.to_string()))?; + Ok(Tunnel { + secure: settled.secure, + direct: settled.direct, + }) +} + +impl Tunnel { + /// Offer the session on `127.0.0.1:port` for one connection (the game's + /// client); the returned thread ends when that connection does. + pub fn serve_loopback(self) -> io::Result<(u16, JoinHandle<()>)> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0))?; + let port = listener.local_addr()?.port(); + listener.set_nonblocking(true)?; + let thread = std::thread::Builder::new() + .name("horizon-friends-guest".into()) + .spawn(move || { + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let plain = loop { + match listener.accept() { + Ok((s, from)) if from.ip().is_loopback() => break Some(s), + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + if std::time::Instant::now() > deadline { + break None; + } + std::thread::sleep(Duration::from_millis(10)); + } + Err(_) => break None, + } + }; + drop(listener); + let (reader, writer) = self.secure; + let Some(plain) = plain else { + let _ = reader.get_ref().shutdown(std::net::Shutdown::Both); + return; + }; + let _ = plain.set_nonblocking(false); + if let Ok(t) = crate::pump(plain, reader, writer) { + let _ = t.join(); + } + })?; + Ok((port, thread)) + } + /// The protected stream itself (tests, or a client that reads it + /// directly). + pub fn into_stream( + self, + ) -> ( + crypto::SecureReader, + crypto::SecureWriter, + ) { + self.secure + } +} diff --git a/crates/horizon-friends/src/host.rs b/crates/horizon-friends/src/host.rs new file mode 100644 index 000000000..567d8ff42 --- /dev/null +++ b/crates/horizon-friends/src/host.rs @@ -0,0 +1,401 @@ +//! The host's end: registers with the relay for a code and keeps that +//! control connection open; for each guest the relay announces, attaches a +//! new connection, agrees on the key, settles on a path and connects the +//! session to the integrated server's port. Everything runs on its own +//! threads; the game polls [`Host::poll`] and [`Host::state`]. +use crate::{ + Code, PathMode, + crypto::{self, Role}, + direct, resolve_relay, session, + wire::{self, FromRelay, ToRelay}, +}; +use std::{ + collections::HashMap, + io::{self, BufReader}, + net::{Shutdown, SocketAddr, TcpStream}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU32, Ordering}, + mpsc::{Receiver, Sender, channel}, + }, + thread::JoinHandle, + time::{Duration, Instant}, +}; + +/// Wrong codes tried before the host retires its code for a new one. +pub const MAX_WRONG: u32 = 5; +/// Guests that can be connecting at once (IntegratedServer.MAX_PLAYERS). +pub const MAX_SESSIONS: usize = 8; + +#[derive(Clone)] +pub struct HostConfig { + pub relay: String, + /// The integrated server's port (Open to LAN's). + pub lan: SocketAddr, + pub protocol: i32, + pub game: String, + pub path: PathMode, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum HostState { + /// Waiting for the relay's answer. + Registering, + Sharing(Code), + /// The relay is unreachable or refused; retrying. + Retrying(String), + Stopped, +} + +#[derive(Debug)] +pub enum HostEvent { + /// A new code (the first, or after the relay restarted or wrong codes). + Code(Code), + /// A guest's session is carried directly or through the relay. + Connected { + session: u64, + direct: bool, + }, + Disconnected { + session: u64, + }, + /// Someone used the nameplate with the wrong secret. + WrongCode { + count: u32, + }, +} + +struct Shared { + config: HostConfig, + stop: AtomicBool, + state: Mutex, + events: Mutex>, + /// The current control connection (shut down to stop). + control: Mutex>, + /// Inbound direct streams waiting for their session. + routes: Mutex, Instant)>>, + sessions: Mutex, + wrong: AtomicU32, +} +impl Shared { + fn emit(&self, e: HostEvent) { + let _ = self.events.lock().unwrap().send(e); + } + fn set(&self, s: HostState) { + *self.state.lock().unwrap() = s; + } + fn stopped(&self) -> bool { + self.stop.load(Ordering::Relaxed) + } +} + +pub struct Host { + shared: Arc, + events: Receiver, + thread: Option>, + accept: Option>, +} + +impl Host { + /// Start sharing in the background: the code arrives as an event. + pub fn start(config: HostConfig) -> io::Result { + let (tx, events) = channel(); + let listeners = if config.path == PathMode::RelayOnly { + None + } else { + direct::listen(0).ok() + }; + let shared = Arc::new(Shared { + config, + stop: AtomicBool::new(false), + state: Mutex::new(HostState::Registering), + events: Mutex::new(tx), + control: Mutex::new(None), + routes: Mutex::new(HashMap::new()), + sessions: Mutex::new(0), + wrong: AtomicU32::new(0), + }); + let port = listeners.as_ref().map(|(_, p)| *p); + let accept = match listeners { + Some((l, _)) => { + let s = shared.clone(); + Some( + std::thread::Builder::new() + .name("horizon-friends-listen".into()) + .spawn(move || route_inbound(l, s))?, + ) + } + None => None, + }; + let s = shared.clone(); + let thread = std::thread::Builder::new() + .name("horizon-friends-host".into()) + .spawn(move || run(s, port))?; + Ok(Self { + shared, + events, + thread: Some(thread), + accept, + }) + } + pub fn state(&self) -> HostState { + self.shared.state.lock().unwrap().clone() + } + pub fn poll(&self) -> Vec { + self.events.try_iter().collect() + } + /// Guests whose sessions are open (connecting or playing). + pub fn sessions(&self) -> usize { + *self.shared.sessions.lock().unwrap() + } + /// End the code. Sessions already playing go on until the integrated + /// server closes them (stopping the LAN server ends them). + pub fn stop(&mut self) { + self.shared.stop.store(true, Ordering::Relaxed); + if let Some(c) = self.shared.control.lock().unwrap().take() { + let _ = c.shutdown(Shutdown::Both); + } + if let Some(t) = self.thread.take() { + let _ = t.join(); + } + if let Some(t) = self.accept.take() { + let _ = t.join(); + } + self.shared.set(HostState::Stopped); + } +} +impl Drop for Host { + fn drop(&mut self) { + self.stop(); + } +} + +/// Hand inbound direct streams to the session they introduce themselves for. +fn route_inbound(listeners: Vec, shared: Arc) { + while !shared.stopped() { + let mut idle = true; + for l in &listeners { + let Ok((mut s, _)) = l.accept() else { continue }; + idle = false; + let shared = shared.clone(); + let _ = std::thread::Builder::new() + .name("horizon-friends-inbound".into()) + .spawn(move || { + let _ = s.set_nonblocking(false); + let Ok(session) = direct::read_session(&mut s) else { + return; + }; + let route = { + let mut routes = shared.routes.lock().unwrap(); + routes.retain(|_, (_, until)| *until > Instant::now()); + routes.get(&session).map(|(tx, _)| tx.clone()) + }; + if let Some(tx) = route + && direct::answer(&mut s, session).is_ok() + { + let _ = tx.send(s); + } + }); + } + if idle { + std::thread::sleep(Duration::from_millis(20)); + } + } +} + +fn connect_relay(config: &HostConfig, port: Option) -> io::Result<(TcpStream, SocketAddr)> { + let mut last = io::Error::other("no address"); + for relay in resolve_relay(&config.relay)? { + match direct::connect_from(port, relay, Duration::from_secs(5)) { + Ok(s) => { + let _ = s.set_nodelay(true); + return Ok((s, relay)); + } + Err(e) => last = e, + } + } + Err(last) +} + +/// Register, serve the control connection, register again when it drops +/// (a new code) until stopped. +fn run(shared: Arc, port: Option) { + let mut backoff = Duration::from_secs(1); + while !shared.stopped() { + match register(&shared, port) { + Ok(()) => backoff = Duration::from_secs(1), + Err(e) => { + if shared.stopped() { + break; + } + shared.set(HostState::Retrying(e.to_string())); + let until = Instant::now() + backoff; + while Instant::now() < until && !shared.stopped() { + std::thread::sleep(Duration::from_millis(50)); + } + backoff = (backoff * 2).min(Duration::from_secs(30)); + } + } + } +} + +fn register(shared: &Arc, port: Option) -> io::Result<()> { + let config = &shared.config; + // The control connection uses any port: the direct port stays free for + // the attach connections, whose observed address is the one that counts. + let (mut control, relay) = connect_relay(config, None)?; + control.set_read_timeout(Some(Duration::from_secs(70)))?; + wire::send( + &mut control, + &ToRelay::Host { + v: wire::VERSION, + protocol: config.protocol, + game: config.game.clone(), + }, + )?; + let mut reader = BufReader::new(control.try_clone()?); + let (nameplate, token) = match wire::recv_buffered::<_, FromRelay>(&mut reader)? { + FromRelay::Registered { + nameplate, token, .. + } => (nameplate, token), + FromRelay::Error { code, .. } => { + return Err(io::Error::other(format!( + "the invite service refused: {}", + code.as_str() + ))); + } + other => return Err(io::Error::other(format!("unexpected answer {other:?}"))), + }; + let code = Code::new(&nameplate)?; + *shared.control.lock().unwrap() = Some(control.try_clone()?); + if shared.stopped() { + let _ = control.shutdown(Shutdown::Both); + return Ok(()); + } + shared.wrong.store(0, Ordering::Relaxed); + shared.set(HostState::Sharing(code.clone())); + shared.emit(HostEvent::Code(code.clone())); + loop { + let message = match wire::recv_buffered::<_, FromRelay>(&mut reader) { + Ok(m) => m, + Err(e) => { + shared.control.lock().unwrap().take(); + return if shared.stopped() || shared.wrong.load(Ordering::Relaxed) >= MAX_WRONG { + Ok(()) + } else { + Err(e) + }; + } + }; + match message { + FromRelay::Ping => wire::send(&mut control, &ToRelay::Pong)?, + FromRelay::Join { session } => { + let mut open = shared.sessions.lock().unwrap(); + if *open >= MAX_SESSIONS { + continue; + } + *open += 1; + drop(open); + let (shared, code, token, control) = ( + shared.clone(), + code.clone(), + token.clone(), + control.try_clone()?, + ); + let _ = std::thread::Builder::new() + .name("horizon-friends-session".into()) + .spawn(move || { + serve(&shared, &code, &token, session, port, relay, &control); + *shared.sessions.lock().unwrap() -= 1; + }); + } + _ => {} + } + } +} + +/// One guest: attach, agree on the key, settle, connect to the integrated +/// server, carry the session until it ends. +fn serve( + shared: &Arc, + code: &Code, + token: &str, + session: u64, + port: Option, + relay: SocketAddr, + control: &TcpStream, +) { + let config = &shared.config; + let Ok((mut stream, _)) = connect_relay(config, port) else { + return; + }; + let _ = stream.set_read_timeout(Some(Duration::from_secs(15))); + let attach = ToRelay::Attach { + v: wire::VERSION, + session, + token: token.to_owned(), + }; + if wire::send(&mut stream, &attach).is_err() { + return; + } + let observed = match wire::recv::<_, FromRelay>(&mut stream) { + Ok(FromRelay::Bridged { observed, .. }) => { + observed.and_then(|o| o.parse::().ok()) + } + _ => return, + }; + let Ok(key) = crypto::exchange(&mut stream, &code.secret, &code.nameplate, session) else { + return; + }; + let relayed = match session::secure(stream, &key, Role::Host) { + Ok(s) => s, + Err(e) => { + if crypto::is_wrong_code(&e) { + let count = shared.wrong.fetch_add(1, Ordering::Relaxed) + 1; + shared.emit(HostEvent::WrongCode { count }); + if count >= MAX_WRONG { + // Retire the code: dropping the control connection ends + // it at the relay, and `run` registers a new one. + let _ = control.shutdown(Shutdown::Both); + } + } + return; + } + }; + let candidates = port.map_or_else(Vec::new, |p| direct::candidates(p, Some(relay), observed)); + let shared_routes = shared.clone(); + let settled = session::settle( + relayed, + &key, + Role::Host, + config.path, + session, + port, + candidates, + Box::new(move |tx, deadline, _stop| { + shared_routes + .routes + .lock() + .unwrap() + .insert(session, (tx, deadline + Duration::from_millis(500))); + }), + ); + shared.routes.lock().unwrap().remove(&session); + let Ok(settled) = settled else { + return; + }; + let Ok(server) = TcpStream::connect_timeout(&config.lan, Duration::from_secs(5)) else { + let (r, _) = settled.secure; + let _ = r.get_ref().shutdown(Shutdown::Both); + return; + }; + let (r, w) = settled.secure; + shared.emit(HostEvent::Connected { + session, + direct: settled.direct, + }); + if let Ok(t) = crate::pump(server, r, w) { + let _ = t.join(); + } + shared.emit(HostEvent::Disconnected { session }); +} diff --git a/crates/horizon-friends/src/lib.rs b/crates/horizon-friends/src/lib.rs new file mode 100644 index 000000000..9c86b10fa --- /dev/null +++ b/crates/horizon-friends/src/lib.rs @@ -0,0 +1,124 @@ +//! Play with friends: a host shares the world it opened to LAN behind an +//! invite code; guests join with the code, directly when the network allows +//! and through the relay otherwise. Both ends carry the game's TCP stream +//! unchanged between loopback ports (the integrated server on the host, a +//! local port the guest's client connects to), end-to-end encrypted with a +//! key only the holders of the code can derive. The design, the protocol and +//! what the relay sees are in docs/play-with-friends.md. +pub mod code; +pub mod crypto; +pub mod direct; +pub mod guest; +pub mod host; +mod session; +pub mod wire; + +pub use code::Code; +pub use guest::{JoinError, Tunnel}; +pub use host::{Host, HostConfig, HostEvent, HostState}; + +use std::{ + io, + net::{SocketAddr, TcpStream, ToSocketAddrs}, + thread::JoinHandle, +}; + +/// Fill `buf` from the OS random generator. +pub(crate) fn fill_random(buf: &mut [u8]) -> io::Result<()> { + getrandom::fill(buf).map_err(|e| io::Error::other(e.to_string())) +} + +/// Which paths a session may take (`HORIZON_FRIENDS_PATH`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PathMode { + /// Direct when the ends reach each other, else the relay. + Auto, + /// Never connect directly: the other end doesn't learn this one's address. + RelayOnly, + /// Never fall back to the relay (tests). + DirectOnly, +} +impl PathMode { + pub fn from_env() -> Self { + match std::env::var("HORIZON_FRIENDS_PATH").as_deref() { + Ok("relay") => Self::RelayOnly, + Ok("direct") => Self::DirectOnly, + _ => Self::Auto, + } + } +} + +/// The relay's address: `HORIZON_RELAY_URL` at run time, else the one +/// compiled in from the same variable. None until a relay is deployed. +pub fn relay_url() -> Option { + let runtime = std::env::var("HORIZON_RELAY_URL").ok(); + let compiled = option_env!("HORIZON_RELAY_URL").map(str::to_owned); + runtime + .into_iter() + .chain(compiled) + .map(|s| s.trim().to_owned()) + .find(|s| !s.is_empty()) +} + +/// `host:port`, optionally written `tcp://host:port`. +pub fn resolve_relay(url: &str) -> io::Result> { + let address = url + .trim() + .trim_start_matches("tcp://") + .trim_end_matches('/'); + let found: Vec = address.to_socket_addrs()?.collect(); + if found.is_empty() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "the relay's name didn't resolve", + )); + } + Ok(found) +} + +/// Carry one connection both ways between a plain stream (the game's) and a +/// protected one, until either side ends; then both are closed. +pub fn pump( + plain: TcpStream, + mut reader: crypto::SecureReader, + mut writer: crypto::SecureWriter, +) -> io::Result> { + use io::{Read, Write}; + for s in [&plain, reader.get_ref(), writer.get_ref()] { + s.set_read_timeout(None)?; + let _ = s.set_nodelay(true); + } + let mut plain_in = plain.try_clone()?; + let mut plain_out = plain; + let outbound = std::thread::Builder::new() + .name("horizon-friends-out".into()) + .spawn(move || { + let mut buf = vec![0u8; crypto::MAX_PAYLOAD]; + loop { + match plain_in.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => { + if writer.write_data(&buf[..n]).is_err() { + break; + } + } + } + } + let _ = writer.get_ref().shutdown(std::net::Shutdown::Both); + let _ = plain_in.shutdown(std::net::Shutdown::Both); + })?; + std::thread::Builder::new() + .name("horizon-friends-in".into()) + .spawn(move || { + while let Ok(Some(frame)) = reader.read_frame() { + if let crypto::Frame::Data(d) = frame + && plain_out.write_all(&d).is_err() + { + break; + } + } + let _ = plain_out.shutdown(std::net::Shutdown::Both); + let _ = reader.get_ref().shutdown(std::net::Shutdown::Both); + let _ = outbound.join(); + }) +} diff --git a/crates/horizon-friends/src/session.rs b/crates/horizon-friends/src/session.rs new file mode 100644 index 000000000..389aa85df --- /dev/null +++ b/crates/horizon-friends/src/session.rs @@ -0,0 +1,195 @@ +//! What both ends do once the relay has bridged them and the key is agreed: +//! protect the relayed stream, swap candidates over it, race direct streams +//! for [`direct::RACE`] and settle on one. The guest decides and says so; +//! the host follows. +use crate::{ + PathMode, + crypto::{self, Frame, Key, Role, SecureReader, SecureWriter}, + direct, +}; +use serde::{Deserialize, Serialize}; +use std::{ + io, + net::{SocketAddr, TcpStream}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc::{Sender, channel}, + }, + time::{Duration, Instant}, +}; + +pub(crate) type Secure = (SecureReader, SecureWriter); + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "t", rename_all = "snake_case")] +enum Control { + /// Where this end can be reached; `direct` false when it won't try. + Hello { + direct: bool, + candidates: Vec, + }, + /// Sent by the guest as the first frame of the direct stream it keeps. + Use, + /// The guest's decision, on the relayed stream. + Path { direct: bool }, +} + +fn send(w: &mut SecureWriter, c: &Control) -> io::Result<()> { + w.write_control(&serde_json::to_vec(c).map_err(io::Error::other)?) +} +fn recv(r: &mut SecureReader) -> io::Result { + match r.read_frame()? { + Some(Frame::Control(c)) => { + serde_json::from_slice(&c).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) + } + Some(Frame::Data(_)) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "data before the path", + )), + None => Err(io::ErrorKind::UnexpectedEof.into()), + } +} + +pub(crate) fn secure(stream: TcpStream, key: &Key, role: Role) -> io::Result { + crypto::secure(stream.try_clone()?, stream, key, role) +} + +pub(crate) struct Established { + pub secure: Secure, + pub direct: bool, +} + +/// Start accepting this session's inbound direct streams (already +/// introduced) into the sender until the deadline or the stop flag. +pub(crate) type Listen<'a> = Box, Instant, Arc) + 'a>; + +/// Settle a session on the relayed stream `relay` (already protected) or a +/// direct one. +pub(crate) fn settle( + relay: Secure, + key: &Key, + role: Role, + mode: PathMode, + session: u64, + direct_port: Option, + candidates: Vec, + listen: Listen<'_>, +) -> io::Result { + let (mut rr, mut rw) = relay; + rr.get_ref() + .set_read_timeout(Some(Duration::from_secs(15)))?; + let wants = mode != PathMode::RelayOnly && direct_port.is_some(); + send( + &mut rw, + &Control::Hello { + direct: wants, + candidates: if wants { + candidates.iter().map(SocketAddr::to_string).collect() + } else { + Vec::new() + }, + }, + )?; + let Control::Hello { + direct: they_want, + candidates: theirs, + } = recv(&mut rr)? + else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "expected hello")); + }; + let theirs: Vec = theirs + .iter() + .filter_map(|a| a.parse().ok()) + .take(16) + .collect(); + let stop = Arc::new(AtomicBool::new(false)); + let (atx, arx) = channel::(); + if wants && they_want { + let deadline = Instant::now() + direct::RACE; + let (raw_tx, raw_rx) = channel::(); + listen(raw_tx.clone(), deadline, stop.clone()); + direct::dial(direct_port, theirs, session, deadline, stop.clone(), raw_tx); + let key = key.clone(); + let _ = std::thread::Builder::new() + .name("horizon-friends-auth".into()) + .spawn(move || { + for raw in raw_rx { + let (key, atx) = (key.clone(), atx.clone()); + let _ = std::thread::Builder::new() + .name("horizon-friends-auth".into()) + .spawn(move || { + let Ok(_) = raw.set_read_timeout(Some(Duration::from_secs(6))) else { + return; + }; + let Ok(mut s) = secure(raw, &key, role) else { + return; + }; + // The host only keeps the stream the guest marks. + if role == Role::Host && !matches!(recv(&mut s.0), Ok(Control::Use)) { + return; + } + let _ = atx.send(s); + }); + } + }); + } else { + drop(atx); + } + let finish = |(r, w): Secure, direct: bool| -> io::Result { + r.get_ref().set_read_timeout(None)?; + Ok(Established { + secure: (r, w), + direct, + }) + }; + let fallback = |rr: SecureReader, rw: SecureWriter| { + if mode == PathMode::DirectOnly { + Err(io::Error::new( + io::ErrorKind::TimedOut, + "no direct connection", + )) + } else { + finish((rr, rw), false) + } + }; + match role { + Role::Guest => { + let chosen = if wants && they_want { + let left = direct::RACE + Duration::from_millis(500); + arx.recv_timeout(left).ok() + } else { + None + }; + stop.store(true, Ordering::Relaxed); + if let Some((r, mut w)) = chosen + && send(&mut w, &Control::Use).is_ok() + { + send(&mut rw, &Control::Path { direct: true })?; + let _ = rr.get_ref().shutdown(std::net::Shutdown::Both); + return finish((r, w), true); + } + send(&mut rw, &Control::Path { direct: false })?; + fallback(rr, rw) + } + Role::Host => { + let Control::Path { direct } = recv(&mut rr)? else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "expected the path", + )); + }; + if direct { + let chosen = arx.recv_timeout(Duration::from_secs(5)); + stop.store(true, Ordering::Relaxed); + let _ = rr.get_ref().shutdown(std::net::Shutdown::Both); + let chosen = chosen.map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "the direct stream vanished") + })?; + return finish(chosen, true); + } + stop.store(true, Ordering::Relaxed); + fallback(rr, rw) + } + } +} diff --git a/crates/horizon-friends/src/wire.rs b/crates/horizon-friends/src/wire.rs new file mode 100644 index 000000000..5991bdf2f --- /dev/null +++ b/crates/horizon-friends/src/wire.rs @@ -0,0 +1,206 @@ +//! The relay protocol: one JSON object per line until a connection is +//! bridged, then the raw session stream (see docs/play-with-friends.md). +use serde::{Deserialize, Serialize}; +use std::io::{self, BufRead, Read, Write}; + +/// Protocol version both the relay and the clients speak. +pub const VERSION: u32 = 1; +/// The longest line either side accepts. +pub const MAX_LINE: usize = 4096; + +/// Lines a client sends. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum ToRelay { + /// Register a world; the connection stays open as the host's control + /// connection. + Host { + v: u32, + protocol: i32, + game: String, + }, + /// Join the world behind a nameplate. + Join { + v: u32, + nameplate: String, + protocol: i32, + game: String, + }, + /// The host's new connection for a session the relay announced. + Attach { + v: u32, + session: u64, + token: String, + }, + Pong, +} + +/// Lines the relay sends. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum FromRelay { + Registered { + nameplate: String, + token: String, + /// The address the relay saw the connection come from. + observed: Option, + }, + /// A guest is joining: attach a new connection for the session. + Join { + session: u64, + }, + /// The connection now carries the session: raw bytes from here on. + Bridged { + session: u64, + observed: Option, + }, + Ping, + Error { + code: ErrorCode, + /// The host's game version, for `version_mismatch`. + #[serde(default, skip_serializing_if = "Option::is_none")] + game: Option, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ErrorCode { + UnknownCode, + Ended, + VersionMismatch, + HostUnreachable, + Full, + RateLimited, + BadRequest, +} +impl ErrorCode { + pub fn as_str(self) -> &'static str { + match self { + Self::UnknownCode => "unknown_code", + Self::Ended => "ended", + Self::VersionMismatch => "version_mismatch", + Self::HostUnreachable => "host_unreachable", + Self::Full => "full", + Self::RateLimited => "rate_limited", + Self::BadRequest => "bad_request", + } + } +} + +/// Write one message as a line. +pub fn send(w: &mut W, message: &T) -> io::Result<()> { + let mut line = serde_json::to_vec(message).map_err(io::Error::other)?; + line.push(b'\n'); + w.write_all(&line)?; + w.flush() +} + +/// Read one line of at most [`MAX_LINE`] bytes, a byte at a time so nothing +/// after it is consumed: the bytes that follow `bridged` belong to the +/// session. None at a clean end of stream. +pub fn read_line(r: &mut R) -> io::Result>> { + let mut line = Vec::new(); + let mut byte = [0u8; 1]; + loop { + match r.read(&mut byte) { + Ok(0) if line.is_empty() => return Ok(None), + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), + Ok(_) if byte[0] == b'\n' => return Ok(Some(line)), + Ok(_) => { + if line.len() == MAX_LINE { + return Err(io::Error::new(io::ErrorKind::InvalidData, "line too long")); + } + line.push(byte[0]); + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } +} + +/// Read and decode one message; an end of stream is an error. +pub fn recv Deserialize<'de>>(r: &mut R) -> io::Result { + let line = read_line(r)?.ok_or(io::ErrorKind::UnexpectedEof)?; + serde_json::from_slice(&line).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +} + +/// The same over a buffered reader (the host's control connection, where +/// nothing follows the lines). +pub fn recv_buffered Deserialize<'de>>(r: &mut R) -> io::Result { + let mut line = Vec::new(); + let n = r.take(MAX_LINE as u64 + 1).read_until(b'\n', &mut line)?; + if n == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + if line.last() != Some(&b'\n') { + return Err(io::Error::new(io::ErrorKind::InvalidData, "line too long")); + } + serde_json::from_slice(&line).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lines_round_trip() { + let mut buf = Vec::new(); + send( + &mut buf, + &ToRelay::Join { + v: VERSION, + nameplate: "K7QM".into(), + protocol: 777, + game: "26.3".into(), + }, + ) + .unwrap(); + assert_eq!( + String::from_utf8(buf.clone()).unwrap(), + "{\"op\":\"join\",\"v\":1,\"nameplate\":\"K7QM\",\"protocol\":777,\"game\":\"26.3\"}\n" + ); + send( + &mut buf, + &FromRelay::Error { + code: ErrorCode::VersionMismatch, + game: Some("26.2".into()), + }, + ) + .unwrap(); + buf.extend_from_slice(b"raw"); + let mut r = &buf[..]; + let first: ToRelay = recv(&mut r).unwrap(); + assert!(matches!(first, ToRelay::Join { protocol: 777, .. })); + let second: FromRelay = recv(&mut r).unwrap(); + assert_eq!( + second, + FromRelay::Error { + code: ErrorCode::VersionMismatch, + game: Some("26.2".into()) + } + ); + // Nothing after the line was consumed. + assert_eq!(r, b"raw"); + } + + #[test] + fn long_lines_and_garbage_are_refused() { + let long = vec![b'a'; MAX_LINE + 1]; + assert!(read_line(&mut &long[..]).is_err()); + let mut exact = vec![b'a'; MAX_LINE]; + exact.push(b'\n'); + assert_eq!(read_line(&mut &exact[..]).unwrap().unwrap().len(), MAX_LINE); + assert!(recv::<_, ToRelay>(&mut &b"{\"op\":\"nope\"}\n"[..]).is_err()); + assert!(recv::<_, ToRelay>(&mut &b""[..]).is_err()); + assert!(read_line(&mut &b""[..]).unwrap().is_none()); + assert!(read_line(&mut &b"partial"[..]).is_err()); + let mut buffered = io::BufReader::new(&long[..]); + assert!(recv_buffered::<_, ToRelay>(&mut buffered).is_err()); + let mut buffered = io::BufReader::new(&b"{\"op\":\"pong\"}\n"[..]); + assert_eq!( + recv_buffered::<_, ToRelay>(&mut buffered).unwrap(), + ToRelay::Pong + ); + } +} diff --git a/crates/horizon-relay/Cargo.toml b/crates/horizon-relay/Cargo.toml new file mode 100644 index 000000000..f6d3edb40 --- /dev/null +++ b/crates/horizon-relay/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "horizon-relay" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "Play with friends' relay: hands out invite codes, introduces hosts and guests, relays the encrypted stream when a direct connection fails" + +[dependencies] +horizon-friends = { path = "../horizon-friends" } +getrandom = "0.3" +serde_json = "1.0.151" + +[[bin]] +name = "horizon-relay" +path = "src/main.rs" + +[lints] +workspace = true diff --git a/crates/horizon-relay/Dockerfile b/crates/horizon-relay/Dockerfile new file mode 100644 index 000000000..7cae743e4 --- /dev/null +++ b/crates/horizon-relay/Dockerfile @@ -0,0 +1,15 @@ +# The Play with friends relay (docs/play-with-friends.md, "Deploying the relay"). +# Build context: the repository root (the workspace's manifests and lockfile): +# docker build -f crates/horizon-relay/Dockerfile -t horizon-relay . +FROM rust:1-slim-bookworm AS build +WORKDIR /src +COPY . . +RUN cargo build --release --locked -p horizon-relay \ + && cp target/release/horizon-relay /horizon-relay + +FROM debian:bookworm-slim +COPY --from=build /horizon-relay /usr/local/bin/horizon-relay +ENV PORT=7000 +EXPOSE 7000 +USER nobody +CMD ["horizon-relay"] diff --git a/crates/horizon-relay/railway.toml b/crates/horizon-relay/railway.toml new file mode 100644 index 000000000..b6916e4d4 --- /dev/null +++ b/crates/horizon-relay/railway.toml @@ -0,0 +1,13 @@ +# Railway service for the Play with friends relay. Set the service's config +# file path to crates/horizon-relay/railway.toml (docs/play-with-friends.md). +[build] +builder = "DOCKERFILE" +dockerfilePath = "crates/horizon-relay/Dockerfile" +watchPatterns = ["crates/horizon-relay/**", "crates/horizon-friends/**", "Cargo.lock"] + +[deploy] +healthcheckPath = "/health" +healthcheckTimeout = 30 +restartPolicyType = "ON_FAILURE" +restartPolicyMaxRetries = 10 +numReplicas = 1 diff --git a/crates/horizon-relay/src/lib.rs b/crates/horizon-relay/src/lib.rs new file mode 100644 index 000000000..48b9ac50e --- /dev/null +++ b/crates/horizon-relay/src/lib.rs @@ -0,0 +1,652 @@ +//! Play with friends' relay. Hosts register and get a nameplate (the +//! relay-visible half of an invite code); guests resolve it; the relay +//! pairs each guest's connection with a fresh one from the host and copies +//! bytes between them. Those bytes are the end-to-end encrypted session +//! (docs/play-with-friends.md): the relay can't read them, only count them. +//! State is in memory; nothing about players is logged. +pub mod limits; + +use horizon_friends::{ + code, + wire::{self, ErrorCode, FromRelay, ToRelay}, +}; +use limits::{Bucket, Limiter, Rates}; +use std::{ + collections::HashMap, + io::{self, Read, Write}, + net::{Shutdown, SocketAddr, TcpListener, TcpStream}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + mpsc::{Receiver, RecvTimeoutError, Sender, SyncSender, channel, sync_channel}, + }, + thread::JoinHandle, + time::{Duration, Instant}, +}; + +#[derive(Clone, Debug)] +pub struct Config { + pub max_hosts: usize, + pub max_bridges: usize, + pub max_pending_per_host: usize, + /// Bytes a second each way per relayed stream. + pub session_rate: f64, + /// Bytes a second for everything relayed. + pub total_rate: f64, + pub rates: Rates, + /// Count limits per client address (off when every client comes from + /// the same proxy address). + pub per_address: bool, + /// Unknown codes a minute across all clients, and the burst. + pub global_unknown_per_min: f64, + pub global_unknown_burst: f64, + /// Read a PROXY protocol v1 line before anything else. + pub proxy_protocol: bool, + pub idle: Duration, + pub ping_every: Duration, + pub host_timeout: Duration, + pub first_line: Duration, + pub attach_wait: Duration, + pub ended_kept: Duration, +} +impl Default for Config { + fn default() -> Self { + Self { + max_hosts: 1000, + max_bridges: 200, + max_pending_per_host: 8, + session_rate: 1024.0 * 1024.0, + total_rate: 100.0 * 1_000_000.0 / 8.0, + rates: Rates::default(), + per_address: true, + global_unknown_per_min: 120.0, + global_unknown_burst: 30.0, + proxy_protocol: false, + idle: Duration::from_secs(120), + ping_every: Duration::from_secs(20), + host_timeout: Duration::from_secs(60), + first_line: Duration::from_secs(10), + attach_wait: Duration::from_secs(10), + ended_kept: Duration::from_secs(15 * 60), + } + } +} +impl Config { + /// The defaults with the `RELAY_*` environment variables applied. + pub fn from_env() -> Self { + fn var(name: &str) -> Option { + std::env::var(name).ok()?.trim().parse().ok() + } + let mut c = Self::default(); + if let Some(v) = var("RELAY_MAX_HOSTS") { + c.max_hosts = v; + } + if let Some(v) = var("RELAY_MAX_BRIDGES") { + c.max_bridges = v; + } + if let Some(v) = var("RELAY_MAX_PENDING_PER_HOST") { + c.max_pending_per_host = v; + } + if let Some(v) = var::("RELAY_SESSION_KBPS") { + c.session_rate = v * 1024.0; + } + if let Some(v) = var::("RELAY_TOTAL_MBPS") { + c.total_rate = v * 1_000_000.0 / 8.0; + } + if let Some(v) = var::("RELAY_GLOBAL_UNKNOWN_PER_MIN") { + c.global_unknown_per_min = v; + } + if let Some(v) = var::("RELAY_PER_ADDRESS_LIMITS") { + c.per_address = v != 0; + } + if let Some(v) = var::("RELAY_PROXY_PROTOCOL") { + c.proxy_protocol = v != 0; + } + c + } +} + +/// What the relay reports (and logs): counts only. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Counts { + pub hosts: usize, + pub bridges: usize, + pub registrations: u64, + pub joins: u64, + pub unknown: u64, + pub rate_limited: u64, + pub relayed_bytes: u64, +} +impl Counts { + pub fn json(&self) -> String { + format!( + "{{\"ok\":true,\"hosts\":{},\"bridges\":{},\"registrations\":{},\"joins\":{},\"unknown\":{},\"rate_limited\":{},\"relayed_bytes\":{}}}", + self.hosts, + self.bridges, + self.registrations, + self.joins, + self.unknown, + self.rate_limited, + self.relayed_bytes + ) + } +} + +/// What the host's control connection sends next. +enum Out { + Line(FromRelay), + /// The host stopped answering or went away. + Closed, +} +struct HostEntry { + token: String, + protocol: i32, + game: String, + to_host: Sender, + pending: usize, +} +struct Pending { + nameplate: String, + attach: SyncSender<(TcpStream, SocketAddr)>, +} +#[derive(Default)] +struct Inner { + hosts: HashMap, + ended: HashMap, + pending: HashMap, +} + +struct State { + config: Config, + inner: Mutex, + limiter: Mutex, + global_unknown: Mutex, + total: Mutex, + bridges: AtomicUsize, + registrations: AtomicU64, + joins: AtomicU64, + unknown: AtomicU64, + rate_limited: AtomicU64, + relayed_bytes: AtomicU64, + stop: AtomicBool, +} + +#[derive(Clone)] +pub struct Relay { + state: Arc, +} + +fn random_u64() -> u64 { + let mut b = [0u8; 8]; + getrandom::fill(&mut b).expect("OS random generator"); + u64::from_le_bytes(b) +} +fn random_token() -> String { + let mut b = [0u8; 16]; + getrandom::fill(&mut b).expect("OS random generator"); + b.iter().map(|x| format!("{x:02x}")).collect() +} + +impl Relay { + pub fn new(config: Config) -> Self { + let now = Instant::now(); + Self { + state: Arc::new(State { + limiter: Mutex::new(Limiter::new(config.rates.clone(), config.per_address)), + global_unknown: Mutex::new(Bucket::new( + config.global_unknown_per_min / 60.0, + config.global_unknown_burst, + now, + )), + total: Mutex::new(Bucket::new(config.total_rate, config.total_rate / 4.0, now)), + config, + inner: Mutex::new(Inner::default()), + bridges: AtomicUsize::new(0), + registrations: AtomicU64::new(0), + joins: AtomicU64::new(0), + unknown: AtomicU64::new(0), + rate_limited: AtomicU64::new(0), + relayed_bytes: AtomicU64::new(0), + stop: AtomicBool::new(false), + }), + } + } + /// A relay on a free loopback port, served on a thread (tests). + pub fn spawn_local(config: Config) -> io::Result<(Self, SocketAddr, JoinHandle<()>)> { + let listener = TcpListener::bind("127.0.0.1:0")?; + let address = listener.local_addr()?; + let relay = Self::new(config); + let r = relay.clone(); + let t = std::thread::spawn(move || r.serve(listener)); + Ok((relay, address, t)) + } + pub fn counts(&self) -> Counts { + counts(&self.state) + } + pub fn stop(&self) { + self.state.stop.store(true, Ordering::Relaxed); + } + /// Accept connections until [`Relay::stop`]. + pub fn serve(&self, listener: TcpListener) { + let _ = listener.set_nonblocking(true); + while !self.state.stop.load(Ordering::Relaxed) { + match listener.accept() { + Ok((stream, peer)) => { + let state = self.state.clone(); + let _ = + std::thread::Builder::new() + .name("relay-conn".into()) + .spawn(move || { + let _ = stream.set_nonblocking(false); + let _ = handle(&state, stream, peer); + }); + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(_) => std::thread::sleep(Duration::from_millis(50)), + } + } + } +} + +fn counts(s: &State) -> Counts { + Counts { + hosts: s.inner.lock().unwrap().hosts.len(), + bridges: s.bridges.load(Ordering::Relaxed), + registrations: s.registrations.load(Ordering::Relaxed), + joins: s.joins.load(Ordering::Relaxed), + unknown: s.unknown.load(Ordering::Relaxed), + rate_limited: s.rate_limited.load(Ordering::Relaxed), + relayed_bytes: s.relayed_bytes.load(Ordering::Relaxed), + } +} + +fn refuse(stream: &mut TcpStream, code: ErrorCode) -> io::Result<()> { + wire::send(stream, &FromRelay::Error { code, game: None }) +} + +/// `PROXY TCP4 `: the client's address. +fn proxy_line(line: &[u8]) -> Option { + let text = std::str::from_utf8(line).ok()?.trim_end_matches('\r'); + let parts: Vec<&str> = text.split(' ').collect(); + match parts.as_slice() { + ["PROXY", "TCP4" | "TCP6", src, _, sport, _] => { + Some(SocketAddr::new(src.parse().ok()?, sport.parse().ok()?)) + } + _ => None, + } +} + +fn handle(state: &Arc, mut stream: TcpStream, mut peer: SocketAddr) -> io::Result<()> { + if state.stop.load(Ordering::Relaxed) { + return Ok(()); + } + stream.set_read_timeout(Some(state.config.first_line))?; + let _ = stream.set_nodelay(true); + let mut line = wire::read_line(&mut stream)?.ok_or(io::ErrorKind::UnexpectedEof)?; + if state.config.proxy_protocol { + peer = proxy_line(&line) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "no PROXY line"))?; + line = wire::read_line(&mut stream)?.ok_or(io::ErrorKind::UnexpectedEof)?; + } + if line.starts_with(b"GET ") { + return http(state, &mut stream, &line, peer); + } + let Ok(hello) = serde_json::from_slice::(&line) else { + return refuse(&mut stream, ErrorCode::BadRequest); + }; + match hello { + ToRelay::Host { v, protocol, game } if v == wire::VERSION => { + host(state, stream, peer, protocol, game) + } + ToRelay::Join { + v, + nameplate, + protocol, + .. + } if v == wire::VERSION => join(state, stream, peer, &nameplate, protocol), + ToRelay::Attach { v, session, token } if v == wire::VERSION => { + attach(state, stream, peer, session, &token) + } + _ => refuse(&mut stream, ErrorCode::BadRequest), + } +} + +fn http(state: &State, stream: &mut TcpStream, line: &[u8], peer: SocketAddr) -> io::Result<()> { + // Drain the request headers (bounded) before answering. + let mut headers = 0; + while let Some(h) = wire::read_line(stream)? { + headers += 1; + if h.is_empty() || h == b"\r" || headers > 64 { + break; + } + } + let path = std::str::from_utf8(line) + .ok() + .and_then(|l| l.split(' ').nth(1)) + .unwrap_or(""); + let (status, body) = match path { + "/health" => ("200 OK", counts(state).json()), + // The address the relay sees for the caller: checks that the hosting + // platform's proxy passes client addresses through. + "/whoami" => ("200 OK", format!("{{\"address\":\"{peer}\"}}")), + _ => ("404 Not Found", "{\"ok\":false}".to_owned()), + }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()) +} + +fn host( + state: &Arc, + mut stream: TcpStream, + peer: SocketAddr, + protocol: i32, + game: String, +) -> io::Result<()> { + let now = Instant::now(); + if !state.limiter.lock().unwrap().allow_register(peer.ip(), now) { + state.rate_limited.fetch_add(1, Ordering::Relaxed); + return refuse(&mut stream, ErrorCode::RateLimited); + } + let (tx, rx) = channel(); + let token = random_token(); + let nameplate = { + let mut inner = state.inner.lock().unwrap(); + let kept = state.config.ended_kept; + inner + .ended + .retain(|_, t| now.saturating_duration_since(*t) < kept); + let mut found = None; + if inner.hosts.len() < state.config.max_hosts { + for _ in 0..64 { + let n = code::nameplate(random_u64() as u32); + if !inner.hosts.contains_key(&n) && !inner.ended.contains_key(&n) { + found = Some(n); + break; + } + } + } + if let Some(n) = &found { + inner.hosts.insert( + n.clone(), + HostEntry { + token: token.clone(), + protocol, + game: game.chars().take(32).collect(), + to_host: tx.clone(), + pending: 0, + }, + ); + } + found + }; + let Some(nameplate) = nameplate else { + return refuse(&mut stream, ErrorCode::Full); + }; + state.registrations.fetch_add(1, Ordering::Relaxed); + let result = serve_host(state, &mut stream, peer, &nameplate, &token, tx, rx); + { + let mut inner = state.inner.lock().unwrap(); + if inner + .hosts + .get(&nameplate) + .is_some_and(|h| h.token == token) + { + inner.hosts.remove(&nameplate); + inner.ended.insert(nameplate, Instant::now()); + } + } + let _ = stream.shutdown(Shutdown::Both); + result +} + +/// The host's control connection: joins and pings out, pongs in. A host +/// silent for `host_timeout` is gone. +fn serve_host( + state: &Arc, + stream: &mut TcpStream, + peer: SocketAddr, + nameplate: &str, + token: &str, + tx: Sender, + rx: Receiver, +) -> io::Result<()> { + wire::send( + stream, + &FromRelay::Registered { + nameplate: nameplate.to_owned(), + token: token.to_owned(), + observed: Some(peer.to_string()), + }, + )?; + let mut reader = stream.try_clone()?; + reader.set_read_timeout(Some(state.config.host_timeout))?; + std::thread::Builder::new() + .name("relay-host-read".into()) + .spawn(move || { + while let Ok(Some(_)) = wire::read_line(&mut reader) {} + let _ = tx.send(Out::Closed); + })?; + while !state.stop.load(Ordering::Relaxed) { + match rx.recv_timeout(state.config.ping_every) { + Ok(Out::Line(m)) => wire::send(stream, &m)?, + Ok(Out::Closed) | Err(RecvTimeoutError::Disconnected) => break, + Err(RecvTimeoutError::Timeout) => wire::send(stream, &FromRelay::Ping)?, + } + } + Ok(()) +} + +enum Lookup { + Found(Sender), + Unknown, + Ended, + Version(String), + Full, +} + +fn join( + state: &Arc, + mut stream: TcpStream, + peer: SocketAddr, + nameplate: &str, + protocol: i32, +) -> io::Result<()> { + let now = Instant::now(); + state.joins.fetch_add(1, Ordering::Relaxed); + // The global budget for unknown codes is checked before the lookup, so a + // refusal says nothing about whether the nameplate exists. + let allowed = state.limiter.lock().unwrap().allow_join(peer.ip(), now) + && state.global_unknown.lock().unwrap().has(now); + if !allowed { + state.rate_limited.fetch_add(1, Ordering::Relaxed); + return refuse(&mut stream, ErrorCode::RateLimited); + } + let Some(nameplate) = code::parse_nameplate(nameplate) else { + return refuse(&mut stream, ErrorCode::BadRequest); + }; + let session = random_u64(); + let (atx, arx) = sync_channel(1); + let lookup = { + let mut inner = state.inner.lock().unwrap(); + let ended = inner + .ended + .get(&nameplate) + .is_some_and(|t| now.saturating_duration_since(*t) < state.config.ended_kept); + let busy = state.bridges.load(Ordering::Relaxed) >= state.config.max_bridges; + let lookup = match inner.hosts.get_mut(&nameplate) { + None if ended => Lookup::Ended, + None => Lookup::Unknown, + Some(h) if h.protocol != protocol => Lookup::Version(h.game.clone()), + Some(h) if busy || h.pending >= state.config.max_pending_per_host => Lookup::Full, + Some(h) => { + h.pending += 1; + Lookup::Found(h.to_host.clone()) + } + }; + if matches!(lookup, Lookup::Found(_)) { + inner.pending.insert( + session, + Pending { + nameplate: nameplate.clone(), + attach: atx, + }, + ); + } + lookup + }; + let to_host = match lookup { + Lookup::Found(tx) => tx, + Lookup::Unknown => { + state.unknown.fetch_add(1, Ordering::Relaxed); + state.limiter.lock().unwrap().unknown(peer.ip(), now); + state.global_unknown.lock().unwrap().try_take(now); + return refuse(&mut stream, ErrorCode::UnknownCode); + } + Lookup::Ended => return refuse(&mut stream, ErrorCode::Ended), + Lookup::Full => return refuse(&mut stream, ErrorCode::Full), + Lookup::Version(game) => { + return wire::send( + &mut stream, + &FromRelay::Error { + code: ErrorCode::VersionMismatch, + game: Some(game), + }, + ); + } + }; + let _ = to_host.send(Out::Line(FromRelay::Join { session })); + let attached = arx.recv_timeout(state.config.attach_wait); + { + let mut inner = state.inner.lock().unwrap(); + inner.pending.remove(&session); + if let Some(h) = inner.hosts.get_mut(&nameplate) { + h.pending = h.pending.saturating_sub(1); + } + } + match attached { + Ok((host, host_peer)) => bridge(state, stream, peer, host, host_peer, session), + Err(_) => refuse(&mut stream, ErrorCode::HostUnreachable), + } +} + +fn attach( + state: &Arc, + mut stream: TcpStream, + peer: SocketAddr, + session: u64, + token: &str, +) -> io::Result<()> { + let target = { + let inner = state.inner.lock().unwrap(); + inner + .pending + .get(&session) + .filter(|p| { + inner + .hosts + .get(&p.nameplate) + .is_some_and(|h| h.token == token) + }) + .map(|p| p.attach.clone()) + }; + match target { + Some(tx) => match tx.try_send((stream, peer)) { + Ok(()) => Ok(()), + Err( + std::sync::mpsc::TrySendError::Full((mut stream, _)) + | std::sync::mpsc::TrySendError::Disconnected((mut stream, _)), + ) => refuse(&mut stream, ErrorCode::BadRequest), + }, + None => refuse(&mut stream, ErrorCode::BadRequest), + } +} + +/// Tell both ends they are bridged, then copy each way until either ends. +fn bridge( + state: &Arc, + mut guest: TcpStream, + guest_peer: SocketAddr, + mut host: TcpStream, + host_peer: SocketAddr, + session: u64, +) -> io::Result<()> { + state.bridges.fetch_add(1, Ordering::Relaxed); + let result = (|| -> io::Result<()> { + wire::send( + &mut guest, + &FromRelay::Bridged { + session, + observed: Some(guest_peer.to_string()), + }, + )?; + wire::send( + &mut host, + &FromRelay::Bridged { + session, + observed: Some(host_peer.to_string()), + }, + )?; + for s in [&guest, &host] { + s.set_read_timeout(Some(state.config.idle))?; + let _ = s.set_nodelay(true); + } + let (g, h, st) = (guest.try_clone()?, host.try_clone()?, state.clone()); + let up = std::thread::Builder::new() + .name("relay-copy".into()) + .spawn(move || copy(&st, g, h))?; + copy(state, host, guest); + let _ = up.join(); + Ok(()) + })(); + state.bridges.fetch_sub(1, Ordering::Relaxed); + result +} + +fn copy(state: &State, mut from: TcpStream, mut to: TcpStream) { + let rate = state.config.session_rate; + let mut bucket = Bucket::new(rate, (rate / 2.0).max(16384.0), Instant::now()); + let mut buf = vec![0u8; 16 * 1024]; + loop { + let n = match from.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + let now = Instant::now(); + let wait = bucket + .take(n as f64, now) + .max(state.total.lock().unwrap().take(n as f64, now)); + if !wait.is_zero() { + std::thread::sleep(wait); + } + if to.write_all(&buf[..n]).is_err() { + break; + } + state.relayed_bytes.fetch_add(n as u64, Ordering::Relaxed); + } + let _ = from.shutdown(Shutdown::Both); + let _ = to.shutdown(Shutdown::Both); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn proxy_lines() { + assert_eq!( + proxy_line(b"PROXY TCP4 203.0.113.7 10.0.0.1 51000 7000\r"), + Some("203.0.113.7:51000".parse().unwrap()) + ); + assert_eq!( + proxy_line(b"PROXY TCP6 2001:db8::1 2001:db8::2 51000 7000"), + Some("[2001:db8::1]:51000".parse().unwrap()) + ); + assert_eq!(proxy_line(b"PROXY UNKNOWN"), None); + assert_eq!(proxy_line(b"{\"op\":\"pong\"}"), None); + } +} diff --git a/crates/horizon-relay/src/limits.rs b/crates/horizon-relay/src/limits.rs new file mode 100644 index 000000000..adf199919 --- /dev/null +++ b/crates/horizon-relay/src/limits.rs @@ -0,0 +1,275 @@ +//! Token buckets for request rates and relayed bandwidth, and the +//! per-address state behind them (kept in memory, forgotten after ten idle +//! minutes). +use std::{ + collections::{HashMap, VecDeque}, + net::IpAddr, + time::{Duration, Instant}, +}; + +/// A token bucket: `rate` tokens a second up to `burst`. +#[derive(Clone, Debug)] +pub struct Bucket { + tokens: f64, + last: Instant, + rate: f64, + burst: f64, +} +impl Bucket { + pub fn new(rate: f64, burst: f64, now: Instant) -> Self { + Self { + tokens: burst, + last: now, + rate, + burst, + } + } + fn refill(&mut self, now: Instant) { + let dt = now.saturating_duration_since(self.last).as_secs_f64(); + self.tokens = (self.tokens + dt * self.rate).min(self.burst); + self.last = now; + } + /// Take one token if there is one. + pub fn try_take(&mut self, now: Instant) -> bool { + self.refill(now); + if self.tokens >= 1.0 { + self.tokens -= 1.0; + true + } else { + false + } + } + pub fn has(&mut self, now: Instant) -> bool { + self.refill(now); + self.tokens >= 1.0 + } + /// Take `n` tokens, going into debt if needed; how long to wait before + /// sending them so the average stays at the rate. + pub fn take(&mut self, n: f64, now: Instant) -> Duration { + self.refill(now); + self.tokens -= n; + if self.tokens >= 0.0 || self.rate <= 0.0 { + Duration::ZERO + } else { + Duration::from_secs_f64(-self.tokens / self.rate) + } + } +} + +/// Addresses are counted by IPv4 address and by IPv6 /64 (one home). +pub fn address_key(ip: IpAddr) -> IpAddr { + match ip { + IpAddr::V6(v6) => match v6.to_ipv4_mapped() { + Some(v4) => IpAddr::V4(v4), + None => { + let s = v6.segments(); + IpAddr::V6(std::net::Ipv6Addr::new(s[0], s[1], s[2], s[3], 0, 0, 0, 0)) + } + }, + v4 => v4, + } +} + +#[derive(Clone, Debug)] +pub struct Rates { + pub register_per_min: f64, + pub register_burst: f64, + pub join_per_min: f64, + pub join_burst: f64, + /// Unknown codes an address may try in `unknown_window` before it is + /// blocked for `block`. + pub unknown_limit: usize, + pub unknown_window: Duration, + pub block: Duration, +} +impl Default for Rates { + fn default() -> Self { + Self { + register_per_min: 6.0, + register_burst: 3.0, + join_per_min: 12.0, + join_burst: 6.0, + unknown_limit: 5, + unknown_window: Duration::from_secs(600), + block: Duration::from_secs(600), + } + } +} + +struct Address { + register: Bucket, + join: Bucket, + unknown: VecDeque, + blocked_until: Option, + seen: Instant, +} + +/// Per-address limits; disabled (everything allowed) when the relay can't +/// see client addresses. +pub struct Limiter { + rates: Rates, + enabled: bool, + addresses: HashMap, + last_sweep: Instant, +} +impl Limiter { + pub fn new(rates: Rates, enabled: bool) -> Self { + Self { + rates, + enabled, + addresses: HashMap::new(), + last_sweep: Instant::now(), + } + } + fn entry(&mut self, ip: IpAddr, now: Instant) -> &mut Address { + if now.saturating_duration_since(self.last_sweep) > Duration::from_secs(60) { + self.last_sweep = now; + let keep = Duration::from_secs(600).max(self.rates.block); + self.addresses.retain(|_, a| { + now.saturating_duration_since(a.seen) < keep + || a.blocked_until.is_some_and(|b| b > now) + }); + } + let r = &self.rates; + let a = self + .addresses + .entry(address_key(ip)) + .or_insert_with(|| Address { + register: Bucket::new(r.register_per_min / 60.0, r.register_burst, now), + join: Bucket::new(r.join_per_min / 60.0, r.join_burst, now), + unknown: VecDeque::new(), + blocked_until: None, + seen: now, + }); + a.seen = now; + a + } + pub fn allow_register(&mut self, ip: IpAddr, now: Instant) -> bool { + !self.enabled || self.entry(ip, now).register.try_take(now) + } + pub fn allow_join(&mut self, ip: IpAddr, now: Instant) -> bool { + if !self.enabled { + return true; + } + let a = self.entry(ip, now); + if a.blocked_until.is_some_and(|b| b > now) { + return false; + } + a.blocked_until = None; + a.join.try_take(now) + } + /// An unknown code from this address; blocks it past the limit. + pub fn unknown(&mut self, ip: IpAddr, now: Instant) { + if !self.enabled { + return; + } + let (limit, window, block) = ( + self.rates.unknown_limit, + self.rates.unknown_window, + self.rates.block, + ); + let a = self.entry(ip, now); + a.unknown.push_back(now); + while a + .unknown + .front() + .is_some_and(|t| now.saturating_duration_since(*t) > window) + { + a.unknown.pop_front(); + } + if a.unknown.len() >= limit { + a.blocked_until = Some(now + block); + a.unknown.clear(); + } + } + pub fn tracked(&self) -> usize { + self.addresses.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn buckets_refill_at_their_rate() { + let t = Instant::now(); + let mut b = Bucket::new(1.0, 2.0, t); + assert!(b.try_take(t)); + assert!(b.try_take(t)); + assert!(!b.try_take(t)); + assert!(!b.try_take(t + Duration::from_millis(900))); + assert!(b.try_take(t + Duration::from_millis(1000))); + // Never above the burst. + assert!(b.try_take(t + Duration::from_secs(100))); + assert!(b.try_take(t + Duration::from_secs(100))); + assert!(!b.try_take(t + Duration::from_secs(100))); + } + + #[test] + fn bandwidth_debt_turns_into_a_wait() { + let t = Instant::now(); + let mut b = Bucket::new(1000.0, 500.0, t); + assert_eq!(b.take(500.0, t), Duration::ZERO); + assert_eq!(b.take(1000.0, t), Duration::from_secs(1)); + // After the wait the debt is paid. + assert_eq!(b.take(0.0, t + Duration::from_secs(1)), Duration::ZERO); + } + + #[test] + fn ipv6_counts_by_home() { + let a: IpAddr = "2001:db8:1:2:aaaa::1".parse().unwrap(); + let b: IpAddr = "2001:db8:1:2:bbbb::9".parse().unwrap(); + let c: IpAddr = "2001:db8:1:3::1".parse().unwrap(); + assert_eq!(address_key(a), address_key(b)); + assert_ne!(address_key(a), address_key(c)); + let mapped: IpAddr = "::ffff:203.0.113.7".parse().unwrap(); + assert_eq!( + address_key(mapped), + "203.0.113.7".parse::().unwrap() + ); + } + + #[test] + fn unknown_codes_block_an_address_for_a_while() { + let t = Instant::now(); + let ip: IpAddr = "203.0.113.7".parse().unwrap(); + let other: IpAddr = "203.0.113.8".parse().unwrap(); + let mut l = Limiter::new( + Rates { + join_burst: 100.0, + ..Rates::default() + }, + true, + ); + for _ in 0..4 { + assert!(l.allow_join(ip, t)); + l.unknown(ip, t); + } + assert!(l.allow_join(ip, t)); + l.unknown(ip, t); + assert!(!l.allow_join(ip, t)); + assert!(!l.allow_join(ip, t + Duration::from_secs(599))); + assert!(l.allow_join(other, t)); + assert!(l.allow_join(ip, t + Duration::from_secs(601))); + // Unknown codes spread beyond the window don't add up. + let mut l = Limiter::new(Rates::default(), true); + for i in 0..10 { + l.unknown(ip, t + Duration::from_secs(i * 200)); + } + assert!(l.allow_join(ip, t + Duration::from_secs(2000))); + } + + #[test] + fn join_and_register_rates() { + let t = Instant::now(); + let ip: IpAddr = "198.51.100.1".parse().unwrap(); + let mut l = Limiter::new(Rates::default(), true); + assert_eq!((0..10).filter(|_| l.allow_register(ip, t)).count(), 3); + assert_eq!((0..10).filter(|_| l.allow_join(ip, t)).count(), 6); + assert!(l.allow_join(ip, t + Duration::from_secs(5))); + let mut off = Limiter::new(Rates::default(), false); + assert!((0..100).all(|_| off.allow_join(ip, t))); + assert_eq!(off.tracked(), 0); + } +} diff --git a/crates/horizon-relay/src/main.rs b/crates/horizon-relay/src/main.rs new file mode 100644 index 000000000..f14bdfe05 --- /dev/null +++ b/crates/horizon-relay/src/main.rs @@ -0,0 +1,38 @@ +//! `horizon-relay`: listens on `$PORT` (7000 when unset), limits from the +//! `RELAY_*` variables (docs/play-with-friends.md). Prints counts every five +//! minutes, nothing else. +use horizon_relay::{Config, Relay}; +use std::{net::TcpListener, time::Duration}; + +fn main() { + let port: u16 = std::env::var("PORT") + .ok() + .and_then(|p| p.trim().parse().ok()) + .unwrap_or(7000); + // Dual-stack where the system allows it (Railway's private network is + // IPv6), else IPv4. + let listener = TcpListener::bind(("::", port)) + .or_else(|_| TcpListener::bind(("0.0.0.0", port))) + .unwrap_or_else(|e| { + eprintln!("horizon-relay: can't listen on port {port}: {e}"); + std::process::exit(1); + }); + let config = Config::from_env(); + println!( + "horizon-relay {} listening on port {port} (hosts {}, bridges {}, {} KiB/s per stream, per-address limits {})", + env!("CARGO_PKG_VERSION"), + config.max_hosts, + config.max_bridges, + (config.session_rate / 1024.0) as u64, + if config.per_address { "on" } else { "off" }, + ); + let relay = Relay::new(config); + let r = relay.clone(); + std::thread::spawn(move || { + loop { + std::thread::sleep(Duration::from_secs(300)); + println!("{}", r.counts().json()); + } + }); + relay.serve(listener); +} diff --git a/crates/horizon-relay/tests/tunnel.rs b/crates/horizon-relay/tests/tunnel.rs new file mode 100644 index 000000000..d988edb88 --- /dev/null +++ b/crates/horizon-relay/tests/tunnel.rs @@ -0,0 +1,349 @@ +//! A relay, a host tunnel in front of an echo server standing in for the +//! integrated server, and guest tunnels, all on loopback in one process. +use horizon_friends::{ + Code, Host, HostConfig, HostEvent, JoinError, PathMode, + guest::{self, GuestConfig}, +}; +use horizon_relay::{Config, Relay, limits::Rates}; +use std::{ + io::{Read, Write}, + net::{SocketAddr, TcpListener, TcpStream}, + time::{Duration, Instant}, +}; + +const PROTOCOL: i32 = 777; + +/// Echoes every connection back until it closes. +fn echo_server() -> SocketAddr { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let a = l.local_addr().unwrap(); + std::thread::spawn(move || { + for s in l.incoming() { + let Ok(mut s) = s else { continue }; + std::thread::spawn(move || { + let mut out = s.try_clone().unwrap(); + let _ = std::io::copy(&mut s, &mut out); + }); + } + }); + a +} + +fn relay(config: Config) -> (Relay, String) { + let (relay, address, _) = Relay::spawn_local(config).unwrap(); + (relay, address.to_string()) +} + +fn generous() -> Config { + Config { + rates: Rates { + join_per_min: 600.0, + join_burst: 100.0, + ..Rates::default() + }, + ..Config::default() + } +} + +fn host(relay: &str, lan: SocketAddr, path: PathMode) -> (Host, Code) { + let host = Host::start(HostConfig { + relay: relay.to_owned(), + lan, + protocol: PROTOCOL, + game: "26.3".into(), + path, + }) + .unwrap(); + let code = wait_event(&host, |e| match e { + HostEvent::Code(c) => Some(c.clone()), + _ => None, + }); + (host, code) +} + +thread_local! { + /// Events polled but not yet waited for. + static BACKLOG: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; +} + +/// The first event (in order) that `f` picks; the others stay queued. +fn wait_event(host: &Host, mut f: impl FnMut(&HostEvent) -> Option) -> T { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + let found = BACKLOG.with_borrow_mut(|backlog| { + backlog.extend(host.poll()); + let i = backlog.iter().position(|e| f(e).is_some())?; + f(&backlog.remove(i)) + }); + if let Some(t) = found { + return t; + } + std::thread::sleep(Duration::from_millis(10)); + } + panic!("no such host event"); +} + +fn join(relay: &str, code: &Code, path: PathMode) -> Result { + guest::join(&GuestConfig { + relay: relay.to_owned(), + code: code.clone(), + protocol: PROTOCOL, + game: "26.3".into(), + path, + }) +} + +/// What the game's client does: connect to the loopback port, send, read +/// the echo back. +fn round_trip(tunnel: guest::Tunnel, payload: &[u8]) { + let (port, done) = tunnel.serve_loopback().unwrap(); + let mut s = TcpStream::connect(("127.0.0.1", port)).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(20))).unwrap(); + let mut w = s.try_clone().unwrap(); + let send = payload.to_vec(); + let writer = std::thread::spawn(move || w.write_all(&send).unwrap()); + let mut got = vec![0u8; payload.len()]; + s.read_exact(&mut got).unwrap(); + writer.join().unwrap(); + assert!(got == payload, "echo differs"); + drop(s); + done.join().unwrap(); +} + +fn payload(n: usize) -> Vec { + (0..n).map(|i| (i * 7 + i / 251) as u8).collect() +} + +#[test] +fn the_direct_path_carries_the_stream() { + let (relay, address) = relay(generous()); + let (host, code) = host(&address, echo_server(), PathMode::Auto); + assert_eq!(code.nameplate.len(), 4); + let tunnel = join(&address, &code, PathMode::Auto).unwrap(); + assert!( + tunnel.direct, + "loopback ends should reach each other directly" + ); + let data = payload(1 << 20); + round_trip(tunnel, &data); + assert!(wait_event(&host, |e| match e { + HostEvent::Connected { direct, .. } => Some(*direct), + _ => None, + })); + // Only the handshake went through the relay. + let counts = relay.counts(); + assert!(counts.relayed_bytes < 16 * 1024, "{counts:?}"); + assert_eq!(counts.hosts, 1); + wait_event(&host, |e| { + matches!(e, HostEvent::Disconnected { .. }).then_some(()) + }); + assert_eq!(host.sessions(), 0); +} + +#[test] +fn the_relay_path_carries_the_stream() { + let (relay, address) = relay(generous()); + let (host, code) = host(&address, echo_server(), PathMode::Auto); + let tunnel = join(&address, &code, PathMode::RelayOnly).unwrap(); + assert!(!tunnel.direct); + let data = payload(1 << 20); + round_trip(tunnel, &data); + assert!(!wait_event(&host, |e| match e { + HostEvent::Connected { direct, .. } => Some(*direct), + _ => None, + })); + // Both directions, encrypted and framed: a little more than twice. + let relayed = relay.counts().relayed_bytes; + assert!(relayed > 2 * data.len() as u64, "{relayed}"); + // Two guests at once, one per path. + let a = join(&address, &code, PathMode::RelayOnly).unwrap(); + let b = join(&address, &code, PathMode::Auto).unwrap(); + let ta = std::thread::spawn(move || round_trip(a, &payload(50_000))); + round_trip(b, &payload(70_000)); + ta.join().unwrap(); +} + +#[test] +fn the_host_can_refuse_direct_connections() { + let (_relay, address) = relay(generous()); + let (_host, code) = host(&address, echo_server(), PathMode::RelayOnly); + let tunnel = join(&address, &code, PathMode::Auto).unwrap(); + assert!(!tunnel.direct); + round_trip(tunnel, b"relay only"); + // A guest that insists on a direct path gets none. + assert!(matches!( + join(&address, &code, PathMode::DirectOnly), + Err(JoinError::Failed(_)) + )); +} + +#[test] +fn wrong_secrets_fail_and_retire_the_code() { + let (_relay, address) = relay(generous()); + let (host, code) = host(&address, echo_server(), PathMode::Auto); + let wrong = Code { + nameplate: code.nameplate.clone(), + secret: if code.secret == "0000" { + "0001".into() + } else { + "0000".into() + }, + }; + for i in 1..=horizon_friends::host::MAX_WRONG { + let e = join(&address, &wrong, PathMode::Auto).err().unwrap(); + assert!(matches!(e, JoinError::WrongCode), "{e}"); + let count = wait_event(&host, |e| match e { + HostEvent::WrongCode { count } => Some(*count), + _ => None, + }); + assert_eq!(count, i); + } + // A new code replaces the old one, which has ended. + let fresh = wait_event(&host, |e| match e { + HostEvent::Code(c) => Some(c.clone()), + _ => None, + }); + assert_ne!(fresh.nameplate, code.nameplate); + assert!(matches!( + join(&address, &code, PathMode::Auto), + Err(JoinError::Ended) + )); + round_trip(join(&address, &fresh, PathMode::Auto).unwrap(), b"fresh"); +} + +#[test] +fn unknown_codes_versions_and_ended_invites() { + let (relay, address) = relay(generous()); + let (mut host, code) = host(&address, echo_server(), PathMode::Auto); + let other = guest::join(&GuestConfig { + relay: address.clone(), + code: code.clone(), + protocol: PROTOCOL - 1, + game: "26.2".into(), + path: PathMode::Auto, + }); + match other { + Err(e @ JoinError::VersionMismatch { .. }) => { + assert_eq!( + e.to_string(), + "Your friend plays Minecraft 26.3; you have 26.2." + ) + } + other => panic!("{:?}", other.err()), + } + host.stop(); + assert!(matches!( + join(&address, &code, PathMode::Auto), + Err(JoinError::Ended) + )); + // A nameplate nobody holds; five of them block this address for a while. + let free = (0..) + .map(|i| horizon_friends::code::nameplate(i * 7919)) + .find(|n| *n != code.nameplate) + .unwrap(); + let unknown = Code { + nameplate: free, + secret: "0000".into(), + }; + for _ in 0..5 { + assert!(matches!( + join(&address, &unknown, PathMode::Auto), + Err(JoinError::UnknownCode) + )); + } + assert!(matches!( + join(&address, &code, PathMode::Auto), + Err(JoinError::RateLimited) + )); + let counts = relay.counts(); + assert_eq!(counts.unknown, 5); + assert!(counts.rate_limited >= 1); + assert_eq!(counts.hosts, 0); +} + +#[test] +fn a_host_that_never_attaches_is_unreachable() { + use horizon_friends::wire::{self, FromRelay, ToRelay}; + let (_relay, address) = relay(Config { + attach_wait: Duration::from_millis(500), + ..generous() + }); + let mut control = TcpStream::connect(&address).unwrap(); + wire::send( + &mut control, + &ToRelay::Host { + v: wire::VERSION, + protocol: PROTOCOL, + game: "26.3".into(), + }, + ) + .unwrap(); + let FromRelay::Registered { + nameplate, token, .. + } = wire::recv(&mut control).unwrap() + else { + panic!() + }; + let code = Code { + nameplate, + secret: "0000".into(), + }; + assert!(matches!( + join(&address, &code, PathMode::Auto), + Err(JoinError::HostUnreachable) + )); + // The relay announced the session; attaching late or with another token fails. + let FromRelay::Join { session } = wire::recv(&mut control).unwrap() else { + panic!() + }; + let mut late = TcpStream::connect(&address).unwrap(); + wire::send( + &mut late, + &ToRelay::Attach { + v: wire::VERSION, + session, + token, + }, + ) + .unwrap(); + assert!(matches!( + wire::recv::<_, FromRelay>(&mut late).unwrap(), + FromRelay::Error { .. } + )); +} + +#[test] +fn relayed_bandwidth_is_capped() { + let rate = 200_000.0; + let (_relay, address) = relay(Config { + session_rate: rate, + ..generous() + }); + let (_host, code) = host(&address, echo_server(), PathMode::RelayOnly); + let tunnel = join(&address, &code, PathMode::RelayOnly).unwrap(); + let started = Instant::now(); + round_trip(tunnel, &payload(600_000)); + // The first 100 kB pass at once, the rest at 200 kB/s. + assert!( + started.elapsed() >= Duration::from_secs(2), + "{:?}", + started.elapsed() + ); +} + +#[test] +fn health_and_whoami_answer_http() { + let (_relay, address) = relay(Config::default()); + let get = |path: &str| { + let mut s = TcpStream::connect(&address).unwrap(); + write!(s, "GET {path} HTTP/1.1\r\nHost: relay\r\n\r\n").unwrap(); + let mut out = String::new(); + s.read_to_string(&mut out).unwrap(); + out + }; + let health = get("/health"); + assert!(health.starts_with("HTTP/1.1 200 OK\r\n"), "{health}"); + assert!(health.ends_with("{\"ok\":true,\"hosts\":0,\"bridges\":0,\"registrations\":0,\"joins\":0,\"unknown\":0,\"rate_limited\":0,\"relayed_bytes\":0}")); + assert!(get("/whoami").contains("\"address\":\"127.0.0.1:")); + assert!(get("/").starts_with("HTTP/1.1 404")); +} diff --git a/docs/play-with-friends.md b/docs/play-with-friends.md new file mode 100644 index 000000000..6ea3449c8 --- /dev/null +++ b/docs/play-with-friends.md @@ -0,0 +1,259 @@ +# Play with friends (invite codes) + +A player hosts their single-player world with one click and gets a short code such as +`K7QM-3XPD`. Friends paste the code (on the multiplayer screen in game, or on the launcher's +Play page) and join. No port forwarding, no accounts. + +It is built on what Open to LAN already does: the integrated server (`crates/horizon-client/src/lan`) +listens on a local TCP port and speaks the vanilla protocol. Play with friends adds a tunnel in front +of that port and a small relay service (`crates/horizon-relay`) that introduces the two ends. Neither +the integrated server nor the guest's client changes: the tunnel carries exactly the bytes vanilla's +protocol would send, between a loopback port on each side. + +``` + guest's game host's game + ┌──────────────┐ 127.0.0.1:A ┌────────────┐ direct TCP ┌────────────┐ 127.0.0.1:LAN ┌──────────────┐ + │ client │──────────────▶│ guest │◀═══ (preferred) ════▶│ host │────────────────▶│ integrated │ + │ (unchanged) │ │ tunnel │ │ tunnel │ │ server (LAN) │ + └──────────────┘ └─────┬──────┘ └─────┬──────┘ └──────────────┘ + │ ┌──────────────────┐ │ + └─────▶│ horizon-relay │◀────────┘ + join, PAKE, │ codes, rendezvous│ register, control, + or the whole│ byte relay │ attach per guest + stream └──────────────────┘ +``` + +## What vanilla has + +Vanilla 26.3 (read with `tools/rea_vanilla.py`) has two related things, and neither is this: + +- **Open to LAN** (`IntegratedServer.publishServer`, `ShareToLanScreen`, `LanServerPinger`, + `PublishCommand`): the integrated server listens on a port and announces itself by multicast on the + local network. This is the baseline Horizon follows exactly: Play with friends uses the very same + server, the same `IntegratedServer.MAX_PLAYERS` (8), the same offline-mode login, game mode and + command settings as Open to LAN. Hosting through a code *is* opening to LAN, plus the tunnel. +- **Realms invite codes** (`com.mojang.realmsclient.dto.InviteCode`, `RealmsInviteCodesScreen`): + codes that add a player to a Realm, a server Mojang hosts. They need accounts and Mojang's + service; nothing in the client tunnels a local world. + +So the code, the relay and the screens are Horizon's own, in the launcher's design language (the +existing `ui_theme` widgets, so the redesign in flight restyles them with everything else). + +## Identity + +Like LAN today: offline mode. A guest joins with the name their game uses; the server gives them the +offline UUID for it; the host's own profile is reserved so nobody can take the host's name. Online-mode +identity (the host checks guests against Mojang's session servers when both are signed in to a +Microsoft account) can come later on top of the same tunnel, since the tunnel carries the login +packets unchanged. Because every tunnelled guest reaches the integrated server from `127.0.0.1`, +`/ban-ip` has nothing useful to act on for them (the LAN host has no such command anyway). + +## Codes + +A code is 8 symbols of Crockford base32 (`0-9 A-Z` without `I L O U`), shown as two groups of four: +`K7QM-3XPD`. Input ignores case, spaces and dashes, and reads `O` as `0` and `I`/`L` as `1`. + +- The first four symbols are the **nameplate** (20 bits). The relay allocates it at random among the + free ones when the host registers, and uses it to find the host. It is the only part the relay sees. +- The last four are the **secret** (20 bits). The host's game draws it from the OS random generator + and never sends it anywhere. It is the password of the key exchange below. + +A code lives as long as the host keeps sharing: when the host stops sharing, closes the world or loses +its connection to the relay, the nameplate is freed (the relay keeps a bare "ended" mark for 15 minutes +so a guest gets "this invite has ended" rather than "unknown code"). + +Guessing: to join a world uninvited, someone needs an active nameplate (the relay limits resolution +attempts per address and in total, see below) and then the secret, for which each attempt is one online run of the +key exchange (20 bits: one chance in a million). After 5 failed exchanges the host's game retires the +code and shows a new one, so a nameplate found by luck is worth at most 5 tries. + +## Encryption: SPAKE2, then ChaCha20-Poly1305 + +The relay must not be able to read or change game traffic, but all the two ends share is a short +code, and the relay sees part of it. A key derived directly from the code would fall to an offline +search in seconds, so the ends run a **password-authenticated key exchange** (SPAKE2 over Ed25519, +the RustCrypto `spake2` crate, the construction magic-wormhole uses for the same problem): + +1. Over the relayed connection, each end sends one SPAKE2 message built from the secret and bound to + the nameplate and the session number. Each derives a 32-byte key `K`. With different secrets the + keys differ, and nothing in the messages helps an offline guess: a dishonest relay gets exactly one + guess per session, and a wrong guess shows up as a failed join. +2. Every stream that carries the session (the relayed one, and each direct connection tried) then + exchanges two fresh 16-byte randoms in the clear and derives its own keys with HKDF-SHA256 from `K`, + the randoms and the direction. Frames are ChaCha20-Poly1305 with a per-direction counter nonce, at + most 16 KiB of payload each. The first frame each way is a key confirmation: a wrong code or a + tampered byte ends the stream. + +What the relay **can** see: the nameplate, the IP addresses and ports of the host and of each guest +(it is their TCP peer), when sessions start and end, and the size and timing of the encrypted frames +of relayed sessions. What it **can't**: the secret half of the code, the key, the game bytes (names, +chat, positions, the world), or the candidate addresses the ends exchange (they travel encrypted). + +The relay connection itself is plain TCP (a TLS front would hide the nameplate from the network +path, but adds certificates to a service that holds nothing worth hiding). + +## Connecting: direct first, then the relay + +Every session starts through the relay, so the key exchange always has a path. Then: + +1. Over the encrypted relayed stream, the ends swap their **candidates**: the local addresses they can + be reached on (found by asking the OS which source address it would use toward the relay, toward + the public IPv4 internet and toward the public IPv6 internet; this finds the LAN address, a global + IPv6 address and a tailnet address when there is one) plus the address the relay saw them come + from (their public, reflexive address), all with the port of their **direct socket**. The guest's + join connection and the host's attach connection leave from that port, so the address the relay + saw is the NAT's mapping of it. (The host's long-lived control connection uses another port, so + it never holds the direct port's connection to the relay.) +2. Both ends then dial every candidate of the other at once, for up to 3 seconds, retrying every 250 + ms, from their direct socket's port (`SO_REUSEADDR`/`SO_REUSEPORT`, so the same port also listens and + carried the relay connection). Dialling from both sides at once is **TCP simultaneous open**: each + side's outgoing SYN opens its own NAT for the other's, which gets through NATs that map a local port + to the same public port for every destination (most home routers) and stateful firewalls (IPv6). + On the same LAN, on a tailnet, with IPv6 or with a forwarded port the plain dial wins first. +3. Each direct stream that connects is authenticated with the session key as above. The guest keeps + the first one that passes, says so over the relayed stream, and both ends close the rest. If none + passes in time, the session stays on the relayed stream: the relay copies the encrypted bytes. + +Why TCP and not UDP hole punching with QUIC: punching UDP succeeds more often across NATs (published +measurements put it around 80% against 60–65% for TCP), but it needs a reliable transport over UDP, +which means QUIC (an async runtime and a QUIC/TLS stack in the game) for a stream that then still needs +the same end-to-end layer. The relay fallback makes every session connect either way; the direct path +only saves latency and relay bandwidth. The tunnel takes any byte stream (`Read + Write`), so a QUIC +transport can be added later beside the TCP one without touching the rest. + +The reflexive candidate is only as good as the relay's view of addresses: if the hosting platform's +TCP proxy hides client addresses (Railway's documentation doesn't say), the relay sees the proxy's +and that candidate simply never connects; the LAN, IPv6, tailnet and forwarded-port paths still work. +`GET /whoami` on the relay shows what it sees, to check after deploying. + +`HORIZON_FRIENDS_PATH=relay` skips the direct attempts (players who don't want to share their address +with the other end, and the tests); `=direct` refuses the relay fallback (tests). + +## The relay protocol + +One TCP port. The first line of each connection is a JSON object (at most 4 KiB, within 10 seconds); +a connection whose first bytes are `GET ` gets an HTTP answer instead (`GET /health`, for the hosting +platform's health check), so one port serves both. + +| From | Line | Meaning | +| --- | --- | --- | +| host | `{"op":"host","v":1,"protocol":P,"game":"26.3"}` | register; the relay answers `{"op":"registered","nameplate":"K7QM","token":T,"observed":"ip:port"}` | +| guest | `{"op":"join","v":1,"nameplate":"K7QM","protocol":P,"game":"26.3"}` | resolve; the relay announces the session to the host, waits for its attach, then answers `{"op":"bridged","session":S,"observed":…}` or `{"op":"error","code":…}` | +| relay → host | `{"op":"join","session":S}` | a guest wants in: open a new connection and attach | +| host | `{"op":"attach","v":1,"session":S,"token":T}` (new connection) | the relay pairs it with the guest's connection, answers `{"op":"bridged","session":S,"observed":…}` to both, then copies bytes | +| relay ↔ host | `{"op":"ping"}` / `{"op":"pong"}` | every 20 s; a host silent for 60 s is dropped and its code ends | + +Error codes the guest can get: `unknown_code`, `ended` (the host stopped sharing), `version_mismatch` +(with the host's game version: the relay compares protocol numbers, so a guest on another Minecraft +version is told before anything else happens), `host_unreachable` (the host didn't attach within 10 +seconds), `full` (the host has 8 guests connecting or the relay is at capacity), `rate_limited`, +`bad_request`. + +After `bridged`, the stream is the session: the SPAKE2 messages, then the encrypted frames. + +## Limits + +All set by environment variables on the relay (defaults in brackets): + +- `RELAY_MAX_HOSTS` [1000] codes at once; `RELAY_MAX_BRIDGES` [200] relayed streams at once; + `RELAY_MAX_PENDING_PER_HOST` [8] guests joining one host at the same time. +- Per client address (IPv6 grouped by /64): 6 host registrations a minute (burst 3); 12 joins a minute + (burst 6); 5 unknown codes in 10 minutes block joins from that address for 10 minutes. + `RELAY_PER_ADDRESS_LIMITS=0` turns these off for a proxy that shows every client as one address; + `RELAY_PROXY_PROTOCOL=1` reads the client's address from a PROXY protocol v1 line instead. +- For everyone together: `RELAY_GLOBAL_UNKNOWN_PER_MIN` [120] unknown codes a minute (burst 30); past + it, joins are refused as rate limited before the code is looked up, so a refusal says nothing about + whether a code exists. Even if one attacker had the whole budget: with 1000 worlds shared, an active + nameplate turns up about every 9 minutes, and each is worth 5 one-in-a-million tries before its + code is retired, so a break-in takes on the order of thousands of years. +- `RELAY_SESSION_KBPS` [1024] KiB/s each way per relayed stream; `RELAY_TOTAL_MBPS` [100] Mbit/s for + the whole relay; a relayed stream with no byte either way for 120 s is closed (the game's keep-alive + sends one every 15 s, so only dead streams idle). +- A first line must arrive within 10 s; hosts answer pings within 60 s. + +## Privacy + +No accounts. The relay keeps only what a live session needs, in memory: active nameplates and the +host connection behind each, pending sessions, rate-limit counters per address (forgotten after 10 +minutes). It logs **counts** every 5 minutes (hosts, relayed streams, joins, unknown codes, +rate-limited requests, relayed bytes) and nothing else: no addresses, no codes, no names. +`/health` answers counts too. + +A direct connection shows each player's IP address to the other, as a LAN game or a server address +would. `HORIZON_FRIENDS_PATH=relay` avoids it at the cost of relay bandwidth. + +## What players see + +Host (pause menu, Horizon style): **Invite friends** beside Open to LAN. It opens the world to the +network if it isn't (same settings as Open to LAN), registers with the relay and shows the code with a +**Copy** button, the friends connected, and **Stop sharing**. Stopping sharing ends the code; it leaves +Open to LAN as it was before sharing started. + +Guest: **Join a friend** on the multiplayer screen (a code field) and a **Join a friend** row on the +launcher's Play page, which starts the game straight into the join. Errors are plain: "No world uses +this code", "This invite has ended", "The code doesn't match (check the last four characters)", +"Your friend's game didn't answer", "Your friend plays Minecraft X; you have Y". + +Configuration: `HORIZON_RELAY_URL=host:port` at run time, else the address compiled in from the same +variable at build time. **Empty by default**: until a relay is deployed, the buttons say "Not available +yet" and do nothing else. + +## Failure modes + +| Failure | What happens | +| --- | --- | +| Relay down or not configured | Host: "Couldn't reach the invite service" (the world stays open to LAN if it was). Guest: same message. | +| Host closes the world / quits / loses the network | The relay drops the code ("ended"); connected guests see the usual disconnect. | +| Relay restarts | Codes end; hosts reconnect and get a **new** code (shown in the panel). Direct sessions keep running; relayed ones drop. | +| Direct attempt half works (one SYN through) | Only authenticated streams count; the rest are closed after 3 s. | +| A guest's NAT blocks everything but the relay | Relayed, at most `RELAY_SESSION_KBPS`. | +| Wrong secret | That guest gets "The code doesn't match"; after 5, the host gets a new code. | +| Guest on another version | Refused by the relay before any key exchange, with both versions in the message. | + +## Deploying the relay + +**Needs Ludovic**: nothing is deployed yet, and the game's default relay address is empty. + +The relay is one static binary, `horizon-relay`, with no disk state; it fits Railway's smallest +instance. Files: `crates/horizon-relay/Dockerfile` (build context: the repository root) and +`crates/horizon-relay/railway.toml`. + +1. New Railway service from the `ludovic111/horizon` repository; in its settings set the config file + path to `crates/horizon-relay/railway.toml` (it selects the Dockerfile and the health check). +2. Variables: `PORT=7000` (the relay listens on `$PORT`); optionally the limits above. **One + replica only** (`numReplicas = 1` in `railway.toml`): codes live in the relay's memory, and the + TCP proxy spreads connections over replicas at random. +3. Networking: add a **TCP proxy** to port 7000. Railway gives an address like + `example.proxy.rlwy.net:12345`. (An HTTP domain is not needed; the health check reaches the service + directly on `$PORT`.) +4. Health check: `GET /health` on `$PORT` (in `railway.toml`), answers `200` with counts. +5. Check addresses: `curl http:///whoami` from a home connection. If it shows your + public address, keep the defaults. If it shows a Railway address, set `RELAY_PER_ADDRESS_LIMITS=0` + (the global limits remain) and expect fewer direct connections across NATs. +6. Point the game at it: set the repository variable `HORIZON_RELAY_URL` to the TCP proxy address so + release builds compile it in, or test first with `HORIZON_RELAY_URL=… horizon-client`. + +Cost and bandwidth: the relay idles at a few MB of memory and no CPU. Sessions that connect directly +cost a few kilobytes of relay traffic each. A relayed player costs what the game sends: joining (the +chunks around the spawn) is a few MB, playing is roughly 10–50 KB/s, flying over new terrain up to a +few hundred KB/s (capped at `RELAY_SESSION_KBPS`). A relayed player-hour is therefore around 50–200 MB +of egress; 100 relayed player-hours a month is 5–20 GB. Check Railway's current egress, memory and +CPU prices against that; `RELAY_TOTAL_MBPS` bounds the worst case. + +## Testing + +- Rust unit tests: codes (alphabet, normalisation, parsing), frames and the key exchange (wrong secret, + tampering, nonce order), rate limiters and bandwidth buckets. +- `crates/horizon-relay/tests/tunnel.rs`: a relay, a host tunnel in front of an echo server standing in + for the integrated server, and guest tunnels, in one process: direct path, forced relay path, wrong + secret (and the 5-strike new code), unknown and ended codes, version mismatch, rate limits. +- `tools/test_friends.py`: a relay, a host Horizon with an integrated world and a guest Horizon, through + the direct path and the forced relay path; chat, movement and a placed block seen by both. +- Two machines over the tailnet (relay on one, host and guest on two others). + +## Pull requests + +1. `horizon-friends` (codes, wire, key exchange, tunnel ends) and `horizon-relay` (the service, its + Dockerfile and Railway config) with their tests. Nothing player-facing. +2. Host: Invite friends in the pause menu, `horizon-ctl friends …`. +3. Guest: Join a friend in game and in the launcher, `tools/test_friends.py`.