From 0605ee689f05152bd9a53980bc84675eb8bc8f09 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:57:49 +0200 Subject: [PATCH] LAN: compress a large packet once for every connection Each player's writer thread compressed every packet it sent, so a chunk column sent to fifty players was deflated fifty times; compression was most of the writer threads' time and about half of the server's CPU. Bodies of 8 KiB or more (chunk columns, mostly) now go through a shared cache of compressed frame contents keyed by their length, two independently keyed 64-bit hashes and the threshold, holding the newest 32 MiB. The bytes on the wire are the same: frame::encode is split into frame::contents and frame::prefixed, and ConnWriter::send_contents sends a frame from its contents. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/lan/conn.rs | 113 +++++++++++++++++++++- crates/horizon-protocol/src/connection.rs | 7 +- crates/horizon-protocol/src/frame.rs | 19 +++- 3 files changed, 133 insertions(+), 6 deletions(-) diff --git a/crates/horizon-client/src/lan/conn.rs b/crates/horizon-client/src/lan/conn.rs index 77faa7e6f..6229c73b5 100644 --- a/crates/horizon-client/src/lan/conn.rs +++ b/crates/horizon-client/src/lan/conn.rs @@ -748,6 +748,82 @@ fn read_loop( } } +/// Packet bodies at least this large are compressed once for every +/// connection: chunk columns, mostly, which every player near them receives. +const SHARED_FRAME_MIN: usize = 8 << 10; +/// The compressed frames kept, in bytes; the oldest go first. +const SHARED_FRAME_BYTES: usize = 32 << 20; + +/// Compressed frame contents by the body they were made from: its length, +/// two independently keyed 64-bit hashes of it, and the threshold. +struct SharedFrames { + hashers: (std::hash::RandomState, std::hash::RandomState), + frames: Mutex, + /// [`SHARED_FRAME_BYTES`]. + limit: usize, +} +type FrameKey = (usize, u64, u64, usize); +#[derive(Default)] +struct FrameCache { + frames: crate::fast_hash::FxHashMap>, + order: std::collections::VecDeque, + bytes: usize, +} +impl SharedFrames { + fn get() -> &'static Self { + static FRAMES: std::sync::OnceLock = std::sync::OnceLock::new(); + FRAMES.get_or_init(|| Self::new(SHARED_FRAME_BYTES)) + } + fn new(limit: usize) -> Self { + Self { + hashers: Default::default(), + frames: Mutex::default(), + limit, + } + } + /// `frame::contents(body, Some(threshold))`, compressed only when no + /// connection has sent the same body lately. + fn contents(&self, body: &[u8], threshold: usize) -> horizon_protocol::Result> { + use std::hash::BuildHasher; + let key = ( + body.len(), + self.hashers.0.hash_one(body), + self.hashers.1.hash_one(body), + threshold, + ); + let cache = || self.frames.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(contents) = cache().frames.get(&key) { + return Ok(contents.clone()); + } + let contents: Arc<[u8]> = horizon_protocol::frame::contents(body, Some(threshold))?.into(); + let mut cache = cache(); + if cache.frames.insert(key, contents.clone()).is_none() { + cache.order.push_back(key); + cache.bytes += contents.len(); + while cache.bytes > self.limit + && let Some(old) = cache.order.pop_front() + { + if let Some(c) = cache.frames.remove(&old) { + cache.bytes -= c.len(); + } + } + } + Ok(contents) + } +} + +/// Send one packet body: a large one compressed through [`SharedFrames`], +/// so a chunk column sent to fifty players is compressed once, not fifty +/// times. The bytes on the wire are the same. +fn send_body(writer: &mut ConnWriter, body: &[u8]) -> horizon_protocol::Result<()> { + match writer.threshold { + Some(t) if body.len() >= t.max(SHARED_FRAME_MIN) => { + writer.send_contents(&SharedFrames::get().contents(body, t)?) + } + _ => writer.send(body), + } +} + fn write_loop( mut writer: ConnWriter, rx: Receiver, @@ -761,7 +837,7 @@ fn write_loop( loop { let mut write = |body: Vec| { pending.fetch_sub(body.len(), Ordering::Relaxed); - writer.send(&body).is_ok() && pending.load(Ordering::Relaxed) <= MAX_PENDING + send_body(&mut writer, &body).is_ok() && pending.load(Ordering::Relaxed) <= MAX_PENDING }; match rx.recv_timeout(Duration::from_millis(200)) { Ok(Out::Packet(body)) => { @@ -830,6 +906,41 @@ mod tests { .collect() } + #[test] + fn shared_frames_are_the_frames_compressed_here() { + use horizon_protocol::frame; + let frames = SharedFrames::new(usize::MAX); + let a: Vec = (0..40_000u32).map(|i| (i * 7 % 13) as u8).collect(); + let mut b = a.clone(); + b[20_000] ^= 1; + for (body, threshold) in [(&a, 256), (&b, 256), (&a, 256), (&a, 64), (&b, 256)] { + let shared = frames.contents(body, threshold).unwrap(); + assert_eq!(*shared, *frame::contents(body, Some(threshold)).unwrap()); + } + // One frame per body and threshold. + assert_eq!(frames.frames.lock().unwrap().frames.len(), 3); + } + + #[test] + fn shared_frames_keep_the_newest_within_their_limit() { + let first = SharedFrames::new(0) + .contents(&[1; 9000], 256) + .unwrap() + .len(); + let frames = SharedFrames::new(first * 2); + for n in 0..5u8 { + frames.contents(&[n; 9000], 256).unwrap(); + } + let cache = frames.frames.lock().unwrap(); + assert!((1..=2).contains(&cache.frames.len())); + assert_eq!(cache.order.len(), cache.frames.len()); + assert_eq!( + cache.bytes, + cache.frames.values().map(|c| c.len()).sum::() + ); + assert!(cache.bytes <= first * 2); + } + #[test] fn held_packets_go_together_and_in_order() { let (tx, rx) = channel(); diff --git a/crates/horizon-protocol/src/connection.rs b/crates/horizon-protocol/src/connection.rs index 4deda020c..8f7ae28cc 100644 --- a/crates/horizon-protocol/src/connection.rs +++ b/crates/horizon-protocol/src/connection.rs @@ -289,7 +289,12 @@ impl ConnReader { impl ConnWriter { /// Send one packet body (VarInt id + payload). pub fn send(&mut self, body: &[u8]) -> Result<()> { - let mut framed = frame::encode(body, self.threshold)?; + self.send_contents(&frame::contents(body, self.threshold)?) + } + /// Send a frame whose contents [`frame::contents`] made with this + /// writer's threshold. + pub fn send_contents(&mut self, contents: &[u8]) -> Result<()> { + let mut framed = frame::prefixed(contents); if let Some(c) = &mut self.encrypt { c.encrypt(&mut framed); } diff --git a/crates/horizon-protocol/src/frame.rs b/crates/horizon-protocol/src/frame.rs index 9a1402e0b..a403148f5 100644 --- a/crates/horizon-protocol/src/frame.rs +++ b/crates/horizon-protocol/src/frame.rs @@ -16,6 +16,12 @@ pub const MAX_UNCOMPRESSED: usize = 8_388_608; /// length prefix, and with compression enabled the uncompressed-length /// field (0 when below the threshold). pub fn encode(body: &[u8], threshold: Option) -> Result> { + Ok(prefixed(&contents(body, threshold)?)) +} + +/// What [`encode`] puts after the length prefix: the same for every +/// connection with the same threshold. +pub fn contents(body: &[u8], threshold: Option) -> Result> { let inner = match threshold { None => body.to_vec(), Some(t) if body.len() < t => { @@ -38,10 +44,15 @@ pub fn encode(body: &[u8], threshold: Option) -> Result> { inner.len() ))); } - let mut out = Vec::with_capacity(inner.len() + 3); - write_varint(&mut out, inner.len() as i32); - out.extend_from_slice(&inner); - Ok(out) + Ok(inner) +} + +/// A frame: the length prefix, then the [`contents`]. +pub fn prefixed(contents: &[u8]) -> Vec { + let mut out = Vec::with_capacity(contents.len() + 3); + write_varint(&mut out, contents.len() as i32); + out.extend_from_slice(contents); + out } /// Split one complete frame off the front of `buffer`, if available.