From cdfcc3d4d446e59c075e6fd0018a8cc599519cdc Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:15:35 +0200 Subject: [PATCH] Server resource packs: the client's pack manager and downloads Vanilla's ServerPackManager, DownloadedPackSource and DownloadQueue, without the game around them yet: the prompt state, each pack's download and activation, the statuses in vanilla's order (onSuccess repeats SUCCESSFULLY_LOADED, a failed reload recovers with the packs that loaded), pop, and leaving. Downloads follow HttpUtil.downloadFile: downloads//, a cached file whose hash matches is reused, a wrong hash fails, no hash downloads to a temporary file renamed to its SHA-1, 250 MiB at most, vanilla's X-Minecraft-* headers, a log.json entry per download, and 20 files kept. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/horizon-client/Cargo.toml | 2 + crates/horizon-client/src/lib.rs | 1 + crates/horizon-client/src/server_packs.rs | 618 ++++++++++++++++++ .../src/server_packs/download.rs | 482 ++++++++++++++ 5 files changed, 1104 insertions(+) create mode 100644 crates/horizon-client/src/server_packs.rs create mode 100644 crates/horizon-client/src/server_packs/download.rs diff --git a/Cargo.lock b/Cargo.lock index 1a1b288c9..d04c0a647 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1441,6 +1441,7 @@ dependencies = [ "rustix 1.1.5", "serde", "serde_json", + "sha1", "sha2", "symphonia", "toml", diff --git a/crates/horizon-client/Cargo.toml b/crates/horizon-client/Cargo.toml index 300872765..b11989ba8 100644 --- a/crates/horizon-client/Cargo.toml +++ b/crates/horizon-client/Cargo.toml @@ -27,6 +27,8 @@ bytemuck = { version = "1.24", features = ["derive"] } glam = "0.30" png = "0.18" sha2 = "0.10.9" +# Server resource packs are checked and cached by their SHA-1, as vanilla does. +sha1 = "0.10" serde = { version = "1", features = ["derive"] } serde_json = { version = "1.0.151", features = ["raw_value"] } # Shader packs: pack.toml, and WGSL checked before it reaches wgpu. diff --git a/crates/horizon-client/src/lib.rs b/crates/horizon-client/src/lib.rs index dddf87c0d..7ec79af5e 100644 --- a/crates/horizon-client/src/lib.rs +++ b/crates/horizon-client/src/lib.rs @@ -158,6 +158,7 @@ pub mod screen_effects; pub mod screen_layout; pub mod sculk; pub use horizon_server_files as server; +pub mod server_packs; pub mod settings; pub mod shader_pack; pub mod shadows; diff --git a/crates/horizon-client/src/server_packs.rs b/crates/horizon-client/src/server_packs.rs new file mode 100644 index 000000000..8e11217b5 --- /dev/null +++ b/crates/horizon-client/src/server_packs.rs @@ -0,0 +1,618 @@ +//! Server resource packs (`ServerPackManager`, `DownloadedPackSource`): the +//! packs a server pushes, the player's answer to its prompt, their downloads, +//! their activation on top of the player's own packs, and the status each one +//! reports back to the server. The game drives it: it shows the prompt, +//! starts the downloads [`ServerPacks::tick`] asks for and reloads the +//! resources with [`ServerPacks::active`] when it asks for a reload. +pub mod download; + +use std::path::PathBuf; + +/// `ServerboundResourcePackPacket.Action`, by its protocol number. +pub mod action { + pub const SUCCESSFULLY_LOADED: i32 = 0; + pub const DECLINED: i32 = 1; + pub const FAILED_DOWNLOAD: i32 = 2; + pub const ACCEPTED: i32 = 3; + pub const DOWNLOADED: i32 = 4; + pub const INVALID_URL: i32 = 5; + pub const FAILED_RELOAD: i32 = 6; + pub const DISCARDED: i32 = 7; +} + +/// `ServerPackManager.PackPromptStatus`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Prompt { + Pending, + Allowed, + Declined, +} +impl Prompt { + /// `ConnectScreen.convertPackStatus`: a server list entry's setting + /// (servers.dat `acceptTextures`; none is Prompt). + pub fn for_server(accept_textures: Option) -> Self { + match accept_textures { + Some(true) => Self::Allowed, + Some(false) => Self::Declined, + None => Self::Pending, + } + } +} + +/// `ClientCommonPacketListenerImpl.parseResourcePackUrl`: only http and https. +pub fn valid_url(url: &str) -> bool { + url.split_once(':').is_some_and(|(scheme, _)| { + scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https") + }) +} + +/// `DownloadedPackSource.tryParseSha1Hash`: 40 hex digits, else no check. +pub fn parse_sha1(hash: &str) -> Option { + (hash.len() == 40 && hash.bytes().all(|b| b.is_ascii_hexdigit())) + .then(|| hash.to_ascii_lowercase()) +} + +/// One pack to download (`DownloadQueue.DownloadRequest`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Request { + pub id: u128, + pub url: String, + pub sha1: Option, +} + +/// `DownloadQueue.BatchResult`: the file each downloaded pack is in, and the +/// packs that failed. +#[derive(Debug, Default)] +pub struct Batch { + pub downloaded: Vec<(u128, PathBuf)>, + pub failed: Vec, +} + +/// What the game has to do after a [`ServerPacks::tick`]. +#[derive(Debug, Default, PartialEq)] +pub struct Tick { + /// Download these, then hand the result to [`ServerPacks::downloaded`]. + pub download: Vec, + /// Reload the resources with these packs on top, bottom first + /// (`PackReloadConfig.scheduleReload`), then call [`ServerPacks::reloaded`]. + pub reload: Option>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Download { + Requested, + Pending, + Done, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Activation { + Inactive, + Pending, + Active, +} +/// `ServerPackManager.RemovalReason`, with the status it reports. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Removal { + DownloadFailed, + ActivationFailed, + Declined, + Discarded, + ServerRemoved, + ServerReplaced, +} +impl Removal { + fn reply(self) -> Option { + match self { + Self::DownloadFailed => Some(action::FAILED_DOWNLOAD), + Self::ActivationFailed => Some(action::FAILED_RELOAD), + Self::Declined => Some(action::DECLINED), + Self::Discarded => Some(action::DISCARDED), + Self::ServerRemoved | Self::ServerReplaced => None, + } + } +} + +/// `ServerPackManager.ServerPackData`. +#[derive(Debug)] +struct Pack { + id: u128, + url: String, + sha1: Option, + path: Option, + removal: Option, + download: Download, + activation: Activation, + accepted: bool, +} +impl Pack { + fn remove(&mut self, reason: Removal) { + self.removal.get_or_insert(reason); + } + fn removed(&self) -> bool { + self.removal.is_some() + } +} + +/// The reload in progress: the packs it loads and unloads (the callbacks +/// `triggerReloadIfNeeded` gives `scheduleReload`). +#[derive(Debug)] +struct Reload { + load: Vec, + unload: Vec, + /// It reloads what was active before a failed one (`isRecovery`). + recovery: bool, +} + +/// `ServerPackManager` with the feedback `DownloadedPackSource` gives it. +#[derive(Debug)] +pub struct ServerPacks { + packs: Vec, + prompt: Prompt, + reload: Option, + /// `registerForUpdate`: something changed, [`Self::tick`] again. + update: bool, + /// Report to the server (`configureForServerControl`); otherwise the + /// statuses are only logged, as for a local world. + reporting: bool, + replies: Vec<(u128, i32)>, +} +impl Default for ServerPacks { + fn default() -> Self { + Self { + packs: Vec::new(), + prompt: Prompt::Pending, + reload: None, + update: false, + reporting: false, + replies: Vec::new(), + } + } +} +impl ServerPacks { + fn report(&mut self, id: u128, status: i32) { + log::debug!("Horizon: server pack {id:032x} changed status to {status}"); + if self.reporting { + self.replies.push((id, status)); + } + } + /// The statuses to send the server, oldest first. + pub fn take_replies(&mut self) -> Vec<(u128, i32)> { + std::mem::take(&mut self.replies) + } + /// Whether [`Self::tick`] has something to do. + pub fn wants_tick(&self) -> bool { + self.update + } + + /// `DownloadedPackSource.configureForServerControl`. + pub fn connect(&mut self, prompt: Prompt) { + self.reporting = true; + match prompt { + Prompt::Allowed => self.allow(), + Prompt::Declined => self.reject(), + Prompt::Pending => self.prompt = Prompt::Pending, + } + } + /// `DownloadedPackSource.configureForLocalWorld`: single player's own + /// server; its packs are taken, and nothing is reported. + pub fn local_world(&mut self) { + self.reporting = false; + self.allow(); + } + /// `DownloadedPackSource.cleanupAfterDisconnect`: every pack goes, and + /// the next server starts from its own setting. + pub fn disconnect(&mut self) { + self.pop_all(); + self.reporting = false; + self.replies.clear(); + self.prompt = Prompt::Pending; + } + + pub fn push(&mut self, id: u128, url: &str, sha1: Option) { + if self.prompt == Prompt::Declined { + self.report(id, action::DECLINED); + return; + } + for pack in self.packs.iter_mut().filter(|p| p.id == id) { + pack.remove(Removal::ServerReplaced); + } + self.packs.push(Pack { + id, + url: url.to_owned(), + sha1, + path: None, + removal: None, + download: Download::Requested, + activation: Activation::Inactive, + accepted: false, + }); + if self.prompt == Prompt::Allowed { + self.accept(self.packs.len() - 1); + } + self.update = true; + } + fn accept(&mut self, i: usize) { + self.packs[i].accepted = true; + let id = self.packs[i].id; + self.report(id, action::ACCEPTED); + } + pub fn pop(&mut self, id: u128) { + if let Some(pack) = self.packs.iter_mut().find(|p| !p.removed() && p.id == id) { + pack.remove(Removal::ServerRemoved); + self.update = true; + } + } + pub fn pop_all(&mut self) { + for pack in &mut self.packs { + pack.remove(Removal::ServerRemoved); + } + self.update = true; + } + /// `allowServerPacks`: the prompt said yes (or the server is Enabled). + pub fn allow(&mut self) { + self.prompt = Prompt::Allowed; + for i in 0..self.packs.len() { + if !self.packs[i].accepted && !self.packs[i].removed() { + self.accept(i); + } + } + self.update = true; + } + /// `rejectServerPacks`: the prompt said no (or the server is Disabled). + pub fn reject(&mut self) { + self.prompt = Prompt::Declined; + for pack in self.packs.iter_mut().filter(|p| !p.accepted) { + pack.remove(Removal::Declined); + } + self.update = true; + } + + /// `ServerPackManager.tick`: start the downloads the accepted packs + /// need; once none is left, reload if the active packs should change; + /// then forget the packs that are gone, reporting why. + pub fn tick(&mut self) -> Tick { + self.update = false; + let mut out = Tick::default(); + let mut downloading = false; + for pack in self.packs.iter_mut().filter(|p| !p.removed() && p.accepted) { + downloading |= pack.download != Download::Done; + if pack.download == Download::Requested { + pack.download = Download::Pending; + out.download.push(Request { + id: pack.id, + url: pack.url.clone(), + sha1: pack.sha1.clone(), + }); + } + } + if !downloading { + out.reload = self.reload_if_needed(); + } + let mut gone = Vec::new(); + self.packs.retain(|p| { + let keep = p.activation != Activation::Inactive || p.removal.is_none(); + if !keep && let Some(reply) = p.removal.and_then(Removal::reply) { + gone.push((p.id, reply)); + } + keep + }); + for (id, reply) in gone { + self.report(id, reply); + } + out + } + fn reload_if_needed(&mut self) -> Option> { + if self.reload.is_some() + || self + .packs + .iter() + .any(|p| p.activation == Activation::Pending) + { + return None; + } + let mut changed = false; + let (mut load, mut unload) = (Vec::new(), Vec::new()); + for pack in &self.packs { + let wanted = pack.accepted && pack.download == Download::Done && !pack.removed(); + match pack.activation { + Activation::Inactive if wanted => { + load.push(pack.id); + changed = true; + } + Activation::Active if wanted => load.push(pack.id), + Activation::Active => { + unload.push(pack.id); + changed = true; + } + _ => {} + } + } + if !changed { + return None; + } + for pack in &mut self.packs { + let loading = load.contains(&pack.id) && pack.activation != Activation::Active; + if loading || unload.contains(&pack.id) { + pack.activation = Activation::Pending; + } + } + let paths = self.paths(&load); + self.reload = Some(Reload { + load, + unload, + recovery: false, + }); + Some(paths) + } + /// The packs to load (`PackReloadConfig.Callbacks.packsToLoad`), bottom first. + fn paths(&self, ids: &[u128]) -> Vec<(u128, PathBuf)> { + ids.iter() + .filter_map(|id| { + let pack = self + .packs + .iter() + .find(|p| p.id == *id && p.path.is_some())?; + Some((*id, pack.path.clone()?)) + }) + .collect() + } + + /// `ServerPackManager.onDownload`. + pub fn downloaded(&mut self, batch: Batch) { + if !batch.failed.is_empty() { + for pack in self + .packs + .iter_mut() + .filter(|p| p.activation != Activation::Active) + { + pack.remove(if batch.failed.contains(&pack.id) { + Removal::DownloadFailed + } else { + Removal::Discarded + }); + } + } + let mut done = Vec::new(); + for (id, path) in batch.downloaded { + if let Some(pack) = self + .packs + .iter_mut() + .find(|p| p.id == id && p.download == Download::Pending) + { + pack.download = Download::Done; + pack.path = Some(path); + if !pack.removed() { + done.push(id); + } + } + } + for id in done { + self.report(id, action::DOWNLOADED); + } + self.update = true; + } + + /// The reload [`Tick::reload`] asked for finished (`Callbacks.onSuccess`), + /// or its packs could not be loaded (`onFailure`). After a first failure + /// the packs that were active are loaded again: the packs to reload with + /// are returned, and this is called again with that reload's outcome. + pub fn reloaded(&mut self, ok: bool) -> Option> { + let mut reload = self.reload.take()?; + self.update = true; + if ok { + let mut applied = Vec::new(); + for pack in &mut self.packs { + if reload.load.contains(&pack.id) { + pack.activation = Activation::Active; + if pack.removal.is_none() { + applied.push(pack.id); + } + } else if reload.unload.contains(&pack.id) { + pack.activation = Activation::Inactive; + } + } + for id in applied { + self.report(id, action::SUCCESSFULLY_LOADED); + } + return None; + } + if reload.recovery { + // onFailure(true): nothing of the server's is loaded. + for pack in &mut self.packs { + if pack.activation == Activation::Pending { + pack.activation = Activation::Inactive; + } + } + return None; + } + reload.load.clear(); + for pack in &mut self.packs { + match pack.activation { + Activation::Inactive => pack.remove(Removal::Discarded), + Activation::Pending => { + pack.activation = Activation::Inactive; + pack.remove(Removal::ActivationFailed); + } + Activation::Active => reload.load.push(pack.id), + } + } + reload.recovery = true; + let paths = self.paths(&reload.load); + self.reload = Some(reload); + Some(paths) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use action::*; + + fn ok(id: u128) -> Batch { + Batch { + downloaded: vec![(id, PathBuf::from(format!("/p/{id}")))], + failed: vec![], + } + } + /// Runs the updates as `createUpdateScheduler` does, downloading and + /// reloading with `download` and `reload` as the game would. + fn settle( + sp: &mut ServerPacks, + download: &mut dyn FnMut(&[Request]) -> Batch, + reload: &mut dyn FnMut(&[(u128, PathBuf)]) -> bool, + ) { + let mut rounds = 0; + while sp.wants_tick() { + rounds += 1; + assert!(rounds < 50); + let t = sp.tick(); + if !t.download.is_empty() { + let batch = download(&t.download); + sp.downloaded(batch); + } + if let Some(mut packs) = t.reload { + while let Some(back) = sp.reloaded(reload(&packs)) { + packs = back; + } + } + } + } + + #[test] + fn an_enabled_server_s_pack_is_accepted_downloaded_and_loaded() { + let mut sp = ServerPacks::default(); + sp.connect(Prompt::Allowed); + sp.push(1, "http://a/p.zip", None); + let mut loaded = Vec::new(); + settle(&mut sp, &mut |r| ok(r[0].id), &mut |p| { + loaded = p.to_vec(); + true + }); + assert_eq!( + sp.take_replies(), + [(1, ACCEPTED), (1, DOWNLOADED), (1, SUCCESSFULLY_LOADED)] + ); + assert_eq!(loaded, [(1, PathBuf::from("/p/1"))]); + } + + #[test] + fn the_prompt_holds_packs_until_answered() { + let mut sp = ServerPacks::default(); + sp.connect(Prompt::Pending); + sp.push(1, "http://a/p.zip", None); + settle(&mut sp, &mut |_| unreachable!(), &mut |_| unreachable!()); + assert!(sp.take_replies().is_empty()); + sp.reject(); + settle(&mut sp, &mut |_| unreachable!(), &mut |_| unreachable!()); + assert_eq!(sp.take_replies(), [(1, DECLINED)]); + // Declined: later pushes are declined at once. + sp.push(2, "http://a/q.zip", None); + assert_eq!(sp.take_replies(), [(2, DECLINED)]); + } + + #[test] + fn a_failed_download_discards_the_rest_of_the_batch() { + let mut sp = ServerPacks::default(); + sp.connect(Prompt::Allowed); + sp.push(1, "http://a/p.zip", None); + sp.push(2, "http://a/dead.zip", None); + settle( + &mut sp, + &mut |_| Batch { + downloaded: vec![(1, PathBuf::from("/p/1"))], + failed: vec![2], + }, + &mut |_| unreachable!(), + ); + assert_eq!( + sp.take_replies(), + [ + (1, ACCEPTED), + (2, ACCEPTED), + (1, DISCARDED), + (2, FAILED_DOWNLOAD) + ] + ); + } + + #[test] + fn a_pack_that_does_not_load_fails_its_reload() { + let mut sp = ServerPacks::default(); + sp.connect(Prompt::Allowed); + sp.push(1, "http://a/p.zip", None); + let mut reloads = Vec::new(); + settle(&mut sp, &mut |r| ok(r[0].id), &mut |p| { + reloads.push(p.to_vec()); + reloads.len() > 1 + }); + assert_eq!( + sp.take_replies(), + [(1, ACCEPTED), (1, DOWNLOADED), (1, FAILED_RELOAD)] + ); + // The recovery reload has none of the server's packs. + assert_eq!(reloads[1], []); + } + + #[test] + fn popping_and_disconnecting_unload_the_packs() { + let mut sp = ServerPacks::default(); + sp.connect(Prompt::Allowed); + sp.push(1, "http://a/p.zip", None); + sp.push(2, "http://a/q.zip", None); + let last = std::cell::RefCell::new(Vec::new()); + let mut reload = |p: &[(u128, PathBuf)]| { + *last.borrow_mut() = p.to_vec(); + true + }; + settle( + &mut sp, + &mut |r| Batch { + downloaded: r + .iter() + .map(|q| (q.id, PathBuf::from(format!("/p/{}", q.id)))) + .collect(), + failed: vec![], + }, + &mut reload, + ); + sp.take_replies(); + sp.pop(1); + settle(&mut sp, &mut |_| unreachable!(), &mut reload); + // Callbacks.onSuccess reports every pack the reload loaded, again. + assert_eq!(sp.take_replies(), [(2, SUCCESSFULLY_LOADED)]); + assert_eq!(*last.borrow(), [(2, PathBuf::from("/p/2"))]); + sp.disconnect(); + settle(&mut sp, &mut |_| unreachable!(), &mut reload); + assert!(last.borrow().is_empty()); + assert!(sp.packs.is_empty()); + } + + #[test] + fn a_pushed_id_replaces_its_pack() { + let mut sp = ServerPacks::default(); + sp.connect(Prompt::Allowed); + sp.push(1, "http://a/old.zip", None); + sp.push(1, "http://a/new.zip", None); + let mut urls = Vec::new(); + settle( + &mut sp, + &mut |r| { + urls.extend(r.iter().map(|q| q.url.clone())); + ok(1) + }, + &mut |_| true, + ); + assert_eq!(urls, ["http://a/new.zip"]); + } + + #[test] + fn urls_and_hashes_are_read_as_vanilla_reads_them() { + assert!(valid_url("https://example.com/p.zip")); + assert!(valid_url("HTTP://example.com/p.zip")); + assert!(!valid_url("ftp://example.com/p.zip")); + assert!(!valid_url("example.com/p.zip")); + let sha1 = "0123456789ABCDEF0123456789abcdef01234567"; + assert_eq!(parse_sha1(sha1), Some(sha1.to_ascii_lowercase())); + assert_eq!(parse_sha1("abc"), None); + } +} diff --git a/crates/horizon-client/src/server_packs/download.rs b/crates/horizon-client/src/server_packs/download.rs new file mode 100644 index 000000000..595612df8 --- /dev/null +++ b/crates/horizon-client/src/server_packs/download.rs @@ -0,0 +1,482 @@ +//! Server pack downloads into the game directory's `downloads/` cache, as +//! vanilla keeps it (`DownloadQueue`, `HttpUtil.downloadFile`, +//! `DownloadCacheCleaner`): `downloads//`, one +//! batch at a time on a thread of its own, every download logged in +//! `downloads/log.json`, and the cache cut to its 20 newest files on start. +use super::{Batch, Request}; +use sha1::{Digest, Sha1}; +use std::{ + fs, + io::{self, Read, Write}, + path::{Path, PathBuf}, + sync::mpsc::{Receiver, Sender, channel}, + time::SystemTime, +}; + +/// `DownloadedPackSource`'s MAX_PACK_SIZE_BYTES (250 MiB). +pub const MAX_SIZE: u64 = 262_144_000; +/// `DownloadQueue.MAX_KEPT_PACKS`. +const MAX_KEPT: usize = 20; + +/// `createDownloadHeaders`: who is downloading, with which game. +pub fn headers(name: &str, uuid: u128) -> Vec<(&'static str, String)> { + use horizon_core::version::{RESOURCE_PACK_FORMAT, VANILLA_VERSION, VERSION_ID}; + vec![ + ("X-Minecraft-Username", name.to_owned()), + ("X-Minecraft-UUID", format!("{uuid:032x}")), + ("X-Minecraft-Version", VANILLA_VERSION.to_owned()), + ("X-Minecraft-Version-ID", VERSION_ID.to_owned()), + ( + "X-Minecraft-Pack-Format", + format!("{}.{}", RESOURCE_PACK_FORMAT.0, RESOURCE_PACK_FORMAT.1), + ), + ("User-Agent", format!("Minecraft Java/{VANILLA_VERSION}")), + ] +} + +type Job = (Vec, Vec<(&'static str, String)>); + +/// `DownloadQueue`: batches download in order, on one thread. +pub struct Queue { + jobs: Sender, + done: Receiver, +} +impl Queue { + pub fn open(dir: PathBuf) -> io::Result { + fs::create_dir_all(&dir)?; + vacuum(&dir, MAX_KEPT); + let (jobs, todo) = channel::(); + let (finished, done) = channel(); + std::thread::Builder::new() + .name("download-queue".into()) + .spawn(move || { + for (requests, headers) in todo { + if finished.send(run(&dir, &requests, &headers)).is_err() { + break; + } + } + })?; + Ok(Self { jobs, done }) + } + pub fn download(&self, requests: Vec, headers: Vec<(&'static str, String)>) { + let _ = self.jobs.send((requests, headers)); + } + /// The batches finished since the last call. + pub fn finished(&self) -> Vec { + self.done.try_iter().collect() + } +} + +/// `DownloadQueue.runDownload`: each pack into `//`, logged. +fn run(dir: &Path, requests: &[Request], headers: &[(&'static str, String)]) -> Batch { + let mut batch = Batch::default(); + for request in requests { + let id = horizon_protocol::crypto::uuid_string(request.id); + let result = download_file(&dir.join(&id), request, headers, MAX_SIZE); + // DownloadQueue.LogEntry, its fields in the codec's order. + let mut entry = vec![ + ("id", serde_json::Value::from(id)), + ("url", request.url.as_str().into()), + ( + "time", + chrono::Utc::now() + .to_rfc3339_opts(chrono::SecondsFormat::AutoSi, true) + .into(), + ), + ]; + if let Some(sha1) = &request.sha1 { + entry.push(("hash", sha1.as_str().into())); + } + entry.push(match &result { + Ok(path) => match fs::metadata(path) { + Ok(meta) => { + let name = path.strip_prefix(dir).unwrap_or(path); + let name = name.to_string_lossy().replace('\\', "/"); + ( + "file", + serde_json::json!({"name": name, "size": meta.len()}), + ) + } + Err(_) => ("error", "no_access".into()), + }, + Err(e) => { + log::error!("Horizon: failed to download {}: {e}", request.url); + ("error", "download_failed".into()) + } + }); + if let Err(e) = append_log(&dir.join("log.json"), &entry) { + log::error!("Horizon: failed to log download of {}: {e}", request.url); + } + match result { + Ok(path) => batch.downloaded.push((request.id, path)), + Err(_) => batch.failed.push(request.id), + } + } + batch +} + +/// `JsonEventLog.write`: one JSON object a line, as Gson writes it (with +/// its HTML escaping). +fn append_log(path: &Path, entry: &[(&str, serde_json::Value)]) -> io::Result<()> { + let fields: Vec = entry + .iter() + .map(|(k, v)| format!("{}:{v}", serde_json::Value::from(*k))) + .collect(); + let mut line = String::new(); + for c in format!("{{{}}}", fields.join(",")).chars() { + match c { + '<' | '>' | '&' | '=' | '\'' => line.push_str(&format!("\\u{:04x}", c as u32)), + c => line.push(c), + } + } + line.push('\n'); + fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)? + .write_all(line.as_bytes()) +} + +fn sha1_of(path: &Path) -> io::Result { + let mut hasher = Sha1::new(); + io::copy(&mut fs::File::open(path)?, &mut hasher)?; + Ok(hex(&hasher.finalize())) +} +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} +/// `checkExistingFile`: the file is there with that hash. +fn matches(path: &Path, sha1: &str) -> bool { + path.exists() && sha1_of(path).is_ok_and(|actual| actual == sha1) +} +/// `updateModificationTime`: used now, so the cleaner keeps it longest. +fn touch(path: &Path) { + let _ = fs::File::options() + .append(true) + .open(path) + .and_then(|f| f.set_modified(SystemTime::now())); +} + +/// `HttpUtil.downloadFile`: into `/`. With a hash the cached file +/// is used when it matches, and a download that does not is an error; +/// without one the file is named after the hash of what came. +pub fn download_file( + dir: &Path, + request: &Request, + headers: &[(&'static str, String)], + max_size: u64, +) -> Result { + let target = request.sha1.as_ref().map(|sha1| dir.join(sha1)); + if let (Some(target), Some(sha1)) = (&target, &request.sha1) { + if matches(target, sha1) { + log::info!("Horizon: returning cached file since actual hash matches requested"); + touch(target); + return Ok(target.clone()); + } + if target.exists() { + log::warn!( + "Horizon: existing file {} not found or had mismatched hash", + target.display() + ); + } + fs::remove_file(target) + .or_else(|e| { + if e.kind() == io::ErrorKind::NotFound { + Ok(()) + } else { + Err(e) + } + }) + .map_err(|e| format!("Failed to remove existing file {}: {e}", target.display()))?; + } + let mut call = ureq::AgentBuilder::new().build().get(&request.url); + for (name, value) in headers { + call = call.set(name, value); + } + let response = call.call().map_err(|e| e.to_string())?; + let size = response + .header("Content-Length") + .and_then(|v| v.trim().parse::().ok()); + fs::create_dir_all(dir).map_err(|e| e.to_string())?; + if let Some(size) = size + && size > max_size + { + return Err(format!( + "Filesize is bigger than maximum allowed (file is {size}, limit is {max_size})" + )); + } + let mut input = response.into_reader(); + match (&target, &request.sha1) { + (Some(target), Some(sha1)) => { + let actual = save_hashed(&mut input, target, max_size).map_err(|e| e.to_string())?; + if &actual != sha1 { + return Err(format!( + "Hash of downloaded file ({actual}) did not match requested ({sha1})" + )); + } + Ok(target.clone()) + } + _ => { + let tmp = dir.join(format!("download{:016x}.tmp", rand_suffix())); + let result = save_hashed(&mut input, &tmp, max_size).and_then(|actual| { + let path = dir.join(&actual); + if matches(&path, &actual) { + touch(&path); + } else { + fs::rename(&tmp, &path)?; + } + Ok(path) + }); + let _ = fs::remove_file(&tmp); + result.map_err(|e| e.to_string()) + } + } +} +fn rand_suffix() -> u64 { + use std::hash::{BuildHasher, RandomState}; + RandomState::new().hash_one(std::time::Instant::now()) +} +/// `downloadAndHash`: write what comes, at most `max_size` bytes. +fn save_hashed(input: &mut dyn Read, path: &Path, max_size: u64) -> io::Result { + let mut out = fs::File::create(path)?; + let mut hasher = Sha1::new(); + let mut buffer = [0u8; 8196]; + let mut read_so_far = 0u64; + loop { + let n = input.read(&mut buffer)?; + if n == 0 { + break; + } + read_so_far += n as u64; + if read_so_far > max_size { + return Err(io::Error::other(format!( + "Filesize was bigger than maximum allowed (got >= {read_so_far}, limit was {max_size})" + ))); + } + out.write_all(&buffer[..n])?; + hasher.update(&buffer[..n]); + } + Ok(hex(&hasher.finalize())) +} + +/// `DownloadCacheCleaner.vacuumCacheDir`: keep the `max_files` files most +/// worth keeping (in the pack folders: the newest, and one per pack before a +/// second of any), then drop the folders left empty. +pub fn vacuum(dir: &Path, max_files: usize) { + let mut files = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + let Ok(entries) = fs::read_dir(&d) else { + continue; + }; + for e in entries.flatten() { + let path = e.path(); + let Ok(meta) = e.metadata() else { continue }; + if meta.is_dir() { + stack.push(path); + } else if meta.is_file() && path.parent() != Some(dir) { + let time = meta.modified().unwrap_or(SystemTime::UNIX_EPOCH); + files.push((path, time)); + } + } + } + let Some(remove) = files.len().checked_sub(max_files).filter(|n| *n > 0) else { + return; + }; + // Newest first, then each file's rank in its folder (0 for a folder's + // newest), reversed, and stably sorted by rank, highest first. + files.sort_by_key(|f| std::cmp::Reverse(f.1)); + let mut counts = std::collections::HashMap::::new(); + let mut ranked: Vec<(PathBuf, usize)> = files + .into_iter() + .map(|(path, _)| { + let count = counts + .entry(path.parent().unwrap_or(dir).to_path_buf()) + .or_default(); + let rank = *count; + *count += 1; + (path, rank) + }) + .collect(); + ranked.reverse(); + ranked.sort_by_key(|f| std::cmp::Reverse(f.1)); + let mut emptied = std::collections::HashSet::new(); + for (path, rank) in ranked.into_iter().take(remove) { + match fs::remove_file(&path) { + Ok(()) if rank == 0 => { + emptied.insert(path.parent().unwrap_or(dir).to_path_buf()); + } + Ok(()) => {} + Err(e) => log::warn!( + "Horizon: failed to delete cache file {}: {e}", + path.display() + ), + } + } + emptied.remove(dir); + for d in emptied { + let _ = fs::remove_dir(d); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + + /// A one-shot HTTP server answering `status` with `body`; the request it got. + fn serve(status: &str, body: Vec) -> (String, std::thread::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/pack.zip", listener.local_addr().unwrap()); + let status = status.to_owned(); + let handle = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + let mut buf = [0u8; 1024]; + while !request.ends_with(b"\r\n\r\n") { + let n = stream.read(&mut buf).unwrap(); + request.extend_from_slice(&buf[..n]); + } + let head = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(head.as_bytes()).unwrap(); + stream.write_all(&body).unwrap(); + String::from_utf8_lossy(&request).into_owned() + }); + (url, handle) + } + fn sha1(bytes: &[u8]) -> String { + hex(&Sha1::digest(bytes)) + } + fn scratch() -> PathBuf { + let dir = std::env::temp_dir().join(format!("horizon-packs-{:016x}", rand_suffix())); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn a_pack_lands_under_its_hash_and_is_reused_from_the_cache() { + let dir = scratch(); + let body = b"PK pack".to_vec(); + let (url, server) = serve("200 OK", body.clone()); + let request = Request { + id: 7, + url, + sha1: Some(sha1(&body)), + }; + let path = download_file(&dir, &request, &headers("Alex", 0xab), MAX_SIZE).unwrap(); + assert_eq!(path, dir.join(sha1(&body))); + assert_eq!(fs::read(&path).unwrap(), body); + let sent = server.join().unwrap(); + assert!(sent.contains("X-Minecraft-Username: Alex"), "{sent}"); + assert!(sent.contains("X-Minecraft-Pack-Format: 97.1"), "{sent}"); + // Cached: no server this time. + let cached = Request { + url: "http://127.0.0.1:9/gone.zip".into(), + ..request + }; + assert_eq!(download_file(&dir, &cached, &[], MAX_SIZE).unwrap(), path); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn a_wrong_hash_a_dead_url_and_a_large_pack_fail() { + let dir = scratch(); + let (url, server) = serve("200 OK", b"other bytes".to_vec()); + let wrong = Request { + id: 1, + url, + sha1: Some("0".repeat(40)), + }; + let err = download_file(&dir, &wrong, &[], MAX_SIZE).unwrap_err(); + assert!(err.contains("did not match requested"), "{err}"); + server.join().unwrap(); + let dead = Request { + id: 2, + url: "http://127.0.0.1:9/pack.zip".into(), + sha1: None, + }; + assert!(download_file(&dir.join("dead"), &dead, &[], MAX_SIZE).is_err()); + assert!(!dir.join("dead").exists()); + let (url, server) = serve("200 OK", vec![0; 100]); + let large = Request { + id: 3, + url, + sha1: None, + }; + let err = download_file(&dir, &large, &[], 10).unwrap_err(); + assert!(err.contains("bigger than maximum"), "{err}"); + let _ = server.join(); + let (url, server) = serve("404 Not Found", Vec::new()); + let missing = Request { + id: 4, + url, + sha1: None, + }; + assert!(download_file(&dir, &missing, &[], MAX_SIZE).is_err()); + server.join().unwrap(); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn the_queue_logs_each_download() { + let dir = scratch(); + let (url, server) = serve("200 OK", b"zip".to_vec()); + let queue = Queue::open(dir.clone()).unwrap(); + let id = 0x5b2c_7a4e_1f3d_4c8b_9a6e_2d4f_8b1c_3e5a; + queue.download( + vec![Request { + id, + url, + sha1: None, + }], + Vec::new(), + ); + let batch = loop { + if let Some(b) = queue.finished().pop() { + break b; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + }; + server.join().unwrap(); + let file = dir + .join("5b2c7a4e-1f3d-4c8b-9a6e-2d4f8b1c3e5a") + .join(sha1(b"zip")); + assert_eq!(batch.downloaded, [(id, file)]); + let log = fs::read_to_string(dir.join("log.json")).unwrap(); + let entry: serde_json::Value = serde_json::from_str(log.trim()).unwrap(); + assert_eq!(entry["id"], "5b2c7a4e-1f3d-4c8b-9a6e-2d4f8b1c3e5a"); + assert_eq!(entry["file"]["size"], 3); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn the_cleaner_keeps_the_newest_file_of_each_pack_longest() { + let dir = scratch(); + let at = |secs| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(secs); + for (pack, file, time) in [ + ("a", "1", 10), + ("a", "2", 20), + ("a", "3", 30), + ("b", "1", 5), + ] { + fs::create_dir_all(dir.join(pack)).unwrap(); + let path = dir.join(pack).join(file); + fs::write(&path, b"x").unwrap(); + fs::File::options() + .append(true) + .open(&path) + .unwrap() + .set_modified(at(time)) + .unwrap(); + } + fs::write(dir.join("log.json"), b"").unwrap(); + vacuum(&dir, 2); + assert!(dir.join("a/3").exists() && dir.join("b/1").exists()); + assert!(!dir.join("a/1").exists() && !dir.join("a/2").exists()); + assert!(dir.join("log.json").exists()); + fs::remove_dir_all(dir).unwrap(); + } +}