Server resource packs: the client's pack manager and downloads

Vanilla's ServerPackManager, DownloadedPackSource and DownloadQueue, without
the game around them yet: the prompt state, each pack's download and
activation, the statuses in vanilla's order (onSuccess repeats
SUCCESSFULLY_LOADED, a failed reload recovers with the packs that loaded),
pop, and leaving.

Downloads follow HttpUtil.downloadFile: downloads/<pack id>/<sha1>, a cached
file whose hash matches is reused, a wrong hash fails, no hash downloads to a
temporary file renamed to its SHA-1, 250 MiB at most, vanilla's X-Minecraft-*
headers, a log.json entry per download, and 20 files kept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-24 17:15:35 +02:00
parent 535f2d2a04
commit cdfcc3d4d4
5 changed files with 1104 additions and 0 deletions
Generated
+1
View File
@@ -1441,6 +1441,7 @@ dependencies = [
"rustix 1.1.5",
"serde",
"serde_json",
"sha1",
"sha2",
"symphonia",
"toml",
+2
View File
@@ -27,6 +27,8 @@ bytemuck = { version = "1.24", features = ["derive"] }
glam = "0.30"
png = "0.18"
sha2 = "0.10.9"
# Server resource packs are checked and cached by their SHA-1, as vanilla does.
sha1 = "0.10"
serde = { version = "1", features = ["derive"] }
serde_json = { version = "1.0.151", features = ["raw_value"] }
# Shader packs: pack.toml, and WGSL checked before it reaches wgpu.
+1
View File
@@ -158,6 +158,7 @@ pub mod screen_effects;
pub mod screen_layout;
pub mod sculk;
pub use horizon_server_files as server;
pub mod server_packs;
pub mod settings;
pub mod shader_pack;
pub mod shadows;
+618
View File
@@ -0,0 +1,618 @@
//! Server resource packs (`ServerPackManager`, `DownloadedPackSource`): the
//! packs a server pushes, the player's answer to its prompt, their downloads,
//! their activation on top of the player's own packs, and the status each one
//! reports back to the server. The game drives it: it shows the prompt,
//! starts the downloads [`ServerPacks::tick`] asks for and reloads the
//! resources with [`ServerPacks::active`] when it asks for a reload.
pub mod download;
use std::path::PathBuf;
/// `ServerboundResourcePackPacket.Action`, by its protocol number.
pub mod action {
pub const SUCCESSFULLY_LOADED: i32 = 0;
pub const DECLINED: i32 = 1;
pub const FAILED_DOWNLOAD: i32 = 2;
pub const ACCEPTED: i32 = 3;
pub const DOWNLOADED: i32 = 4;
pub const INVALID_URL: i32 = 5;
pub const FAILED_RELOAD: i32 = 6;
pub const DISCARDED: i32 = 7;
}
/// `ServerPackManager.PackPromptStatus`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Prompt {
Pending,
Allowed,
Declined,
}
impl Prompt {
/// `ConnectScreen.convertPackStatus`: a server list entry's setting
/// (servers.dat `acceptTextures`; none is Prompt).
pub fn for_server(accept_textures: Option<bool>) -> Self {
match accept_textures {
Some(true) => Self::Allowed,
Some(false) => Self::Declined,
None => Self::Pending,
}
}
}
/// `ClientCommonPacketListenerImpl.parseResourcePackUrl`: only http and https.
pub fn valid_url(url: &str) -> bool {
url.split_once(':').is_some_and(|(scheme, _)| {
scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https")
})
}
/// `DownloadedPackSource.tryParseSha1Hash`: 40 hex digits, else no check.
pub fn parse_sha1(hash: &str) -> Option<String> {
(hash.len() == 40 && hash.bytes().all(|b| b.is_ascii_hexdigit()))
.then(|| hash.to_ascii_lowercase())
}
/// One pack to download (`DownloadQueue.DownloadRequest`).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Request {
pub id: u128,
pub url: String,
pub sha1: Option<String>,
}
/// `DownloadQueue.BatchResult`: the file each downloaded pack is in, and the
/// packs that failed.
#[derive(Debug, Default)]
pub struct Batch {
pub downloaded: Vec<(u128, PathBuf)>,
pub failed: Vec<u128>,
}
/// What the game has to do after a [`ServerPacks::tick`].
#[derive(Debug, Default, PartialEq)]
pub struct Tick {
/// Download these, then hand the result to [`ServerPacks::downloaded`].
pub download: Vec<Request>,
/// Reload the resources with these packs on top, bottom first
/// (`PackReloadConfig.scheduleReload`), then call [`ServerPacks::reloaded`].
pub reload: Option<Vec<(u128, PathBuf)>>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Download {
Requested,
Pending,
Done,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Activation {
Inactive,
Pending,
Active,
}
/// `ServerPackManager.RemovalReason`, with the status it reports.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Removal {
DownloadFailed,
ActivationFailed,
Declined,
Discarded,
ServerRemoved,
ServerReplaced,
}
impl Removal {
fn reply(self) -> Option<i32> {
match self {
Self::DownloadFailed => Some(action::FAILED_DOWNLOAD),
Self::ActivationFailed => Some(action::FAILED_RELOAD),
Self::Declined => Some(action::DECLINED),
Self::Discarded => Some(action::DISCARDED),
Self::ServerRemoved | Self::ServerReplaced => None,
}
}
}
/// `ServerPackManager.ServerPackData`.
#[derive(Debug)]
struct Pack {
id: u128,
url: String,
sha1: Option<String>,
path: Option<PathBuf>,
removal: Option<Removal>,
download: Download,
activation: Activation,
accepted: bool,
}
impl Pack {
fn remove(&mut self, reason: Removal) {
self.removal.get_or_insert(reason);
}
fn removed(&self) -> bool {
self.removal.is_some()
}
}
/// The reload in progress: the packs it loads and unloads (the callbacks
/// `triggerReloadIfNeeded` gives `scheduleReload`).
#[derive(Debug)]
struct Reload {
load: Vec<u128>,
unload: Vec<u128>,
/// It reloads what was active before a failed one (`isRecovery`).
recovery: bool,
}
/// `ServerPackManager` with the feedback `DownloadedPackSource` gives it.
#[derive(Debug)]
pub struct ServerPacks {
packs: Vec<Pack>,
prompt: Prompt,
reload: Option<Reload>,
/// `registerForUpdate`: something changed, [`Self::tick`] again.
update: bool,
/// Report to the server (`configureForServerControl`); otherwise the
/// statuses are only logged, as for a local world.
reporting: bool,
replies: Vec<(u128, i32)>,
}
impl Default for ServerPacks {
fn default() -> Self {
Self {
packs: Vec::new(),
prompt: Prompt::Pending,
reload: None,
update: false,
reporting: false,
replies: Vec::new(),
}
}
}
impl ServerPacks {
fn report(&mut self, id: u128, status: i32) {
log::debug!("Horizon: server pack {id:032x} changed status to {status}");
if self.reporting {
self.replies.push((id, status));
}
}
/// The statuses to send the server, oldest first.
pub fn take_replies(&mut self) -> Vec<(u128, i32)> {
std::mem::take(&mut self.replies)
}
/// Whether [`Self::tick`] has something to do.
pub fn wants_tick(&self) -> bool {
self.update
}
/// `DownloadedPackSource.configureForServerControl`.
pub fn connect(&mut self, prompt: Prompt) {
self.reporting = true;
match prompt {
Prompt::Allowed => self.allow(),
Prompt::Declined => self.reject(),
Prompt::Pending => self.prompt = Prompt::Pending,
}
}
/// `DownloadedPackSource.configureForLocalWorld`: single player's own
/// server; its packs are taken, and nothing is reported.
pub fn local_world(&mut self) {
self.reporting = false;
self.allow();
}
/// `DownloadedPackSource.cleanupAfterDisconnect`: every pack goes, and
/// the next server starts from its own setting.
pub fn disconnect(&mut self) {
self.pop_all();
self.reporting = false;
self.replies.clear();
self.prompt = Prompt::Pending;
}
pub fn push(&mut self, id: u128, url: &str, sha1: Option<String>) {
if self.prompt == Prompt::Declined {
self.report(id, action::DECLINED);
return;
}
for pack in self.packs.iter_mut().filter(|p| p.id == id) {
pack.remove(Removal::ServerReplaced);
}
self.packs.push(Pack {
id,
url: url.to_owned(),
sha1,
path: None,
removal: None,
download: Download::Requested,
activation: Activation::Inactive,
accepted: false,
});
if self.prompt == Prompt::Allowed {
self.accept(self.packs.len() - 1);
}
self.update = true;
}
fn accept(&mut self, i: usize) {
self.packs[i].accepted = true;
let id = self.packs[i].id;
self.report(id, action::ACCEPTED);
}
pub fn pop(&mut self, id: u128) {
if let Some(pack) = self.packs.iter_mut().find(|p| !p.removed() && p.id == id) {
pack.remove(Removal::ServerRemoved);
self.update = true;
}
}
pub fn pop_all(&mut self) {
for pack in &mut self.packs {
pack.remove(Removal::ServerRemoved);
}
self.update = true;
}
/// `allowServerPacks`: the prompt said yes (or the server is Enabled).
pub fn allow(&mut self) {
self.prompt = Prompt::Allowed;
for i in 0..self.packs.len() {
if !self.packs[i].accepted && !self.packs[i].removed() {
self.accept(i);
}
}
self.update = true;
}
/// `rejectServerPacks`: the prompt said no (or the server is Disabled).
pub fn reject(&mut self) {
self.prompt = Prompt::Declined;
for pack in self.packs.iter_mut().filter(|p| !p.accepted) {
pack.remove(Removal::Declined);
}
self.update = true;
}
/// `ServerPackManager.tick`: start the downloads the accepted packs
/// need; once none is left, reload if the active packs should change;
/// then forget the packs that are gone, reporting why.
pub fn tick(&mut self) -> Tick {
self.update = false;
let mut out = Tick::default();
let mut downloading = false;
for pack in self.packs.iter_mut().filter(|p| !p.removed() && p.accepted) {
downloading |= pack.download != Download::Done;
if pack.download == Download::Requested {
pack.download = Download::Pending;
out.download.push(Request {
id: pack.id,
url: pack.url.clone(),
sha1: pack.sha1.clone(),
});
}
}
if !downloading {
out.reload = self.reload_if_needed();
}
let mut gone = Vec::new();
self.packs.retain(|p| {
let keep = p.activation != Activation::Inactive || p.removal.is_none();
if !keep && let Some(reply) = p.removal.and_then(Removal::reply) {
gone.push((p.id, reply));
}
keep
});
for (id, reply) in gone {
self.report(id, reply);
}
out
}
fn reload_if_needed(&mut self) -> Option<Vec<(u128, PathBuf)>> {
if self.reload.is_some()
|| self
.packs
.iter()
.any(|p| p.activation == Activation::Pending)
{
return None;
}
let mut changed = false;
let (mut load, mut unload) = (Vec::new(), Vec::new());
for pack in &self.packs {
let wanted = pack.accepted && pack.download == Download::Done && !pack.removed();
match pack.activation {
Activation::Inactive if wanted => {
load.push(pack.id);
changed = true;
}
Activation::Active if wanted => load.push(pack.id),
Activation::Active => {
unload.push(pack.id);
changed = true;
}
_ => {}
}
}
if !changed {
return None;
}
for pack in &mut self.packs {
let loading = load.contains(&pack.id) && pack.activation != Activation::Active;
if loading || unload.contains(&pack.id) {
pack.activation = Activation::Pending;
}
}
let paths = self.paths(&load);
self.reload = Some(Reload {
load,
unload,
recovery: false,
});
Some(paths)
}
/// The packs to load (`PackReloadConfig.Callbacks.packsToLoad`), bottom first.
fn paths(&self, ids: &[u128]) -> Vec<(u128, PathBuf)> {
ids.iter()
.filter_map(|id| {
let pack = self
.packs
.iter()
.find(|p| p.id == *id && p.path.is_some())?;
Some((*id, pack.path.clone()?))
})
.collect()
}
/// `ServerPackManager.onDownload`.
pub fn downloaded(&mut self, batch: Batch) {
if !batch.failed.is_empty() {
for pack in self
.packs
.iter_mut()
.filter(|p| p.activation != Activation::Active)
{
pack.remove(if batch.failed.contains(&pack.id) {
Removal::DownloadFailed
} else {
Removal::Discarded
});
}
}
let mut done = Vec::new();
for (id, path) in batch.downloaded {
if let Some(pack) = self
.packs
.iter_mut()
.find(|p| p.id == id && p.download == Download::Pending)
{
pack.download = Download::Done;
pack.path = Some(path);
if !pack.removed() {
done.push(id);
}
}
}
for id in done {
self.report(id, action::DOWNLOADED);
}
self.update = true;
}
/// The reload [`Tick::reload`] asked for finished (`Callbacks.onSuccess`),
/// or its packs could not be loaded (`onFailure`). After a first failure
/// the packs that were active are loaded again: the packs to reload with
/// are returned, and this is called again with that reload's outcome.
pub fn reloaded(&mut self, ok: bool) -> Option<Vec<(u128, PathBuf)>> {
let mut reload = self.reload.take()?;
self.update = true;
if ok {
let mut applied = Vec::new();
for pack in &mut self.packs {
if reload.load.contains(&pack.id) {
pack.activation = Activation::Active;
if pack.removal.is_none() {
applied.push(pack.id);
}
} else if reload.unload.contains(&pack.id) {
pack.activation = Activation::Inactive;
}
}
for id in applied {
self.report(id, action::SUCCESSFULLY_LOADED);
}
return None;
}
if reload.recovery {
// onFailure(true): nothing of the server's is loaded.
for pack in &mut self.packs {
if pack.activation == Activation::Pending {
pack.activation = Activation::Inactive;
}
}
return None;
}
reload.load.clear();
for pack in &mut self.packs {
match pack.activation {
Activation::Inactive => pack.remove(Removal::Discarded),
Activation::Pending => {
pack.activation = Activation::Inactive;
pack.remove(Removal::ActivationFailed);
}
Activation::Active => reload.load.push(pack.id),
}
}
reload.recovery = true;
let paths = self.paths(&reload.load);
self.reload = Some(reload);
Some(paths)
}
}
#[cfg(test)]
mod tests {
use super::*;
use action::*;
fn ok(id: u128) -> Batch {
Batch {
downloaded: vec![(id, PathBuf::from(format!("/p/{id}")))],
failed: vec![],
}
}
/// Runs the updates as `createUpdateScheduler` does, downloading and
/// reloading with `download` and `reload` as the game would.
fn settle(
sp: &mut ServerPacks,
download: &mut dyn FnMut(&[Request]) -> Batch,
reload: &mut dyn FnMut(&[(u128, PathBuf)]) -> bool,
) {
let mut rounds = 0;
while sp.wants_tick() {
rounds += 1;
assert!(rounds < 50);
let t = sp.tick();
if !t.download.is_empty() {
let batch = download(&t.download);
sp.downloaded(batch);
}
if let Some(mut packs) = t.reload {
while let Some(back) = sp.reloaded(reload(&packs)) {
packs = back;
}
}
}
}
#[test]
fn an_enabled_server_s_pack_is_accepted_downloaded_and_loaded() {
let mut sp = ServerPacks::default();
sp.connect(Prompt::Allowed);
sp.push(1, "http://a/p.zip", None);
let mut loaded = Vec::new();
settle(&mut sp, &mut |r| ok(r[0].id), &mut |p| {
loaded = p.to_vec();
true
});
assert_eq!(
sp.take_replies(),
[(1, ACCEPTED), (1, DOWNLOADED), (1, SUCCESSFULLY_LOADED)]
);
assert_eq!(loaded, [(1, PathBuf::from("/p/1"))]);
}
#[test]
fn the_prompt_holds_packs_until_answered() {
let mut sp = ServerPacks::default();
sp.connect(Prompt::Pending);
sp.push(1, "http://a/p.zip", None);
settle(&mut sp, &mut |_| unreachable!(), &mut |_| unreachable!());
assert!(sp.take_replies().is_empty());
sp.reject();
settle(&mut sp, &mut |_| unreachable!(), &mut |_| unreachable!());
assert_eq!(sp.take_replies(), [(1, DECLINED)]);
// Declined: later pushes are declined at once.
sp.push(2, "http://a/q.zip", None);
assert_eq!(sp.take_replies(), [(2, DECLINED)]);
}
#[test]
fn a_failed_download_discards_the_rest_of_the_batch() {
let mut sp = ServerPacks::default();
sp.connect(Prompt::Allowed);
sp.push(1, "http://a/p.zip", None);
sp.push(2, "http://a/dead.zip", None);
settle(
&mut sp,
&mut |_| Batch {
downloaded: vec![(1, PathBuf::from("/p/1"))],
failed: vec![2],
},
&mut |_| unreachable!(),
);
assert_eq!(
sp.take_replies(),
[
(1, ACCEPTED),
(2, ACCEPTED),
(1, DISCARDED),
(2, FAILED_DOWNLOAD)
]
);
}
#[test]
fn a_pack_that_does_not_load_fails_its_reload() {
let mut sp = ServerPacks::default();
sp.connect(Prompt::Allowed);
sp.push(1, "http://a/p.zip", None);
let mut reloads = Vec::new();
settle(&mut sp, &mut |r| ok(r[0].id), &mut |p| {
reloads.push(p.to_vec());
reloads.len() > 1
});
assert_eq!(
sp.take_replies(),
[(1, ACCEPTED), (1, DOWNLOADED), (1, FAILED_RELOAD)]
);
// The recovery reload has none of the server's packs.
assert_eq!(reloads[1], []);
}
#[test]
fn popping_and_disconnecting_unload_the_packs() {
let mut sp = ServerPacks::default();
sp.connect(Prompt::Allowed);
sp.push(1, "http://a/p.zip", None);
sp.push(2, "http://a/q.zip", None);
let last = std::cell::RefCell::new(Vec::new());
let mut reload = |p: &[(u128, PathBuf)]| {
*last.borrow_mut() = p.to_vec();
true
};
settle(
&mut sp,
&mut |r| Batch {
downloaded: r
.iter()
.map(|q| (q.id, PathBuf::from(format!("/p/{}", q.id))))
.collect(),
failed: vec![],
},
&mut reload,
);
sp.take_replies();
sp.pop(1);
settle(&mut sp, &mut |_| unreachable!(), &mut reload);
// Callbacks.onSuccess reports every pack the reload loaded, again.
assert_eq!(sp.take_replies(), [(2, SUCCESSFULLY_LOADED)]);
assert_eq!(*last.borrow(), [(2, PathBuf::from("/p/2"))]);
sp.disconnect();
settle(&mut sp, &mut |_| unreachable!(), &mut reload);
assert!(last.borrow().is_empty());
assert!(sp.packs.is_empty());
}
#[test]
fn a_pushed_id_replaces_its_pack() {
let mut sp = ServerPacks::default();
sp.connect(Prompt::Allowed);
sp.push(1, "http://a/old.zip", None);
sp.push(1, "http://a/new.zip", None);
let mut urls = Vec::new();
settle(
&mut sp,
&mut |r| {
urls.extend(r.iter().map(|q| q.url.clone()));
ok(1)
},
&mut |_| true,
);
assert_eq!(urls, ["http://a/new.zip"]);
}
#[test]
fn urls_and_hashes_are_read_as_vanilla_reads_them() {
assert!(valid_url("https://example.com/p.zip"));
assert!(valid_url("HTTP://example.com/p.zip"));
assert!(!valid_url("ftp://example.com/p.zip"));
assert!(!valid_url("example.com/p.zip"));
let sha1 = "0123456789ABCDEF0123456789abcdef01234567";
assert_eq!(parse_sha1(sha1), Some(sha1.to_ascii_lowercase()));
assert_eq!(parse_sha1("abc"), None);
}
}
@@ -0,0 +1,482 @@
//! Server pack downloads into the game directory's `downloads/` cache, as
//! vanilla keeps it (`DownloadQueue`, `HttpUtil.downloadFile`,
//! `DownloadCacheCleaner`): `downloads/<pack id>/<SHA-1 of the file>`, one
//! batch at a time on a thread of its own, every download logged in
//! `downloads/log.json`, and the cache cut to its 20 newest files on start.
use super::{Batch, Request};
use sha1::{Digest, Sha1};
use std::{
fs,
io::{self, Read, Write},
path::{Path, PathBuf},
sync::mpsc::{Receiver, Sender, channel},
time::SystemTime,
};
/// `DownloadedPackSource`'s MAX_PACK_SIZE_BYTES (250 MiB).
pub const MAX_SIZE: u64 = 262_144_000;
/// `DownloadQueue.MAX_KEPT_PACKS`.
const MAX_KEPT: usize = 20;
/// `createDownloadHeaders`: who is downloading, with which game.
pub fn headers(name: &str, uuid: u128) -> Vec<(&'static str, String)> {
use horizon_core::version::{RESOURCE_PACK_FORMAT, VANILLA_VERSION, VERSION_ID};
vec![
("X-Minecraft-Username", name.to_owned()),
("X-Minecraft-UUID", format!("{uuid:032x}")),
("X-Minecraft-Version", VANILLA_VERSION.to_owned()),
("X-Minecraft-Version-ID", VERSION_ID.to_owned()),
(
"X-Minecraft-Pack-Format",
format!("{}.{}", RESOURCE_PACK_FORMAT.0, RESOURCE_PACK_FORMAT.1),
),
("User-Agent", format!("Minecraft Java/{VANILLA_VERSION}")),
]
}
type Job = (Vec<Request>, Vec<(&'static str, String)>);
/// `DownloadQueue`: batches download in order, on one thread.
pub struct Queue {
jobs: Sender<Job>,
done: Receiver<Batch>,
}
impl Queue {
pub fn open(dir: PathBuf) -> io::Result<Self> {
fs::create_dir_all(&dir)?;
vacuum(&dir, MAX_KEPT);
let (jobs, todo) = channel::<Job>();
let (finished, done) = channel();
std::thread::Builder::new()
.name("download-queue".into())
.spawn(move || {
for (requests, headers) in todo {
if finished.send(run(&dir, &requests, &headers)).is_err() {
break;
}
}
})?;
Ok(Self { jobs, done })
}
pub fn download(&self, requests: Vec<Request>, headers: Vec<(&'static str, String)>) {
let _ = self.jobs.send((requests, headers));
}
/// The batches finished since the last call.
pub fn finished(&self) -> Vec<Batch> {
self.done.try_iter().collect()
}
}
/// `DownloadQueue.runDownload`: each pack into `<dir>/<id>/`, logged.
fn run(dir: &Path, requests: &[Request], headers: &[(&'static str, String)]) -> Batch {
let mut batch = Batch::default();
for request in requests {
let id = horizon_protocol::crypto::uuid_string(request.id);
let result = download_file(&dir.join(&id), request, headers, MAX_SIZE);
// DownloadQueue.LogEntry, its fields in the codec's order.
let mut entry = vec![
("id", serde_json::Value::from(id)),
("url", request.url.as_str().into()),
(
"time",
chrono::Utc::now()
.to_rfc3339_opts(chrono::SecondsFormat::AutoSi, true)
.into(),
),
];
if let Some(sha1) = &request.sha1 {
entry.push(("hash", sha1.as_str().into()));
}
entry.push(match &result {
Ok(path) => match fs::metadata(path) {
Ok(meta) => {
let name = path.strip_prefix(dir).unwrap_or(path);
let name = name.to_string_lossy().replace('\\', "/");
(
"file",
serde_json::json!({"name": name, "size": meta.len()}),
)
}
Err(_) => ("error", "no_access".into()),
},
Err(e) => {
log::error!("Horizon: failed to download {}: {e}", request.url);
("error", "download_failed".into())
}
});
if let Err(e) = append_log(&dir.join("log.json"), &entry) {
log::error!("Horizon: failed to log download of {}: {e}", request.url);
}
match result {
Ok(path) => batch.downloaded.push((request.id, path)),
Err(_) => batch.failed.push(request.id),
}
}
batch
}
/// `JsonEventLog.write`: one JSON object a line, as Gson writes it (with
/// its HTML escaping).
fn append_log(path: &Path, entry: &[(&str, serde_json::Value)]) -> io::Result<()> {
let fields: Vec<String> = entry
.iter()
.map(|(k, v)| format!("{}:{v}", serde_json::Value::from(*k)))
.collect();
let mut line = String::new();
for c in format!("{{{}}}", fields.join(",")).chars() {
match c {
'<' | '>' | '&' | '=' | '\'' => line.push_str(&format!("\\u{:04x}", c as u32)),
c => line.push(c),
}
}
line.push('\n');
fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)?
.write_all(line.as_bytes())
}
fn sha1_of(path: &Path) -> io::Result<String> {
let mut hasher = Sha1::new();
io::copy(&mut fs::File::open(path)?, &mut hasher)?;
Ok(hex(&hasher.finalize()))
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
/// `checkExistingFile`: the file is there with that hash.
fn matches(path: &Path, sha1: &str) -> bool {
path.exists() && sha1_of(path).is_ok_and(|actual| actual == sha1)
}
/// `updateModificationTime`: used now, so the cleaner keeps it longest.
fn touch(path: &Path) {
let _ = fs::File::options()
.append(true)
.open(path)
.and_then(|f| f.set_modified(SystemTime::now()));
}
/// `HttpUtil.downloadFile`: into `<dir>/<SHA-1>`. With a hash the cached file
/// is used when it matches, and a download that does not is an error;
/// without one the file is named after the hash of what came.
pub fn download_file(
dir: &Path,
request: &Request,
headers: &[(&'static str, String)],
max_size: u64,
) -> Result<PathBuf, String> {
let target = request.sha1.as_ref().map(|sha1| dir.join(sha1));
if let (Some(target), Some(sha1)) = (&target, &request.sha1) {
if matches(target, sha1) {
log::info!("Horizon: returning cached file since actual hash matches requested");
touch(target);
return Ok(target.clone());
}
if target.exists() {
log::warn!(
"Horizon: existing file {} not found or had mismatched hash",
target.display()
);
}
fs::remove_file(target)
.or_else(|e| {
if e.kind() == io::ErrorKind::NotFound {
Ok(())
} else {
Err(e)
}
})
.map_err(|e| format!("Failed to remove existing file {}: {e}", target.display()))?;
}
let mut call = ureq::AgentBuilder::new().build().get(&request.url);
for (name, value) in headers {
call = call.set(name, value);
}
let response = call.call().map_err(|e| e.to_string())?;
let size = response
.header("Content-Length")
.and_then(|v| v.trim().parse::<u64>().ok());
fs::create_dir_all(dir).map_err(|e| e.to_string())?;
if let Some(size) = size
&& size > max_size
{
return Err(format!(
"Filesize is bigger than maximum allowed (file is {size}, limit is {max_size})"
));
}
let mut input = response.into_reader();
match (&target, &request.sha1) {
(Some(target), Some(sha1)) => {
let actual = save_hashed(&mut input, target, max_size).map_err(|e| e.to_string())?;
if &actual != sha1 {
return Err(format!(
"Hash of downloaded file ({actual}) did not match requested ({sha1})"
));
}
Ok(target.clone())
}
_ => {
let tmp = dir.join(format!("download{:016x}.tmp", rand_suffix()));
let result = save_hashed(&mut input, &tmp, max_size).and_then(|actual| {
let path = dir.join(&actual);
if matches(&path, &actual) {
touch(&path);
} else {
fs::rename(&tmp, &path)?;
}
Ok(path)
});
let _ = fs::remove_file(&tmp);
result.map_err(|e| e.to_string())
}
}
}
fn rand_suffix() -> u64 {
use std::hash::{BuildHasher, RandomState};
RandomState::new().hash_one(std::time::Instant::now())
}
/// `downloadAndHash`: write what comes, at most `max_size` bytes.
fn save_hashed(input: &mut dyn Read, path: &Path, max_size: u64) -> io::Result<String> {
let mut out = fs::File::create(path)?;
let mut hasher = Sha1::new();
let mut buffer = [0u8; 8196];
let mut read_so_far = 0u64;
loop {
let n = input.read(&mut buffer)?;
if n == 0 {
break;
}
read_so_far += n as u64;
if read_so_far > max_size {
return Err(io::Error::other(format!(
"Filesize was bigger than maximum allowed (got >= {read_so_far}, limit was {max_size})"
)));
}
out.write_all(&buffer[..n])?;
hasher.update(&buffer[..n]);
}
Ok(hex(&hasher.finalize()))
}
/// `DownloadCacheCleaner.vacuumCacheDir`: keep the `max_files` files most
/// worth keeping (in the pack folders: the newest, and one per pack before a
/// second of any), then drop the folders left empty.
pub fn vacuum(dir: &Path, max_files: usize) {
let mut files = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(d) = stack.pop() {
let Ok(entries) = fs::read_dir(&d) else {
continue;
};
for e in entries.flatten() {
let path = e.path();
let Ok(meta) = e.metadata() else { continue };
if meta.is_dir() {
stack.push(path);
} else if meta.is_file() && path.parent() != Some(dir) {
let time = meta.modified().unwrap_or(SystemTime::UNIX_EPOCH);
files.push((path, time));
}
}
}
let Some(remove) = files.len().checked_sub(max_files).filter(|n| *n > 0) else {
return;
};
// Newest first, then each file's rank in its folder (0 for a folder's
// newest), reversed, and stably sorted by rank, highest first.
files.sort_by_key(|f| std::cmp::Reverse(f.1));
let mut counts = std::collections::HashMap::<PathBuf, usize>::new();
let mut ranked: Vec<(PathBuf, usize)> = files
.into_iter()
.map(|(path, _)| {
let count = counts
.entry(path.parent().unwrap_or(dir).to_path_buf())
.or_default();
let rank = *count;
*count += 1;
(path, rank)
})
.collect();
ranked.reverse();
ranked.sort_by_key(|f| std::cmp::Reverse(f.1));
let mut emptied = std::collections::HashSet::new();
for (path, rank) in ranked.into_iter().take(remove) {
match fs::remove_file(&path) {
Ok(()) if rank == 0 => {
emptied.insert(path.parent().unwrap_or(dir).to_path_buf());
}
Ok(()) => {}
Err(e) => log::warn!(
"Horizon: failed to delete cache file {}: {e}",
path.display()
),
}
}
emptied.remove(dir);
for d in emptied {
let _ = fs::remove_dir(d);
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::TcpListener;
/// A one-shot HTTP server answering `status` with `body`; the request it got.
fn serve(status: &str, body: Vec<u8>) -> (String, std::thread::JoinHandle<String>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}/pack.zip", listener.local_addr().unwrap());
let status = status.to_owned();
let handle = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let mut request = Vec::new();
let mut buf = [0u8; 1024];
while !request.ends_with(b"\r\n\r\n") {
let n = stream.read(&mut buf).unwrap();
request.extend_from_slice(&buf[..n]);
}
let head = format!(
"HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
stream.write_all(head.as_bytes()).unwrap();
stream.write_all(&body).unwrap();
String::from_utf8_lossy(&request).into_owned()
});
(url, handle)
}
fn sha1(bytes: &[u8]) -> String {
hex(&Sha1::digest(bytes))
}
fn scratch() -> PathBuf {
let dir = std::env::temp_dir().join(format!("horizon-packs-{:016x}", rand_suffix()));
fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn a_pack_lands_under_its_hash_and_is_reused_from_the_cache() {
let dir = scratch();
let body = b"PK pack".to_vec();
let (url, server) = serve("200 OK", body.clone());
let request = Request {
id: 7,
url,
sha1: Some(sha1(&body)),
};
let path = download_file(&dir, &request, &headers("Alex", 0xab), MAX_SIZE).unwrap();
assert_eq!(path, dir.join(sha1(&body)));
assert_eq!(fs::read(&path).unwrap(), body);
let sent = server.join().unwrap();
assert!(sent.contains("X-Minecraft-Username: Alex"), "{sent}");
assert!(sent.contains("X-Minecraft-Pack-Format: 97.1"), "{sent}");
// Cached: no server this time.
let cached = Request {
url: "http://127.0.0.1:9/gone.zip".into(),
..request
};
assert_eq!(download_file(&dir, &cached, &[], MAX_SIZE).unwrap(), path);
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn a_wrong_hash_a_dead_url_and_a_large_pack_fail() {
let dir = scratch();
let (url, server) = serve("200 OK", b"other bytes".to_vec());
let wrong = Request {
id: 1,
url,
sha1: Some("0".repeat(40)),
};
let err = download_file(&dir, &wrong, &[], MAX_SIZE).unwrap_err();
assert!(err.contains("did not match requested"), "{err}");
server.join().unwrap();
let dead = Request {
id: 2,
url: "http://127.0.0.1:9/pack.zip".into(),
sha1: None,
};
assert!(download_file(&dir.join("dead"), &dead, &[], MAX_SIZE).is_err());
assert!(!dir.join("dead").exists());
let (url, server) = serve("200 OK", vec![0; 100]);
let large = Request {
id: 3,
url,
sha1: None,
};
let err = download_file(&dir, &large, &[], 10).unwrap_err();
assert!(err.contains("bigger than maximum"), "{err}");
let _ = server.join();
let (url, server) = serve("404 Not Found", Vec::new());
let missing = Request {
id: 4,
url,
sha1: None,
};
assert!(download_file(&dir, &missing, &[], MAX_SIZE).is_err());
server.join().unwrap();
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn the_queue_logs_each_download() {
let dir = scratch();
let (url, server) = serve("200 OK", b"zip".to_vec());
let queue = Queue::open(dir.clone()).unwrap();
let id = 0x5b2c_7a4e_1f3d_4c8b_9a6e_2d4f_8b1c_3e5a;
queue.download(
vec![Request {
id,
url,
sha1: None,
}],
Vec::new(),
);
let batch = loop {
if let Some(b) = queue.finished().pop() {
break b;
}
std::thread::sleep(std::time::Duration::from_millis(10));
};
server.join().unwrap();
let file = dir
.join("5b2c7a4e-1f3d-4c8b-9a6e-2d4f8b1c3e5a")
.join(sha1(b"zip"));
assert_eq!(batch.downloaded, [(id, file)]);
let log = fs::read_to_string(dir.join("log.json")).unwrap();
let entry: serde_json::Value = serde_json::from_str(log.trim()).unwrap();
assert_eq!(entry["id"], "5b2c7a4e-1f3d-4c8b-9a6e-2d4f8b1c3e5a");
assert_eq!(entry["file"]["size"], 3);
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn the_cleaner_keeps_the_newest_file_of_each_pack_longest() {
let dir = scratch();
let at = |secs| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(secs);
for (pack, file, time) in [
("a", "1", 10),
("a", "2", 20),
("a", "3", 30),
("b", "1", 5),
] {
fs::create_dir_all(dir.join(pack)).unwrap();
let path = dir.join(pack).join(file);
fs::write(&path, b"x").unwrap();
fs::File::options()
.append(true)
.open(&path)
.unwrap()
.set_modified(at(time))
.unwrap();
}
fs::write(dir.join("log.json"), b"").unwrap();
vacuum(&dir, 2);
assert!(dir.join("a/3").exists() && dir.join("b/1").exists());
assert!(!dir.join("a/1").exists() && !dir.join("a/2").exists());
assert!(dir.join("log.json").exists());
fs::remove_dir_all(dir).unwrap();
}
}