diff --git a/crates/horizon-client/examples/save_bench.rs b/crates/horizon-client/examples/save_bench.rs index 2e3084290..30625be65 100644 --- a/crates/horizon-client/examples/save_bench.rs +++ b/crates/horizon-client/examples/save_bench.rs @@ -87,7 +87,7 @@ fn main() -> Result<()> { let _ = save.tables.set(tables.clone()); let started = Instant::now(); save.save_later(&chunks, registry, &biomes)?; - save.flush(); + save.flush().unwrap(); times[2].push(started.elapsed().as_secs_f64()); drop(save); diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index 6baab9863..d1177ac2e 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -118,6 +118,7 @@ mod remote_projectiles; mod render_control; mod render_tick; mod respawn_anchor; +mod saving; mod scoreboard; mod screens; mod sculk; @@ -1366,6 +1367,7 @@ struct Game<'a> { text_selection: horizon_client::clipboard::AppendSelection, /// The world was saved for the application ending (`save_for_exit`). saved_for_exit: bool, + pending_save: Option, /// The block a guest is acting on (game/lan_actions.rs). forced_hit: Option, /// The part of a right click a guest's packet asks for (game/lan_actions.rs). @@ -7240,13 +7242,21 @@ impl Game<'_> { } pub(super) fn save_for_exit(&mut self) -> Result<()> { + // Native application termination can bypass CloseRequested. Finish any + // save screen synchronously here, before the process can go away. + while self.pending_save.is_some() { + self.poll_closing_save()?; + if self.pending_save.is_some() { + std::thread::sleep(Duration::from_millis(1)); + } + } if self.title.is_some() || self.saved_for_exit { return Ok(()); } - self.saved_for_exit = true; if let Some(server) = self.integrated.take() { // The server thread saves the world. - server.stop(); + server.stop()?; + self.saved_for_exit = true; return Ok(()); } self.close_container(); @@ -7254,9 +7264,14 @@ impl Game<'_> { self.stats.custom("leave_game", 1); } self.lan_close(false); - self.save_game() + self.save_game()?; + self.saved_for_exit = true; + Ok(()) } fn close_world(&mut self) -> Result<()> { + self.begin_closing_save(false) + } + fn finish_close_world(&mut self) -> Result<()> { self.ambience.clear(&mut self.audio); // updateLevelInEngines(null): every sound stops; the music manager // lets go of its track. @@ -7265,13 +7280,7 @@ impl Game<'_> { if self.remote.is_some() { return self.leave_server(None); } - self.finish_perf(); - self.stats.custom("leave_game", 1); - self.save_game()?; - self.lan_close(false); - // Closing the level storage: once the last writes are on disk the - // session lock goes, so the title can delete or back up this world. - self.save.flush(); + // The save screen has finished every write before releasing storage. self.session_lock = None; horizon_client::crash::clear_detail("World"); self.debug_charts.reset(); @@ -9037,6 +9046,9 @@ impl Game<'_> { } self.save_level(wait)?; self.save_parked_levels(wait)?; + self.save_game_metadata() + } + fn save_game_metadata(&mut self) -> Result<()> { self.save_maps(); self.save_random_sequences(); self.save_fight(); @@ -9062,11 +9074,12 @@ impl Game<'_> { let chunks = self.with_entities(keys.iter().filter_map(|k| self.world.chunks.get(k))); self.save .save_later(&chunks, self.pack.assets.registry, self.res.biomes())?; - if wait { - self.save.flush(); - } - self.save_entities(wait)?; + // Prepare entities while the chunk writer runs, then use one barrier. + self.save_entities(false)?; self.save_raids(); + if wait { + self.save.flush()?; + } Ok(()) } fn render(&mut self) -> Result { @@ -10079,11 +10092,36 @@ impl ApplicationHandler for Game<'_> { { return; } + if self.pending_save.is_some() { + match event { + WindowEvent::CloseRequested => { + self.pending_save.as_mut().unwrap().quit = true; + } + WindowEvent::Focused(focused) => { + self.focused = focused; + if !focused { + self.capture(false); + } + } + WindowEvent::Resized(size) => { + if let Some(r) = &mut self.renderer { + r.resize(size.width, size.height); + } + } + WindowEvent::RedrawRequested => { + if let Err(error) = self.render_closing_save() { + self.error = Some(error); + event_loop.exit(); + } + } + _ => {} + } + return; + } let result: Result<()> = (|| { match event { WindowEvent::CloseRequested => { - self.save_for_exit()?; - event_loop.exit(); + self.begin_closing_save(true)?; } WindowEvent::Resized(size) => { if let Some(r) = &mut self.renderer { @@ -10724,7 +10762,8 @@ impl ApplicationHandler for Game<'_> { } } fn device_event(&mut self, _: &ActiveEventLoop, _: DeviceId, event: DeviceEvent) { - if self.captured + if self.pending_save.is_none() + && self.captured && self.focused && self.screen == Screen::Playing && let DeviceEvent::MouseMotion { delta: (x, y) } = event @@ -10735,12 +10774,34 @@ impl ApplicationHandler for Game<'_> { } } fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) { + if self.pending_save.is_some() { + match self.poll_closing_save() { + Ok(true) => event_loop.exit(), + Ok(false) => { + if let Some(window) = &self.window { + window.request_redraw(); + } + // Yield between polls even when disk I/O is slow. + event_loop.set_control_flow(ControlFlow::WaitUntil( + Instant::now() + Duration::from_millis(4), + )); + } + Err(error) => { + self.error = Some(error); + event_loop.exit(); + } + } + return; + } if let Err(error) = self.resume_world_entry() { self.error = Some(error); event_loop.exit(); return; } self.poll_control(event_loop); + if self.pending_save.is_some() { + return; + } self.poll_pads(event_loop); if self.options.frames.is_some() && self.started.elapsed() > self.options.timeout { self.error = Some("frame-limited run timed out".into()); @@ -11087,6 +11148,7 @@ impl<'a> Game<'a> { clipboard: Default::default(), text_selection: Default::default(), saved_for_exit: false, + pending_save: None, forced_hit: None, guest_use: None, lan_events: Vec::new(), diff --git a/crates/horizon-client/src/game/control.rs b/crates/horizon-client/src/game/control.rs index 06ebf31d6..5cdf46110 100644 --- a/crates/horizon-client/src/game/control.rs +++ b/crates/horizon-client/src/game/control.rs @@ -117,6 +117,9 @@ impl Game<'_> { } while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) { self.answer(request, event_loop); + if self.pending_save.is_some() { + break; + } } if self.control.is_some() { self.watch_events(); @@ -140,6 +143,9 @@ impl Game<'_> { event_loop: &ActiveEventLoop, ) -> Result> { let words: Vec<&str> = line.split_whitespace().collect(); + if self.pending_save.is_some() && words.first() != Some(&"quit") { + return Err("a world save is in progress".into()); + } let arg = |i: usize| { words .get(i) @@ -625,6 +631,9 @@ impl Game<'_> { server .call(move |g| g.control_save(all).map_err(|e| e.to_string())) .ok_or("the integrated server stopped")??; + } else if self.remote.is_none() { + self.begin_control_save(all, reply.clone())?; + return Ok(None); } else { self.control_save(all)?; } @@ -637,8 +646,7 @@ impl Game<'_> { ok() } "quit" => { - self.save_for_exit()?; - event_loop.exit(); + self.begin_closing_save(true)?; ok() } other => match self.station_command(&words)? { diff --git a/crates/horizon-client/src/game/integrated.rs b/crates/horizon-client/src/game/integrated.rs index bfbc8d66c..7c0ab2bbf 100644 --- a/crates/horizon-client/src/game/integrated.rs +++ b/crates/horizon-client/src/game/integrated.rs @@ -27,7 +27,7 @@ use std::{ net::TcpListener, path::Path, sync::{ - Arc, + Arc, Mutex, atomic::{AtomicBool, AtomicI32, AtomicU16, AtomicU32, Ordering}, mpsc::{Receiver, Sender, channel, sync_channel}, }, @@ -41,6 +41,7 @@ struct Flags { /// `Minecraft.pause`: a single-player pause screen is open. paused: AtomicBool, stop: AtomicBool, + failure: Mutex>, /// `IntegratedServer.getPort`: the port it is open to the LAN on, 0 /// while it is not. published: AtomicU16, @@ -96,19 +97,42 @@ impl Integrated { rx.recv_timeout(Duration::from_secs(10)).ok() } /// `IntegratedServer.halt(true)`: returns once the world is saved and let go. - pub fn stop(mut self) { - self.halt(); + pub fn stop(mut self) -> Result<()> { + self.halt() } - fn halt(&mut self) { + /// Stop on the server thread; the window polls completion while drawing. + pub fn request_stop(&self) { self.flags.stop.store(true, Ordering::Relaxed); + } + pub fn stopped(&self) -> bool { + self.thread + .as_ref() + .is_none_or(|thread| thread.is_finished()) + } + fn halt(&mut self) -> Result<()> { + self.request_stop(); if let Some(thread) = self.thread.take() { - let _ = thread.join(); + thread + .join() + .map_err(|_| "the integrated server panicked")?; + } + match self + .flags + .failure + .lock() + .map_err(|_| "server status poisoned")? + .as_ref() + { + Some(error) => Err(error.clone().into()), + None => Ok(()), } } } impl Drop for Integrated { fn drop(&mut self) { - self.halt(); + if let Err(error) = self.halt() { + log::error!("Horizon: integrated server shutdown: {error}"); + } } } @@ -224,12 +248,13 @@ impl Game<'_> { options, settings, launch, - server_flags, + server_flags.clone(), server_tasks, &ready_tx, ); if let Err(e) = result { log::error!("Horizon: integrated server: {e}"); + *server_flags.failure.lock().unwrap() = Some(e.to_string()); let _ = ready_tx.send(Err(e.to_string())); } })?; @@ -386,7 +411,7 @@ impl Game<'_> { std::thread::sleep(Duration::from_millis(100)); self.save_game()?; self.lan_close(false); - self.save.flush(); + self.save.flush()?; self.session_lock = None; Ok(()) } @@ -663,3 +688,42 @@ fn serve( let mut server = Game::new(res, pack, gui, options, settings, audio)?; server.run_integrated(launch, flags, tasks, ready) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shutdown_request_does_not_join_a_server_still_saving() { + let flags = Arc::new(Flags::default()); + let (release, wait) = channel(); + let (tasks, _) = channel(); + let server = Integrated { + flags: flags.clone(), + thread: Some(std::thread::spawn(move || wait.recv().unwrap())), + tasks, + world_name: String::new(), + }; + server.request_stop(); + assert!(flags.stop.load(Ordering::Relaxed)); + assert!(!server.stopped()); + release.send(()).unwrap(); + server.stop().unwrap(); + } + + #[test] + fn shutdown_reports_a_failed_server_save() { + let flags = Arc::new(Flags::default()); + let saving = flags.clone(); + let (tasks, _) = channel(); + let server = Integrated { + flags, + thread: Some(std::thread::spawn(move || { + *saving.failure.lock().unwrap() = Some("disk write failed".into()); + })), + tasks, + world_name: String::new(), + }; + assert_eq!(server.stop().unwrap_err().to_string(), "disk write failed"); + } +} diff --git a/crates/horizon-client/src/game/persist.rs b/crates/horizon-client/src/game/persist.rs index 7a967a024..d266ad1f0 100644 --- a/crates/horizon-client/src/game/persist.rs +++ b/crates/horizon-client/src/game/persist.rs @@ -559,7 +559,7 @@ impl Game<'_> { /// The entity NBT of every simulated entity in `keys`, plus the unknown /// entities kept for them. With `remove`, the entities leave the world /// (their chunks unload). - fn entity_chunks( + pub(super) fn entity_chunks( &mut self, keys: &HashSet<(i32, i32)>, remove: bool, @@ -858,6 +858,14 @@ impl Game<'_> { self.spawn_loaded_entities(); let keys: HashSet<(i32, i32)> = self.world.chunks.keys().copied().collect(); let chunks = self.entity_chunks(&keys, false); + self.prune_entity_memory(); + if wait { + self.save.save_entities(chunks) + } else { + self.save.save_entities_later(chunks) + } + } + pub(super) fn prune_entity_memory(&mut self) { // Forget entities that no longer exist. let alive: HashSet = self .mobs @@ -873,11 +881,6 @@ impl Game<'_> { .collect(); self.persist.uuids.retain(|id, _| alive.contains(id)); self.persist.raw.retain(|id, _| alive.contains(id)); - if wait { - self.save.save_entities(chunks) - } else { - self.save.save_entities_later(chunks) - } } /// Save and remove the entities of chunks that unloaded. pub(super) fn unload_entities(&mut self, unloaded: &[Arc]) { diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index c10c3f439..604964367 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -361,6 +361,9 @@ impl Game<'_> { /// Disconnect and return to the multiplayer screen (DisconnectedScreen). pub(super) fn leave_server(&mut self, reason: Option) -> Result<()> { + if reason.is_none() && self.integrated.as_ref().is_some_and(|s| !s.stopped()) { + return self.begin_closing_save(false); + } self.finish_perf(); if reason.is_none() { // Leaving on purpose: nothing for auto-reconnect to retry. @@ -372,7 +375,7 @@ impl Game<'_> { // Single player's server thread saves the world before the title shows. let single_player = match self.integrated.take() { Some(server) => { - server.stop(); + server.stop()?; true } None => false, diff --git a/crates/horizon-client/src/game/saving.rs b/crates/horizon-client/src/game/saving.rs new file mode 100644 index 000000000..d73681634 --- /dev/null +++ b/crates/horizon-client/src/game/saving.rs @@ -0,0 +1,249 @@ +//! The save screen freezes simulation, but keeps pumping window events while +//! snapshots are prepared in small batches and the region workers drain. +use super::{Game, loading_frame}; +use horizon_client::{Result, hud::HudBatch}; +use std::{ + collections::{HashSet, VecDeque}, + time::{Duration, Instant}, +}; + +pub(super) struct ClosingSave { + pub quit: bool, + close: bool, + reply: Option>, + started: Instant, + stage: Stage, +} +enum Stage { + Generation, + Chunks(VecDeque<(i32, i32)>), + Entities(VecDeque<(i32, i32)>), + Writes, + Server, +} + +// Bound the work in both sparse and block-entity-heavy worlds. A single chunk +// is indivisible; check the time budget after each snapshot, not each batch. +const SNAPSHOTS_PER_FRAME: usize = 16; +const SNAPSHOT_BUDGET: Duration = Duration::from_millis(2); + +impl Game<'_> { + pub(super) fn begin_closing_save(&mut self, quit: bool) -> Result<()> { + if let Some(save) = &mut self.pending_save { + save.quit |= quit; + return Ok(()); + } + if self.title.is_some() || self.saved_for_exit { + // A quit from the title still needs the event loop to finish. + self.pending_save = Some(ClosingSave { + quit, + close: true, + reply: None, + started: Instant::now(), + stage: Stage::Writes, + }); + return Ok(()); + } + if self.integrated.is_some() { + self.integrated.as_ref().unwrap().request_stop(); + self.pending_save = Some(ClosingSave { + quit, + close: true, + reply: None, + started: Instant::now(), + stage: Stage::Server, + }); + } else if self.remote.is_some() { + if !quit { + return self.leave_server(None); + } + self.pending_save = Some(ClosingSave { + quit, + close: true, + reply: None, + started: Instant::now(), + stage: Stage::Writes, + }); + } else { + self.close_container(); + self.stats.custom("leave_game", 1); + self.lan_close(false); + self.pending_save = Some(ClosingSave { + quit, + close: true, + reply: None, + started: Instant::now(), + stage: Stage::Generation, + }); + } + self.capture(false); + Ok(()) + } + + /// Control saves acknowledge only after durable completion, while their + /// snapshot work is spread across event-loop turns too. + pub(super) fn begin_control_save( + &mut self, + all: bool, + reply: std::sync::mpsc::Sender, + ) -> Result<()> { + if self.pending_save.is_some() { + return Err("a world save is already in progress".into()); + } + if all { + self.world + .modified + .extend(self.world.chunks.keys().copied()); + } + self.keys.clear(); + self.physical_keys.clear(); + self.left = false; + self.right = false; + self.pending_save = Some(ClosingSave { + quit: false, + close: false, + reply: Some(reply), + started: Instant::now(), + stage: Stage::Generation, + }); + Ok(()) + } + + /// `true` means the window can exit. The session lock stays in Game until + /// every snapshot has been queued and every writer has synced its files. + pub(super) fn poll_closing_save(&mut self) -> Result { + let Some(mut save) = self.pending_save.take() else { + return Ok(false); + }; + let result = (|| -> Result { + match &mut save.stage { + Stage::Server => { + let server = self + .integrated + .as_ref() + .ok_or("the integrated server stopped with an error")?; + if !server.stopped() { + return Ok(false); + } + // Drop/join only after completion; leave_server's teardown + // can then run without blocking on the server's writes. + if save.quit { + self.integrated.take().unwrap().stop()?; + } else { + self.leave_server(None)?; + } + Ok(true) + } + Stage::Generation => { + if save.started.elapsed() > Duration::from_secs(30) { + return Err("chunk generation did not finish before saving".into()); + } + if self.streaming.poll_save(&mut self.world) { + save.stage = Stage::Chunks(self.world.modified.iter().copied().collect()); + } + Ok(false) + } + Stage::Chunks(keys) => { + let started = Instant::now(); + for _ in 0..SNAPSHOTS_PER_FRAME { + let Some(&key) = keys.front() else { break }; + let chunks = self.with_entities(self.world.chunks.get(&key)); + self.save.save_later( + &chunks, + self.pack.assets.registry, + self.res.biomes(), + )?; + self.world.modified.remove(&key); + keys.pop_front(); + if started.elapsed() >= SNAPSHOT_BUDGET { + break; + } + } + if keys.is_empty() { + self.spawn_loaded_entities(); + save.stage = Stage::Entities(self.world.chunks.keys().copied().collect()); + } + Ok(false) + } + Stage::Entities(keys) => { + let started = Instant::now(); + for _ in 0..SNAPSHOTS_PER_FRAME { + let Some(&key) = keys.front() else { break }; + let chunks = self.entity_chunks(&HashSet::from([key]), false); + self.save.save_entities_later(chunks)?; + keys.pop_front(); + if started.elapsed() >= SNAPSHOT_BUDGET { + break; + } + } + if keys.is_empty() { + self.prune_entity_memory(); + self.save_raids(); + self.save_game_metadata()?; + save.stage = Stage::Writes; + } + Ok(false) + } + Stage::Writes => self.save.writes_finished(), + } + })(); + match result { + Ok(true) => { + if let Some(reply) = save.reply.take() { + let _ = reply.send(serde_json::json!({ "ok": true }).to_string()); + } + if save.quit && !save.close { + // A window close during a control save still has to close + // containers and save the resulting inventory changes. + self.begin_closing_save(true)?; + return Ok(false); + } + if save.quit { + self.saved_for_exit = true; + Ok(true) + } else { + if save.close && self.title.is_none() { + self.finish_perf(); + self.finish_close_world()?; + } + self.clock.reset(); + self.previous = Instant::now(); + Ok(false) + } + } + other => { + if let Err(error) = &other + && let Some(reply) = save.reply.take() + { + let _ = + reply.send(serde_json::json!({ "error": error.to_string() }).to_string()); + } + // Keep ownership (and the lock) on errors too. Native shutdown + // sees the same durable-write failure instead of declaring success. + self.pending_save = Some(save); + other + } + } + } + + pub(super) fn render_closing_save(&mut self) -> Result<()> { + let Some(renderer) = &mut self.renderer else { + return Ok(()); + }; + let mut hud = HudBatch::new([renderer.config.width, renderer.config.height]); + let progress = match self.pending_save.as_ref().map(|save| &save.stage) { + Some(Stage::Chunks(_)) => 0.25, + Some(Stage::Entities(_)) => 0.5, + Some(Stage::Writes) => 0.9, + _ => 0.0, + }; + horizon_client::level_loading::draw_progress( + &mut hud, + &self.gui.atlas, + "Saving world...", + progress, + ); + renderer.render(&loading_frame(&hud))?; + Ok(()) + } +} diff --git a/crates/horizon-client/src/save.rs b/crates/horizon-client/src/save.rs index dffaeb46d..83c81bf67 100644 --- a/crates/horizon-client/src/save.rs +++ b/crates/horizon-client/src/save.rs @@ -1487,6 +1487,7 @@ struct Writer { #[derive(Default)] struct Outstanding { count: Mutex, + error: Mutex>, done: Condvar, } /// One dimension folder (`dimensions/minecraft/`). Region files are @@ -1622,11 +1623,22 @@ impl SaveDir { let mut writer = self.writer.lock().map_err(|_| "writer poisoned")?; let writer = writer.get_or_insert_with(|| self.start_writer()); if writer.send.send((entities, batch)).is_err() { + *self + .outstanding + .error + .lock() + .map_err(|_| "writer poisoned")? = Some("the region writer stopped".into()); + *self + .outstanding + .count + .lock() + .map_err(|_| "writer poisoned")? -= 1; + self.outstanding.done.notify_all(); return Err("the region writer stopped".into()); } } if wait { - self.flush(); + self.flush()?; } Ok(()) } @@ -1657,11 +1669,16 @@ impl SaveDir { .filter(|(k, _, seq)| newest(k, *seq)) .map(|(k, r, _)| (*k, r)) .collect(); - if let Err(e) = write_region_records(&root, cache, folder, &records) { - log::warn!("Horizon: failed to write {folder} files: {e}"); + let result = write_region_records(&root, cache, folder, &records); + if let Err(e) = &result { + let message = format!("failed to write {folder} files: {e}"); + log::warn!("Horizon: {message}"); + *outstanding.error.lock().unwrap() = Some(message); } - // Readers find the files (and a fresh region cache) from here on. - if let Ok(mut pending) = pending.lock() { + // Keep failed snapshots readable; never report them as durable. + if result.is_ok() + && let Ok(mut pending) = pending.lock() + { for (k, _, seq) in &batch { if pending .get(&(entities, k.0, k.1)) @@ -1681,15 +1698,41 @@ impl SaveDir { Writer { send, thread } } /// Wait until every queued write is on disk. - pub fn flush(&self) { - let Ok(mut count) = self.outstanding.count.lock() else { - return; - }; + pub fn flush(&self) -> Result<()> { + let mut count = self + .outstanding + .count + .lock() + .map_err(|_| "writer poisoned")?; while *count > 0 { - count = match self.outstanding.done.wait(count) { - Ok(c) => c, - Err(_) => return, - }; + count = self + .outstanding + .done + .wait(count) + .map_err(|_| "writer poisoned")?; + } + self.write_result() + } + /// Poll durability without holding up the event loop. Errors remain visible + /// to every waiter, including a later synchronous shutdown. + pub fn writes_finished(&self) -> Result { + let count = self + .outstanding + .count + .lock() + .map_err(|_| "writer poisoned")?; + self.write_result()?; + Ok(*count == 0) + } + fn write_result(&self) -> Result<()> { + match &*self + .outstanding + .error + .lock() + .map_err(|_| "writer poisoned")? + { + Some(error) => Err(error.clone().into()), + None => Ok(()), } } /// Write `chunks` into their region files and wait until they are on disk. @@ -3593,6 +3636,62 @@ mod tests { super::upgrade_older_world(&dir).unwrap(); let _ = std::fs::remove_dir_all(&dir); } + #[test] + fn write_completion_polls_without_waiting_and_keeps_latest_snapshot() { + let dir = std::env::temp_dir().join(format!("horizon-save-poll-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let save = super::SaveDir::new(dir.clone()); + let lock = super::SessionLock::acquire(&dir).unwrap(); + let path = save.region_path("entities", 0, 0); + let region = super::cached_region(&save.entity_regions, (0, 0), &path, true) + .unwrap() + .unwrap(); + // Hold the actual region lock, rather than relying on disk speed or sleeps. + let blocked = region.lock().unwrap(); + let entity = |version| Tag::compound([("version", Tag::Int(version))]); + save.save_entities_later(vec![((1, 2), vec![entity(1)])]) + .unwrap(); + save.save_entities_later(vec![((1, 2), vec![entity(2)])]) + .unwrap(); + assert!(!save.writes_finished().unwrap()); + assert_eq!(save.load_entities(1, 2), vec![entity(2)]); + assert!(super::SessionLock::acquire(&dir).is_err()); + drop(blocked); + save.flush().unwrap(); + assert!(save.writes_finished().unwrap()); + let reopened = super::SaveDir::new(dir.clone()); + assert_eq!(reopened.load_entities(1, 2), vec![entity(2)]); + save.save_entities_later(vec![((1, 2), vec![])]).unwrap(); + save.flush().unwrap(); + assert!( + super::SaveDir::new(dir.clone()) + .load_entities(1, 2) + .is_empty() + ); + drop((reopened, save, lock)); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn failed_async_writes_are_readable_and_never_reported_durable() { + let dir = std::env::temp_dir().join(format!("horizon-save-failure-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + // A file where a directory is needed causes a real writer I/O error. + std::fs::write(dir.join("entities"), b"blocked").unwrap(); + let save = super::SaveDir::new(dir.clone()); + let entity = Tag::compound([("version", Tag::Int(3))]); + save.save_entities_later(vec![((0, 0), vec![entity.clone()])]) + .unwrap(); + assert!(save.flush().is_err()); + assert!(save.writes_finished().is_err()); + assert_eq!(save.load_entities(0, 0), vec![entity]); + // The failure stays visible on subsequent barriers too. + assert!(save.flush().is_err()); + drop(save); + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn session_lock_is_exclusive() { let dir = std::env::temp_dir().join(format!("horizon-lock-{}", std::process::id())); diff --git a/crates/horizon-client/src/streaming.rs b/crates/horizon-client/src/streaming.rs index 3d791729c..58b155883 100644 --- a/crates/horizon-client/src/streaming.rs +++ b/crates/horizon-client/src/streaming.rs @@ -1314,6 +1314,30 @@ impl Streaming { } self.written.clear(); } + /// Finish generation for a closing world without waiting on the event + /// loop. No new generation is scheduled while the save screen is shown. + /// Proto snapshots are checked out to workers before chunk writes, just as + /// in `save(true)`, so older generation cannot overwrite the final chunks. + pub fn poll_save(&mut self, world: &mut World) -> bool { + let started = std::time::Instant::now(); + while let Ok(done) = self.results.try_recv() { + match done { + Done::Decorated(x, z, region) => self.receive_decorated(x, z, region), + Done::Generated(chunk) => self.receive_generated(chunk, world), + Done::Saved(protos) => self.receive_saved(protos), + other => self.deferred.push(other), + } + if started.elapsed() >= std::time::Duration::from_millis(2) { + return false; + } + } + if self.decorating > 0 || !self.finishing.is_empty() || self.saving > 0 { + return false; + } + let written: Vec<_> = self.written.drain().collect(); + self.save_checked_out(written); + self.saving == 0 + } fn in_range(&self, x: i32, z: i32, margin: i32) -> bool { self.ranges().in_range(x, z, margin) } diff --git a/crates/horizon-client/tests/generation_saves.rs b/crates/horizon-client/tests/generation_saves.rs index ce91a1096..df47d1fa3 100644 --- a/crates/horizon-client/tests/generation_saves.rs +++ b/crates/horizon-client/tests/generation_saves.rs @@ -155,7 +155,11 @@ impl Session { self.streaming.save(&mut self.world, false); } fn save(&mut self) { - self.streaming.save(&mut self.world, true); + let deadline = Instant::now() + Duration::from_secs(120); + while !self.streaming.poll_save(&mut self.world) { + assert!(Instant::now() < deadline, "incremental save never finished"); + thread::sleep(Duration::from_millis(1)); + } for key in std::mem::take(&mut self.world.modified) { if let Some(chunk) = self.world.chunks.get(&key) { self.generator.save_chunk(chunk); @@ -269,7 +273,7 @@ fn saved_proto_chunks_restore_exactly() { generator.decorate(&mut worker, &mut region); assert!(generator.is_decorated(®ion[4])); generator.save_protos(®ion.iter().collect::>()); - save.flush(); + save.flush().unwrap(); let restored: Vec = keys .iter() .map(|&(x, z)| generator.proto(&mut worker, x, z))