Prepare world save snapshots incrementally and poll durable completion
This commit is contained in:
@@ -87,7 +87,7 @@ fn main() -> Result<()> {
|
||||
let _ = save.tables.set(tables.clone());
|
||||
let started = Instant::now();
|
||||
save.save_later(&chunks, registry, &biomes)?;
|
||||
save.flush();
|
||||
save.flush().unwrap();
|
||||
times[2].push(started.elapsed().as_secs_f64());
|
||||
drop(save);
|
||||
|
||||
|
||||
@@ -118,6 +118,7 @@ mod remote_projectiles;
|
||||
mod render_control;
|
||||
mod render_tick;
|
||||
mod respawn_anchor;
|
||||
mod saving;
|
||||
mod scoreboard;
|
||||
mod screens;
|
||||
mod sculk;
|
||||
@@ -1366,6 +1367,7 @@ struct Game<'a> {
|
||||
text_selection: horizon_client::clipboard::AppendSelection,
|
||||
/// The world was saved for the application ending (`save_for_exit`).
|
||||
saved_for_exit: bool,
|
||||
pending_save: Option<saving::ClosingSave>,
|
||||
/// The block a guest is acting on (game/lan_actions.rs).
|
||||
forced_hit: Option<Hit>,
|
||||
/// The part of a right click a guest's packet asks for (game/lan_actions.rs).
|
||||
@@ -7240,13 +7242,21 @@ impl Game<'_> {
|
||||
}
|
||||
|
||||
pub(super) fn save_for_exit(&mut self) -> Result<()> {
|
||||
// Native application termination can bypass CloseRequested. Finish any
|
||||
// save screen synchronously here, before the process can go away.
|
||||
while self.pending_save.is_some() {
|
||||
self.poll_closing_save()?;
|
||||
if self.pending_save.is_some() {
|
||||
std::thread::sleep(Duration::from_millis(1));
|
||||
}
|
||||
}
|
||||
if self.title.is_some() || self.saved_for_exit {
|
||||
return Ok(());
|
||||
}
|
||||
self.saved_for_exit = true;
|
||||
if let Some(server) = self.integrated.take() {
|
||||
// The server thread saves the world.
|
||||
server.stop();
|
||||
server.stop()?;
|
||||
self.saved_for_exit = true;
|
||||
return Ok(());
|
||||
}
|
||||
self.close_container();
|
||||
@@ -7254,9 +7264,14 @@ impl Game<'_> {
|
||||
self.stats.custom("leave_game", 1);
|
||||
}
|
||||
self.lan_close(false);
|
||||
self.save_game()
|
||||
self.save_game()?;
|
||||
self.saved_for_exit = true;
|
||||
Ok(())
|
||||
}
|
||||
fn close_world(&mut self) -> Result<()> {
|
||||
self.begin_closing_save(false)
|
||||
}
|
||||
fn finish_close_world(&mut self) -> Result<()> {
|
||||
self.ambience.clear(&mut self.audio);
|
||||
// updateLevelInEngines(null): every sound stops; the music manager
|
||||
// lets go of its track.
|
||||
@@ -7265,13 +7280,7 @@ impl Game<'_> {
|
||||
if self.remote.is_some() {
|
||||
return self.leave_server(None);
|
||||
}
|
||||
self.finish_perf();
|
||||
self.stats.custom("leave_game", 1);
|
||||
self.save_game()?;
|
||||
self.lan_close(false);
|
||||
// Closing the level storage: once the last writes are on disk the
|
||||
// session lock goes, so the title can delete or back up this world.
|
||||
self.save.flush();
|
||||
// The save screen has finished every write before releasing storage.
|
||||
self.session_lock = None;
|
||||
horizon_client::crash::clear_detail("World");
|
||||
self.debug_charts.reset();
|
||||
@@ -9037,6 +9046,9 @@ impl Game<'_> {
|
||||
}
|
||||
self.save_level(wait)?;
|
||||
self.save_parked_levels(wait)?;
|
||||
self.save_game_metadata()
|
||||
}
|
||||
fn save_game_metadata(&mut self) -> Result<()> {
|
||||
self.save_maps();
|
||||
self.save_random_sequences();
|
||||
self.save_fight();
|
||||
@@ -9062,11 +9074,12 @@ impl Game<'_> {
|
||||
let chunks = self.with_entities(keys.iter().filter_map(|k| self.world.chunks.get(k)));
|
||||
self.save
|
||||
.save_later(&chunks, self.pack.assets.registry, self.res.biomes())?;
|
||||
if wait {
|
||||
self.save.flush();
|
||||
}
|
||||
self.save_entities(wait)?;
|
||||
// Prepare entities while the chunk writer runs, then use one barrier.
|
||||
self.save_entities(false)?;
|
||||
self.save_raids();
|
||||
if wait {
|
||||
self.save.flush()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn render(&mut self) -> Result<bool> {
|
||||
@@ -10079,11 +10092,36 @@ impl ApplicationHandler for Game<'_> {
|
||||
{
|
||||
return;
|
||||
}
|
||||
if self.pending_save.is_some() {
|
||||
match event {
|
||||
WindowEvent::CloseRequested => {
|
||||
self.pending_save.as_mut().unwrap().quit = true;
|
||||
}
|
||||
WindowEvent::Focused(focused) => {
|
||||
self.focused = focused;
|
||||
if !focused {
|
||||
self.capture(false);
|
||||
}
|
||||
}
|
||||
WindowEvent::Resized(size) => {
|
||||
if let Some(r) = &mut self.renderer {
|
||||
r.resize(size.width, size.height);
|
||||
}
|
||||
}
|
||||
WindowEvent::RedrawRequested => {
|
||||
if let Err(error) = self.render_closing_save() {
|
||||
self.error = Some(error);
|
||||
event_loop.exit();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
return;
|
||||
}
|
||||
let result: Result<()> = (|| {
|
||||
match event {
|
||||
WindowEvent::CloseRequested => {
|
||||
self.save_for_exit()?;
|
||||
event_loop.exit();
|
||||
self.begin_closing_save(true)?;
|
||||
}
|
||||
WindowEvent::Resized(size) => {
|
||||
if let Some(r) = &mut self.renderer {
|
||||
@@ -10724,7 +10762,8 @@ impl ApplicationHandler for Game<'_> {
|
||||
}
|
||||
}
|
||||
fn device_event(&mut self, _: &ActiveEventLoop, _: DeviceId, event: DeviceEvent) {
|
||||
if self.captured
|
||||
if self.pending_save.is_none()
|
||||
&& self.captured
|
||||
&& self.focused
|
||||
&& self.screen == Screen::Playing
|
||||
&& let DeviceEvent::MouseMotion { delta: (x, y) } = event
|
||||
@@ -10735,12 +10774,34 @@ impl ApplicationHandler for Game<'_> {
|
||||
}
|
||||
}
|
||||
fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) {
|
||||
if self.pending_save.is_some() {
|
||||
match self.poll_closing_save() {
|
||||
Ok(true) => event_loop.exit(),
|
||||
Ok(false) => {
|
||||
if let Some(window) = &self.window {
|
||||
window.request_redraw();
|
||||
}
|
||||
// Yield between polls even when disk I/O is slow.
|
||||
event_loop.set_control_flow(ControlFlow::WaitUntil(
|
||||
Instant::now() + Duration::from_millis(4),
|
||||
));
|
||||
}
|
||||
Err(error) => {
|
||||
self.error = Some(error);
|
||||
event_loop.exit();
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if let Err(error) = self.resume_world_entry() {
|
||||
self.error = Some(error);
|
||||
event_loop.exit();
|
||||
return;
|
||||
}
|
||||
self.poll_control(event_loop);
|
||||
if self.pending_save.is_some() {
|
||||
return;
|
||||
}
|
||||
self.poll_pads(event_loop);
|
||||
if self.options.frames.is_some() && self.started.elapsed() > self.options.timeout {
|
||||
self.error = Some("frame-limited run timed out".into());
|
||||
@@ -11087,6 +11148,7 @@ impl<'a> Game<'a> {
|
||||
clipboard: Default::default(),
|
||||
text_selection: Default::default(),
|
||||
saved_for_exit: false,
|
||||
pending_save: None,
|
||||
forced_hit: None,
|
||||
guest_use: None,
|
||||
lan_events: Vec::new(),
|
||||
|
||||
@@ -117,6 +117,9 @@ impl Game<'_> {
|
||||
}
|
||||
while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) {
|
||||
self.answer(request, event_loop);
|
||||
if self.pending_save.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if self.control.is_some() {
|
||||
self.watch_events();
|
||||
@@ -140,6 +143,9 @@ impl Game<'_> {
|
||||
event_loop: &ActiveEventLoop,
|
||||
) -> Result<Option<Value>> {
|
||||
let words: Vec<&str> = line.split_whitespace().collect();
|
||||
if self.pending_save.is_some() && words.first() != Some(&"quit") {
|
||||
return Err("a world save is in progress".into());
|
||||
}
|
||||
let arg = |i: usize| {
|
||||
words
|
||||
.get(i)
|
||||
@@ -625,6 +631,9 @@ impl Game<'_> {
|
||||
server
|
||||
.call(move |g| g.control_save(all).map_err(|e| e.to_string()))
|
||||
.ok_or("the integrated server stopped")??;
|
||||
} else if self.remote.is_none() {
|
||||
self.begin_control_save(all, reply.clone())?;
|
||||
return Ok(None);
|
||||
} else {
|
||||
self.control_save(all)?;
|
||||
}
|
||||
@@ -637,8 +646,7 @@ impl Game<'_> {
|
||||
ok()
|
||||
}
|
||||
"quit" => {
|
||||
self.save_for_exit()?;
|
||||
event_loop.exit();
|
||||
self.begin_closing_save(true)?;
|
||||
ok()
|
||||
}
|
||||
other => match self.station_command(&words)? {
|
||||
|
||||
@@ -27,7 +27,7 @@ use std::{
|
||||
net::TcpListener,
|
||||
path::Path,
|
||||
sync::{
|
||||
Arc,
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, AtomicI32, AtomicU16, AtomicU32, Ordering},
|
||||
mpsc::{Receiver, Sender, channel, sync_channel},
|
||||
},
|
||||
@@ -41,6 +41,7 @@ struct Flags {
|
||||
/// `Minecraft.pause`: a single-player pause screen is open.
|
||||
paused: AtomicBool,
|
||||
stop: AtomicBool,
|
||||
failure: Mutex<Option<String>>,
|
||||
/// `IntegratedServer.getPort`: the port it is open to the LAN on, 0
|
||||
/// while it is not.
|
||||
published: AtomicU16,
|
||||
@@ -96,19 +97,42 @@ impl Integrated {
|
||||
rx.recv_timeout(Duration::from_secs(10)).ok()
|
||||
}
|
||||
/// `IntegratedServer.halt(true)`: returns once the world is saved and let go.
|
||||
pub fn stop(mut self) {
|
||||
self.halt();
|
||||
pub fn stop(mut self) -> Result<()> {
|
||||
self.halt()
|
||||
}
|
||||
fn halt(&mut self) {
|
||||
/// Stop on the server thread; the window polls completion while drawing.
|
||||
pub fn request_stop(&self) {
|
||||
self.flags.stop.store(true, Ordering::Relaxed);
|
||||
}
|
||||
pub fn stopped(&self) -> bool {
|
||||
self.thread
|
||||
.as_ref()
|
||||
.is_none_or(|thread| thread.is_finished())
|
||||
}
|
||||
fn halt(&mut self) -> Result<()> {
|
||||
self.request_stop();
|
||||
if let Some(thread) = self.thread.take() {
|
||||
let _ = thread.join();
|
||||
thread
|
||||
.join()
|
||||
.map_err(|_| "the integrated server panicked")?;
|
||||
}
|
||||
match self
|
||||
.flags
|
||||
.failure
|
||||
.lock()
|
||||
.map_err(|_| "server status poisoned")?
|
||||
.as_ref()
|
||||
{
|
||||
Some(error) => Err(error.clone().into()),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Drop for Integrated {
|
||||
fn drop(&mut self) {
|
||||
self.halt();
|
||||
if let Err(error) = self.halt() {
|
||||
log::error!("Horizon: integrated server shutdown: {error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -224,12 +248,13 @@ impl Game<'_> {
|
||||
options,
|
||||
settings,
|
||||
launch,
|
||||
server_flags,
|
||||
server_flags.clone(),
|
||||
server_tasks,
|
||||
&ready_tx,
|
||||
);
|
||||
if let Err(e) = result {
|
||||
log::error!("Horizon: integrated server: {e}");
|
||||
*server_flags.failure.lock().unwrap() = Some(e.to_string());
|
||||
let _ = ready_tx.send(Err(e.to_string()));
|
||||
}
|
||||
})?;
|
||||
@@ -386,7 +411,7 @@ impl Game<'_> {
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
self.save_game()?;
|
||||
self.lan_close(false);
|
||||
self.save.flush();
|
||||
self.save.flush()?;
|
||||
self.session_lock = None;
|
||||
Ok(())
|
||||
}
|
||||
@@ -663,3 +688,42 @@ fn serve(
|
||||
let mut server = Game::new(res, pack, gui, options, settings, audio)?;
|
||||
server.run_integrated(launch, flags, tasks, ready)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn shutdown_request_does_not_join_a_server_still_saving() {
|
||||
let flags = Arc::new(Flags::default());
|
||||
let (release, wait) = channel();
|
||||
let (tasks, _) = channel();
|
||||
let server = Integrated {
|
||||
flags: flags.clone(),
|
||||
thread: Some(std::thread::spawn(move || wait.recv().unwrap())),
|
||||
tasks,
|
||||
world_name: String::new(),
|
||||
};
|
||||
server.request_stop();
|
||||
assert!(flags.stop.load(Ordering::Relaxed));
|
||||
assert!(!server.stopped());
|
||||
release.send(()).unwrap();
|
||||
server.stop().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shutdown_reports_a_failed_server_save() {
|
||||
let flags = Arc::new(Flags::default());
|
||||
let saving = flags.clone();
|
||||
let (tasks, _) = channel();
|
||||
let server = Integrated {
|
||||
flags,
|
||||
thread: Some(std::thread::spawn(move || {
|
||||
*saving.failure.lock().unwrap() = Some("disk write failed".into());
|
||||
})),
|
||||
tasks,
|
||||
world_name: String::new(),
|
||||
};
|
||||
assert_eq!(server.stop().unwrap_err().to_string(), "disk write failed");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -559,7 +559,7 @@ impl Game<'_> {
|
||||
/// The entity NBT of every simulated entity in `keys`, plus the unknown
|
||||
/// entities kept for them. With `remove`, the entities leave the world
|
||||
/// (their chunks unload).
|
||||
fn entity_chunks(
|
||||
pub(super) fn entity_chunks(
|
||||
&mut self,
|
||||
keys: &HashSet<(i32, i32)>,
|
||||
remove: bool,
|
||||
@@ -858,6 +858,14 @@ impl Game<'_> {
|
||||
self.spawn_loaded_entities();
|
||||
let keys: HashSet<(i32, i32)> = self.world.chunks.keys().copied().collect();
|
||||
let chunks = self.entity_chunks(&keys, false);
|
||||
self.prune_entity_memory();
|
||||
if wait {
|
||||
self.save.save_entities(chunks)
|
||||
} else {
|
||||
self.save.save_entities_later(chunks)
|
||||
}
|
||||
}
|
||||
pub(super) fn prune_entity_memory(&mut self) {
|
||||
// Forget entities that no longer exist.
|
||||
let alive: HashSet<u32> = self
|
||||
.mobs
|
||||
@@ -873,11 +881,6 @@ impl Game<'_> {
|
||||
.collect();
|
||||
self.persist.uuids.retain(|id, _| alive.contains(id));
|
||||
self.persist.raw.retain(|id, _| alive.contains(id));
|
||||
if wait {
|
||||
self.save.save_entities(chunks)
|
||||
} else {
|
||||
self.save.save_entities_later(chunks)
|
||||
}
|
||||
}
|
||||
/// Save and remove the entities of chunks that unloaded.
|
||||
pub(super) fn unload_entities(&mut self, unloaded: &[Arc<Chunk>]) {
|
||||
|
||||
@@ -361,6 +361,9 @@ impl Game<'_> {
|
||||
|
||||
/// Disconnect and return to the multiplayer screen (DisconnectedScreen).
|
||||
pub(super) fn leave_server(&mut self, reason: Option<String>) -> Result<()> {
|
||||
if reason.is_none() && self.integrated.as_ref().is_some_and(|s| !s.stopped()) {
|
||||
return self.begin_closing_save(false);
|
||||
}
|
||||
self.finish_perf();
|
||||
if reason.is_none() {
|
||||
// Leaving on purpose: nothing for auto-reconnect to retry.
|
||||
@@ -372,7 +375,7 @@ impl Game<'_> {
|
||||
// Single player's server thread saves the world before the title shows.
|
||||
let single_player = match self.integrated.take() {
|
||||
Some(server) => {
|
||||
server.stop();
|
||||
server.stop()?;
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
//! The save screen freezes simulation, but keeps pumping window events while
|
||||
//! snapshots are prepared in small batches and the region workers drain.
|
||||
use super::{Game, loading_frame};
|
||||
use horizon_client::{Result, hud::HudBatch};
|
||||
use std::{
|
||||
collections::{HashSet, VecDeque},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
pub(super) struct ClosingSave {
|
||||
pub quit: bool,
|
||||
close: bool,
|
||||
reply: Option<std::sync::mpsc::Sender<String>>,
|
||||
started: Instant,
|
||||
stage: Stage,
|
||||
}
|
||||
enum Stage {
|
||||
Generation,
|
||||
Chunks(VecDeque<(i32, i32)>),
|
||||
Entities(VecDeque<(i32, i32)>),
|
||||
Writes,
|
||||
Server,
|
||||
}
|
||||
|
||||
// Bound the work in both sparse and block-entity-heavy worlds. A single chunk
|
||||
// is indivisible; check the time budget after each snapshot, not each batch.
|
||||
const SNAPSHOTS_PER_FRAME: usize = 16;
|
||||
const SNAPSHOT_BUDGET: Duration = Duration::from_millis(2);
|
||||
|
||||
impl Game<'_> {
|
||||
pub(super) fn begin_closing_save(&mut self, quit: bool) -> Result<()> {
|
||||
if let Some(save) = &mut self.pending_save {
|
||||
save.quit |= quit;
|
||||
return Ok(());
|
||||
}
|
||||
if self.title.is_some() || self.saved_for_exit {
|
||||
// A quit from the title still needs the event loop to finish.
|
||||
self.pending_save = Some(ClosingSave {
|
||||
quit,
|
||||
close: true,
|
||||
reply: None,
|
||||
started: Instant::now(),
|
||||
stage: Stage::Writes,
|
||||
});
|
||||
return Ok(());
|
||||
}
|
||||
if self.integrated.is_some() {
|
||||
self.integrated.as_ref().unwrap().request_stop();
|
||||
self.pending_save = Some(ClosingSave {
|
||||
quit,
|
||||
close: true,
|
||||
reply: None,
|
||||
started: Instant::now(),
|
||||
stage: Stage::Server,
|
||||
});
|
||||
} else if self.remote.is_some() {
|
||||
if !quit {
|
||||
return self.leave_server(None);
|
||||
}
|
||||
self.pending_save = Some(ClosingSave {
|
||||
quit,
|
||||
close: true,
|
||||
reply: None,
|
||||
started: Instant::now(),
|
||||
stage: Stage::Writes,
|
||||
});
|
||||
} else {
|
||||
self.close_container();
|
||||
self.stats.custom("leave_game", 1);
|
||||
self.lan_close(false);
|
||||
self.pending_save = Some(ClosingSave {
|
||||
quit,
|
||||
close: true,
|
||||
reply: None,
|
||||
started: Instant::now(),
|
||||
stage: Stage::Generation,
|
||||
});
|
||||
}
|
||||
self.capture(false);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Control saves acknowledge only after durable completion, while their
|
||||
/// snapshot work is spread across event-loop turns too.
|
||||
pub(super) fn begin_control_save(
|
||||
&mut self,
|
||||
all: bool,
|
||||
reply: std::sync::mpsc::Sender<String>,
|
||||
) -> Result<()> {
|
||||
if self.pending_save.is_some() {
|
||||
return Err("a world save is already in progress".into());
|
||||
}
|
||||
if all {
|
||||
self.world
|
||||
.modified
|
||||
.extend(self.world.chunks.keys().copied());
|
||||
}
|
||||
self.keys.clear();
|
||||
self.physical_keys.clear();
|
||||
self.left = false;
|
||||
self.right = false;
|
||||
self.pending_save = Some(ClosingSave {
|
||||
quit: false,
|
||||
close: false,
|
||||
reply: Some(reply),
|
||||
started: Instant::now(),
|
||||
stage: Stage::Generation,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `true` means the window can exit. The session lock stays in Game until
|
||||
/// every snapshot has been queued and every writer has synced its files.
|
||||
pub(super) fn poll_closing_save(&mut self) -> Result<bool> {
|
||||
let Some(mut save) = self.pending_save.take() else {
|
||||
return Ok(false);
|
||||
};
|
||||
let result = (|| -> Result<bool> {
|
||||
match &mut save.stage {
|
||||
Stage::Server => {
|
||||
let server = self
|
||||
.integrated
|
||||
.as_ref()
|
||||
.ok_or("the integrated server stopped with an error")?;
|
||||
if !server.stopped() {
|
||||
return Ok(false);
|
||||
}
|
||||
// Drop/join only after completion; leave_server's teardown
|
||||
// can then run without blocking on the server's writes.
|
||||
if save.quit {
|
||||
self.integrated.take().unwrap().stop()?;
|
||||
} else {
|
||||
self.leave_server(None)?;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
Stage::Generation => {
|
||||
if save.started.elapsed() > Duration::from_secs(30) {
|
||||
return Err("chunk generation did not finish before saving".into());
|
||||
}
|
||||
if self.streaming.poll_save(&mut self.world) {
|
||||
save.stage = Stage::Chunks(self.world.modified.iter().copied().collect());
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
Stage::Chunks(keys) => {
|
||||
let started = Instant::now();
|
||||
for _ in 0..SNAPSHOTS_PER_FRAME {
|
||||
let Some(&key) = keys.front() else { break };
|
||||
let chunks = self.with_entities(self.world.chunks.get(&key));
|
||||
self.save.save_later(
|
||||
&chunks,
|
||||
self.pack.assets.registry,
|
||||
self.res.biomes(),
|
||||
)?;
|
||||
self.world.modified.remove(&key);
|
||||
keys.pop_front();
|
||||
if started.elapsed() >= SNAPSHOT_BUDGET {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if keys.is_empty() {
|
||||
self.spawn_loaded_entities();
|
||||
save.stage = Stage::Entities(self.world.chunks.keys().copied().collect());
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
Stage::Entities(keys) => {
|
||||
let started = Instant::now();
|
||||
for _ in 0..SNAPSHOTS_PER_FRAME {
|
||||
let Some(&key) = keys.front() else { break };
|
||||
let chunks = self.entity_chunks(&HashSet::from([key]), false);
|
||||
self.save.save_entities_later(chunks)?;
|
||||
keys.pop_front();
|
||||
if started.elapsed() >= SNAPSHOT_BUDGET {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if keys.is_empty() {
|
||||
self.prune_entity_memory();
|
||||
self.save_raids();
|
||||
self.save_game_metadata()?;
|
||||
save.stage = Stage::Writes;
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
Stage::Writes => self.save.writes_finished(),
|
||||
}
|
||||
})();
|
||||
match result {
|
||||
Ok(true) => {
|
||||
if let Some(reply) = save.reply.take() {
|
||||
let _ = reply.send(serde_json::json!({ "ok": true }).to_string());
|
||||
}
|
||||
if save.quit && !save.close {
|
||||
// A window close during a control save still has to close
|
||||
// containers and save the resulting inventory changes.
|
||||
self.begin_closing_save(true)?;
|
||||
return Ok(false);
|
||||
}
|
||||
if save.quit {
|
||||
self.saved_for_exit = true;
|
||||
Ok(true)
|
||||
} else {
|
||||
if save.close && self.title.is_none() {
|
||||
self.finish_perf();
|
||||
self.finish_close_world()?;
|
||||
}
|
||||
self.clock.reset();
|
||||
self.previous = Instant::now();
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
other => {
|
||||
if let Err(error) = &other
|
||||
&& let Some(reply) = save.reply.take()
|
||||
{
|
||||
let _ =
|
||||
reply.send(serde_json::json!({ "error": error.to_string() }).to_string());
|
||||
}
|
||||
// Keep ownership (and the lock) on errors too. Native shutdown
|
||||
// sees the same durable-write failure instead of declaring success.
|
||||
self.pending_save = Some(save);
|
||||
other
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn render_closing_save(&mut self) -> Result<()> {
|
||||
let Some(renderer) = &mut self.renderer else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut hud = HudBatch::new([renderer.config.width, renderer.config.height]);
|
||||
let progress = match self.pending_save.as_ref().map(|save| &save.stage) {
|
||||
Some(Stage::Chunks(_)) => 0.25,
|
||||
Some(Stage::Entities(_)) => 0.5,
|
||||
Some(Stage::Writes) => 0.9,
|
||||
_ => 0.0,
|
||||
};
|
||||
horizon_client::level_loading::draw_progress(
|
||||
&mut hud,
|
||||
&self.gui.atlas,
|
||||
"Saving world...",
|
||||
progress,
|
||||
);
|
||||
renderer.render(&loading_frame(&hud))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1487,6 +1487,7 @@ struct Writer {
|
||||
#[derive(Default)]
|
||||
struct Outstanding {
|
||||
count: Mutex<usize>,
|
||||
error: Mutex<Option<String>>,
|
||||
done: Condvar,
|
||||
}
|
||||
/// One dimension folder (`dimensions/minecraft/<name>`). Region files are
|
||||
@@ -1622,11 +1623,22 @@ impl SaveDir {
|
||||
let mut writer = self.writer.lock().map_err(|_| "writer poisoned")?;
|
||||
let writer = writer.get_or_insert_with(|| self.start_writer());
|
||||
if writer.send.send((entities, batch)).is_err() {
|
||||
*self
|
||||
.outstanding
|
||||
.error
|
||||
.lock()
|
||||
.map_err(|_| "writer poisoned")? = Some("the region writer stopped".into());
|
||||
*self
|
||||
.outstanding
|
||||
.count
|
||||
.lock()
|
||||
.map_err(|_| "writer poisoned")? -= 1;
|
||||
self.outstanding.done.notify_all();
|
||||
return Err("the region writer stopped".into());
|
||||
}
|
||||
}
|
||||
if wait {
|
||||
self.flush();
|
||||
self.flush()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1657,11 +1669,16 @@ impl SaveDir {
|
||||
.filter(|(k, _, seq)| newest(k, *seq))
|
||||
.map(|(k, r, _)| (*k, r))
|
||||
.collect();
|
||||
if let Err(e) = write_region_records(&root, cache, folder, &records) {
|
||||
log::warn!("Horizon: failed to write {folder} files: {e}");
|
||||
let result = write_region_records(&root, cache, folder, &records);
|
||||
if let Err(e) = &result {
|
||||
let message = format!("failed to write {folder} files: {e}");
|
||||
log::warn!("Horizon: {message}");
|
||||
*outstanding.error.lock().unwrap() = Some(message);
|
||||
}
|
||||
// Readers find the files (and a fresh region cache) from here on.
|
||||
if let Ok(mut pending) = pending.lock() {
|
||||
// Keep failed snapshots readable; never report them as durable.
|
||||
if result.is_ok()
|
||||
&& let Ok(mut pending) = pending.lock()
|
||||
{
|
||||
for (k, _, seq) in &batch {
|
||||
if pending
|
||||
.get(&(entities, k.0, k.1))
|
||||
@@ -1681,15 +1698,41 @@ impl SaveDir {
|
||||
Writer { send, thread }
|
||||
}
|
||||
/// Wait until every queued write is on disk.
|
||||
pub fn flush(&self) {
|
||||
let Ok(mut count) = self.outstanding.count.lock() else {
|
||||
return;
|
||||
};
|
||||
pub fn flush(&self) -> Result<()> {
|
||||
let mut count = self
|
||||
.outstanding
|
||||
.count
|
||||
.lock()
|
||||
.map_err(|_| "writer poisoned")?;
|
||||
while *count > 0 {
|
||||
count = match self.outstanding.done.wait(count) {
|
||||
Ok(c) => c,
|
||||
Err(_) => return,
|
||||
};
|
||||
count = self
|
||||
.outstanding
|
||||
.done
|
||||
.wait(count)
|
||||
.map_err(|_| "writer poisoned")?;
|
||||
}
|
||||
self.write_result()
|
||||
}
|
||||
/// Poll durability without holding up the event loop. Errors remain visible
|
||||
/// to every waiter, including a later synchronous shutdown.
|
||||
pub fn writes_finished(&self) -> Result<bool> {
|
||||
let count = self
|
||||
.outstanding
|
||||
.count
|
||||
.lock()
|
||||
.map_err(|_| "writer poisoned")?;
|
||||
self.write_result()?;
|
||||
Ok(*count == 0)
|
||||
}
|
||||
fn write_result(&self) -> Result<()> {
|
||||
match &*self
|
||||
.outstanding
|
||||
.error
|
||||
.lock()
|
||||
.map_err(|_| "writer poisoned")?
|
||||
{
|
||||
Some(error) => Err(error.clone().into()),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
/// Write `chunks` into their region files and wait until they are on disk.
|
||||
@@ -3593,6 +3636,62 @@ mod tests {
|
||||
super::upgrade_older_world(&dir).unwrap();
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
#[test]
|
||||
fn write_completion_polls_without_waiting_and_keeps_latest_snapshot() {
|
||||
let dir = std::env::temp_dir().join(format!("horizon-save-poll-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let save = super::SaveDir::new(dir.clone());
|
||||
let lock = super::SessionLock::acquire(&dir).unwrap();
|
||||
let path = save.region_path("entities", 0, 0);
|
||||
let region = super::cached_region(&save.entity_regions, (0, 0), &path, true)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
// Hold the actual region lock, rather than relying on disk speed or sleeps.
|
||||
let blocked = region.lock().unwrap();
|
||||
let entity = |version| Tag::compound([("version", Tag::Int(version))]);
|
||||
save.save_entities_later(vec![((1, 2), vec![entity(1)])])
|
||||
.unwrap();
|
||||
save.save_entities_later(vec![((1, 2), vec![entity(2)])])
|
||||
.unwrap();
|
||||
assert!(!save.writes_finished().unwrap());
|
||||
assert_eq!(save.load_entities(1, 2), vec![entity(2)]);
|
||||
assert!(super::SessionLock::acquire(&dir).is_err());
|
||||
drop(blocked);
|
||||
save.flush().unwrap();
|
||||
assert!(save.writes_finished().unwrap());
|
||||
let reopened = super::SaveDir::new(dir.clone());
|
||||
assert_eq!(reopened.load_entities(1, 2), vec![entity(2)]);
|
||||
save.save_entities_later(vec![((1, 2), vec![])]).unwrap();
|
||||
save.flush().unwrap();
|
||||
assert!(
|
||||
super::SaveDir::new(dir.clone())
|
||||
.load_entities(1, 2)
|
||||
.is_empty()
|
||||
);
|
||||
drop((reopened, save, lock));
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_async_writes_are_readable_and_never_reported_durable() {
|
||||
let dir = std::env::temp_dir().join(format!("horizon-save-failure-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
// A file where a directory is needed causes a real writer I/O error.
|
||||
std::fs::write(dir.join("entities"), b"blocked").unwrap();
|
||||
let save = super::SaveDir::new(dir.clone());
|
||||
let entity = Tag::compound([("version", Tag::Int(3))]);
|
||||
save.save_entities_later(vec![((0, 0), vec![entity.clone()])])
|
||||
.unwrap();
|
||||
assert!(save.flush().is_err());
|
||||
assert!(save.writes_finished().is_err());
|
||||
assert_eq!(save.load_entities(0, 0), vec![entity]);
|
||||
// The failure stays visible on subsequent barriers too.
|
||||
assert!(save.flush().is_err());
|
||||
drop(save);
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_lock_is_exclusive() {
|
||||
let dir = std::env::temp_dir().join(format!("horizon-lock-{}", std::process::id()));
|
||||
|
||||
@@ -1314,6 +1314,30 @@ impl Streaming {
|
||||
}
|
||||
self.written.clear();
|
||||
}
|
||||
/// Finish generation for a closing world without waiting on the event
|
||||
/// loop. No new generation is scheduled while the save screen is shown.
|
||||
/// Proto snapshots are checked out to workers before chunk writes, just as
|
||||
/// in `save(true)`, so older generation cannot overwrite the final chunks.
|
||||
pub fn poll_save(&mut self, world: &mut World) -> bool {
|
||||
let started = std::time::Instant::now();
|
||||
while let Ok(done) = self.results.try_recv() {
|
||||
match done {
|
||||
Done::Decorated(x, z, region) => self.receive_decorated(x, z, region),
|
||||
Done::Generated(chunk) => self.receive_generated(chunk, world),
|
||||
Done::Saved(protos) => self.receive_saved(protos),
|
||||
other => self.deferred.push(other),
|
||||
}
|
||||
if started.elapsed() >= std::time::Duration::from_millis(2) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if self.decorating > 0 || !self.finishing.is_empty() || self.saving > 0 {
|
||||
return false;
|
||||
}
|
||||
let written: Vec<_> = self.written.drain().collect();
|
||||
self.save_checked_out(written);
|
||||
self.saving == 0
|
||||
}
|
||||
fn in_range(&self, x: i32, z: i32, margin: i32) -> bool {
|
||||
self.ranges().in_range(x, z, margin)
|
||||
}
|
||||
|
||||
@@ -155,7 +155,11 @@ impl Session {
|
||||
self.streaming.save(&mut self.world, false);
|
||||
}
|
||||
fn save(&mut self) {
|
||||
self.streaming.save(&mut self.world, true);
|
||||
let deadline = Instant::now() + Duration::from_secs(120);
|
||||
while !self.streaming.poll_save(&mut self.world) {
|
||||
assert!(Instant::now() < deadline, "incremental save never finished");
|
||||
thread::sleep(Duration::from_millis(1));
|
||||
}
|
||||
for key in std::mem::take(&mut self.world.modified) {
|
||||
if let Some(chunk) = self.world.chunks.get(&key) {
|
||||
self.generator.save_chunk(chunk);
|
||||
@@ -269,7 +273,7 @@ fn saved_proto_chunks_restore_exactly() {
|
||||
generator.decorate(&mut worker, &mut region);
|
||||
assert!(generator.is_decorated(®ion[4]));
|
||||
generator.save_protos(®ion.iter().collect::<Vec<_>>());
|
||||
save.flush();
|
||||
save.flush().unwrap();
|
||||
let restored: Vec<Proto> = keys
|
||||
.iter()
|
||||
.map(|&(x, z)| generator.proto(&mut worker, x, z))
|
||||
|
||||
Reference in New Issue
Block a user