From c2b7f3a4ea2dc5fdb86f4aa6469eabc4b2a26e22 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:55:54 +0200 Subject: [PATCH] Slow-frame and slow-tick lines show the CPU time of each span HORIZON_SPIKES spans keep the thread's CPU time beside their wall time, and the slow-frame line gives the render thread's CPU time in the frame. A span that mostly waited (for the GPU, a lock or a core) prints both (`submit 30.2/0.1`), so a frame that waited reads apart from one that worked. Found the cause of #678's long frames: the render thread losing its core in queue.submit. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-client/src/game.rs | 21 +++-- crates/horizon-client/src/game/levels.rs | 7 +- crates/horizon-client/src/spans.rs | 100 ++++++++++++++++++++--- docs/graphics.md | 2 +- 4 files changed, 103 insertions(+), 27 deletions(-) diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index 2df6aa9b0..518fba2db 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -10933,6 +10933,10 @@ impl ApplicationHandler for Game<'_> { WindowEvent::RedrawRequested => { self.observe_fullscreen(); let now = Instant::now(); + let frame_cpu = self + .options + .spikes + .and_then(|_| horizon_client::spans::thread_cpu()); let elapsed = now.duration_since(self.previous); self.frame_ticks = elapsed.as_secs_f32() * 20.0; self.previous = now; @@ -10999,15 +11003,16 @@ impl ApplicationHandler for Game<'_> { let cpu = now.elapsed().saturating_sub(r.last_wait).as_secs_f32() * 1000.0; let spans = horizon_client::spans::take(); if cpu > threshold { - let list: Vec = spans - .iter() - .filter(|(_, ms)| *ms >= 0.5) - .map(|(n, ms)| format!("{n} {ms:.1}")) - .collect(); + // The render thread's own CPU time in the frame: + // far below the wall time, the frame waited. + let worked = horizon_client::spans::thread_cpu() + .zip(frame_cpu) + .map_or_else(String::new, |(end, start)| { + format!(", cpu {:.1} ms", (end - start).as_secs_f32() * 1000.0) + }); log::info!( - "Horizon: slow frame {cpu:.1} ms ({} ticks): {}", - steps, - list.join(", ") + "Horizon: slow frame {cpu:.1} ms{worked} ({steps} ticks): {}", + horizon_client::spans::describe(&spans) ); } } diff --git a/crates/horizon-client/src/game/levels.rs b/crates/horizon-client/src/game/levels.rs index 31b34a9d0..7df5541b4 100644 --- a/crates/horizon-client/src/game/levels.rs +++ b/crates/horizon-client/src/game/levels.rs @@ -312,15 +312,10 @@ impl Game<'_> { .map_or_else(String::new, |(end, start)| { format!(", cpu {:.1} ms", (end - start).as_secs_f32() * 1000.0) }); - let list: Vec = spans - .iter() - .filter(|(_, ms)| *ms >= 0.5) - .map(|(n, ms)| format!("{n} {ms:.1}")) - .collect(); log::info!( "Horizon: slow tick {wall:.1} ms{cpu} ({} mobs): {}", self.mobs.len(), - list.join(", ") + horizon_client::spans::describe(&spans) ); } result diff --git a/crates/horizon-client/src/spans.rs b/crates/horizon-client/src/spans.rs index 5c37f2bfc..ce69be8ce 100644 --- a/crates/horizon-client/src/spans.rs +++ b/crates/horizon-client/src/spans.rs @@ -1,8 +1,10 @@ //! Named timings within a frame, for finding what makes a frame slow: //! `HORIZON_SPIKES=MS` prints the spans of every frame whose CPU time //! exceeds MS milliseconds, and of every server tick (dedicated or -//! integrated) longer than MS. Costs one clock read per span when enabled -//! and a relaxed load otherwise. +//! integrated) longer than MS. Each span keeps its wall time and the +//! thread's CPU time inside it, so a span that waited (for a lock, the GPU +//! or a core on a busy machine) shows apart from one that worked. Costs two +//! clock reads per span when enabled and a relaxed load otherwise. use std::{ cell::RefCell, sync::atomic::{AtomicBool, Ordering}, @@ -11,7 +13,15 @@ use std::{ static ENABLED: AtomicBool = AtomicBool::new(false); thread_local! { - static SPANS: RefCell> = const { RefCell::new(Vec::new()) }; + static SPANS: RefCell> = const { RefCell::new(Vec::new()) }; +} + +/// A named timing: wall milliseconds and the thread's CPU milliseconds in it. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Span { + pub name: &'static str, + pub ms: f32, + pub cpu_ms: f32, } pub fn enable(on: bool) { @@ -23,10 +33,13 @@ pub fn time(name: &'static str, f: impl FnOnce() -> T) -> T { if !ENABLED.load(Ordering::Relaxed) { return f(); } - let started = Instant::now(); + let (started, cpu) = (Instant::now(), thread_cpu()); let out = f(); let ms = started.elapsed().as_secs_f32() * 1000.0; - SPANS.with(|s| s.borrow_mut().push((name, ms))); + let cpu_ms = thread_cpu() + .zip(cpu) + .map_or(ms, |(end, start)| (end - start).as_secs_f32() * 1000.0); + SPANS.with(|s| s.borrow_mut().push(Span { name, ms, cpu_ms })); out } /// The CPU time this thread has used, where the platform tells it: a tick @@ -42,21 +55,84 @@ pub fn thread_cpu() -> Option { None } /// The frame's spans, summed by name, slowest first; clears them. -pub fn take() -> Vec<(&'static str, f32)> { +pub fn take() -> Vec { let spans = SPANS.with(|s| std::mem::take(&mut *s.borrow_mut())); - let mut sums: Vec<(&'static str, f32)> = Vec::new(); - for (name, ms) in spans { - match sums.iter_mut().find(|(n, _)| *n == name) { - Some(entry) => entry.1 += ms, - None => sums.push((name, ms)), + let mut sums: Vec = Vec::new(); + for span in spans { + match sums.iter_mut().find(|s| s.name == span.name) { + Some(sum) => { + sum.ms += span.ms; + sum.cpu_ms += span.cpu_ms; + } + None => sums.push(span), } } - sums.sort_by(|a, b| b.1.total_cmp(&a.1)); + sums.sort_by(|a, b| b.ms.total_cmp(&a.ms)); sums } +/// `name wall` for spans of at least 0.5 ms, with `/cpu` when the thread +/// worked for less than nine tenths of it: the slow-frame and slow-tick lines. +pub fn describe(spans: &[Span]) -> String { + let list: Vec = spans + .iter() + .filter(|s| s.ms >= 0.5) + .map(|s| { + if s.cpu_ms < s.ms * 0.9 { + format!("{} {:.1}/{:.1}", s.name, s.ms, s.cpu_ms) + } else { + format!("{} {:.1}", s.name, s.ms) + } + }) + .collect(); + list.join(", ") +} #[cfg(test)] mod tests { + use super::Span; + + /// Spans of one name add up; a span the thread spent mostly waiting + /// shows its CPU time after the wall time, short ones are left out. + #[test] + fn spans_sum_by_name_and_show_waits() { + super::SPANS.with(|s| { + s.borrow_mut().extend([ + Span { + name: "tick", + ms: 2.0, + cpu_ms: 2.0, + }, + Span { + name: "submit", + ms: 30.0, + cpu_ms: 0.2, + }, + Span { + name: "tick", + ms: 3.0, + cpu_ms: 2.9, + }, + Span { + name: "hud", + ms: 0.3, + cpu_ms: 0.3, + }, + ]); + }); + let spans = super::take(); + assert_eq!( + spans[0], + Span { + name: "submit", + ms: 30.0, + cpu_ms: 0.2 + } + ); + assert_eq!((spans[1].name, spans[1].ms), ("tick", 5.0)); + assert_eq!(super::describe(&spans), "submit 30.0/0.2, tick 5.0"); + assert!(super::take().is_empty()); + } + /// The thread's CPU clock moves with work done on it, not with time /// spent asleep. #[cfg(unix)] diff --git a/docs/graphics.md b/docs/graphics.md index b1c3c0209..34032a641 100644 --- a/docs/graphics.md +++ b/docs/graphics.md @@ -64,7 +64,7 @@ python3 tools/graphics_bench.py --compare baseline-4080 mine-gpu # markdown ta vscreen is a headless compositor whose frames are software-rendered (lavapipe), so its frame times measure the CPU, rasterisation included. `--offscreen` (`horizon-client --offscreen`) renders into a texture on the most powerful GPU instead of the window, which measures the RTX 4080 under the same compositor. `--distances`, `--scenes` and `--build` pick a subset or another build. Results go to `benchmarks/graphics/