Slow-frame and slow-tick lines show the CPU time of each span
HORIZON_SPIKES spans keep the thread's CPU time beside their wall time, and the slow-frame line gives the render thread's CPU time in the frame. A span that mostly waited (for the GPU, a lock or a core) prints both (`submit 30.2/0.1`), so a frame that waited reads apart from one that worked. Found the cause of #678's long frames: the render thread losing its core in queue.submit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10933,6 +10933,10 @@ impl ApplicationHandler for Game<'_> {
|
||||
WindowEvent::RedrawRequested => {
|
||||
self.observe_fullscreen();
|
||||
let now = Instant::now();
|
||||
let frame_cpu = self
|
||||
.options
|
||||
.spikes
|
||||
.and_then(|_| horizon_client::spans::thread_cpu());
|
||||
let elapsed = now.duration_since(self.previous);
|
||||
self.frame_ticks = elapsed.as_secs_f32() * 20.0;
|
||||
self.previous = now;
|
||||
@@ -10999,15 +11003,16 @@ impl ApplicationHandler for Game<'_> {
|
||||
let cpu = now.elapsed().saturating_sub(r.last_wait).as_secs_f32() * 1000.0;
|
||||
let spans = horizon_client::spans::take();
|
||||
if cpu > threshold {
|
||||
let list: Vec<String> = spans
|
||||
.iter()
|
||||
.filter(|(_, ms)| *ms >= 0.5)
|
||||
.map(|(n, ms)| format!("{n} {ms:.1}"))
|
||||
.collect();
|
||||
// The render thread's own CPU time in the frame:
|
||||
// far below the wall time, the frame waited.
|
||||
let worked = horizon_client::spans::thread_cpu()
|
||||
.zip(frame_cpu)
|
||||
.map_or_else(String::new, |(end, start)| {
|
||||
format!(", cpu {:.1} ms", (end - start).as_secs_f32() * 1000.0)
|
||||
});
|
||||
log::info!(
|
||||
"Horizon: slow frame {cpu:.1} ms ({} ticks): {}",
|
||||
steps,
|
||||
list.join(", ")
|
||||
"Horizon: slow frame {cpu:.1} ms{worked} ({steps} ticks): {}",
|
||||
horizon_client::spans::describe(&spans)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -312,15 +312,10 @@ impl Game<'_> {
|
||||
.map_or_else(String::new, |(end, start)| {
|
||||
format!(", cpu {:.1} ms", (end - start).as_secs_f32() * 1000.0)
|
||||
});
|
||||
let list: Vec<String> = spans
|
||||
.iter()
|
||||
.filter(|(_, ms)| *ms >= 0.5)
|
||||
.map(|(n, ms)| format!("{n} {ms:.1}"))
|
||||
.collect();
|
||||
log::info!(
|
||||
"Horizon: slow tick {wall:.1} ms{cpu} ({} mobs): {}",
|
||||
self.mobs.len(),
|
||||
list.join(", ")
|
||||
horizon_client::spans::describe(&spans)
|
||||
);
|
||||
}
|
||||
result
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
//! Named timings within a frame, for finding what makes a frame slow:
|
||||
//! `HORIZON_SPIKES=MS` prints the spans of every frame whose CPU time
|
||||
//! exceeds MS milliseconds, and of every server tick (dedicated or
|
||||
//! integrated) longer than MS. Costs one clock read per span when enabled
|
||||
//! and a relaxed load otherwise.
|
||||
//! integrated) longer than MS. Each span keeps its wall time and the
|
||||
//! thread's CPU time inside it, so a span that waited (for a lock, the GPU
|
||||
//! or a core on a busy machine) shows apart from one that worked. Costs two
|
||||
//! clock reads per span when enabled and a relaxed load otherwise.
|
||||
use std::{
|
||||
cell::RefCell,
|
||||
sync::atomic::{AtomicBool, Ordering},
|
||||
@@ -11,7 +13,15 @@ use std::{
|
||||
|
||||
static ENABLED: AtomicBool = AtomicBool::new(false);
|
||||
thread_local! {
|
||||
static SPANS: RefCell<Vec<(&'static str, f32)>> = const { RefCell::new(Vec::new()) };
|
||||
static SPANS: RefCell<Vec<Span>> = const { RefCell::new(Vec::new()) };
|
||||
}
|
||||
|
||||
/// A named timing: wall milliseconds and the thread's CPU milliseconds in it.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Span {
|
||||
pub name: &'static str,
|
||||
pub ms: f32,
|
||||
pub cpu_ms: f32,
|
||||
}
|
||||
|
||||
pub fn enable(on: bool) {
|
||||
@@ -23,10 +33,13 @@ pub fn time<T>(name: &'static str, f: impl FnOnce() -> T) -> T {
|
||||
if !ENABLED.load(Ordering::Relaxed) {
|
||||
return f();
|
||||
}
|
||||
let started = Instant::now();
|
||||
let (started, cpu) = (Instant::now(), thread_cpu());
|
||||
let out = f();
|
||||
let ms = started.elapsed().as_secs_f32() * 1000.0;
|
||||
SPANS.with(|s| s.borrow_mut().push((name, ms)));
|
||||
let cpu_ms = thread_cpu()
|
||||
.zip(cpu)
|
||||
.map_or(ms, |(end, start)| (end - start).as_secs_f32() * 1000.0);
|
||||
SPANS.with(|s| s.borrow_mut().push(Span { name, ms, cpu_ms }));
|
||||
out
|
||||
}
|
||||
/// The CPU time this thread has used, where the platform tells it: a tick
|
||||
@@ -42,21 +55,84 @@ pub fn thread_cpu() -> Option<Duration> {
|
||||
None
|
||||
}
|
||||
/// The frame's spans, summed by name, slowest first; clears them.
|
||||
pub fn take() -> Vec<(&'static str, f32)> {
|
||||
pub fn take() -> Vec<Span> {
|
||||
let spans = SPANS.with(|s| std::mem::take(&mut *s.borrow_mut()));
|
||||
let mut sums: Vec<(&'static str, f32)> = Vec::new();
|
||||
for (name, ms) in spans {
|
||||
match sums.iter_mut().find(|(n, _)| *n == name) {
|
||||
Some(entry) => entry.1 += ms,
|
||||
None => sums.push((name, ms)),
|
||||
let mut sums: Vec<Span> = Vec::new();
|
||||
for span in spans {
|
||||
match sums.iter_mut().find(|s| s.name == span.name) {
|
||||
Some(sum) => {
|
||||
sum.ms += span.ms;
|
||||
sum.cpu_ms += span.cpu_ms;
|
||||
}
|
||||
None => sums.push(span),
|
||||
}
|
||||
}
|
||||
sums.sort_by(|a, b| b.1.total_cmp(&a.1));
|
||||
sums.sort_by(|a, b| b.ms.total_cmp(&a.ms));
|
||||
sums
|
||||
}
|
||||
/// `name wall` for spans of at least 0.5 ms, with `/cpu` when the thread
|
||||
/// worked for less than nine tenths of it: the slow-frame and slow-tick lines.
|
||||
pub fn describe(spans: &[Span]) -> String {
|
||||
let list: Vec<String> = spans
|
||||
.iter()
|
||||
.filter(|s| s.ms >= 0.5)
|
||||
.map(|s| {
|
||||
if s.cpu_ms < s.ms * 0.9 {
|
||||
format!("{} {:.1}/{:.1}", s.name, s.ms, s.cpu_ms)
|
||||
} else {
|
||||
format!("{} {:.1}", s.name, s.ms)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
list.join(", ")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::Span;
|
||||
|
||||
/// Spans of one name add up; a span the thread spent mostly waiting
|
||||
/// shows its CPU time after the wall time, short ones are left out.
|
||||
#[test]
|
||||
fn spans_sum_by_name_and_show_waits() {
|
||||
super::SPANS.with(|s| {
|
||||
s.borrow_mut().extend([
|
||||
Span {
|
||||
name: "tick",
|
||||
ms: 2.0,
|
||||
cpu_ms: 2.0,
|
||||
},
|
||||
Span {
|
||||
name: "submit",
|
||||
ms: 30.0,
|
||||
cpu_ms: 0.2,
|
||||
},
|
||||
Span {
|
||||
name: "tick",
|
||||
ms: 3.0,
|
||||
cpu_ms: 2.9,
|
||||
},
|
||||
Span {
|
||||
name: "hud",
|
||||
ms: 0.3,
|
||||
cpu_ms: 0.3,
|
||||
},
|
||||
]);
|
||||
});
|
||||
let spans = super::take();
|
||||
assert_eq!(
|
||||
spans[0],
|
||||
Span {
|
||||
name: "submit",
|
||||
ms: 30.0,
|
||||
cpu_ms: 0.2
|
||||
}
|
||||
);
|
||||
assert_eq!((spans[1].name, spans[1].ms), ("tick", 5.0));
|
||||
assert_eq!(super::describe(&spans), "submit 30.0/0.2, tick 5.0");
|
||||
assert!(super::take().is_empty());
|
||||
}
|
||||
|
||||
/// The thread's CPU clock moves with work done on it, not with time
|
||||
/// spent asleep.
|
||||
#[cfg(unix)]
|
||||
|
||||
+1
-1
@@ -64,7 +64,7 @@ python3 tools/graphics_bench.py --compare baseline-4080 mine-gpu # markdown ta
|
||||
|
||||
vscreen is a headless compositor whose frames are software-rendered (lavapipe), so its frame times measure the CPU, rasterisation included. `--offscreen` (`horizon-client --offscreen`) renders into a texture on the most powerful GPU instead of the window, which measures the RTX 4080 under the same compositor. `--distances`, `--scenes` and `--build` pick a subset or another build. Results go to `benchmarks/graphics/<label>.json`, with the load average before and after: the desktop is shared, and other sessions' builds move the numbers.
|
||||
|
||||
Two tools find where a slow frame goes: `HORIZON_SPIKES=MS` prints the timed spans (ticks by system, streaming, uploads, saving, renderer) of every frame slower than MS milliseconds, and a frame-pointer build (`RUSTFLAGS="-C force-frame-pointers=yes"`) profiles with `perf record --call-graph fp`.
|
||||
Two tools find where a slow frame goes: `HORIZON_SPIKES=MS` prints the timed spans (ticks by system, streaming, uploads, saving, renderer) of every frame slower than MS milliseconds, with the render thread's CPU time in the frame and, after a span that mostly waited (for the GPU, a lock or a core), its CPU time (`submit 30.2/0.1`), and a frame-pointer build (`RUSTFLAGS="-C force-frame-pointers=yes"`) profiles with `perf record --call-graph fp`.
|
||||
|
||||
## Results
|
||||
|
||||
|
||||
Reference in New Issue
Block a user