Merge main into feat/mod-minimap-api

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-26 08:55:02 +01:00
17 changed files with 559 additions and 177 deletions
+4 -4
View File
@@ -23,10 +23,10 @@ env:
jobs:
parity-main:
# The Mac mini's release runner (label added 2026-10-26), not the two `horizon-game` runners: behind PR parity jobs
# this run waited 2 h while every merge replaced the pending one, and releases wait for it. Releases need it green
# first, so the two never compete. Not the desktop: a job slowed down because the maintainer came back could time
# out, and a failure here bisects and comments on merges.
# A Mac mini runner of its own (ludovics-mac-mini-2 holds only this label since 2026-10-26): behind PR parity jobs
# this run waited 2 h while every merge replaced the pending one, and on the release runner a release then waited
# behind the next run. PR parity jobs have ludovics-mac-mini and the desktop. Not the desktop: a job slowed down
# because the maintainer came back could time out, and a failure here bisects and comments on merges.
runs-on: [self-hosted, macOS, horizon-parity-main]
timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures
steps:
+8 -1
View File
@@ -52,7 +52,14 @@ jobs:
- name: The prepared output is the one the preparer's OUTPUT_VERSION says
env:
BASE_SHA: ${{ inputs.base }}
run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA"
run: |
# The job runs main's workflow on the head's files: a head made before OUTPUT_VERSION (#1947) has neither
# the tool nor the number, so there is nothing to check.
if [ ! -f tools/prepare_output.py ] || ! grep -q OUTPUT_VERSION crates/horizon-prepare/src/lib.rs; then
echo "This head's preparer has no OUTPUT_VERSION: nothing to check"
exit 0
fi
python3 tools/prepare_output.py check --cache --base "$BASE_SHA"
- name: The version constants are the pinned jar's
run: python3 tools/gen_version_consts.py --check
- name: The harnesses this change can affect
@@ -84,6 +84,33 @@ impl Displayer {
}
}
/// Horizon's own lines: the shader pack's GPU time, then its costliest passes (`passes`: each
/// pass's median milliseconds, in the order they run).
pub fn shader_pass_lines(passes: &[(String, f32)]) -> Vec<String> {
const SHOWN: usize = 8;
if passes.is_empty() {
return Vec::new();
}
let total: f32 = passes.iter().map(|p| p.1).sum();
let mut lines = vec![format!(
"Shader pack GPU: {total:.2} ms in {} passes",
passes.len()
)];
let mut costliest: Vec<&(String, f32)> = passes.iter().collect();
costliest.sort_by(|a, b| b.1.total_cmp(&a.1));
lines.extend(
costliest
.iter()
.take(SHOWN)
.map(|(name, ms)| format!("{name}: {ms:.2} ms")),
);
if costliest.len() > SHOWN {
let rest: f32 = costliest[SHOWN..].iter().map(|p| p.1).sum();
lines.push(format!("{} more: {rest:.2} ms", costliest.len() - SHOWN));
}
lines
}
/// `Mth.wrapDegrees`.
fn wrap_degrees(angle: f32) -> f32 {
let wrapped = angle % 360.0;
@@ -337,4 +364,15 @@ mod tests {
(1.0, [640, 360])
);
}
#[test]
fn shader_pass_lines_list_the_costliest_passes_first() {
assert!(shader_pass_lines(&[]).is_empty());
let passes: Vec<(String, f32)> = (0..10).map(|i| (format!("p{i}"), i as f32)).collect();
let lines = shader_pass_lines(&passes);
assert_eq!(lines[0], "Shader pack GPU: 45.00 ms in 10 passes");
assert_eq!(lines[1], "p9: 9.00 ms");
assert_eq!(lines[8], "p2: 2.00 ms");
assert_eq!(lines[9], "2 more: 1.00 ms");
}
}
+10 -3
View File
@@ -2203,7 +2203,7 @@ impl Game<'_> {
self.player_drop(thrown);
}
/// `Player.drop`: the statistics and a thrown item entity in front of the player.
fn player_drop(&mut self, thrown: ItemStack) {
pub(super) fn player_drop(&mut self, thrown: ItemStack) {
let Some(eye) = self.eye_position() else {
return;
};
@@ -6512,7 +6512,11 @@ impl Game<'_> {
use horizon_client::menu::Input;
let button = i32::from(right);
if self.remote.is_some() {
self.remote_throw(right, input);
if self.screen == Screen::Creative {
self.remote_creative_drop(right);
} else {
self.remote_throw(right, input);
}
return;
}
let before = self.carried.clone();
@@ -10496,7 +10500,10 @@ impl Game<'_> {
menu_blur: self.menu_blur(),
};
let result = match self.renderer.as_mut() {
Some(renderer) => horizon_client::spans::time("renderer", || renderer.render(&frame)),
Some(renderer) => {
renderer.set_pass_timing(self.debug);
horizon_client::spans::time("renderer", || renderer.render(&frame))
}
None => Err("renderer".into()),
};
// Restore storage even when rendering fails (for example, a lost surface).
@@ -415,6 +415,13 @@ impl Game<'_> {
],
);
}
// Horizon's own: the shader pack's GPU time per pass (an empty group would move others).
let pass_lines = self.renderer.as_ref().map_or_else(Vec::new, |r| {
debug_overlay::shader_pass_lines(&r.shader_pass_times())
});
if !pass_lines.is_empty() {
d.group("shader pack", pass_lines);
}
// tps: DebugEntryTps for local and remote connections.
if self.player.is_some() {
let [tx, rx] = self.debug_charts.packet_rates.map(|n| n.round() as i64);
@@ -38,6 +38,9 @@ pub(super) struct GuestPlay {
use_delay: u32,
/// The shield's ItemCooldowns entry: an axe disabled it for this many ticks.
pub(super) shield_cooldown: u32,
/// `ServerGamePacketListenerImpl.dropSpamThrottler` (`TickThrottler(20, 1480)`): 20 for each
/// item a creative player drops from the creative screen, one less each tick.
creative_drops: u32,
/// The guest's `Player.getCooldowns()` (`Game::cooldowns` for its turn).
pub(super) cooldowns: horizon_client::cooldowns::Cooldowns,
attack_ticks: u32,
@@ -1086,6 +1089,35 @@ impl Game<'_> {
}
S::MoveVehicle(p) => self.guest_move_vehicle(host, conn, &p),
S::PaddleBoat(p) => self.guest_paddle_boat(host, conn, [p.left, p.right]),
// handleSetCreativeModeSlot's drop: a slot below 0 is a stack thrown out of the
// creative screen (`handleCreativeModeItemDrop`), dropped from the player's hand.
S::SetCreativeModeSlot(p) if p.slot < 0 => {
if !host.hub.guests[gi].creative() {
return;
}
let Some(stack) = self.item_from_network(host, &p.item) else {
return;
};
if stack.count
> self
.res
.items()
.get(&stack.item)
.map_or(64, |i| i.max_stack)
{
return;
}
let play = host.play.entry(conn).or_default();
if play.creative_drops >= 1480 {
eprintln!(
"Player {} was dropping items too fast in creative mode, ignoring.",
host.hub.guests[gi].name
);
return;
}
play.creative_drops += 20;
self.as_guest(host, conn, |g| g.player_drop(stack));
}
S::SetCreativeModeSlot(p) => {
let g = &mut host.hub.guests[gi];
if !g.creative() || !(0..46).contains(&p.slot) {
@@ -1805,6 +1837,7 @@ impl Game<'_> {
// Player.tick's ItemCooldowns.tick, once a tick for each player
// (the shield an axe disabled, a thrown pearl).
play.shield_cooldown = play.shield_cooldown.saturating_sub(1);
play.creative_drops = play.creative_drops.saturating_sub(1);
// ServerItemCooldowns.onCooldownEnded: a finished entry is told to
// the guest (a duration of 0 removes it).
for group in play.cooldowns.tick() {
+28
View File
@@ -4828,6 +4828,34 @@ impl Game<'_> {
carried,
}));
}
/// `CreativeModeInventoryScreen.slotClicked` outside the window: the server knows nothing of
/// a stack picked from the creative screen, so the carried stack (an item tab's right click:
/// one of it) is dropped through `handleCreativeModeItemDrop`, a SetCreativeModeSlot to slot
/// -1, and the hand swings as `Player.drop` predicts.
pub(super) fn remote_creative_drop(&mut self, right: bool) {
let one = right && !self.creative_inventory_open();
let Some(carried) = self.carried.as_mut() else {
return;
};
let thrown = if one && carried.count > 1 {
carried.count -= 1;
ItemStack {
count: 1,
..carried.clone()
}
} else {
self.carried.take().expect("carried")
};
if !self.creative {
return;
}
self.swing_arm();
let item = self.to_network(&Some(thrown));
self.send(S::SetCreativeModeSlot(play::SetCreativeModeSlot {
slot: -1,
item,
}));
}
pub(super) fn remote_close_container(&mut self) {
let id = self
.remote
+42
View File
@@ -502,6 +502,8 @@ pub struct Renderer {
pack_post: Option<(i32, Option<pack_post::PackPost>)>,
/// The post passes of one of Horizon's own shader packs; never beside `pack_post`'s.
native_post: Option<native_post::NativePost>,
/// Time the shader pack's passes on the GPU (F3, or `HORIZON_SHADER_PACK_TIMING`).
pass_timing: bool,
/// The entity vertices Horizon draws itself while a shader pack draws the others.
pack_entity_rest: Vec<Vertex>,
/// Set 1 of the cloud pipeline, for a shader pack's `gbuffers_clouds`.
@@ -2413,6 +2415,7 @@ impl Renderer {
world_post,
pack_post: None,
native_post: None,
pass_timing: pack_timing::PassTiming::logging(),
pack_entity_rest: Vec::new(),
cloud_bindings,
pack_weather: None,
@@ -2992,6 +2995,34 @@ impl Renderer {
.terrain_inputs(self.pack_terrain_flat),
)
}
/// Times the shader pack's passes on the GPU while `on` (the F3 overlay is open), or always
/// with `HORIZON_SHADER_PACK_TIMING`.
pub fn set_pass_timing(&mut self, on: bool) {
self.pass_timing = on || pack_timing::PassTiming::logging();
}
/// The running shader pack's timer, of either kind.
fn pass_timer<'a>(
pack: &'a mut Option<(i32, Option<pack_post::PackPost>)>,
native: &'a mut Option<native_post::NativePost>,
) -> Option<&'a mut Option<pack_timing::PassTiming>> {
match (pack, native) {
(Some((_, Some(post))), _) => Some(&mut post.timing),
(_, Some(post)) => Some(&mut post.timing),
_ => None,
}
}
/// Each pass of the running shader pack with its median GPU time in milliseconds, while
/// they are timed (`set_pass_timing`).
pub fn shader_pass_times(&self) -> Vec<(String, f32)> {
let timer = match (&self.pack_post, &self.native_post) {
(Some((_, Some(post))), _) => post.timing.as_ref(),
(_, Some(post)) => post.timing.as_ref(),
_ => None,
};
timer
.map(pack_timing::PassTiming::medians)
.unwrap_or_default()
}
/// The dimension the shader pack was last set for.
pub fn shader_pack_dimension(&self) -> Option<i32> {
self.pack_post.as_ref().map(|p| p.0)
@@ -4016,6 +4047,14 @@ impl Renderer {
let pack_post = !frame.title && self.pack_post.as_ref().is_some_and(|p| p.1.is_some());
// One of Horizon's own packs reads the same world target.
let native_post = !frame.title && self.native_post.is_some();
if let Some(timer) = Self::pass_timer(&mut self.pack_post, &mut self.native_post)
&& timer.is_some() != self.pass_timing
{
*timer = self
.pass_timing
.then(|| pack_timing::PassTiming::new(&self.device, &self.queue))
.flatten();
}
let pack_reads_world = pack_post || native_post;
// The pack's program reads the vertices this frame's draws bind.
let pack_draws_terrain = pack_terrain && flat_frame == self.pack_terrain_flat;
@@ -4649,6 +4688,9 @@ impl Renderer {
timer.submitted(&self.queue);
timer.collect(&self.device, &mut self.gpu_times);
}
if let Some(Some(timer)) = Self::pass_timer(&mut self.pack_post, &mut self.native_post) {
timer.submitted(&self.queue);
}
if let Some((path, buffer)) = readback {
self.screenshot_result = Some(
self.write_screenshot(&path, &buffer)
@@ -27,6 +27,10 @@ pub struct NativePost {
groups: Option<(WorldViews, Vec<wgpu::BindGroup>)>,
start: std::time::Instant,
frames: u32,
/// The screen's colour format, the world target's too.
format: wgpu::TextureFormat,
/// GPU time of each pass, while the renderer asks for it (`pack_timing.rs`).
pub timing: Option<super::pack_timing::PassTiming>,
}
fn texture_entry(binding: u32, sample_type: wgpu::TextureSampleType) -> wgpu::BindGroupLayoutEntry {
@@ -165,6 +169,8 @@ impl NativePost {
groups: None,
start: std::time::Instant::now(),
frames: 0,
format,
timing: None,
})
}
@@ -287,6 +293,10 @@ impl NativePost {
) {
self.prepare(device, world, size);
queue.write_buffer(&self.frame, 0, bytemuck::bytes_of(inputs));
if let Some(t) = &mut self.timing {
t.begin(device, queue);
t.mark(device, encoder, (world.0, self.format));
}
let Some((_, groups)) = &self.groups else {
return;
};
@@ -307,6 +317,7 @@ impl NativePost {
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some(label),
color_attachments: &attachments,
timestamp_writes: self.timing.as_mut().and_then(|t| t.writes(label)),
..Default::default()
});
pass.set_pipeline(pipeline);
+3 -3
View File
@@ -297,8 +297,8 @@ pub struct PackPost {
/// with the page buffer it was made for.
block_ids: Option<(wgpu::BindGroupLayout, wgpu::Buffer)>,
page_groups: HashMap<u32, (wgpu::Buffer, wgpu::BindGroup)>,
/// `HORIZON_SHADER_PACK_TIMING`: GPU time of each pass, in the log.
timing: Option<super::pack_timing::PassTiming>,
/// GPU time of each pass, while the renderer asks for it (`pack_timing.rs`).
pub timing: Option<super::pack_timing::PassTiming>,
/// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`.
custom: horizon_shaders::Custom,
atlas: Option<wgpu::TextureView>,
@@ -1430,7 +1430,7 @@ impl PackPost {
draws,
block_ids,
page_groups: HashMap::new(),
timing: super::pack_timing::PassTiming::new(device, queue),
timing: None,
custom,
atlas: world.atlas.as_ref().map(|a| a.0.clone()),
lightmap: world.lightmap,
+222 -68
View File
@@ -1,31 +1,53 @@
//! Developers' aid: `HORIZON_SHADER_PACK_TIMING=1` times every pass of a shader pack on the
//! GPU, one frame in `EVERY`, and logs each pass's median every `REPORT` timed frames. The
//! frame after a timed one waits for the GPU before reading the timestamps (Metal resolves
//! them without waiting for the passes that write them), so timing slows the game.
//! GPU time of each pass of a shader pack, of either kind: shown in the F3 overlay while it is
//! open, and logged every `REPORT` timed frames with `HORIZON_SHADER_PACK_TIMING=1`. One frame
//! at a time is timed, and the CPU never waits for it: its timestamps are resolved in a later
//! frame, once the GPU has finished it (Metal resolves them without waiting for the passes that
//! write them), then read back when mapped. The next frame after that is timed again.
use std::collections::VecDeque;
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
const EVERY: u32 = 30;
const REPORT: usize = 30;
/// Passes timed per frame at most (two timestamps each).
const CAPACITY: u32 = 64;
/// Timed frames each pass's median is taken over.
const WINDOW: usize = 15;
pub(super) struct PassTiming {
enum Stage {
/// The next frame is timed.
Idle,
/// The frame being recorded is timed.
Recording,
/// Submitted; the flag is set once the GPU has finished it.
Running(Arc<AtomicBool>),
/// Resolved and copied; the flag is set once the copy is mapped.
Reading(Arc<AtomicBool>, u64),
}
pub struct PassTiming {
queries: wgpu::QuerySet,
resolve: wgpu::Buffer,
read: wgpu::Buffer,
period_ns: f32,
frame: u32,
/// The passes timed in the frame being recorded or waiting to be read, in order.
stage: Stage,
/// The passes of the timed frame, in order.
names: Vec<String>,
timed: bool,
samples: Vec<(String, Vec<f32>)>,
/// Each pass's last `WINDOW` times, in milliseconds.
samples: Vec<(String, VecDeque<f32>)>,
/// Log the medians every `REPORT` timed frames.
log: bool,
/// `mark`'s pipeline, for the target format it was made for.
marker: Option<(wgpu::TextureFormat, wgpu::RenderPipeline)>,
reports: usize,
}
impl PassTiming {
/// `None` when the device can't time passes.
pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Option<Self> {
if std::env::var_os("HORIZON_SHADER_PACK_TIMING").is_none()
|| !device.features().contains(wgpu::Features::TIMESTAMP_QUERY)
{
if !device.features().contains(wgpu::Features::TIMESTAMP_QUERY) {
return None;
}
let size = u64::from(CAPACITY) * 2 * 8;
@@ -52,51 +74,133 @@ impl PassTiming {
wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
),
period_ns: queue.get_timestamp_period(),
frame: 0,
stage: Stage::Idle,
names: Vec::new(),
timed: false,
samples: Vec::new(),
log: Self::logging(),
marker: None,
reports: 0,
})
}
/// At the start of a frame: reads the last timed frame's passes, and decides whether this
/// one is timed.
/// Whether `HORIZON_SHADER_PACK_TIMING` asks for timing without the F3 overlay.
pub fn logging() -> bool {
static LOG: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*LOG.get_or_init(|| std::env::var_os("HORIZON_SHADER_PACK_TIMING").is_some())
}
/// At the start of a frame's passes: moves the last timed frame along, and decides whether
/// this one is timed.
pub fn begin(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {
if self.timed && !self.names.is_empty() {
self.collect(device, queue);
match &self.stage {
// A timed frame that was never submitted: time this one instead.
Stage::Recording => self.stage = Stage::Idle,
Stage::Running(done) if done.load(Ordering::Acquire) => {
let bytes = self.names.len() as u64 * 16;
let mut encoder = device.create_command_encoder(&Default::default());
encoder.resolve_query_set(
&self.queries,
0..self.names.len() as u32 * 2,
&self.resolve,
0,
);
encoder.copy_buffer_to_buffer(&self.resolve, 0, &self.read, 0, bytes);
queue.submit([encoder.finish()]);
let mapped = Arc::new(AtomicBool::new(false));
let flag = mapped.clone();
self.read
.slice(..bytes)
.map_async(wgpu::MapMode::Read, move |r| {
flag.store(r.is_ok(), Ordering::Release)
});
self.stage = Stage::Reading(mapped, bytes);
}
Stage::Reading(mapped, bytes) if mapped.load(Ordering::Acquire) => {
let bytes = *bytes;
self.collect(bytes);
self.stage = Stage::Idle;
}
_ => {}
}
if matches!(self.stage, Stage::Idle) {
self.names.clear();
self.stage = Stage::Recording;
}
self.names.clear();
self.frame += 1;
self.timed = self.frame.is_multiple_of(EVERY);
}
/// Timestamp writes for the pass `name`, in a timed frame.
pub fn writes(&mut self, name: &str) -> Option<wgpu::RenderPassTimestampWrites<'_>> {
let i = self.push(name)?;
Some(stamps(&self.queries, i))
}
/// The index of the timed frame's next pass, `name`.
fn push(&mut self, name: &str) -> Option<u32> {
let i = self.names.len() as u32;
if !self.timed || i >= CAPACITY {
if !matches!(self.stage, Stage::Recording) || i >= CAPACITY {
return None;
}
self.names.push(name.to_owned());
Some(wgpu::RenderPassTimestampWrites {
query_set: &self.queries,
beginning_of_pass_write_index: Some(2 * i),
end_of_pass_write_index: Some(2 * i + 1),
})
Some(i)
}
fn collect(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {
// The timed frame first, then the resolve, so every timestamp is written.
_ = device.poll(wgpu::PollType::wait_indefinitely());
let count = self.names.len() as u32 * 2;
let bytes = u64::from(count) * 8;
let mut encoder = device.create_command_encoder(&Default::default());
encoder.resolve_query_set(&self.queries, 0..count, &self.resolve, 0);
encoder.copy_buffer_to_buffer(&self.resolve, 0, &self.read, 0, bytes);
queue.submit([encoder.finish()]);
/// A pass that ends once everything drawn into `target` before it has, in a timed frame:
/// the next pass counts from there, not from when the GPU queued it behind them.
pub fn mark(
&mut self,
device: &wgpu::Device,
encoder: &mut wgpu::CommandEncoder,
(target, format): super::timing::Target,
) {
if self.marker.as_ref().is_none_or(|(f, _)| *f != format) {
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("Shader pack timing mark"),
source: wgpu::ShaderSource::Wgsl(super::timing::MARKER.into()),
});
let pipeline = super::timing::end_pipeline(device, &module, format);
self.marker = Some((format, pipeline));
}
// An unnamed pass: its end counts, it is never listed.
let Some(i) = self.push("") else {
return;
};
let (_, pipeline) = self.marker.as_ref().expect("made above");
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("Shader pack timing mark"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
view: target,
depth_slice: None,
resolve_target: None,
ops: wgpu::Operations {
load: wgpu::LoadOp::Load,
store: wgpu::StoreOp::Store,
},
})],
timestamp_writes: Some(stamps(&self.queries, i)),
..Default::default()
});
pass.set_pipeline(pipeline);
pass.set_scissor_rect(0, 0, 1, 1);
pass.draw(0..3, 0..1);
}
/// After the frame is submitted.
pub fn submitted(&mut self, queue: &wgpu::Queue) {
if !matches!(self.stage, Stage::Recording) {
return;
}
if self.names.is_empty() {
self.stage = Stage::Idle;
return;
}
let done = Arc::new(AtomicBool::new(false));
let flag = done.clone();
queue.on_submitted_work_done(move || flag.store(true, Ordering::Release));
self.stage = Stage::Running(done);
}
fn collect(&mut self, bytes: u64) {
let slice = self.read.slice(..bytes);
slice.map_async(wgpu::MapMode::Read, |_| {});
_ = device.poll(wgpu::PollType::wait_indefinitely());
let Ok(data) = slice.get_mapped_range() else {
return;
};
@@ -108,41 +212,91 @@ impl PassTiming {
.collect();
drop(data);
self.read.unmap();
// Passes overlap on tiled GPUs (a pass begins before the one before it ends): a pass
// costs the time from the later of its beginning and the previous pass's end.
let mut previous_end = 0;
for (name, pair) in self.names.iter().zip(stamps.as_chunks::<2>().0) {
if pair[0] == 0 || pair[1] < pair[0] {
let times = pass_times(&stamps, self.period_ns);
for (name, ms) in self.names.iter().zip(times) {
let Some(ms) = ms.filter(|_| !name.is_empty()) else {
continue;
};
let i = match self.samples.iter().position(|s| s.0 == *name) {
Some(i) => i,
None => {
self.samples.push((name.clone(), VecDeque::new()));
self.samples.len() - 1
}
};
let window = &mut self.samples[i].1;
if window.len() == WINDOW {
window.pop_front();
}
let from = pair[0].max(previous_end).min(pair[1]);
previous_end = pair[1];
let ms = (pair[1] - from) as f32 * self.period_ns / 1e6;
match self.samples.iter_mut().find(|s| s.0 == *name) {
Some(s) => s.1.push(ms),
None => self.samples.push((name.clone(), vec![ms])),
}
window.push_back(ms);
}
self.reports += 1;
if self.reports.is_multiple_of(REPORT) {
self.report();
if self.log && self.reports.is_multiple_of(REPORT) {
let medians = self.medians();
let total: f32 = medians.iter().map(|m| m.1).sum();
let parts: Vec<String> = medians
.iter()
.map(|(n, ms)| format!("{n} {ms:.3}"))
.collect();
log::info!(
"shader pack GPU time per pass (median ms of the last {WINDOW} timed frames): {} = {total:.3}",
parts.join(", ")
);
}
}
fn report(&mut self) {
let mut total = 0.0;
let mut parts = Vec::new();
// A pass that did not run in this window (no entities, the sky out of sight) is left out.
for (name, v) in self.samples.iter_mut().filter(|s| !s.1.is_empty()) {
v.sort_by(f32::total_cmp);
let median = v[v.len() / 2];
total += median;
parts.push(format!("{name} {median:.3}"));
v.clear();
}
log::info!(
"shader pack GPU time per pass (median ms of {REPORT} frames): {} = {total:.3}",
parts.join(", ")
);
/// Each pass's median time in milliseconds over its last timed frames, in the order the
/// passes first ran.
pub fn medians(&self) -> Vec<(String, f32)> {
self.samples
.iter()
.filter(|s| !s.1.is_empty())
.map(|(name, v)| {
let mut v: Vec<f32> = v.iter().copied().collect();
v.sort_by(f32::total_cmp);
(name.clone(), v[v.len() / 2])
})
.collect()
}
}
/// Pass `i`'s beginning and end timestamps.
fn stamps(queries: &wgpu::QuerySet, i: u32) -> wgpu::RenderPassTimestampWrites<'_> {
wgpu::RenderPassTimestampWrites {
query_set: queries,
beginning_of_pass_write_index: Some(2 * i),
end_of_pass_write_index: Some(2 * i + 1),
}
}
/// Each pass's time in milliseconds from its (beginning, end) timestamps, `None` for a pass
/// the GPU left unstamped. Passes overlap on tiled GPUs (a pass begins before the one before it
/// ends): a pass costs the time from the later of its beginning and the previous pass's end.
fn pass_times(stamps: &[u64], period_ns: f32) -> Vec<Option<f32>> {
let mut previous_end = 0;
stamps
.as_chunks::<2>()
.0
.iter()
.map(|&[begin, end]| {
if begin == 0 || end < begin {
return None;
}
let from = begin.max(previous_end).min(end);
previous_end = end;
Some((end - from) as f32 * period_ns / 1e6)
})
.collect()
}
#[cfg(test)]
mod tests {
use super::pass_times;
#[test]
fn overlapping_passes_cost_from_the_previous_end() {
// Pass 2 begins before pass 1 ends; pass 3 was never stamped.
let stamps = [1_000_000, 3_000_000, 2_000_000, 5_000_000, 0, 0];
assert_eq!(pass_times(&stamps, 1.0), [Some(2.0), Some(2.0), None]);
}
}
+2 -2
View File
@@ -60,7 +60,7 @@ struct Markers {
/// The start's one thread; the end's triangle over the whole target (the
/// pass's scissor keeps one pixel), shading nothing it writes.
const MARKER: &str =
pub(super) const MARKER: &str =
"@vertex fn cover(@builtin(vertex_index) i: u32) -> @builtin(position) vec4<f32> {
return vec4<f32>(f32(i & 1u) * 4.0 - 1.0, f32(i >> 1u) * 4.0 - 1.0, 0.0, 1.0);
}
@@ -254,7 +254,7 @@ impl GpuTimer {
}
/// The end marker's pipeline for a `format` target, every write masked.
fn end_pipeline(
pub(super) fn end_pipeline(
device: &wgpu::Device,
module: &wgpu::ShaderModule,
format: wgpu::TextureFormat,
+1 -1
View File
@@ -53,7 +53,7 @@ A harness can declare, at the top level of its script:
### Which harnesses a change affects
`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `.local/parity-runner/reach-cache.json`. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses.
`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `reach-cache.json` of the shared parity folder (beside `passes/`), so a new worktree starts warm; a harness's search stops as soon as it reaches a changed file, and after `PARITY_REACH_BUDGET` seconds (default 180) the remaining harnesses count every source their probes link, so choosing never holds a slow machine for long. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses.
## The gate
+5 -5
View File
@@ -127,11 +127,11 @@ one keeps drawing), the launcher before it installs one, and the registry's CI b
6. Every entry point translated for Vulkan (SPIR-V), Metal and DirectX 12 (HLSL), with the options at their
defaults, minimums and maximums.
`[budget] gpu_ms` is the GPU time your pack aims to add to a frame; the game's overlay will show each pass's time
against it.
`[budget] gpu_ms` is the GPU time your pack aims to add to a frame. To compare, open F3 in the game: while it is
open, each pass is timed on the GPU and the overlay shows the pack's total and its costliest passes, by their `name`
(the median of the last 15 timed frames). Timing waits for nothing, so it does not slow the game.
## Next
The renderer hooks (shared with the Iris/OptiFine loader), then the post stage in the game with hot reload and the
GPU time per pass, Options > Shader packs listing these packs, then the material and sky stages, and the registry's
`shader` kind with installs from the launcher.
Options > Shader packs listing these packs, then the material and sky stages, and the registry's `shader` kind with
installs from the launcher.
+10
View File
@@ -101,6 +101,16 @@ class ReachTests(unittest.TestCase):
(self.root / "crates/a/src/unused.rs").unlink()
self.assertIn("crates/a/src/unused.rs", self.narrowed())
def test_a_search_told_to_stop_still_keeps_the_file_it_stopped_at(self):
graph = parity_reach.SourceGraph(self.root, parity.rust_code)
kept = parity_reach.narrow(self.dependencies, PROBE, graph, {"crates/a/src/used.rs"}, {"crates/a/src/used.rs"})
self.assertEqual(kept["crates/a/src/used.rs"], (True, set()))
self.assertFalse(graph.results, "a partial search is not remembered")
# A stop file outside the harness's sources doesn't end its search.
full = parity_reach.narrow(self.dependencies, PROBE, graph, (), {"crates/b/src/lib.rs"})
self.assertIn("crates/a/src/display.rs", full)
self.assertTrue(graph.results)
def test_unknown_probes_keep_everything(self):
self.assertEqual(set(self.narrowed(artifacts=[])), set(self.dependencies))
self.assertEqual(set(self.narrowed(artifacts=[["example", "missing", "release", ["-p", "a"], True]])),
+31 -10
View File
@@ -1245,14 +1245,20 @@ def affected_harnesses(found, records, changed, reach=True, base=None):
commit), only declarations of modules the harness doesn't reach changed (tools/parity_reach.py)."""
if changed & SHARED_INPUTS:
return set(found)
graph = None
graph, stop = None, frozenset()
# Parses are keyed by content, so every worktree sharing the vanilla data shares them.
cache_path = (SHARED if SHARED.is_dir() else STATE_DIR) / "reach-cache.json"
if reach and any(path.startswith("crates/") for path in changed):
cache_path = STATE_DIR / "reach-cache.json"
try:
cache = json.loads(cache_path.read_text(encoding="utf-8"))
except (OSError, ValueError):
cache = {}
graph = parity_reach.SourceGraph(ROOT, rust_code, cache)
# A changed file that differs in more than its `mod` declarations counts as soon as a harness reaches it,
# so the search for that harness can stop there.
stop = frozenset(path for path in changed if parity_reach.is_workspace_rust(path)
and parity_reach.declarations_changed(ROOT, path, (), base, True))
budget, started, over = float(os.environ.get("PARITY_REACH_BUDGET") or 180), time.monotonic(), False
affected = set()
for name, script in found.items():
record = records.get(name) or {}
@@ -1265,16 +1271,31 @@ def affected_harnesses(found, records, changed, reach=True, base=None):
affected.add(name)
continue
plain = {path.split(KEEP)[0] for path in dependencies}
if graph is not None and plain & changed:
narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed)
if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed(
ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()):
affected.add(name)
elif plain & changed:
hit = plain & changed
if not hit:
continue
if graph is None or any(not path.startswith("crates/") for path in hit):
affected.add(name) # no narrowing, or a file narrowing always keeps (tools/, Java oracles)
continue
if not over and time.monotonic() - started > budget:
over = True
print(f"note: choosing harnesses took over {budget:.0f} s (PARITY_REACH_BUDGET): the rest count every "
"source their probes link", flush=True)
if over:
affected.add(name)
continue
narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed, stop)
if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed(
ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()):
affected.add(name)
if graph is not None:
STATE_DIR.mkdir(parents=True, exist_ok=True)
cache_path.write_text(json.dumps(graph.saved_cache()), encoding="utf-8")
saved = graph.saved_cache()
if len(cache) + len(saved) < 4 * max(len(saved), 1):
saved = {**cache, **saved} # other worktrees' versions of the files too, within bounds
cache_path.parent.mkdir(parents=True, exist_ok=True)
temporary = cache_path.with_suffix(f".{os.getpid()}.tmp")
temporary.write_text(json.dumps(saved), encoding="utf-8")
temporary.replace(cache_path)
return affected
+104 -80
View File
@@ -25,8 +25,9 @@ assets, fonts) that only unreached sources name; a dependency that no longer exi
a change adds take part.
"""
import functools
import hashlib
from pathlib import Path, PurePosixPath
from pathlib import Path
import re
import subprocess
@@ -478,9 +479,10 @@ class SourceGraph:
seeds=seeds, node_sets={})
return self.tables
def reachable(self, entries, universe, reasons=None):
def reachable(self, entries, universe, reasons=None, stop=frozenset()):
"""The files of universe (workspace Rust sources) holding an item reachable from entries (by the rules in
this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging)."""
this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging). The
search ends early, with only part of the answer, once it reaches a file of stop."""
# Searching every source of the crates involved (not only those Cargo listed) only adds reach, and lets
# harnesses with the same entries (the game binary's) share one search.
crates = frozenset(crate_of(path) for path in universe)
@@ -488,11 +490,12 @@ class SourceGraph:
if reasons is None and key in self.results:
return self.results[key] & set(universe)
everything = {path for path in self.files if crate_of(path) in crates} | set(universe)
files = self.search(entries, everything, reasons)
self.results[key] = files
files, complete = self.search(entries, everything, reasons, stop)
if complete:
self.results[key] = files
return files & set(universe)
def search(self, entries, universe, reasons):
def search(self, entries, universe, reasons, stop=frozenset()):
targets = {self.target(path) for path in entries}
universe = {path for path in universe if (self.is_library(path) or self.target(path) in targets)
and path not in self.test_only}
@@ -600,71 +603,80 @@ class SourceGraph:
files.add(path)
if reasons is not None:
reasons[path] = why
if path in stop:
raise Stop
for path in universe:
if path.endswith("/src/lib.rs") or path in entries or path not in self.parent:
open_module(path, "crate root or entry")
for node in seeds:
reach(node, "entry" if node[0] in entries else "always counted")
for path in entries & universe:
files.add(path)
if reasons is not None:
reasons.setdefault(path, "entry")
while queue:
node = queue.pop()
path, index, member = node
item = item_of(node)
crate = crate_of(path)
label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"]
source = sets_of(node, item)
why = f"{path} ({label})"
# Plain mentions reach items of this file and of the modules above it.
ancestor, new = path, source["mentions"]
while ancestor is not None and new:
known = local.setdefault(ancestor, set())
new = new - known
known |= new
try:
for path in universe:
if path.endswith("/src/lib.rs") or path in entries or path not in self.parent:
open_module(path, "crate root or entry")
for node in seeds:
reach(node, "entry" if node[0] in entries else "always counted")
for path in entries & universe:
files.add(path)
if reasons is not None:
reasons.setdefault(path, "entry")
while queue:
node = queue.pop()
path, index, member = node
item = item_of(node)
crate = crate_of(path)
label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"]
source = sets_of(node, item)
why = f"{path} ({label})"
# Plain mentions reach items of this file and of the modules above it.
ancestor, new = path, source["mentions"]
while ancestor is not None and new:
known = local.setdefault(ancestor, set())
new = new - known
known |= new
for name in new:
wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}")
ancestor = self.parent.get(ancestor)
if item["kind"] == "use" and item["glob"]:
for name in item["names"]:
for target in sees.get(crate, {crate}):
for module in list(by_module.get((target, name), ())) + \
([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []):
if module not in globbed and module in universe:
globbed.add(module)
wake([(module, index, None) for index in range(len(self.parse(module)["items"]))]
if self.parse(module) else [], f"{why} glob-imports {name}")
visible_to = sees.get(crate, {crate})
new = source["paths"] - done["paths"].setdefault(crate, set())
done["paths"][crate] |= new
for name in new:
wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}")
ancestor = self.parent.get(ancestor)
if item["kind"] == "use" and item["glob"]:
for name in item["names"]:
for target in sees.get(crate, {crate}):
for module in list(by_module.get((target, name), ())) + \
([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []):
if module not in globbed and module in universe:
globbed.add(module)
wake([(module, index, None) for index in range(len(self.parse(module)["items"]))]
if self.parse(module) else [], f"{why} glob-imports {name}")
visible_to = sees.get(crate, {crate})
new = source["paths"] - done["paths"].setdefault(crate, set())
done["paths"][crate] |= new
for name in new:
for target in visible_to:
path_names.setdefault(target, set()).add(name)
for module in by_module.get((target, name), ()):
if module in universe and self.parent[module] in opened:
open_module(module, f"{why} names {name}")
for key, table, wanted in (("qualified", qualified, lambda other: True),
("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"),
("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)):
new = source[key] - done[key].setdefault(crate, set())
done[key][crate] |= new
for name in new:
table.setdefault(name, set()).add(crate)
wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)],
f"{why} {key} {name}")
if item["kind"] in TYPE_KINDS and member is None:
own = types.setdefault(crate, set())
local_types.setdefault(path, set()).update(names(node))
if not names(node) <= own:
own |= names(node)
visible_cache.clear()
seeing = seen_by.get(crate, {crate})
for name in names(node): # also when the crate has one by that name already: impls of this one
wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing],
f"implements for {name}, defined in {path}")
return files
for target in visible_to:
path_names.setdefault(target, set()).add(name)
for module in by_module.get((target, name), ()):
if module in universe and self.parent[module] in opened:
open_module(module, f"{why} names {name}")
for key, table, wanted in (("qualified", qualified, lambda other: True),
("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"),
("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)):
new = source[key] - done[key].setdefault(crate, set())
done[key][crate] |= new
for name in new:
table.setdefault(name, set()).add(crate)
wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)],
f"{why} {key} {name}")
if item["kind"] in TYPE_KINDS and member is None:
own = types.setdefault(crate, set())
local_types.setdefault(path, set()).update(names(node))
if not names(node) <= own:
own |= names(node)
visible_cache.clear()
seeing = seen_by.get(crate, {crate})
for name in names(node): # also when the crate has one by that name already: impls of this one
wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing],
f"implements for {name}, defined in {path}")
except Stop:
return files, False
return files, True
class Stop(Exception):
"""A search reached a file it was told to stop at."""
def impl_subject(header):
@@ -682,13 +694,19 @@ def impl_subject(header):
return parameters, re.split(r"\bwhere\b", header)[0]
_CRATES = {}
def crate_of(path):
return path.split("/", 2)[1]
"""The crate folder of a crates/<crate>/... path (called millions of times by a search: cached)."""
crate = _CRATES.get(path)
if crate is None:
crate = _CRATES[path] = path.split("/", 2)[1]
return crate
def is_workspace_rust(path):
parts = PurePosixPath(path).parts
return len(parts) > 2 and parts[0] == "crates" and path.endswith(".rs")
return path.startswith("crates/") and path.endswith(".rs") and path.count("/") >= 2
DECLARATION_LINE = re.compile(rf"\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+({IDENT})\s*;")
@@ -718,6 +736,11 @@ def declarations_changed(root, path, children, base, whole):
missing."""
if base is None:
return True
return _declarations_changed(str(root), path, frozenset(children), base, whole)
@functools.lru_cache(maxsize=None)
def _declarations_changed(root, path, children, base, whole):
try:
head = (Path(root) / path).read_text(encoding="utf-8", errors="replace")
before = subprocess.run(["git", "show", f"{base}:{path}"], cwd=root, capture_output=True, text=True, check=True).stdout
@@ -726,36 +749,37 @@ def declarations_changed(root, path, children, base, whole):
return declarations(head, children, whole) != declarations(before, children, whole)
def narrow(dependencies, artifacts, graph, changed=()):
def narrow(dependencies, artifacts, graph, changed=(), stop=frozenset()):
"""A harness's dependencies narrowed to what its entry points can reach: {dependency: (whole, children)}, where
whole says the file counts for its own content (else only through its `mod` declarations of children, as a
module ancestor of a reached file) and children are its reached child modules. Without known entry points (a
probe or binary the harness built that isn't found here), every dependency is kept whole."""
probe or binary the harness built that isn't found here), every dependency is kept whole. The search may end
as soon as it reaches a file of stop (changed files that count once reached): then only part is returned."""
plain = [dependency.split("#keep:")[0] for dependency in dependencies]
crates = {PurePosixPath(path).parts[1] for path in plain if path.startswith("crates/")}
crates = {crate_of(path) for path in plain if path.startswith("crates/") and path.count("/") >= 2}
rust = {path for path in plain if is_workspace_rust(path)}
added = {path for path in changed if is_workspace_rust(path) and PurePosixPath(path).parts[1] in crates
added = {path for path in changed if is_workspace_rust(path) and crate_of(path) in crates
and path not in rust and (graph.root / path).is_file()}
entries = graph.entries(artifacts, rust)
if entries is None:
return {dependency: (True, None) for dependency in dependencies}
missing = {path for path in rust if graph.parse(path) is None}
reached = graph.reachable(entries, rust | added | entries) | missing
reached = graph.reachable(entries, rust | added | entries, stop=frozenset(stop) & (rust | added)) | missing
named, named_in_crate = set(), {}
for path in rust | added:
parsed = graph.parse(path)
file_names = set(parsed["file_names"]) if parsed else set()
if path in reached:
named |= file_names
named_in_crate.setdefault(PurePosixPath(path).parts[1], set()).update(file_names)
named_in_crate.setdefault(crate_of(path), set()).update(file_names)
kept = {}
for dependency, path in zip(dependencies, plain):
if is_workspace_rust(path):
if path in reached:
kept[dependency] = (True, set())
elif not path.startswith("crates/") or path.endswith(("Cargo.toml", "build.rs")) \
or PurePosixPath(path).name in named \
or PurePosixPath(path).name not in named_in_crate.get(PurePosixPath(path).parts[1], ()):
or path.rsplit("/", 1)[-1] in named \
or path.rsplit("/", 1)[-1] not in named_in_crate.get(crate_of(path), ()):
kept[dependency] = (True, None)
for path in sorted(added & reached):
kept[path] = (True, set())