diff --git a/.github/workflows/parity-main.yml b/.github/workflows/parity-main.yml index 13be3c36d..c47005ab0 100644 --- a/.github/workflows/parity-main.yml +++ b/.github/workflows/parity-main.yml @@ -23,10 +23,10 @@ env: jobs: parity-main: - # The Mac mini's release runner (label added 2026-10-26), not the two `horizon-game` runners: behind PR parity jobs - # this run waited 2 h while every merge replaced the pending one, and releases wait for it. Releases need it green - # first, so the two never compete. Not the desktop: a job slowed down because the maintainer came back could time - # out, and a failure here bisects and comments on merges. + # A Mac mini runner of its own (ludovics-mac-mini-2 holds only this label since 2026-10-26): behind PR parity jobs + # this run waited 2 h while every merge replaced the pending one, and on the release runner a release then waited + # behind the next run. PR parity jobs have ludovics-mac-mini and the desktop. Not the desktop: a job slowed down + # because the maintainer came back could time out, and a failure here bisects and comments on merges. runs-on: [self-hosted, macOS, horizon-parity-main] timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures steps: diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index b48be1d05..1fc31d8cb 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -52,7 +52,14 @@ jobs: - name: The prepared output is the one the preparer's OUTPUT_VERSION says env: BASE_SHA: ${{ inputs.base }} - run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA" + run: | + # The job runs main's workflow on the head's files: a head made before OUTPUT_VERSION (#1947) has neither + # the tool nor the number, so there is nothing to check. + if [ ! -f tools/prepare_output.py ] || ! grep -q OUTPUT_VERSION crates/horizon-prepare/src/lib.rs; then + echo "This head's preparer has no OUTPUT_VERSION: nothing to check" + exit 0 + fi + python3 tools/prepare_output.py check --cache --base "$BASE_SHA" - name: The version constants are the pinned jar's run: python3 tools/gen_version_consts.py --check - name: The harnesses this change can affect diff --git a/crates/horizon-client/src/debug_overlay.rs b/crates/horizon-client/src/debug_overlay.rs index 3448ea411..7140905a8 100644 --- a/crates/horizon-client/src/debug_overlay.rs +++ b/crates/horizon-client/src/debug_overlay.rs @@ -84,6 +84,33 @@ impl Displayer { } } +/// Horizon's own lines: the shader pack's GPU time, then its costliest passes (`passes`: each +/// pass's median milliseconds, in the order they run). +pub fn shader_pass_lines(passes: &[(String, f32)]) -> Vec { + const SHOWN: usize = 8; + if passes.is_empty() { + return Vec::new(); + } + let total: f32 = passes.iter().map(|p| p.1).sum(); + let mut lines = vec![format!( + "Shader pack GPU: {total:.2} ms in {} passes", + passes.len() + )]; + let mut costliest: Vec<&(String, f32)> = passes.iter().collect(); + costliest.sort_by(|a, b| b.1.total_cmp(&a.1)); + lines.extend( + costliest + .iter() + .take(SHOWN) + .map(|(name, ms)| format!("{name}: {ms:.2} ms")), + ); + if costliest.len() > SHOWN { + let rest: f32 = costliest[SHOWN..].iter().map(|p| p.1).sum(); + lines.push(format!("{} more: {rest:.2} ms", costliest.len() - SHOWN)); + } + lines +} + /// `Mth.wrapDegrees`. fn wrap_degrees(angle: f32) -> f32 { let wrapped = angle % 360.0; @@ -337,4 +364,15 @@ mod tests { (1.0, [640, 360]) ); } + + #[test] + fn shader_pass_lines_list_the_costliest_passes_first() { + assert!(shader_pass_lines(&[]).is_empty()); + let passes: Vec<(String, f32)> = (0..10).map(|i| (format!("p{i}"), i as f32)).collect(); + let lines = shader_pass_lines(&passes); + assert_eq!(lines[0], "Shader pack GPU: 45.00 ms in 10 passes"); + assert_eq!(lines[1], "p9: 9.00 ms"); + assert_eq!(lines[8], "p2: 2.00 ms"); + assert_eq!(lines[9], "2 more: 1.00 ms"); + } } diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index 8f9c1957d..b7da6ca1a 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -2203,7 +2203,7 @@ impl Game<'_> { self.player_drop(thrown); } /// `Player.drop`: the statistics and a thrown item entity in front of the player. - fn player_drop(&mut self, thrown: ItemStack) { + pub(super) fn player_drop(&mut self, thrown: ItemStack) { let Some(eye) = self.eye_position() else { return; }; @@ -6512,7 +6512,11 @@ impl Game<'_> { use horizon_client::menu::Input; let button = i32::from(right); if self.remote.is_some() { - self.remote_throw(right, input); + if self.screen == Screen::Creative { + self.remote_creative_drop(right); + } else { + self.remote_throw(right, input); + } return; } let before = self.carried.clone(); @@ -10496,7 +10500,10 @@ impl Game<'_> { menu_blur: self.menu_blur(), }; let result = match self.renderer.as_mut() { - Some(renderer) => horizon_client::spans::time("renderer", || renderer.render(&frame)), + Some(renderer) => { + renderer.set_pass_timing(self.debug); + horizon_client::spans::time("renderer", || renderer.render(&frame)) + } None => Err("renderer".into()), }; // Restore storage even when rendering fails (for example, a lost surface). diff --git a/crates/horizon-client/src/game/debug_keys.rs b/crates/horizon-client/src/game/debug_keys.rs index 61e2b2f05..16e7fc6e7 100644 --- a/crates/horizon-client/src/game/debug_keys.rs +++ b/crates/horizon-client/src/game/debug_keys.rs @@ -415,6 +415,13 @@ impl Game<'_> { ], ); } + // Horizon's own: the shader pack's GPU time per pass (an empty group would move others). + let pass_lines = self.renderer.as_ref().map_or_else(Vec::new, |r| { + debug_overlay::shader_pass_lines(&r.shader_pass_times()) + }); + if !pass_lines.is_empty() { + d.group("shader pack", pass_lines); + } // tps: DebugEntryTps for local and remote connections. if self.player.is_some() { let [tx, rx] = self.debug_charts.packet_rates.map(|n| n.round() as i64); diff --git a/crates/horizon-client/src/game/lan_actions.rs b/crates/horizon-client/src/game/lan_actions.rs index 29dfc91d2..76839ffd1 100644 --- a/crates/horizon-client/src/game/lan_actions.rs +++ b/crates/horizon-client/src/game/lan_actions.rs @@ -38,6 +38,9 @@ pub(super) struct GuestPlay { use_delay: u32, /// The shield's ItemCooldowns entry: an axe disabled it for this many ticks. pub(super) shield_cooldown: u32, + /// `ServerGamePacketListenerImpl.dropSpamThrottler` (`TickThrottler(20, 1480)`): 20 for each + /// item a creative player drops from the creative screen, one less each tick. + creative_drops: u32, /// The guest's `Player.getCooldowns()` (`Game::cooldowns` for its turn). pub(super) cooldowns: horizon_client::cooldowns::Cooldowns, attack_ticks: u32, @@ -1086,6 +1089,35 @@ impl Game<'_> { } S::MoveVehicle(p) => self.guest_move_vehicle(host, conn, &p), S::PaddleBoat(p) => self.guest_paddle_boat(host, conn, [p.left, p.right]), + // handleSetCreativeModeSlot's drop: a slot below 0 is a stack thrown out of the + // creative screen (`handleCreativeModeItemDrop`), dropped from the player's hand. + S::SetCreativeModeSlot(p) if p.slot < 0 => { + if !host.hub.guests[gi].creative() { + return; + } + let Some(stack) = self.item_from_network(host, &p.item) else { + return; + }; + if stack.count + > self + .res + .items() + .get(&stack.item) + .map_or(64, |i| i.max_stack) + { + return; + } + let play = host.play.entry(conn).or_default(); + if play.creative_drops >= 1480 { + eprintln!( + "Player {} was dropping items too fast in creative mode, ignoring.", + host.hub.guests[gi].name + ); + return; + } + play.creative_drops += 20; + self.as_guest(host, conn, |g| g.player_drop(stack)); + } S::SetCreativeModeSlot(p) => { let g = &mut host.hub.guests[gi]; if !g.creative() || !(0..46).contains(&p.slot) { @@ -1805,6 +1837,7 @@ impl Game<'_> { // Player.tick's ItemCooldowns.tick, once a tick for each player // (the shield an axe disabled, a thrown pearl). play.shield_cooldown = play.shield_cooldown.saturating_sub(1); + play.creative_drops = play.creative_drops.saturating_sub(1); // ServerItemCooldowns.onCooldownEnded: a finished entry is told to // the guest (a duration of 0 removes it). for group in play.cooldowns.tick() { diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index 3a2dd1d69..3b272a890 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -4828,6 +4828,34 @@ impl Game<'_> { carried, })); } + /// `CreativeModeInventoryScreen.slotClicked` outside the window: the server knows nothing of + /// a stack picked from the creative screen, so the carried stack (an item tab's right click: + /// one of it) is dropped through `handleCreativeModeItemDrop`, a SetCreativeModeSlot to slot + /// -1, and the hand swings as `Player.drop` predicts. + pub(super) fn remote_creative_drop(&mut self, right: bool) { + let one = right && !self.creative_inventory_open(); + let Some(carried) = self.carried.as_mut() else { + return; + }; + let thrown = if one && carried.count > 1 { + carried.count -= 1; + ItemStack { + count: 1, + ..carried.clone() + } + } else { + self.carried.take().expect("carried") + }; + if !self.creative { + return; + } + self.swing_arm(); + let item = self.to_network(&Some(thrown)); + self.send(S::SetCreativeModeSlot(play::SetCreativeModeSlot { + slot: -1, + item, + })); + } pub(super) fn remote_close_container(&mut self) { let id = self .remote diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 5db82b19a..24e386bc7 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -502,6 +502,8 @@ pub struct Renderer { pack_post: Option<(i32, Option)>, /// The post passes of one of Horizon's own shader packs; never beside `pack_post`'s. native_post: Option, + /// Time the shader pack's passes on the GPU (F3, or `HORIZON_SHADER_PACK_TIMING`). + pass_timing: bool, /// The entity vertices Horizon draws itself while a shader pack draws the others. pack_entity_rest: Vec, /// Set 1 of the cloud pipeline, for a shader pack's `gbuffers_clouds`. @@ -2413,6 +2415,7 @@ impl Renderer { world_post, pack_post: None, native_post: None, + pass_timing: pack_timing::PassTiming::logging(), pack_entity_rest: Vec::new(), cloud_bindings, pack_weather: None, @@ -2992,6 +2995,34 @@ impl Renderer { .terrain_inputs(self.pack_terrain_flat), ) } + /// Times the shader pack's passes on the GPU while `on` (the F3 overlay is open), or always + /// with `HORIZON_SHADER_PACK_TIMING`. + pub fn set_pass_timing(&mut self, on: bool) { + self.pass_timing = on || pack_timing::PassTiming::logging(); + } + /// The running shader pack's timer, of either kind. + fn pass_timer<'a>( + pack: &'a mut Option<(i32, Option)>, + native: &'a mut Option, + ) -> Option<&'a mut Option> { + match (pack, native) { + (Some((_, Some(post))), _) => Some(&mut post.timing), + (_, Some(post)) => Some(&mut post.timing), + _ => None, + } + } + /// Each pass of the running shader pack with its median GPU time in milliseconds, while + /// they are timed (`set_pass_timing`). + pub fn shader_pass_times(&self) -> Vec<(String, f32)> { + let timer = match (&self.pack_post, &self.native_post) { + (Some((_, Some(post))), _) => post.timing.as_ref(), + (_, Some(post)) => post.timing.as_ref(), + _ => None, + }; + timer + .map(pack_timing::PassTiming::medians) + .unwrap_or_default() + } /// The dimension the shader pack was last set for. pub fn shader_pack_dimension(&self) -> Option { self.pack_post.as_ref().map(|p| p.0) @@ -4016,6 +4047,14 @@ impl Renderer { let pack_post = !frame.title && self.pack_post.as_ref().is_some_and(|p| p.1.is_some()); // One of Horizon's own packs reads the same world target. let native_post = !frame.title && self.native_post.is_some(); + if let Some(timer) = Self::pass_timer(&mut self.pack_post, &mut self.native_post) + && timer.is_some() != self.pass_timing + { + *timer = self + .pass_timing + .then(|| pack_timing::PassTiming::new(&self.device, &self.queue)) + .flatten(); + } let pack_reads_world = pack_post || native_post; // The pack's program reads the vertices this frame's draws bind. let pack_draws_terrain = pack_terrain && flat_frame == self.pack_terrain_flat; @@ -4649,6 +4688,9 @@ impl Renderer { timer.submitted(&self.queue); timer.collect(&self.device, &mut self.gpu_times); } + if let Some(Some(timer)) = Self::pass_timer(&mut self.pack_post, &mut self.native_post) { + timer.submitted(&self.queue); + } if let Some((path, buffer)) = readback { self.screenshot_result = Some( self.write_screenshot(&path, &buffer) diff --git a/crates/horizon-client/src/gfx/native_post.rs b/crates/horizon-client/src/gfx/native_post.rs index c4af862d7..cf99b1936 100644 --- a/crates/horizon-client/src/gfx/native_post.rs +++ b/crates/horizon-client/src/gfx/native_post.rs @@ -27,6 +27,10 @@ pub struct NativePost { groups: Option<(WorldViews, Vec)>, start: std::time::Instant, frames: u32, + /// The screen's colour format, the world target's too. + format: wgpu::TextureFormat, + /// GPU time of each pass, while the renderer asks for it (`pack_timing.rs`). + pub timing: Option, } fn texture_entry(binding: u32, sample_type: wgpu::TextureSampleType) -> wgpu::BindGroupLayoutEntry { @@ -165,6 +169,8 @@ impl NativePost { groups: None, start: std::time::Instant::now(), frames: 0, + format, + timing: None, }) } @@ -287,6 +293,10 @@ impl NativePost { ) { self.prepare(device, world, size); queue.write_buffer(&self.frame, 0, bytemuck::bytes_of(inputs)); + if let Some(t) = &mut self.timing { + t.begin(device, queue); + t.mark(device, encoder, (world.0, self.format)); + } let Some((_, groups)) = &self.groups else { return; }; @@ -307,6 +317,7 @@ impl NativePost { let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some(label), color_attachments: &attachments, + timestamp_writes: self.timing.as_mut().and_then(|t| t.writes(label)), ..Default::default() }); pass.set_pipeline(pipeline); diff --git a/crates/horizon-client/src/gfx/pack_post.rs b/crates/horizon-client/src/gfx/pack_post.rs index 19f0d7349..b9320277a 100644 --- a/crates/horizon-client/src/gfx/pack_post.rs +++ b/crates/horizon-client/src/gfx/pack_post.rs @@ -297,8 +297,8 @@ pub struct PackPost { /// with the page buffer it was made for. block_ids: Option<(wgpu::BindGroupLayout, wgpu::Buffer)>, page_groups: HashMap, - /// `HORIZON_SHADER_PACK_TIMING`: GPU time of each pass, in the log. - timing: Option, + /// GPU time of each pass, while the renderer asks for it (`pack_timing.rs`). + pub timing: Option, /// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`. custom: horizon_shaders::Custom, atlas: Option, @@ -1430,7 +1430,7 @@ impl PackPost { draws, block_ids, page_groups: HashMap::new(), - timing: super::pack_timing::PassTiming::new(device, queue), + timing: None, custom, atlas: world.atlas.as_ref().map(|a| a.0.clone()), lightmap: world.lightmap, diff --git a/crates/horizon-client/src/gfx/pack_timing.rs b/crates/horizon-client/src/gfx/pack_timing.rs index acfdb698f..24b97d7af 100644 --- a/crates/horizon-client/src/gfx/pack_timing.rs +++ b/crates/horizon-client/src/gfx/pack_timing.rs @@ -1,31 +1,53 @@ -//! Developers' aid: `HORIZON_SHADER_PACK_TIMING=1` times every pass of a shader pack on the -//! GPU, one frame in `EVERY`, and logs each pass's median every `REPORT` timed frames. The -//! frame after a timed one waits for the GPU before reading the timestamps (Metal resolves -//! them without waiting for the passes that write them), so timing slows the game. +//! GPU time of each pass of a shader pack, of either kind: shown in the F3 overlay while it is +//! open, and logged every `REPORT` timed frames with `HORIZON_SHADER_PACK_TIMING=1`. One frame +//! at a time is timed, and the CPU never waits for it: its timestamps are resolved in a later +//! frame, once the GPU has finished it (Metal resolves them without waiting for the passes that +//! write them), then read back when mapped. The next frame after that is timed again. + +use std::collections::VecDeque; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; -const EVERY: u32 = 30; const REPORT: usize = 30; /// Passes timed per frame at most (two timestamps each). const CAPACITY: u32 = 64; +/// Timed frames each pass's median is taken over. +const WINDOW: usize = 15; -pub(super) struct PassTiming { +enum Stage { + /// The next frame is timed. + Idle, + /// The frame being recorded is timed. + Recording, + /// Submitted; the flag is set once the GPU has finished it. + Running(Arc), + /// Resolved and copied; the flag is set once the copy is mapped. + Reading(Arc, u64), +} + +pub struct PassTiming { queries: wgpu::QuerySet, resolve: wgpu::Buffer, read: wgpu::Buffer, period_ns: f32, - frame: u32, - /// The passes timed in the frame being recorded or waiting to be read, in order. + stage: Stage, + /// The passes of the timed frame, in order. names: Vec, - timed: bool, - samples: Vec<(String, Vec)>, + /// Each pass's last `WINDOW` times, in milliseconds. + samples: Vec<(String, VecDeque)>, + /// Log the medians every `REPORT` timed frames. + log: bool, + /// `mark`'s pipeline, for the target format it was made for. + marker: Option<(wgpu::TextureFormat, wgpu::RenderPipeline)>, reports: usize, } impl PassTiming { + /// `None` when the device can't time passes. pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Option { - if std::env::var_os("HORIZON_SHADER_PACK_TIMING").is_none() - || !device.features().contains(wgpu::Features::TIMESTAMP_QUERY) - { + if !device.features().contains(wgpu::Features::TIMESTAMP_QUERY) { return None; } let size = u64::from(CAPACITY) * 2 * 8; @@ -52,51 +74,133 @@ impl PassTiming { wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, ), period_ns: queue.get_timestamp_period(), - frame: 0, + stage: Stage::Idle, names: Vec::new(), - timed: false, samples: Vec::new(), + log: Self::logging(), + marker: None, reports: 0, }) } - /// At the start of a frame: reads the last timed frame's passes, and decides whether this - /// one is timed. + /// Whether `HORIZON_SHADER_PACK_TIMING` asks for timing without the F3 overlay. + pub fn logging() -> bool { + static LOG: std::sync::OnceLock = std::sync::OnceLock::new(); + *LOG.get_or_init(|| std::env::var_os("HORIZON_SHADER_PACK_TIMING").is_some()) + } + + /// At the start of a frame's passes: moves the last timed frame along, and decides whether + /// this one is timed. pub fn begin(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) { - if self.timed && !self.names.is_empty() { - self.collect(device, queue); + match &self.stage { + // A timed frame that was never submitted: time this one instead. + Stage::Recording => self.stage = Stage::Idle, + Stage::Running(done) if done.load(Ordering::Acquire) => { + let bytes = self.names.len() as u64 * 16; + let mut encoder = device.create_command_encoder(&Default::default()); + encoder.resolve_query_set( + &self.queries, + 0..self.names.len() as u32 * 2, + &self.resolve, + 0, + ); + encoder.copy_buffer_to_buffer(&self.resolve, 0, &self.read, 0, bytes); + queue.submit([encoder.finish()]); + let mapped = Arc::new(AtomicBool::new(false)); + let flag = mapped.clone(); + self.read + .slice(..bytes) + .map_async(wgpu::MapMode::Read, move |r| { + flag.store(r.is_ok(), Ordering::Release) + }); + self.stage = Stage::Reading(mapped, bytes); + } + Stage::Reading(mapped, bytes) if mapped.load(Ordering::Acquire) => { + let bytes = *bytes; + self.collect(bytes); + self.stage = Stage::Idle; + } + _ => {} + } + if matches!(self.stage, Stage::Idle) { + self.names.clear(); + self.stage = Stage::Recording; } - self.names.clear(); - self.frame += 1; - self.timed = self.frame.is_multiple_of(EVERY); } /// Timestamp writes for the pass `name`, in a timed frame. pub fn writes(&mut self, name: &str) -> Option> { + let i = self.push(name)?; + Some(stamps(&self.queries, i)) + } + + /// The index of the timed frame's next pass, `name`. + fn push(&mut self, name: &str) -> Option { let i = self.names.len() as u32; - if !self.timed || i >= CAPACITY { + if !matches!(self.stage, Stage::Recording) || i >= CAPACITY { return None; } self.names.push(name.to_owned()); - Some(wgpu::RenderPassTimestampWrites { - query_set: &self.queries, - beginning_of_pass_write_index: Some(2 * i), - end_of_pass_write_index: Some(2 * i + 1), - }) + Some(i) } - fn collect(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) { - // The timed frame first, then the resolve, so every timestamp is written. - _ = device.poll(wgpu::PollType::wait_indefinitely()); - let count = self.names.len() as u32 * 2; - let bytes = u64::from(count) * 8; - let mut encoder = device.create_command_encoder(&Default::default()); - encoder.resolve_query_set(&self.queries, 0..count, &self.resolve, 0); - encoder.copy_buffer_to_buffer(&self.resolve, 0, &self.read, 0, bytes); - queue.submit([encoder.finish()]); + /// A pass that ends once everything drawn into `target` before it has, in a timed frame: + /// the next pass counts from there, not from when the GPU queued it behind them. + pub fn mark( + &mut self, + device: &wgpu::Device, + encoder: &mut wgpu::CommandEncoder, + (target, format): super::timing::Target, + ) { + if self.marker.as_ref().is_none_or(|(f, _)| *f != format) { + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("Shader pack timing mark"), + source: wgpu::ShaderSource::Wgsl(super::timing::MARKER.into()), + }); + let pipeline = super::timing::end_pipeline(device, &module, format); + self.marker = Some((format, pipeline)); + } + // An unnamed pass: its end counts, it is never listed. + let Some(i) = self.push("") else { + return; + }; + let (_, pipeline) = self.marker.as_ref().expect("made above"); + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Shader pack timing mark"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: target, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + timestamp_writes: Some(stamps(&self.queries, i)), + ..Default::default() + }); + pass.set_pipeline(pipeline); + pass.set_scissor_rect(0, 0, 1, 1); + pass.draw(0..3, 0..1); + } + + /// After the frame is submitted. + pub fn submitted(&mut self, queue: &wgpu::Queue) { + if !matches!(self.stage, Stage::Recording) { + return; + } + if self.names.is_empty() { + self.stage = Stage::Idle; + return; + } + let done = Arc::new(AtomicBool::new(false)); + let flag = done.clone(); + queue.on_submitted_work_done(move || flag.store(true, Ordering::Release)); + self.stage = Stage::Running(done); + } + + fn collect(&mut self, bytes: u64) { let slice = self.read.slice(..bytes); - slice.map_async(wgpu::MapMode::Read, |_| {}); - _ = device.poll(wgpu::PollType::wait_indefinitely()); let Ok(data) = slice.get_mapped_range() else { return; }; @@ -108,41 +212,91 @@ impl PassTiming { .collect(); drop(data); self.read.unmap(); - // Passes overlap on tiled GPUs (a pass begins before the one before it ends): a pass - // costs the time from the later of its beginning and the previous pass's end. - let mut previous_end = 0; - for (name, pair) in self.names.iter().zip(stamps.as_chunks::<2>().0) { - if pair[0] == 0 || pair[1] < pair[0] { + let times = pass_times(&stamps, self.period_ns); + for (name, ms) in self.names.iter().zip(times) { + let Some(ms) = ms.filter(|_| !name.is_empty()) else { continue; + }; + let i = match self.samples.iter().position(|s| s.0 == *name) { + Some(i) => i, + None => { + self.samples.push((name.clone(), VecDeque::new())); + self.samples.len() - 1 + } + }; + let window = &mut self.samples[i].1; + if window.len() == WINDOW { + window.pop_front(); } - let from = pair[0].max(previous_end).min(pair[1]); - previous_end = pair[1]; - let ms = (pair[1] - from) as f32 * self.period_ns / 1e6; - match self.samples.iter_mut().find(|s| s.0 == *name) { - Some(s) => s.1.push(ms), - None => self.samples.push((name.clone(), vec![ms])), - } + window.push_back(ms); } self.reports += 1; - if self.reports.is_multiple_of(REPORT) { - self.report(); + if self.log && self.reports.is_multiple_of(REPORT) { + let medians = self.medians(); + let total: f32 = medians.iter().map(|m| m.1).sum(); + let parts: Vec = medians + .iter() + .map(|(n, ms)| format!("{n} {ms:.3}")) + .collect(); + log::info!( + "shader pack GPU time per pass (median ms of the last {WINDOW} timed frames): {} = {total:.3}", + parts.join(", ") + ); } } - fn report(&mut self) { - let mut total = 0.0; - let mut parts = Vec::new(); - // A pass that did not run in this window (no entities, the sky out of sight) is left out. - for (name, v) in self.samples.iter_mut().filter(|s| !s.1.is_empty()) { - v.sort_by(f32::total_cmp); - let median = v[v.len() / 2]; - total += median; - parts.push(format!("{name} {median:.3}")); - v.clear(); - } - log::info!( - "shader pack GPU time per pass (median ms of {REPORT} frames): {} = {total:.3}", - parts.join(", ") - ); + /// Each pass's median time in milliseconds over its last timed frames, in the order the + /// passes first ran. + pub fn medians(&self) -> Vec<(String, f32)> { + self.samples + .iter() + .filter(|s| !s.1.is_empty()) + .map(|(name, v)| { + let mut v: Vec = v.iter().copied().collect(); + v.sort_by(f32::total_cmp); + (name.clone(), v[v.len() / 2]) + }) + .collect() + } +} + +/// Pass `i`'s beginning and end timestamps. +fn stamps(queries: &wgpu::QuerySet, i: u32) -> wgpu::RenderPassTimestampWrites<'_> { + wgpu::RenderPassTimestampWrites { + query_set: queries, + beginning_of_pass_write_index: Some(2 * i), + end_of_pass_write_index: Some(2 * i + 1), + } +} + +/// Each pass's time in milliseconds from its (beginning, end) timestamps, `None` for a pass +/// the GPU left unstamped. Passes overlap on tiled GPUs (a pass begins before the one before it +/// ends): a pass costs the time from the later of its beginning and the previous pass's end. +fn pass_times(stamps: &[u64], period_ns: f32) -> Vec> { + let mut previous_end = 0; + stamps + .as_chunks::<2>() + .0 + .iter() + .map(|&[begin, end]| { + if begin == 0 || end < begin { + return None; + } + let from = begin.max(previous_end).min(end); + previous_end = end; + Some((end - from) as f32 * period_ns / 1e6) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::pass_times; + + #[test] + fn overlapping_passes_cost_from_the_previous_end() { + // Pass 2 begins before pass 1 ends; pass 3 was never stamped. + let stamps = [1_000_000, 3_000_000, 2_000_000, 5_000_000, 0, 0]; + assert_eq!(pass_times(&stamps, 1.0), [Some(2.0), Some(2.0), None]); } } diff --git a/crates/horizon-client/src/gfx/timing.rs b/crates/horizon-client/src/gfx/timing.rs index 5fc566fe7..7d2b17b3b 100644 --- a/crates/horizon-client/src/gfx/timing.rs +++ b/crates/horizon-client/src/gfx/timing.rs @@ -60,7 +60,7 @@ struct Markers { /// The start's one thread; the end's triangle over the whole target (the /// pass's scissor keeps one pixel), shading nothing it writes. -const MARKER: &str = +pub(super) const MARKER: &str = "@vertex fn cover(@builtin(vertex_index) i: u32) -> @builtin(position) vec4 { return vec4(f32(i & 1u) * 4.0 - 1.0, f32(i >> 1u) * 4.0 - 1.0, 0.0, 1.0); } @@ -254,7 +254,7 @@ impl GpuTimer { } /// The end marker's pipeline for a `format` target, every write masked. -fn end_pipeline( +pub(super) fn end_pipeline( device: &wgpu::Device, module: &wgpu::ShaderModule, format: wgpu::TextureFormat, diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index e8e030c83..846a24e18 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -53,7 +53,7 @@ A harness can declare, at the top level of its script: ### Which harnesses a change affects -`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `.local/parity-runner/reach-cache.json`. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses. +`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `reach-cache.json` of the shared parity folder (beside `passes/`), so a new worktree starts warm; a harness's search stops as soon as it reaches a changed file, and after `PARITY_REACH_BUDGET` seconds (default 180) the remaining harnesses count every source their probes link, so choosing never holds a slow machine for long. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses. ## The gate diff --git a/docs/shaders.md b/docs/shaders.md index ef8521cfe..8ad1f24c8 100644 --- a/docs/shaders.md +++ b/docs/shaders.md @@ -127,11 +127,11 @@ one keeps drawing), the launcher before it installs one, and the registry's CI b 6. Every entry point translated for Vulkan (SPIR-V), Metal and DirectX 12 (HLSL), with the options at their defaults, minimums and maximums. -`[budget] gpu_ms` is the GPU time your pack aims to add to a frame; the game's overlay will show each pass's time -against it. +`[budget] gpu_ms` is the GPU time your pack aims to add to a frame. To compare, open F3 in the game: while it is +open, each pass is timed on the GPU and the overlay shows the pack's total and its costliest passes, by their `name` +(the median of the last 15 timed frames). Timing waits for nothing, so it does not slow the game. ## Next -The renderer hooks (shared with the Iris/OptiFine loader), then the post stage in the game with hot reload and the -GPU time per pass, Options > Shader packs listing these packs, then the material and sky stages, and the registry's -`shader` kind with installs from the launcher. +Options > Shader packs listing these packs, then the material and sky stages, and the registry's `shader` kind with +installs from the launcher. diff --git a/tests/test_parity_reach.py b/tests/test_parity_reach.py index 9b7464b47..07e3df9e3 100644 --- a/tests/test_parity_reach.py +++ b/tests/test_parity_reach.py @@ -101,6 +101,16 @@ class ReachTests(unittest.TestCase): (self.root / "crates/a/src/unused.rs").unlink() self.assertIn("crates/a/src/unused.rs", self.narrowed()) + def test_a_search_told_to_stop_still_keeps_the_file_it_stopped_at(self): + graph = parity_reach.SourceGraph(self.root, parity.rust_code) + kept = parity_reach.narrow(self.dependencies, PROBE, graph, {"crates/a/src/used.rs"}, {"crates/a/src/used.rs"}) + self.assertEqual(kept["crates/a/src/used.rs"], (True, set())) + self.assertFalse(graph.results, "a partial search is not remembered") + # A stop file outside the harness's sources doesn't end its search. + full = parity_reach.narrow(self.dependencies, PROBE, graph, (), {"crates/b/src/lib.rs"}) + self.assertIn("crates/a/src/display.rs", full) + self.assertTrue(graph.results) + def test_unknown_probes_keep_everything(self): self.assertEqual(set(self.narrowed(artifacts=[])), set(self.dependencies)) self.assertEqual(set(self.narrowed(artifacts=[["example", "missing", "release", ["-p", "a"], True]])), diff --git a/tools/parity.py b/tools/parity.py index 3f18b922d..285e7268b 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -1245,14 +1245,20 @@ def affected_harnesses(found, records, changed, reach=True, base=None): commit), only declarations of modules the harness doesn't reach changed (tools/parity_reach.py).""" if changed & SHARED_INPUTS: return set(found) - graph = None + graph, stop = None, frozenset() + # Parses are keyed by content, so every worktree sharing the vanilla data shares them. + cache_path = (SHARED if SHARED.is_dir() else STATE_DIR) / "reach-cache.json" if reach and any(path.startswith("crates/") for path in changed): - cache_path = STATE_DIR / "reach-cache.json" try: cache = json.loads(cache_path.read_text(encoding="utf-8")) except (OSError, ValueError): cache = {} graph = parity_reach.SourceGraph(ROOT, rust_code, cache) + # A changed file that differs in more than its `mod` declarations counts as soon as a harness reaches it, + # so the search for that harness can stop there. + stop = frozenset(path for path in changed if parity_reach.is_workspace_rust(path) + and parity_reach.declarations_changed(ROOT, path, (), base, True)) + budget, started, over = float(os.environ.get("PARITY_REACH_BUDGET") or 180), time.monotonic(), False affected = set() for name, script in found.items(): record = records.get(name) or {} @@ -1265,16 +1271,31 @@ def affected_harnesses(found, records, changed, reach=True, base=None): affected.add(name) continue plain = {path.split(KEEP)[0] for path in dependencies} - if graph is not None and plain & changed: - narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed) - if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed( - ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()): - affected.add(name) - elif plain & changed: + hit = plain & changed + if not hit: + continue + if graph is None or any(not path.startswith("crates/") for path in hit): + affected.add(name) # no narrowing, or a file narrowing always keeps (tools/, Java oracles) + continue + if not over and time.monotonic() - started > budget: + over = True + print(f"note: choosing harnesses took over {budget:.0f} s (PARITY_REACH_BUDGET): the rest count every " + "source their probes link", flush=True) + if over: + affected.add(name) + continue + narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed, stop) + if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed( + ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()): affected.add(name) if graph is not None: - STATE_DIR.mkdir(parents=True, exist_ok=True) - cache_path.write_text(json.dumps(graph.saved_cache()), encoding="utf-8") + saved = graph.saved_cache() + if len(cache) + len(saved) < 4 * max(len(saved), 1): + saved = {**cache, **saved} # other worktrees' versions of the files too, within bounds + cache_path.parent.mkdir(parents=True, exist_ok=True) + temporary = cache_path.with_suffix(f".{os.getpid()}.tmp") + temporary.write_text(json.dumps(saved), encoding="utf-8") + temporary.replace(cache_path) return affected diff --git a/tools/parity_reach.py b/tools/parity_reach.py index 0c3051b86..dc61f6215 100644 --- a/tools/parity_reach.py +++ b/tools/parity_reach.py @@ -25,8 +25,9 @@ assets, fonts) that only unreached sources name; a dependency that no longer exi a change adds take part. """ +import functools import hashlib -from pathlib import Path, PurePosixPath +from pathlib import Path import re import subprocess @@ -478,9 +479,10 @@ class SourceGraph: seeds=seeds, node_sets={}) return self.tables - def reachable(self, entries, universe, reasons=None): + def reachable(self, entries, universe, reasons=None, stop=frozenset()): """The files of universe (workspace Rust sources) holding an item reachable from entries (by the rules in - this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging).""" + this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging). The + search ends early, with only part of the answer, once it reaches a file of stop.""" # Searching every source of the crates involved (not only those Cargo listed) only adds reach, and lets # harnesses with the same entries (the game binary's) share one search. crates = frozenset(crate_of(path) for path in universe) @@ -488,11 +490,12 @@ class SourceGraph: if reasons is None and key in self.results: return self.results[key] & set(universe) everything = {path for path in self.files if crate_of(path) in crates} | set(universe) - files = self.search(entries, everything, reasons) - self.results[key] = files + files, complete = self.search(entries, everything, reasons, stop) + if complete: + self.results[key] = files return files & set(universe) - def search(self, entries, universe, reasons): + def search(self, entries, universe, reasons, stop=frozenset()): targets = {self.target(path) for path in entries} universe = {path for path in universe if (self.is_library(path) or self.target(path) in targets) and path not in self.test_only} @@ -600,71 +603,80 @@ class SourceGraph: files.add(path) if reasons is not None: reasons[path] = why + if path in stop: + raise Stop - for path in universe: - if path.endswith("/src/lib.rs") or path in entries or path not in self.parent: - open_module(path, "crate root or entry") - for node in seeds: - reach(node, "entry" if node[0] in entries else "always counted") - for path in entries & universe: - files.add(path) - if reasons is not None: - reasons.setdefault(path, "entry") - while queue: - node = queue.pop() - path, index, member = node - item = item_of(node) - crate = crate_of(path) - label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"] - source = sets_of(node, item) - why = f"{path} ({label})" - # Plain mentions reach items of this file and of the modules above it. - ancestor, new = path, source["mentions"] - while ancestor is not None and new: - known = local.setdefault(ancestor, set()) - new = new - known - known |= new + try: + for path in universe: + if path.endswith("/src/lib.rs") or path in entries or path not in self.parent: + open_module(path, "crate root or entry") + for node in seeds: + reach(node, "entry" if node[0] in entries else "always counted") + for path in entries & universe: + files.add(path) + if reasons is not None: + reasons.setdefault(path, "entry") + while queue: + node = queue.pop() + path, index, member = node + item = item_of(node) + crate = crate_of(path) + label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"] + source = sets_of(node, item) + why = f"{path} ({label})" + # Plain mentions reach items of this file and of the modules above it. + ancestor, new = path, source["mentions"] + while ancestor is not None and new: + known = local.setdefault(ancestor, set()) + new = new - known + known |= new + for name in new: + wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}") + ancestor = self.parent.get(ancestor) + if item["kind"] == "use" and item["glob"]: + for name in item["names"]: + for target in sees.get(crate, {crate}): + for module in list(by_module.get((target, name), ())) + \ + ([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []): + if module not in globbed and module in universe: + globbed.add(module) + wake([(module, index, None) for index in range(len(self.parse(module)["items"]))] + if self.parse(module) else [], f"{why} glob-imports {name}") + visible_to = sees.get(crate, {crate}) + new = source["paths"] - done["paths"].setdefault(crate, set()) + done["paths"][crate] |= new for name in new: - wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}") - ancestor = self.parent.get(ancestor) - if item["kind"] == "use" and item["glob"]: - for name in item["names"]: - for target in sees.get(crate, {crate}): - for module in list(by_module.get((target, name), ())) + \ - ([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []): - if module not in globbed and module in universe: - globbed.add(module) - wake([(module, index, None) for index in range(len(self.parse(module)["items"]))] - if self.parse(module) else [], f"{why} glob-imports {name}") - visible_to = sees.get(crate, {crate}) - new = source["paths"] - done["paths"].setdefault(crate, set()) - done["paths"][crate] |= new - for name in new: - for target in visible_to: - path_names.setdefault(target, set()).add(name) - for module in by_module.get((target, name), ()): - if module in universe and self.parent[module] in opened: - open_module(module, f"{why} names {name}") - for key, table, wanted in (("qualified", qualified, lambda other: True), - ("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"), - ("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)): - new = source[key] - done[key].setdefault(crate, set()) - done[key][crate] |= new - for name in new: - table.setdefault(name, set()).add(crate) - wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)], - f"{why} {key} {name}") - if item["kind"] in TYPE_KINDS and member is None: - own = types.setdefault(crate, set()) - local_types.setdefault(path, set()).update(names(node)) - if not names(node) <= own: - own |= names(node) - visible_cache.clear() - seeing = seen_by.get(crate, {crate}) - for name in names(node): # also when the crate has one by that name already: impls of this one - wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing], - f"implements for {name}, defined in {path}") - return files + for target in visible_to: + path_names.setdefault(target, set()).add(name) + for module in by_module.get((target, name), ()): + if module in universe and self.parent[module] in opened: + open_module(module, f"{why} names {name}") + for key, table, wanted in (("qualified", qualified, lambda other: True), + ("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"), + ("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)): + new = source[key] - done[key].setdefault(crate, set()) + done[key][crate] |= new + for name in new: + table.setdefault(name, set()).add(crate) + wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)], + f"{why} {key} {name}") + if item["kind"] in TYPE_KINDS and member is None: + own = types.setdefault(crate, set()) + local_types.setdefault(path, set()).update(names(node)) + if not names(node) <= own: + own |= names(node) + visible_cache.clear() + seeing = seen_by.get(crate, {crate}) + for name in names(node): # also when the crate has one by that name already: impls of this one + wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing], + f"implements for {name}, defined in {path}") + except Stop: + return files, False + return files, True + + +class Stop(Exception): + """A search reached a file it was told to stop at.""" def impl_subject(header): @@ -682,13 +694,19 @@ def impl_subject(header): return parameters, re.split(r"\bwhere\b", header)[0] +_CRATES = {} + + def crate_of(path): - return path.split("/", 2)[1] + """The crate folder of a crates//... path (called millions of times by a search: cached).""" + crate = _CRATES.get(path) + if crate is None: + crate = _CRATES[path] = path.split("/", 2)[1] + return crate def is_workspace_rust(path): - parts = PurePosixPath(path).parts - return len(parts) > 2 and parts[0] == "crates" and path.endswith(".rs") + return path.startswith("crates/") and path.endswith(".rs") and path.count("/") >= 2 DECLARATION_LINE = re.compile(rf"\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+({IDENT})\s*;") @@ -718,6 +736,11 @@ def declarations_changed(root, path, children, base, whole): missing.""" if base is None: return True + return _declarations_changed(str(root), path, frozenset(children), base, whole) + + +@functools.lru_cache(maxsize=None) +def _declarations_changed(root, path, children, base, whole): try: head = (Path(root) / path).read_text(encoding="utf-8", errors="replace") before = subprocess.run(["git", "show", f"{base}:{path}"], cwd=root, capture_output=True, text=True, check=True).stdout @@ -726,36 +749,37 @@ def declarations_changed(root, path, children, base, whole): return declarations(head, children, whole) != declarations(before, children, whole) -def narrow(dependencies, artifacts, graph, changed=()): +def narrow(dependencies, artifacts, graph, changed=(), stop=frozenset()): """A harness's dependencies narrowed to what its entry points can reach: {dependency: (whole, children)}, where whole says the file counts for its own content (else only through its `mod` declarations of children, as a module ancestor of a reached file) and children are its reached child modules. Without known entry points (a - probe or binary the harness built that isn't found here), every dependency is kept whole.""" + probe or binary the harness built that isn't found here), every dependency is kept whole. The search may end + as soon as it reaches a file of stop (changed files that count once reached): then only part is returned.""" plain = [dependency.split("#keep:")[0] for dependency in dependencies] - crates = {PurePosixPath(path).parts[1] for path in plain if path.startswith("crates/")} + crates = {crate_of(path) for path in plain if path.startswith("crates/") and path.count("/") >= 2} rust = {path for path in plain if is_workspace_rust(path)} - added = {path for path in changed if is_workspace_rust(path) and PurePosixPath(path).parts[1] in crates + added = {path for path in changed if is_workspace_rust(path) and crate_of(path) in crates and path not in rust and (graph.root / path).is_file()} entries = graph.entries(artifacts, rust) if entries is None: return {dependency: (True, None) for dependency in dependencies} missing = {path for path in rust if graph.parse(path) is None} - reached = graph.reachable(entries, rust | added | entries) | missing + reached = graph.reachable(entries, rust | added | entries, stop=frozenset(stop) & (rust | added)) | missing named, named_in_crate = set(), {} for path in rust | added: parsed = graph.parse(path) file_names = set(parsed["file_names"]) if parsed else set() if path in reached: named |= file_names - named_in_crate.setdefault(PurePosixPath(path).parts[1], set()).update(file_names) + named_in_crate.setdefault(crate_of(path), set()).update(file_names) kept = {} for dependency, path in zip(dependencies, plain): if is_workspace_rust(path): if path in reached: kept[dependency] = (True, set()) elif not path.startswith("crates/") or path.endswith(("Cargo.toml", "build.rs")) \ - or PurePosixPath(path).name in named \ - or PurePosixPath(path).name not in named_in_crate.get(PurePosixPath(path).parts[1], ()): + or path.rsplit("/", 1)[-1] in named \ + or path.rsplit("/", 1)[-1] not in named_in_crate.get(crate_of(path), ()): kept[dependency] = (True, None) for path in sorted(added & reached): kept[path] = (True, set())