Merge main into feat/mod-minimap-api
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -23,10 +23,10 @@ env:
|
||||
|
||||
jobs:
|
||||
parity-main:
|
||||
# The Mac mini's release runner (label added 2026-10-26), not the two `horizon-game` runners: behind PR parity jobs
|
||||
# this run waited 2 h while every merge replaced the pending one, and releases wait for it. Releases need it green
|
||||
# first, so the two never compete. Not the desktop: a job slowed down because the maintainer came back could time
|
||||
# out, and a failure here bisects and comments on merges.
|
||||
# A Mac mini runner of its own (ludovics-mac-mini-2 holds only this label since 2026-10-26): behind PR parity jobs
|
||||
# this run waited 2 h while every merge replaced the pending one, and on the release runner a release then waited
|
||||
# behind the next run. PR parity jobs have ludovics-mac-mini and the desktop. Not the desktop: a job slowed down
|
||||
# because the maintainer came back could time out, and a failure here bisects and comments on merges.
|
||||
runs-on: [self-hosted, macOS, horizon-parity-main]
|
||||
timeout-minutes: 240 # a core-crate batch reruns ~300 harnesses (~60-90 min), a bisect a few rounds of the failures
|
||||
steps:
|
||||
|
||||
@@ -52,7 +52,14 @@ jobs:
|
||||
- name: The prepared output is the one the preparer's OUTPUT_VERSION says
|
||||
env:
|
||||
BASE_SHA: ${{ inputs.base }}
|
||||
run: python3 tools/prepare_output.py check --cache --base "$BASE_SHA"
|
||||
run: |
|
||||
# The job runs main's workflow on the head's files: a head made before OUTPUT_VERSION (#1947) has neither
|
||||
# the tool nor the number, so there is nothing to check.
|
||||
if [ ! -f tools/prepare_output.py ] || ! grep -q OUTPUT_VERSION crates/horizon-prepare/src/lib.rs; then
|
||||
echo "This head's preparer has no OUTPUT_VERSION: nothing to check"
|
||||
exit 0
|
||||
fi
|
||||
python3 tools/prepare_output.py check --cache --base "$BASE_SHA"
|
||||
- name: The version constants are the pinned jar's
|
||||
run: python3 tools/gen_version_consts.py --check
|
||||
- name: The harnesses this change can affect
|
||||
|
||||
@@ -84,6 +84,33 @@ impl Displayer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Horizon's own lines: the shader pack's GPU time, then its costliest passes (`passes`: each
|
||||
/// pass's median milliseconds, in the order they run).
|
||||
pub fn shader_pass_lines(passes: &[(String, f32)]) -> Vec<String> {
|
||||
const SHOWN: usize = 8;
|
||||
if passes.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let total: f32 = passes.iter().map(|p| p.1).sum();
|
||||
let mut lines = vec![format!(
|
||||
"Shader pack GPU: {total:.2} ms in {} passes",
|
||||
passes.len()
|
||||
)];
|
||||
let mut costliest: Vec<&(String, f32)> = passes.iter().collect();
|
||||
costliest.sort_by(|a, b| b.1.total_cmp(&a.1));
|
||||
lines.extend(
|
||||
costliest
|
||||
.iter()
|
||||
.take(SHOWN)
|
||||
.map(|(name, ms)| format!("{name}: {ms:.2} ms")),
|
||||
);
|
||||
if costliest.len() > SHOWN {
|
||||
let rest: f32 = costliest[SHOWN..].iter().map(|p| p.1).sum();
|
||||
lines.push(format!("{} more: {rest:.2} ms", costliest.len() - SHOWN));
|
||||
}
|
||||
lines
|
||||
}
|
||||
|
||||
/// `Mth.wrapDegrees`.
|
||||
fn wrap_degrees(angle: f32) -> f32 {
|
||||
let wrapped = angle % 360.0;
|
||||
@@ -337,4 +364,15 @@ mod tests {
|
||||
(1.0, [640, 360])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shader_pass_lines_list_the_costliest_passes_first() {
|
||||
assert!(shader_pass_lines(&[]).is_empty());
|
||||
let passes: Vec<(String, f32)> = (0..10).map(|i| (format!("p{i}"), i as f32)).collect();
|
||||
let lines = shader_pass_lines(&passes);
|
||||
assert_eq!(lines[0], "Shader pack GPU: 45.00 ms in 10 passes");
|
||||
assert_eq!(lines[1], "p9: 9.00 ms");
|
||||
assert_eq!(lines[8], "p2: 2.00 ms");
|
||||
assert_eq!(lines[9], "2 more: 1.00 ms");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2203,7 +2203,7 @@ impl Game<'_> {
|
||||
self.player_drop(thrown);
|
||||
}
|
||||
/// `Player.drop`: the statistics and a thrown item entity in front of the player.
|
||||
fn player_drop(&mut self, thrown: ItemStack) {
|
||||
pub(super) fn player_drop(&mut self, thrown: ItemStack) {
|
||||
let Some(eye) = self.eye_position() else {
|
||||
return;
|
||||
};
|
||||
@@ -6512,7 +6512,11 @@ impl Game<'_> {
|
||||
use horizon_client::menu::Input;
|
||||
let button = i32::from(right);
|
||||
if self.remote.is_some() {
|
||||
self.remote_throw(right, input);
|
||||
if self.screen == Screen::Creative {
|
||||
self.remote_creative_drop(right);
|
||||
} else {
|
||||
self.remote_throw(right, input);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let before = self.carried.clone();
|
||||
@@ -10496,7 +10500,10 @@ impl Game<'_> {
|
||||
menu_blur: self.menu_blur(),
|
||||
};
|
||||
let result = match self.renderer.as_mut() {
|
||||
Some(renderer) => horizon_client::spans::time("renderer", || renderer.render(&frame)),
|
||||
Some(renderer) => {
|
||||
renderer.set_pass_timing(self.debug);
|
||||
horizon_client::spans::time("renderer", || renderer.render(&frame))
|
||||
}
|
||||
None => Err("renderer".into()),
|
||||
};
|
||||
// Restore storage even when rendering fails (for example, a lost surface).
|
||||
|
||||
@@ -415,6 +415,13 @@ impl Game<'_> {
|
||||
],
|
||||
);
|
||||
}
|
||||
// Horizon's own: the shader pack's GPU time per pass (an empty group would move others).
|
||||
let pass_lines = self.renderer.as_ref().map_or_else(Vec::new, |r| {
|
||||
debug_overlay::shader_pass_lines(&r.shader_pass_times())
|
||||
});
|
||||
if !pass_lines.is_empty() {
|
||||
d.group("shader pack", pass_lines);
|
||||
}
|
||||
// tps: DebugEntryTps for local and remote connections.
|
||||
if self.player.is_some() {
|
||||
let [tx, rx] = self.debug_charts.packet_rates.map(|n| n.round() as i64);
|
||||
|
||||
@@ -38,6 +38,9 @@ pub(super) struct GuestPlay {
|
||||
use_delay: u32,
|
||||
/// The shield's ItemCooldowns entry: an axe disabled it for this many ticks.
|
||||
pub(super) shield_cooldown: u32,
|
||||
/// `ServerGamePacketListenerImpl.dropSpamThrottler` (`TickThrottler(20, 1480)`): 20 for each
|
||||
/// item a creative player drops from the creative screen, one less each tick.
|
||||
creative_drops: u32,
|
||||
/// The guest's `Player.getCooldowns()` (`Game::cooldowns` for its turn).
|
||||
pub(super) cooldowns: horizon_client::cooldowns::Cooldowns,
|
||||
attack_ticks: u32,
|
||||
@@ -1086,6 +1089,35 @@ impl Game<'_> {
|
||||
}
|
||||
S::MoveVehicle(p) => self.guest_move_vehicle(host, conn, &p),
|
||||
S::PaddleBoat(p) => self.guest_paddle_boat(host, conn, [p.left, p.right]),
|
||||
// handleSetCreativeModeSlot's drop: a slot below 0 is a stack thrown out of the
|
||||
// creative screen (`handleCreativeModeItemDrop`), dropped from the player's hand.
|
||||
S::SetCreativeModeSlot(p) if p.slot < 0 => {
|
||||
if !host.hub.guests[gi].creative() {
|
||||
return;
|
||||
}
|
||||
let Some(stack) = self.item_from_network(host, &p.item) else {
|
||||
return;
|
||||
};
|
||||
if stack.count
|
||||
> self
|
||||
.res
|
||||
.items()
|
||||
.get(&stack.item)
|
||||
.map_or(64, |i| i.max_stack)
|
||||
{
|
||||
return;
|
||||
}
|
||||
let play = host.play.entry(conn).or_default();
|
||||
if play.creative_drops >= 1480 {
|
||||
eprintln!(
|
||||
"Player {} was dropping items too fast in creative mode, ignoring.",
|
||||
host.hub.guests[gi].name
|
||||
);
|
||||
return;
|
||||
}
|
||||
play.creative_drops += 20;
|
||||
self.as_guest(host, conn, |g| g.player_drop(stack));
|
||||
}
|
||||
S::SetCreativeModeSlot(p) => {
|
||||
let g = &mut host.hub.guests[gi];
|
||||
if !g.creative() || !(0..46).contains(&p.slot) {
|
||||
@@ -1805,6 +1837,7 @@ impl Game<'_> {
|
||||
// Player.tick's ItemCooldowns.tick, once a tick for each player
|
||||
// (the shield an axe disabled, a thrown pearl).
|
||||
play.shield_cooldown = play.shield_cooldown.saturating_sub(1);
|
||||
play.creative_drops = play.creative_drops.saturating_sub(1);
|
||||
// ServerItemCooldowns.onCooldownEnded: a finished entry is told to
|
||||
// the guest (a duration of 0 removes it).
|
||||
for group in play.cooldowns.tick() {
|
||||
|
||||
@@ -4828,6 +4828,34 @@ impl Game<'_> {
|
||||
carried,
|
||||
}));
|
||||
}
|
||||
/// `CreativeModeInventoryScreen.slotClicked` outside the window: the server knows nothing of
|
||||
/// a stack picked from the creative screen, so the carried stack (an item tab's right click:
|
||||
/// one of it) is dropped through `handleCreativeModeItemDrop`, a SetCreativeModeSlot to slot
|
||||
/// -1, and the hand swings as `Player.drop` predicts.
|
||||
pub(super) fn remote_creative_drop(&mut self, right: bool) {
|
||||
let one = right && !self.creative_inventory_open();
|
||||
let Some(carried) = self.carried.as_mut() else {
|
||||
return;
|
||||
};
|
||||
let thrown = if one && carried.count > 1 {
|
||||
carried.count -= 1;
|
||||
ItemStack {
|
||||
count: 1,
|
||||
..carried.clone()
|
||||
}
|
||||
} else {
|
||||
self.carried.take().expect("carried")
|
||||
};
|
||||
if !self.creative {
|
||||
return;
|
||||
}
|
||||
self.swing_arm();
|
||||
let item = self.to_network(&Some(thrown));
|
||||
self.send(S::SetCreativeModeSlot(play::SetCreativeModeSlot {
|
||||
slot: -1,
|
||||
item,
|
||||
}));
|
||||
}
|
||||
pub(super) fn remote_close_container(&mut self) {
|
||||
let id = self
|
||||
.remote
|
||||
|
||||
@@ -502,6 +502,8 @@ pub struct Renderer {
|
||||
pack_post: Option<(i32, Option<pack_post::PackPost>)>,
|
||||
/// The post passes of one of Horizon's own shader packs; never beside `pack_post`'s.
|
||||
native_post: Option<native_post::NativePost>,
|
||||
/// Time the shader pack's passes on the GPU (F3, or `HORIZON_SHADER_PACK_TIMING`).
|
||||
pass_timing: bool,
|
||||
/// The entity vertices Horizon draws itself while a shader pack draws the others.
|
||||
pack_entity_rest: Vec<Vertex>,
|
||||
/// Set 1 of the cloud pipeline, for a shader pack's `gbuffers_clouds`.
|
||||
@@ -2413,6 +2415,7 @@ impl Renderer {
|
||||
world_post,
|
||||
pack_post: None,
|
||||
native_post: None,
|
||||
pass_timing: pack_timing::PassTiming::logging(),
|
||||
pack_entity_rest: Vec::new(),
|
||||
cloud_bindings,
|
||||
pack_weather: None,
|
||||
@@ -2992,6 +2995,34 @@ impl Renderer {
|
||||
.terrain_inputs(self.pack_terrain_flat),
|
||||
)
|
||||
}
|
||||
/// Times the shader pack's passes on the GPU while `on` (the F3 overlay is open), or always
|
||||
/// with `HORIZON_SHADER_PACK_TIMING`.
|
||||
pub fn set_pass_timing(&mut self, on: bool) {
|
||||
self.pass_timing = on || pack_timing::PassTiming::logging();
|
||||
}
|
||||
/// The running shader pack's timer, of either kind.
|
||||
fn pass_timer<'a>(
|
||||
pack: &'a mut Option<(i32, Option<pack_post::PackPost>)>,
|
||||
native: &'a mut Option<native_post::NativePost>,
|
||||
) -> Option<&'a mut Option<pack_timing::PassTiming>> {
|
||||
match (pack, native) {
|
||||
(Some((_, Some(post))), _) => Some(&mut post.timing),
|
||||
(_, Some(post)) => Some(&mut post.timing),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
/// Each pass of the running shader pack with its median GPU time in milliseconds, while
|
||||
/// they are timed (`set_pass_timing`).
|
||||
pub fn shader_pass_times(&self) -> Vec<(String, f32)> {
|
||||
let timer = match (&self.pack_post, &self.native_post) {
|
||||
(Some((_, Some(post))), _) => post.timing.as_ref(),
|
||||
(_, Some(post)) => post.timing.as_ref(),
|
||||
_ => None,
|
||||
};
|
||||
timer
|
||||
.map(pack_timing::PassTiming::medians)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
/// The dimension the shader pack was last set for.
|
||||
pub fn shader_pack_dimension(&self) -> Option<i32> {
|
||||
self.pack_post.as_ref().map(|p| p.0)
|
||||
@@ -4016,6 +4047,14 @@ impl Renderer {
|
||||
let pack_post = !frame.title && self.pack_post.as_ref().is_some_and(|p| p.1.is_some());
|
||||
// One of Horizon's own packs reads the same world target.
|
||||
let native_post = !frame.title && self.native_post.is_some();
|
||||
if let Some(timer) = Self::pass_timer(&mut self.pack_post, &mut self.native_post)
|
||||
&& timer.is_some() != self.pass_timing
|
||||
{
|
||||
*timer = self
|
||||
.pass_timing
|
||||
.then(|| pack_timing::PassTiming::new(&self.device, &self.queue))
|
||||
.flatten();
|
||||
}
|
||||
let pack_reads_world = pack_post || native_post;
|
||||
// The pack's program reads the vertices this frame's draws bind.
|
||||
let pack_draws_terrain = pack_terrain && flat_frame == self.pack_terrain_flat;
|
||||
@@ -4649,6 +4688,9 @@ impl Renderer {
|
||||
timer.submitted(&self.queue);
|
||||
timer.collect(&self.device, &mut self.gpu_times);
|
||||
}
|
||||
if let Some(Some(timer)) = Self::pass_timer(&mut self.pack_post, &mut self.native_post) {
|
||||
timer.submitted(&self.queue);
|
||||
}
|
||||
if let Some((path, buffer)) = readback {
|
||||
self.screenshot_result = Some(
|
||||
self.write_screenshot(&path, &buffer)
|
||||
|
||||
@@ -27,6 +27,10 @@ pub struct NativePost {
|
||||
groups: Option<(WorldViews, Vec<wgpu::BindGroup>)>,
|
||||
start: std::time::Instant,
|
||||
frames: u32,
|
||||
/// The screen's colour format, the world target's too.
|
||||
format: wgpu::TextureFormat,
|
||||
/// GPU time of each pass, while the renderer asks for it (`pack_timing.rs`).
|
||||
pub timing: Option<super::pack_timing::PassTiming>,
|
||||
}
|
||||
|
||||
fn texture_entry(binding: u32, sample_type: wgpu::TextureSampleType) -> wgpu::BindGroupLayoutEntry {
|
||||
@@ -165,6 +169,8 @@ impl NativePost {
|
||||
groups: None,
|
||||
start: std::time::Instant::now(),
|
||||
frames: 0,
|
||||
format,
|
||||
timing: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -287,6 +293,10 @@ impl NativePost {
|
||||
) {
|
||||
self.prepare(device, world, size);
|
||||
queue.write_buffer(&self.frame, 0, bytemuck::bytes_of(inputs));
|
||||
if let Some(t) = &mut self.timing {
|
||||
t.begin(device, queue);
|
||||
t.mark(device, encoder, (world.0, self.format));
|
||||
}
|
||||
let Some((_, groups)) = &self.groups else {
|
||||
return;
|
||||
};
|
||||
@@ -307,6 +317,7 @@ impl NativePost {
|
||||
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some(label),
|
||||
color_attachments: &attachments,
|
||||
timestamp_writes: self.timing.as_mut().and_then(|t| t.writes(label)),
|
||||
..Default::default()
|
||||
});
|
||||
pass.set_pipeline(pipeline);
|
||||
|
||||
@@ -297,8 +297,8 @@ pub struct PackPost {
|
||||
/// with the page buffer it was made for.
|
||||
block_ids: Option<(wgpu::BindGroupLayout, wgpu::Buffer)>,
|
||||
page_groups: HashMap<u32, (wgpu::Buffer, wgpu::BindGroup)>,
|
||||
/// `HORIZON_SHADER_PACK_TIMING`: GPU time of each pass, in the log.
|
||||
timing: Option<super::pack_timing::PassTiming>,
|
||||
/// GPU time of each pass, while the renderer asks for it (`pack_timing.rs`).
|
||||
pub timing: Option<super::pack_timing::PassTiming>,
|
||||
/// `shaders.properties`' custom uniforms, after the frame's in `uniforms.values`.
|
||||
custom: horizon_shaders::Custom,
|
||||
atlas: Option<wgpu::TextureView>,
|
||||
@@ -1430,7 +1430,7 @@ impl PackPost {
|
||||
draws,
|
||||
block_ids,
|
||||
page_groups: HashMap::new(),
|
||||
timing: super::pack_timing::PassTiming::new(device, queue),
|
||||
timing: None,
|
||||
custom,
|
||||
atlas: world.atlas.as_ref().map(|a| a.0.clone()),
|
||||
lightmap: world.lightmap,
|
||||
|
||||
@@ -1,31 +1,53 @@
|
||||
//! Developers' aid: `HORIZON_SHADER_PACK_TIMING=1` times every pass of a shader pack on the
|
||||
//! GPU, one frame in `EVERY`, and logs each pass's median every `REPORT` timed frames. The
|
||||
//! frame after a timed one waits for the GPU before reading the timestamps (Metal resolves
|
||||
//! them without waiting for the passes that write them), so timing slows the game.
|
||||
//! GPU time of each pass of a shader pack, of either kind: shown in the F3 overlay while it is
|
||||
//! open, and logged every `REPORT` timed frames with `HORIZON_SHADER_PACK_TIMING=1`. One frame
|
||||
//! at a time is timed, and the CPU never waits for it: its timestamps are resolved in a later
|
||||
//! frame, once the GPU has finished it (Metal resolves them without waiting for the passes that
|
||||
//! write them), then read back when mapped. The next frame after that is timed again.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
const EVERY: u32 = 30;
|
||||
const REPORT: usize = 30;
|
||||
/// Passes timed per frame at most (two timestamps each).
|
||||
const CAPACITY: u32 = 64;
|
||||
/// Timed frames each pass's median is taken over.
|
||||
const WINDOW: usize = 15;
|
||||
|
||||
pub(super) struct PassTiming {
|
||||
enum Stage {
|
||||
/// The next frame is timed.
|
||||
Idle,
|
||||
/// The frame being recorded is timed.
|
||||
Recording,
|
||||
/// Submitted; the flag is set once the GPU has finished it.
|
||||
Running(Arc<AtomicBool>),
|
||||
/// Resolved and copied; the flag is set once the copy is mapped.
|
||||
Reading(Arc<AtomicBool>, u64),
|
||||
}
|
||||
|
||||
pub struct PassTiming {
|
||||
queries: wgpu::QuerySet,
|
||||
resolve: wgpu::Buffer,
|
||||
read: wgpu::Buffer,
|
||||
period_ns: f32,
|
||||
frame: u32,
|
||||
/// The passes timed in the frame being recorded or waiting to be read, in order.
|
||||
stage: Stage,
|
||||
/// The passes of the timed frame, in order.
|
||||
names: Vec<String>,
|
||||
timed: bool,
|
||||
samples: Vec<(String, Vec<f32>)>,
|
||||
/// Each pass's last `WINDOW` times, in milliseconds.
|
||||
samples: Vec<(String, VecDeque<f32>)>,
|
||||
/// Log the medians every `REPORT` timed frames.
|
||||
log: bool,
|
||||
/// `mark`'s pipeline, for the target format it was made for.
|
||||
marker: Option<(wgpu::TextureFormat, wgpu::RenderPipeline)>,
|
||||
reports: usize,
|
||||
}
|
||||
|
||||
impl PassTiming {
|
||||
/// `None` when the device can't time passes.
|
||||
pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Option<Self> {
|
||||
if std::env::var_os("HORIZON_SHADER_PACK_TIMING").is_none()
|
||||
|| !device.features().contains(wgpu::Features::TIMESTAMP_QUERY)
|
||||
{
|
||||
if !device.features().contains(wgpu::Features::TIMESTAMP_QUERY) {
|
||||
return None;
|
||||
}
|
||||
let size = u64::from(CAPACITY) * 2 * 8;
|
||||
@@ -52,51 +74,133 @@ impl PassTiming {
|
||||
wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
|
||||
),
|
||||
period_ns: queue.get_timestamp_period(),
|
||||
frame: 0,
|
||||
stage: Stage::Idle,
|
||||
names: Vec::new(),
|
||||
timed: false,
|
||||
samples: Vec::new(),
|
||||
log: Self::logging(),
|
||||
marker: None,
|
||||
reports: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// At the start of a frame: reads the last timed frame's passes, and decides whether this
|
||||
/// one is timed.
|
||||
/// Whether `HORIZON_SHADER_PACK_TIMING` asks for timing without the F3 overlay.
|
||||
pub fn logging() -> bool {
|
||||
static LOG: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||
*LOG.get_or_init(|| std::env::var_os("HORIZON_SHADER_PACK_TIMING").is_some())
|
||||
}
|
||||
|
||||
/// At the start of a frame's passes: moves the last timed frame along, and decides whether
|
||||
/// this one is timed.
|
||||
pub fn begin(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {
|
||||
if self.timed && !self.names.is_empty() {
|
||||
self.collect(device, queue);
|
||||
match &self.stage {
|
||||
// A timed frame that was never submitted: time this one instead.
|
||||
Stage::Recording => self.stage = Stage::Idle,
|
||||
Stage::Running(done) if done.load(Ordering::Acquire) => {
|
||||
let bytes = self.names.len() as u64 * 16;
|
||||
let mut encoder = device.create_command_encoder(&Default::default());
|
||||
encoder.resolve_query_set(
|
||||
&self.queries,
|
||||
0..self.names.len() as u32 * 2,
|
||||
&self.resolve,
|
||||
0,
|
||||
);
|
||||
encoder.copy_buffer_to_buffer(&self.resolve, 0, &self.read, 0, bytes);
|
||||
queue.submit([encoder.finish()]);
|
||||
let mapped = Arc::new(AtomicBool::new(false));
|
||||
let flag = mapped.clone();
|
||||
self.read
|
||||
.slice(..bytes)
|
||||
.map_async(wgpu::MapMode::Read, move |r| {
|
||||
flag.store(r.is_ok(), Ordering::Release)
|
||||
});
|
||||
self.stage = Stage::Reading(mapped, bytes);
|
||||
}
|
||||
Stage::Reading(mapped, bytes) if mapped.load(Ordering::Acquire) => {
|
||||
let bytes = *bytes;
|
||||
self.collect(bytes);
|
||||
self.stage = Stage::Idle;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if matches!(self.stage, Stage::Idle) {
|
||||
self.names.clear();
|
||||
self.stage = Stage::Recording;
|
||||
}
|
||||
self.names.clear();
|
||||
self.frame += 1;
|
||||
self.timed = self.frame.is_multiple_of(EVERY);
|
||||
}
|
||||
|
||||
/// Timestamp writes for the pass `name`, in a timed frame.
|
||||
pub fn writes(&mut self, name: &str) -> Option<wgpu::RenderPassTimestampWrites<'_>> {
|
||||
let i = self.push(name)?;
|
||||
Some(stamps(&self.queries, i))
|
||||
}
|
||||
|
||||
/// The index of the timed frame's next pass, `name`.
|
||||
fn push(&mut self, name: &str) -> Option<u32> {
|
||||
let i = self.names.len() as u32;
|
||||
if !self.timed || i >= CAPACITY {
|
||||
if !matches!(self.stage, Stage::Recording) || i >= CAPACITY {
|
||||
return None;
|
||||
}
|
||||
self.names.push(name.to_owned());
|
||||
Some(wgpu::RenderPassTimestampWrites {
|
||||
query_set: &self.queries,
|
||||
beginning_of_pass_write_index: Some(2 * i),
|
||||
end_of_pass_write_index: Some(2 * i + 1),
|
||||
})
|
||||
Some(i)
|
||||
}
|
||||
|
||||
fn collect(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {
|
||||
// The timed frame first, then the resolve, so every timestamp is written.
|
||||
_ = device.poll(wgpu::PollType::wait_indefinitely());
|
||||
let count = self.names.len() as u32 * 2;
|
||||
let bytes = u64::from(count) * 8;
|
||||
let mut encoder = device.create_command_encoder(&Default::default());
|
||||
encoder.resolve_query_set(&self.queries, 0..count, &self.resolve, 0);
|
||||
encoder.copy_buffer_to_buffer(&self.resolve, 0, &self.read, 0, bytes);
|
||||
queue.submit([encoder.finish()]);
|
||||
/// A pass that ends once everything drawn into `target` before it has, in a timed frame:
|
||||
/// the next pass counts from there, not from when the GPU queued it behind them.
|
||||
pub fn mark(
|
||||
&mut self,
|
||||
device: &wgpu::Device,
|
||||
encoder: &mut wgpu::CommandEncoder,
|
||||
(target, format): super::timing::Target,
|
||||
) {
|
||||
if self.marker.as_ref().is_none_or(|(f, _)| *f != format) {
|
||||
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
|
||||
label: Some("Shader pack timing mark"),
|
||||
source: wgpu::ShaderSource::Wgsl(super::timing::MARKER.into()),
|
||||
});
|
||||
let pipeline = super::timing::end_pipeline(device, &module, format);
|
||||
self.marker = Some((format, pipeline));
|
||||
}
|
||||
// An unnamed pass: its end counts, it is never listed.
|
||||
let Some(i) = self.push("") else {
|
||||
return;
|
||||
};
|
||||
let (_, pipeline) = self.marker.as_ref().expect("made above");
|
||||
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some("Shader pack timing mark"),
|
||||
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
|
||||
view: target,
|
||||
depth_slice: None,
|
||||
resolve_target: None,
|
||||
ops: wgpu::Operations {
|
||||
load: wgpu::LoadOp::Load,
|
||||
store: wgpu::StoreOp::Store,
|
||||
},
|
||||
})],
|
||||
timestamp_writes: Some(stamps(&self.queries, i)),
|
||||
..Default::default()
|
||||
});
|
||||
pass.set_pipeline(pipeline);
|
||||
pass.set_scissor_rect(0, 0, 1, 1);
|
||||
pass.draw(0..3, 0..1);
|
||||
}
|
||||
|
||||
/// After the frame is submitted.
|
||||
pub fn submitted(&mut self, queue: &wgpu::Queue) {
|
||||
if !matches!(self.stage, Stage::Recording) {
|
||||
return;
|
||||
}
|
||||
if self.names.is_empty() {
|
||||
self.stage = Stage::Idle;
|
||||
return;
|
||||
}
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
let flag = done.clone();
|
||||
queue.on_submitted_work_done(move || flag.store(true, Ordering::Release));
|
||||
self.stage = Stage::Running(done);
|
||||
}
|
||||
|
||||
fn collect(&mut self, bytes: u64) {
|
||||
let slice = self.read.slice(..bytes);
|
||||
slice.map_async(wgpu::MapMode::Read, |_| {});
|
||||
_ = device.poll(wgpu::PollType::wait_indefinitely());
|
||||
let Ok(data) = slice.get_mapped_range() else {
|
||||
return;
|
||||
};
|
||||
@@ -108,41 +212,91 @@ impl PassTiming {
|
||||
.collect();
|
||||
drop(data);
|
||||
self.read.unmap();
|
||||
// Passes overlap on tiled GPUs (a pass begins before the one before it ends): a pass
|
||||
// costs the time from the later of its beginning and the previous pass's end.
|
||||
let mut previous_end = 0;
|
||||
for (name, pair) in self.names.iter().zip(stamps.as_chunks::<2>().0) {
|
||||
if pair[0] == 0 || pair[1] < pair[0] {
|
||||
let times = pass_times(&stamps, self.period_ns);
|
||||
for (name, ms) in self.names.iter().zip(times) {
|
||||
let Some(ms) = ms.filter(|_| !name.is_empty()) else {
|
||||
continue;
|
||||
};
|
||||
let i = match self.samples.iter().position(|s| s.0 == *name) {
|
||||
Some(i) => i,
|
||||
None => {
|
||||
self.samples.push((name.clone(), VecDeque::new()));
|
||||
self.samples.len() - 1
|
||||
}
|
||||
};
|
||||
let window = &mut self.samples[i].1;
|
||||
if window.len() == WINDOW {
|
||||
window.pop_front();
|
||||
}
|
||||
let from = pair[0].max(previous_end).min(pair[1]);
|
||||
previous_end = pair[1];
|
||||
let ms = (pair[1] - from) as f32 * self.period_ns / 1e6;
|
||||
match self.samples.iter_mut().find(|s| s.0 == *name) {
|
||||
Some(s) => s.1.push(ms),
|
||||
None => self.samples.push((name.clone(), vec![ms])),
|
||||
}
|
||||
window.push_back(ms);
|
||||
}
|
||||
self.reports += 1;
|
||||
if self.reports.is_multiple_of(REPORT) {
|
||||
self.report();
|
||||
if self.log && self.reports.is_multiple_of(REPORT) {
|
||||
let medians = self.medians();
|
||||
let total: f32 = medians.iter().map(|m| m.1).sum();
|
||||
let parts: Vec<String> = medians
|
||||
.iter()
|
||||
.map(|(n, ms)| format!("{n} {ms:.3}"))
|
||||
.collect();
|
||||
log::info!(
|
||||
"shader pack GPU time per pass (median ms of the last {WINDOW} timed frames): {} = {total:.3}",
|
||||
parts.join(", ")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn report(&mut self) {
|
||||
let mut total = 0.0;
|
||||
let mut parts = Vec::new();
|
||||
// A pass that did not run in this window (no entities, the sky out of sight) is left out.
|
||||
for (name, v) in self.samples.iter_mut().filter(|s| !s.1.is_empty()) {
|
||||
v.sort_by(f32::total_cmp);
|
||||
let median = v[v.len() / 2];
|
||||
total += median;
|
||||
parts.push(format!("{name} {median:.3}"));
|
||||
v.clear();
|
||||
}
|
||||
log::info!(
|
||||
"shader pack GPU time per pass (median ms of {REPORT} frames): {} = {total:.3}",
|
||||
parts.join(", ")
|
||||
);
|
||||
/// Each pass's median time in milliseconds over its last timed frames, in the order the
|
||||
/// passes first ran.
|
||||
pub fn medians(&self) -> Vec<(String, f32)> {
|
||||
self.samples
|
||||
.iter()
|
||||
.filter(|s| !s.1.is_empty())
|
||||
.map(|(name, v)| {
|
||||
let mut v: Vec<f32> = v.iter().copied().collect();
|
||||
v.sort_by(f32::total_cmp);
|
||||
(name.clone(), v[v.len() / 2])
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Pass `i`'s beginning and end timestamps.
|
||||
fn stamps(queries: &wgpu::QuerySet, i: u32) -> wgpu::RenderPassTimestampWrites<'_> {
|
||||
wgpu::RenderPassTimestampWrites {
|
||||
query_set: queries,
|
||||
beginning_of_pass_write_index: Some(2 * i),
|
||||
end_of_pass_write_index: Some(2 * i + 1),
|
||||
}
|
||||
}
|
||||
|
||||
/// Each pass's time in milliseconds from its (beginning, end) timestamps, `None` for a pass
|
||||
/// the GPU left unstamped. Passes overlap on tiled GPUs (a pass begins before the one before it
|
||||
/// ends): a pass costs the time from the later of its beginning and the previous pass's end.
|
||||
fn pass_times(stamps: &[u64], period_ns: f32) -> Vec<Option<f32>> {
|
||||
let mut previous_end = 0;
|
||||
stamps
|
||||
.as_chunks::<2>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|&[begin, end]| {
|
||||
if begin == 0 || end < begin {
|
||||
return None;
|
||||
}
|
||||
let from = begin.max(previous_end).min(end);
|
||||
previous_end = end;
|
||||
Some((end - from) as f32 * period_ns / 1e6)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::pass_times;
|
||||
|
||||
#[test]
|
||||
fn overlapping_passes_cost_from_the_previous_end() {
|
||||
// Pass 2 begins before pass 1 ends; pass 3 was never stamped.
|
||||
let stamps = [1_000_000, 3_000_000, 2_000_000, 5_000_000, 0, 0];
|
||||
assert_eq!(pass_times(&stamps, 1.0), [Some(2.0), Some(2.0), None]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ struct Markers {
|
||||
|
||||
/// The start's one thread; the end's triangle over the whole target (the
|
||||
/// pass's scissor keeps one pixel), shading nothing it writes.
|
||||
const MARKER: &str =
|
||||
pub(super) const MARKER: &str =
|
||||
"@vertex fn cover(@builtin(vertex_index) i: u32) -> @builtin(position) vec4<f32> {
|
||||
return vec4<f32>(f32(i & 1u) * 4.0 - 1.0, f32(i >> 1u) * 4.0 - 1.0, 0.0, 1.0);
|
||||
}
|
||||
@@ -254,7 +254,7 @@ impl GpuTimer {
|
||||
}
|
||||
|
||||
/// The end marker's pipeline for a `format` target, every write masked.
|
||||
fn end_pipeline(
|
||||
pub(super) fn end_pipeline(
|
||||
device: &wgpu::Device,
|
||||
module: &wgpu::ShaderModule,
|
||||
format: wgpu::TextureFormat,
|
||||
|
||||
@@ -53,7 +53,7 @@ A harness can declare, at the top level of its script:
|
||||
|
||||
### Which harnesses a change affects
|
||||
|
||||
`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `.local/parity-runner/reach-cache.json`. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses.
|
||||
`--changed` and `--changed-files` narrow each harness's Rust sources to those its probes can reach (`tools/parity_reach.py`), so a change to one module of `horizon-client` no longer reruns every harness whose probe links the crate. The narrowing works on items, not files: from the probe modules a harness ran (or the binary it built), a free item (fn, type, const, static) is reached when a reached item names it in its own module, through a path or an import of a module that is named, or through a glob import; an inherent impl's methods count one by one once their type is reached and their name is mentioned; a trait impl counts once its trait and its type are reached (std's traits and types always are); a macro counts once invoked. Names match by spelling, so the result only over-approximates. A reached file's parent modules count through their `mod` declarations of it, compared with the merge base; Rust files a change adds take part, a dependency that was deleted counts as changed, code under `#[cfg(test)]` never counts, and a harness whose probe can't be found keeps every source. Parses are cached by content in `reach-cache.json` of the shared parity folder (beside `passes/`), so a new worktree starts warm; a harness's search stops as soon as it reaches a changed file, and after `PARITY_REACH_BUDGET` seconds (default 180) the remaining harnesses count every source their probes link, so choosing never holds a slow machine for long. `--by-crate` turns the narrowing off. On the last 20 pull requests merged on 2026-10-25, the harnesses picked went from 2255 to 1061 (2384 to 1592 harness-minutes; game-binary harnesses, which link all of `game/`, set the floor), and every harness a CI parity run failed in the week before was still picked; every source file a sample of probes executed (LLVM `-C instrument-coverage`) is among those kept. Building the graph takes 15 s cold, under a second warm, and up to a minute and a half on a change that reaches many harnesses.
|
||||
|
||||
## The gate
|
||||
|
||||
|
||||
+5
-5
@@ -127,11 +127,11 @@ one keeps drawing), the launcher before it installs one, and the registry's CI b
|
||||
6. Every entry point translated for Vulkan (SPIR-V), Metal and DirectX 12 (HLSL), with the options at their
|
||||
defaults, minimums and maximums.
|
||||
|
||||
`[budget] gpu_ms` is the GPU time your pack aims to add to a frame; the game's overlay will show each pass's time
|
||||
against it.
|
||||
`[budget] gpu_ms` is the GPU time your pack aims to add to a frame. To compare, open F3 in the game: while it is
|
||||
open, each pass is timed on the GPU and the overlay shows the pack's total and its costliest passes, by their `name`
|
||||
(the median of the last 15 timed frames). Timing waits for nothing, so it does not slow the game.
|
||||
|
||||
## Next
|
||||
|
||||
The renderer hooks (shared with the Iris/OptiFine loader), then the post stage in the game with hot reload and the
|
||||
GPU time per pass, Options > Shader packs listing these packs, then the material and sky stages, and the registry's
|
||||
`shader` kind with installs from the launcher.
|
||||
Options > Shader packs listing these packs, then the material and sky stages, and the registry's `shader` kind with
|
||||
installs from the launcher.
|
||||
|
||||
@@ -101,6 +101,16 @@ class ReachTests(unittest.TestCase):
|
||||
(self.root / "crates/a/src/unused.rs").unlink()
|
||||
self.assertIn("crates/a/src/unused.rs", self.narrowed())
|
||||
|
||||
def test_a_search_told_to_stop_still_keeps_the_file_it_stopped_at(self):
|
||||
graph = parity_reach.SourceGraph(self.root, parity.rust_code)
|
||||
kept = parity_reach.narrow(self.dependencies, PROBE, graph, {"crates/a/src/used.rs"}, {"crates/a/src/used.rs"})
|
||||
self.assertEqual(kept["crates/a/src/used.rs"], (True, set()))
|
||||
self.assertFalse(graph.results, "a partial search is not remembered")
|
||||
# A stop file outside the harness's sources doesn't end its search.
|
||||
full = parity_reach.narrow(self.dependencies, PROBE, graph, (), {"crates/b/src/lib.rs"})
|
||||
self.assertIn("crates/a/src/display.rs", full)
|
||||
self.assertTrue(graph.results)
|
||||
|
||||
def test_unknown_probes_keep_everything(self):
|
||||
self.assertEqual(set(self.narrowed(artifacts=[])), set(self.dependencies))
|
||||
self.assertEqual(set(self.narrowed(artifacts=[["example", "missing", "release", ["-p", "a"], True]])),
|
||||
|
||||
+31
-10
@@ -1245,14 +1245,20 @@ def affected_harnesses(found, records, changed, reach=True, base=None):
|
||||
commit), only declarations of modules the harness doesn't reach changed (tools/parity_reach.py)."""
|
||||
if changed & SHARED_INPUTS:
|
||||
return set(found)
|
||||
graph = None
|
||||
graph, stop = None, frozenset()
|
||||
# Parses are keyed by content, so every worktree sharing the vanilla data shares them.
|
||||
cache_path = (SHARED if SHARED.is_dir() else STATE_DIR) / "reach-cache.json"
|
||||
if reach and any(path.startswith("crates/") for path in changed):
|
||||
cache_path = STATE_DIR / "reach-cache.json"
|
||||
try:
|
||||
cache = json.loads(cache_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
cache = {}
|
||||
graph = parity_reach.SourceGraph(ROOT, rust_code, cache)
|
||||
# A changed file that differs in more than its `mod` declarations counts as soon as a harness reaches it,
|
||||
# so the search for that harness can stop there.
|
||||
stop = frozenset(path for path in changed if parity_reach.is_workspace_rust(path)
|
||||
and parity_reach.declarations_changed(ROOT, path, (), base, True))
|
||||
budget, started, over = float(os.environ.get("PARITY_REACH_BUDGET") or 180), time.monotonic(), False
|
||||
affected = set()
|
||||
for name, script in found.items():
|
||||
record = records.get(name) or {}
|
||||
@@ -1265,16 +1271,31 @@ def affected_harnesses(found, records, changed, reach=True, base=None):
|
||||
affected.add(name)
|
||||
continue
|
||||
plain = {path.split(KEEP)[0] for path in dependencies}
|
||||
if graph is not None and plain & changed:
|
||||
narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed)
|
||||
if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed(
|
||||
ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()):
|
||||
affected.add(name)
|
||||
elif plain & changed:
|
||||
hit = plain & changed
|
||||
if not hit:
|
||||
continue
|
||||
if graph is None or any(not path.startswith("crates/") for path in hit):
|
||||
affected.add(name) # no narrowing, or a file narrowing always keeps (tools/, Java oracles)
|
||||
continue
|
||||
if not over and time.monotonic() - started > budget:
|
||||
over = True
|
||||
print(f"note: choosing harnesses took over {budget:.0f} s (PARITY_REACH_BUDGET): the rest count every "
|
||||
"source their probes link", flush=True)
|
||||
if over:
|
||||
affected.add(name)
|
||||
continue
|
||||
narrowed = parity_reach.narrow(dependencies, recorded_artifacts(record), graph, changed, stop)
|
||||
if any(path.split(KEEP)[0] in changed and (children is None or parity_reach.declarations_changed(
|
||||
ROOT, path.split(KEEP)[0], children, base, whole)) for path, (whole, children) in narrowed.items()):
|
||||
affected.add(name)
|
||||
if graph is not None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cache_path.write_text(json.dumps(graph.saved_cache()), encoding="utf-8")
|
||||
saved = graph.saved_cache()
|
||||
if len(cache) + len(saved) < 4 * max(len(saved), 1):
|
||||
saved = {**cache, **saved} # other worktrees' versions of the files too, within bounds
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = cache_path.with_suffix(f".{os.getpid()}.tmp")
|
||||
temporary.write_text(json.dumps(saved), encoding="utf-8")
|
||||
temporary.replace(cache_path)
|
||||
return affected
|
||||
|
||||
|
||||
|
||||
+104
-80
@@ -25,8 +25,9 @@ assets, fonts) that only unreached sources name; a dependency that no longer exi
|
||||
a change adds take part.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import hashlib
|
||||
from pathlib import Path, PurePosixPath
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
@@ -478,9 +479,10 @@ class SourceGraph:
|
||||
seeds=seeds, node_sets={})
|
||||
return self.tables
|
||||
|
||||
def reachable(self, entries, universe, reasons=None):
|
||||
def reachable(self, entries, universe, reasons=None, stop=frozenset()):
|
||||
"""The files of universe (workspace Rust sources) holding an item reachable from entries (by the rules in
|
||||
this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging)."""
|
||||
this module's doc), and the entries. reasons, a dict, gets why each file was reached (for debugging). The
|
||||
search ends early, with only part of the answer, once it reaches a file of stop."""
|
||||
# Searching every source of the crates involved (not only those Cargo listed) only adds reach, and lets
|
||||
# harnesses with the same entries (the game binary's) share one search.
|
||||
crates = frozenset(crate_of(path) for path in universe)
|
||||
@@ -488,11 +490,12 @@ class SourceGraph:
|
||||
if reasons is None and key in self.results:
|
||||
return self.results[key] & set(universe)
|
||||
everything = {path for path in self.files if crate_of(path) in crates} | set(universe)
|
||||
files = self.search(entries, everything, reasons)
|
||||
self.results[key] = files
|
||||
files, complete = self.search(entries, everything, reasons, stop)
|
||||
if complete:
|
||||
self.results[key] = files
|
||||
return files & set(universe)
|
||||
|
||||
def search(self, entries, universe, reasons):
|
||||
def search(self, entries, universe, reasons, stop=frozenset()):
|
||||
targets = {self.target(path) for path in entries}
|
||||
universe = {path for path in universe if (self.is_library(path) or self.target(path) in targets)
|
||||
and path not in self.test_only}
|
||||
@@ -600,71 +603,80 @@ class SourceGraph:
|
||||
files.add(path)
|
||||
if reasons is not None:
|
||||
reasons[path] = why
|
||||
if path in stop:
|
||||
raise Stop
|
||||
|
||||
for path in universe:
|
||||
if path.endswith("/src/lib.rs") or path in entries or path not in self.parent:
|
||||
open_module(path, "crate root or entry")
|
||||
for node in seeds:
|
||||
reach(node, "entry" if node[0] in entries else "always counted")
|
||||
for path in entries & universe:
|
||||
files.add(path)
|
||||
if reasons is not None:
|
||||
reasons.setdefault(path, "entry")
|
||||
while queue:
|
||||
node = queue.pop()
|
||||
path, index, member = node
|
||||
item = item_of(node)
|
||||
crate = crate_of(path)
|
||||
label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"]
|
||||
source = sets_of(node, item)
|
||||
why = f"{path} ({label})"
|
||||
# Plain mentions reach items of this file and of the modules above it.
|
||||
ancestor, new = path, source["mentions"]
|
||||
while ancestor is not None and new:
|
||||
known = local.setdefault(ancestor, set())
|
||||
new = new - known
|
||||
known |= new
|
||||
try:
|
||||
for path in universe:
|
||||
if path.endswith("/src/lib.rs") or path in entries or path not in self.parent:
|
||||
open_module(path, "crate root or entry")
|
||||
for node in seeds:
|
||||
reach(node, "entry" if node[0] in entries else "always counted")
|
||||
for path in entries & universe:
|
||||
files.add(path)
|
||||
if reasons is not None:
|
||||
reasons.setdefault(path, "entry")
|
||||
while queue:
|
||||
node = queue.pop()
|
||||
path, index, member = node
|
||||
item = item_of(node)
|
||||
crate = crate_of(path)
|
||||
label = (item.get("names") or [item["kind"]])[0] if member is None else item["members"][member]["name"]
|
||||
source = sets_of(node, item)
|
||||
why = f"{path} ({label})"
|
||||
# Plain mentions reach items of this file and of the modules above it.
|
||||
ancestor, new = path, source["mentions"]
|
||||
while ancestor is not None and new:
|
||||
known = local.setdefault(ancestor, set())
|
||||
new = new - known
|
||||
known |= new
|
||||
for name in new:
|
||||
wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}")
|
||||
ancestor = self.parent.get(ancestor)
|
||||
if item["kind"] == "use" and item["glob"]:
|
||||
for name in item["names"]:
|
||||
for target in sees.get(crate, {crate}):
|
||||
for module in list(by_module.get((target, name), ())) + \
|
||||
([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []):
|
||||
if module not in globbed and module in universe:
|
||||
globbed.add(module)
|
||||
wake([(module, index, None) for index in range(len(self.parse(module)["items"]))]
|
||||
if self.parse(module) else [], f"{why} glob-imports {name}")
|
||||
visible_to = sees.get(crate, {crate})
|
||||
new = source["paths"] - done["paths"].setdefault(crate, set())
|
||||
done["paths"][crate] |= new
|
||||
for name in new:
|
||||
wake(by_file_name.get((ancestor, name), ()), f"{why} mentions {name}")
|
||||
ancestor = self.parent.get(ancestor)
|
||||
if item["kind"] == "use" and item["glob"]:
|
||||
for name in item["names"]:
|
||||
for target in sees.get(crate, {crate}):
|
||||
for module in list(by_module.get((target, name), ())) + \
|
||||
([f"crates/{idents[name]}/src/lib.rs"] if name in idents else []):
|
||||
if module not in globbed and module in universe:
|
||||
globbed.add(module)
|
||||
wake([(module, index, None) for index in range(len(self.parse(module)["items"]))]
|
||||
if self.parse(module) else [], f"{why} glob-imports {name}")
|
||||
visible_to = sees.get(crate, {crate})
|
||||
new = source["paths"] - done["paths"].setdefault(crate, set())
|
||||
done["paths"][crate] |= new
|
||||
for name in new:
|
||||
for target in visible_to:
|
||||
path_names.setdefault(target, set()).add(name)
|
||||
for module in by_module.get((target, name), ()):
|
||||
if module in universe and self.parent[module] in opened:
|
||||
open_module(module, f"{why} names {name}")
|
||||
for key, table, wanted in (("qualified", qualified, lambda other: True),
|
||||
("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"),
|
||||
("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)):
|
||||
new = source[key] - done[key].setdefault(crate, set())
|
||||
done[key][crate] |= new
|
||||
for name in new:
|
||||
table.setdefault(name, set()).add(crate)
|
||||
wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)],
|
||||
f"{why} {key} {name}")
|
||||
if item["kind"] in TYPE_KINDS and member is None:
|
||||
own = types.setdefault(crate, set())
|
||||
local_types.setdefault(path, set()).update(names(node))
|
||||
if not names(node) <= own:
|
||||
own |= names(node)
|
||||
visible_cache.clear()
|
||||
seeing = seen_by.get(crate, {crate})
|
||||
for name in names(node): # also when the crate has one by that name already: impls of this one
|
||||
wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing],
|
||||
f"implements for {name}, defined in {path}")
|
||||
return files
|
||||
for target in visible_to:
|
||||
path_names.setdefault(target, set()).add(name)
|
||||
for module in by_module.get((target, name), ()):
|
||||
if module in universe and self.parent[module] in opened:
|
||||
open_module(module, f"{why} names {name}")
|
||||
for key, table, wanted in (("qualified", qualified, lambda other: True),
|
||||
("invokes", invoked, lambda other: item_of(other)["kind"] == "macro"),
|
||||
("mentions", mentioned, lambda other: other[2] is not None or other[0] in globbed)):
|
||||
new = source[key] - done[key].setdefault(crate, set())
|
||||
done[key][crate] |= new
|
||||
for name in new:
|
||||
table.setdefault(name, set()).add(crate)
|
||||
wake([other for other in by_name.get(name, ()) if crate_of(other[0]) in visible_to and wanted(other)],
|
||||
f"{why} {key} {name}")
|
||||
if item["kind"] in TYPE_KINDS and member is None:
|
||||
own = types.setdefault(crate, set())
|
||||
local_types.setdefault(path, set()).update(names(node))
|
||||
if not names(node) <= own:
|
||||
own |= names(node)
|
||||
visible_cache.clear()
|
||||
seeing = seen_by.get(crate, {crate})
|
||||
for name in names(node): # also when the crate has one by that name already: impls of this one
|
||||
wake([other for other in by_type.get(name, ()) if crate_of(other[0]) in seeing],
|
||||
f"implements for {name}, defined in {path}")
|
||||
except Stop:
|
||||
return files, False
|
||||
return files, True
|
||||
|
||||
|
||||
class Stop(Exception):
|
||||
"""A search reached a file it was told to stop at."""
|
||||
|
||||
|
||||
def impl_subject(header):
|
||||
@@ -682,13 +694,19 @@ def impl_subject(header):
|
||||
return parameters, re.split(r"\bwhere\b", header)[0]
|
||||
|
||||
|
||||
_CRATES = {}
|
||||
|
||||
|
||||
def crate_of(path):
|
||||
return path.split("/", 2)[1]
|
||||
"""The crate folder of a crates/<crate>/... path (called millions of times by a search: cached)."""
|
||||
crate = _CRATES.get(path)
|
||||
if crate is None:
|
||||
crate = _CRATES[path] = path.split("/", 2)[1]
|
||||
return crate
|
||||
|
||||
|
||||
def is_workspace_rust(path):
|
||||
parts = PurePosixPath(path).parts
|
||||
return len(parts) > 2 and parts[0] == "crates" and path.endswith(".rs")
|
||||
return path.startswith("crates/") and path.endswith(".rs") and path.count("/") >= 2
|
||||
|
||||
|
||||
DECLARATION_LINE = re.compile(rf"\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+({IDENT})\s*;")
|
||||
@@ -718,6 +736,11 @@ def declarations_changed(root, path, children, base, whole):
|
||||
missing."""
|
||||
if base is None:
|
||||
return True
|
||||
return _declarations_changed(str(root), path, frozenset(children), base, whole)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _declarations_changed(root, path, children, base, whole):
|
||||
try:
|
||||
head = (Path(root) / path).read_text(encoding="utf-8", errors="replace")
|
||||
before = subprocess.run(["git", "show", f"{base}:{path}"], cwd=root, capture_output=True, text=True, check=True).stdout
|
||||
@@ -726,36 +749,37 @@ def declarations_changed(root, path, children, base, whole):
|
||||
return declarations(head, children, whole) != declarations(before, children, whole)
|
||||
|
||||
|
||||
def narrow(dependencies, artifacts, graph, changed=()):
|
||||
def narrow(dependencies, artifacts, graph, changed=(), stop=frozenset()):
|
||||
"""A harness's dependencies narrowed to what its entry points can reach: {dependency: (whole, children)}, where
|
||||
whole says the file counts for its own content (else only through its `mod` declarations of children, as a
|
||||
module ancestor of a reached file) and children are its reached child modules. Without known entry points (a
|
||||
probe or binary the harness built that isn't found here), every dependency is kept whole."""
|
||||
probe or binary the harness built that isn't found here), every dependency is kept whole. The search may end
|
||||
as soon as it reaches a file of stop (changed files that count once reached): then only part is returned."""
|
||||
plain = [dependency.split("#keep:")[0] for dependency in dependencies]
|
||||
crates = {PurePosixPath(path).parts[1] for path in plain if path.startswith("crates/")}
|
||||
crates = {crate_of(path) for path in plain if path.startswith("crates/") and path.count("/") >= 2}
|
||||
rust = {path for path in plain if is_workspace_rust(path)}
|
||||
added = {path for path in changed if is_workspace_rust(path) and PurePosixPath(path).parts[1] in crates
|
||||
added = {path for path in changed if is_workspace_rust(path) and crate_of(path) in crates
|
||||
and path not in rust and (graph.root / path).is_file()}
|
||||
entries = graph.entries(artifacts, rust)
|
||||
if entries is None:
|
||||
return {dependency: (True, None) for dependency in dependencies}
|
||||
missing = {path for path in rust if graph.parse(path) is None}
|
||||
reached = graph.reachable(entries, rust | added | entries) | missing
|
||||
reached = graph.reachable(entries, rust | added | entries, stop=frozenset(stop) & (rust | added)) | missing
|
||||
named, named_in_crate = set(), {}
|
||||
for path in rust | added:
|
||||
parsed = graph.parse(path)
|
||||
file_names = set(parsed["file_names"]) if parsed else set()
|
||||
if path in reached:
|
||||
named |= file_names
|
||||
named_in_crate.setdefault(PurePosixPath(path).parts[1], set()).update(file_names)
|
||||
named_in_crate.setdefault(crate_of(path), set()).update(file_names)
|
||||
kept = {}
|
||||
for dependency, path in zip(dependencies, plain):
|
||||
if is_workspace_rust(path):
|
||||
if path in reached:
|
||||
kept[dependency] = (True, set())
|
||||
elif not path.startswith("crates/") or path.endswith(("Cargo.toml", "build.rs")) \
|
||||
or PurePosixPath(path).name in named \
|
||||
or PurePosixPath(path).name not in named_in_crate.get(PurePosixPath(path).parts[1], ()):
|
||||
or path.rsplit("/", 1)[-1] in named \
|
||||
or path.rsplit("/", 1)[-1] not in named_in_crate.get(crate_of(path), ()):
|
||||
kept[dependency] = (True, None)
|
||||
for path in sorted(added & reached):
|
||||
kept[path] = (True, set())
|
||||
|
||||
Reference in New Issue
Block a user