Merge pull request #1434 from ludovic111/perf/metal-cull-flat
🤖 Agent: merged after verify and CI parity passed. Metal GPU occlusion culling: on the M4 Pro at RD 32, FPS is +8% to +48% (interleaved A/B against main); the hidden sections match Vulkan's; parity --changed passed 104/104.
This commit is contained in:
@@ -83,6 +83,40 @@ mod world_post;
|
||||
const COMPLETE_COLUMN: u32 = (1 << (crate::chunk::HEIGHT / 16)) - 1;
|
||||
/// A section's instance: its camera-relative origin and, in `w`, the step of
|
||||
/// its fixed-point UVs (`terrain.wgsl`'s `terrain_uv`).
|
||||
/// The world pass again after occlusion culling's test: it loads both
|
||||
/// targets (phase 2 draws over phase 1, reading the test's results).
|
||||
fn world_pass_after_culling<'e>(
|
||||
encoder: &'e mut wgpu::CommandEncoder,
|
||||
view: &wgpu::TextureView,
|
||||
depth: &wgpu::TextureView,
|
||||
pack: bool,
|
||||
) -> wgpu::RenderPass<'e> {
|
||||
encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some("Horizon world, after culling"),
|
||||
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
|
||||
view,
|
||||
resolve_target: None,
|
||||
depth_slice: None,
|
||||
ops: wgpu::Operations {
|
||||
load: wgpu::LoadOp::Load,
|
||||
store: wgpu::StoreOp::Store,
|
||||
},
|
||||
})],
|
||||
depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment {
|
||||
view: depth,
|
||||
depth_ops: Some(wgpu::Operations {
|
||||
load: wgpu::LoadOp::Load,
|
||||
store: if pack {
|
||||
wgpu::StoreOp::Store
|
||||
} else {
|
||||
wgpu::StoreOp::Discard
|
||||
},
|
||||
}),
|
||||
stencil_ops: None,
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
fn section_instance(origin: [f32; 3]) -> [f32; 4] {
|
||||
[
|
||||
origin[0],
|
||||
@@ -316,7 +350,8 @@ pub struct Renderer {
|
||||
flat: Option<flat::Flat>,
|
||||
/// `flat` draws this frame's sections (off: the per-section draws).
|
||||
flat_on: bool,
|
||||
/// Occlusion culling of solid section draws, where multi-draw is native.
|
||||
/// Occlusion culling of solid section draws, where the GPU's instance
|
||||
/// counts reach a multi-draw's entries (Vulkan, Metal).
|
||||
cull: Option<cull::Cull>,
|
||||
cull_on: bool,
|
||||
sky: wgpu::RenderPipeline,
|
||||
@@ -1455,7 +1490,6 @@ impl Renderer {
|
||||
usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
let depth = Self::depth(&device, &config);
|
||||
let (atlas_pixels, atlas_width, atlas_height) = hud_atlas;
|
||||
let atlas = device.create_texture_with_data(
|
||||
&queue,
|
||||
@@ -2211,14 +2245,25 @@ impl Renderer {
|
||||
.features()
|
||||
.contains(wgpu::Features::INDIRECT_FIRST_INSTANCE);
|
||||
// HORIZON_GPU_CULL=0 turns occlusion culling off, `all` culls every
|
||||
// frame however little it hides (A/B checks).
|
||||
// frame however few its draws and little it hides, `shader` tests in
|
||||
// the shader as on Metal (A/B checks).
|
||||
let cull_env = std::env::var("HORIZON_GPU_CULL");
|
||||
let cull = (cull_env.as_deref() != Ok("0") && cull::Cull::supported(&adapter, &device))
|
||||
.then(|| {
|
||||
let mut cull = cull::Cull::new(&device, config.format.remove_srgb_suffix());
|
||||
let cull = cull::Cull::supported(&adapter, &device)
|
||||
.filter(|_| cull_env.as_deref() != Ok("0"))
|
||||
.map(|depth_test| match cull_env.as_deref() {
|
||||
Ok("shader") => cull::DepthTest::InShader,
|
||||
_ => depth_test,
|
||||
})
|
||||
.map(|depth_test| {
|
||||
let mut cull =
|
||||
cull::Cull::new(&device, config.format.remove_srgb_suffix(), depth_test);
|
||||
cull.always = cull_env.as_deref() == Ok("all");
|
||||
if !cull.always && adapter.get_info().backend == wgpu::Backend::Metal {
|
||||
cull.min_draws = cull::TILED_MIN_DRAWS;
|
||||
}
|
||||
cull
|
||||
});
|
||||
let depth = Self::depth(&device, &config, cull.as_ref());
|
||||
let world_post = world_post::Post::new(&device, config.format.remove_srgb_suffix());
|
||||
let uploads = upload::Uploads::new(&device);
|
||||
let section_uploads = upload::Uploads::with_chunk(&device, SECTION_UPLOAD_CHUNK);
|
||||
@@ -2431,7 +2476,13 @@ impl Renderer {
|
||||
self.surface = Some(surface);
|
||||
Ok(())
|
||||
}
|
||||
fn depth(device: &wgpu::Device, config: &wgpu::SurfaceConfiguration) -> wgpu::TextureView {
|
||||
/// The world's depth, which occlusion culling's test may read.
|
||||
fn depth(
|
||||
device: &wgpu::Device,
|
||||
config: &wgpu::SurfaceConfiguration,
|
||||
cull: Option<&cull::Cull>,
|
||||
) -> wgpu::TextureView {
|
||||
let read = cull.is_some_and(|c| c.depth_test() == cull::DepthTest::InShader);
|
||||
device
|
||||
.create_texture(&wgpu::TextureDescriptor {
|
||||
label: Some("Horizon depth"),
|
||||
@@ -2444,7 +2495,11 @@ impl Renderer {
|
||||
sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format: wgpu::TextureFormat::Depth32Float,
|
||||
usage: wgpu::TextureUsages::RENDER_ATTACHMENT,
|
||||
usage: if read {
|
||||
wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING
|
||||
} else {
|
||||
wgpu::TextureUsages::RENDER_ATTACHMENT
|
||||
},
|
||||
view_formats: &[],
|
||||
})
|
||||
.create_view(&Default::default())
|
||||
@@ -2495,7 +2550,7 @@ impl Renderer {
|
||||
} else {
|
||||
self.offscreen = Some(Self::offscreen_target(&self.device, &self.config));
|
||||
}
|
||||
self.depth = Self::depth(&self.device, &self.config);
|
||||
self.depth = Self::depth(&self.device, &self.config, self.cull.as_ref());
|
||||
}
|
||||
/// Whether sections are drawn as one index list per frame (`flat`).
|
||||
pub fn flat_draws(&self) -> bool {
|
||||
@@ -3215,15 +3270,13 @@ impl Renderer {
|
||||
.inverse()
|
||||
.transform_point3(Vec3::ZERO);
|
||||
// Occlusion culling leaves the sections its last result did not see
|
||||
// to phase 2, after the others, with instance counts of 0.
|
||||
// to phase 2, after the others, with instance counts of 0. Whether
|
||||
// the frame has draws enough goes by the last one's.
|
||||
let mut cull = self
|
||||
.cull
|
||||
.as_mut()
|
||||
.filter(|_| {
|
||||
self.cull_on
|
||||
&& !frame.title
|
||||
&& self.multi_draw
|
||||
&& !(self.flat_on && self.flat.is_some())
|
||||
.filter(|c| {
|
||||
self.cull_on && !frame.title && self.multi_draw && self.section_draws >= c.min_draws
|
||||
})
|
||||
.and_then(|cull| cull.begin(&self.device).then_some(cull));
|
||||
self.phase2_draws.clear();
|
||||
@@ -3610,6 +3663,16 @@ impl Renderer {
|
||||
frame.hud,
|
||||
[self.config.width, self.config.height],
|
||||
);
|
||||
if culling && let Some(cull) = &mut self.cull {
|
||||
cull.upload(
|
||||
&self.device,
|
||||
&mut self.uploads,
|
||||
&mut encoder,
|
||||
&self.draws[phase1_draws as usize..solid_draws as usize],
|
||||
&self.instances,
|
||||
&self.globals,
|
||||
);
|
||||
}
|
||||
let flat_frame = match &mut self.flat {
|
||||
Some(flat)
|
||||
if self.flat_on && !self.draws.is_empty() && self.draws.len() >= flat.min_draws =>
|
||||
@@ -3617,6 +3680,7 @@ impl Renderer {
|
||||
let sections = &self.sections;
|
||||
let lods = &self.lods;
|
||||
let sorted = &self.sorted_indices;
|
||||
let phase2 = self.cull.as_ref().filter(|_| culling).map(|c| &c.phase2);
|
||||
crate::spans::time("flatten", || {
|
||||
flat.prepare(
|
||||
&self.device,
|
||||
@@ -3624,7 +3688,9 @@ impl Renderer {
|
||||
&mut self.uploads,
|
||||
&self.draws,
|
||||
&self.draw_pages,
|
||||
phase1_draws as usize,
|
||||
solid_draws as usize,
|
||||
phase2,
|
||||
visible
|
||||
.iter()
|
||||
.filter_map(|v| {
|
||||
@@ -3659,16 +3725,6 @@ impl Renderer {
|
||||
0,
|
||||
bytemuck::cast_slice(&self.draws),
|
||||
);
|
||||
if culling && let Some(cull) = &mut self.cull {
|
||||
cull.upload(
|
||||
&self.device,
|
||||
&mut self.uploads,
|
||||
&mut encoder,
|
||||
&self.draws[phase1_draws as usize..solid_draws as usize],
|
||||
&self.instances,
|
||||
&self.globals,
|
||||
);
|
||||
}
|
||||
}
|
||||
let view_target = target_texture.create_view(&wgpu::TextureViewDescriptor {
|
||||
format: Some(self.config.format.remove_srgb_suffix()),
|
||||
@@ -3720,6 +3776,9 @@ impl Renderer {
|
||||
Some((color, depth)) => (color, depth),
|
||||
None => (&view_target, &self.depth),
|
||||
};
|
||||
if culling && let Some(cull) = &mut self.cull {
|
||||
cull.bind_depth(&self.device, world_depth);
|
||||
}
|
||||
let cull = self.cull.as_ref().filter(|_| culling && solid_draws > 0);
|
||||
if shadow_cfg.is_some() {
|
||||
let shadow_view = self.world_post.shadow_depth().clone();
|
||||
@@ -3830,9 +3889,28 @@ impl Renderer {
|
||||
match self.flat.as_ref().filter(|_| flat_frame) {
|
||||
Some(flat) => {
|
||||
pass.set_pipeline(&flat.solid);
|
||||
flat.draw(&mut pass, &flat.calls[..flat.solid_calls], |p| {
|
||||
self.arena.buffer(p)
|
||||
});
|
||||
let vertices = |p| self.arena.buffer(p);
|
||||
match cull {
|
||||
// As below: phase 1, the test, then phase 2's draws
|
||||
// that the test saw, copied after it.
|
||||
Some(cull) => {
|
||||
flat.draw(&mut pass, &flat.calls[..flat.phase1_calls], vertices);
|
||||
cull.test(&mut pass);
|
||||
drop(pass);
|
||||
cull.test_pass(&mut encoder);
|
||||
flat.flatten_phase2(&mut encoder);
|
||||
pass = world_pass_after_culling(
|
||||
&mut encoder,
|
||||
world_view,
|
||||
world_depth,
|
||||
pack,
|
||||
);
|
||||
pass.set_bind_group(0, &self.group, &[]);
|
||||
pass.set_pipeline(&flat.solid);
|
||||
flat.draw_phase2(&mut pass, vertices);
|
||||
}
|
||||
None => flat.draw(&mut pass, &flat.calls[..flat.solid_calls], vertices),
|
||||
}
|
||||
pass.set_bind_group(0, &self.group, &[]);
|
||||
pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32);
|
||||
}
|
||||
@@ -3842,40 +3920,20 @@ impl Renderer {
|
||||
pass.set_vertex_buffer(1, self.instances.slice(..));
|
||||
match cull {
|
||||
Some(cull) => {
|
||||
// Phase 1 and the boxes against its depth, then
|
||||
// Phase 1 and the boxes against its depth (in
|
||||
// its pass, or one of their own on Metal), then
|
||||
// what they found in sight, in a pass that loads
|
||||
// both targets (it reads the test's results).
|
||||
self.draw_sections(&mut pass, &self.indirect, 0, 0, phase1_draws);
|
||||
cull.test(&mut pass);
|
||||
drop(pass);
|
||||
let attachments = [Some(wgpu::RenderPassColorAttachment {
|
||||
view: world_view,
|
||||
resolve_target: None,
|
||||
depth_slice: None,
|
||||
ops: wgpu::Operations {
|
||||
load: wgpu::LoadOp::Load,
|
||||
store: wgpu::StoreOp::Store,
|
||||
},
|
||||
})];
|
||||
pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some("Horizon world, after culling"),
|
||||
color_attachments: &attachments,
|
||||
depth_stencil_attachment: Some(
|
||||
wgpu::RenderPassDepthStencilAttachment {
|
||||
view: world_depth,
|
||||
depth_ops: Some(wgpu::Operations {
|
||||
load: wgpu::LoadOp::Load,
|
||||
store: if pack {
|
||||
wgpu::StoreOp::Store
|
||||
} else {
|
||||
wgpu::StoreOp::Discard
|
||||
},
|
||||
}),
|
||||
stencil_ops: None,
|
||||
},
|
||||
),
|
||||
..Default::default()
|
||||
});
|
||||
cull.test_pass(&mut encoder);
|
||||
pass = world_pass_after_culling(
|
||||
&mut encoder,
|
||||
world_view,
|
||||
world_depth,
|
||||
pack,
|
||||
);
|
||||
pass.set_bind_group(0, &self.group, &[]);
|
||||
pass.set_pipeline(&self.solid);
|
||||
pass.set_index_buffer(
|
||||
|
||||
@@ -22,8 +22,16 @@
|
||||
//! Every solid draw is an entry of one of the two lists, so the CPU records
|
||||
//! as many draws as without culling: drivers that unroll an indirect
|
||||
//! multi-draw on the CPU (Mesa's on Intel Gen9) pay per entry. Phase 2's
|
||||
//! counts come from the GPU, which needs a native multi-draw and forced
|
||||
//! early depth tests: Vulkan. Translucent draws are not culled.
|
||||
//! counts come from the GPU, which reads them from the draws themselves:
|
||||
//! Vulkan's native multi-draw, and Metal's, which wgpu replays as one
|
||||
//! indirect draw per entry. Translucent draws are not culled.
|
||||
//!
|
||||
//! Metal cannot force the depth test before a fragment shader that writes
|
||||
//! storage (naga writes no `[[early_fragment_tests]]`), so there the boxes
|
||||
//! are drawn in a pass of their own between the phases, which reads phase
|
||||
//! 1's depth as a texture and makes the same comparison in the shader: the
|
||||
//! fragment's depth against the stored one, greater or equal passing. The
|
||||
//! same fragments pass, so the same sections are seen.
|
||||
use std::num::NonZeroU64;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
@@ -38,6 +46,13 @@ const RETEST: u32 = 8;
|
||||
/// more than they save. Else one frame in `PROBE` tests every section.
|
||||
const MIN_SHARE: f32 = 0.1;
|
||||
const PROBE: u32 = 64;
|
||||
/// On Metal, section draws from which a frame culls. Apple's GPUs store and
|
||||
/// reload the whole frame at each of the passes the test adds, a fixed cost
|
||||
/// that the sections it hides repay only at high render distances: on an M4
|
||||
/// Pro at 1080p and render distance 12 (1,000–1,600 draws) culling added
|
||||
/// about 0.05 ms of GPU time while hiding 15–30% of the solid indices; at
|
||||
/// 32 (drawn flat, over this many draws) it saved 0.25–1.6 ms.
|
||||
pub const TILED_MIN_DRAWS: usize = 4096;
|
||||
/// Boxes over more than this share of the screen count as seen untested.
|
||||
const BIG: f32 = 0.25;
|
||||
/// Results read back at once, some frames late so the CPU never waits.
|
||||
@@ -74,10 +89,40 @@ struct Group {
|
||||
group: wgpu::BindGroup,
|
||||
}
|
||||
|
||||
/// Where the boxes' depth test runs.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum DepthTest {
|
||||
/// Forced before the fragment shader, in phase 1's pass (Vulkan).
|
||||
Early,
|
||||
/// In the fragment shader, against phase 1's depth read as a texture,
|
||||
/// in a pass of its own (Metal).
|
||||
InShader,
|
||||
}
|
||||
|
||||
/// The Vulkan entry point: the depth test forced early, which naga cannot
|
||||
/// write for Metal (so it is not in `cull.wgsl`, which every backend takes).
|
||||
const EARLY_TEST: &str = "
|
||||
@fragment
|
||||
@early_depth_test(force)
|
||||
fn fs_box_early(in: Box) -> @location(0) vec4<f32> {
|
||||
see(in.test);
|
||||
return vec4(0.0);
|
||||
}
|
||||
";
|
||||
|
||||
/// The test's depth texture binding (`InShader`), for the view it was made with.
|
||||
struct DepthGroup {
|
||||
view: wgpu::TextureView,
|
||||
group: wgpu::BindGroup,
|
||||
}
|
||||
|
||||
pub struct Cull {
|
||||
pipeline: wgpu::RenderPipeline,
|
||||
layout: wgpu::BindGroupLayout,
|
||||
group: Option<Group>,
|
||||
depth_test: DepthTest,
|
||||
depth_layout: Option<wgpu::BindGroupLayout>,
|
||||
depth_group: Option<DepthGroup>,
|
||||
params: wgpu::Buffer,
|
||||
/// Phase 2's draws: the solid draws after phase 1's in the frame's list.
|
||||
pub phase2: wgpu::Buffer,
|
||||
@@ -103,6 +148,8 @@ pub struct Cull {
|
||||
pub hidden: ([usize; 2], f32),
|
||||
/// Cull every frame, whatever the hidden share (A/B checks).
|
||||
pub always: bool,
|
||||
/// Frames with fewer section draws than this do not cull.
|
||||
pub min_draws: usize,
|
||||
}
|
||||
|
||||
fn buffer(
|
||||
@@ -134,20 +181,33 @@ impl Cull {
|
||||
pub fn features(adapter: &wgpu::Adapter) -> wgpu::Features {
|
||||
adapter.features() & wgpu::Features::SHADER_EARLY_DEPTH_TEST
|
||||
}
|
||||
/// Whether the device draws an indirect multi-draw natively and runs
|
||||
/// fragment shaders that write storage only past an early depth test.
|
||||
pub fn supported(adapter: &wgpu::Adapter, device: &wgpu::Device) -> bool {
|
||||
adapter.get_info().backend == wgpu::Backend::Vulkan
|
||||
&& device.features().contains(
|
||||
wgpu::Features::SHADER_EARLY_DEPTH_TEST | wgpu::Features::INDIRECT_FIRST_INSTANCE,
|
||||
)
|
||||
/// Where the device can run the test, if it can: Vulkan with early
|
||||
/// depth tests, or Metal, both drawing an indirect multi-draw's entries
|
||||
/// with the instance counts the GPU wrote, with fragment shaders that
|
||||
/// write storage.
|
||||
pub fn supported(adapter: &wgpu::Adapter, device: &wgpu::Device) -> Option<DepthTest> {
|
||||
let depth_test = match adapter.get_info().backend {
|
||||
wgpu::Backend::Vulkan
|
||||
if device
|
||||
.features()
|
||||
.contains(wgpu::Features::SHADER_EARLY_DEPTH_TEST) =>
|
||||
{
|
||||
DepthTest::Early
|
||||
}
|
||||
wgpu::Backend::Metal => DepthTest::InShader,
|
||||
_ => return None,
|
||||
};
|
||||
(device
|
||||
.features()
|
||||
.contains(wgpu::Features::INDIRECT_FIRST_INSTANCE)
|
||||
&& adapter.get_downlevel_capabilities().flags.contains(
|
||||
wgpu::DownlevelFlags::VERTEX_STORAGE
|
||||
| wgpu::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE,
|
||||
)
|
||||
))
|
||||
.then_some(depth_test)
|
||||
}
|
||||
/// `color` is the world pass's colour format.
|
||||
pub fn new(device: &wgpu::Device, color: wgpu::TextureFormat) -> Self {
|
||||
pub fn new(device: &wgpu::Device, color: wgpu::TextureFormat, depth_test: DepthTest) -> Self {
|
||||
let entry = |binding, visibility, ty| wgpu::BindGroupLayoutEntry {
|
||||
binding,
|
||||
visibility,
|
||||
@@ -180,13 +240,41 @@ impl Cull {
|
||||
});
|
||||
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
|
||||
label: Some("Section culling"),
|
||||
source: wgpu::ShaderSource::Wgsl(include_str!("cull.wgsl").into()),
|
||||
source: wgpu::ShaderSource::Wgsl(source(depth_test).into()),
|
||||
});
|
||||
let depth_layout = (depth_test == DepthTest::InShader).then(|| {
|
||||
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
|
||||
label: Some("Section culling depth"),
|
||||
entries: &[entry(
|
||||
0,
|
||||
fragment,
|
||||
wgpu::BindingType::Texture {
|
||||
sample_type: wgpu::TextureSampleType::Depth,
|
||||
view_dimension: wgpu::TextureViewDimension::D2,
|
||||
multisampled: false,
|
||||
},
|
||||
)],
|
||||
})
|
||||
});
|
||||
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
|
||||
label: Some("Section culling"),
|
||||
bind_group_layouts: &[Some(&layout)],
|
||||
bind_group_layouts: &[Some(&layout), depth_layout.as_ref()],
|
||||
..Default::default()
|
||||
});
|
||||
// In the shader: no colour target, and phase 1's depth attached read
|
||||
// only (a pass needs an attachment), its test left to the shader.
|
||||
let (fragment_entry, targets, depth_compare) = match depth_test {
|
||||
DepthTest::Early => (
|
||||
"fs_box_early",
|
||||
&[Some(wgpu::ColorTargetState {
|
||||
format: color,
|
||||
blend: None,
|
||||
write_mask: wgpu::ColorWrites::empty(),
|
||||
})][..],
|
||||
wgpu::CompareFunction::GreaterEqual,
|
||||
),
|
||||
DepthTest::InShader => ("fs_box", &[][..], wgpu::CompareFunction::Always),
|
||||
};
|
||||
let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
|
||||
label: Some("Section culling"),
|
||||
layout: Some(&pipeline_layout),
|
||||
@@ -204,20 +292,16 @@ impl Cull {
|
||||
depth_stencil: Some(wgpu::DepthStencilState {
|
||||
format: wgpu::TextureFormat::Depth32Float,
|
||||
depth_write_enabled: Some(false),
|
||||
depth_compare: Some(wgpu::CompareFunction::GreaterEqual),
|
||||
depth_compare: Some(depth_compare),
|
||||
stencil: Default::default(),
|
||||
bias: Default::default(),
|
||||
}),
|
||||
multisample: Default::default(),
|
||||
fragment: Some(wgpu::FragmentState {
|
||||
module: &module,
|
||||
entry_point: Some("fs_box"),
|
||||
entry_point: Some(fragment_entry),
|
||||
compilation_options: Default::default(),
|
||||
targets: &[Some(wgpu::ColorTargetState {
|
||||
format: color,
|
||||
blend: None,
|
||||
write_mask: wgpu::ColorWrites::empty(),
|
||||
})],
|
||||
targets,
|
||||
}),
|
||||
multiview_mask: None,
|
||||
cache: None,
|
||||
@@ -226,6 +310,9 @@ impl Cull {
|
||||
pipeline,
|
||||
layout,
|
||||
group: None,
|
||||
depth_test,
|
||||
depth_layout,
|
||||
depth_group: None,
|
||||
params: buffer(
|
||||
device,
|
||||
"Section culling parameters",
|
||||
@@ -259,6 +346,7 @@ impl Cull {
|
||||
stamp: 0,
|
||||
hidden: ([0; 2], 1.0),
|
||||
always: false,
|
||||
min_draws: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -399,17 +487,69 @@ impl Cull {
|
||||
}
|
||||
}
|
||||
|
||||
/// In the world pass after phase 1: draw the tested boxes against its
|
||||
/// depth. Leaves the pass's pipeline and group 0 changed.
|
||||
pub fn depth_test(&self) -> DepthTest {
|
||||
self.depth_test
|
||||
}
|
||||
|
||||
/// In the world pass after phase 1 (`Early`): draw the tested boxes
|
||||
/// against its depth. Leaves the pass's pipeline and group 0 changed.
|
||||
pub fn test(&self, pass: &mut wgpu::RenderPass<'_>) {
|
||||
let Some(Group { group, .. }) = &self.group else {
|
||||
return;
|
||||
};
|
||||
if self.depth_test != DepthTest::Early || self.test_data.is_empty() {
|
||||
return;
|
||||
}
|
||||
pass.set_pipeline(&self.pipeline);
|
||||
pass.set_bind_group(0, group, &[]);
|
||||
pass.draw(0..36, 0..self.test_data.len() as u32);
|
||||
}
|
||||
|
||||
/// Before the world pass (`InShader`): the binding of the depth the
|
||||
/// world is drawn with this frame.
|
||||
pub fn bind_depth(&mut self, device: &wgpu::Device, depth: &wgpu::TextureView) {
|
||||
let Some(layout) = &self.depth_layout else {
|
||||
return;
|
||||
};
|
||||
if self.depth_group.as_ref().is_none_or(|g| g.view != *depth) {
|
||||
self.depth_group = Some(DepthGroup {
|
||||
view: depth.clone(),
|
||||
group: device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("Section culling depth"),
|
||||
layout,
|
||||
entries: &[wgpu::BindGroupEntry {
|
||||
binding: 0,
|
||||
resource: wgpu::BindingResource::TextureView(depth),
|
||||
}],
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Between the phases (`InShader`): draw the tested boxes in a pass of
|
||||
/// their own that reads phase 1's depth, bound by `bind_depth`.
|
||||
pub fn test_pass(&self, encoder: &mut wgpu::CommandEncoder) {
|
||||
let (Some(Group { group, .. }), Some(DepthGroup { view, group: depth })) =
|
||||
(&self.group, &self.depth_group)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if self.test_data.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some("Section culling"),
|
||||
color_attachments: &[],
|
||||
depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment {
|
||||
view,
|
||||
depth_ops: None,
|
||||
stencil_ops: None,
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
pass.set_pipeline(&self.pipeline);
|
||||
pass.set_bind_group(0, group, &[]);
|
||||
pass.set_bind_group(1, depth, &[]);
|
||||
pass.draw(0..36, 0..self.test_data.len() as u32);
|
||||
}
|
||||
|
||||
@@ -498,16 +638,33 @@ impl Cull {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn shader_validates() {
|
||||
let module = naga::front::wgsl::parse_str(include_str!("cull.wgsl")).unwrap();
|
||||
naga::valid::Validator::new(
|
||||
naga::valid::ValidationFlags::all(),
|
||||
naga::valid::Capabilities::all(),
|
||||
)
|
||||
.validate(&module)
|
||||
.unwrap();
|
||||
/// The test's shader for `depth_test`.
|
||||
fn source(depth_test: DepthTest) -> String {
|
||||
let common = include_str!("cull.wgsl");
|
||||
match depth_test {
|
||||
DepthTest::Early => format!("{common}{EARLY_TEST}"),
|
||||
DepthTest::InShader => common.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn shaders_validate() {
|
||||
for (depth_test, capabilities) in [
|
||||
(DepthTest::Early, naga::valid::Capabilities::all()),
|
||||
// What wgpu's Metal backend allows: no early depth test.
|
||||
(
|
||||
DepthTest::InShader,
|
||||
naga::valid::Capabilities::all() - naga::valid::Capabilities::EARLY_DEPTH_TEST,
|
||||
),
|
||||
] {
|
||||
let module = naga::front::wgsl::parse_str(&source(depth_test)).unwrap();
|
||||
naga::valid::Validator::new(naga::valid::ValidationFlags::all(), capabilities)
|
||||
.validate(&module)
|
||||
.unwrap_or_else(|e| panic!("{depth_test:?}: {e:?}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,16 +86,28 @@ fn vs_box(@builtin(vertex_index) v: u32, @builtin(instance_index) t: u32) -> Box
|
||||
return out;
|
||||
}
|
||||
|
||||
// Runs only for fragments that pass the depth test (forced early).
|
||||
@fragment
|
||||
@early_depth_test(force)
|
||||
fn fs_box(in: Box) -> @location(0) vec4<f32> {
|
||||
if results[in.test] != params.stamp {
|
||||
results[in.test] = params.stamp;
|
||||
let test = tests[in.test];
|
||||
// The box is seen at this fragment: store the stamp, and give its phase 2
|
||||
// draws their instance.
|
||||
fn see(t: u32) {
|
||||
if results[t] != params.stamp {
|
||||
results[t] = params.stamp;
|
||||
let test = tests[t];
|
||||
for (var k = 0u; k < test.z; k++) {
|
||||
phase2[(test.y + k) * 5u + 1u] = 1u;
|
||||
}
|
||||
}
|
||||
return vec4(0.0);
|
||||
}
|
||||
|
||||
// Phase 1's depth, where the test runs in the shader (gfx/cull.rs).
|
||||
@group(1) @binding(0) var depth: texture_depth_2d;
|
||||
|
||||
// The world pass's reversed depth test, greater or equal passing, made here:
|
||||
// where it cannot be forced before a shader that writes storage, a fragment
|
||||
// that fails it would still write. Where it can, `fs_box_early` (cull.rs)
|
||||
// runs only for the fragments that pass it.
|
||||
@fragment
|
||||
fn fs_box(in: Box) {
|
||||
if in.position.z >= textureLoad(depth, vec2<u32>(in.position.xy), 0) {
|
||||
see(in.test);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,11 +11,37 @@
|
||||
//! so each block names one section: its id here, which indexes the frame's
|
||||
//! camera-relative origins (computed on the CPU exactly as for the instance
|
||||
//! attribute).
|
||||
//!
|
||||
//! With occlusion culling (`cull`), phase 1's draws and the translucent ones
|
||||
//! are copied before the world pass, as without it. Phase 2's draws are
|
||||
//! copied after the test, which tells which ones it saw: each run of a page
|
||||
//! reserves room for all of its draws, the copy packs the seen ones at its
|
||||
//! start, and the run is drawn indirectly with the count the copy reached.
|
||||
use super::arena::GRAIN;
|
||||
|
||||
/// One section draw for `flatten.wgsl`: index count, first sorted index (or
|
||||
/// `QUADS` for a solid draw), base vertex, first index in the list.
|
||||
/// `QUADS` for a solid draw; phase 2: its entry in phase 2's draws), base
|
||||
/// vertex, first index in the list (phase 2: its draw call in phase 2).
|
||||
type Range = [u32; 4];
|
||||
|
||||
/// Where a group of ranges takes its indices.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Source {
|
||||
/// Solid draws: quads.
|
||||
Quads,
|
||||
/// Occlusion culling's phase 2: quads, of the sections its test saw.
|
||||
Seen,
|
||||
/// Translucent draws: this sorted index page.
|
||||
Sorted(u32),
|
||||
}
|
||||
|
||||
/// The frame's draw calls, as `plan` lays them out.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct Plan {
|
||||
phase1_calls: usize,
|
||||
solid_calls: usize,
|
||||
total: u64,
|
||||
}
|
||||
const QUADS: u32 = u32::MAX;
|
||||
const GROUP_LIMIT: u32 = 65535;
|
||||
|
||||
@@ -38,6 +64,15 @@ pub struct Flat {
|
||||
pub translucent: wgpu::RenderPipeline,
|
||||
flatten: wgpu::ComputePipeline,
|
||||
compute_layout: wgpu::BindGroupLayout,
|
||||
flatten_seen: wgpu::ComputePipeline,
|
||||
seen_layout: wgpu::BindGroupLayout,
|
||||
/// Phase 2's draw calls (`DrawIndexedIndirectArgs`), whose index counts
|
||||
/// `flatten_seen` makes.
|
||||
phase2_calls: wgpu::Buffer,
|
||||
phase2_call_capacity: u64,
|
||||
phase2_call_data: Vec<[u32; 5]>,
|
||||
/// The copy of phase 2's draws, and its ranges.
|
||||
phase2_group: Option<(wgpu::BindGroup, u32)>,
|
||||
draw_layout: wgpu::BindGroupLayout,
|
||||
blocks_per_page: u64,
|
||||
pages: Vec<Option<PageTable>>,
|
||||
@@ -52,12 +87,14 @@ pub struct Flat {
|
||||
ranges: wgpu::Buffer,
|
||||
range_capacity: u64,
|
||||
range_data: Vec<Range>,
|
||||
/// Segments of `range_data` by index source: (sorted index page or
|
||||
/// `None` for solid draws, first range, ranges).
|
||||
segments: Vec<(Option<u32>, u64, u64)>,
|
||||
/// Segments of `range_data` by index source: (source, first range,
|
||||
/// ranges).
|
||||
segments: Vec<(Source, u64, u64)>,
|
||||
empty: wgpu::Buffer,
|
||||
/// This frame's draw calls: solid ones, then translucent ones.
|
||||
/// This frame's draw calls: solid ones (phase 1's, then phase 2's when
|
||||
/// culling), then translucent ones.
|
||||
pub calls: Vec<Call>,
|
||||
pub phase1_calls: usize,
|
||||
pub solid_calls: usize,
|
||||
/// Frames with fewer section draws keep the per-section draws.
|
||||
pub min_draws: usize,
|
||||
@@ -121,42 +158,60 @@ fn buffer(
|
||||
/// Lays out the frame's index list: each draw's range (destinations in draw
|
||||
/// order) grouped by index source into `ranges`, every group padded to
|
||||
/// `align` entries for binding, with `segments` naming each group; and the
|
||||
/// draw calls, one per run of draws in the same vertex page, solid and
|
||||
/// translucent apart. Returns the first translucent call and the list's
|
||||
/// length.
|
||||
/// draw calls, one per run of draws in the same vertex page, phase 1's solid
|
||||
/// ones, phase 2's (draws `phase1_draws..solid_draws`) and translucent ones
|
||||
/// apart. A phase 2 range names its draw in phase 2 and its call in phase 2
|
||||
/// instead of a destination: the call's room holds all of its draws.
|
||||
fn plan(
|
||||
draws: &[[u32; 5]],
|
||||
pages: &[(u32, Option<u32>)],
|
||||
phase1_draws: usize,
|
||||
solid_draws: usize,
|
||||
align: u64,
|
||||
calls: &mut Vec<Call>,
|
||||
ranges: &mut Vec<Range>,
|
||||
segments: &mut Vec<(Option<u32>, u64, u64)>,
|
||||
) -> (usize, u64) {
|
||||
segments: &mut Vec<(Source, u64, u64)>,
|
||||
) -> Plan {
|
||||
calls.clear();
|
||||
ranges.clear();
|
||||
segments.clear();
|
||||
let mut solid_calls = None;
|
||||
let (mut phase1_calls, mut solid_calls) = (None, None);
|
||||
let mut total = 0u64;
|
||||
let mut by_source: Vec<(Option<u32>, Vec<Range>)> = Vec::new();
|
||||
let mut by_source: Vec<(Source, Vec<Range>)> = Vec::new();
|
||||
for (i, (d, &(page, sorted))) in draws.iter().zip(pages).enumerate() {
|
||||
let first = if sorted.is_some() { d[2] } else { QUADS };
|
||||
let range = [d[0], first, d[3], total as u32];
|
||||
match by_source.iter_mut().find(|(s, _)| *s == sorted) {
|
||||
Some((_, list)) => list.push(range),
|
||||
None => by_source.push((sorted, vec![range])),
|
||||
if i == phase1_draws {
|
||||
phase1_calls.get_or_insert(calls.len());
|
||||
}
|
||||
if i == solid_draws {
|
||||
solid_calls = Some(calls.len());
|
||||
}
|
||||
match calls.last_mut() {
|
||||
Some(last) if last.page == page && i != solid_draws => last.count += d[0],
|
||||
Some(last) if last.page == page && i != phase1_draws && i != solid_draws => {
|
||||
last.count += d[0]
|
||||
}
|
||||
_ => calls.push(Call {
|
||||
page,
|
||||
first: total as u32,
|
||||
count: d[0],
|
||||
}),
|
||||
}
|
||||
let (source, range) = match sorted {
|
||||
Some(p) => (Source::Sorted(p), [d[0], d[2], d[3], total as u32]),
|
||||
None if (phase1_draws..solid_draws).contains(&i) => (
|
||||
Source::Seen,
|
||||
[
|
||||
d[0],
|
||||
(i - phase1_draws) as u32,
|
||||
d[3],
|
||||
(calls.len() - 1 - phase1_calls.unwrap_or(0)) as u32,
|
||||
],
|
||||
),
|
||||
None => (Source::Quads, [d[0], QUADS, d[3], total as u32]),
|
||||
};
|
||||
match by_source.iter_mut().find(|(s, _)| *s == source) {
|
||||
Some((_, list)) => list.push(range),
|
||||
None => by_source.push((source, vec![range])),
|
||||
}
|
||||
total += u64::from(d[0]);
|
||||
}
|
||||
for (source, list) in by_source {
|
||||
@@ -165,7 +220,12 @@ fn plan(
|
||||
let padded = (ranges.len() as u64).next_multiple_of(align);
|
||||
ranges.resize(padded as usize, [0; 4]);
|
||||
}
|
||||
(solid_calls.unwrap_or(calls.len()), total)
|
||||
let solid_calls = solid_calls.unwrap_or(calls.len());
|
||||
Plan {
|
||||
phase1_calls: phase1_calls.unwrap_or(calls.len()).min(solid_calls),
|
||||
solid_calls,
|
||||
total,
|
||||
}
|
||||
}
|
||||
|
||||
impl Flat {
|
||||
@@ -201,24 +261,37 @@ impl Flat {
|
||||
storage(2, compute, false),
|
||||
],
|
||||
});
|
||||
let seen_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
|
||||
label: Some("Flatten phase 2 section draws"),
|
||||
entries: &[
|
||||
storage(0, compute, true),
|
||||
storage(2, compute, false),
|
||||
storage(3, compute, true),
|
||||
storage(4, compute, false),
|
||||
],
|
||||
});
|
||||
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
|
||||
label: Some("Flatten section draws"),
|
||||
source: wgpu::ShaderSource::Wgsl(include_str!("flatten.wgsl").into()),
|
||||
});
|
||||
let flatten = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
|
||||
label: Some("Flatten section draws"),
|
||||
layout: Some(
|
||||
&device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
|
||||
label: Some("Flatten section draws"),
|
||||
bind_group_layouts: &[Some(&compute_layout)],
|
||||
..Default::default()
|
||||
}),
|
||||
),
|
||||
module: &module,
|
||||
entry_point: Some("flatten"),
|
||||
compilation_options: Default::default(),
|
||||
cache: None,
|
||||
});
|
||||
let compute_pipeline = |layout: &wgpu::BindGroupLayout, entry| {
|
||||
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
|
||||
label: Some("Flatten section draws"),
|
||||
layout: Some(
|
||||
&device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
|
||||
label: Some("Flatten section draws"),
|
||||
bind_group_layouts: &[Some(layout)],
|
||||
..Default::default()
|
||||
}),
|
||||
),
|
||||
module: &module,
|
||||
entry_point: Some(entry),
|
||||
compilation_options: Default::default(),
|
||||
cache: None,
|
||||
})
|
||||
};
|
||||
let flatten = compute_pipeline(&compute_layout, "flatten");
|
||||
let flatten_seen = compute_pipeline(&seen_layout, "flatten_seen");
|
||||
let storage_usage = wgpu::BufferUsages::STORAGE;
|
||||
Self {
|
||||
solid: pipeline(&layout, "Terrain solid (flat)", "fs_solid", None),
|
||||
@@ -230,6 +303,17 @@ impl Flat {
|
||||
),
|
||||
flatten,
|
||||
compute_layout,
|
||||
flatten_seen,
|
||||
seen_layout,
|
||||
phase2_calls: buffer(
|
||||
device,
|
||||
"Flat section draw calls, phase 2",
|
||||
20,
|
||||
wgpu::BufferUsages::INDIRECT | storage_usage,
|
||||
),
|
||||
phase2_call_capacity: 1,
|
||||
phase2_call_data: Vec::new(),
|
||||
phase2_group: None,
|
||||
draw_layout,
|
||||
blocks_per_page: page_vertices.div_ceil(GRAIN),
|
||||
pages: Vec::new(),
|
||||
@@ -251,6 +335,7 @@ impl Flat {
|
||||
segments: Vec::new(),
|
||||
empty: buffer(device, "No sorted indices", 16, storage_usage),
|
||||
calls: Vec::new(),
|
||||
phase1_calls: 0,
|
||||
solid_calls: 0,
|
||||
min_draws: if std::env::var("HORIZON_FLAT_DRAWS").as_deref() == Ok("1") {
|
||||
0
|
||||
@@ -321,6 +406,9 @@ impl Flat {
|
||||
/// the origins of the sections they draw by id. Records the copy into
|
||||
/// `encoder`; `calls` then lists the frame's draw calls. False when the
|
||||
/// frame cannot be drawn flat (the per-section draws then apply).
|
||||
///
|
||||
/// When culling, the solid draws from `phase1_draws` are phase 2's, which
|
||||
/// `phase2` holds as the test leaves them: `flatten_phase2` copies them.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn prepare(
|
||||
&mut self,
|
||||
@@ -329,10 +417,18 @@ impl Flat {
|
||||
uploads: &mut super::upload::Uploads,
|
||||
draws: &[[u32; 5]],
|
||||
pages: &[(u32, Option<u32>)],
|
||||
phase1_draws: usize,
|
||||
solid_draws: usize,
|
||||
phase2: Option<&wgpu::Buffer>,
|
||||
origins: impl Iterator<Item = (u32, [f32; 4])>,
|
||||
sorted_page: impl Fn(u32) -> wgpu::Buffer,
|
||||
) -> bool {
|
||||
self.phase2_group = None;
|
||||
let phase1_draws = if phase2.is_some() {
|
||||
phase1_draws
|
||||
} else {
|
||||
solid_draws
|
||||
};
|
||||
// Origins by section id.
|
||||
self.origin_data.clear();
|
||||
for (id, o) in origins {
|
||||
@@ -362,18 +458,25 @@ impl Flat {
|
||||
bytemuck::cast_slice(&self.origin_data),
|
||||
);
|
||||
}
|
||||
let (solid_calls, total) = plan(
|
||||
let Plan {
|
||||
phase1_calls,
|
||||
solid_calls,
|
||||
total,
|
||||
} = plan(
|
||||
draws,
|
||||
pages,
|
||||
phase1_draws,
|
||||
solid_draws,
|
||||
self.range_align,
|
||||
&mut self.calls,
|
||||
&mut self.range_data,
|
||||
&mut self.segments,
|
||||
);
|
||||
self.phase1_calls = phase1_calls;
|
||||
self.solid_calls = solid_calls;
|
||||
if total == 0 || total * 4 > device.limits().max_storage_buffer_binding_size {
|
||||
self.calls.clear();
|
||||
self.phase1_calls = 0;
|
||||
self.solid_calls = 0;
|
||||
return false;
|
||||
}
|
||||
@@ -401,22 +504,82 @@ impl Flat {
|
||||
0,
|
||||
bytemuck::cast_slice(&self.range_data),
|
||||
);
|
||||
// Phase 2's calls start empty, each at the start of its room.
|
||||
self.phase2_call_data.clear();
|
||||
self.phase2_call_data.extend(
|
||||
self.calls[phase1_calls..solid_calls]
|
||||
.iter()
|
||||
.map(|c| [0, 1, c.first, 0, 0]),
|
||||
);
|
||||
let phase2_calls = self.phase2_call_data.len() as u64;
|
||||
if phase2_calls > self.phase2_call_capacity {
|
||||
self.phase2_call_capacity = phase2_calls.next_power_of_two();
|
||||
self.phase2_calls = buffer(
|
||||
device,
|
||||
"Flat section draw calls, phase 2",
|
||||
self.phase2_call_capacity * 20,
|
||||
wgpu::BufferUsages::INDIRECT | wgpu::BufferUsages::STORAGE,
|
||||
);
|
||||
}
|
||||
if phase2_calls > 0 {
|
||||
uploads.buffer(
|
||||
encoder,
|
||||
&self.phase2_calls,
|
||||
0,
|
||||
bytemuck::cast_slice(&self.phase2_call_data),
|
||||
);
|
||||
}
|
||||
let ranges_at = |first: u64, count: u64| {
|
||||
wgpu::BindingResource::Buffer(wgpu::BufferBinding {
|
||||
buffer: &self.ranges,
|
||||
offset: first * 16,
|
||||
size: wgpu::BufferSize::new(count * 16),
|
||||
})
|
||||
};
|
||||
if let (Some(phase2), Some(&(_, first, count))) = (
|
||||
phase2,
|
||||
self.segments.iter().find(|(s, ..)| *s == Source::Seen),
|
||||
) {
|
||||
let group = device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("Flatten phase 2 section draws"),
|
||||
layout: &self.seen_layout,
|
||||
entries: &[
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 0,
|
||||
resource: ranges_at(first, count),
|
||||
},
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 2,
|
||||
resource: self.indices.as_entire_binding(),
|
||||
},
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 3,
|
||||
resource: phase2.as_entire_binding(),
|
||||
},
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 4,
|
||||
resource: self.phase2_calls.as_entire_binding(),
|
||||
},
|
||||
],
|
||||
});
|
||||
self.phase2_group = Some((group, count as u32));
|
||||
}
|
||||
let groups: Vec<(wgpu::BindGroup, u32)> = self
|
||||
.segments
|
||||
.iter()
|
||||
.map(|&(source, first, count)| {
|
||||
let sorted = source.map(&sorted_page);
|
||||
.filter_map(|&(source, first, count)| {
|
||||
let sorted = match source {
|
||||
Source::Quads => None,
|
||||
Source::Seen => return None,
|
||||
Source::Sorted(page) => Some(sorted_page(page)),
|
||||
};
|
||||
let group = device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("Flatten section draws"),
|
||||
layout: &self.compute_layout,
|
||||
entries: &[
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 0,
|
||||
resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding {
|
||||
buffer: &self.ranges,
|
||||
offset: first * 16,
|
||||
size: wgpu::BufferSize::new(count * 16),
|
||||
}),
|
||||
resource: ranges_at(first, count),
|
||||
},
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 1,
|
||||
@@ -428,7 +591,7 @@ impl Flat {
|
||||
},
|
||||
],
|
||||
});
|
||||
(group, count as u32)
|
||||
Some((group, count as u32))
|
||||
})
|
||||
.collect();
|
||||
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
|
||||
@@ -464,6 +627,19 @@ impl Flat {
|
||||
}
|
||||
true
|
||||
}
|
||||
/// After occlusion culling's test: copies the phase 2 draws it saw.
|
||||
pub fn flatten_phase2(&self, encoder: &mut wgpu::CommandEncoder) {
|
||||
let Some((group, count)) = &self.phase2_group else {
|
||||
return;
|
||||
};
|
||||
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
|
||||
label: Some("Flatten phase 2 section draws"),
|
||||
timestamp_writes: None,
|
||||
});
|
||||
pass.set_pipeline(&self.flatten_seen);
|
||||
pass.set_bind_group(0, group, &[]);
|
||||
pass.dispatch_workgroups((*count).min(GROUP_LIMIT), count.div_ceil(GROUP_LIMIT), 1);
|
||||
}
|
||||
/// Draws `calls` with a flat pipeline set and `vertices` giving each
|
||||
/// page's vertex buffer.
|
||||
pub fn draw<'a>(
|
||||
@@ -474,15 +650,38 @@ impl Flat {
|
||||
) {
|
||||
pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32);
|
||||
for call in calls {
|
||||
let table = self.pages[call.page as usize]
|
||||
.as_ref()
|
||||
.and_then(|t| t.group.as_ref())
|
||||
.expect("prepared page");
|
||||
pass.set_vertex_buffer(0, vertices(call.page).slice(..));
|
||||
pass.set_bind_group(1, table, &[]);
|
||||
self.bind_page(pass, call.page, &vertices);
|
||||
pass.draw_indexed(call.first..call.first + call.count, 0, 0..1);
|
||||
}
|
||||
}
|
||||
/// Draws phase 2's calls, as `flatten_phase2` left them.
|
||||
pub fn draw_phase2<'a>(
|
||||
&self,
|
||||
pass: &mut wgpu::RenderPass<'_>,
|
||||
vertices: impl Fn(u32) -> &'a wgpu::Buffer,
|
||||
) {
|
||||
pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32);
|
||||
for (k, call) in self.calls[self.phase1_calls..self.solid_calls]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
self.bind_page(pass, call.page, &vertices);
|
||||
pass.draw_indexed_indirect(&self.phase2_calls, k as u64 * 20);
|
||||
}
|
||||
}
|
||||
fn bind_page<'a>(
|
||||
&self,
|
||||
pass: &mut wgpu::RenderPass<'_>,
|
||||
page: u32,
|
||||
vertices: &impl Fn(u32) -> &'a wgpu::Buffer,
|
||||
) {
|
||||
let table = self.pages[page as usize]
|
||||
.as_ref()
|
||||
.and_then(|t| t.group.as_ref())
|
||||
.expect("prepared page");
|
||||
pass.set_vertex_buffer(0, vertices(page).slice(..));
|
||||
pass.set_bind_group(1, table, &[]);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -516,16 +715,22 @@ mod tests {
|
||||
fn plan_draws_runs_of_a_page_in_order() {
|
||||
let (draws, pages, solid) = sample();
|
||||
let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new());
|
||||
let (solid_calls, total) = plan(
|
||||
let Plan {
|
||||
phase1_calls,
|
||||
solid_calls,
|
||||
total,
|
||||
} = plan(
|
||||
&draws,
|
||||
&pages,
|
||||
solid,
|
||||
solid,
|
||||
4,
|
||||
&mut calls,
|
||||
&mut ranges,
|
||||
&mut segments,
|
||||
);
|
||||
assert_eq!(total, 6 * 13);
|
||||
assert_eq!(phase1_calls, solid_calls);
|
||||
let call = |page, first, count| Call { page, first, count };
|
||||
// Pages 0, 0 | 1 | 0, then translucent 0, 0 (never merged with solid).
|
||||
assert_eq!(
|
||||
@@ -539,16 +744,65 @@ mod tests {
|
||||
);
|
||||
assert_eq!(solid_calls, 3);
|
||||
// Solid ranges, then each sorted page's, every group padded to 4.
|
||||
assert_eq!(segments, [(None, 0, 4), (Some(0), 4, 1), (Some(1), 8, 1)]);
|
||||
assert_eq!(
|
||||
segments,
|
||||
[
|
||||
(Source::Quads, 0, 4),
|
||||
(Source::Sorted(0), 4, 1),
|
||||
(Source::Sorted(1), 8, 1)
|
||||
]
|
||||
);
|
||||
assert_eq!(ranges[3], [24, QUADS, 1024, 36]);
|
||||
assert_eq!(ranges[4], [12, 6, 768, 60]);
|
||||
assert_eq!(ranges[8], [6, 0, 40, 72]);
|
||||
assert_eq!(ranges.len(), 12);
|
||||
}
|
||||
|
||||
/// Phase 2's draws (here the third and fourth) get calls of their own,
|
||||
/// each with room for all of its draws, and ranges naming their draw
|
||||
/// and call in phase 2.
|
||||
#[test]
|
||||
fn plan_gives_phase_2_calls_of_its_own() {
|
||||
let (draws, pages, solid) = sample();
|
||||
let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new());
|
||||
let plan = plan(
|
||||
&draws,
|
||||
&pages,
|
||||
2,
|
||||
solid,
|
||||
4,
|
||||
&mut calls,
|
||||
&mut ranges,
|
||||
&mut segments,
|
||||
);
|
||||
assert_eq!(
|
||||
plan,
|
||||
Plan {
|
||||
phase1_calls: 1,
|
||||
solid_calls: 3,
|
||||
total: 6 * 13
|
||||
}
|
||||
);
|
||||
let call = |page, first, count| Call { page, first, count };
|
||||
// Phase 1: page 0 | phase 2: page 1, page 0 | translucent: page 0.
|
||||
assert_eq!(
|
||||
calls,
|
||||
[
|
||||
call(0, 0, 30),
|
||||
call(1, 30, 6),
|
||||
call(0, 36, 24),
|
||||
call(0, 60, 18)
|
||||
]
|
||||
);
|
||||
assert_eq!(segments[1], (Source::Seen, 4, 2));
|
||||
assert_eq!(ranges[4], [6, 0, 256, 0]);
|
||||
assert_eq!(ranges[5], [24, 1, 1024, 1]);
|
||||
}
|
||||
|
||||
/// The compute pass writes exactly the indices the per-section draws
|
||||
/// read: quads 0 1 2 0 2 3 for solid draws, sorted indices otherwise,
|
||||
/// plus the base vertex.
|
||||
/// plus the base vertex. With culling, phase 2's runs hold the indices
|
||||
/// of the draws the test saw, in order, and their calls count them.
|
||||
#[test]
|
||||
fn flatten_writes_the_indices_of_the_per_section_draws() {
|
||||
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle());
|
||||
@@ -564,101 +818,170 @@ mod tests {
|
||||
use wgpu::util::DeviceExt;
|
||||
let (draws, pages, solid) = sample();
|
||||
let align = (u64::from(device.limits().min_storage_buffer_offset_alignment) / 16).max(1);
|
||||
let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new());
|
||||
let (_, total) = plan(
|
||||
&draws,
|
||||
&pages,
|
||||
solid,
|
||||
align,
|
||||
&mut calls,
|
||||
&mut ranges,
|
||||
&mut segments,
|
||||
);
|
||||
let sorted: [Vec<u32>; 2] = [
|
||||
(0..32).map(|i| (i * 7) % 32).collect(),
|
||||
(0..8).rev().collect(),
|
||||
];
|
||||
let mut expected = Vec::new();
|
||||
for (d, (_, page)) in draws.iter().zip(&pages) {
|
||||
for k in 0..d[0] {
|
||||
let i = match page {
|
||||
None => (k / 6) * 4 + [0, 1, 2, 0, 2, 3][(k % 6) as usize],
|
||||
Some(p) => sorted[*p as usize][(d[2] + k) as usize],
|
||||
};
|
||||
expected.push(d[3] + i);
|
||||
}
|
||||
}
|
||||
let storage = |entries: &[u32]| {
|
||||
let indices = |d: &[u32; 5], sorted_page: Option<u32>| -> Vec<u32> {
|
||||
(0..d[0])
|
||||
.map(|k| {
|
||||
d[3] + match sorted_page {
|
||||
None => (k / 6) * 4 + [0, 1, 2, 0, 2, 3][(k % 6) as usize],
|
||||
Some(p) => sorted[p as usize][(d[2] + k) as usize],
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let storage = |entries: &[u32], usage| {
|
||||
device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
|
||||
label: None,
|
||||
contents: bytemuck::cast_slice(entries),
|
||||
usage: wgpu::BufferUsages::STORAGE,
|
||||
usage: wgpu::BufferUsages::STORAGE | usage,
|
||||
})
|
||||
};
|
||||
let range_buffer = storage(bytemuck::cast_slice(&ranges));
|
||||
let sorted_buffers = [storage(&sorted[0]), storage(&sorted[1]), storage(&[0; 4])];
|
||||
let out = device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: None,
|
||||
size: total * 4,
|
||||
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
let read = device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: None,
|
||||
size: total * 4,
|
||||
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
let none = wgpu::BufferUsages::empty();
|
||||
let sorted_buffers = [
|
||||
storage(&sorted[0], none),
|
||||
storage(&sorted[1], none),
|
||||
storage(&[0; 4], none),
|
||||
];
|
||||
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
|
||||
label: None,
|
||||
source: wgpu::ShaderSource::Wgsl(include_str!("flatten.wgsl").into()),
|
||||
});
|
||||
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
|
||||
label: None,
|
||||
layout: None,
|
||||
module: &module,
|
||||
entry_point: Some("flatten"),
|
||||
compilation_options: Default::default(),
|
||||
cache: None,
|
||||
});
|
||||
let mut encoder = device.create_command_encoder(&Default::default());
|
||||
{
|
||||
let mut pass = encoder.begin_compute_pass(&Default::default());
|
||||
pass.set_pipeline(&pipeline);
|
||||
for &(source, first, count) in &segments {
|
||||
let group = device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: None,
|
||||
layout: &pipeline.get_bind_group_layout(0),
|
||||
entries: &[
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 0,
|
||||
resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding {
|
||||
buffer: &range_buffer,
|
||||
offset: first * 16,
|
||||
size: wgpu::BufferSize::new(count * 16),
|
||||
}),
|
||||
},
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 1,
|
||||
resource: sorted_buffers[source.map_or(2, |p| p as usize)]
|
||||
.as_entire_binding(),
|
||||
},
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 2,
|
||||
resource: out.as_entire_binding(),
|
||||
},
|
||||
],
|
||||
});
|
||||
pass.set_bind_group(0, &group, &[]);
|
||||
pass.dispatch_workgroups(count as u32, 1, 1);
|
||||
let pipeline = |entry| {
|
||||
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
|
||||
label: None,
|
||||
layout: None,
|
||||
module: &module,
|
||||
entry_point: Some(entry),
|
||||
compilation_options: Default::default(),
|
||||
cache: None,
|
||||
})
|
||||
};
|
||||
let (flatten, flatten_seen) = (pipeline("flatten"), pipeline("flatten_seen"));
|
||||
let read_back = |buffer: &wgpu::Buffer, words: u64| -> Vec<u32> {
|
||||
let read = device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: None,
|
||||
size: words * 4,
|
||||
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
let mut encoder = device.create_command_encoder(&Default::default());
|
||||
encoder.copy_buffer_to_buffer(buffer, 0, &read, 0, words * 4);
|
||||
queue.submit([encoder.finish()]);
|
||||
read.slice(..).map_async(wgpu::MapMode::Read, |_| {});
|
||||
device.poll(wgpu::PollType::wait_indefinitely()).unwrap();
|
||||
bytemuck::cast_slice(&read.slice(..).get_mapped_range().unwrap()).to_vec()
|
||||
};
|
||||
// Without culling, then with the third and fourth draws in phase 2,
|
||||
// of which the test saw only the fourth.
|
||||
for phase1 in [solid, 2] {
|
||||
let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new());
|
||||
let plan = plan(
|
||||
&draws,
|
||||
&pages,
|
||||
phase1,
|
||||
solid,
|
||||
align,
|
||||
&mut calls,
|
||||
&mut ranges,
|
||||
&mut segments,
|
||||
);
|
||||
// Phase 2's draws as the test leaves them: instance counts of 0,
|
||||
// 1 where it saw the section.
|
||||
let mut phase2: Vec<[u32; 5]> = draws[phase1..solid]
|
||||
.iter()
|
||||
.map(|&[count, _, first, base, instance]| [count, 0, first, base, instance])
|
||||
.collect();
|
||||
if let Some(last) = phase2.last_mut() {
|
||||
last[1] = 1;
|
||||
}
|
||||
let phase2_calls: Vec<[u32; 5]> = calls[plan.phase1_calls..plan.solid_calls]
|
||||
.iter()
|
||||
.map(|c| [0, 1, c.first, 0, 0])
|
||||
.collect();
|
||||
let range_buffer = storage(bytemuck::cast_slice(&ranges), none);
|
||||
let phase2_buffer = storage(
|
||||
bytemuck::cast_slice(&[phase2.as_slice(), &[[0; 5]]].concat()),
|
||||
none,
|
||||
);
|
||||
let calls_buffer = storage(
|
||||
bytemuck::cast_slice(&[phase2_calls.as_slice(), &[[0; 5]]].concat()),
|
||||
wgpu::BufferUsages::COPY_SRC,
|
||||
);
|
||||
let out = storage(&vec![0; plan.total as usize], wgpu::BufferUsages::COPY_SRC);
|
||||
let mut encoder = device.create_command_encoder(&Default::default());
|
||||
{
|
||||
let mut pass = encoder.begin_compute_pass(&Default::default());
|
||||
for &(source, first, count) in &segments {
|
||||
let ranges = wgpu::BindingResource::Buffer(wgpu::BufferBinding {
|
||||
buffer: &range_buffer,
|
||||
offset: first * 16,
|
||||
size: wgpu::BufferSize::new(count * 16),
|
||||
});
|
||||
let entry = |binding, resource| wgpu::BindGroupEntry { binding, resource };
|
||||
let (pipeline, entries) = match source {
|
||||
Source::Seen => (
|
||||
&flatten_seen,
|
||||
vec![
|
||||
entry(0, ranges),
|
||||
entry(2, out.as_entire_binding()),
|
||||
entry(3, phase2_buffer.as_entire_binding()),
|
||||
entry(4, calls_buffer.as_entire_binding()),
|
||||
],
|
||||
),
|
||||
Source::Quads | Source::Sorted(_) => {
|
||||
let page = match source {
|
||||
Source::Sorted(p) => p as usize,
|
||||
_ => 2,
|
||||
};
|
||||
(
|
||||
&flatten,
|
||||
vec![
|
||||
entry(0, ranges),
|
||||
entry(1, sorted_buffers[page].as_entire_binding()),
|
||||
entry(2, out.as_entire_binding()),
|
||||
],
|
||||
)
|
||||
}
|
||||
};
|
||||
let group = device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: None,
|
||||
layout: &pipeline.get_bind_group_layout(0),
|
||||
entries: &entries,
|
||||
});
|
||||
pass.set_pipeline(pipeline);
|
||||
pass.set_bind_group(0, &group, &[]);
|
||||
pass.dispatch_workgroups(count as u32, 1, 1);
|
||||
}
|
||||
}
|
||||
queue.submit([encoder.finish()]);
|
||||
let got = read_back(&out, plan.total);
|
||||
// Phase 1's and the translucent calls: each of their draws, in order.
|
||||
let expected: Vec<u32> = (0..draws.len())
|
||||
.filter(|i| !(phase1..solid).contains(i))
|
||||
.flat_map(|i| indices(&draws[i], pages[i].1))
|
||||
.collect();
|
||||
let drawn: Vec<u32> = calls
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(c, _)| !(plan.phase1_calls..plan.solid_calls).contains(c))
|
||||
.flat_map(|(_, c)| got[c.first as usize..(c.first + c.count) as usize].to_vec())
|
||||
.collect();
|
||||
assert_eq!(drawn, expected);
|
||||
// Phase 2: page 1's run drew nothing; page 0's holds the fourth
|
||||
// draw's indices.
|
||||
if phase1 < solid {
|
||||
let counts = read_back(&calls_buffer, phase2_calls.len() as u64 * 5);
|
||||
assert_eq!(counts[0], 0);
|
||||
assert_eq!(counts[5], draws[3][0]);
|
||||
let run = &calls[plan.phase1_calls + 1];
|
||||
assert_eq!(
|
||||
got[run.first as usize..(run.first + draws[3][0]) as usize],
|
||||
indices(&draws[3], None)
|
||||
);
|
||||
}
|
||||
}
|
||||
encoder.copy_buffer_to_buffer(&out, 0, &read, 0, total * 4);
|
||||
queue.submit([encoder.finish()]);
|
||||
read.slice(..).map_async(wgpu::MapMode::Read, |_| {});
|
||||
device.poll(wgpu::PollType::wait_indefinitely()).unwrap();
|
||||
let got: Vec<u32> =
|
||||
bytemuck::cast_slice(&read.slice(..).get_mapped_range().unwrap()).to_vec();
|
||||
assert_eq!(got, expected);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,9 +4,11 @@
|
||||
// translucent ones read their sorted indices.
|
||||
struct Range {
|
||||
count: u32,
|
||||
// First sorted index, or QUADS for a solid draw.
|
||||
// First sorted index, or QUADS for a solid draw (`flatten_seen`: its
|
||||
// entry in occlusion culling's phase 2 draws).
|
||||
first: u32,
|
||||
base: u32,
|
||||
// First index in the list (`flatten_seen`: its draw call).
|
||||
dst: u32,
|
||||
}
|
||||
const QUADS: u32 = 0xffffffffu;
|
||||
@@ -14,6 +16,13 @@ const GROUP: u32 = 64u;
|
||||
@group(0) @binding(0) var<storage, read> ranges: array<Range>;
|
||||
@group(0) @binding(1) var<storage, read> sorted: array<u32>;
|
||||
@group(0) @binding(2) var<storage, read_write> out: array<u32>;
|
||||
|
||||
// Index `k` of a solid draw: quad k / 6's vertices 0 1 2 0 2 3.
|
||||
fn quad(k: u32) -> u32 {
|
||||
let c = k % 6u;
|
||||
return (k / 6u) * 4u + select(select(c - 2u, 0u, c == 3u), c, c < 3u);
|
||||
}
|
||||
|
||||
@compute @workgroup_size(64)
|
||||
fn flatten(@builtin(workgroup_id) group: vec3<u32>,
|
||||
@builtin(local_invocation_index) lane: u32) {
|
||||
@@ -25,11 +34,43 @@ fn flatten(@builtin(workgroup_id) group: vec3<u32>,
|
||||
for (var k = lane; k < d.count; k += GROUP) {
|
||||
var i: u32;
|
||||
if d.first == QUADS {
|
||||
let c = k % 6u;
|
||||
i = (k / 6u) * 4u + select(select(c - 2u, 0u, c == 3u), c, c < 3u);
|
||||
i = quad(k);
|
||||
} else {
|
||||
i = sorted[d.first + k];
|
||||
}
|
||||
out[d.dst + k] = d.base + i;
|
||||
}
|
||||
}
|
||||
|
||||
// Occlusion culling's phase 2 (gfx/cull.rs), after its test: the draws as
|
||||
// the test left them (`DrawIndexedIndirectArgs`, the instance count 1 where
|
||||
// it saw the section), and phase 2's draw calls, whose index counts start
|
||||
// at 0. A draw the test saw takes the next indices of its call's run.
|
||||
@group(0) @binding(3) var<storage, read> draws: array<u32>;
|
||||
@group(0) @binding(4) var<storage, read_write> calls: array<atomic<u32>>;
|
||||
const UNSEEN: u32 = 0xffffffffu;
|
||||
var<workgroup> start: u32;
|
||||
|
||||
@compute @workgroup_size(64)
|
||||
fn flatten_seen(@builtin(workgroup_id) group: vec3<u32>,
|
||||
@builtin(local_invocation_index) lane: u32) {
|
||||
let r = group.y * 65535u + group.x;
|
||||
if r >= arrayLength(&ranges) {
|
||||
return;
|
||||
}
|
||||
let d = ranges[r];
|
||||
if lane == 0u {
|
||||
var s = UNSEEN;
|
||||
if draws[d.first * 5u + 1u] != 0u {
|
||||
s = atomicLoad(&calls[d.dst * 5u + 2u]) + atomicAdd(&calls[d.dst * 5u], d.count);
|
||||
}
|
||||
start = s;
|
||||
}
|
||||
let s = workgroupUniformLoad(&start);
|
||||
if s == UNSEEN {
|
||||
return;
|
||||
}
|
||||
for (var k = lane; k < d.count; k += GROUP) {
|
||||
out[s + k] = d.base + quad(k);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user