Merge pull request #1434 from ludovic111/perf/metal-cull-flat

🤖 Agent: merged after verify and CI parity passed. Metal GPU occlusion culling: on the M4 Pro at RD 32, FPS is +8% to +48% (interleaved A/B against main); the hidden sections match Vulkan's; parity --changed passed 104/104.
This commit is contained in:
Ludovic Marie
2026-10-23 16:03:14 +02:00
committed by GitHub
5 changed files with 824 additions and 233 deletions
+115 -57
View File
@@ -83,6 +83,40 @@ mod world_post;
const COMPLETE_COLUMN: u32 = (1 << (crate::chunk::HEIGHT / 16)) - 1;
/// A section's instance: its camera-relative origin and, in `w`, the step of
/// its fixed-point UVs (`terrain.wgsl`'s `terrain_uv`).
/// The world pass again after occlusion culling's test: it loads both
/// targets (phase 2 draws over phase 1, reading the test's results).
fn world_pass_after_culling<'e>(
encoder: &'e mut wgpu::CommandEncoder,
view: &wgpu::TextureView,
depth: &wgpu::TextureView,
pack: bool,
) -> wgpu::RenderPass<'e> {
encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("Horizon world, after culling"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
view,
resolve_target: None,
depth_slice: None,
ops: wgpu::Operations {
load: wgpu::LoadOp::Load,
store: wgpu::StoreOp::Store,
},
})],
depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment {
view: depth,
depth_ops: Some(wgpu::Operations {
load: wgpu::LoadOp::Load,
store: if pack {
wgpu::StoreOp::Store
} else {
wgpu::StoreOp::Discard
},
}),
stencil_ops: None,
}),
..Default::default()
})
}
fn section_instance(origin: [f32; 3]) -> [f32; 4] {
[
origin[0],
@@ -316,7 +350,8 @@ pub struct Renderer {
flat: Option<flat::Flat>,
/// `flat` draws this frame's sections (off: the per-section draws).
flat_on: bool,
/// Occlusion culling of solid section draws, where multi-draw is native.
/// Occlusion culling of solid section draws, where the GPU's instance
/// counts reach a multi-draw's entries (Vulkan, Metal).
cull: Option<cull::Cull>,
cull_on: bool,
sky: wgpu::RenderPipeline,
@@ -1455,7 +1490,6 @@ impl Renderer {
usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
let depth = Self::depth(&device, &config);
let (atlas_pixels, atlas_width, atlas_height) = hud_atlas;
let atlas = device.create_texture_with_data(
&queue,
@@ -2211,14 +2245,25 @@ impl Renderer {
.features()
.contains(wgpu::Features::INDIRECT_FIRST_INSTANCE);
// HORIZON_GPU_CULL=0 turns occlusion culling off, `all` culls every
// frame however little it hides (A/B checks).
// frame however few its draws and little it hides, `shader` tests in
// the shader as on Metal (A/B checks).
let cull_env = std::env::var("HORIZON_GPU_CULL");
let cull = (cull_env.as_deref() != Ok("0") && cull::Cull::supported(&adapter, &device))
.then(|| {
let mut cull = cull::Cull::new(&device, config.format.remove_srgb_suffix());
let cull = cull::Cull::supported(&adapter, &device)
.filter(|_| cull_env.as_deref() != Ok("0"))
.map(|depth_test| match cull_env.as_deref() {
Ok("shader") => cull::DepthTest::InShader,
_ => depth_test,
})
.map(|depth_test| {
let mut cull =
cull::Cull::new(&device, config.format.remove_srgb_suffix(), depth_test);
cull.always = cull_env.as_deref() == Ok("all");
if !cull.always && adapter.get_info().backend == wgpu::Backend::Metal {
cull.min_draws = cull::TILED_MIN_DRAWS;
}
cull
});
let depth = Self::depth(&device, &config, cull.as_ref());
let world_post = world_post::Post::new(&device, config.format.remove_srgb_suffix());
let uploads = upload::Uploads::new(&device);
let section_uploads = upload::Uploads::with_chunk(&device, SECTION_UPLOAD_CHUNK);
@@ -2431,7 +2476,13 @@ impl Renderer {
self.surface = Some(surface);
Ok(())
}
fn depth(device: &wgpu::Device, config: &wgpu::SurfaceConfiguration) -> wgpu::TextureView {
/// The world's depth, which occlusion culling's test may read.
fn depth(
device: &wgpu::Device,
config: &wgpu::SurfaceConfiguration,
cull: Option<&cull::Cull>,
) -> wgpu::TextureView {
let read = cull.is_some_and(|c| c.depth_test() == cull::DepthTest::InShader);
device
.create_texture(&wgpu::TextureDescriptor {
label: Some("Horizon depth"),
@@ -2444,7 +2495,11 @@ impl Renderer {
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Depth32Float,
usage: wgpu::TextureUsages::RENDER_ATTACHMENT,
usage: if read {
wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING
} else {
wgpu::TextureUsages::RENDER_ATTACHMENT
},
view_formats: &[],
})
.create_view(&Default::default())
@@ -2495,7 +2550,7 @@ impl Renderer {
} else {
self.offscreen = Some(Self::offscreen_target(&self.device, &self.config));
}
self.depth = Self::depth(&self.device, &self.config);
self.depth = Self::depth(&self.device, &self.config, self.cull.as_ref());
}
/// Whether sections are drawn as one index list per frame (`flat`).
pub fn flat_draws(&self) -> bool {
@@ -3215,15 +3270,13 @@ impl Renderer {
.inverse()
.transform_point3(Vec3::ZERO);
// Occlusion culling leaves the sections its last result did not see
// to phase 2, after the others, with instance counts of 0.
// to phase 2, after the others, with instance counts of 0. Whether
// the frame has draws enough goes by the last one's.
let mut cull = self
.cull
.as_mut()
.filter(|_| {
self.cull_on
&& !frame.title
&& self.multi_draw
&& !(self.flat_on && self.flat.is_some())
.filter(|c| {
self.cull_on && !frame.title && self.multi_draw && self.section_draws >= c.min_draws
})
.and_then(|cull| cull.begin(&self.device).then_some(cull));
self.phase2_draws.clear();
@@ -3610,6 +3663,16 @@ impl Renderer {
frame.hud,
[self.config.width, self.config.height],
);
if culling && let Some(cull) = &mut self.cull {
cull.upload(
&self.device,
&mut self.uploads,
&mut encoder,
&self.draws[phase1_draws as usize..solid_draws as usize],
&self.instances,
&self.globals,
);
}
let flat_frame = match &mut self.flat {
Some(flat)
if self.flat_on && !self.draws.is_empty() && self.draws.len() >= flat.min_draws =>
@@ -3617,6 +3680,7 @@ impl Renderer {
let sections = &self.sections;
let lods = &self.lods;
let sorted = &self.sorted_indices;
let phase2 = self.cull.as_ref().filter(|_| culling).map(|c| &c.phase2);
crate::spans::time("flatten", || {
flat.prepare(
&self.device,
@@ -3624,7 +3688,9 @@ impl Renderer {
&mut self.uploads,
&self.draws,
&self.draw_pages,
phase1_draws as usize,
solid_draws as usize,
phase2,
visible
.iter()
.filter_map(|v| {
@@ -3659,16 +3725,6 @@ impl Renderer {
0,
bytemuck::cast_slice(&self.draws),
);
if culling && let Some(cull) = &mut self.cull {
cull.upload(
&self.device,
&mut self.uploads,
&mut encoder,
&self.draws[phase1_draws as usize..solid_draws as usize],
&self.instances,
&self.globals,
);
}
}
let view_target = target_texture.create_view(&wgpu::TextureViewDescriptor {
format: Some(self.config.format.remove_srgb_suffix()),
@@ -3720,6 +3776,9 @@ impl Renderer {
Some((color, depth)) => (color, depth),
None => (&view_target, &self.depth),
};
if culling && let Some(cull) = &mut self.cull {
cull.bind_depth(&self.device, world_depth);
}
let cull = self.cull.as_ref().filter(|_| culling && solid_draws > 0);
if shadow_cfg.is_some() {
let shadow_view = self.world_post.shadow_depth().clone();
@@ -3830,9 +3889,28 @@ impl Renderer {
match self.flat.as_ref().filter(|_| flat_frame) {
Some(flat) => {
pass.set_pipeline(&flat.solid);
flat.draw(&mut pass, &flat.calls[..flat.solid_calls], |p| {
self.arena.buffer(p)
});
let vertices = |p| self.arena.buffer(p);
match cull {
// As below: phase 1, the test, then phase 2's draws
// that the test saw, copied after it.
Some(cull) => {
flat.draw(&mut pass, &flat.calls[..flat.phase1_calls], vertices);
cull.test(&mut pass);
drop(pass);
cull.test_pass(&mut encoder);
flat.flatten_phase2(&mut encoder);
pass = world_pass_after_culling(
&mut encoder,
world_view,
world_depth,
pack,
);
pass.set_bind_group(0, &self.group, &[]);
pass.set_pipeline(&flat.solid);
flat.draw_phase2(&mut pass, vertices);
}
None => flat.draw(&mut pass, &flat.calls[..flat.solid_calls], vertices),
}
pass.set_bind_group(0, &self.group, &[]);
pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32);
}
@@ -3842,40 +3920,20 @@ impl Renderer {
pass.set_vertex_buffer(1, self.instances.slice(..));
match cull {
Some(cull) => {
// Phase 1 and the boxes against its depth, then
// Phase 1 and the boxes against its depth (in
// its pass, or one of their own on Metal), then
// what they found in sight, in a pass that loads
// both targets (it reads the test's results).
self.draw_sections(&mut pass, &self.indirect, 0, 0, phase1_draws);
cull.test(&mut pass);
drop(pass);
let attachments = [Some(wgpu::RenderPassColorAttachment {
view: world_view,
resolve_target: None,
depth_slice: None,
ops: wgpu::Operations {
load: wgpu::LoadOp::Load,
store: wgpu::StoreOp::Store,
},
})];
pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("Horizon world, after culling"),
color_attachments: &attachments,
depth_stencil_attachment: Some(
wgpu::RenderPassDepthStencilAttachment {
view: world_depth,
depth_ops: Some(wgpu::Operations {
load: wgpu::LoadOp::Load,
store: if pack {
wgpu::StoreOp::Store
} else {
wgpu::StoreOp::Discard
},
}),
stencil_ops: None,
},
),
..Default::default()
});
cull.test_pass(&mut encoder);
pass = world_pass_after_culling(
&mut encoder,
world_view,
world_depth,
pack,
);
pass.set_bind_group(0, &self.group, &[]);
pass.set_pipeline(&self.solid);
pass.set_index_buffer(
+190 -33
View File
@@ -22,8 +22,16 @@
//! Every solid draw is an entry of one of the two lists, so the CPU records
//! as many draws as without culling: drivers that unroll an indirect
//! multi-draw on the CPU (Mesa's on Intel Gen9) pay per entry. Phase 2's
//! counts come from the GPU, which needs a native multi-draw and forced
//! early depth tests: Vulkan. Translucent draws are not culled.
//! counts come from the GPU, which reads them from the draws themselves:
//! Vulkan's native multi-draw, and Metal's, which wgpu replays as one
//! indirect draw per entry. Translucent draws are not culled.
//!
//! Metal cannot force the depth test before a fragment shader that writes
//! storage (naga writes no `[[early_fragment_tests]]`), so there the boxes
//! are drawn in a pass of their own between the phases, which reads phase
//! 1's depth as a texture and makes the same comparison in the shader: the
//! fragment's depth against the stored one, greater or equal passing. The
//! same fragments pass, so the same sections are seen.
use std::num::NonZeroU64;
use std::sync::{
Arc,
@@ -38,6 +46,13 @@ const RETEST: u32 = 8;
/// more than they save. Else one frame in `PROBE` tests every section.
const MIN_SHARE: f32 = 0.1;
const PROBE: u32 = 64;
/// On Metal, section draws from which a frame culls. Apple's GPUs store and
/// reload the whole frame at each of the passes the test adds, a fixed cost
/// that the sections it hides repay only at high render distances: on an M4
/// Pro at 1080p and render distance 12 (1,000–1,600 draws) culling added
/// about 0.05 ms of GPU time while hiding 15–30% of the solid indices; at
/// 32 (drawn flat, over this many draws) it saved 0.25–1.6 ms.
pub const TILED_MIN_DRAWS: usize = 4096;
/// Boxes over more than this share of the screen count as seen untested.
const BIG: f32 = 0.25;
/// Results read back at once, some frames late so the CPU never waits.
@@ -74,10 +89,40 @@ struct Group {
group: wgpu::BindGroup,
}
/// Where the boxes' depth test runs.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DepthTest {
/// Forced before the fragment shader, in phase 1's pass (Vulkan).
Early,
/// In the fragment shader, against phase 1's depth read as a texture,
/// in a pass of its own (Metal).
InShader,
}
/// The Vulkan entry point: the depth test forced early, which naga cannot
/// write for Metal (so it is not in `cull.wgsl`, which every backend takes).
const EARLY_TEST: &str = "
@fragment
@early_depth_test(force)
fn fs_box_early(in: Box) -> @location(0) vec4<f32> {
see(in.test);
return vec4(0.0);
}
";
/// The test's depth texture binding (`InShader`), for the view it was made with.
struct DepthGroup {
view: wgpu::TextureView,
group: wgpu::BindGroup,
}
pub struct Cull {
pipeline: wgpu::RenderPipeline,
layout: wgpu::BindGroupLayout,
group: Option<Group>,
depth_test: DepthTest,
depth_layout: Option<wgpu::BindGroupLayout>,
depth_group: Option<DepthGroup>,
params: wgpu::Buffer,
/// Phase 2's draws: the solid draws after phase 1's in the frame's list.
pub phase2: wgpu::Buffer,
@@ -103,6 +148,8 @@ pub struct Cull {
pub hidden: ([usize; 2], f32),
/// Cull every frame, whatever the hidden share (A/B checks).
pub always: bool,
/// Frames with fewer section draws than this do not cull.
pub min_draws: usize,
}
fn buffer(
@@ -134,20 +181,33 @@ impl Cull {
pub fn features(adapter: &wgpu::Adapter) -> wgpu::Features {
adapter.features() & wgpu::Features::SHADER_EARLY_DEPTH_TEST
}
/// Whether the device draws an indirect multi-draw natively and runs
/// fragment shaders that write storage only past an early depth test.
pub fn supported(adapter: &wgpu::Adapter, device: &wgpu::Device) -> bool {
adapter.get_info().backend == wgpu::Backend::Vulkan
&& device.features().contains(
wgpu::Features::SHADER_EARLY_DEPTH_TEST | wgpu::Features::INDIRECT_FIRST_INSTANCE,
)
/// Where the device can run the test, if it can: Vulkan with early
/// depth tests, or Metal, both drawing an indirect multi-draw's entries
/// with the instance counts the GPU wrote, with fragment shaders that
/// write storage.
pub fn supported(adapter: &wgpu::Adapter, device: &wgpu::Device) -> Option<DepthTest> {
let depth_test = match adapter.get_info().backend {
wgpu::Backend::Vulkan
if device
.features()
.contains(wgpu::Features::SHADER_EARLY_DEPTH_TEST) =>
{
DepthTest::Early
}
wgpu::Backend::Metal => DepthTest::InShader,
_ => return None,
};
(device
.features()
.contains(wgpu::Features::INDIRECT_FIRST_INSTANCE)
&& adapter.get_downlevel_capabilities().flags.contains(
wgpu::DownlevelFlags::VERTEX_STORAGE
| wgpu::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE,
)
))
.then_some(depth_test)
}
/// `color` is the world pass's colour format.
pub fn new(device: &wgpu::Device, color: wgpu::TextureFormat) -> Self {
pub fn new(device: &wgpu::Device, color: wgpu::TextureFormat, depth_test: DepthTest) -> Self {
let entry = |binding, visibility, ty| wgpu::BindGroupLayoutEntry {
binding,
visibility,
@@ -180,13 +240,41 @@ impl Cull {
});
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("Section culling"),
source: wgpu::ShaderSource::Wgsl(include_str!("cull.wgsl").into()),
source: wgpu::ShaderSource::Wgsl(source(depth_test).into()),
});
let depth_layout = (depth_test == DepthTest::InShader).then(|| {
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("Section culling depth"),
entries: &[entry(
0,
fragment,
wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Depth,
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
)],
})
});
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("Section culling"),
bind_group_layouts: &[Some(&layout)],
bind_group_layouts: &[Some(&layout), depth_layout.as_ref()],
..Default::default()
});
// In the shader: no colour target, and phase 1's depth attached read
// only (a pass needs an attachment), its test left to the shader.
let (fragment_entry, targets, depth_compare) = match depth_test {
DepthTest::Early => (
"fs_box_early",
&[Some(wgpu::ColorTargetState {
format: color,
blend: None,
write_mask: wgpu::ColorWrites::empty(),
})][..],
wgpu::CompareFunction::GreaterEqual,
),
DepthTest::InShader => ("fs_box", &[][..], wgpu::CompareFunction::Always),
};
let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("Section culling"),
layout: Some(&pipeline_layout),
@@ -204,20 +292,16 @@ impl Cull {
depth_stencil: Some(wgpu::DepthStencilState {
format: wgpu::TextureFormat::Depth32Float,
depth_write_enabled: Some(false),
depth_compare: Some(wgpu::CompareFunction::GreaterEqual),
depth_compare: Some(depth_compare),
stencil: Default::default(),
bias: Default::default(),
}),
multisample: Default::default(),
fragment: Some(wgpu::FragmentState {
module: &module,
entry_point: Some("fs_box"),
entry_point: Some(fragment_entry),
compilation_options: Default::default(),
targets: &[Some(wgpu::ColorTargetState {
format: color,
blend: None,
write_mask: wgpu::ColorWrites::empty(),
})],
targets,
}),
multiview_mask: None,
cache: None,
@@ -226,6 +310,9 @@ impl Cull {
pipeline,
layout,
group: None,
depth_test,
depth_layout,
depth_group: None,
params: buffer(
device,
"Section culling parameters",
@@ -259,6 +346,7 @@ impl Cull {
stamp: 0,
hidden: ([0; 2], 1.0),
always: false,
min_draws: 0,
}
}
@@ -399,17 +487,69 @@ impl Cull {
}
}
/// In the world pass after phase 1: draw the tested boxes against its
/// depth. Leaves the pass's pipeline and group 0 changed.
pub fn depth_test(&self) -> DepthTest {
self.depth_test
}
/// In the world pass after phase 1 (`Early`): draw the tested boxes
/// against its depth. Leaves the pass's pipeline and group 0 changed.
pub fn test(&self, pass: &mut wgpu::RenderPass<'_>) {
let Some(Group { group, .. }) = &self.group else {
return;
};
if self.depth_test != DepthTest::Early || self.test_data.is_empty() {
return;
}
pass.set_pipeline(&self.pipeline);
pass.set_bind_group(0, group, &[]);
pass.draw(0..36, 0..self.test_data.len() as u32);
}
/// Before the world pass (`InShader`): the binding of the depth the
/// world is drawn with this frame.
pub fn bind_depth(&mut self, device: &wgpu::Device, depth: &wgpu::TextureView) {
let Some(layout) = &self.depth_layout else {
return;
};
if self.depth_group.as_ref().is_none_or(|g| g.view != *depth) {
self.depth_group = Some(DepthGroup {
view: depth.clone(),
group: device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("Section culling depth"),
layout,
entries: &[wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::TextureView(depth),
}],
}),
});
}
}
/// Between the phases (`InShader`): draw the tested boxes in a pass of
/// their own that reads phase 1's depth, bound by `bind_depth`.
pub fn test_pass(&self, encoder: &mut wgpu::CommandEncoder) {
let (Some(Group { group, .. }), Some(DepthGroup { view, group: depth })) =
(&self.group, &self.depth_group)
else {
return;
};
if self.test_data.is_empty() {
return;
}
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("Section culling"),
color_attachments: &[],
depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment {
view,
depth_ops: None,
stencil_ops: None,
}),
..Default::default()
});
pass.set_pipeline(&self.pipeline);
pass.set_bind_group(0, group, &[]);
pass.set_bind_group(1, depth, &[]);
pass.draw(0..36, 0..self.test_data.len() as u32);
}
@@ -498,16 +638,33 @@ impl Cull {
}
}
#[cfg(test)]
mod tests {
#[test]
fn shader_validates() {
let module = naga::front::wgsl::parse_str(include_str!("cull.wgsl")).unwrap();
naga::valid::Validator::new(
naga::valid::ValidationFlags::all(),
naga::valid::Capabilities::all(),
)
.validate(&module)
.unwrap();
/// The test's shader for `depth_test`.
fn source(depth_test: DepthTest) -> String {
let common = include_str!("cull.wgsl");
match depth_test {
DepthTest::Early => format!("{common}{EARLY_TEST}"),
DepthTest::InShader => common.to_owned(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn shaders_validate() {
for (depth_test, capabilities) in [
(DepthTest::Early, naga::valid::Capabilities::all()),
// What wgpu's Metal backend allows: no early depth test.
(
DepthTest::InShader,
naga::valid::Capabilities::all() - naga::valid::Capabilities::EARLY_DEPTH_TEST,
),
] {
let module = naga::front::wgsl::parse_str(&source(depth_test)).unwrap();
naga::valid::Validator::new(naga::valid::ValidationFlags::all(), capabilities)
.validate(&module)
.unwrap_or_else(|e| panic!("{depth_test:?}: {e:?}"));
}
}
}
+20 -8
View File
@@ -86,16 +86,28 @@ fn vs_box(@builtin(vertex_index) v: u32, @builtin(instance_index) t: u32) -> Box
return out;
}
// Runs only for fragments that pass the depth test (forced early).
@fragment
@early_depth_test(force)
fn fs_box(in: Box) -> @location(0) vec4<f32> {
if results[in.test] != params.stamp {
results[in.test] = params.stamp;
let test = tests[in.test];
// The box is seen at this fragment: store the stamp, and give its phase 2
// draws their instance.
fn see(t: u32) {
if results[t] != params.stamp {
results[t] = params.stamp;
let test = tests[t];
for (var k = 0u; k < test.z; k++) {
phase2[(test.y + k) * 5u + 1u] = 1u;
}
}
return vec4(0.0);
}
// Phase 1's depth, where the test runs in the shader (gfx/cull.rs).
@group(1) @binding(0) var depth: texture_depth_2d;
// The world pass's reversed depth test, greater or equal passing, made here:
// where it cannot be forced before a shader that writes storage, a fragment
// that fails it would still write. Where it can, `fs_box_early` (cull.rs)
// runs only for the fragments that pass it.
@fragment
fn fs_box(in: Box) {
if in.position.z >= textureLoad(depth, vec2<u32>(in.position.xy), 0) {
see(in.test);
}
}
+455 -132
View File
@@ -11,11 +11,37 @@
//! so each block names one section: its id here, which indexes the frame's
//! camera-relative origins (computed on the CPU exactly as for the instance
//! attribute).
//!
//! With occlusion culling (`cull`), phase 1's draws and the translucent ones
//! are copied before the world pass, as without it. Phase 2's draws are
//! copied after the test, which tells which ones it saw: each run of a page
//! reserves room for all of its draws, the copy packs the seen ones at its
//! start, and the run is drawn indirectly with the count the copy reached.
use super::arena::GRAIN;
/// One section draw for `flatten.wgsl`: index count, first sorted index (or
/// `QUADS` for a solid draw), base vertex, first index in the list.
/// `QUADS` for a solid draw; phase 2: its entry in phase 2's draws), base
/// vertex, first index in the list (phase 2: its draw call in phase 2).
type Range = [u32; 4];
/// Where a group of ranges takes its indices.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Source {
/// Solid draws: quads.
Quads,
/// Occlusion culling's phase 2: quads, of the sections its test saw.
Seen,
/// Translucent draws: this sorted index page.
Sorted(u32),
}
/// The frame's draw calls, as `plan` lays them out.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Plan {
phase1_calls: usize,
solid_calls: usize,
total: u64,
}
const QUADS: u32 = u32::MAX;
const GROUP_LIMIT: u32 = 65535;
@@ -38,6 +64,15 @@ pub struct Flat {
pub translucent: wgpu::RenderPipeline,
flatten: wgpu::ComputePipeline,
compute_layout: wgpu::BindGroupLayout,
flatten_seen: wgpu::ComputePipeline,
seen_layout: wgpu::BindGroupLayout,
/// Phase 2's draw calls (`DrawIndexedIndirectArgs`), whose index counts
/// `flatten_seen` makes.
phase2_calls: wgpu::Buffer,
phase2_call_capacity: u64,
phase2_call_data: Vec<[u32; 5]>,
/// The copy of phase 2's draws, and its ranges.
phase2_group: Option<(wgpu::BindGroup, u32)>,
draw_layout: wgpu::BindGroupLayout,
blocks_per_page: u64,
pages: Vec<Option<PageTable>>,
@@ -52,12 +87,14 @@ pub struct Flat {
ranges: wgpu::Buffer,
range_capacity: u64,
range_data: Vec<Range>,
/// Segments of `range_data` by index source: (sorted index page or
/// `None` for solid draws, first range, ranges).
segments: Vec<(Option<u32>, u64, u64)>,
/// Segments of `range_data` by index source: (source, first range,
/// ranges).
segments: Vec<(Source, u64, u64)>,
empty: wgpu::Buffer,
/// This frame's draw calls: solid ones, then translucent ones.
/// This frame's draw calls: solid ones (phase 1's, then phase 2's when
/// culling), then translucent ones.
pub calls: Vec<Call>,
pub phase1_calls: usize,
pub solid_calls: usize,
/// Frames with fewer section draws keep the per-section draws.
pub min_draws: usize,
@@ -121,42 +158,60 @@ fn buffer(
/// Lays out the frame's index list: each draw's range (destinations in draw
/// order) grouped by index source into `ranges`, every group padded to
/// `align` entries for binding, with `segments` naming each group; and the
/// draw calls, one per run of draws in the same vertex page, solid and
/// translucent apart. Returns the first translucent call and the list's
/// length.
/// draw calls, one per run of draws in the same vertex page, phase 1's solid
/// ones, phase 2's (draws `phase1_draws..solid_draws`) and translucent ones
/// apart. A phase 2 range names its draw in phase 2 and its call in phase 2
/// instead of a destination: the call's room holds all of its draws.
fn plan(
draws: &[[u32; 5]],
pages: &[(u32, Option<u32>)],
phase1_draws: usize,
solid_draws: usize,
align: u64,
calls: &mut Vec<Call>,
ranges: &mut Vec<Range>,
segments: &mut Vec<(Option<u32>, u64, u64)>,
) -> (usize, u64) {
segments: &mut Vec<(Source, u64, u64)>,
) -> Plan {
calls.clear();
ranges.clear();
segments.clear();
let mut solid_calls = None;
let (mut phase1_calls, mut solid_calls) = (None, None);
let mut total = 0u64;
let mut by_source: Vec<(Option<u32>, Vec<Range>)> = Vec::new();
let mut by_source: Vec<(Source, Vec<Range>)> = Vec::new();
for (i, (d, &(page, sorted))) in draws.iter().zip(pages).enumerate() {
let first = if sorted.is_some() { d[2] } else { QUADS };
let range = [d[0], first, d[3], total as u32];
match by_source.iter_mut().find(|(s, _)| *s == sorted) {
Some((_, list)) => list.push(range),
None => by_source.push((sorted, vec![range])),
if i == phase1_draws {
phase1_calls.get_or_insert(calls.len());
}
if i == solid_draws {
solid_calls = Some(calls.len());
}
match calls.last_mut() {
Some(last) if last.page == page && i != solid_draws => last.count += d[0],
Some(last) if last.page == page && i != phase1_draws && i != solid_draws => {
last.count += d[0]
}
_ => calls.push(Call {
page,
first: total as u32,
count: d[0],
}),
}
let (source, range) = match sorted {
Some(p) => (Source::Sorted(p), [d[0], d[2], d[3], total as u32]),
None if (phase1_draws..solid_draws).contains(&i) => (
Source::Seen,
[
d[0],
(i - phase1_draws) as u32,
d[3],
(calls.len() - 1 - phase1_calls.unwrap_or(0)) as u32,
],
),
None => (Source::Quads, [d[0], QUADS, d[3], total as u32]),
};
match by_source.iter_mut().find(|(s, _)| *s == source) {
Some((_, list)) => list.push(range),
None => by_source.push((source, vec![range])),
}
total += u64::from(d[0]);
}
for (source, list) in by_source {
@@ -165,7 +220,12 @@ fn plan(
let padded = (ranges.len() as u64).next_multiple_of(align);
ranges.resize(padded as usize, [0; 4]);
}
(solid_calls.unwrap_or(calls.len()), total)
let solid_calls = solid_calls.unwrap_or(calls.len());
Plan {
phase1_calls: phase1_calls.unwrap_or(calls.len()).min(solid_calls),
solid_calls,
total,
}
}
impl Flat {
@@ -201,24 +261,37 @@ impl Flat {
storage(2, compute, false),
],
});
let seen_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("Flatten phase 2 section draws"),
entries: &[
storage(0, compute, true),
storage(2, compute, false),
storage(3, compute, true),
storage(4, compute, false),
],
});
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("Flatten section draws"),
source: wgpu::ShaderSource::Wgsl(include_str!("flatten.wgsl").into()),
});
let flatten = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some("Flatten section draws"),
layout: Some(
&device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("Flatten section draws"),
bind_group_layouts: &[Some(&compute_layout)],
..Default::default()
}),
),
module: &module,
entry_point: Some("flatten"),
compilation_options: Default::default(),
cache: None,
});
let compute_pipeline = |layout: &wgpu::BindGroupLayout, entry| {
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some("Flatten section draws"),
layout: Some(
&device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("Flatten section draws"),
bind_group_layouts: &[Some(layout)],
..Default::default()
}),
),
module: &module,
entry_point: Some(entry),
compilation_options: Default::default(),
cache: None,
})
};
let flatten = compute_pipeline(&compute_layout, "flatten");
let flatten_seen = compute_pipeline(&seen_layout, "flatten_seen");
let storage_usage = wgpu::BufferUsages::STORAGE;
Self {
solid: pipeline(&layout, "Terrain solid (flat)", "fs_solid", None),
@@ -230,6 +303,17 @@ impl Flat {
),
flatten,
compute_layout,
flatten_seen,
seen_layout,
phase2_calls: buffer(
device,
"Flat section draw calls, phase 2",
20,
wgpu::BufferUsages::INDIRECT | storage_usage,
),
phase2_call_capacity: 1,
phase2_call_data: Vec::new(),
phase2_group: None,
draw_layout,
blocks_per_page: page_vertices.div_ceil(GRAIN),
pages: Vec::new(),
@@ -251,6 +335,7 @@ impl Flat {
segments: Vec::new(),
empty: buffer(device, "No sorted indices", 16, storage_usage),
calls: Vec::new(),
phase1_calls: 0,
solid_calls: 0,
min_draws: if std::env::var("HORIZON_FLAT_DRAWS").as_deref() == Ok("1") {
0
@@ -321,6 +406,9 @@ impl Flat {
/// the origins of the sections they draw by id. Records the copy into
/// `encoder`; `calls` then lists the frame's draw calls. False when the
/// frame cannot be drawn flat (the per-section draws then apply).
///
/// When culling, the solid draws from `phase1_draws` are phase 2's, which
/// `phase2` holds as the test leaves them: `flatten_phase2` copies them.
#[allow(clippy::too_many_arguments)]
pub fn prepare(
&mut self,
@@ -329,10 +417,18 @@ impl Flat {
uploads: &mut super::upload::Uploads,
draws: &[[u32; 5]],
pages: &[(u32, Option<u32>)],
phase1_draws: usize,
solid_draws: usize,
phase2: Option<&wgpu::Buffer>,
origins: impl Iterator<Item = (u32, [f32; 4])>,
sorted_page: impl Fn(u32) -> wgpu::Buffer,
) -> bool {
self.phase2_group = None;
let phase1_draws = if phase2.is_some() {
phase1_draws
} else {
solid_draws
};
// Origins by section id.
self.origin_data.clear();
for (id, o) in origins {
@@ -362,18 +458,25 @@ impl Flat {
bytemuck::cast_slice(&self.origin_data),
);
}
let (solid_calls, total) = plan(
let Plan {
phase1_calls,
solid_calls,
total,
} = plan(
draws,
pages,
phase1_draws,
solid_draws,
self.range_align,
&mut self.calls,
&mut self.range_data,
&mut self.segments,
);
self.phase1_calls = phase1_calls;
self.solid_calls = solid_calls;
if total == 0 || total * 4 > device.limits().max_storage_buffer_binding_size {
self.calls.clear();
self.phase1_calls = 0;
self.solid_calls = 0;
return false;
}
@@ -401,22 +504,82 @@ impl Flat {
0,
bytemuck::cast_slice(&self.range_data),
);
// Phase 2's calls start empty, each at the start of its room.
self.phase2_call_data.clear();
self.phase2_call_data.extend(
self.calls[phase1_calls..solid_calls]
.iter()
.map(|c| [0, 1, c.first, 0, 0]),
);
let phase2_calls = self.phase2_call_data.len() as u64;
if phase2_calls > self.phase2_call_capacity {
self.phase2_call_capacity = phase2_calls.next_power_of_two();
self.phase2_calls = buffer(
device,
"Flat section draw calls, phase 2",
self.phase2_call_capacity * 20,
wgpu::BufferUsages::INDIRECT | wgpu::BufferUsages::STORAGE,
);
}
if phase2_calls > 0 {
uploads.buffer(
encoder,
&self.phase2_calls,
0,
bytemuck::cast_slice(&self.phase2_call_data),
);
}
let ranges_at = |first: u64, count: u64| {
wgpu::BindingResource::Buffer(wgpu::BufferBinding {
buffer: &self.ranges,
offset: first * 16,
size: wgpu::BufferSize::new(count * 16),
})
};
if let (Some(phase2), Some(&(_, first, count))) = (
phase2,
self.segments.iter().find(|(s, ..)| *s == Source::Seen),
) {
let group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("Flatten phase 2 section draws"),
layout: &self.seen_layout,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: ranges_at(first, count),
},
wgpu::BindGroupEntry {
binding: 2,
resource: self.indices.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 3,
resource: phase2.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 4,
resource: self.phase2_calls.as_entire_binding(),
},
],
});
self.phase2_group = Some((group, count as u32));
}
let groups: Vec<(wgpu::BindGroup, u32)> = self
.segments
.iter()
.map(|&(source, first, count)| {
let sorted = source.map(&sorted_page);
.filter_map(|&(source, first, count)| {
let sorted = match source {
Source::Quads => None,
Source::Seen => return None,
Source::Sorted(page) => Some(sorted_page(page)),
};
let group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("Flatten section draws"),
layout: &self.compute_layout,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding {
buffer: &self.ranges,
offset: first * 16,
size: wgpu::BufferSize::new(count * 16),
}),
resource: ranges_at(first, count),
},
wgpu::BindGroupEntry {
binding: 1,
@@ -428,7 +591,7 @@ impl Flat {
},
],
});
(group, count as u32)
Some((group, count as u32))
})
.collect();
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
@@ -464,6 +627,19 @@ impl Flat {
}
true
}
/// After occlusion culling's test: copies the phase 2 draws it saw.
pub fn flatten_phase2(&self, encoder: &mut wgpu::CommandEncoder) {
let Some((group, count)) = &self.phase2_group else {
return;
};
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("Flatten phase 2 section draws"),
timestamp_writes: None,
});
pass.set_pipeline(&self.flatten_seen);
pass.set_bind_group(0, group, &[]);
pass.dispatch_workgroups((*count).min(GROUP_LIMIT), count.div_ceil(GROUP_LIMIT), 1);
}
/// Draws `calls` with a flat pipeline set and `vertices` giving each
/// page's vertex buffer.
pub fn draw<'a>(
@@ -474,15 +650,38 @@ impl Flat {
) {
pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32);
for call in calls {
let table = self.pages[call.page as usize]
.as_ref()
.and_then(|t| t.group.as_ref())
.expect("prepared page");
pass.set_vertex_buffer(0, vertices(call.page).slice(..));
pass.set_bind_group(1, table, &[]);
self.bind_page(pass, call.page, &vertices);
pass.draw_indexed(call.first..call.first + call.count, 0, 0..1);
}
}
/// Draws phase 2's calls, as `flatten_phase2` left them.
pub fn draw_phase2<'a>(
&self,
pass: &mut wgpu::RenderPass<'_>,
vertices: impl Fn(u32) -> &'a wgpu::Buffer,
) {
pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32);
for (k, call) in self.calls[self.phase1_calls..self.solid_calls]
.iter()
.enumerate()
{
self.bind_page(pass, call.page, &vertices);
pass.draw_indexed_indirect(&self.phase2_calls, k as u64 * 20);
}
}
fn bind_page<'a>(
&self,
pass: &mut wgpu::RenderPass<'_>,
page: u32,
vertices: &impl Fn(u32) -> &'a wgpu::Buffer,
) {
let table = self.pages[page as usize]
.as_ref()
.and_then(|t| t.group.as_ref())
.expect("prepared page");
pass.set_vertex_buffer(0, vertices(page).slice(..));
pass.set_bind_group(1, table, &[]);
}
}
#[cfg(test)]
@@ -516,16 +715,22 @@ mod tests {
fn plan_draws_runs_of_a_page_in_order() {
let (draws, pages, solid) = sample();
let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new());
let (solid_calls, total) = plan(
let Plan {
phase1_calls,
solid_calls,
total,
} = plan(
&draws,
&pages,
solid,
solid,
4,
&mut calls,
&mut ranges,
&mut segments,
);
assert_eq!(total, 6 * 13);
assert_eq!(phase1_calls, solid_calls);
let call = |page, first, count| Call { page, first, count };
// Pages 0, 0 | 1 | 0, then translucent 0, 0 (never merged with solid).
assert_eq!(
@@ -539,16 +744,65 @@ mod tests {
);
assert_eq!(solid_calls, 3);
// Solid ranges, then each sorted page's, every group padded to 4.
assert_eq!(segments, [(None, 0, 4), (Some(0), 4, 1), (Some(1), 8, 1)]);
assert_eq!(
segments,
[
(Source::Quads, 0, 4),
(Source::Sorted(0), 4, 1),
(Source::Sorted(1), 8, 1)
]
);
assert_eq!(ranges[3], [24, QUADS, 1024, 36]);
assert_eq!(ranges[4], [12, 6, 768, 60]);
assert_eq!(ranges[8], [6, 0, 40, 72]);
assert_eq!(ranges.len(), 12);
}
/// Phase 2's draws (here the third and fourth) get calls of their own,
/// each with room for all of its draws, and ranges naming their draw
/// and call in phase 2.
#[test]
fn plan_gives_phase_2_calls_of_its_own() {
let (draws, pages, solid) = sample();
let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new());
let plan = plan(
&draws,
&pages,
2,
solid,
4,
&mut calls,
&mut ranges,
&mut segments,
);
assert_eq!(
plan,
Plan {
phase1_calls: 1,
solid_calls: 3,
total: 6 * 13
}
);
let call = |page, first, count| Call { page, first, count };
// Phase 1: page 0 | phase 2: page 1, page 0 | translucent: page 0.
assert_eq!(
calls,
[
call(0, 0, 30),
call(1, 30, 6),
call(0, 36, 24),
call(0, 60, 18)
]
);
assert_eq!(segments[1], (Source::Seen, 4, 2));
assert_eq!(ranges[4], [6, 0, 256, 0]);
assert_eq!(ranges[5], [24, 1, 1024, 1]);
}
/// The compute pass writes exactly the indices the per-section draws
/// read: quads 0 1 2 0 2 3 for solid draws, sorted indices otherwise,
/// plus the base vertex.
/// plus the base vertex. With culling, phase 2's runs hold the indices
/// of the draws the test saw, in order, and their calls count them.
#[test]
fn flatten_writes_the_indices_of_the_per_section_draws() {
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle());
@@ -564,101 +818,170 @@ mod tests {
use wgpu::util::DeviceExt;
let (draws, pages, solid) = sample();
let align = (u64::from(device.limits().min_storage_buffer_offset_alignment) / 16).max(1);
let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new());
let (_, total) = plan(
&draws,
&pages,
solid,
align,
&mut calls,
&mut ranges,
&mut segments,
);
let sorted: [Vec<u32>; 2] = [
(0..32).map(|i| (i * 7) % 32).collect(),
(0..8).rev().collect(),
];
let mut expected = Vec::new();
for (d, (_, page)) in draws.iter().zip(&pages) {
for k in 0..d[0] {
let i = match page {
None => (k / 6) * 4 + [0, 1, 2, 0, 2, 3][(k % 6) as usize],
Some(p) => sorted[*p as usize][(d[2] + k) as usize],
};
expected.push(d[3] + i);
}
}
let storage = |entries: &[u32]| {
let indices = |d: &[u32; 5], sorted_page: Option<u32>| -> Vec<u32> {
(0..d[0])
.map(|k| {
d[3] + match sorted_page {
None => (k / 6) * 4 + [0, 1, 2, 0, 2, 3][(k % 6) as usize],
Some(p) => sorted[p as usize][(d[2] + k) as usize],
}
})
.collect()
};
let storage = |entries: &[u32], usage| {
device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: None,
contents: bytemuck::cast_slice(entries),
usage: wgpu::BufferUsages::STORAGE,
usage: wgpu::BufferUsages::STORAGE | usage,
})
};
let range_buffer = storage(bytemuck::cast_slice(&ranges));
let sorted_buffers = [storage(&sorted[0]), storage(&sorted[1]), storage(&[0; 4])];
let out = device.create_buffer(&wgpu::BufferDescriptor {
label: None,
size: total * 4,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
});
let read = device.create_buffer(&wgpu::BufferDescriptor {
label: None,
size: total * 4,
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
let none = wgpu::BufferUsages::empty();
let sorted_buffers = [
storage(&sorted[0], none),
storage(&sorted[1], none),
storage(&[0; 4], none),
];
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: None,
source: wgpu::ShaderSource::Wgsl(include_str!("flatten.wgsl").into()),
});
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: None,
layout: None,
module: &module,
entry_point: Some("flatten"),
compilation_options: Default::default(),
cache: None,
});
let mut encoder = device.create_command_encoder(&Default::default());
{
let mut pass = encoder.begin_compute_pass(&Default::default());
pass.set_pipeline(&pipeline);
for &(source, first, count) in &segments {
let group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: None,
layout: &pipeline.get_bind_group_layout(0),
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding {
buffer: &range_buffer,
offset: first * 16,
size: wgpu::BufferSize::new(count * 16),
}),
},
wgpu::BindGroupEntry {
binding: 1,
resource: sorted_buffers[source.map_or(2, |p| p as usize)]
.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 2,
resource: out.as_entire_binding(),
},
],
});
pass.set_bind_group(0, &group, &[]);
pass.dispatch_workgroups(count as u32, 1, 1);
let pipeline = |entry| {
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: None,
layout: None,
module: &module,
entry_point: Some(entry),
compilation_options: Default::default(),
cache: None,
})
};
let (flatten, flatten_seen) = (pipeline("flatten"), pipeline("flatten_seen"));
let read_back = |buffer: &wgpu::Buffer, words: u64| -> Vec<u32> {
let read = device.create_buffer(&wgpu::BufferDescriptor {
label: None,
size: words * 4,
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
let mut encoder = device.create_command_encoder(&Default::default());
encoder.copy_buffer_to_buffer(buffer, 0, &read, 0, words * 4);
queue.submit([encoder.finish()]);
read.slice(..).map_async(wgpu::MapMode::Read, |_| {});
device.poll(wgpu::PollType::wait_indefinitely()).unwrap();
bytemuck::cast_slice(&read.slice(..).get_mapped_range().unwrap()).to_vec()
};
// Without culling, then with the third and fourth draws in phase 2,
// of which the test saw only the fourth.
for phase1 in [solid, 2] {
let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new());
let plan = plan(
&draws,
&pages,
phase1,
solid,
align,
&mut calls,
&mut ranges,
&mut segments,
);
// Phase 2's draws as the test leaves them: instance counts of 0,
// 1 where it saw the section.
let mut phase2: Vec<[u32; 5]> = draws[phase1..solid]
.iter()
.map(|&[count, _, first, base, instance]| [count, 0, first, base, instance])
.collect();
if let Some(last) = phase2.last_mut() {
last[1] = 1;
}
let phase2_calls: Vec<[u32; 5]> = calls[plan.phase1_calls..plan.solid_calls]
.iter()
.map(|c| [0, 1, c.first, 0, 0])
.collect();
let range_buffer = storage(bytemuck::cast_slice(&ranges), none);
let phase2_buffer = storage(
bytemuck::cast_slice(&[phase2.as_slice(), &[[0; 5]]].concat()),
none,
);
let calls_buffer = storage(
bytemuck::cast_slice(&[phase2_calls.as_slice(), &[[0; 5]]].concat()),
wgpu::BufferUsages::COPY_SRC,
);
let out = storage(&vec![0; plan.total as usize], wgpu::BufferUsages::COPY_SRC);
let mut encoder = device.create_command_encoder(&Default::default());
{
let mut pass = encoder.begin_compute_pass(&Default::default());
for &(source, first, count) in &segments {
let ranges = wgpu::BindingResource::Buffer(wgpu::BufferBinding {
buffer: &range_buffer,
offset: first * 16,
size: wgpu::BufferSize::new(count * 16),
});
let entry = |binding, resource| wgpu::BindGroupEntry { binding, resource };
let (pipeline, entries) = match source {
Source::Seen => (
&flatten_seen,
vec![
entry(0, ranges),
entry(2, out.as_entire_binding()),
entry(3, phase2_buffer.as_entire_binding()),
entry(4, calls_buffer.as_entire_binding()),
],
),
Source::Quads | Source::Sorted(_) => {
let page = match source {
Source::Sorted(p) => p as usize,
_ => 2,
};
(
&flatten,
vec![
entry(0, ranges),
entry(1, sorted_buffers[page].as_entire_binding()),
entry(2, out.as_entire_binding()),
],
)
}
};
let group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: None,
layout: &pipeline.get_bind_group_layout(0),
entries: &entries,
});
pass.set_pipeline(pipeline);
pass.set_bind_group(0, &group, &[]);
pass.dispatch_workgroups(count as u32, 1, 1);
}
}
queue.submit([encoder.finish()]);
let got = read_back(&out, plan.total);
// Phase 1's and the translucent calls: each of their draws, in order.
let expected: Vec<u32> = (0..draws.len())
.filter(|i| !(phase1..solid).contains(i))
.flat_map(|i| indices(&draws[i], pages[i].1))
.collect();
let drawn: Vec<u32> = calls
.iter()
.enumerate()
.filter(|(c, _)| !(plan.phase1_calls..plan.solid_calls).contains(c))
.flat_map(|(_, c)| got[c.first as usize..(c.first + c.count) as usize].to_vec())
.collect();
assert_eq!(drawn, expected);
// Phase 2: page 1's run drew nothing; page 0's holds the fourth
// draw's indices.
if phase1 < solid {
let counts = read_back(&calls_buffer, phase2_calls.len() as u64 * 5);
assert_eq!(counts[0], 0);
assert_eq!(counts[5], draws[3][0]);
let run = &calls[plan.phase1_calls + 1];
assert_eq!(
got[run.first as usize..(run.first + draws[3][0]) as usize],
indices(&draws[3], None)
);
}
}
encoder.copy_buffer_to_buffer(&out, 0, &read, 0, total * 4);
queue.submit([encoder.finish()]);
read.slice(..).map_async(wgpu::MapMode::Read, |_| {});
device.poll(wgpu::PollType::wait_indefinitely()).unwrap();
let got: Vec<u32> =
bytemuck::cast_slice(&read.slice(..).get_mapped_range().unwrap()).to_vec();
assert_eq!(got, expected);
}
}
+44 -3
View File
@@ -4,9 +4,11 @@
// translucent ones read their sorted indices.
struct Range {
count: u32,
// First sorted index, or QUADS for a solid draw.
// First sorted index, or QUADS for a solid draw (`flatten_seen`: its
// entry in occlusion culling's phase 2 draws).
first: u32,
base: u32,
// First index in the list (`flatten_seen`: its draw call).
dst: u32,
}
const QUADS: u32 = 0xffffffffu;
@@ -14,6 +16,13 @@ const GROUP: u32 = 64u;
@group(0) @binding(0) var<storage, read> ranges: array<Range>;
@group(0) @binding(1) var<storage, read> sorted: array<u32>;
@group(0) @binding(2) var<storage, read_write> out: array<u32>;
// Index `k` of a solid draw: quad k / 6's vertices 0 1 2 0 2 3.
fn quad(k: u32) -> u32 {
let c = k % 6u;
return (k / 6u) * 4u + select(select(c - 2u, 0u, c == 3u), c, c < 3u);
}
@compute @workgroup_size(64)
fn flatten(@builtin(workgroup_id) group: vec3<u32>,
@builtin(local_invocation_index) lane: u32) {
@@ -25,11 +34,43 @@ fn flatten(@builtin(workgroup_id) group: vec3<u32>,
for (var k = lane; k < d.count; k += GROUP) {
var i: u32;
if d.first == QUADS {
let c = k % 6u;
i = (k / 6u) * 4u + select(select(c - 2u, 0u, c == 3u), c, c < 3u);
i = quad(k);
} else {
i = sorted[d.first + k];
}
out[d.dst + k] = d.base + i;
}
}
// Occlusion culling's phase 2 (gfx/cull.rs), after its test: the draws as
// the test left them (`DrawIndexedIndirectArgs`, the instance count 1 where
// it saw the section), and phase 2's draw calls, whose index counts start
// at 0. A draw the test saw takes the next indices of its call's run.
@group(0) @binding(3) var<storage, read> draws: array<u32>;
@group(0) @binding(4) var<storage, read_write> calls: array<atomic<u32>>;
const UNSEEN: u32 = 0xffffffffu;
var<workgroup> start: u32;
@compute @workgroup_size(64)
fn flatten_seen(@builtin(workgroup_id) group: vec3<u32>,
@builtin(local_invocation_index) lane: u32) {
let r = group.y * 65535u + group.x;
if r >= arrayLength(&ranges) {
return;
}
let d = ranges[r];
if lane == 0u {
var s = UNSEEN;
if draws[d.first * 5u + 1u] != 0u {
s = atomicLoad(&calls[d.dst * 5u + 2u]) + atomicAdd(&calls[d.dst * 5u], d.count);
}
start = s;
}
let s = workgroupUniformLoad(&start);
if s == UNSEEN {
return;
}
for (var k = lane; k < d.count; k += GROUP) {
out[s + k] = d.base + quad(k);
}
}