diff --git a/crates/horizon-client/src/gfx.rs b/crates/horizon-client/src/gfx.rs index 6a154d0de..82a9603e2 100644 --- a/crates/horizon-client/src/gfx.rs +++ b/crates/horizon-client/src/gfx.rs @@ -83,6 +83,40 @@ mod world_post; const COMPLETE_COLUMN: u32 = (1 << (crate::chunk::HEIGHT / 16)) - 1; /// A section's instance: its camera-relative origin and, in `w`, the step of /// its fixed-point UVs (`terrain.wgsl`'s `terrain_uv`). +/// The world pass again after occlusion culling's test: it loads both +/// targets (phase 2 draws over phase 1, reading the test's results). +fn world_pass_after_culling<'e>( + encoder: &'e mut wgpu::CommandEncoder, + view: &wgpu::TextureView, + depth: &wgpu::TextureView, + pack: bool, +) -> wgpu::RenderPass<'e> { + encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Horizon world, after culling"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view, + resolve_target: None, + depth_slice: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: depth, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Load, + store: if pack { + wgpu::StoreOp::Store + } else { + wgpu::StoreOp::Discard + }, + }), + stencil_ops: None, + }), + ..Default::default() + }) +} fn section_instance(origin: [f32; 3]) -> [f32; 4] { [ origin[0], @@ -316,7 +350,8 @@ pub struct Renderer { flat: Option, /// `flat` draws this frame's sections (off: the per-section draws). flat_on: bool, - /// Occlusion culling of solid section draws, where multi-draw is native. + /// Occlusion culling of solid section draws, where the GPU's instance + /// counts reach a multi-draw's entries (Vulkan, Metal). cull: Option, cull_on: bool, sky: wgpu::RenderPipeline, @@ -1455,7 +1490,6 @@ impl Renderer { usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, mapped_at_creation: false, }); - let depth = Self::depth(&device, &config); let (atlas_pixels, atlas_width, atlas_height) = hud_atlas; let atlas = device.create_texture_with_data( &queue, @@ -2211,14 +2245,25 @@ impl Renderer { .features() .contains(wgpu::Features::INDIRECT_FIRST_INSTANCE); // HORIZON_GPU_CULL=0 turns occlusion culling off, `all` culls every - // frame however little it hides (A/B checks). + // frame however few its draws and little it hides, `shader` tests in + // the shader as on Metal (A/B checks). let cull_env = std::env::var("HORIZON_GPU_CULL"); - let cull = (cull_env.as_deref() != Ok("0") && cull::Cull::supported(&adapter, &device)) - .then(|| { - let mut cull = cull::Cull::new(&device, config.format.remove_srgb_suffix()); + let cull = cull::Cull::supported(&adapter, &device) + .filter(|_| cull_env.as_deref() != Ok("0")) + .map(|depth_test| match cull_env.as_deref() { + Ok("shader") => cull::DepthTest::InShader, + _ => depth_test, + }) + .map(|depth_test| { + let mut cull = + cull::Cull::new(&device, config.format.remove_srgb_suffix(), depth_test); cull.always = cull_env.as_deref() == Ok("all"); + if !cull.always && adapter.get_info().backend == wgpu::Backend::Metal { + cull.min_draws = cull::TILED_MIN_DRAWS; + } cull }); + let depth = Self::depth(&device, &config, cull.as_ref()); let world_post = world_post::Post::new(&device, config.format.remove_srgb_suffix()); let uploads = upload::Uploads::new(&device); let section_uploads = upload::Uploads::with_chunk(&device, SECTION_UPLOAD_CHUNK); @@ -2431,7 +2476,13 @@ impl Renderer { self.surface = Some(surface); Ok(()) } - fn depth(device: &wgpu::Device, config: &wgpu::SurfaceConfiguration) -> wgpu::TextureView { + /// The world's depth, which occlusion culling's test may read. + fn depth( + device: &wgpu::Device, + config: &wgpu::SurfaceConfiguration, + cull: Option<&cull::Cull>, + ) -> wgpu::TextureView { + let read = cull.is_some_and(|c| c.depth_test() == cull::DepthTest::InShader); device .create_texture(&wgpu::TextureDescriptor { label: Some("Horizon depth"), @@ -2444,7 +2495,11 @@ impl Renderer { sample_count: 1, dimension: wgpu::TextureDimension::D2, format: wgpu::TextureFormat::Depth32Float, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + usage: if read { + wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING + } else { + wgpu::TextureUsages::RENDER_ATTACHMENT + }, view_formats: &[], }) .create_view(&Default::default()) @@ -2495,7 +2550,7 @@ impl Renderer { } else { self.offscreen = Some(Self::offscreen_target(&self.device, &self.config)); } - self.depth = Self::depth(&self.device, &self.config); + self.depth = Self::depth(&self.device, &self.config, self.cull.as_ref()); } /// Whether sections are drawn as one index list per frame (`flat`). pub fn flat_draws(&self) -> bool { @@ -3215,15 +3270,13 @@ impl Renderer { .inverse() .transform_point3(Vec3::ZERO); // Occlusion culling leaves the sections its last result did not see - // to phase 2, after the others, with instance counts of 0. + // to phase 2, after the others, with instance counts of 0. Whether + // the frame has draws enough goes by the last one's. let mut cull = self .cull .as_mut() - .filter(|_| { - self.cull_on - && !frame.title - && self.multi_draw - && !(self.flat_on && self.flat.is_some()) + .filter(|c| { + self.cull_on && !frame.title && self.multi_draw && self.section_draws >= c.min_draws }) .and_then(|cull| cull.begin(&self.device).then_some(cull)); self.phase2_draws.clear(); @@ -3610,6 +3663,16 @@ impl Renderer { frame.hud, [self.config.width, self.config.height], ); + if culling && let Some(cull) = &mut self.cull { + cull.upload( + &self.device, + &mut self.uploads, + &mut encoder, + &self.draws[phase1_draws as usize..solid_draws as usize], + &self.instances, + &self.globals, + ); + } let flat_frame = match &mut self.flat { Some(flat) if self.flat_on && !self.draws.is_empty() && self.draws.len() >= flat.min_draws => @@ -3617,6 +3680,7 @@ impl Renderer { let sections = &self.sections; let lods = &self.lods; let sorted = &self.sorted_indices; + let phase2 = self.cull.as_ref().filter(|_| culling).map(|c| &c.phase2); crate::spans::time("flatten", || { flat.prepare( &self.device, @@ -3624,7 +3688,9 @@ impl Renderer { &mut self.uploads, &self.draws, &self.draw_pages, + phase1_draws as usize, solid_draws as usize, + phase2, visible .iter() .filter_map(|v| { @@ -3659,16 +3725,6 @@ impl Renderer { 0, bytemuck::cast_slice(&self.draws), ); - if culling && let Some(cull) = &mut self.cull { - cull.upload( - &self.device, - &mut self.uploads, - &mut encoder, - &self.draws[phase1_draws as usize..solid_draws as usize], - &self.instances, - &self.globals, - ); - } } let view_target = target_texture.create_view(&wgpu::TextureViewDescriptor { format: Some(self.config.format.remove_srgb_suffix()), @@ -3720,6 +3776,9 @@ impl Renderer { Some((color, depth)) => (color, depth), None => (&view_target, &self.depth), }; + if culling && let Some(cull) = &mut self.cull { + cull.bind_depth(&self.device, world_depth); + } let cull = self.cull.as_ref().filter(|_| culling && solid_draws > 0); if shadow_cfg.is_some() { let shadow_view = self.world_post.shadow_depth().clone(); @@ -3830,9 +3889,28 @@ impl Renderer { match self.flat.as_ref().filter(|_| flat_frame) { Some(flat) => { pass.set_pipeline(&flat.solid); - flat.draw(&mut pass, &flat.calls[..flat.solid_calls], |p| { - self.arena.buffer(p) - }); + let vertices = |p| self.arena.buffer(p); + match cull { + // As below: phase 1, the test, then phase 2's draws + // that the test saw, copied after it. + Some(cull) => { + flat.draw(&mut pass, &flat.calls[..flat.phase1_calls], vertices); + cull.test(&mut pass); + drop(pass); + cull.test_pass(&mut encoder); + flat.flatten_phase2(&mut encoder); + pass = world_pass_after_culling( + &mut encoder, + world_view, + world_depth, + pack, + ); + pass.set_bind_group(0, &self.group, &[]); + pass.set_pipeline(&flat.solid); + flat.draw_phase2(&mut pass, vertices); + } + None => flat.draw(&mut pass, &flat.calls[..flat.solid_calls], vertices), + } pass.set_bind_group(0, &self.group, &[]); pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32); } @@ -3842,40 +3920,20 @@ impl Renderer { pass.set_vertex_buffer(1, self.instances.slice(..)); match cull { Some(cull) => { - // Phase 1 and the boxes against its depth, then + // Phase 1 and the boxes against its depth (in + // its pass, or one of their own on Metal), then // what they found in sight, in a pass that loads // both targets (it reads the test's results). self.draw_sections(&mut pass, &self.indirect, 0, 0, phase1_draws); cull.test(&mut pass); drop(pass); - let attachments = [Some(wgpu::RenderPassColorAttachment { - view: world_view, - resolve_target: None, - depth_slice: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Load, - store: wgpu::StoreOp::Store, - }, - })]; - pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("Horizon world, after culling"), - color_attachments: &attachments, - depth_stencil_attachment: Some( - wgpu::RenderPassDepthStencilAttachment { - view: world_depth, - depth_ops: Some(wgpu::Operations { - load: wgpu::LoadOp::Load, - store: if pack { - wgpu::StoreOp::Store - } else { - wgpu::StoreOp::Discard - }, - }), - stencil_ops: None, - }, - ), - ..Default::default() - }); + cull.test_pass(&mut encoder); + pass = world_pass_after_culling( + &mut encoder, + world_view, + world_depth, + pack, + ); pass.set_bind_group(0, &self.group, &[]); pass.set_pipeline(&self.solid); pass.set_index_buffer( diff --git a/crates/horizon-client/src/gfx/cull.rs b/crates/horizon-client/src/gfx/cull.rs index bcb738de3..57049eb4d 100644 --- a/crates/horizon-client/src/gfx/cull.rs +++ b/crates/horizon-client/src/gfx/cull.rs @@ -22,8 +22,16 @@ //! Every solid draw is an entry of one of the two lists, so the CPU records //! as many draws as without culling: drivers that unroll an indirect //! multi-draw on the CPU (Mesa's on Intel Gen9) pay per entry. Phase 2's -//! counts come from the GPU, which needs a native multi-draw and forced -//! early depth tests: Vulkan. Translucent draws are not culled. +//! counts come from the GPU, which reads them from the draws themselves: +//! Vulkan's native multi-draw, and Metal's, which wgpu replays as one +//! indirect draw per entry. Translucent draws are not culled. +//! +//! Metal cannot force the depth test before a fragment shader that writes +//! storage (naga writes no `[[early_fragment_tests]]`), so there the boxes +//! are drawn in a pass of their own between the phases, which reads phase +//! 1's depth as a texture and makes the same comparison in the shader: the +//! fragment's depth against the stored one, greater or equal passing. The +//! same fragments pass, so the same sections are seen. use std::num::NonZeroU64; use std::sync::{ Arc, @@ -38,6 +46,13 @@ const RETEST: u32 = 8; /// more than they save. Else one frame in `PROBE` tests every section. const MIN_SHARE: f32 = 0.1; const PROBE: u32 = 64; +/// On Metal, section draws from which a frame culls. Apple's GPUs store and +/// reload the whole frame at each of the passes the test adds, a fixed cost +/// that the sections it hides repay only at high render distances: on an M4 +/// Pro at 1080p and render distance 12 (1,000–1,600 draws) culling added +/// about 0.05 ms of GPU time while hiding 15–30% of the solid indices; at +/// 32 (drawn flat, over this many draws) it saved 0.25–1.6 ms. +pub const TILED_MIN_DRAWS: usize = 4096; /// Boxes over more than this share of the screen count as seen untested. const BIG: f32 = 0.25; /// Results read back at once, some frames late so the CPU never waits. @@ -74,10 +89,40 @@ struct Group { group: wgpu::BindGroup, } +/// Where the boxes' depth test runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DepthTest { + /// Forced before the fragment shader, in phase 1's pass (Vulkan). + Early, + /// In the fragment shader, against phase 1's depth read as a texture, + /// in a pass of its own (Metal). + InShader, +} + +/// The Vulkan entry point: the depth test forced early, which naga cannot +/// write for Metal (so it is not in `cull.wgsl`, which every backend takes). +const EARLY_TEST: &str = " +@fragment +@early_depth_test(force) +fn fs_box_early(in: Box) -> @location(0) vec4 { + see(in.test); + return vec4(0.0); +} +"; + +/// The test's depth texture binding (`InShader`), for the view it was made with. +struct DepthGroup { + view: wgpu::TextureView, + group: wgpu::BindGroup, +} + pub struct Cull { pipeline: wgpu::RenderPipeline, layout: wgpu::BindGroupLayout, group: Option, + depth_test: DepthTest, + depth_layout: Option, + depth_group: Option, params: wgpu::Buffer, /// Phase 2's draws: the solid draws after phase 1's in the frame's list. pub phase2: wgpu::Buffer, @@ -103,6 +148,8 @@ pub struct Cull { pub hidden: ([usize; 2], f32), /// Cull every frame, whatever the hidden share (A/B checks). pub always: bool, + /// Frames with fewer section draws than this do not cull. + pub min_draws: usize, } fn buffer( @@ -134,20 +181,33 @@ impl Cull { pub fn features(adapter: &wgpu::Adapter) -> wgpu::Features { adapter.features() & wgpu::Features::SHADER_EARLY_DEPTH_TEST } - /// Whether the device draws an indirect multi-draw natively and runs - /// fragment shaders that write storage only past an early depth test. - pub fn supported(adapter: &wgpu::Adapter, device: &wgpu::Device) -> bool { - adapter.get_info().backend == wgpu::Backend::Vulkan - && device.features().contains( - wgpu::Features::SHADER_EARLY_DEPTH_TEST | wgpu::Features::INDIRECT_FIRST_INSTANCE, - ) + /// Where the device can run the test, if it can: Vulkan with early + /// depth tests, or Metal, both drawing an indirect multi-draw's entries + /// with the instance counts the GPU wrote, with fragment shaders that + /// write storage. + pub fn supported(adapter: &wgpu::Adapter, device: &wgpu::Device) -> Option { + let depth_test = match adapter.get_info().backend { + wgpu::Backend::Vulkan + if device + .features() + .contains(wgpu::Features::SHADER_EARLY_DEPTH_TEST) => + { + DepthTest::Early + } + wgpu::Backend::Metal => DepthTest::InShader, + _ => return None, + }; + (device + .features() + .contains(wgpu::Features::INDIRECT_FIRST_INSTANCE) && adapter.get_downlevel_capabilities().flags.contains( wgpu::DownlevelFlags::VERTEX_STORAGE | wgpu::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE, - ) + )) + .then_some(depth_test) } /// `color` is the world pass's colour format. - pub fn new(device: &wgpu::Device, color: wgpu::TextureFormat) -> Self { + pub fn new(device: &wgpu::Device, color: wgpu::TextureFormat, depth_test: DepthTest) -> Self { let entry = |binding, visibility, ty| wgpu::BindGroupLayoutEntry { binding, visibility, @@ -180,13 +240,41 @@ impl Cull { }); let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("Section culling"), - source: wgpu::ShaderSource::Wgsl(include_str!("cull.wgsl").into()), + source: wgpu::ShaderSource::Wgsl(source(depth_test).into()), + }); + let depth_layout = (depth_test == DepthTest::InShader).then(|| { + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Section culling depth"), + entries: &[entry( + 0, + fragment, + wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Depth, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + )], + }) }); let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("Section culling"), - bind_group_layouts: &[Some(&layout)], + bind_group_layouts: &[Some(&layout), depth_layout.as_ref()], ..Default::default() }); + // In the shader: no colour target, and phase 1's depth attached read + // only (a pass needs an attachment), its test left to the shader. + let (fragment_entry, targets, depth_compare) = match depth_test { + DepthTest::Early => ( + "fs_box_early", + &[Some(wgpu::ColorTargetState { + format: color, + blend: None, + write_mask: wgpu::ColorWrites::empty(), + })][..], + wgpu::CompareFunction::GreaterEqual, + ), + DepthTest::InShader => ("fs_box", &[][..], wgpu::CompareFunction::Always), + }; let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Section culling"), layout: Some(&pipeline_layout), @@ -204,20 +292,16 @@ impl Cull { depth_stencil: Some(wgpu::DepthStencilState { format: wgpu::TextureFormat::Depth32Float, depth_write_enabled: Some(false), - depth_compare: Some(wgpu::CompareFunction::GreaterEqual), + depth_compare: Some(depth_compare), stencil: Default::default(), bias: Default::default(), }), multisample: Default::default(), fragment: Some(wgpu::FragmentState { module: &module, - entry_point: Some("fs_box"), + entry_point: Some(fragment_entry), compilation_options: Default::default(), - targets: &[Some(wgpu::ColorTargetState { - format: color, - blend: None, - write_mask: wgpu::ColorWrites::empty(), - })], + targets, }), multiview_mask: None, cache: None, @@ -226,6 +310,9 @@ impl Cull { pipeline, layout, group: None, + depth_test, + depth_layout, + depth_group: None, params: buffer( device, "Section culling parameters", @@ -259,6 +346,7 @@ impl Cull { stamp: 0, hidden: ([0; 2], 1.0), always: false, + min_draws: 0, } } @@ -399,17 +487,69 @@ impl Cull { } } - /// In the world pass after phase 1: draw the tested boxes against its - /// depth. Leaves the pass's pipeline and group 0 changed. + pub fn depth_test(&self) -> DepthTest { + self.depth_test + } + + /// In the world pass after phase 1 (`Early`): draw the tested boxes + /// against its depth. Leaves the pass's pipeline and group 0 changed. pub fn test(&self, pass: &mut wgpu::RenderPass<'_>) { let Some(Group { group, .. }) = &self.group else { return; }; + if self.depth_test != DepthTest::Early || self.test_data.is_empty() { + return; + } + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, group, &[]); + pass.draw(0..36, 0..self.test_data.len() as u32); + } + + /// Before the world pass (`InShader`): the binding of the depth the + /// world is drawn with this frame. + pub fn bind_depth(&mut self, device: &wgpu::Device, depth: &wgpu::TextureView) { + let Some(layout) = &self.depth_layout else { + return; + }; + if self.depth_group.as_ref().is_none_or(|g| g.view != *depth) { + self.depth_group = Some(DepthGroup { + view: depth.clone(), + group: device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Section culling depth"), + layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(depth), + }], + }), + }); + } + } + + /// Between the phases (`InShader`): draw the tested boxes in a pass of + /// their own that reads phase 1's depth, bound by `bind_depth`. + pub fn test_pass(&self, encoder: &mut wgpu::CommandEncoder) { + let (Some(Group { group, .. }), Some(DepthGroup { view, group: depth })) = + (&self.group, &self.depth_group) + else { + return; + }; if self.test_data.is_empty() { return; } + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Section culling"), + color_attachments: &[], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view, + depth_ops: None, + stencil_ops: None, + }), + ..Default::default() + }); pass.set_pipeline(&self.pipeline); pass.set_bind_group(0, group, &[]); + pass.set_bind_group(1, depth, &[]); pass.draw(0..36, 0..self.test_data.len() as u32); } @@ -498,16 +638,33 @@ impl Cull { } } -#[cfg(test)] -mod tests { - #[test] - fn shader_validates() { - let module = naga::front::wgsl::parse_str(include_str!("cull.wgsl")).unwrap(); - naga::valid::Validator::new( - naga::valid::ValidationFlags::all(), - naga::valid::Capabilities::all(), - ) - .validate(&module) - .unwrap(); +/// The test's shader for `depth_test`. +fn source(depth_test: DepthTest) -> String { + let common = include_str!("cull.wgsl"); + match depth_test { + DepthTest::Early => format!("{common}{EARLY_TEST}"), + DepthTest::InShader => common.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shaders_validate() { + for (depth_test, capabilities) in [ + (DepthTest::Early, naga::valid::Capabilities::all()), + // What wgpu's Metal backend allows: no early depth test. + ( + DepthTest::InShader, + naga::valid::Capabilities::all() - naga::valid::Capabilities::EARLY_DEPTH_TEST, + ), + ] { + let module = naga::front::wgsl::parse_str(&source(depth_test)).unwrap(); + naga::valid::Validator::new(naga::valid::ValidationFlags::all(), capabilities) + .validate(&module) + .unwrap_or_else(|e| panic!("{depth_test:?}: {e:?}")); + } } } diff --git a/crates/horizon-client/src/gfx/cull.wgsl b/crates/horizon-client/src/gfx/cull.wgsl index faa495caf..b06d4ccc2 100644 --- a/crates/horizon-client/src/gfx/cull.wgsl +++ b/crates/horizon-client/src/gfx/cull.wgsl @@ -86,16 +86,28 @@ fn vs_box(@builtin(vertex_index) v: u32, @builtin(instance_index) t: u32) -> Box return out; } -// Runs only for fragments that pass the depth test (forced early). -@fragment -@early_depth_test(force) -fn fs_box(in: Box) -> @location(0) vec4 { - if results[in.test] != params.stamp { - results[in.test] = params.stamp; - let test = tests[in.test]; +// The box is seen at this fragment: store the stamp, and give its phase 2 +// draws their instance. +fn see(t: u32) { + if results[t] != params.stamp { + results[t] = params.stamp; + let test = tests[t]; for (var k = 0u; k < test.z; k++) { phase2[(test.y + k) * 5u + 1u] = 1u; } } - return vec4(0.0); +} + +// Phase 1's depth, where the test runs in the shader (gfx/cull.rs). +@group(1) @binding(0) var depth: texture_depth_2d; + +// The world pass's reversed depth test, greater or equal passing, made here: +// where it cannot be forced before a shader that writes storage, a fragment +// that fails it would still write. Where it can, `fs_box_early` (cull.rs) +// runs only for the fragments that pass it. +@fragment +fn fs_box(in: Box) { + if in.position.z >= textureLoad(depth, vec2(in.position.xy), 0) { + see(in.test); + } } diff --git a/crates/horizon-client/src/gfx/flat.rs b/crates/horizon-client/src/gfx/flat.rs index a1ae20d09..2d0a8d27c 100644 --- a/crates/horizon-client/src/gfx/flat.rs +++ b/crates/horizon-client/src/gfx/flat.rs @@ -11,11 +11,37 @@ //! so each block names one section: its id here, which indexes the frame's //! camera-relative origins (computed on the CPU exactly as for the instance //! attribute). +//! +//! With occlusion culling (`cull`), phase 1's draws and the translucent ones +//! are copied before the world pass, as without it. Phase 2's draws are +//! copied after the test, which tells which ones it saw: each run of a page +//! reserves room for all of its draws, the copy packs the seen ones at its +//! start, and the run is drawn indirectly with the count the copy reached. use super::arena::GRAIN; /// One section draw for `flatten.wgsl`: index count, first sorted index (or -/// `QUADS` for a solid draw), base vertex, first index in the list. +/// `QUADS` for a solid draw; phase 2: its entry in phase 2's draws), base +/// vertex, first index in the list (phase 2: its draw call in phase 2). type Range = [u32; 4]; + +/// Where a group of ranges takes its indices. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Source { + /// Solid draws: quads. + Quads, + /// Occlusion culling's phase 2: quads, of the sections its test saw. + Seen, + /// Translucent draws: this sorted index page. + Sorted(u32), +} + +/// The frame's draw calls, as `plan` lays them out. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Plan { + phase1_calls: usize, + solid_calls: usize, + total: u64, +} const QUADS: u32 = u32::MAX; const GROUP_LIMIT: u32 = 65535; @@ -38,6 +64,15 @@ pub struct Flat { pub translucent: wgpu::RenderPipeline, flatten: wgpu::ComputePipeline, compute_layout: wgpu::BindGroupLayout, + flatten_seen: wgpu::ComputePipeline, + seen_layout: wgpu::BindGroupLayout, + /// Phase 2's draw calls (`DrawIndexedIndirectArgs`), whose index counts + /// `flatten_seen` makes. + phase2_calls: wgpu::Buffer, + phase2_call_capacity: u64, + phase2_call_data: Vec<[u32; 5]>, + /// The copy of phase 2's draws, and its ranges. + phase2_group: Option<(wgpu::BindGroup, u32)>, draw_layout: wgpu::BindGroupLayout, blocks_per_page: u64, pages: Vec>, @@ -52,12 +87,14 @@ pub struct Flat { ranges: wgpu::Buffer, range_capacity: u64, range_data: Vec, - /// Segments of `range_data` by index source: (sorted index page or - /// `None` for solid draws, first range, ranges). - segments: Vec<(Option, u64, u64)>, + /// Segments of `range_data` by index source: (source, first range, + /// ranges). + segments: Vec<(Source, u64, u64)>, empty: wgpu::Buffer, - /// This frame's draw calls: solid ones, then translucent ones. + /// This frame's draw calls: solid ones (phase 1's, then phase 2's when + /// culling), then translucent ones. pub calls: Vec, + pub phase1_calls: usize, pub solid_calls: usize, /// Frames with fewer section draws keep the per-section draws. pub min_draws: usize, @@ -121,42 +158,60 @@ fn buffer( /// Lays out the frame's index list: each draw's range (destinations in draw /// order) grouped by index source into `ranges`, every group padded to /// `align` entries for binding, with `segments` naming each group; and the -/// draw calls, one per run of draws in the same vertex page, solid and -/// translucent apart. Returns the first translucent call and the list's -/// length. +/// draw calls, one per run of draws in the same vertex page, phase 1's solid +/// ones, phase 2's (draws `phase1_draws..solid_draws`) and translucent ones +/// apart. A phase 2 range names its draw in phase 2 and its call in phase 2 +/// instead of a destination: the call's room holds all of its draws. fn plan( draws: &[[u32; 5]], pages: &[(u32, Option)], + phase1_draws: usize, solid_draws: usize, align: u64, calls: &mut Vec, ranges: &mut Vec, - segments: &mut Vec<(Option, u64, u64)>, -) -> (usize, u64) { + segments: &mut Vec<(Source, u64, u64)>, +) -> Plan { calls.clear(); ranges.clear(); segments.clear(); - let mut solid_calls = None; + let (mut phase1_calls, mut solid_calls) = (None, None); let mut total = 0u64; - let mut by_source: Vec<(Option, Vec)> = Vec::new(); + let mut by_source: Vec<(Source, Vec)> = Vec::new(); for (i, (d, &(page, sorted))) in draws.iter().zip(pages).enumerate() { - let first = if sorted.is_some() { d[2] } else { QUADS }; - let range = [d[0], first, d[3], total as u32]; - match by_source.iter_mut().find(|(s, _)| *s == sorted) { - Some((_, list)) => list.push(range), - None => by_source.push((sorted, vec![range])), + if i == phase1_draws { + phase1_calls.get_or_insert(calls.len()); } if i == solid_draws { solid_calls = Some(calls.len()); } match calls.last_mut() { - Some(last) if last.page == page && i != solid_draws => last.count += d[0], + Some(last) if last.page == page && i != phase1_draws && i != solid_draws => { + last.count += d[0] + } _ => calls.push(Call { page, first: total as u32, count: d[0], }), } + let (source, range) = match sorted { + Some(p) => (Source::Sorted(p), [d[0], d[2], d[3], total as u32]), + None if (phase1_draws..solid_draws).contains(&i) => ( + Source::Seen, + [ + d[0], + (i - phase1_draws) as u32, + d[3], + (calls.len() - 1 - phase1_calls.unwrap_or(0)) as u32, + ], + ), + None => (Source::Quads, [d[0], QUADS, d[3], total as u32]), + }; + match by_source.iter_mut().find(|(s, _)| *s == source) { + Some((_, list)) => list.push(range), + None => by_source.push((source, vec![range])), + } total += u64::from(d[0]); } for (source, list) in by_source { @@ -165,7 +220,12 @@ fn plan( let padded = (ranges.len() as u64).next_multiple_of(align); ranges.resize(padded as usize, [0; 4]); } - (solid_calls.unwrap_or(calls.len()), total) + let solid_calls = solid_calls.unwrap_or(calls.len()); + Plan { + phase1_calls: phase1_calls.unwrap_or(calls.len()).min(solid_calls), + solid_calls, + total, + } } impl Flat { @@ -201,24 +261,37 @@ impl Flat { storage(2, compute, false), ], }); + let seen_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Flatten phase 2 section draws"), + entries: &[ + storage(0, compute, true), + storage(2, compute, false), + storage(3, compute, true), + storage(4, compute, false), + ], + }); let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("Flatten section draws"), source: wgpu::ShaderSource::Wgsl(include_str!("flatten.wgsl").into()), }); - let flatten = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { - label: Some("Flatten section draws"), - layout: Some( - &device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("Flatten section draws"), - bind_group_layouts: &[Some(&compute_layout)], - ..Default::default() - }), - ), - module: &module, - entry_point: Some("flatten"), - compilation_options: Default::default(), - cache: None, - }); + let compute_pipeline = |layout: &wgpu::BindGroupLayout, entry| { + device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("Flatten section draws"), + layout: Some( + &device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("Flatten section draws"), + bind_group_layouts: &[Some(layout)], + ..Default::default() + }), + ), + module: &module, + entry_point: Some(entry), + compilation_options: Default::default(), + cache: None, + }) + }; + let flatten = compute_pipeline(&compute_layout, "flatten"); + let flatten_seen = compute_pipeline(&seen_layout, "flatten_seen"); let storage_usage = wgpu::BufferUsages::STORAGE; Self { solid: pipeline(&layout, "Terrain solid (flat)", "fs_solid", None), @@ -230,6 +303,17 @@ impl Flat { ), flatten, compute_layout, + flatten_seen, + seen_layout, + phase2_calls: buffer( + device, + "Flat section draw calls, phase 2", + 20, + wgpu::BufferUsages::INDIRECT | storage_usage, + ), + phase2_call_capacity: 1, + phase2_call_data: Vec::new(), + phase2_group: None, draw_layout, blocks_per_page: page_vertices.div_ceil(GRAIN), pages: Vec::new(), @@ -251,6 +335,7 @@ impl Flat { segments: Vec::new(), empty: buffer(device, "No sorted indices", 16, storage_usage), calls: Vec::new(), + phase1_calls: 0, solid_calls: 0, min_draws: if std::env::var("HORIZON_FLAT_DRAWS").as_deref() == Ok("1") { 0 @@ -321,6 +406,9 @@ impl Flat { /// the origins of the sections they draw by id. Records the copy into /// `encoder`; `calls` then lists the frame's draw calls. False when the /// frame cannot be drawn flat (the per-section draws then apply). + /// + /// When culling, the solid draws from `phase1_draws` are phase 2's, which + /// `phase2` holds as the test leaves them: `flatten_phase2` copies them. #[allow(clippy::too_many_arguments)] pub fn prepare( &mut self, @@ -329,10 +417,18 @@ impl Flat { uploads: &mut super::upload::Uploads, draws: &[[u32; 5]], pages: &[(u32, Option)], + phase1_draws: usize, solid_draws: usize, + phase2: Option<&wgpu::Buffer>, origins: impl Iterator, sorted_page: impl Fn(u32) -> wgpu::Buffer, ) -> bool { + self.phase2_group = None; + let phase1_draws = if phase2.is_some() { + phase1_draws + } else { + solid_draws + }; // Origins by section id. self.origin_data.clear(); for (id, o) in origins { @@ -362,18 +458,25 @@ impl Flat { bytemuck::cast_slice(&self.origin_data), ); } - let (solid_calls, total) = plan( + let Plan { + phase1_calls, + solid_calls, + total, + } = plan( draws, pages, + phase1_draws, solid_draws, self.range_align, &mut self.calls, &mut self.range_data, &mut self.segments, ); + self.phase1_calls = phase1_calls; self.solid_calls = solid_calls; if total == 0 || total * 4 > device.limits().max_storage_buffer_binding_size { self.calls.clear(); + self.phase1_calls = 0; self.solid_calls = 0; return false; } @@ -401,22 +504,82 @@ impl Flat { 0, bytemuck::cast_slice(&self.range_data), ); + // Phase 2's calls start empty, each at the start of its room. + self.phase2_call_data.clear(); + self.phase2_call_data.extend( + self.calls[phase1_calls..solid_calls] + .iter() + .map(|c| [0, 1, c.first, 0, 0]), + ); + let phase2_calls = self.phase2_call_data.len() as u64; + if phase2_calls > self.phase2_call_capacity { + self.phase2_call_capacity = phase2_calls.next_power_of_two(); + self.phase2_calls = buffer( + device, + "Flat section draw calls, phase 2", + self.phase2_call_capacity * 20, + wgpu::BufferUsages::INDIRECT | wgpu::BufferUsages::STORAGE, + ); + } + if phase2_calls > 0 { + uploads.buffer( + encoder, + &self.phase2_calls, + 0, + bytemuck::cast_slice(&self.phase2_call_data), + ); + } + let ranges_at = |first: u64, count: u64| { + wgpu::BindingResource::Buffer(wgpu::BufferBinding { + buffer: &self.ranges, + offset: first * 16, + size: wgpu::BufferSize::new(count * 16), + }) + }; + if let (Some(phase2), Some(&(_, first, count))) = ( + phase2, + self.segments.iter().find(|(s, ..)| *s == Source::Seen), + ) { + let group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Flatten phase 2 section draws"), + layout: &self.seen_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: ranges_at(first, count), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: self.indices.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: phase2.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 4, + resource: self.phase2_calls.as_entire_binding(), + }, + ], + }); + self.phase2_group = Some((group, count as u32)); + } let groups: Vec<(wgpu::BindGroup, u32)> = self .segments .iter() - .map(|&(source, first, count)| { - let sorted = source.map(&sorted_page); + .filter_map(|&(source, first, count)| { + let sorted = match source { + Source::Quads => None, + Source::Seen => return None, + Source::Sorted(page) => Some(sorted_page(page)), + }; let group = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("Flatten section draws"), layout: &self.compute_layout, entries: &[ wgpu::BindGroupEntry { binding: 0, - resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { - buffer: &self.ranges, - offset: first * 16, - size: wgpu::BufferSize::new(count * 16), - }), + resource: ranges_at(first, count), }, wgpu::BindGroupEntry { binding: 1, @@ -428,7 +591,7 @@ impl Flat { }, ], }); - (group, count as u32) + Some((group, count as u32)) }) .collect(); let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { @@ -464,6 +627,19 @@ impl Flat { } true } + /// After occlusion culling's test: copies the phase 2 draws it saw. + pub fn flatten_phase2(&self, encoder: &mut wgpu::CommandEncoder) { + let Some((group, count)) = &self.phase2_group else { + return; + }; + let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { + label: Some("Flatten phase 2 section draws"), + timestamp_writes: None, + }); + pass.set_pipeline(&self.flatten_seen); + pass.set_bind_group(0, group, &[]); + pass.dispatch_workgroups((*count).min(GROUP_LIMIT), count.div_ceil(GROUP_LIMIT), 1); + } /// Draws `calls` with a flat pipeline set and `vertices` giving each /// page's vertex buffer. pub fn draw<'a>( @@ -474,15 +650,38 @@ impl Flat { ) { pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32); for call in calls { - let table = self.pages[call.page as usize] - .as_ref() - .and_then(|t| t.group.as_ref()) - .expect("prepared page"); - pass.set_vertex_buffer(0, vertices(call.page).slice(..)); - pass.set_bind_group(1, table, &[]); + self.bind_page(pass, call.page, &vertices); pass.draw_indexed(call.first..call.first + call.count, 0, 0..1); } } + /// Draws phase 2's calls, as `flatten_phase2` left them. + pub fn draw_phase2<'a>( + &self, + pass: &mut wgpu::RenderPass<'_>, + vertices: impl Fn(u32) -> &'a wgpu::Buffer, + ) { + pass.set_index_buffer(self.indices.slice(..), wgpu::IndexFormat::Uint32); + for (k, call) in self.calls[self.phase1_calls..self.solid_calls] + .iter() + .enumerate() + { + self.bind_page(pass, call.page, &vertices); + pass.draw_indexed_indirect(&self.phase2_calls, k as u64 * 20); + } + } + fn bind_page<'a>( + &self, + pass: &mut wgpu::RenderPass<'_>, + page: u32, + vertices: &impl Fn(u32) -> &'a wgpu::Buffer, + ) { + let table = self.pages[page as usize] + .as_ref() + .and_then(|t| t.group.as_ref()) + .expect("prepared page"); + pass.set_vertex_buffer(0, vertices(page).slice(..)); + pass.set_bind_group(1, table, &[]); + } } #[cfg(test)] @@ -516,16 +715,22 @@ mod tests { fn plan_draws_runs_of_a_page_in_order() { let (draws, pages, solid) = sample(); let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new()); - let (solid_calls, total) = plan( + let Plan { + phase1_calls, + solid_calls, + total, + } = plan( &draws, &pages, solid, + solid, 4, &mut calls, &mut ranges, &mut segments, ); assert_eq!(total, 6 * 13); + assert_eq!(phase1_calls, solid_calls); let call = |page, first, count| Call { page, first, count }; // Pages 0, 0 | 1 | 0, then translucent 0, 0 (never merged with solid). assert_eq!( @@ -539,16 +744,65 @@ mod tests { ); assert_eq!(solid_calls, 3); // Solid ranges, then each sorted page's, every group padded to 4. - assert_eq!(segments, [(None, 0, 4), (Some(0), 4, 1), (Some(1), 8, 1)]); + assert_eq!( + segments, + [ + (Source::Quads, 0, 4), + (Source::Sorted(0), 4, 1), + (Source::Sorted(1), 8, 1) + ] + ); assert_eq!(ranges[3], [24, QUADS, 1024, 36]); assert_eq!(ranges[4], [12, 6, 768, 60]); assert_eq!(ranges[8], [6, 0, 40, 72]); assert_eq!(ranges.len(), 12); } + /// Phase 2's draws (here the third and fourth) get calls of their own, + /// each with room for all of its draws, and ranges naming their draw + /// and call in phase 2. + #[test] + fn plan_gives_phase_2_calls_of_its_own() { + let (draws, pages, solid) = sample(); + let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new()); + let plan = plan( + &draws, + &pages, + 2, + solid, + 4, + &mut calls, + &mut ranges, + &mut segments, + ); + assert_eq!( + plan, + Plan { + phase1_calls: 1, + solid_calls: 3, + total: 6 * 13 + } + ); + let call = |page, first, count| Call { page, first, count }; + // Phase 1: page 0 | phase 2: page 1, page 0 | translucent: page 0. + assert_eq!( + calls, + [ + call(0, 0, 30), + call(1, 30, 6), + call(0, 36, 24), + call(0, 60, 18) + ] + ); + assert_eq!(segments[1], (Source::Seen, 4, 2)); + assert_eq!(ranges[4], [6, 0, 256, 0]); + assert_eq!(ranges[5], [24, 1, 1024, 1]); + } + /// The compute pass writes exactly the indices the per-section draws /// read: quads 0 1 2 0 2 3 for solid draws, sorted indices otherwise, - /// plus the base vertex. + /// plus the base vertex. With culling, phase 2's runs hold the indices + /// of the draws the test saw, in order, and their calls count them. #[test] fn flatten_writes_the_indices_of_the_per_section_draws() { let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); @@ -564,101 +818,170 @@ mod tests { use wgpu::util::DeviceExt; let (draws, pages, solid) = sample(); let align = (u64::from(device.limits().min_storage_buffer_offset_alignment) / 16).max(1); - let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new()); - let (_, total) = plan( - &draws, - &pages, - solid, - align, - &mut calls, - &mut ranges, - &mut segments, - ); let sorted: [Vec; 2] = [ (0..32).map(|i| (i * 7) % 32).collect(), (0..8).rev().collect(), ]; - let mut expected = Vec::new(); - for (d, (_, page)) in draws.iter().zip(&pages) { - for k in 0..d[0] { - let i = match page { - None => (k / 6) * 4 + [0, 1, 2, 0, 2, 3][(k % 6) as usize], - Some(p) => sorted[*p as usize][(d[2] + k) as usize], - }; - expected.push(d[3] + i); - } - } - let storage = |entries: &[u32]| { + let indices = |d: &[u32; 5], sorted_page: Option| -> Vec { + (0..d[0]) + .map(|k| { + d[3] + match sorted_page { + None => (k / 6) * 4 + [0, 1, 2, 0, 2, 3][(k % 6) as usize], + Some(p) => sorted[p as usize][(d[2] + k) as usize], + } + }) + .collect() + }; + let storage = |entries: &[u32], usage| { device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: None, contents: bytemuck::cast_slice(entries), - usage: wgpu::BufferUsages::STORAGE, + usage: wgpu::BufferUsages::STORAGE | usage, }) }; - let range_buffer = storage(bytemuck::cast_slice(&ranges)); - let sorted_buffers = [storage(&sorted[0]), storage(&sorted[1]), storage(&[0; 4])]; - let out = device.create_buffer(&wgpu::BufferDescriptor { - label: None, - size: total * 4, - usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC, - mapped_at_creation: false, - }); - let read = device.create_buffer(&wgpu::BufferDescriptor { - label: None, - size: total * 4, - usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, - mapped_at_creation: false, - }); + let none = wgpu::BufferUsages::empty(); + let sorted_buffers = [ + storage(&sorted[0], none), + storage(&sorted[1], none), + storage(&[0; 4], none), + ]; let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: None, source: wgpu::ShaderSource::Wgsl(include_str!("flatten.wgsl").into()), }); - let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { - label: None, - layout: None, - module: &module, - entry_point: Some("flatten"), - compilation_options: Default::default(), - cache: None, - }); - let mut encoder = device.create_command_encoder(&Default::default()); - { - let mut pass = encoder.begin_compute_pass(&Default::default()); - pass.set_pipeline(&pipeline); - for &(source, first, count) in &segments { - let group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: None, - layout: &pipeline.get_bind_group_layout(0), - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { - buffer: &range_buffer, - offset: first * 16, - size: wgpu::BufferSize::new(count * 16), - }), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: sorted_buffers[source.map_or(2, |p| p as usize)] - .as_entire_binding(), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: out.as_entire_binding(), - }, - ], - }); - pass.set_bind_group(0, &group, &[]); - pass.dispatch_workgroups(count as u32, 1, 1); + let pipeline = |entry| { + device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: None, + layout: None, + module: &module, + entry_point: Some(entry), + compilation_options: Default::default(), + cache: None, + }) + }; + let (flatten, flatten_seen) = (pipeline("flatten"), pipeline("flatten_seen")); + let read_back = |buffer: &wgpu::Buffer, words: u64| -> Vec { + let read = device.create_buffer(&wgpu::BufferDescriptor { + label: None, + size: words * 4, + usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let mut encoder = device.create_command_encoder(&Default::default()); + encoder.copy_buffer_to_buffer(buffer, 0, &read, 0, words * 4); + queue.submit([encoder.finish()]); + read.slice(..).map_async(wgpu::MapMode::Read, |_| {}); + device.poll(wgpu::PollType::wait_indefinitely()).unwrap(); + bytemuck::cast_slice(&read.slice(..).get_mapped_range().unwrap()).to_vec() + }; + // Without culling, then with the third and fourth draws in phase 2, + // of which the test saw only the fourth. + for phase1 in [solid, 2] { + let (mut calls, mut ranges, mut segments) = (Vec::new(), Vec::new(), Vec::new()); + let plan = plan( + &draws, + &pages, + phase1, + solid, + align, + &mut calls, + &mut ranges, + &mut segments, + ); + // Phase 2's draws as the test leaves them: instance counts of 0, + // 1 where it saw the section. + let mut phase2: Vec<[u32; 5]> = draws[phase1..solid] + .iter() + .map(|&[count, _, first, base, instance]| [count, 0, first, base, instance]) + .collect(); + if let Some(last) = phase2.last_mut() { + last[1] = 1; + } + let phase2_calls: Vec<[u32; 5]> = calls[plan.phase1_calls..plan.solid_calls] + .iter() + .map(|c| [0, 1, c.first, 0, 0]) + .collect(); + let range_buffer = storage(bytemuck::cast_slice(&ranges), none); + let phase2_buffer = storage( + bytemuck::cast_slice(&[phase2.as_slice(), &[[0; 5]]].concat()), + none, + ); + let calls_buffer = storage( + bytemuck::cast_slice(&[phase2_calls.as_slice(), &[[0; 5]]].concat()), + wgpu::BufferUsages::COPY_SRC, + ); + let out = storage(&vec![0; plan.total as usize], wgpu::BufferUsages::COPY_SRC); + let mut encoder = device.create_command_encoder(&Default::default()); + { + let mut pass = encoder.begin_compute_pass(&Default::default()); + for &(source, first, count) in &segments { + let ranges = wgpu::BindingResource::Buffer(wgpu::BufferBinding { + buffer: &range_buffer, + offset: first * 16, + size: wgpu::BufferSize::new(count * 16), + }); + let entry = |binding, resource| wgpu::BindGroupEntry { binding, resource }; + let (pipeline, entries) = match source { + Source::Seen => ( + &flatten_seen, + vec![ + entry(0, ranges), + entry(2, out.as_entire_binding()), + entry(3, phase2_buffer.as_entire_binding()), + entry(4, calls_buffer.as_entire_binding()), + ], + ), + Source::Quads | Source::Sorted(_) => { + let page = match source { + Source::Sorted(p) => p as usize, + _ => 2, + }; + ( + &flatten, + vec![ + entry(0, ranges), + entry(1, sorted_buffers[page].as_entire_binding()), + entry(2, out.as_entire_binding()), + ], + ) + } + }; + let group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &pipeline.get_bind_group_layout(0), + entries: &entries, + }); + pass.set_pipeline(pipeline); + pass.set_bind_group(0, &group, &[]); + pass.dispatch_workgroups(count as u32, 1, 1); + } + } + queue.submit([encoder.finish()]); + let got = read_back(&out, plan.total); + // Phase 1's and the translucent calls: each of their draws, in order. + let expected: Vec = (0..draws.len()) + .filter(|i| !(phase1..solid).contains(i)) + .flat_map(|i| indices(&draws[i], pages[i].1)) + .collect(); + let drawn: Vec = calls + .iter() + .enumerate() + .filter(|(c, _)| !(plan.phase1_calls..plan.solid_calls).contains(c)) + .flat_map(|(_, c)| got[c.first as usize..(c.first + c.count) as usize].to_vec()) + .collect(); + assert_eq!(drawn, expected); + // Phase 2: page 1's run drew nothing; page 0's holds the fourth + // draw's indices. + if phase1 < solid { + let counts = read_back(&calls_buffer, phase2_calls.len() as u64 * 5); + assert_eq!(counts[0], 0); + assert_eq!(counts[5], draws[3][0]); + let run = &calls[plan.phase1_calls + 1]; + assert_eq!( + got[run.first as usize..(run.first + draws[3][0]) as usize], + indices(&draws[3], None) + ); } } - encoder.copy_buffer_to_buffer(&out, 0, &read, 0, total * 4); - queue.submit([encoder.finish()]); - read.slice(..).map_async(wgpu::MapMode::Read, |_| {}); - device.poll(wgpu::PollType::wait_indefinitely()).unwrap(); - let got: Vec = - bytemuck::cast_slice(&read.slice(..).get_mapped_range().unwrap()).to_vec(); - assert_eq!(got, expected); } } diff --git a/crates/horizon-client/src/gfx/flatten.wgsl b/crates/horizon-client/src/gfx/flatten.wgsl index 2f57f8775..d722d7b9f 100644 --- a/crates/horizon-client/src/gfx/flatten.wgsl +++ b/crates/horizon-client/src/gfx/flatten.wgsl @@ -4,9 +4,11 @@ // translucent ones read their sorted indices. struct Range { count: u32, - // First sorted index, or QUADS for a solid draw. + // First sorted index, or QUADS for a solid draw (`flatten_seen`: its + // entry in occlusion culling's phase 2 draws). first: u32, base: u32, + // First index in the list (`flatten_seen`: its draw call). dst: u32, } const QUADS: u32 = 0xffffffffu; @@ -14,6 +16,13 @@ const GROUP: u32 = 64u; @group(0) @binding(0) var ranges: array; @group(0) @binding(1) var sorted: array; @group(0) @binding(2) var out: array; + +// Index `k` of a solid draw: quad k / 6's vertices 0 1 2 0 2 3. +fn quad(k: u32) -> u32 { + let c = k % 6u; + return (k / 6u) * 4u + select(select(c - 2u, 0u, c == 3u), c, c < 3u); +} + @compute @workgroup_size(64) fn flatten(@builtin(workgroup_id) group: vec3, @builtin(local_invocation_index) lane: u32) { @@ -25,11 +34,43 @@ fn flatten(@builtin(workgroup_id) group: vec3, for (var k = lane; k < d.count; k += GROUP) { var i: u32; if d.first == QUADS { - let c = k % 6u; - i = (k / 6u) * 4u + select(select(c - 2u, 0u, c == 3u), c, c < 3u); + i = quad(k); } else { i = sorted[d.first + k]; } out[d.dst + k] = d.base + i; } } + +// Occlusion culling's phase 2 (gfx/cull.rs), after its test: the draws as +// the test left them (`DrawIndexedIndirectArgs`, the instance count 1 where +// it saw the section), and phase 2's draw calls, whose index counts start +// at 0. A draw the test saw takes the next indices of its call's run. +@group(0) @binding(3) var draws: array; +@group(0) @binding(4) var calls: array>; +const UNSEEN: u32 = 0xffffffffu; +var start: u32; + +@compute @workgroup_size(64) +fn flatten_seen(@builtin(workgroup_id) group: vec3, + @builtin(local_invocation_index) lane: u32) { + let r = group.y * 65535u + group.x; + if r >= arrayLength(&ranges) { + return; + } + let d = ranges[r]; + if lane == 0u { + var s = UNSEEN; + if draws[d.first * 5u + 1u] != 0u { + s = atomicLoad(&calls[d.dst * 5u + 2u]) + atomicAdd(&calls[d.dst * 5u], d.count); + } + start = s; + } + let s = workgroupUniformLoad(&start); + if s == UNSEEN { + return; + } + for (var k = lane; k < d.count; k += GROUP) { + out[s + k] = d.base + quad(k); + } +}