diff --git a/Cargo.lock b/Cargo.lock index 7036ea5ca..7b1770094 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1435,6 +1435,7 @@ name = "horizon-shaders" version = "0.1.0" dependencies = [ "horizon-core", + "naga", "zip", ] @@ -1951,6 +1952,7 @@ dependencies = [ "naga-types", "num-traits", "once_cell", + "pp-rs", "rustc-hash 1.1.0", "spirv", "thiserror 2.0.21", @@ -2627,6 +2629,15 @@ dependencies = [ "zerovec", ] +[[package]] +name = "pp-rs" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb458bb7f6e250e6eb79d5026badc10a3ebb8f9a15d1fff0f13d17c71f4d6dee" +dependencies = [ + "unicode-xid", +] + [[package]] name = "presser" version = "0.3.1" diff --git a/crates/horizon-shaders/Cargo.toml b/crates/horizon-shaders/Cargo.toml index 750949e83..38e068b94 100644 --- a/crates/horizon-shaders/Cargo.toml +++ b/crates/horizon-shaders/Cargo.toml @@ -8,6 +8,7 @@ description = "Shader packs (the shaders/ folder format): reading packs, their . [dependencies] horizon-core = { path = "../horizon-core" } +naga = { version = "30.0.1", features = ["glsl-in", "wgsl-in"] } zip = { version = "2", default-features = false, features = ["deflate"] } [lints] diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index 14f85b758..bae3c0dcd 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -7,6 +7,7 @@ //! - [`Properties`]: `shaders.properties` and the id files, with their conditional directives (3.1, 1.5). //! - [`Lang`]: the pack's translated names (8.4). //! - [`preprocess`]: a stage file through `#include` and the GLSL preprocessor, lines mapped back (1.4, 1.5). +//! - [`translate_composite`]: a composite-style program into naga modules and its interface (1.6, 2.2). //! - [`Macros`] and [`eval_condition`]: the macros packs test, and the `#if` arithmetic (1.5). mod condition; @@ -16,6 +17,7 @@ mod macros; mod pack; mod preprocess; mod properties; +mod translate; pub use condition::eval_condition; pub use discover::{Found, discover}; @@ -24,3 +26,4 @@ pub use macros::{MC_VERSION, Macros, RENDER_STAGES}; pub use pack::{Pack, PackError}; pub use preprocess::{Diagnostic, Origin, Preprocessed, preprocess}; pub use properties::{Entry, Note, Properties}; +pub use translate::{Error, Interface, Program, Sampler, Target, Uniform, translate_composite}; diff --git a/crates/horizon-shaders/src/translate/mod.rs b/crates/horizon-shaders/src/translate/mod.rs new file mode 100644 index 000000000..9cc2ea07a --- /dev/null +++ b/crates/horizon-shaders/src/translate/mod.rs @@ -0,0 +1,436 @@ +//! A program's preprocessed GLSL (any version from 110 to 460, compatibility or core) turned +//! into the Vulkan-flavoured GLSL 450 naga reads, then into validated naga modules (spec 1.6, +//! 2.2). The rewrite works on preprocessing tokens and never adds or removes a line of the pack's +//! code, so naga's line numbers map back through the preprocessor to the pack's files. +//! +//! What it does: +//! - Loose uniforms go into one uniform block shared by both stages (`bool` stored as `int`). +//! - Each sampler is split into a texture and a sampler object, combined again where it is used +//! (`sampler2D(t, s)`), or passed as two arguments to the pack's own functions. +//! - The compatibility built-ins (`gl_Vertex`, `gl_MultiTexCoordN`, `gl_*Matrix`, `ftransform()`, +//! `gl_FragData`, `gl_FragColor`, `gl_TexCoord`, `texture2D`...) become declarations of ours, +//! named with the reserved prefix `hz_`; varyings get locations by name across the stages. +//! - The pack's `main` becomes `hz_main`, called by a `main` of ours that sets the inputs and +//! adapts GL's conventions (spec 5.5): clip y is negated for every target but the window, so +//! memory rows are GL's rows and `gl_FragCoord` needs no change, and clip z goes from GL's +//! [-w, w] to [0, w], so stored depth is GL's window depth. +//! +//! Composite-style programs draw one quad from (0,0) to (1,1) with an identity model-view and an +//! orthographic projection onto it, the texture coordinate equal to the position (OQ-6). + +use std::collections::BTreeMap; +use std::fmt::Write; + +use naga::ShaderStage; +use naga::front::glsl::{Frontend, Options}; +use naga::valid::{Capabilities, ValidationFlags, Validator}; + +use crate::Preprocessed; + +mod rewrite; +mod scan; + +use scan::Stage; + +/// What a program's stages need from the game. +#[derive(Debug, Default, PartialEq)] +pub struct Interface { + /// The uniform block at binding 0: its size and each value's name, GLSL type and offset. + /// Ours come first: `hz_target` (the target's width, height, 1/width, 1/height), and the + /// fixed-function fog the compatibility `gl_Fog` reads (OQ-28): `hz_fog_color` (vanilla's fog + /// colour) and `hz_fog` (density, start, end, 1 / (end - start)). + pub uniform_size: u32, + pub uniforms: Vec, + /// Each sampler: texture at `binding`, sampler object at `binding + 1`. + pub samplers: Vec, + /// The fragment stage's output locations. + pub outputs: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Uniform { + pub name: String, + /// The type the pack declared (`float`, `ivec2`, `mat4`; `bool` is stored as an `int`). + pub ty: String, + pub offset: u32, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Sampler { + pub name: String, + /// The type the pack declared: `sampler2D`, `sampler2DShadow`, `usampler2D`, `sampler3D`... + pub ty: String, + pub binding: u32, +} + +/// A program that failed, at a pack file and line. +#[derive(Clone, Debug, PartialEq)] +pub struct Error { + pub file: String, + pub line: u32, + pub message: String, +} + +pub struct Program { + pub vertex: naga::Module, + pub fragment: naga::Module, + pub interface: Interface, +} + +/// Where the program draws, which decides the conventions of 5.5. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Target { + /// Pack buffers: rows stored as GL stores them. + Buffers, + /// The window (`final`). + Window, +} + +/// Translates a composite-style program (`composite`, `deferred`, `final`...). +pub fn translate_composite( + vertex: &Preprocessed, + fragment: &Preprocessed, + target: Target, +) -> Result> { + let mut v = Stage::scan(vertex, ShaderStage::Vertex); + let mut f = Stage::scan(fragment, ShaderStage::Fragment); + let mut errors: Vec = v.errors.drain(..).chain(f.errors.drain(..)).collect(); + + // One uniform block and one set of samplers for both stages. + let mut uniforms: BTreeMap = BTreeMap::new(); + let mut samplers: BTreeMap = BTreeMap::new(); + for stage in [&v, &f] { + for (name, ty, at) in &stage.uniforms { + if let Some(old) = uniforms.insert(name.clone(), ty.clone()) + && &old != ty + { + errors.push(stage.error( + *at, + &format!("`{name}` is `{old}` in one stage and `{ty}` in the other"), + )); + } + } + for (name, ty) in &stage.samplers { + samplers.insert(name.clone(), ty.clone()); + } + } + let samplers: Vec = samplers + .into_iter() + .enumerate() + .map(|(i, (name, ty))| Sampler { + name, + ty, + binding: 1 + 2 * i as u32, + }) + .collect(); + + // Varyings by name: the vertex outputs, then fragment inputs the vertex stage lacks. + let mut locations: BTreeMap = BTreeMap::new(); + for var in v.outs.iter().chain(&f.ins) { + let next = locations.len() as u32; + locations + .entry(var.name.clone()) + .or_insert((next, var.decl.clone())); + } + + let header = |stage: &Stage| { + let mut h = String::from("#version 450\n"); + h.push_str("layout(set = 0, binding = 0) uniform HzUniforms { vec4 hz_target; vec4 hz_fog_color; vec4 hz_fog;"); + for (name, ty) in &uniforms { + match ty.as_str() { + "bool" => write!(h, " int hz_bool_{name};"), + _ => match ty.split_once('[') { + Some((base, size)) => write!(h, " {base} {name}[{size};"), + None => write!(h, " {ty} {name};"), + }, + } + .unwrap(); + } + h.push_str(" };\n"); + for s in &samplers { + let (texture, sampler) = split_sampler(&s.ty); + writeln!( + h, + "layout(set = 0, binding = {}) uniform {texture} {}; layout(set = 0, binding = {}) uniform {sampler} hz_s_{};", + s.binding, s.name, s.binding + 1, s.name + ) + .unwrap(); + } + let direction = if stage.kind == ShaderStage::Vertex { + "out" + } else { + "in" + }; + for (name, (location, decl)) in &locations { + writeln!( + h, + "layout(location = {location}) {direction} {}", + decl.replace("@", name) + ) + .unwrap(); + } + h.push_str(&stage.prelude()); + h + }; + let flip = if target == Target::Buffers { "-" } else { "" }; + let fog = + " hz_Fog = hz_FogParameters(hz_fog_color, hz_fog.x, hz_fog.y, hz_fog.z, hz_fog.w);\n"; + let mut vertex_main = String::from( + "\nvoid main() {\n vec2 hz_p = hz_quad[gl_VertexIndex];\n hz_Vertex = vec4(hz_p, 0.0, 1.0);\n \ + hz_MultiTexCoord0 = hz_Vertex;\n hz_Color = vec4(1.0);\n hz_Normal = vec3(0.0, 0.0, 1.0);\n", + ); + for (name, value) in &v.attributes { + writeln!(vertex_main, " {name} = {value};").unwrap(); + } + vertex_main.push_str(fog); + for name in &v.inits { + writeln!(vertex_main, " {name} = hz_init_{name}();").unwrap(); + } + write!( + vertex_main, + " hz_main();\n gl_Position = vec4(gl_Position.x, {flip}gl_Position.y, (gl_Position.z + gl_Position.w) * 0.5, gl_Position.w);\n}}\n" + ) + .unwrap(); + let mut fragment_main = String::from("\nvoid main() {\n"); + fragment_main.push_str(fog); + if target == Target::Window && f.frag_coord { + fragment_main.push_str(" hz_FragCoord = vec4(gl_FragCoord.x, hz_target.y - gl_FragCoord.y, gl_FragCoord.zw);\n"); + } + for name in &f.inits { + writeln!(fragment_main, " {name} = hz_init_{name}();").unwrap(); + } + fragment_main.push_str(" hz_main();\n}\n"); + + let v_text = format!( + "{}{}{vertex_main}", + header(&v), + v.rewrite(&samplers, &uniforms) + ); + let f_text = format!( + "{}{}{fragment_main}", + header(&f), + f.rewrite(&samplers, &uniforms) + ); + if !errors.is_empty() { + return Err(errors); + } + let vertex_module = compile(&v, &v_text, header(&v).lines().count()); + let fragment_module = compile(&f, &f_text, header(&f).lines().count()); + let (vertex, fragment) = match (vertex_module, fragment_module) { + (Ok(v), Ok(f)) => (v, f), + (a, b) => return Err(a.err().into_iter().chain(b.err()).flatten().collect()), + }; + let (uniform_size, offsets) = block_layout(&fragment); + let uniforms = offsets + .into_iter() + .filter_map(|(member, offset)| { + let name = member + .strip_prefix("hz_bool_") + .unwrap_or(&member) + .to_owned(); + let ty = if name.starts_with("hz_") { + "vec4".to_owned() + } else { + uniforms.get(&name)?.clone() + }; + Some(Uniform { name, ty, offset }) + }) + .collect(); + let outputs = f.outputs.iter().copied().collect(); + Ok(Program { + vertex, + fragment, + interface: Interface { + uniform_size, + uniforms, + samplers, + outputs, + }, + }) +} + +/// `sampler2DShadow` → (`texture2D`, `samplerShadow`); `isampler3D` → (`itexture3D`, `sampler`). +pub(super) fn split_sampler(ty: &str) -> (String, &'static str) { + let shadow = ty.ends_with("Shadow"); + let base = ty + .trim_end_matches("Shadow") + .replacen("sampler", "texture", 1); + (base, if shadow { "samplerShadow" } else { "sampler" }) +} + +/// Parses and validates one stage; errors at the pack's files and lines. +fn compile(stage: &Stage, text: &str, header_lines: usize) -> Result> { + let at = |span: naga::Span, message: String| { + let start = span.to_range().map_or(0, |r| r.start); + let line = text[..start.min(text.len())].matches('\n').count(); + match line + .checked_sub(header_lines) + .and_then(|l| stage.pp.lines.get(l)) + { + Some(o) => Error { + file: stage + .pp + .files + .get(o.file as usize) + .cloned() + .unwrap_or_default(), + line: o.line, + message, + }, + None => Error { + file: "".into(), + line: line as u32 + 1, + message, + }, + } + }; + let module = Frontend::default() + .parse(&Options::from(stage.kind), text) + .map_err(|e| { + e.errors + .into_iter() + .map(|e| at(e.meta, e.kind.to_string())) + .collect::>() + })?; + Validator::new(ValidationFlags::all(), Capabilities::all()) + .validate(&module) + .map_err(|e| { + let span = e.spans().next().map_or(naga::Span::default(), |(s, _)| *s); + // The validator nests its reasons: say them all. + let inner = e.into_inner(); + let mut message = inner.to_string(); + let mut source = std::error::Error::source(&inner); + while let Some(s) = source { + write!(message, ": {s}").unwrap(); + source = s.source(); + } + vec![at(span, message)] + })?; + Ok(module) +} + +/// The uniform block's size and member offsets, from naga's layout. +fn block_layout(module: &naga::Module) -> (u32, Vec<(String, u32)>) { + for (_, var) in module.global_variables.iter() { + if var.space != naga::AddressSpace::Uniform { + continue; + } + if let naga::TypeInner::Struct { members, span } = &module.types[var.ty].inner { + let offsets = members + .iter() + .filter_map(|m| Some((m.name.clone()?, m.offset))) + .collect(); + return (*span, offsets); + } + } + (16, vec![("hz_target".into(), 0)]) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pack::tests::zip; + use crate::{Macros, Pack, preprocess}; + + fn program(files: &[(&str, &str)], target: Target) -> Result> { + let files: Vec<(String, &str)> = files + .iter() + .map(|(p, t)| (format!("shaders/{p}"), *t)) + .collect(); + let files: Vec<(&str, &str)> = files.iter().map(|(p, t)| (p.as_str(), *t)).collect(); + let pack = Pack::from_zip_bytes("P.zip", zip(&files)).unwrap(); + let m = Macros::standard(); + translate_composite( + &preprocess(&pack, "composite.vsh", &m), + &preprocess(&pack, "composite.fsh", &m), + target, + ) + } + + const VSH_120: &str = "#version 120\nvarying vec2 texcoord;\nvoid main() {\n gl_Position = ftransform();\n \ + texcoord = (gl_TextureMatrix[0] * gl_MultiTexCoord0).xy;\n}\n"; + + #[test] + fn compatibility_programs_translate() { + let fsh = "#version 120\nuniform sampler2D colortex0;\nuniform sampler2D depthtex0;\nuniform float viewWidth;\n\ + uniform bool hideGUI;\nuniform int frameCounter;\nvarying vec2 texcoord;\n\ + vec3 tint(sampler2D s, vec2 c) { return texture2D(s, c).rgb; }\nvoid main() {\n \ + vec3 c = tint(colortex0, texcoord) * texture2D(depthtex0, texcoord).r;\n if (hideGUI) c *= 0.5;\n \ + /* DRAWBUFFERS:04 */\n gl_FragData[0] = vec4(c, 1.0);\n gl_FragData[1] = vec4(frameCounter / viewWidth);\n}\n"; + let p = program( + &[("composite.vsh", VSH_120), ("composite.fsh", fsh)], + Target::Buffers, + ) + .unwrap(); + let i = &p.interface; + let samplers: Vec<(&str, u32)> = i + .samplers + .iter() + .map(|s| (s.name.as_str(), s.binding)) + .collect(); + assert_eq!(samplers, [("colortex0", 1), ("depthtex0", 3)]); + let uniforms: Vec<(&str, &str)> = i + .uniforms + .iter() + .map(|u| (u.name.as_str(), u.ty.as_str())) + .collect(); + assert_eq!( + uniforms, + [ + ("hz_target", "vec4"), + ("hz_fog_color", "vec4"), + ("hz_fog", "vec4"), + ("frameCounter", "int"), + ("hideGUI", "bool"), + ("viewWidth", "float") + ] + ); + assert_eq!(i.uniforms[0].offset, 0); + assert!(i.uniform_size >= 28); + assert_eq!(i.outputs, [0, 1]); + } + + #[test] + fn core_programs_translate() { + let vsh = "#version 330 core\nin vec3 vaPosition;\nin vec2 vaUV0;\nuniform mat4 modelViewMatrix, projectionMatrix;\n\ + out vec2 uv;\nflat out int flatId;\nvoid main() {\n \ + gl_Position = projectionMatrix * modelViewMatrix * vec4(vaPosition, 1.0);\n uv = vaUV0;\n flatId = 1;\n}\n"; + let fsh = "#version 330 core\nin vec2 uv;\nuniform sampler2D colortex0;\nuniform usampler2D colortex5;\n\ + layout(location = 2) out vec4 color;\nvoid main() {\n \ + if (texture(colortex0, uv).a < 0.1) discard;\n color = texture(colortex0, uv) + float(texelFetch(colortex5, ivec2(gl_FragCoord.xy), 0).r);\n}\n"; + let p = program( + &[("composite.vsh", vsh), ("composite.fsh", fsh)], + Target::Window, + ) + .unwrap(); + assert_eq!(p.interface.outputs, [2]); + assert!( + p.interface + .uniforms + .iter() + .any(|u| u.name == "projectionMatrix" && u.ty == "mat4") + ); + } + + #[test] + fn errors_point_at_the_pack_file_and_line() { + let fsh = + "#version 120\n#include \"/lib/bad.glsl\"\nvoid main() { gl_FragColor = vec4(f()); }\n"; + let bad = "// fine\nfloat f() {\n return undefinedThing;\n}\n"; + let errors = program( + &[ + ("composite.vsh", VSH_120), + ("composite.fsh", fsh), + ("lib/bad.glsl", bad), + ], + Target::Buffers, + ) + .err() + .unwrap(); + assert_eq!( + (errors[0].file.as_str(), errors[0].line), + ("lib/bad.glsl", 3), + "{errors:?}" + ); + } +} diff --git a/crates/horizon-shaders/src/translate/rewrite.rs b/crates/horizon-shaders/src/translate/rewrite.rs new file mode 100644 index 000000000..312575139 --- /dev/null +++ b/crates/horizon-shaders/src/translate/rewrite.rs @@ -0,0 +1,257 @@ +//! The second pass: the stage's code written again for naga, line for line, and the +//! declarations of ours it needs. + +use std::collections::{BTreeMap, HashMap}; +use std::fmt::Write; + +use naga::ShaderStage; + +use super::scan::{LATER_KEYWORDS, Stage, UNIFORM_STATE, VERTEX_INPUTS, is_ident, significant}; +use super::{Sampler, split_sampler}; + +impl Stage<'_> { + /// Declarations of ours this stage uses (after the uniform block, samplers and varyings). + pub(super) fn prelude(&self) -> String { + let mut p = String::from( + "float hz_modf(float x, out float i) { i = trunc(x); return x - i; }\n\ + vec2 hz_modf(vec2 x, out vec2 i) { i = trunc(x); return x - i; }\n\ + vec3 hz_modf(vec3 x, out vec3 i) { i = trunc(x); return x - i; }\n\ + vec4 hz_modf(vec4 x, out vec4 i) { i = trunc(x); return x - i; }\n\ + struct hz_FogParameters { vec4 color; float density; float start; float end; float scale; };\n\ + hz_FogParameters hz_Fog;\n\ + const mat4 hz_ModelViewMatrix = mat4(1.0); const mat4 hz_ProjectionMatrix = mat4(2.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, -1.0, -1.0, 0.0, 1.0);\n\ + const mat4 hz_ModelViewProjectionMatrix = hz_ProjectionMatrix; const mat4 hz_ModelViewMatrixInverse = mat4(1.0); const mat3 hz_NormalMatrix = mat3(1.0);\n\ + const mat4 hz_ProjectionMatrixInverse = mat4(0.5, 0.0, 0.0, 0.0, 0.0, 0.5, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.5, 0.5, 0.0, 1.0);\n\ + const mat4 hz_TextureMatrix[8] = mat4[8](mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0));\n", + ); + if self.kind == ShaderStage::Vertex { + p.push_str( + "const vec2 hz_quad[6] = vec2[6](vec2(0.0, 0.0), vec2(1.0, 0.0), vec2(1.0, 1.0), vec2(0.0, 0.0), vec2(1.0, 1.0), vec2(0.0, 1.0));\n\ + vec4 hz_Vertex; vec4 hz_Color; vec4 hz_SecondaryColor; vec3 hz_Normal; float hz_FogCoord;\n\ + vec4 hz_MultiTexCoord0; vec4 hz_MultiTexCoord1; vec4 hz_MultiTexCoord2; vec4 hz_MultiTexCoord3;\n\ + vec4 hz_MultiTexCoord4; vec4 hz_MultiTexCoord5; vec4 hz_MultiTexCoord6; vec4 hz_MultiTexCoord7;\n", + ); + } else { + for n in &self.outputs { + if !self.frag_outs.iter().any(|(at, _)| at == n) { + writeln!(p, "layout(location = {n}) out vec4 hz_out{n};").unwrap(); + } + } + for (_, decl) in &self.frag_outs { + writeln!(p, "{decl}").unwrap(); + } + p.push_str("vec4 hz_FragCoord;\n"); + } + p + } + + /// The pack's code with the rewrites above; the same number of lines. + pub(super) fn rewrite( + &self, + samplers: &[Sampler], + uniforms: &BTreeMap, + ) -> String { + let removed = |i: usize| self.removed.iter().any(|r| r.contains(&i)); + let sampler_ty: HashMap<&str, &str> = samplers + .iter() + .map(|s| (s.name.as_str(), s.ty.as_str())) + .collect(); + let old = self.pp.version.as_ref().is_none_or(|(v, _)| *v < 400); + let fragment = self.kind == ShaderStage::Fragment; + let mut out = String::with_capacity(self.pp.code.len() + 1024); + // Per open bracket: the function called, and the closing text to add. + let mut calls: Vec<(&str, &str, usize)> = Vec::new(); + // Sampler parameters of the function being written: name → type. + let mut params: HashMap = HashMap::new(); + let mut depth = 0; + let mut skip_until = 0; + let next = |i: usize| (i + 1..self.tokens.len()).find(|&j| significant(self.tokens[j])); + let prev = |i: usize| (0..i).rev().find(|&j| significant(self.tokens[j])); + for (i, &t) in self.tokens.iter().enumerate() { + if i < skip_until { + continue; + } + if let Some(text) = self.replaced.get(&i) { + out.push_str(text); + continue; + } + if removed(i) { + out.extend(t.chars().filter(|&c| c == '\n')); + if t.contains(' ') && !t.contains('\n') { + out.push(' '); + } + continue; + } + let called = next(i).is_some_and(|j| self.tokens[j] == "("); + match t { + "{" => depth += 1, + "}" => { + depth -= 1; + if depth == 0 { + params.clear(); + } + } + "(" => { + let callee = prev(i).map_or("", |j| self.tokens[j]); + let close = if matches!(callee, "shadow2D" | "shadow2DLod" | "shadow2DProj") { + ")" + } else { + "" + }; + calls.push((callee, close, 0)); + } + ")" => { + if let Some((_, close, _)) = calls.pop() { + out.push_str(close); + } + } + "," => { + if let Some(c) = calls.last_mut() { + c.2 += 1; + } + } + _ => {} + } + // `T x[] = T[](a, b)`: naga gives unsized arrays no size, so the count is written in. + if t == "[" + && let Some(close) = next(i).filter(|&j| self.tokens[j] == "]") + { + if let Some(n) = self.array_length(close) { + write!(out, "[{n}]").unwrap(); + skip_until = close + 1; + continue; + } + } + if !is_ident(t) { + out.push_str(t); + continue; + } + // A function definition's sampler parameters: `sampler2D s` → `texture2D s, sampler hz_s_s`. + if depth == 0 && t.contains("sampler") && calls.len() == 1 { + if let Some(n) = next(i).filter(|&n| is_ident(self.tokens[n])) { + let name = self.tokens[n]; + let (texture, sampler) = split_sampler(t); + write!(out, "{texture} {name}, {sampler} hz_s_{name}").unwrap(); + params.insert(name.to_owned(), t.to_owned()); + skip_until = n + 1; + continue; + } + } + let sampler = params + .get(t) + .map(String::as_str) + .or_else(|| sampler_ty.get(t).copied()); + if let Some(ty) = sampler { + let as_argument = calls.last().is_some_and(|(callee, _, arg)| { + self.sampler_functions + .get(*callee) + .is_some_and(|p| p.get(*arg) == Some(&true)) + }) && prev(i) + .is_some_and(|j| matches!(self.tokens[j], "(" | ",")); + if as_argument { + write!(out, "{t}, hz_s_{t}").unwrap(); + } else { + write!(out, "{ty}({t}, hz_s_{t})").unwrap(); + } + continue; + } + if uniforms.get(t).is_some_and(|ty| ty == "bool") { + write!(out, "(hz_bool_{t} != 0)").unwrap(); + continue; + } + let replaced: Option = match t { + "main" if depth == 0 && called => Some("hz_main".into()), + // GLSL's `modf`, which naga's frontend lacks. + "modf" if called => Some("hz_modf".into()), + "texture2D" | "texture1D" | "texture3D" | "textureCube" if called => { + Some("texture".into()) + } + "texture2DLod" | "texture3DLod" | "texture2DLodARB" | "textureCubeLod" + if called => + { + Some("textureLod".into()) + } + "texture2DProj" if called => Some("textureProj".into()), + "texture2DGrad" | "texture2DGradARB" if called => Some("textureGrad".into()), + "shadow2D" if called => Some("vec4(texture".into()), + "shadow2DLod" if called => Some("vec4(textureLod".into()), + "shadow2DProj" if called => Some("vec4(textureProj".into()), + "ftransform" if called => { + // `ftransform()` → the product it stands for; its `()` stays as a no-op call... so skip it. + let close = next(i).and_then(next); + skip_until = close.map_or(i + 1, |c| c + 1); + Some("(hz_ModelViewProjectionMatrix * hz_Vertex)".into()) + } + "gl_FragColor" => Some("hz_out0".into()), + "gl_FragData" | "gl_TexCoord" => match self.index_after(i) { + Some((n, close)) => { + skip_until = close + 1; + Some(match t { + "gl_FragData" => format!("hz_out{n}"), + _ => format!("hz_TexCoord{n}"), + }) + } + None => None, + }, + "gl_Color" if fragment => Some("hz_FrontColor".into()), + "gl_FragCoord" if !fragment => None, + "gl_FragCoord" => Some("hz_FragCoord".into()), + "gl_FrontColor" => Some("hz_FrontColor".into()), + "gl_Fog" => Some("hz_Fog".into()), + _ if UNIFORM_STATE.iter().any(|b| t.starts_with(b)) => { + Some(format!("hz_{}", &t[3..])) + } + _ if !fragment && VERTEX_INPUTS.iter().any(|b| t.starts_with(b)) => { + Some(format!("hz_{}", &t[3..])) + } + "gl_VertexID" => Some("gl_VertexIndex".into()), + "gl_InstanceID" => Some("gl_InstanceIndex".into()), + _ if old && LATER_KEYWORDS.contains(&t) => Some(format!("hz_kw_{t}")), + _ => None, + }; + out.push_str(replaced.as_deref().unwrap_or(t)); + } + out + } +} + +impl Stage<'_> { + /// The element count of an unsized array whose `[]` ends at `close`: the arguments of the + /// constructor that follows (`[] = T[](...)` or `[](...)`). + pub(super) fn array_length(&self, close: usize) -> Option { + let mut j = close + 1; + let mut seen_eq = false; + while let Some(&t) = self.tokens.get(j) { + match t { + "(" => break, + "=" if !seen_eq => seen_eq = true, + ";" | "," | ")" | "{" => return None, + _ => {} + } + j += 1; + } + let (mut depth, mut args, mut any) = (0, 1, false); + for &t in &self.tokens[j + 1..] { + match t { + "(" | "[" | "{" => depth += 1, + ")" if depth == 0 => return any.then_some(args), + ")" | "]" | "}" => depth -= 1, + "," if depth == 0 => args += 1, + _ if significant(t) => any = true, + _ => {} + } + } + None + } +} + +/// What our vertex `main` gives a vertex input of a composite-style program. +pub(super) fn attribute_value(name: &str, ty: &str) -> String { + match name { + "vaPosition" => "vec3(hz_p, 0.0)".into(), + "vaUV0" => "hz_p".into(), + "vaColor" => "vec4(1.0)".into(), + "vaNormal" => "vec3(0.0, 0.0, 1.0)".into(), + _ => format!("{ty}(0)"), + } +} diff --git a/crates/horizon-shaders/src/translate/scan.rs b/crates/horizon-shaders/src/translate/scan.rs new file mode 100644 index 000000000..1fd91dd94 --- /dev/null +++ b/crates/horizon-shaders/src/translate/scan.rs @@ -0,0 +1,399 @@ +//! The first pass over a stage's tokens: what it declares at global scope, which of its +//! functions take samplers, which outputs and built-ins it uses. + +use std::collections::{BTreeSet, HashMap}; +use std::fmt::Write; + +use naga::ShaderStage; + +use super::Error; +use super::rewrite::attribute_value; +use crate::Preprocessed; +use crate::macros::lex; + +/// A varying: its name and its declaration with `@` where the name goes (`flat ivec2 @[2];`). +pub(super) struct Var { + pub(super) name: String, + pub(super) decl: String, +} + +pub(super) struct Stage<'p> { + pub(super) kind: ShaderStage, + pub(super) pp: &'p Preprocessed, + pub(super) tokens: Vec<&'p str>, + /// Token ranges to blank out (declarations moved into the header). + pub(super) removed: Vec>, + /// Name, type with its array suffix, token of the type. + pub(super) uniforms: Vec<(String, String, usize)>, + pub(super) samplers: Vec<(String, String)>, + pub(super) ins: Vec, + pub(super) outs: Vec, + /// Vertex inputs, now private globals, and the value our `main` gives each. + pub(super) attributes: Vec<(String, String)>, + /// The pack's fragment outputs, declared again with their locations. + pub(super) frag_outs: Vec<(u32, String)>, + /// The pack's functions that take samplers: which parameters are samplers. + pub(super) sampler_functions: HashMap>, + pub(super) outputs: BTreeSet, + pub(super) frag_coord: bool, + /// Tokens written differently, by index. + pub(super) replaced: HashMap, + /// Globals whose initializer runs in our `main` (`g = hz_init_g();`), in order. + pub(super) inits: Vec, + pub(super) errors: Vec, +} + +pub(super) const STORAGE: [&str; 5] = ["uniform", "in", "out", "attribute", "varying"]; +pub(super) const QUALIFIERS: [&str; 11] = [ + "flat", + "smooth", + "noperspective", + "centroid", + "invariant", + "highp", + "mediump", + "lowp", + "precise", + "sample", + "const", +]; +/// The compatibility profile's uniform state (with its `Inverse`/`Transpose` forms). +pub(super) const UNIFORM_STATE: [&str; 5] = [ + "gl_ModelViewMatrix", + "gl_ProjectionMatrix", + "gl_ModelViewProjectionMatrix", + "gl_NormalMatrix", + "gl_TextureMatrix", +]; +/// The compatibility profile's vertex inputs. +pub(super) const VERTEX_INPUTS: [&str; 6] = [ + "gl_Vertex", + "gl_Color", + "gl_SecondaryColor", + "gl_Normal", + "gl_FogCoord", + "gl_MultiTexCoord", +]; +/// Words that became keywords after GLSL 1.x and that old packs use as names. +pub(super) const LATER_KEYWORDS: [&str; 17] = [ + "sample", + "buffer", + "shared", + "filter", + "input", + "output", + "common", + "partition", + "active", + "resource", + "patch", + "subroutine", + "coherent", + "volatile", + "restrict", + "readonly", + "writeonly", +]; + +pub(super) fn significant(t: &str) -> bool { + !t.trim().is_empty() +} + +pub(super) fn is_ident(t: &str) -> bool { + t.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_') +} + +impl<'p> Stage<'p> { + pub(super) fn scan(pp: &'p Preprocessed, kind: ShaderStage) -> Self { + let mut s = Stage { + kind, + pp, + tokens: lex(&pp.code), + removed: Vec::new(), + uniforms: Vec::new(), + samplers: Vec::new(), + ins: Vec::new(), + outs: Vec::new(), + attributes: Vec::new(), + frag_outs: Vec::new(), + sampler_functions: HashMap::new(), + outputs: BTreeSet::new(), + frag_coord: false, + replaced: HashMap::new(), + inits: Vec::new(), + errors: Vec::new(), + }; + let (mut start, mut depth, mut parens) = (0, 0usize, 0usize); + for i in 0..s.tokens.len() { + match s.tokens[i] { + "(" => parens += 1, + ")" => parens = parens.saturating_sub(1), + "{" if depth == 0 && parens == 0 => { + s.global(start, i, true); + depth = 1; + } + "{" => depth += 1, + "}" if depth > 0 => { + depth -= 1; + if depth == 0 { + start = i + 1; + } + } + ";" if depth == 0 && parens == 0 => { + s.global(start, i, false); + start = i + 1; + } + "gl_FragData" | "gl_FragColor" if kind == ShaderStage::Fragment => { + let n = if s.tokens[i] == "gl_FragColor" { + Some(0) + } else { + s.index_after(i).map(|(n, _)| n) + }; + match n { + Some(n) => _ = s.outputs.insert(n), + None => s + .errors + .push(s.error(i, "`gl_FragData` needs a constant index")), + } + } + "gl_FragCoord" => s.frag_coord = true, + "gl_TexCoord" => { + if let Some((n, _)) = s.index_after(i) { + s.builtin_varying(format!("hz_TexCoord{n}")); + } + } + "gl_FrontColor" => s.builtin_varying("hz_FrontColor".into()), + "gl_Color" if kind == ShaderStage::Fragment => { + s.builtin_varying("hz_FrontColor".into()) + } + _ => {} + } + } + s + } + + pub(super) fn builtin_varying(&mut self, name: String) { + let list = if self.kind == ShaderStage::Vertex { + &mut self.outs + } else { + &mut self.ins + }; + if !list.iter().any(|v| v.name == name) { + list.push(Var { + name, + decl: "vec4 @;".into(), + }); + } + } + + /// `[ n ]` right after token `i`: the number and the index of the `]`. + pub(super) fn index_after(&self, i: usize) -> Option<(u32, usize)> { + let mut it = (i + 1..self.tokens.len()).filter(|&j| significant(self.tokens[j])); + let open = it.next()?; + let n = it.next()?; + let close = it.next()?; + (self.tokens[open] == "[" && self.tokens[close] == "]") + .then(|| self.tokens[n].parse().ok().map(|n| (n, close))) + .flatten() + } + + pub(super) fn error(&self, token: usize, message: &str) -> Error { + let line: usize = self.tokens[..token.min(self.tokens.len())] + .iter() + .map(|t| t.matches('\n').count()) + .sum(); + let o = self.pp.lines.get(line).copied().unwrap_or_default(); + Error { + file: self + .pp + .files + .get(o.file as usize) + .cloned() + .unwrap_or_default(), + line: o.line, + message: message.to_owned(), + } + } + + /// A statement at global scope, tokens `from..to` (without its `;`, or before a `{`). + pub(super) fn global(&mut self, from: usize, to: usize, body: bool) { + let words: Vec = (from..to) + .filter(|&i| significant(self.tokens[i])) + .collect(); + // A prototype: a `(` before any `=`, and no storage qualifier. + let paren = words + .iter() + .position(|&i| matches!(self.tokens[i], "(" | "=")); + let prototype = paren.is_some_and(|p| self.tokens[words[p]] == "(") + && !words.iter().any(|&i| STORAGE.contains(&self.tokens[i])); + if body || prototype { + return self.function(&words); + } + let mut k = 0; + let mut storage = None; + let mut location = None; + let mut interpolation = String::new(); + while let Some(&i) = words.get(k) { + let w = self.tokens[i]; + if w == "layout" { + // layout(location = n, ...) + let mut j = k + 1; + while let Some(&t) = words.get(j) { + j += 1; + if self.tokens[t] == "location" { + location = words + .get(j + 1) + .and_then(|&n| self.tokens[n].parse::().ok()); + } + if self.tokens[t] == ")" { + break; + } + } + k = j; + continue; + } + if STORAGE.contains(&w) { + storage = Some(w); + } else if QUALIFIERS.contains(&w) { + if matches!( + w, + "flat" | "smooth" | "noperspective" | "centroid" | "sample" + ) { + write!(interpolation, "{w} ").unwrap(); + } + } else { + break; + } + k += 1; + } + let Some(storage) = storage else { + return self.initializer(&words, to); + }; + let Some(&ty_at) = words.get(k) else { return }; + let ty = self.tokens[ty_at].to_owned(); + if words[k..].iter().any(|&i| self.tokens[i] == "{") { + return; // An interface block: left as the pack wrote it. + } + // Declarators: name [array] [= init], separated by commas. + let mut declarators: Vec<(String, String)> = Vec::new(); + let mut j = k + 1; + while let Some(&n) = words.get(j) { + let name = self.tokens[n].to_owned(); + let mut array = String::new(); + j += 1; + let mut nesting = 0; + while let Some(&t) = words.get(j) { + match self.tokens[t] { + "," if nesting == 0 => break, + "(" | "[" => nesting += 1, + ")" | "]" => nesting -= 1, + _ => {} + } + if self.tokens[t] == "=" && nesting == 0 { + // An initializer: uniforms are set by the game, so it is dropped. + while words.get(j).is_some_and(|&t| self.tokens[t] != ",") { + j += 1; + } + break; + } + array.push_str(self.tokens[t]); + j += 1; + } + j += 1; + declarators.push((name, array)); + } + let fragment = self.kind == ShaderStage::Fragment; + let input = matches!(storage, "in" | "attribute") || (storage == "varying" && fragment); + self.removed.push(from..to + 1); + for (name, array) in declarators { + let decl = format!("{interpolation}{ty} @{array};"); + match storage { + "uniform" if ty.contains("sampler") => self.samplers.push((name, ty.clone())), + "uniform" if ty.contains("image") => self + .errors + .push(self.error(ty_at, &format!("`{name}`: images are not supported yet"))), + "uniform" => self.uniforms.push((name, format!("{ty}{array}"), ty_at)), + _ if input && !fragment => { + let value = attribute_value(&name, &ty); + self.attributes.push((name, value)); + self.removed.pop(); + // A private global of ours now: `in`/`attribute` and qualifiers go. + for &w in &words[..k] { + self.removed.push(w..w + 1); + } + } + _ if input => self.ins.push(Var { name, decl }), + _ if fragment => { + let n = location.unwrap_or(self.outputs.len() as u32); + self.outputs.insert(n); + self.frag_outs + .push((n, format!("layout(location = {n}) out {ty} {name}{array};"))); + } + _ => self.outs.push(Var { name, decl }), + } + } + } + + /// A global `T name = a ? b : c;`: naga can't evaluate a conditional at global scope, so + /// the initializer becomes a function of ours on the same line (`T name; T hz_init_name() { + /// return a ? b : c; }`), called by our `main` before the pack's. + pub(super) fn initializer(&mut self, words: &[usize], to: usize) { + let text = |i: usize| self.tokens[i]; + let Some(eq) = words.iter().position(|&i| text(i) == "=") else { + return; + }; + if !words[eq..].iter().any(|&i| text(i) == "?") + || words[..eq].iter().any(|&i| text(i) == ",") + { + return; + } + let Some(name_at) = words[..eq].iter().rposition(|&i| is_ident(text(i))) else { + return; + }; + let name = text(words[name_at]).to_owned(); + let array: String = words[name_at + 1..eq].iter().map(|&i| text(i)).collect(); + let ty: Vec<&str> = words[..name_at] + .iter() + .map(|&i| text(i)) + .filter(|&t| t != "const") + .collect(); + let Some(ty) = ty.last() else { return }; + for &i in &words[..name_at] { + if text(i) == "const" { + self.removed.push(i..i + 1); + } + } + self.replaced.insert( + words[eq], + format!("; {ty}{array} hz_init_{name}() {{ return"), + ); + self.replaced.insert(to, "; }".into()); + self.inits.push(name); + } + + /// A function definition or prototype: remember which parameters are samplers. + pub(super) fn function(&mut self, words: &[usize]) { + let Some(open) = words.iter().position(|&i| self.tokens[i] == "(") else { + return; + }; + let Some(&name) = open.checked_sub(1).and_then(|n| words.get(n)) else { + return; + }; + let mut params = Vec::new(); + let mut is_sampler = false; + for &i in &words[open + 1..] { + match self.tokens[i] { + "," | ")" => { + params.push(is_sampler); + is_sampler = false; + } + t if t.contains("sampler") => is_sampler = true, + _ => {} + } + } + if params.iter().any(|&p| p) { + self.sampler_functions + .insert(self.tokens[name].to_owned(), params); + } + } +} diff --git a/crates/horizon-shaders/tests/real_packs.rs b/crates/horizon-shaders/tests/real_packs.rs index c252df6e3..4fac4b0d7 100644 --- a/crates/horizon-shaders/tests/real_packs.rs +++ b/crates/horizon-shaders/tests/real_packs.rs @@ -5,7 +5,11 @@ use std::path::Path; use std::time::Instant; -use horizon_shaders::{Lang, Macros, Pack, Properties, discover, preprocess}; +use std::collections::BTreeMap; + +use horizon_shaders::{ + Lang, Macros, Pack, Properties, Target, discover, preprocess, translate_composite, +}; #[test] #[ignore = "needs HORIZON_TEST_SHADER_PACKS"] @@ -86,6 +90,73 @@ fn real_packs_read() { for d in diagnostics.iter().take(8) { println!(" {d}"); } + // Every composite-style program with both stages, translated and validated. + let start = Instant::now(); + let (mut ok, mut failed, mut causes) = (0, 0, BTreeMap::::new()); + for dir in ["", "world0", "world-1", "world1"] { + for file in pack.list(dir) { + let Some(name) = file.strip_suffix(".fsh") else { + continue; + }; + let composite = [ + "composite", + "deferred", + "final", + "prepare", + "begin", + "shadowcomp", + ] + .iter() + .any(|p| { + name.strip_prefix(p) + .is_some_and(|n| n.chars().all(|c| c.is_ascii_digit())) + }); + let path = |ext: &str| { + if dir.is_empty() { + format!("{name}.{ext}") + } else { + format!("{dir}/{name}.{ext}") + } + }; + if !composite || !pack.exists(&path("vsh")) { + continue; + } + let target = if name == "final" { + Target::Window + } else { + Target::Buffers + }; + let v = preprocess(&pack, &path("vsh"), ¯os); + let fr = preprocess(&pack, &path("fsh"), ¯os); + match translate_composite(&v, &fr, target) { + Ok(_) => ok += 1, + Err(errors) => { + failed += 1; + let first = &errors[0]; + let cause: String = first.message.chars().take(90).collect(); + *causes.entry(cause).or_default() += 1; + if std::env::var_os("SHOW_ERRORS").is_some() { + println!( + " {}: {}:{}: {}", + path("fsh"), + first.file, + first.line, + first.message + ); + } + } + } + } + } + println!( + " composite-style programs: {ok} translate, {failed} fail, {:.0} ms", + start.elapsed().as_secs_f64() * 1000.0 + ); + let mut causes: Vec<_> = causes.into_iter().collect(); + causes.sort_by_key(|c| std::cmp::Reverse(c.1)); + for (cause, n) in causes.iter().take(6) { + println!(" {n} x {cause}"); + } assert!(!shaders.entries.is_empty(), "{}: no keys read", f.name); read += 1; }