From eb0a6f74e6d5911cc63e637b6dcd14076c28ac0e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:42:05 +0200 Subject: [PATCH 1/5] Shader packs, clean room 2: macros and #if conditions The integer arithmetic of #if/#elif that a pack's .properties files and, later, its GLSL preprocessor share: defined X and defined(X), decimal, hexadecimal and octal literals, the C operators with C precedence, macros expanded to their values (a macro is not expanded inside its own expansion), and an identifier that is not a macro read as 0. A bad condition says why. The standard macros that need no renderer: MC_VERSION, from the pinned version by the documented rule (26.3 is 260300), and the system's MC_OS_*. Nothing uses it yet: players see no change. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/horizon-shaders/Cargo.toml | 1 + crates/horizon-shaders/src/condition.rs | 265 ++++++++++++++++++++++++ crates/horizon-shaders/src/lib.rs | 5 + crates/horizon-shaders/src/macros.rs | 131 ++++++++++++ 5 files changed, 403 insertions(+) create mode 100644 crates/horizon-shaders/src/condition.rs create mode 100644 crates/horizon-shaders/src/macros.rs diff --git a/Cargo.lock b/Cargo.lock index 6c7279aa7..7036ea5ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1434,6 +1434,7 @@ dependencies = [ name = "horizon-shaders" version = "0.1.0" dependencies = [ + "horizon-core", "zip", ] diff --git a/crates/horizon-shaders/Cargo.toml b/crates/horizon-shaders/Cargo.toml index ed3a24266..750949e83 100644 --- a/crates/horizon-shaders/Cargo.toml +++ b/crates/horizon-shaders/Cargo.toml @@ -7,6 +7,7 @@ rust-version = "1.85" description = "Shader packs (the shaders/ folder format): reading packs, their .properties and .lang files, and later their GLSL" [dependencies] +horizon-core = { path = "../horizon-core" } zip = { version = "2", default-features = false, features = ["deflate"] } [lints] diff --git a/crates/horizon-shaders/src/condition.rs b/crates/horizon-shaders/src/condition.rs new file mode 100644 index 000000000..b911eb985 --- /dev/null +++ b/crates/horizon-shaders/src/condition.rs @@ -0,0 +1,265 @@ +//! The integer arithmetic of `#if` and `#elif` (spec 1.5, GLSL 4.60 §3.3): `defined`, integer +//! literals, macros expanded to their values, and the C operators with C precedence. +//! An identifier that is not a macro counts as 0, as in C: GLSL would reject the line, and a +//! pack would then lose a whole section of its settings over one unknown name. + +use crate::Macros; +use crate::macros::{is_ident, is_ident_start}; + +/// The value of a condition; `Err` says what is wrong with it. +pub fn eval_condition(expr: &str, macros: &Macros) -> Result { + let tokens = tokens(&resolve_defined(expr, macros)?, macros)?; + let mut parser = Parser { + tokens: &tokens, + at: 0, + }; + let value = parser.binary(0)?; + match parser.tokens.get(parser.at) { + None => Ok(value), + Some(t) => Err(format!("unexpected `{}`", t.text())), + } +} + +/// `defined X` and `defined(X)` become `1` or `0` before anything is expanded. +fn resolve_defined(expr: &str, macros: &Macros) -> Result { + let mut out = String::with_capacity(expr.len()); + let mut rest = expr; + while let Some(at) = find_word(rest, "defined") { + out.push_str(&rest[..at]); + let after = rest[at + "defined".len()..].trim_start(); + let (inner, tail) = match after.strip_prefix('(') { + Some(open) => { + let close = open.find(')').ok_or("`defined(` without `)`")?; + (open[..close].trim(), &open[close + 1..]) + } + None => { + let len = after.find(|c: char| !is_ident(c)).unwrap_or(after.len()); + (&after[..len], &after[len..]) + } + }; + if inner.is_empty() || !inner.starts_with(is_ident_start) || !inner.chars().all(is_ident) { + return Err("`defined` needs a macro name".into()); + } + out.push_str(if macros.is_defined(inner) { + " 1 " + } else { + " 0 " + }); + rest = tail; + } + out.push_str(rest); + Ok(out) +} + +/// Where `word` starts in `text` as a whole identifier. +fn find_word(text: &str, word: &str) -> Option { + let mut from = 0; + while let Some(i) = text[from..].find(word).map(|i| i + from) { + let before = text[..i].chars().next_back().is_some_and(is_ident); + let after = text[i + word.len()..].chars().next().is_some_and(is_ident); + if !before && !after { + return Some(i); + } + from = i + word.len(); + } + None +} + +#[derive(Clone, Copy, Debug, PartialEq)] +enum Token { + Num(i64), + Op(&'static str), +} + +impl Token { + fn text(&self) -> String { + match self { + Token::Num(n) => n.to_string(), + Token::Op(op) => (*op).to_owned(), + } + } +} + +const OPS: [&str; 22] = [ + "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "(", ")", "!", "~", "+", "-", "*", "/", "%", + "<", ">", "&", "^", "|", +]; + +fn tokens(expr: &str, macros: &Macros) -> Result, String> { + let expanded = macros.expand(expr); + let mut out = Vec::new(); + let mut rest = expanded.trim_start(); + while let Some(c) = rest.chars().next() { + if c.is_ascii_digit() { + let len = rest + .find(|c: char| !c.is_ascii_alphanumeric()) + .unwrap_or(rest.len()); + if rest[len..].starts_with('.') { + return Err(format!("`{}`: `#if` takes integers only", &rest[..len + 1])); + } + out.push(Token::Num(int_literal(&rest[..len])?)); + rest = &rest[len..]; + } else if is_ident_start(c) { + // Not a macro: 0, as the C preprocessor counts it. + let len = rest.find(|c: char| !is_ident(c)).unwrap_or(rest.len()); + out.push(Token::Num(0)); + rest = &rest[len..]; + } else { + let op = OPS + .iter() + .find(|op| rest.starts_with(**op)) + .ok_or_else(|| format!("unexpected `{c}`"))?; + out.push(Token::Op(op)); + rest = &rest[op.len()..]; + } + rest = rest.trim_start(); + } + Ok(out) +} + +/// Decimal, `0x` hexadecimal or leading-zero octal, with an optional `u` suffix. +fn int_literal(text: &str) -> Result { + let digits = text.strip_suffix(['u', 'U']).unwrap_or(text); + let parsed = if let Some(hex) = digits.strip_prefix("0x").or(digits.strip_prefix("0X")) { + i64::from_str_radix(hex, 16) + } else if digits.len() > 1 && digits.starts_with('0') { + i64::from_str_radix(&digits[1..], 8) + } else { + digits.parse() + }; + parsed.map_err(|_| format!("`{text}` is not an integer")) +} + +struct Parser<'a> { + tokens: &'a [Token], + at: usize, +} + +/// Binary operators from the loosest to the tightest (GLSL §5.1 / §3.3). +const LEVELS: [&[&str]; 10] = [ + &["||"], + &["&&"], + &["|"], + &["^"], + &["&"], + &["==", "!="], + &["<", ">", "<=", ">="], + &["<<", ">>"], + &["+", "-"], + &["*", "/", "%"], +]; + +impl Parser<'_> { + fn next(&mut self) -> Option { + let t = self.tokens.get(self.at).copied(); + self.at += 1; + t + } + + fn binary(&mut self, level: usize) -> Result { + if level == LEVELS.len() { + return self.unary(); + } + let mut left = self.binary(level + 1)?; + while let Some(&Token::Op(op)) = self.tokens.get(self.at) { + if !LEVELS[level].contains(&op) { + break; + } + self.at += 1; + let right = self.binary(level + 1)?; + left = apply(op, left, right)?; + } + Ok(left) + } + + fn unary(&mut self) -> Result { + match self.next() { + Some(Token::Num(n)) => Ok(n), + Some(Token::Op("(")) => { + let value = self.binary(0)?; + match self.next() { + Some(Token::Op(")")) => Ok(value), + _ => Err("missing `)`".into()), + } + } + Some(Token::Op("-")) => Ok(self.unary()?.wrapping_neg()), + Some(Token::Op("+")) => self.unary(), + Some(Token::Op("!")) => Ok((self.unary()? == 0) as i64), + Some(Token::Op("~")) => Ok(!self.unary()?), + Some(t) => Err(format!("unexpected `{}`", t.text())), + None => Err("the condition ends too early".into()), + } + } +} + +fn apply(op: &str, a: i64, b: i64) -> Result { + Ok(match op { + "||" => (a != 0 || b != 0) as i64, + "&&" => (a != 0 && b != 0) as i64, + "|" => a | b, + "^" => a ^ b, + "&" => a & b, + "==" => (a == b) as i64, + "!=" => (a != b) as i64, + "<" => (a < b) as i64, + ">" => (a > b) as i64, + "<=" => (a <= b) as i64, + ">=" => (a >= b) as i64, + "<<" => a.wrapping_shl(b as u32), + ">>" => a.wrapping_shr(b as u32), + "+" => a.wrapping_add(b), + "-" => a.wrapping_sub(b), + "*" => a.wrapping_mul(b), + "/" | "%" if b == 0 => return Err("division by zero".into()), + "/" => a.wrapping_div(b), + _ => a.wrapping_rem(b), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn eval(expr: &str) -> Result { + let mut m = Macros::standard(); + m.define("SHADOW_RES", "2048"); + m.define("WAVES", ""); + m.define("QUALITY", "SHADOW_RES / 1024"); + eval_condition(expr, &m) + } + + #[test] + fn precedence_and_operators_follow_c() { + assert_eq!(eval("1 + 2 * 3"), Ok(7)); + assert_eq!(eval("(1 + 2) * 3"), Ok(9)); + assert_eq!(eval("1 << 2 + 1"), Ok(8)); + assert_eq!(eval("6 & 3 == 3"), Ok(0)); + assert_eq!(eval("1 || 0 && 0"), Ok(1)); + assert_eq!(eval("-7 / 2"), Ok(-3)); + assert_eq!(eval("-7 % 2"), Ok(-1)); + assert_eq!(eval("!0 + ~0"), Ok(0)); + assert_eq!(eval("0x10 + 010 + 3u"), Ok(27)); + } + + #[test] + fn macros_and_defined() { + assert_eq!(eval("MC_VERSION >= 11300"), Ok(1)); + assert_eq!(eval("MC_VERSION <= 12111"), Ok(0)); + assert_eq!(eval("QUALITY == 2"), Ok(1)); + assert_eq!(eval("defined WAVES && defined(SHADOW_RES)"), Ok(1)); + assert_eq!(eval("!defined NOPE"), Ok(1)); + assert_eq!(eval("NOPE == 0"), Ok(1)); + assert_eq!(eval("undefinedFeature"), Ok(0)); + } + + #[test] + fn bad_conditions_say_why() { + assert!(eval("1 / 0").is_err()); + assert!(eval("SHADOW_RES == 0.5").is_err()); + assert!(eval("(1 + 2").is_err()); + assert!(eval("1 2").is_err()); + assert!(eval("").is_err()); + assert!(eval("defined").is_err()); + assert!(eval("1 @ 2").is_err()); + } +} diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index 1c9b46dd1..e7ccb26d6 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -5,11 +5,16 @@ //! - [`Pack`]: a pack as a folder or a zip, with paths relative to its `shaders` folder (spec 1.1). //! - [`discover`]: the packs in a `shaderpacks` folder. //! - [`Lang`]: the pack's translated names (8.4). +//! - [`Macros`] and [`eval_condition`]: the macros packs test, and the `#if` arithmetic (1.5). +mod condition; mod discover; mod lang; +mod macros; mod pack; +pub use condition::eval_condition; pub use discover::{Found, discover}; pub use lang::Lang; +pub use macros::{MC_VERSION, Macros}; pub use pack::{Pack, PackError}; diff --git a/crates/horizon-shaders/src/macros.rs b/crates/horizon-shaders/src/macros.rs new file mode 100644 index 000000000..f3a53722f --- /dev/null +++ b/crates/horizon-shaders/src/macros.rs @@ -0,0 +1,131 @@ +//! Object-like macros: the ones packs test in `#if` lines and use inside property values +//! (spec 1.5, 1.5.1). The GLSL preprocessor's function-like macros are not needed here. + +use std::collections::BTreeMap; + +/// `MC_VERSION` of the version Horizon plays: 26.3 → 260300. +pub const MC_VERSION: u32 = mc_version(horizon_core::version::VERSION_ID); + +/// The documented rule, `major * 10000 + minor * 100 + patch` (spec 1.5.1, OQ-1): `1.21` → 12100, +/// `1.7.10` → 10710. Parts that are not digits count as 0. +pub const fn mc_version(id: &str) -> u32 { + let bytes = id.as_bytes(); + let (mut parts, mut part, mut i) = ([0u32; 3], 0, 0); + while i < bytes.len() && part < 3 { + match bytes[i] { + b'.' => part += 1, + b @ b'0'..=b'9' => parts[part] = parts[part] * 10 + (b - b'0') as u32, + _ => break, + } + i += 1; + } + parts[0] * 10000 + parts[1] * 100 + parts[2] +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Macros(BTreeMap); + +impl Macros { + /// The standard macros that need no renderer: `MC_VERSION` and this system's `MC_OS_*`. + /// The graphics ones arrive with the renderer. + pub fn standard() -> Self { + let mut m = Self::default(); + m.define("MC_VERSION", MC_VERSION.to_string()); + let os = match std::env::consts::OS { + "windows" => "MC_OS_WINDOWS", + "macos" => "MC_OS_MAC", + "linux" => "MC_OS_LINUX", + _ => "MC_OS_OTHER", + }; + m.define(os, ""); + m + } + + pub fn define(&mut self, name: impl Into, value: impl Into) { + self.0.insert(name.into(), value.into()); + } + + pub fn undef(&mut self, name: &str) { + self.0.remove(name); + } + + pub fn is_defined(&self, name: &str) -> bool { + self.0.contains_key(name) + } + + pub fn get(&self, name: &str) -> Option<&str> { + self.0.get(name).map(String::as_str) + } + + /// `text` with every identifier that names a macro replaced by its value, repeatedly; a macro + /// is not expanded again inside its own expansion, so `A` defined as `A + 1` stops. + pub fn expand(&self, text: &str) -> String { + self.expand_hiding(text, &mut Vec::new()) + } + + fn expand_hiding<'a>(&'a self, text: &str, hidden: &mut Vec<&'a str>) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(start) = rest.find(is_ident_start) { + out.push_str(&rest[..start]); + // A digit run glued to letters (`1e5`, `0x1F`) is a number, not a name. + let glued = out + .chars() + .next_back() + .is_some_and(|c| c.is_ascii_alphanumeric() || c == '.'); + let len = rest[start..] + .find(|c: char| !is_ident(c)) + .unwrap_or(rest.len() - start); + let word = &rest[start..start + len]; + match self.0.get_key_value(word) { + Some((name, value)) if !glued && !hidden.contains(&name.as_str()) => { + hidden.push(name); + out.push_str(&self.expand_hiding(value, hidden)); + hidden.pop(); + } + _ => out.push_str(word), + } + rest = &rest[start + len..]; + } + out.push_str(rest); + out + } +} + +pub(crate) fn is_ident_start(c: char) -> bool { + c.is_ascii_alphabetic() || c == '_' +} + +pub(crate) fn is_ident(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '_' +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn versions_follow_the_documented_rule() { + assert_eq!(mc_version("1.21"), 12100); + assert_eq!(mc_version("1.21.0"), 12100); + assert_eq!(mc_version("1.7.10"), 10710); + assert_eq!(mc_version("26.3"), 260300); + assert_eq!(MC_VERSION, 260300); + assert_eq!(Macros::standard().get("MC_VERSION"), Some("260300")); + } + + #[test] + fn expansion_replaces_whole_words_and_stops_on_recursion() { + let mut m = Macros::default(); + m.define("RES", "SIZE"); + m.define("SIZE", "512"); + m.define("LOOP", "LOOP + 1"); + assert_eq!(m.expand("RES RES 0.5"), "512 512 0.5"); + assert_eq!( + m.expand("RESOLUTION colortex4 LOOP"), + "RESOLUTION colortex4 LOOP + 1" + ); + m.define("e5", "x"); + assert_eq!(m.expand("1e5"), "1e5"); + } +} From 397d529db72a17a1def83e1d7c50e9f866beb69e Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:42:33 +0200 Subject: [PATCH 2/5] Shader packs, clean room 3: .properties files with conditional directives shaders.properties and the id files (block, item, entity, dimension): key=value lines in file order with their line numbers, # comments, a value continued on the next line by a trailing backslash, and the conditional directives evaluated with the standard and option macros. A # line is a directive only when its first word is one, so a commented-out key such as #ifdefault=1 stays a comment. #define and #undef apply to the lines after them in every one of these files: a public pack defines a macro inside #ifdef MC_OS_MAC in its shaders.properties and tests it further down. A later duplicate key replaces the earlier one; broken conditionals, lines that are not key=value and duplicates are kept as notes for the pack's log. Nothing uses it yet: players see no change. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-shaders/src/lib.rs | 3 + crates/horizon-shaders/src/properties.rs | 279 +++++++++++++++++++++ crates/horizon-shaders/tests/real_packs.rs | 30 ++- 3 files changed, 305 insertions(+), 7 deletions(-) create mode 100644 crates/horizon-shaders/src/properties.rs diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index e7ccb26d6..bdfef7067 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -4,6 +4,7 @@ //! //! - [`Pack`]: a pack as a folder or a zip, with paths relative to its `shaders` folder (spec 1.1). //! - [`discover`]: the packs in a `shaderpacks` folder. +//! - [`Properties`]: `shaders.properties` and the id files, with their conditional directives (3.1, 1.5). //! - [`Lang`]: the pack's translated names (8.4). //! - [`Macros`] and [`eval_condition`]: the macros packs test, and the `#if` arithmetic (1.5). @@ -12,9 +13,11 @@ mod discover; mod lang; mod macros; mod pack; +mod properties; pub use condition::eval_condition; pub use discover::{Found, discover}; pub use lang::Lang; pub use macros::{MC_VERSION, Macros}; pub use pack::{Pack, PackError}; +pub use properties::{Entry, Note, Properties}; diff --git a/crates/horizon-shaders/src/properties.rs b/crates/horizon-shaders/src/properties.rs new file mode 100644 index 000000000..fc3f00c81 --- /dev/null +++ b/crates/horizon-shaders/src/properties.rs @@ -0,0 +1,279 @@ +//! `.properties` files: `shaders.properties` (spec 3.1) and the id files `block.properties`, +//! `item.properties`, `entity.properties` (4.1). `key=value` lines, `#` comments, a trailing `\` +//! continuing a value on the next line, and conditional directives evaluated with the +//! standard and option macros (1.5 item 2). `#define` and `#undef` work in every one of them: +//! a public pack defines a macro in its `shaders.properties` and tests it further down. +//! No other escape is read: values are names, numbers and expressions. A later duplicate key +//! replaces the earlier one, with a note. + +use crate::{Macros, eval_condition}; + +/// One `key=value`, with the line it starts on (from 1). +#[derive(Clone, Debug, PartialEq)] +pub struct Entry { + pub key: String, + pub value: String, + pub line: u32, +} + +/// Something wrong with a line, for the pack's log. +#[derive(Clone, Debug, PartialEq)] +pub struct Note { + pub line: u32, + pub message: String, +} + +#[derive(Debug, Default, PartialEq)] +pub struct Properties { + /// In file order; a key appears once (a later duplicate replaces the earlier one). + pub entries: Vec, + pub notes: Vec, +} + +/// One branch level of `#if` ... `#endif`. +struct Branch { + /// This branch's lines are read. + on: bool, + /// A branch of this `#if` was already taken (the rest are skipped). + taken: bool, + /// The enclosing level is read. + outer: bool, + seen_else: bool, +} + +impl Properties { + /// Reads `text`; `#define`/`#undef` change the macros for the lines after them only. + pub fn parse(text: &str, macros: &Macros) -> Self { + let mut out = Self::default(); + let mut macros = macros.clone(); + let mut branches: Vec = Vec::new(); + let mut lines = text.lines().zip(1u32..); + while let Some((raw, number)) = lines.next() { + let line = raw.trim(); + let on = branches.last().is_none_or(|b| b.on); + if let Some(directive) = line.strip_prefix('#') { + out.directive(directive.trim_start(), number, &mut branches, &mut macros); + continue; + } + // Continuation lines belong to their entry even inside a skipped branch. + let mut logical = line.to_owned(); + while ends_with_continuation(&logical) { + logical.pop(); + match lines.next() { + Some((next, _)) => logical.push_str(next.trim()), + None => break, + } + } + if !on || logical.is_empty() { + continue; + } + match logical.split_once('=') { + Some((key, value)) if !key.trim().is_empty() => out.insert(Entry { + key: key.trim().to_owned(), + value: value.trim().to_owned(), + line: number, + }), + _ => out.note(number, "not a key=value line"), + } + } + if !branches.is_empty() { + out.note(text.lines().count() as u32, "`#if` without `#endif`"); + } + out + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.entry(key).map(|e| e.value.as_str()) + } + + pub fn entry(&self, key: &str) -> Option<&Entry> { + self.entries.iter().find(|e| e.key == key) + } + + fn insert(&mut self, entry: Entry) { + if let Some(at) = self.entries.iter().position(|e| e.key == entry.key) { + let first = self.entries.remove(at).line; + self.note( + entry.line, + &format!( + "`{}` is set again (first on line {first}); this one counts", + entry.key + ), + ); + } + self.entries.push(entry); + } + + fn note(&mut self, line: u32, message: &str) { + self.notes.push(Note { + line, + message: message.to_owned(), + }); + } + + /// A line starting with `#`. It is a directive only when its first word is one; anything + /// else (`# text`, `#commented.key=1`, `#ifdefault`) is a comment. + fn directive( + &mut self, + line: &str, + number: u32, + branches: &mut Vec, + macros: &mut Macros, + ) { + let word_len = line + .find(|c: char| !c.is_ascii_alphabetic()) + .unwrap_or(line.len()); + let (word, rest) = line.split_at(word_len); + if !(rest.is_empty() || rest.starts_with(char::is_whitespace) || rest.starts_with("//")) { + return; + } + let rest = rest.split("//").next().unwrap_or("").trim(); + let on = branches.last().is_none_or(|b| b.on); + match word { + "if" | "ifdef" | "ifndef" => { + let value = on && self.condition(word, rest, number, macros); + branches.push(Branch { + on: value, + taken: value, + outer: on, + seen_else: false, + }); + } + "elif" | "else" => { + let Some(b) = branches.last_mut() else { + return self.note(number, &format!("`#{word}` without `#if`")); + }; + if b.seen_else { + return self.note(number, &format!("`#{word}` after `#else`")); + } + let take = b.outer && !b.taken; + b.seen_else = word == "else"; + b.on = take && (word == "else" || self.condition("if", rest, number, macros)); + b.taken |= b.on; + } + "endif" => { + if branches.pop().is_none() { + self.note(number, "`#endif` without `#if`"); + } + } + "define" if on => { + let (name, value) = rest.split_once(char::is_whitespace).unwrap_or((rest, "")); + macros.define(name, value.trim()); + } + "undef" if on => macros.undef(rest), + _ => {} + } + } + + fn condition(&mut self, word: &str, rest: &str, number: u32, macros: &Macros) -> bool { + let name = rest.split_whitespace().next().unwrap_or(""); + let result = match word { + "ifdef" => Ok(macros.is_defined(name) as i64), + "ifndef" => Ok(!macros.is_defined(name) as i64), + _ => eval_condition(rest, macros), + }; + result.unwrap_or_else(|e| { + self.note(number, &format!("`#{word} {rest}`: {e}; read as false")); + 0 + }) != 0 + } +} + +/// An odd number of trailing backslashes: the last one joins the next line (`\\` is a literal). +fn ends_with_continuation(line: &str) -> bool { + line.bytes().rev().take_while(|&b| b == b'\\').count() % 2 == 1 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(text: &str) -> Properties { + let mut m = Macros::standard(); + m.define("SHADOWS", ""); + m.define("QUALITY", "2"); + Properties::parse(text, &m) + } + + fn pairs(p: &Properties) -> Vec<(&str, &str)> { + p.entries + .iter() + .map(|e| (e.key.as_str(), e.value.as_str())) + .collect() + } + + #[test] + fn keys_values_and_comments() { + let p = + read("# a comment\n\n sun = false\n#commented.key=1\nscreen=A B [C]\r\nbad line\n"); + assert_eq!(pairs(&p), [("sun", "false"), ("screen", "A B [C]")]); + assert_eq!(p.entry("screen").unwrap().line, 5); + assert_eq!( + p.notes, + [Note { + line: 6, + message: "not a key=value line".into() + }] + ); + } + + #[test] + fn continuation_lines_join() { + let p = read("screen=A B \\\n C D\\\n E\nsliders=X\nliteral=a\\\\\nnext=1"); + assert_eq!( + pairs(&p), + [ + ("screen", "A B C DE"), + ("sliders", "X"), + ("literal", "a\\\\"), + ("next", "1") + ] + ); + assert_eq!(p.entry("sliders").unwrap().line, 4); + } + + #[test] + fn conditionals_pick_one_branch() { + let p = read( + "#if MC_VERSION >= 11300\nnew=1\n#else\nold=1\n#endif\n\ + #ifdef SHADOWS\n#if QUALITY == 1\nq=1\n#elif QUALITY == 2\nq=2\n#elif QUALITY >= 2\nq=3\n#endif\n#endif\n\ + #ifndef SHADOWS\nnone=1\n#elif 0\nnever=1\n#else\nfallback=1\n#endif\n# ifdef SHADOWS // trailing comment\nspaced=1\n#endif", + ); + assert_eq!( + pairs(&p), + [("new", "1"), ("q", "2"), ("fallback", "1"), ("spaced", "1")] + ); + assert!(p.notes.is_empty(), "{:?}", p.notes); + } + + #[test] + fn comment_lines_are_not_directives() { + let p = read("#ifdefault=1\n#iffy\n#endiff\n#else-where\nkey=1"); + assert_eq!(pairs(&p), [("key", "1")]); + assert!(p.notes.is_empty(), "{:?}", p.notes); + } + + #[test] + fn defines_apply_to_the_lines_after_them() { + let text = "#ifdef MC_OS_NONE\n#define WATER 0\n#endif\n#define WATER 8\n\ + #if WATER == 8\nblock.8=minecraft:water\n#endif\n#undef WATER\n#ifdef WATER\nno=1\n#endif"; + let p = Properties::parse(text, &Macros::default()); + assert_eq!(pairs(&p), [("block.8", "minecraft:water")]); + assert!(p.notes.is_empty(), "{:?}", p.notes); + } + + #[test] + fn duplicate_keys_keep_the_last() { + let p = read("a=1\nb=2\na=3"); + assert_eq!(pairs(&p), [("b", "2"), ("a", "3")]); + assert_eq!(p.notes[0].line, 3); + } + + #[test] + fn broken_conditionals_are_noted() { + let p = read("#endif\n#else\n#if 1 +\nx=1\n#endif\n#if 1\n#else\n#else\n"); + let lines: Vec = p.notes.iter().map(|n| n.line).collect(); + assert_eq!(lines, [1, 2, 3, 8, 8]); + assert!(p.entries.is_empty()); + } +} diff --git a/crates/horizon-shaders/tests/real_packs.rs b/crates/horizon-shaders/tests/real_packs.rs index b35aef519..e0ad38c25 100644 --- a/crates/horizon-shaders/tests/real_packs.rs +++ b/crates/horizon-shaders/tests/real_packs.rs @@ -5,7 +5,7 @@ use std::path::Path; use std::time::Instant; -use horizon_shaders::{Lang, Pack, discover}; +use horizon_shaders::{Lang, Macros, Pack, Properties, discover}; #[test] #[ignore = "needs HORIZON_TEST_SHADER_PACKS"] @@ -22,21 +22,37 @@ fn real_packs_read() { for f in found.iter().filter(|f| f.problem.is_none()) { let start = Instant::now(); let pack = Pack::open(&f.path).unwrap(); - let properties = pack.read_text("shaders.properties"); + let macros = Macros::standard(); + let shaders = Properties::parse( + &pack.read_text("shaders.properties").unwrap_or_default(), + ¯os, + ); + let ids: Vec = ["block", "item", "entity", "dimension"] + .iter() + .filter_map(|n| pack.read_text(&format!("{n}.properties"))) + .map(|t| Properties::parse(&t, ¯os)) + .collect(); let lang = Lang::load(&pack, "en_us"); + let elapsed = start.elapsed(); println!( - "{}: {} files at the root, shaders.properties {} bytes, lang {}, {:.2} ms", + "{}: shaders.properties {} keys ({} notes), {} id files {} keys ({} notes), lang {}, {:.2} ms", f.name, - pack.list("").len(), - properties.as_ref().map_or(0, String::len), + shaders.entries.len(), + shaders.notes.len(), + ids.len(), + ids.iter().map(|p| p.entries.len()).sum::(), + ids.iter().map(|p| p.notes.len()).sum::(), if lang == Lang::default() { "none" } else { "read" }, - start.elapsed().as_secs_f64() * 1000.0 + elapsed.as_secs_f64() * 1000.0 ); - assert!(properties.is_some(), "{}: no shaders.properties", f.name); + for n in &shaders.notes { + println!(" shaders.properties:{}: {}", n.line, n.message); + } + assert!(!shaders.entries.is_empty(), "{}: no keys read", f.name); read += 1; } assert!(read > 0, "no packs under {}", dir.display()); From 4c65ed2f5c6c4e089cf4f0bec4b32ab97004e4bc Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:09:07 +0200 Subject: [PATCH 3/5] Shader packs, clean room 4: function-like macros and the standard macros The macro engine the GLSL preprocessor needs, shared with the .properties reader: function-like macros take their arguments (expanded first, raw beside ##, which pastes) when a ( follows their name; a macro is still not expanded inside its own expansion. Text is cut into preprocessing tokens, so numbers such as 1e5 or 0x1F are never mistaken for names. The #if/#elif/#else/#endif state moves out of the .properties reader so the preprocessor uses the same code. Standard macros: MC_GL_VERSION and MC_GLSL_VERSION 460, the 24 MC_RENDER_STAGE_* as ascending integers in draw order, render and shadow quality 1.0, MC_HAND_DEPTH 0.125, MAX_COLOR_BUFFERS 16. Nothing uses it yet: players see no change. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-shaders/src/condition.rs | 104 ++++++++++ crates/horizon-shaders/src/lib.rs | 2 +- crates/horizon-shaders/src/macros.rs | 242 ++++++++++++++++++++--- crates/horizon-shaders/src/properties.rs | 98 +++------ 4 files changed, 343 insertions(+), 103 deletions(-) diff --git a/crates/horizon-shaders/src/condition.rs b/crates/horizon-shaders/src/condition.rs index b911eb985..c476bb663 100644 --- a/crates/horizon-shaders/src/condition.rs +++ b/crates/horizon-shaders/src/condition.rs @@ -20,6 +20,110 @@ pub fn eval_condition(expr: &str, macros: &Macros) -> Result { } } +/// The text after a line's `#`, split into the directive word and the rest without its `//` +/// comment. `None` when the first word is followed by something other than white space, `//` or +/// the end of the line (`#ifdefault=1` is a comment, not `#ifdef`). +pub(crate) fn split_directive(after_hash: &str) -> Option<(&str, &str)> { + let line = after_hash.trim_start(); + let len = line + .find(|c: char| !c.is_ascii_alphabetic()) + .unwrap_or(line.len()); + let (word, rest) = line.split_at(len); + let ok = rest.is_empty() || rest.starts_with(char::is_whitespace) || rest.starts_with("//"); + ok.then(|| (word, rest.split("//").next().unwrap_or("").trim())) +} + +/// One level of `#if` ... `#endif`. +struct Branch { + /// This branch's lines are read. + on: bool, + /// A branch of this `#if` was already taken (the rest are skipped). + taken: bool, + /// The enclosing level is read. + outer: bool, + seen_else: bool, +} + +/// The state of nested conditional directives. +#[derive(Default)] +pub(crate) struct Conditionals(Vec); + +impl Conditionals { + /// Lines here are read. + pub(crate) fn active(&self) -> bool { + self.0.last().is_none_or(|b| b.on) + } + + /// An `#if` is still open. + pub(crate) fn open(&self) -> bool { + !self.0.is_empty() + } + + /// Applies `#word rest` when it is a conditional directive (`Ok(false)` when it is not). A + /// condition that can't be evaluated reads as false and returns why. + pub(crate) fn apply( + &mut self, + word: &str, + rest: &str, + macros: &Macros, + ) -> Result { + let on = self.active(); + match word { + "if" | "ifdef" | "ifndef" => { + let value = if on { + test(word, rest, macros) + } else { + Ok(false) + }; + let v = *value.as_ref().unwrap_or(&false); + self.0.push(Branch { + on: v, + taken: v, + outer: on, + seen_else: false, + }); + value.map(|_| true) + } + "elif" | "else" => { + let b = self + .0 + .last_mut() + .ok_or_else(|| format!("`#{word}` without `#if`"))?; + if b.seen_else { + return Err(format!("`#{word}` after `#else`")); + } + b.seen_else = word == "else"; + let take = b.outer && !b.taken; + let value = match word { + "else" => Ok(take), + _ if take => test("if", rest, macros), + _ => Ok(false), + }; + b.on = *value.as_ref().unwrap_or(&false); + b.taken |= b.on; + value.map(|_| true) + } + "endif" => self + .0 + .pop() + .map(|_| true) + .ok_or_else(|| "`#endif` without `#if`".into()), + _ => Ok(false), + } + } +} + +fn test(word: &str, rest: &str, macros: &Macros) -> Result { + let name = rest.split_whitespace().next().unwrap_or(""); + match word { + "ifdef" => Ok(macros.is_defined(name)), + "ifndef" => Ok(!macros.is_defined(name)), + _ => eval_condition(rest, macros) + .map(|v| v != 0) + .map_err(|e| format!("`#{word} {rest}`: {e}; read as false")), + } +} + /// `defined X` and `defined(X)` become `1` or `0` before anything is expanded. fn resolve_defined(expr: &str, macros: &Macros) -> Result { let mut out = String::with_capacity(expr.len()); diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index bdfef7067..05e51a53d 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -18,6 +18,6 @@ mod properties; pub use condition::eval_condition; pub use discover::{Found, discover}; pub use lang::Lang; -pub use macros::{MC_VERSION, Macros}; +pub use macros::{MC_VERSION, Macros, RENDER_STAGES}; pub use pack::{Pack, PackError}; pub use properties::{Entry, Note, Properties}; diff --git a/crates/horizon-shaders/src/macros.rs b/crates/horizon-shaders/src/macros.rs index f3a53722f..d8a184f41 100644 --- a/crates/horizon-shaders/src/macros.rs +++ b/crates/horizon-shaders/src/macros.rs @@ -22,12 +22,48 @@ pub const fn mc_version(id: &str) -> u32 { parts[0] * 10000 + parts[1] * 100 + parts[2] } +#[derive(Clone, Debug, PartialEq)] +struct Macro { + /// `None` for an object-like macro; the parameter names of a function-like one. + params: Option>, + body: String, +} + #[derive(Clone, Debug, Default, PartialEq)] -pub struct Macros(BTreeMap); +pub struct Macros(BTreeMap); + +/// `MC_RENDER_STAGE_*` in the order the stages draw (spec 5.4); each is its index (OQ-7). +pub const RENDER_STAGES: [&str; 24] = [ + "NONE", + "SKY", + "SUNSET", + "CUSTOM_SKY", + "SUN", + "MOON", + "STARS", + "VOID", + "TERRAIN_SOLID", + "TERRAIN_CUTOUT_MIPPED", + "TERRAIN_CUTOUT", + "ENTITIES", + "BLOCK_ENTITIES", + "DESTROY", + "OUTLINE", + "DEBUG", + "HAND_SOLID", + "TERRAIN_TRANSLUCENT", + "TRIPWIRE", + "PARTICLES", + "CLOUDS", + "RAIN_SNOW", + "WORLD_BORDER", + "HAND_TRANSLUCENT", +]; impl Macros { - /// The standard macros that need no renderer: `MC_VERSION` and this system's `MC_OS_*`. - /// The graphics ones arrive with the renderer. + /// The standard macros that need no renderer (spec 1.5.1): `MC_VERSION`, this system's + /// `MC_OS_*`, the GL and GLSL versions Horizon's translation accepts, the render stages and + /// the host settings at their defaults. The GPU vendor and family arrive with the renderer. pub fn standard() -> Self { let mut m = Self::default(); m.define("MC_VERSION", MC_VERSION.to_string()); @@ -38,11 +74,32 @@ impl Macros { _ => "MC_OS_OTHER", }; m.define(os, ""); + m.define("MC_GL_VERSION", "460"); + m.define("MC_GLSL_VERSION", "460"); + for (i, stage) in RENDER_STAGES.iter().enumerate() { + m.define(format!("MC_RENDER_STAGE_{stage}"), i.to_string()); + } + m.define("MC_RENDER_QUALITY", "1.0"); + m.define("MC_SHADOW_QUALITY", "1.0"); + m.define("MC_HAND_DEPTH", "0.125"); + m.define("MAX_COLOR_BUFFERS", "16"); m } pub fn define(&mut self, name: impl Into, value: impl Into) { - self.0.insert(name.into(), value.into()); + let body = value.into(); + self.0.insert(name.into(), Macro { params: None, body }); + } + + pub fn define_function(&mut self, name: &str, params: Vec, body: &str) { + let body = body.to_owned(); + self.0.insert( + name.to_owned(), + Macro { + params: Some(params), + body, + }, + ); } pub fn undef(&mut self, name: &str) { @@ -53,43 +110,158 @@ impl Macros { self.0.contains_key(name) } + /// The value of an object-like macro. pub fn get(&self, name: &str) -> Option<&str> { - self.0.get(name).map(String::as_str) + self.0 + .get(name) + .filter(|m| m.params.is_none()) + .map(|m| m.body.as_str()) } - /// `text` with every identifier that names a macro replaced by its value, repeatedly; a macro - /// is not expanded again inside its own expansion, so `A` defined as `A + 1` stops. + /// `text` with every macro expanded, repeatedly; a macro is not expanded again inside its own + /// expansion, so `A` defined as `A + 1` stops. Function-like macros take their arguments + /// (expanded first, raw beside `##`, which pastes) when a `(` follows their name. pub fn expand(&self, text: &str) -> String { self.expand_hiding(text, &mut Vec::new()) } fn expand_hiding<'a>(&'a self, text: &str, hidden: &mut Vec<&'a str>) -> String { let mut out = String::with_capacity(text.len()); - let mut rest = text; - while let Some(start) = rest.find(is_ident_start) { - out.push_str(&rest[..start]); - // A digit run glued to letters (`1e5`, `0x1F`) is a number, not a name. - let glued = out - .chars() - .next_back() - .is_some_and(|c| c.is_ascii_alphanumeric() || c == '.'); - let len = rest[start..] - .find(|c: char| !is_ident(c)) - .unwrap_or(rest.len() - start); - let word = &rest[start..start + len]; - match self.0.get_key_value(word) { - Some((name, value)) if !glued && !hidden.contains(&name.as_str()) => { - hidden.push(name); - out.push_str(&self.expand_hiding(value, hidden)); - hidden.pop(); - } - _ => out.push_str(word), + let tokens = lex(text); + let mut i = 0; + while i < tokens.len() { + let word = tokens[i]; + i += 1; + let Some((name, m)) = self.0.get_key_value(word) else { + out.push_str(word); + continue; + }; + if hidden.contains(&name.as_str()) { + out.push_str(word); + continue; } - rest = &rest[start + len..]; + let body = match &m.params { + None => m.body.clone(), + Some(params) => { + let Some((args, next)) = arguments(&tokens, i) else { + out.push_str(word); + continue; + }; + i = next; + self.substitute(params, &args, &m.body, hidden) + } + }; + hidden.push(name); + out.push_str(&self.expand_hiding(&body, hidden)); + hidden.pop(); } - out.push_str(rest); out } + + /// A function-like macro's body with its parameters replaced. + fn substitute<'a>( + &'a self, + params: &[String], + args: &[String], + body: &str, + hidden: &mut Vec<&'a str>, + ) -> String { + let tokens = lex(body); + let pasted = |at: usize, step: isize| { + let mut j = at as isize + step; + while (0..tokens.len() as isize).contains(&j) && tokens[j as usize].trim().is_empty() { + j += step; + } + (0..tokens.len() as isize).contains(&j) && tokens[j as usize] == "##" + }; + let mut out = String::new(); + for (at, token) in tokens.iter().enumerate() { + match params.iter().position(|p| p == token) { + Some(k) => { + let arg = args.get(k).map_or("", |a| a.trim()); + if pasted(at, -1) || pasted(at, 1) { + out.push_str(arg); + } else { + out.push_str(&self.expand_hiding(arg, hidden)); + } + } + None => out.push_str(token), + } + } + // `a ## b` → `ab`. + let mut joined = String::with_capacity(out.len()); + for (k, part) in out.split("##").enumerate() { + joined.push_str(if k == 0 { part.trim_end() } else { part.trim() }); + } + if out.contains("##") { joined } else { out } + } +} + +/// The arguments of a call whose `(` is the next non-blank token from `i`, split at top-level +/// commas, and the index after its `)`. `None` when no `(` follows or it never closes. +fn arguments(tokens: &[&str], mut i: usize) -> Option<(Vec, usize)> { + while tokens.get(i)?.trim().is_empty() { + i += 1; + } + if tokens[i] != "(" { + return None; + } + let (mut depth, mut args, mut from) = (0, Vec::new(), i + 1); + for (j, t) in tokens.iter().enumerate().skip(i + 1) { + match *t { + "(" => depth += 1, + ")" if depth == 0 => { + args.push(tokens[from..j].concat()); + return Some((args, j + 1)); + } + ")" => depth -= 1, + "," if depth == 0 => { + args.push(tokens[from..j].concat()); + from = j + 1; + } + _ => {} + } + } + None +} + +/// Preprocessing tokens: identifiers, numbers (with their suffixes and exponents), runs of +/// white space, `##`, and single characters. Concatenated they give back `text`. +pub(crate) fn lex(text: &str) -> Vec<&str> { + let bytes = text.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < bytes.len() { + let c = bytes[i] as char; + let start = i; + i += 1; + if is_ident_start(c) { + while i < bytes.len() && is_ident(bytes[i] as char) { + i += 1; + } + } else if c.is_ascii_digit() || (c == '.' && bytes.get(i).is_some_and(u8::is_ascii_digit)) { + while i < bytes.len() { + let d = bytes[i] as char; + let exp_sign = (d == '+' || d == '-') && matches!(bytes[i - 1], b'e' | b'E'); + if !(is_ident(d) || d == '.' || exp_sign) { + break; + } + i += 1; + } + } else if c.is_whitespace() { + while i < bytes.len() && (bytes[i] as char).is_whitespace() { + i += 1; + } + } else if c == '#' && bytes.get(i) == Some(&b'#') { + i += 1; + } else { + while !text.is_char_boundary(i) { + i += 1; + } + } + out.push(&text[start..i]); + } + out } pub(crate) fn is_ident_start(c: char) -> bool { @@ -128,4 +300,18 @@ mod tests { m.define("e5", "x"); assert_eq!(m.expand("1e5"), "1e5"); } + + #[test] + fn function_like_macros_take_arguments_and_paste() { + let mut m = Macros::default(); + m.define("N", "4"); + m.define_function("SQ", vec!["x".into()], "((x) * (x))"); + m.define_function("CAT", vec!["a".into(), "b".into()], "a ## b"); + m.define_function("PI", vec![], "3.14"); + assert_eq!(m.expand("SQ(N + 1)"), "((4 + 1) * (4 + 1))"); + assert_eq!(m.expand("SQ(min(N, 2))"), "((min(4, 2)) * (min(4, 2)))"); + assert_eq!(m.expand("CAT(N, 2) CAT(col, ortex4)"), "N2 colortex4"); + assert_eq!(m.expand("PI() SQ SQ(1"), "3.14 SQ SQ(1"); + assert_eq!(m.get("SQ"), None); + } } diff --git a/crates/horizon-shaders/src/properties.rs b/crates/horizon-shaders/src/properties.rs index fc3f00c81..b9d27f3e3 100644 --- a/crates/horizon-shaders/src/properties.rs +++ b/crates/horizon-shaders/src/properties.rs @@ -6,7 +6,8 @@ //! No other escape is read: values are names, numbers and expressions. A later duplicate key //! replaces the earlier one, with a note. -use crate::{Macros, eval_condition}; +use crate::Macros; +use crate::condition::{Conditionals, split_directive}; /// One `key=value`, with the line it starts on (from 1). #[derive(Clone, Debug, PartialEq)] @@ -30,29 +31,20 @@ pub struct Properties { pub notes: Vec, } -/// One branch level of `#if` ... `#endif`. -struct Branch { - /// This branch's lines are read. - on: bool, - /// A branch of this `#if` was already taken (the rest are skipped). - taken: bool, - /// The enclosing level is read. - outer: bool, - seen_else: bool, -} - impl Properties { /// Reads `text`; `#define`/`#undef` change the macros for the lines after them only. pub fn parse(text: &str, macros: &Macros) -> Self { let mut out = Self::default(); let mut macros = macros.clone(); - let mut branches: Vec = Vec::new(); + let mut conditionals = Conditionals::default(); let mut lines = text.lines().zip(1u32..); while let Some((raw, number)) = lines.next() { let line = raw.trim(); - let on = branches.last().is_none_or(|b| b.on); + let on = conditionals.active(); if let Some(directive) = line.strip_prefix('#') { - out.directive(directive.trim_start(), number, &mut branches, &mut macros); + if let Some((word, rest)) = split_directive(directive) { + out.directive(word, rest, number, &mut conditionals, &mut macros); + } continue; } // Continuation lines belong to their entry even inside a skipped branch. @@ -76,7 +68,7 @@ impl Properties { _ => out.note(number, "not a key=value line"), } } - if !branches.is_empty() { + if conditionals.open() { out.note(text.lines().count() as u32, "`#if` without `#endif`"); } out @@ -111,72 +103,30 @@ impl Properties { }); } - /// A line starting with `#`. It is a directive only when its first word is one; anything - /// else (`# text`, `#commented.key=1`, `#ifdefault`) is a comment. + /// A line starting with `#` whose first word may be a directive; any other word makes it a + /// comment. fn directive( &mut self, - line: &str, + word: &str, + rest: &str, number: u32, - branches: &mut Vec, + conditionals: &mut Conditionals, macros: &mut Macros, ) { - let word_len = line - .find(|c: char| !c.is_ascii_alphabetic()) - .unwrap_or(line.len()); - let (word, rest) = line.split_at(word_len); - if !(rest.is_empty() || rest.starts_with(char::is_whitespace) || rest.starts_with("//")) { - return; - } - let rest = rest.split("//").next().unwrap_or("").trim(); - let on = branches.last().is_none_or(|b| b.on); - match word { - "if" | "ifdef" | "ifndef" => { - let value = on && self.condition(word, rest, number, macros); - branches.push(Branch { - on: value, - taken: value, - outer: on, - seen_else: false, - }); - } - "elif" | "else" => { - let Some(b) = branches.last_mut() else { - return self.note(number, &format!("`#{word}` without `#if`")); - }; - if b.seen_else { - return self.note(number, &format!("`#{word}` after `#else`")); + match conditionals.apply(word, rest, macros) { + Err(e) => self.note(number, &e), + Ok(true) => {} + Ok(false) if conditionals.active() => match word { + "define" => { + let (name, value) = rest.split_once(char::is_whitespace).unwrap_or((rest, "")); + macros.define(name, value.trim()); } - let take = b.outer && !b.taken; - b.seen_else = word == "else"; - b.on = take && (word == "else" || self.condition("if", rest, number, macros)); - b.taken |= b.on; - } - "endif" => { - if branches.pop().is_none() { - self.note(number, "`#endif` without `#if`"); - } - } - "define" if on => { - let (name, value) = rest.split_once(char::is_whitespace).unwrap_or((rest, "")); - macros.define(name, value.trim()); - } - "undef" if on => macros.undef(rest), - _ => {} + "undef" => macros.undef(rest), + _ => {} + }, + Ok(false) => {} } } - - fn condition(&mut self, word: &str, rest: &str, number: u32, macros: &Macros) -> bool { - let name = rest.split_whitespace().next().unwrap_or(""); - let result = match word { - "ifdef" => Ok(macros.is_defined(name) as i64), - "ifndef" => Ok(!macros.is_defined(name) as i64), - _ => eval_condition(rest, macros), - }; - result.unwrap_or_else(|e| { - self.note(number, &format!("`#{word} {rest}`: {e}; read as false")); - 0 - }) != 0 - } } /// An odd number of trailing backslashes: the last one joins the next line (`\\` is a literal). From f2399559979c1fd1a0e62c578a83ae4e5a29cd75 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:09:29 +0200 Subject: [PATCH 4/5] Shader packs, clean room 5: the GLSL preprocessor and #include A stage file goes through comment removal, #include, conditionals and macro expansion, and every output line remembers the pack file and line it came from, so later errors point at the pack's own code. - #include "x" from the including file's folder, "/x" from the shaders folder, expanded in live code only (an include inside a false #if is not read), ten levels at most, paths that leave the pack refused; a file is read once per run however often it is included. - #define, #undef, line continuations, __LINE__ and __VERSION__; #version and #extension kept apart for the translation's own header; the block comments of live code kept for the directives packs write in them (DRAWBUFFERS, buffer formats); #error and unknown directives diagnosed. Black box over the public test packs: every stage file of the base and dimension folders (126 to 219 per pack) preprocesses with no diagnostic. Nothing uses it yet: players see no change. Co-Authored-By: Claude Opus 5.5 --- crates/horizon-shaders/src/lib.rs | 3 + crates/horizon-shaders/src/pack.rs | 2 +- crates/horizon-shaders/src/preprocess.rs | 374 +++++++++++++++++++++ crates/horizon-shaders/tests/real_packs.rs | 36 +- 4 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 crates/horizon-shaders/src/preprocess.rs diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index 05e51a53d..14f85b758 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -6,6 +6,7 @@ //! - [`discover`]: the packs in a `shaderpacks` folder. //! - [`Properties`]: `shaders.properties` and the id files, with their conditional directives (3.1, 1.5). //! - [`Lang`]: the pack's translated names (8.4). +//! - [`preprocess`]: a stage file through `#include` and the GLSL preprocessor, lines mapped back (1.4, 1.5). //! - [`Macros`] and [`eval_condition`]: the macros packs test, and the `#if` arithmetic (1.5). mod condition; @@ -13,6 +14,7 @@ mod discover; mod lang; mod macros; mod pack; +mod preprocess; mod properties; pub use condition::eval_condition; @@ -20,4 +22,5 @@ pub use discover::{Found, discover}; pub use lang::Lang; pub use macros::{MC_VERSION, Macros, RENDER_STAGES}; pub use pack::{Pack, PackError}; +pub use preprocess::{Diagnostic, Origin, Preprocessed, preprocess}; pub use properties::{Entry, Note, Properties}; diff --git a/crates/horizon-shaders/src/pack.rs b/crates/horizon-shaders/src/pack.rs index 113db9ac9..ae2230618 100644 --- a/crates/horizon-shaders/src/pack.rs +++ b/crates/horizon-shaders/src/pack.rs @@ -170,7 +170,7 @@ impl Pack { } /// `a/./b/../c` → `a/c`; `None` for a path that climbs out of `shaders`. -fn normalize(path: &str) -> Option { +pub(crate) fn normalize(path: &str) -> Option { let mut parts: Vec<&str> = Vec::new(); for part in path.split(['/', '\\']) { match part { diff --git a/crates/horizon-shaders/src/preprocess.rs b/crates/horizon-shaders/src/preprocess.rs new file mode 100644 index 000000000..4884f28f7 --- /dev/null +++ b/crates/horizon-shaders/src/preprocess.rs @@ -0,0 +1,374 @@ +//! The GLSL preprocessor a program's stage file goes through before translation (spec 1.4, +//! 1.5, 1.6): comments removed, `#include` expanded, conditionals evaluated, macros expanded. +//! Every output line remembers the pack file and line it came from, so errors found later point +//! at the pack's own code. +//! +//! `#include` is expanded where it stands in live code only: an include inside a false `#if` +//! is not read, so it can't fail (OQ-14). Ten levels deep at most, which also stops a file that +//! includes itself. `#version` and `#extension` are taken out of the code and kept apart: the +//! translation writes its own header. + +use std::collections::HashMap; + +use crate::Macros; +use crate::Pack; +use crate::condition::{Conditionals, split_directive}; +use crate::pack::normalize; + +const MAX_DEPTH: usize = 10; + +/// Where a line came from: an index into [`Preprocessed::files`] and a line number from 1. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Origin { + pub file: u16, + pub line: u32, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Diagnostic { + pub origin: Origin, + pub message: String, +} + +#[derive(Debug, Default)] +pub struct Preprocessed { + /// The code, one line per entry of `lines`. + pub code: String, + pub lines: Vec, + /// The pack paths of the files read, the stage file first. + pub files: Vec, + /// `#version` of the stage file: the number and the profile word, if any. + pub version: Option<(u32, Option)>, + /// `#extension` lines of live code, as written after `#extension`. + pub extensions: Vec, + /// The text of every block comment in live code (`DRAWBUFFERS:`, buffer formats). + pub comments: Vec<(Origin, String)>, + pub diagnostics: Vec, +} + +impl Preprocessed { + /// `file:line` of an output line (from 0), for messages. + pub fn locate(&self, line: usize) -> String { + let o = self.lines.get(line).copied().unwrap_or_default(); + let file = self.files.get(o.file as usize).map_or("?", String::as_str); + format!("{file}:{}", o.line) + } +} + +/// Runs the preprocessor on the stage file `path` of `pack` with `macros` defined. +pub fn preprocess(pack: &Pack, path: &str, macros: &Macros) -> Preprocessed { + let mut run = Run { + pack, + macros: macros.clone(), + conditionals: Conditionals::default(), + out: Preprocessed::default(), + code: Vec::new(), + texts: HashMap::new(), + }; + run.file(path, 0, Origin::default()); + if run.conditionals.open() { + run.diagnose(Origin::default(), "`#if` without `#endif`"); + } + run.out.code = run.code.join("\n"); + run.out +} + +struct Run<'p> { + pack: &'p Pack, + macros: Macros, + conditionals: Conditionals, + out: Preprocessed, + code: Vec, + /// Files already read: a guarded header is included many times. + texts: HashMap>, +} + +impl Run<'_> { + fn file(&mut self, path: &str, depth: usize, from: Origin) { + let cached = self + .texts + .entry(path.to_owned()) + .or_insert_with(|| self.pack.read_text(path)); + let Some(text) = cached.clone() else { + return self.diagnose(from, &format!("can't find `{path}`")); + }; + let file = match self.out.files.iter().position(|f| f == path) { + Some(i) => i, + None => { + self.out.files.push(path.to_owned()); + self.out.files.len() - 1 + } + } as u16; + let (text, comments) = strip_comments(&text); + let lines: Vec<&str> = text.split('\n').map(|l| l.trim_end_matches('\r')).collect(); + let mut comments = comments.into_iter().peekable(); + let mut n = 0; + while n < lines.len() { + let first = n; + let mut line = lines[n].to_owned(); + while line.ends_with('\\') && n + 1 < lines.len() { + line.pop(); + n += 1; + line.push_str(lines[n]); + } + n += 1; + let origin = Origin { + file, + line: first as u32 + 1, + }; + self.line(&line, origin, path, depth); + for k in first + 1..n { + self.emit( + String::new(), + Origin { + file, + line: k as u32 + 1, + }, + ); + } + while let Some((_, comment)) = comments.next_if(|(at, _)| *at < n) { + if self.conditionals.active() { + self.out.comments.push((origin, comment)); + } + } + } + } + + fn line(&mut self, line: &str, origin: Origin, path: &str, depth: usize) { + let Some(after) = line.trim_start().strip_prefix('#') else { + let code = if self.conditionals.active() { + self.macros.define("__LINE__", origin.line.to_string()); + self.macros.expand(line) + } else { + String::new() + }; + return self.emit(code, origin); + }; + self.emit(String::new(), origin); + let Some((word, rest)) = split_directive(after) else { + return self.diagnose(origin, &format!("unknown directive `#{}`", after.trim())); + }; + match self.conditionals.apply(word, rest, &self.macros) { + Err(e) => return self.diagnose(origin, &e), + Ok(true) => return, + Ok(false) if !self.conditionals.active() => return, + Ok(false) => {} + } + match word { + "define" => self.define(rest, origin), + "undef" => self.macros.undef(rest), + "include" => { + let target = rest.trim_matches(|c| c == '"' || c == '<' || c == '>'); + match resolve(path, target) { + _ if depth + 1 >= MAX_DEPTH => self.diagnose( + origin, + &format!("`{target}`: includes nest more than {MAX_DEPTH} deep"), + ), + Some(file) => { + // The include line's own (empty) output line stays; the file follows. + self.file(&file, depth + 1, origin); + } + None => self.diagnose(origin, &format!("`{target}` is outside the pack")), + } + } + "version" if depth == 0 && self.out.version.is_none() => { + let mut words = rest.split_whitespace(); + let number = words.next().and_then(|v| v.parse().ok()).unwrap_or(110); + self.out.version = Some((number, words.next().map(str::to_owned))); + self.macros.define("__VERSION__", number.to_string()); + } + "extension" => self.out.extensions.push(rest.to_owned()), + "error" => self.diagnose(origin, &format!("#error {rest}")), + "version" | "pragma" | "line" | "" => {} + _ => self.diagnose(origin, &format!("unknown directive `#{word}`")), + } + } + + /// `#define NAME body` or `#define NAME(a, b) body` (no space before the `(`). + fn define(&mut self, rest: &str, origin: Origin) { + let len = rest + .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_')) + .unwrap_or(rest.len()); + let (name, tail) = rest.split_at(len); + if name.is_empty() { + return self.diagnose(origin, "`#define` without a name"); + } + if let Some(params) = tail.strip_prefix('(') { + let Some((params, body)) = params.split_once(')') else { + return self.diagnose(origin, &format!("`#define {name}(` without `)`")); + }; + let params = params + .split(',') + .map(|p| p.trim().to_owned()) + .filter(|p| !p.is_empty()) + .collect(); + self.macros.define_function(name, params, body.trim()); + } else { + self.macros.define(name, tail.trim()); + } + } + + fn emit(&mut self, code: String, origin: Origin) { + self.code.push(code); + self.out.lines.push(origin); + } + + fn diagnose(&mut self, origin: Origin, message: &str) { + self.out.diagnostics.push(Diagnostic { + origin, + message: message.to_owned(), + }); + } +} + +/// An `#include` path: from the `shaders` folder with a leading `/`, else from the including +/// file's folder (spec 1.4). `None` when it climbs out of the pack. +fn resolve(from_file: &str, target: &str) -> Option { + match target.strip_prefix('/') { + Some(absolute) => normalize(absolute), + None => { + let dir = from_file.rsplit_once('/').map_or("", |(dir, _)| dir); + normalize(&format!("{dir}/{target}")) + } + } +} + +/// `text` with comments replaced by spaces (new lines kept, so line numbers hold), and the body of +/// each block comment with the index of the line it starts on. +fn strip_comments(text: &str) -> (String, Vec<(usize, String)>) { + let mut out = String::with_capacity(text.len()); + let mut comments = Vec::new(); + let (mut rest, mut line) = (text, 0); + while let Some(at) = rest.find('/') { + let (before, from) = rest.split_at(at); + out.push_str(before); + line += before.matches('\n').count(); + if let Some(body) = from.strip_prefix("//") { + let end = body.find('\n').unwrap_or(body.len()); + rest = &body[end..]; + } else if let Some(body) = from.strip_prefix("/*") { + let end = body.find("*/").unwrap_or(body.len()); + let comment = &body[..end]; + comments.push((line, comment.to_owned())); + let newlines = comment.matches('\n').count(); + line += newlines; + out.push(' '); + out.extend(std::iter::repeat_n('\n', newlines)); + rest = body.get(end + 2..).unwrap_or(""); + } else { + out.push('/'); + rest = &from[1..]; + } + } + out.push_str(rest); + (out, comments) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pack::tests::zip; + + fn run(files: &[(&str, &str)]) -> Preprocessed { + let files: Vec<(String, &str)> = files + .iter() + .map(|(p, t)| (format!("shaders/{p}"), *t)) + .collect(); + let files: Vec<(&str, &str)> = files.iter().map(|(p, t)| (p.as_str(), *t)).collect(); + let pack = Pack::from_zip_bytes("P.zip", zip(&files)).unwrap(); + preprocess(&pack, "program/final.fsh", &Macros::standard()) + } + + fn code_lines(p: &Preprocessed) -> Vec<(&str, String)> { + p.code + .lines() + .enumerate() + .filter(|(_, l)| !l.trim().is_empty()) + .map(|(i, l)| (l.trim(), p.locate(i))) + .collect() + } + + #[test] + fn includes_resolve_and_lines_map_back() { + let p = run(&[ + ( + "program/final.fsh", + "#version 330 compatibility\n#include \"/lib/a.glsl\"\nvoid main() {}\n", + ), + ( + "lib/a.glsl", + "#ifndef A\n#define A\n#include \"b.glsl\"\nfloat a;\n#endif\n", + ), + ("lib/b.glsl", "// b\nfloat b;\n#include \"a.glsl\"\n"), + ]); + assert_eq!(p.version, Some((330, Some("compatibility".into())))); + assert_eq!( + code_lines(&p), + [ + ("float b;", "lib/b.glsl:2".into()), + ("float a;", "lib/a.glsl:4".into()), + ("void main() {}", "program/final.fsh:3".into()), + ] + ); + assert!(p.diagnostics.is_empty(), "{:?}", p.diagnostics); + } + + #[test] + fn missing_and_runaway_includes_are_diagnosed_where_they_stand() { + let p = run(&[( + "program/final.fsh", + "#ifdef NOPE\n#include \"gone.glsl\"\n#endif\n#include \"gone.glsl\"\n#include \"final.fsh\"\n", + )]); + let messages: Vec = p + .diagnostics + .iter() + .map(|d| format!("{}: {}", d.origin.line, d.message)) + .collect(); + assert_eq!(messages[0], "4: can't find `program/gone.glsl`"); + assert!( + messages + .last() + .unwrap() + .ends_with("includes nest more than 10 deep") + ); + assert!( + run(&[("program/final.fsh", "#include \"../../x\"")]).diagnostics[0] + .message + .contains("outside") + ); + } + + #[test] + fn macros_expand_like_the_glsl_preprocessor() { + let p = run(&[( + "program/final.fsh", + "#define SQ(x) ((x) * (x))\n#define CAT(a, b) a ## b\n#define N 4\n#define \\\n LONG 1\n\ + float v = SQ(N + 1) + CAT(N, 2) + LONG;\nint l = __LINE__;\n#if MC_VERSION > 12100 && defined MC_RENDER_STAGE_SKY\nint ok;\n#else\nint bad;\n#endif\n", + )]); + let lines: Vec<&str> = code_lines(&p).into_iter().map(|(l, _)| l).collect(); + assert_eq!( + lines, + [ + "float v = ((4 + 1) * (4 + 1)) + N2 + 1;", + "int l = 7;", + "int ok;" + ] + ); + assert_eq!(p.lines.len(), p.code.split('\n').count()); + } + + #[test] + fn comments_extensions_and_errors() { + let p = run(&[( + "program/final.fsh", + "#version 120\n#extension GL_ARB_shader_texture_lod : enable\n/* DRAWBUFFERS:04 */\n#if 0\n/* RENDERTARGETS: 1 */\n#error dead\n#endif\n\ + /* multi\nline */ float x; // gone\n#error live\n#bogus\n", + )]); + assert_eq!(p.extensions, ["GL_ARB_shader_texture_lod : enable"]); + let comments: Vec<&str> = p.comments.iter().map(|(_, c)| c.trim()).collect(); + assert_eq!(comments, ["DRAWBUFFERS:04", "multi\nline"]); + assert_eq!(code_lines(&p), [("float x;", "program/final.fsh:9".into())]); + let messages: Vec<&str> = p.diagnostics.iter().map(|d| d.message.as_str()).collect(); + assert_eq!(messages, ["#error live", "unknown directive `#bogus`"]); + } +} diff --git a/crates/horizon-shaders/tests/real_packs.rs b/crates/horizon-shaders/tests/real_packs.rs index e0ad38c25..c252df6e3 100644 --- a/crates/horizon-shaders/tests/real_packs.rs +++ b/crates/horizon-shaders/tests/real_packs.rs @@ -5,7 +5,7 @@ use std::path::Path; use std::time::Instant; -use horizon_shaders::{Lang, Macros, Pack, Properties, discover}; +use horizon_shaders::{Lang, Macros, Pack, Properties, discover, preprocess}; #[test] #[ignore = "needs HORIZON_TEST_SHADER_PACKS"] @@ -52,6 +52,40 @@ fn real_packs_read() { for n in &shaders.notes { println!(" shaders.properties:{}: {}", n.line, n.message); } + // Every stage file of the base and dimension folders through the preprocessor. + let start = Instant::now(); + let (mut stages, mut lines, mut diagnostics) = (0, 0, Vec::new()); + for dir in ["", "world0", "world-1", "world1"] { + for file in pack.list(dir) { + if [".vsh", ".fsh", ".gsh", ".csh"] + .iter() + .any(|e| file.ends_with(e)) + { + let path = if dir.is_empty() { + file + } else { + format!("{dir}/{file}") + }; + let p = preprocess(&pack, &path, ¯os); + stages += 1; + lines += p.lines.len(); + diagnostics.extend(p.diagnostics.iter().map(|d| { + let file = &p.files[d.origin.file as usize]; + format!("{file}:{}: {}", d.origin.line, d.message) + })); + } + } + } + println!( + " {stages} stage files preprocessed, {lines} lines out, {} diagnostics, {:.1} ms", + diagnostics.len(), + start.elapsed().as_secs_f64() * 1000.0 + ); + diagnostics.sort(); + diagnostics.dedup(); + for d in diagnostics.iter().take(8) { + println!(" {d}"); + } assert!(!shaders.entries.is_empty(), "{}: no keys read", f.name); read += 1; } From b61b25f80fa2afdb7249ccc48c5e2e0f9eb90c68 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:35:37 +0200 Subject: [PATCH 5/5] Shader packs, clean room 6: composite-style programs to naga A program's preprocessed GLSL, any version from 110 to 460, compatibility or core, rewritten into the Vulkan-flavoured GLSL 450 that naga's frontend reads, then parsed and validated into naga modules with the program's interface: its uniform block (offsets from naga's layout), its samplers and their bindings, its output locations. The rewrite works on preprocessing tokens and keeps the pack's lines, so every error points at the pack's own file and line through the preprocessor's line map. - Loose uniforms go into one block shared by both stages; bool is stored as int. Samplers are split into texture and sampler objects, combined again where used, and passed as two arguments to the pack's functions. - The compatibility built-ins become declarations of ours (prefix hz_): gl_Vertex, gl_MultiTexCoordN, the matrices and ftransform(), gl_FragData, gl_FragColor, gl_TexCoord, gl_FrontColor, gl_Fog (from vanilla's fog), texture2D and the other old texture functions, shadow2D; varyings get locations by name across the stages; words that became keywords after GLSL 1.x are renamed in old code. - naga gaps covered: modf, conditional initializers of globals (run by our main), and unsized arrays sized from their constructor. - The pack's main is called by one of ours that draws the quad from (0,0) to (1,1) and adapts GL's conventions: clip y negated for every target but the window, so memory rows are GL's rows and gl_FragCoord needs no change; clip z from GL's [-w, w] to [0, w], so stored depth is GL's window depth. In final, gl_FragCoord.y is counted from the bottom of the window. Black box over the public test packs: all 105 composite-style programs (composite, deferred, final, prepare, begin, shadowcomp of the base and dimension folders) translate and validate. Nothing uses it yet: players see no change. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 11 + crates/horizon-shaders/Cargo.toml | 1 + crates/horizon-shaders/src/lib.rs | 3 + crates/horizon-shaders/src/translate/mod.rs | 436 ++++++++++++++++++ .../horizon-shaders/src/translate/rewrite.rs | 257 +++++++++++ crates/horizon-shaders/src/translate/scan.rs | 399 ++++++++++++++++ crates/horizon-shaders/tests/real_packs.rs | 73 ++- 7 files changed, 1179 insertions(+), 1 deletion(-) create mode 100644 crates/horizon-shaders/src/translate/mod.rs create mode 100644 crates/horizon-shaders/src/translate/rewrite.rs create mode 100644 crates/horizon-shaders/src/translate/scan.rs diff --git a/Cargo.lock b/Cargo.lock index 7036ea5ca..7b1770094 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1435,6 +1435,7 @@ name = "horizon-shaders" version = "0.1.0" dependencies = [ "horizon-core", + "naga", "zip", ] @@ -1951,6 +1952,7 @@ dependencies = [ "naga-types", "num-traits", "once_cell", + "pp-rs", "rustc-hash 1.1.0", "spirv", "thiserror 2.0.21", @@ -2627,6 +2629,15 @@ dependencies = [ "zerovec", ] +[[package]] +name = "pp-rs" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb458bb7f6e250e6eb79d5026badc10a3ebb8f9a15d1fff0f13d17c71f4d6dee" +dependencies = [ + "unicode-xid", +] + [[package]] name = "presser" version = "0.3.1" diff --git a/crates/horizon-shaders/Cargo.toml b/crates/horizon-shaders/Cargo.toml index 750949e83..38e068b94 100644 --- a/crates/horizon-shaders/Cargo.toml +++ b/crates/horizon-shaders/Cargo.toml @@ -8,6 +8,7 @@ description = "Shader packs (the shaders/ folder format): reading packs, their . [dependencies] horizon-core = { path = "../horizon-core" } +naga = { version = "30.0.1", features = ["glsl-in", "wgsl-in"] } zip = { version = "2", default-features = false, features = ["deflate"] } [lints] diff --git a/crates/horizon-shaders/src/lib.rs b/crates/horizon-shaders/src/lib.rs index 14f85b758..bae3c0dcd 100644 --- a/crates/horizon-shaders/src/lib.rs +++ b/crates/horizon-shaders/src/lib.rs @@ -7,6 +7,7 @@ //! - [`Properties`]: `shaders.properties` and the id files, with their conditional directives (3.1, 1.5). //! - [`Lang`]: the pack's translated names (8.4). //! - [`preprocess`]: a stage file through `#include` and the GLSL preprocessor, lines mapped back (1.4, 1.5). +//! - [`translate_composite`]: a composite-style program into naga modules and its interface (1.6, 2.2). //! - [`Macros`] and [`eval_condition`]: the macros packs test, and the `#if` arithmetic (1.5). mod condition; @@ -16,6 +17,7 @@ mod macros; mod pack; mod preprocess; mod properties; +mod translate; pub use condition::eval_condition; pub use discover::{Found, discover}; @@ -24,3 +26,4 @@ pub use macros::{MC_VERSION, Macros, RENDER_STAGES}; pub use pack::{Pack, PackError}; pub use preprocess::{Diagnostic, Origin, Preprocessed, preprocess}; pub use properties::{Entry, Note, Properties}; +pub use translate::{Error, Interface, Program, Sampler, Target, Uniform, translate_composite}; diff --git a/crates/horizon-shaders/src/translate/mod.rs b/crates/horizon-shaders/src/translate/mod.rs new file mode 100644 index 000000000..9cc2ea07a --- /dev/null +++ b/crates/horizon-shaders/src/translate/mod.rs @@ -0,0 +1,436 @@ +//! A program's preprocessed GLSL (any version from 110 to 460, compatibility or core) turned +//! into the Vulkan-flavoured GLSL 450 naga reads, then into validated naga modules (spec 1.6, +//! 2.2). The rewrite works on preprocessing tokens and never adds or removes a line of the pack's +//! code, so naga's line numbers map back through the preprocessor to the pack's files. +//! +//! What it does: +//! - Loose uniforms go into one uniform block shared by both stages (`bool` stored as `int`). +//! - Each sampler is split into a texture and a sampler object, combined again where it is used +//! (`sampler2D(t, s)`), or passed as two arguments to the pack's own functions. +//! - The compatibility built-ins (`gl_Vertex`, `gl_MultiTexCoordN`, `gl_*Matrix`, `ftransform()`, +//! `gl_FragData`, `gl_FragColor`, `gl_TexCoord`, `texture2D`...) become declarations of ours, +//! named with the reserved prefix `hz_`; varyings get locations by name across the stages. +//! - The pack's `main` becomes `hz_main`, called by a `main` of ours that sets the inputs and +//! adapts GL's conventions (spec 5.5): clip y is negated for every target but the window, so +//! memory rows are GL's rows and `gl_FragCoord` needs no change, and clip z goes from GL's +//! [-w, w] to [0, w], so stored depth is GL's window depth. +//! +//! Composite-style programs draw one quad from (0,0) to (1,1) with an identity model-view and an +//! orthographic projection onto it, the texture coordinate equal to the position (OQ-6). + +use std::collections::BTreeMap; +use std::fmt::Write; + +use naga::ShaderStage; +use naga::front::glsl::{Frontend, Options}; +use naga::valid::{Capabilities, ValidationFlags, Validator}; + +use crate::Preprocessed; + +mod rewrite; +mod scan; + +use scan::Stage; + +/// What a program's stages need from the game. +#[derive(Debug, Default, PartialEq)] +pub struct Interface { + /// The uniform block at binding 0: its size and each value's name, GLSL type and offset. + /// Ours come first: `hz_target` (the target's width, height, 1/width, 1/height), and the + /// fixed-function fog the compatibility `gl_Fog` reads (OQ-28): `hz_fog_color` (vanilla's fog + /// colour) and `hz_fog` (density, start, end, 1 / (end - start)). + pub uniform_size: u32, + pub uniforms: Vec, + /// Each sampler: texture at `binding`, sampler object at `binding + 1`. + pub samplers: Vec, + /// The fragment stage's output locations. + pub outputs: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Uniform { + pub name: String, + /// The type the pack declared (`float`, `ivec2`, `mat4`; `bool` is stored as an `int`). + pub ty: String, + pub offset: u32, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Sampler { + pub name: String, + /// The type the pack declared: `sampler2D`, `sampler2DShadow`, `usampler2D`, `sampler3D`... + pub ty: String, + pub binding: u32, +} + +/// A program that failed, at a pack file and line. +#[derive(Clone, Debug, PartialEq)] +pub struct Error { + pub file: String, + pub line: u32, + pub message: String, +} + +pub struct Program { + pub vertex: naga::Module, + pub fragment: naga::Module, + pub interface: Interface, +} + +/// Where the program draws, which decides the conventions of 5.5. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Target { + /// Pack buffers: rows stored as GL stores them. + Buffers, + /// The window (`final`). + Window, +} + +/// Translates a composite-style program (`composite`, `deferred`, `final`...). +pub fn translate_composite( + vertex: &Preprocessed, + fragment: &Preprocessed, + target: Target, +) -> Result> { + let mut v = Stage::scan(vertex, ShaderStage::Vertex); + let mut f = Stage::scan(fragment, ShaderStage::Fragment); + let mut errors: Vec = v.errors.drain(..).chain(f.errors.drain(..)).collect(); + + // One uniform block and one set of samplers for both stages. + let mut uniforms: BTreeMap = BTreeMap::new(); + let mut samplers: BTreeMap = BTreeMap::new(); + for stage in [&v, &f] { + for (name, ty, at) in &stage.uniforms { + if let Some(old) = uniforms.insert(name.clone(), ty.clone()) + && &old != ty + { + errors.push(stage.error( + *at, + &format!("`{name}` is `{old}` in one stage and `{ty}` in the other"), + )); + } + } + for (name, ty) in &stage.samplers { + samplers.insert(name.clone(), ty.clone()); + } + } + let samplers: Vec = samplers + .into_iter() + .enumerate() + .map(|(i, (name, ty))| Sampler { + name, + ty, + binding: 1 + 2 * i as u32, + }) + .collect(); + + // Varyings by name: the vertex outputs, then fragment inputs the vertex stage lacks. + let mut locations: BTreeMap = BTreeMap::new(); + for var in v.outs.iter().chain(&f.ins) { + let next = locations.len() as u32; + locations + .entry(var.name.clone()) + .or_insert((next, var.decl.clone())); + } + + let header = |stage: &Stage| { + let mut h = String::from("#version 450\n"); + h.push_str("layout(set = 0, binding = 0) uniform HzUniforms { vec4 hz_target; vec4 hz_fog_color; vec4 hz_fog;"); + for (name, ty) in &uniforms { + match ty.as_str() { + "bool" => write!(h, " int hz_bool_{name};"), + _ => match ty.split_once('[') { + Some((base, size)) => write!(h, " {base} {name}[{size};"), + None => write!(h, " {ty} {name};"), + }, + } + .unwrap(); + } + h.push_str(" };\n"); + for s in &samplers { + let (texture, sampler) = split_sampler(&s.ty); + writeln!( + h, + "layout(set = 0, binding = {}) uniform {texture} {}; layout(set = 0, binding = {}) uniform {sampler} hz_s_{};", + s.binding, s.name, s.binding + 1, s.name + ) + .unwrap(); + } + let direction = if stage.kind == ShaderStage::Vertex { + "out" + } else { + "in" + }; + for (name, (location, decl)) in &locations { + writeln!( + h, + "layout(location = {location}) {direction} {}", + decl.replace("@", name) + ) + .unwrap(); + } + h.push_str(&stage.prelude()); + h + }; + let flip = if target == Target::Buffers { "-" } else { "" }; + let fog = + " hz_Fog = hz_FogParameters(hz_fog_color, hz_fog.x, hz_fog.y, hz_fog.z, hz_fog.w);\n"; + let mut vertex_main = String::from( + "\nvoid main() {\n vec2 hz_p = hz_quad[gl_VertexIndex];\n hz_Vertex = vec4(hz_p, 0.0, 1.0);\n \ + hz_MultiTexCoord0 = hz_Vertex;\n hz_Color = vec4(1.0);\n hz_Normal = vec3(0.0, 0.0, 1.0);\n", + ); + for (name, value) in &v.attributes { + writeln!(vertex_main, " {name} = {value};").unwrap(); + } + vertex_main.push_str(fog); + for name in &v.inits { + writeln!(vertex_main, " {name} = hz_init_{name}();").unwrap(); + } + write!( + vertex_main, + " hz_main();\n gl_Position = vec4(gl_Position.x, {flip}gl_Position.y, (gl_Position.z + gl_Position.w) * 0.5, gl_Position.w);\n}}\n" + ) + .unwrap(); + let mut fragment_main = String::from("\nvoid main() {\n"); + fragment_main.push_str(fog); + if target == Target::Window && f.frag_coord { + fragment_main.push_str(" hz_FragCoord = vec4(gl_FragCoord.x, hz_target.y - gl_FragCoord.y, gl_FragCoord.zw);\n"); + } + for name in &f.inits { + writeln!(fragment_main, " {name} = hz_init_{name}();").unwrap(); + } + fragment_main.push_str(" hz_main();\n}\n"); + + let v_text = format!( + "{}{}{vertex_main}", + header(&v), + v.rewrite(&samplers, &uniforms) + ); + let f_text = format!( + "{}{}{fragment_main}", + header(&f), + f.rewrite(&samplers, &uniforms) + ); + if !errors.is_empty() { + return Err(errors); + } + let vertex_module = compile(&v, &v_text, header(&v).lines().count()); + let fragment_module = compile(&f, &f_text, header(&f).lines().count()); + let (vertex, fragment) = match (vertex_module, fragment_module) { + (Ok(v), Ok(f)) => (v, f), + (a, b) => return Err(a.err().into_iter().chain(b.err()).flatten().collect()), + }; + let (uniform_size, offsets) = block_layout(&fragment); + let uniforms = offsets + .into_iter() + .filter_map(|(member, offset)| { + let name = member + .strip_prefix("hz_bool_") + .unwrap_or(&member) + .to_owned(); + let ty = if name.starts_with("hz_") { + "vec4".to_owned() + } else { + uniforms.get(&name)?.clone() + }; + Some(Uniform { name, ty, offset }) + }) + .collect(); + let outputs = f.outputs.iter().copied().collect(); + Ok(Program { + vertex, + fragment, + interface: Interface { + uniform_size, + uniforms, + samplers, + outputs, + }, + }) +} + +/// `sampler2DShadow` → (`texture2D`, `samplerShadow`); `isampler3D` → (`itexture3D`, `sampler`). +pub(super) fn split_sampler(ty: &str) -> (String, &'static str) { + let shadow = ty.ends_with("Shadow"); + let base = ty + .trim_end_matches("Shadow") + .replacen("sampler", "texture", 1); + (base, if shadow { "samplerShadow" } else { "sampler" }) +} + +/// Parses and validates one stage; errors at the pack's files and lines. +fn compile(stage: &Stage, text: &str, header_lines: usize) -> Result> { + let at = |span: naga::Span, message: String| { + let start = span.to_range().map_or(0, |r| r.start); + let line = text[..start.min(text.len())].matches('\n').count(); + match line + .checked_sub(header_lines) + .and_then(|l| stage.pp.lines.get(l)) + { + Some(o) => Error { + file: stage + .pp + .files + .get(o.file as usize) + .cloned() + .unwrap_or_default(), + line: o.line, + message, + }, + None => Error { + file: "".into(), + line: line as u32 + 1, + message, + }, + } + }; + let module = Frontend::default() + .parse(&Options::from(stage.kind), text) + .map_err(|e| { + e.errors + .into_iter() + .map(|e| at(e.meta, e.kind.to_string())) + .collect::>() + })?; + Validator::new(ValidationFlags::all(), Capabilities::all()) + .validate(&module) + .map_err(|e| { + let span = e.spans().next().map_or(naga::Span::default(), |(s, _)| *s); + // The validator nests its reasons: say them all. + let inner = e.into_inner(); + let mut message = inner.to_string(); + let mut source = std::error::Error::source(&inner); + while let Some(s) = source { + write!(message, ": {s}").unwrap(); + source = s.source(); + } + vec![at(span, message)] + })?; + Ok(module) +} + +/// The uniform block's size and member offsets, from naga's layout. +fn block_layout(module: &naga::Module) -> (u32, Vec<(String, u32)>) { + for (_, var) in module.global_variables.iter() { + if var.space != naga::AddressSpace::Uniform { + continue; + } + if let naga::TypeInner::Struct { members, span } = &module.types[var.ty].inner { + let offsets = members + .iter() + .filter_map(|m| Some((m.name.clone()?, m.offset))) + .collect(); + return (*span, offsets); + } + } + (16, vec![("hz_target".into(), 0)]) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pack::tests::zip; + use crate::{Macros, Pack, preprocess}; + + fn program(files: &[(&str, &str)], target: Target) -> Result> { + let files: Vec<(String, &str)> = files + .iter() + .map(|(p, t)| (format!("shaders/{p}"), *t)) + .collect(); + let files: Vec<(&str, &str)> = files.iter().map(|(p, t)| (p.as_str(), *t)).collect(); + let pack = Pack::from_zip_bytes("P.zip", zip(&files)).unwrap(); + let m = Macros::standard(); + translate_composite( + &preprocess(&pack, "composite.vsh", &m), + &preprocess(&pack, "composite.fsh", &m), + target, + ) + } + + const VSH_120: &str = "#version 120\nvarying vec2 texcoord;\nvoid main() {\n gl_Position = ftransform();\n \ + texcoord = (gl_TextureMatrix[0] * gl_MultiTexCoord0).xy;\n}\n"; + + #[test] + fn compatibility_programs_translate() { + let fsh = "#version 120\nuniform sampler2D colortex0;\nuniform sampler2D depthtex0;\nuniform float viewWidth;\n\ + uniform bool hideGUI;\nuniform int frameCounter;\nvarying vec2 texcoord;\n\ + vec3 tint(sampler2D s, vec2 c) { return texture2D(s, c).rgb; }\nvoid main() {\n \ + vec3 c = tint(colortex0, texcoord) * texture2D(depthtex0, texcoord).r;\n if (hideGUI) c *= 0.5;\n \ + /* DRAWBUFFERS:04 */\n gl_FragData[0] = vec4(c, 1.0);\n gl_FragData[1] = vec4(frameCounter / viewWidth);\n}\n"; + let p = program( + &[("composite.vsh", VSH_120), ("composite.fsh", fsh)], + Target::Buffers, + ) + .unwrap(); + let i = &p.interface; + let samplers: Vec<(&str, u32)> = i + .samplers + .iter() + .map(|s| (s.name.as_str(), s.binding)) + .collect(); + assert_eq!(samplers, [("colortex0", 1), ("depthtex0", 3)]); + let uniforms: Vec<(&str, &str)> = i + .uniforms + .iter() + .map(|u| (u.name.as_str(), u.ty.as_str())) + .collect(); + assert_eq!( + uniforms, + [ + ("hz_target", "vec4"), + ("hz_fog_color", "vec4"), + ("hz_fog", "vec4"), + ("frameCounter", "int"), + ("hideGUI", "bool"), + ("viewWidth", "float") + ] + ); + assert_eq!(i.uniforms[0].offset, 0); + assert!(i.uniform_size >= 28); + assert_eq!(i.outputs, [0, 1]); + } + + #[test] + fn core_programs_translate() { + let vsh = "#version 330 core\nin vec3 vaPosition;\nin vec2 vaUV0;\nuniform mat4 modelViewMatrix, projectionMatrix;\n\ + out vec2 uv;\nflat out int flatId;\nvoid main() {\n \ + gl_Position = projectionMatrix * modelViewMatrix * vec4(vaPosition, 1.0);\n uv = vaUV0;\n flatId = 1;\n}\n"; + let fsh = "#version 330 core\nin vec2 uv;\nuniform sampler2D colortex0;\nuniform usampler2D colortex5;\n\ + layout(location = 2) out vec4 color;\nvoid main() {\n \ + if (texture(colortex0, uv).a < 0.1) discard;\n color = texture(colortex0, uv) + float(texelFetch(colortex5, ivec2(gl_FragCoord.xy), 0).r);\n}\n"; + let p = program( + &[("composite.vsh", vsh), ("composite.fsh", fsh)], + Target::Window, + ) + .unwrap(); + assert_eq!(p.interface.outputs, [2]); + assert!( + p.interface + .uniforms + .iter() + .any(|u| u.name == "projectionMatrix" && u.ty == "mat4") + ); + } + + #[test] + fn errors_point_at_the_pack_file_and_line() { + let fsh = + "#version 120\n#include \"/lib/bad.glsl\"\nvoid main() { gl_FragColor = vec4(f()); }\n"; + let bad = "// fine\nfloat f() {\n return undefinedThing;\n}\n"; + let errors = program( + &[ + ("composite.vsh", VSH_120), + ("composite.fsh", fsh), + ("lib/bad.glsl", bad), + ], + Target::Buffers, + ) + .err() + .unwrap(); + assert_eq!( + (errors[0].file.as_str(), errors[0].line), + ("lib/bad.glsl", 3), + "{errors:?}" + ); + } +} diff --git a/crates/horizon-shaders/src/translate/rewrite.rs b/crates/horizon-shaders/src/translate/rewrite.rs new file mode 100644 index 000000000..312575139 --- /dev/null +++ b/crates/horizon-shaders/src/translate/rewrite.rs @@ -0,0 +1,257 @@ +//! The second pass: the stage's code written again for naga, line for line, and the +//! declarations of ours it needs. + +use std::collections::{BTreeMap, HashMap}; +use std::fmt::Write; + +use naga::ShaderStage; + +use super::scan::{LATER_KEYWORDS, Stage, UNIFORM_STATE, VERTEX_INPUTS, is_ident, significant}; +use super::{Sampler, split_sampler}; + +impl Stage<'_> { + /// Declarations of ours this stage uses (after the uniform block, samplers and varyings). + pub(super) fn prelude(&self) -> String { + let mut p = String::from( + "float hz_modf(float x, out float i) { i = trunc(x); return x - i; }\n\ + vec2 hz_modf(vec2 x, out vec2 i) { i = trunc(x); return x - i; }\n\ + vec3 hz_modf(vec3 x, out vec3 i) { i = trunc(x); return x - i; }\n\ + vec4 hz_modf(vec4 x, out vec4 i) { i = trunc(x); return x - i; }\n\ + struct hz_FogParameters { vec4 color; float density; float start; float end; float scale; };\n\ + hz_FogParameters hz_Fog;\n\ + const mat4 hz_ModelViewMatrix = mat4(1.0); const mat4 hz_ProjectionMatrix = mat4(2.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, -1.0, -1.0, 0.0, 1.0);\n\ + const mat4 hz_ModelViewProjectionMatrix = hz_ProjectionMatrix; const mat4 hz_ModelViewMatrixInverse = mat4(1.0); const mat3 hz_NormalMatrix = mat3(1.0);\n\ + const mat4 hz_ProjectionMatrixInverse = mat4(0.5, 0.0, 0.0, 0.0, 0.0, 0.5, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.5, 0.5, 0.0, 1.0);\n\ + const mat4 hz_TextureMatrix[8] = mat4[8](mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0));\n", + ); + if self.kind == ShaderStage::Vertex { + p.push_str( + "const vec2 hz_quad[6] = vec2[6](vec2(0.0, 0.0), vec2(1.0, 0.0), vec2(1.0, 1.0), vec2(0.0, 0.0), vec2(1.0, 1.0), vec2(0.0, 1.0));\n\ + vec4 hz_Vertex; vec4 hz_Color; vec4 hz_SecondaryColor; vec3 hz_Normal; float hz_FogCoord;\n\ + vec4 hz_MultiTexCoord0; vec4 hz_MultiTexCoord1; vec4 hz_MultiTexCoord2; vec4 hz_MultiTexCoord3;\n\ + vec4 hz_MultiTexCoord4; vec4 hz_MultiTexCoord5; vec4 hz_MultiTexCoord6; vec4 hz_MultiTexCoord7;\n", + ); + } else { + for n in &self.outputs { + if !self.frag_outs.iter().any(|(at, _)| at == n) { + writeln!(p, "layout(location = {n}) out vec4 hz_out{n};").unwrap(); + } + } + for (_, decl) in &self.frag_outs { + writeln!(p, "{decl}").unwrap(); + } + p.push_str("vec4 hz_FragCoord;\n"); + } + p + } + + /// The pack's code with the rewrites above; the same number of lines. + pub(super) fn rewrite( + &self, + samplers: &[Sampler], + uniforms: &BTreeMap, + ) -> String { + let removed = |i: usize| self.removed.iter().any(|r| r.contains(&i)); + let sampler_ty: HashMap<&str, &str> = samplers + .iter() + .map(|s| (s.name.as_str(), s.ty.as_str())) + .collect(); + let old = self.pp.version.as_ref().is_none_or(|(v, _)| *v < 400); + let fragment = self.kind == ShaderStage::Fragment; + let mut out = String::with_capacity(self.pp.code.len() + 1024); + // Per open bracket: the function called, and the closing text to add. + let mut calls: Vec<(&str, &str, usize)> = Vec::new(); + // Sampler parameters of the function being written: name → type. + let mut params: HashMap = HashMap::new(); + let mut depth = 0; + let mut skip_until = 0; + let next = |i: usize| (i + 1..self.tokens.len()).find(|&j| significant(self.tokens[j])); + let prev = |i: usize| (0..i).rev().find(|&j| significant(self.tokens[j])); + for (i, &t) in self.tokens.iter().enumerate() { + if i < skip_until { + continue; + } + if let Some(text) = self.replaced.get(&i) { + out.push_str(text); + continue; + } + if removed(i) { + out.extend(t.chars().filter(|&c| c == '\n')); + if t.contains(' ') && !t.contains('\n') { + out.push(' '); + } + continue; + } + let called = next(i).is_some_and(|j| self.tokens[j] == "("); + match t { + "{" => depth += 1, + "}" => { + depth -= 1; + if depth == 0 { + params.clear(); + } + } + "(" => { + let callee = prev(i).map_or("", |j| self.tokens[j]); + let close = if matches!(callee, "shadow2D" | "shadow2DLod" | "shadow2DProj") { + ")" + } else { + "" + }; + calls.push((callee, close, 0)); + } + ")" => { + if let Some((_, close, _)) = calls.pop() { + out.push_str(close); + } + } + "," => { + if let Some(c) = calls.last_mut() { + c.2 += 1; + } + } + _ => {} + } + // `T x[] = T[](a, b)`: naga gives unsized arrays no size, so the count is written in. + if t == "[" + && let Some(close) = next(i).filter(|&j| self.tokens[j] == "]") + { + if let Some(n) = self.array_length(close) { + write!(out, "[{n}]").unwrap(); + skip_until = close + 1; + continue; + } + } + if !is_ident(t) { + out.push_str(t); + continue; + } + // A function definition's sampler parameters: `sampler2D s` → `texture2D s, sampler hz_s_s`. + if depth == 0 && t.contains("sampler") && calls.len() == 1 { + if let Some(n) = next(i).filter(|&n| is_ident(self.tokens[n])) { + let name = self.tokens[n]; + let (texture, sampler) = split_sampler(t); + write!(out, "{texture} {name}, {sampler} hz_s_{name}").unwrap(); + params.insert(name.to_owned(), t.to_owned()); + skip_until = n + 1; + continue; + } + } + let sampler = params + .get(t) + .map(String::as_str) + .or_else(|| sampler_ty.get(t).copied()); + if let Some(ty) = sampler { + let as_argument = calls.last().is_some_and(|(callee, _, arg)| { + self.sampler_functions + .get(*callee) + .is_some_and(|p| p.get(*arg) == Some(&true)) + }) && prev(i) + .is_some_and(|j| matches!(self.tokens[j], "(" | ",")); + if as_argument { + write!(out, "{t}, hz_s_{t}").unwrap(); + } else { + write!(out, "{ty}({t}, hz_s_{t})").unwrap(); + } + continue; + } + if uniforms.get(t).is_some_and(|ty| ty == "bool") { + write!(out, "(hz_bool_{t} != 0)").unwrap(); + continue; + } + let replaced: Option = match t { + "main" if depth == 0 && called => Some("hz_main".into()), + // GLSL's `modf`, which naga's frontend lacks. + "modf" if called => Some("hz_modf".into()), + "texture2D" | "texture1D" | "texture3D" | "textureCube" if called => { + Some("texture".into()) + } + "texture2DLod" | "texture3DLod" | "texture2DLodARB" | "textureCubeLod" + if called => + { + Some("textureLod".into()) + } + "texture2DProj" if called => Some("textureProj".into()), + "texture2DGrad" | "texture2DGradARB" if called => Some("textureGrad".into()), + "shadow2D" if called => Some("vec4(texture".into()), + "shadow2DLod" if called => Some("vec4(textureLod".into()), + "shadow2DProj" if called => Some("vec4(textureProj".into()), + "ftransform" if called => { + // `ftransform()` → the product it stands for; its `()` stays as a no-op call... so skip it. + let close = next(i).and_then(next); + skip_until = close.map_or(i + 1, |c| c + 1); + Some("(hz_ModelViewProjectionMatrix * hz_Vertex)".into()) + } + "gl_FragColor" => Some("hz_out0".into()), + "gl_FragData" | "gl_TexCoord" => match self.index_after(i) { + Some((n, close)) => { + skip_until = close + 1; + Some(match t { + "gl_FragData" => format!("hz_out{n}"), + _ => format!("hz_TexCoord{n}"), + }) + } + None => None, + }, + "gl_Color" if fragment => Some("hz_FrontColor".into()), + "gl_FragCoord" if !fragment => None, + "gl_FragCoord" => Some("hz_FragCoord".into()), + "gl_FrontColor" => Some("hz_FrontColor".into()), + "gl_Fog" => Some("hz_Fog".into()), + _ if UNIFORM_STATE.iter().any(|b| t.starts_with(b)) => { + Some(format!("hz_{}", &t[3..])) + } + _ if !fragment && VERTEX_INPUTS.iter().any(|b| t.starts_with(b)) => { + Some(format!("hz_{}", &t[3..])) + } + "gl_VertexID" => Some("gl_VertexIndex".into()), + "gl_InstanceID" => Some("gl_InstanceIndex".into()), + _ if old && LATER_KEYWORDS.contains(&t) => Some(format!("hz_kw_{t}")), + _ => None, + }; + out.push_str(replaced.as_deref().unwrap_or(t)); + } + out + } +} + +impl Stage<'_> { + /// The element count of an unsized array whose `[]` ends at `close`: the arguments of the + /// constructor that follows (`[] = T[](...)` or `[](...)`). + pub(super) fn array_length(&self, close: usize) -> Option { + let mut j = close + 1; + let mut seen_eq = false; + while let Some(&t) = self.tokens.get(j) { + match t { + "(" => break, + "=" if !seen_eq => seen_eq = true, + ";" | "," | ")" | "{" => return None, + _ => {} + } + j += 1; + } + let (mut depth, mut args, mut any) = (0, 1, false); + for &t in &self.tokens[j + 1..] { + match t { + "(" | "[" | "{" => depth += 1, + ")" if depth == 0 => return any.then_some(args), + ")" | "]" | "}" => depth -= 1, + "," if depth == 0 => args += 1, + _ if significant(t) => any = true, + _ => {} + } + } + None + } +} + +/// What our vertex `main` gives a vertex input of a composite-style program. +pub(super) fn attribute_value(name: &str, ty: &str) -> String { + match name { + "vaPosition" => "vec3(hz_p, 0.0)".into(), + "vaUV0" => "hz_p".into(), + "vaColor" => "vec4(1.0)".into(), + "vaNormal" => "vec3(0.0, 0.0, 1.0)".into(), + _ => format!("{ty}(0)"), + } +} diff --git a/crates/horizon-shaders/src/translate/scan.rs b/crates/horizon-shaders/src/translate/scan.rs new file mode 100644 index 000000000..1fd91dd94 --- /dev/null +++ b/crates/horizon-shaders/src/translate/scan.rs @@ -0,0 +1,399 @@ +//! The first pass over a stage's tokens: what it declares at global scope, which of its +//! functions take samplers, which outputs and built-ins it uses. + +use std::collections::{BTreeSet, HashMap}; +use std::fmt::Write; + +use naga::ShaderStage; + +use super::Error; +use super::rewrite::attribute_value; +use crate::Preprocessed; +use crate::macros::lex; + +/// A varying: its name and its declaration with `@` where the name goes (`flat ivec2 @[2];`). +pub(super) struct Var { + pub(super) name: String, + pub(super) decl: String, +} + +pub(super) struct Stage<'p> { + pub(super) kind: ShaderStage, + pub(super) pp: &'p Preprocessed, + pub(super) tokens: Vec<&'p str>, + /// Token ranges to blank out (declarations moved into the header). + pub(super) removed: Vec>, + /// Name, type with its array suffix, token of the type. + pub(super) uniforms: Vec<(String, String, usize)>, + pub(super) samplers: Vec<(String, String)>, + pub(super) ins: Vec, + pub(super) outs: Vec, + /// Vertex inputs, now private globals, and the value our `main` gives each. + pub(super) attributes: Vec<(String, String)>, + /// The pack's fragment outputs, declared again with their locations. + pub(super) frag_outs: Vec<(u32, String)>, + /// The pack's functions that take samplers: which parameters are samplers. + pub(super) sampler_functions: HashMap>, + pub(super) outputs: BTreeSet, + pub(super) frag_coord: bool, + /// Tokens written differently, by index. + pub(super) replaced: HashMap, + /// Globals whose initializer runs in our `main` (`g = hz_init_g();`), in order. + pub(super) inits: Vec, + pub(super) errors: Vec, +} + +pub(super) const STORAGE: [&str; 5] = ["uniform", "in", "out", "attribute", "varying"]; +pub(super) const QUALIFIERS: [&str; 11] = [ + "flat", + "smooth", + "noperspective", + "centroid", + "invariant", + "highp", + "mediump", + "lowp", + "precise", + "sample", + "const", +]; +/// The compatibility profile's uniform state (with its `Inverse`/`Transpose` forms). +pub(super) const UNIFORM_STATE: [&str; 5] = [ + "gl_ModelViewMatrix", + "gl_ProjectionMatrix", + "gl_ModelViewProjectionMatrix", + "gl_NormalMatrix", + "gl_TextureMatrix", +]; +/// The compatibility profile's vertex inputs. +pub(super) const VERTEX_INPUTS: [&str; 6] = [ + "gl_Vertex", + "gl_Color", + "gl_SecondaryColor", + "gl_Normal", + "gl_FogCoord", + "gl_MultiTexCoord", +]; +/// Words that became keywords after GLSL 1.x and that old packs use as names. +pub(super) const LATER_KEYWORDS: [&str; 17] = [ + "sample", + "buffer", + "shared", + "filter", + "input", + "output", + "common", + "partition", + "active", + "resource", + "patch", + "subroutine", + "coherent", + "volatile", + "restrict", + "readonly", + "writeonly", +]; + +pub(super) fn significant(t: &str) -> bool { + !t.trim().is_empty() +} + +pub(super) fn is_ident(t: &str) -> bool { + t.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_') +} + +impl<'p> Stage<'p> { + pub(super) fn scan(pp: &'p Preprocessed, kind: ShaderStage) -> Self { + let mut s = Stage { + kind, + pp, + tokens: lex(&pp.code), + removed: Vec::new(), + uniforms: Vec::new(), + samplers: Vec::new(), + ins: Vec::new(), + outs: Vec::new(), + attributes: Vec::new(), + frag_outs: Vec::new(), + sampler_functions: HashMap::new(), + outputs: BTreeSet::new(), + frag_coord: false, + replaced: HashMap::new(), + inits: Vec::new(), + errors: Vec::new(), + }; + let (mut start, mut depth, mut parens) = (0, 0usize, 0usize); + for i in 0..s.tokens.len() { + match s.tokens[i] { + "(" => parens += 1, + ")" => parens = parens.saturating_sub(1), + "{" if depth == 0 && parens == 0 => { + s.global(start, i, true); + depth = 1; + } + "{" => depth += 1, + "}" if depth > 0 => { + depth -= 1; + if depth == 0 { + start = i + 1; + } + } + ";" if depth == 0 && parens == 0 => { + s.global(start, i, false); + start = i + 1; + } + "gl_FragData" | "gl_FragColor" if kind == ShaderStage::Fragment => { + let n = if s.tokens[i] == "gl_FragColor" { + Some(0) + } else { + s.index_after(i).map(|(n, _)| n) + }; + match n { + Some(n) => _ = s.outputs.insert(n), + None => s + .errors + .push(s.error(i, "`gl_FragData` needs a constant index")), + } + } + "gl_FragCoord" => s.frag_coord = true, + "gl_TexCoord" => { + if let Some((n, _)) = s.index_after(i) { + s.builtin_varying(format!("hz_TexCoord{n}")); + } + } + "gl_FrontColor" => s.builtin_varying("hz_FrontColor".into()), + "gl_Color" if kind == ShaderStage::Fragment => { + s.builtin_varying("hz_FrontColor".into()) + } + _ => {} + } + } + s + } + + pub(super) fn builtin_varying(&mut self, name: String) { + let list = if self.kind == ShaderStage::Vertex { + &mut self.outs + } else { + &mut self.ins + }; + if !list.iter().any(|v| v.name == name) { + list.push(Var { + name, + decl: "vec4 @;".into(), + }); + } + } + + /// `[ n ]` right after token `i`: the number and the index of the `]`. + pub(super) fn index_after(&self, i: usize) -> Option<(u32, usize)> { + let mut it = (i + 1..self.tokens.len()).filter(|&j| significant(self.tokens[j])); + let open = it.next()?; + let n = it.next()?; + let close = it.next()?; + (self.tokens[open] == "[" && self.tokens[close] == "]") + .then(|| self.tokens[n].parse().ok().map(|n| (n, close))) + .flatten() + } + + pub(super) fn error(&self, token: usize, message: &str) -> Error { + let line: usize = self.tokens[..token.min(self.tokens.len())] + .iter() + .map(|t| t.matches('\n').count()) + .sum(); + let o = self.pp.lines.get(line).copied().unwrap_or_default(); + Error { + file: self + .pp + .files + .get(o.file as usize) + .cloned() + .unwrap_or_default(), + line: o.line, + message: message.to_owned(), + } + } + + /// A statement at global scope, tokens `from..to` (without its `;`, or before a `{`). + pub(super) fn global(&mut self, from: usize, to: usize, body: bool) { + let words: Vec = (from..to) + .filter(|&i| significant(self.tokens[i])) + .collect(); + // A prototype: a `(` before any `=`, and no storage qualifier. + let paren = words + .iter() + .position(|&i| matches!(self.tokens[i], "(" | "=")); + let prototype = paren.is_some_and(|p| self.tokens[words[p]] == "(") + && !words.iter().any(|&i| STORAGE.contains(&self.tokens[i])); + if body || prototype { + return self.function(&words); + } + let mut k = 0; + let mut storage = None; + let mut location = None; + let mut interpolation = String::new(); + while let Some(&i) = words.get(k) { + let w = self.tokens[i]; + if w == "layout" { + // layout(location = n, ...) + let mut j = k + 1; + while let Some(&t) = words.get(j) { + j += 1; + if self.tokens[t] == "location" { + location = words + .get(j + 1) + .and_then(|&n| self.tokens[n].parse::().ok()); + } + if self.tokens[t] == ")" { + break; + } + } + k = j; + continue; + } + if STORAGE.contains(&w) { + storage = Some(w); + } else if QUALIFIERS.contains(&w) { + if matches!( + w, + "flat" | "smooth" | "noperspective" | "centroid" | "sample" + ) { + write!(interpolation, "{w} ").unwrap(); + } + } else { + break; + } + k += 1; + } + let Some(storage) = storage else { + return self.initializer(&words, to); + }; + let Some(&ty_at) = words.get(k) else { return }; + let ty = self.tokens[ty_at].to_owned(); + if words[k..].iter().any(|&i| self.tokens[i] == "{") { + return; // An interface block: left as the pack wrote it. + } + // Declarators: name [array] [= init], separated by commas. + let mut declarators: Vec<(String, String)> = Vec::new(); + let mut j = k + 1; + while let Some(&n) = words.get(j) { + let name = self.tokens[n].to_owned(); + let mut array = String::new(); + j += 1; + let mut nesting = 0; + while let Some(&t) = words.get(j) { + match self.tokens[t] { + "," if nesting == 0 => break, + "(" | "[" => nesting += 1, + ")" | "]" => nesting -= 1, + _ => {} + } + if self.tokens[t] == "=" && nesting == 0 { + // An initializer: uniforms are set by the game, so it is dropped. + while words.get(j).is_some_and(|&t| self.tokens[t] != ",") { + j += 1; + } + break; + } + array.push_str(self.tokens[t]); + j += 1; + } + j += 1; + declarators.push((name, array)); + } + let fragment = self.kind == ShaderStage::Fragment; + let input = matches!(storage, "in" | "attribute") || (storage == "varying" && fragment); + self.removed.push(from..to + 1); + for (name, array) in declarators { + let decl = format!("{interpolation}{ty} @{array};"); + match storage { + "uniform" if ty.contains("sampler") => self.samplers.push((name, ty.clone())), + "uniform" if ty.contains("image") => self + .errors + .push(self.error(ty_at, &format!("`{name}`: images are not supported yet"))), + "uniform" => self.uniforms.push((name, format!("{ty}{array}"), ty_at)), + _ if input && !fragment => { + let value = attribute_value(&name, &ty); + self.attributes.push((name, value)); + self.removed.pop(); + // A private global of ours now: `in`/`attribute` and qualifiers go. + for &w in &words[..k] { + self.removed.push(w..w + 1); + } + } + _ if input => self.ins.push(Var { name, decl }), + _ if fragment => { + let n = location.unwrap_or(self.outputs.len() as u32); + self.outputs.insert(n); + self.frag_outs + .push((n, format!("layout(location = {n}) out {ty} {name}{array};"))); + } + _ => self.outs.push(Var { name, decl }), + } + } + } + + /// A global `T name = a ? b : c;`: naga can't evaluate a conditional at global scope, so + /// the initializer becomes a function of ours on the same line (`T name; T hz_init_name() { + /// return a ? b : c; }`), called by our `main` before the pack's. + pub(super) fn initializer(&mut self, words: &[usize], to: usize) { + let text = |i: usize| self.tokens[i]; + let Some(eq) = words.iter().position(|&i| text(i) == "=") else { + return; + }; + if !words[eq..].iter().any(|&i| text(i) == "?") + || words[..eq].iter().any(|&i| text(i) == ",") + { + return; + } + let Some(name_at) = words[..eq].iter().rposition(|&i| is_ident(text(i))) else { + return; + }; + let name = text(words[name_at]).to_owned(); + let array: String = words[name_at + 1..eq].iter().map(|&i| text(i)).collect(); + let ty: Vec<&str> = words[..name_at] + .iter() + .map(|&i| text(i)) + .filter(|&t| t != "const") + .collect(); + let Some(ty) = ty.last() else { return }; + for &i in &words[..name_at] { + if text(i) == "const" { + self.removed.push(i..i + 1); + } + } + self.replaced.insert( + words[eq], + format!("; {ty}{array} hz_init_{name}() {{ return"), + ); + self.replaced.insert(to, "; }".into()); + self.inits.push(name); + } + + /// A function definition or prototype: remember which parameters are samplers. + pub(super) fn function(&mut self, words: &[usize]) { + let Some(open) = words.iter().position(|&i| self.tokens[i] == "(") else { + return; + }; + let Some(&name) = open.checked_sub(1).and_then(|n| words.get(n)) else { + return; + }; + let mut params = Vec::new(); + let mut is_sampler = false; + for &i in &words[open + 1..] { + match self.tokens[i] { + "," | ")" => { + params.push(is_sampler); + is_sampler = false; + } + t if t.contains("sampler") => is_sampler = true, + _ => {} + } + } + if params.iter().any(|&p| p) { + self.sampler_functions + .insert(self.tokens[name].to_owned(), params); + } + } +} diff --git a/crates/horizon-shaders/tests/real_packs.rs b/crates/horizon-shaders/tests/real_packs.rs index c252df6e3..4fac4b0d7 100644 --- a/crates/horizon-shaders/tests/real_packs.rs +++ b/crates/horizon-shaders/tests/real_packs.rs @@ -5,7 +5,11 @@ use std::path::Path; use std::time::Instant; -use horizon_shaders::{Lang, Macros, Pack, Properties, discover, preprocess}; +use std::collections::BTreeMap; + +use horizon_shaders::{ + Lang, Macros, Pack, Properties, Target, discover, preprocess, translate_composite, +}; #[test] #[ignore = "needs HORIZON_TEST_SHADER_PACKS"] @@ -86,6 +90,73 @@ fn real_packs_read() { for d in diagnostics.iter().take(8) { println!(" {d}"); } + // Every composite-style program with both stages, translated and validated. + let start = Instant::now(); + let (mut ok, mut failed, mut causes) = (0, 0, BTreeMap::::new()); + for dir in ["", "world0", "world-1", "world1"] { + for file in pack.list(dir) { + let Some(name) = file.strip_suffix(".fsh") else { + continue; + }; + let composite = [ + "composite", + "deferred", + "final", + "prepare", + "begin", + "shadowcomp", + ] + .iter() + .any(|p| { + name.strip_prefix(p) + .is_some_and(|n| n.chars().all(|c| c.is_ascii_digit())) + }); + let path = |ext: &str| { + if dir.is_empty() { + format!("{name}.{ext}") + } else { + format!("{dir}/{name}.{ext}") + } + }; + if !composite || !pack.exists(&path("vsh")) { + continue; + } + let target = if name == "final" { + Target::Window + } else { + Target::Buffers + }; + let v = preprocess(&pack, &path("vsh"), ¯os); + let fr = preprocess(&pack, &path("fsh"), ¯os); + match translate_composite(&v, &fr, target) { + Ok(_) => ok += 1, + Err(errors) => { + failed += 1; + let first = &errors[0]; + let cause: String = first.message.chars().take(90).collect(); + *causes.entry(cause).or_default() += 1; + if std::env::var_os("SHOW_ERRORS").is_some() { + println!( + " {}: {}:{}: {}", + path("fsh"), + first.file, + first.line, + first.message + ); + } + } + } + } + } + println!( + " composite-style programs: {ok} translate, {failed} fail, {:.0} ms", + start.elapsed().as_secs_f64() * 1000.0 + ); + let mut causes: Vec<_> = causes.into_iter().collect(); + causes.sort_by_key(|c| std::cmp::Reverse(c.1)); + for (cause, n) in causes.iter().take(6) { + println!(" {n} x {cause}"); + } assert!(!shaders.entries.is_empty(), "{}: no keys read", f.name); read += 1; }