Merge pull request #1601 from ludovic111/feat/shader-packs-translate

Shader packs, clean room 6: composite-style programs to naga
This commit is contained in:
Ludovic Marie
2026-10-25 14:19:52 +01:00
committed by GitHub
12 changed files with 2537 additions and 7 deletions
Generated
+12
View File
@@ -1438,6 +1438,8 @@ dependencies = [
name = "horizon-shaders"
version = "0.1.0"
dependencies = [
"horizon-core",
"naga",
"zip",
]
@@ -1973,6 +1975,7 @@ dependencies = [
"naga-types",
"num-traits",
"once_cell",
"pp-rs",
"rustc-hash 1.1.0",
"spirv",
"thiserror 2.0.21",
@@ -2649,6 +2652,15 @@ dependencies = [
"zerovec",
]
[[package]]
name = "pp-rs"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb458bb7f6e250e6eb79d5026badc10a3ebb8f9a15d1fff0f13d17c71f4d6dee"
dependencies = [
"unicode-xid",
]
[[package]]
name = "presser"
version = "0.3.1"
+2
View File
@@ -7,6 +7,8 @@ rust-version = "1.85"
description = "Shader packs (the shaders/ folder format): reading packs, their .properties and .lang files, and later their GLSL"
[dependencies]
horizon-core = { path = "../horizon-core" }
naga = { version = "30.0.1", features = ["glsl-in", "wgsl-in"] }
zip = { version = "2", default-features = false, features = ["deflate"] }
[lints]
+369
View File
@@ -0,0 +1,369 @@
//! The integer arithmetic of `#if` and `#elif` (spec 1.5, GLSL 4.60 §3.3): `defined`, integer
//! literals, macros expanded to their values, and the C operators with C precedence.
//! An identifier that is not a macro counts as 0, as in C: GLSL would reject the line, and a
//! pack would then lose a whole section of its settings over one unknown name.
use crate::Macros;
use crate::macros::{is_ident, is_ident_start};
/// The value of a condition; `Err` says what is wrong with it.
pub fn eval_condition(expr: &str, macros: &Macros) -> Result<i64, String> {
let tokens = tokens(&resolve_defined(expr, macros)?, macros)?;
let mut parser = Parser {
tokens: &tokens,
at: 0,
};
let value = parser.binary(0)?;
match parser.tokens.get(parser.at) {
None => Ok(value),
Some(t) => Err(format!("unexpected `{}`", t.text())),
}
}
/// The text after a line's `#`, split into the directive word and the rest without its `//`
/// comment. `None` when the first word is followed by something other than white space, `//` or
/// the end of the line (`#ifdefault=1` is a comment, not `#ifdef`).
pub(crate) fn split_directive(after_hash: &str) -> Option<(&str, &str)> {
let line = after_hash.trim_start();
let len = line
.find(|c: char| !c.is_ascii_alphabetic())
.unwrap_or(line.len());
let (word, rest) = line.split_at(len);
let ok = rest.is_empty() || rest.starts_with(char::is_whitespace) || rest.starts_with("//");
ok.then(|| (word, rest.split("//").next().unwrap_or("").trim()))
}
/// One level of `#if` ... `#endif`.
struct Branch {
/// This branch's lines are read.
on: bool,
/// A branch of this `#if` was already taken (the rest are skipped).
taken: bool,
/// The enclosing level is read.
outer: bool,
seen_else: bool,
}
/// The state of nested conditional directives.
#[derive(Default)]
pub(crate) struct Conditionals(Vec<Branch>);
impl Conditionals {
/// Lines here are read.
pub(crate) fn active(&self) -> bool {
self.0.last().is_none_or(|b| b.on)
}
/// An `#if` is still open.
pub(crate) fn open(&self) -> bool {
!self.0.is_empty()
}
/// Applies `#word rest` when it is a conditional directive (`Ok(false)` when it is not). A
/// condition that can't be evaluated reads as false and returns why.
pub(crate) fn apply(
&mut self,
word: &str,
rest: &str,
macros: &Macros,
) -> Result<bool, String> {
let on = self.active();
match word {
"if" | "ifdef" | "ifndef" => {
let value = if on {
test(word, rest, macros)
} else {
Ok(false)
};
let v = *value.as_ref().unwrap_or(&false);
self.0.push(Branch {
on: v,
taken: v,
outer: on,
seen_else: false,
});
value.map(|_| true)
}
"elif" | "else" => {
let b = self
.0
.last_mut()
.ok_or_else(|| format!("`#{word}` without `#if`"))?;
if b.seen_else {
return Err(format!("`#{word}` after `#else`"));
}
b.seen_else = word == "else";
let take = b.outer && !b.taken;
let value = match word {
"else" => Ok(take),
_ if take => test("if", rest, macros),
_ => Ok(false),
};
b.on = *value.as_ref().unwrap_or(&false);
b.taken |= b.on;
value.map(|_| true)
}
"endif" => self
.0
.pop()
.map(|_| true)
.ok_or_else(|| "`#endif` without `#if`".into()),
_ => Ok(false),
}
}
}
fn test(word: &str, rest: &str, macros: &Macros) -> Result<bool, String> {
let name = rest.split_whitespace().next().unwrap_or("");
match word {
"ifdef" => Ok(macros.is_defined(name)),
"ifndef" => Ok(!macros.is_defined(name)),
_ => eval_condition(rest, macros)
.map(|v| v != 0)
.map_err(|e| format!("`#{word} {rest}`: {e}; read as false")),
}
}
/// `defined X` and `defined(X)` become `1` or `0` before anything is expanded.
fn resolve_defined(expr: &str, macros: &Macros) -> Result<String, String> {
let mut out = String::with_capacity(expr.len());
let mut rest = expr;
while let Some(at) = find_word(rest, "defined") {
out.push_str(&rest[..at]);
let after = rest[at + "defined".len()..].trim_start();
let (inner, tail) = match after.strip_prefix('(') {
Some(open) => {
let close = open.find(')').ok_or("`defined(` without `)`")?;
(open[..close].trim(), &open[close + 1..])
}
None => {
let len = after.find(|c: char| !is_ident(c)).unwrap_or(after.len());
(&after[..len], &after[len..])
}
};
if inner.is_empty() || !inner.starts_with(is_ident_start) || !inner.chars().all(is_ident) {
return Err("`defined` needs a macro name".into());
}
out.push_str(if macros.is_defined(inner) {
" 1 "
} else {
" 0 "
});
rest = tail;
}
out.push_str(rest);
Ok(out)
}
/// Where `word` starts in `text` as a whole identifier.
fn find_word(text: &str, word: &str) -> Option<usize> {
let mut from = 0;
while let Some(i) = text[from..].find(word).map(|i| i + from) {
let before = text[..i].chars().next_back().is_some_and(is_ident);
let after = text[i + word.len()..].chars().next().is_some_and(is_ident);
if !before && !after {
return Some(i);
}
from = i + word.len();
}
None
}
#[derive(Clone, Copy, Debug, PartialEq)]
enum Token {
Num(i64),
Op(&'static str),
}
impl Token {
fn text(&self) -> String {
match self {
Token::Num(n) => n.to_string(),
Token::Op(op) => (*op).to_owned(),
}
}
}
const OPS: [&str; 22] = [
"<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "(", ")", "!", "~", "+", "-", "*", "/", "%",
"<", ">", "&", "^", "|",
];
fn tokens(expr: &str, macros: &Macros) -> Result<Vec<Token>, String> {
let expanded = macros.expand(expr);
let mut out = Vec::new();
let mut rest = expanded.trim_start();
while let Some(c) = rest.chars().next() {
if c.is_ascii_digit() {
let len = rest
.find(|c: char| !c.is_ascii_alphanumeric())
.unwrap_or(rest.len());
if rest[len..].starts_with('.') {
return Err(format!("`{}`: `#if` takes integers only", &rest[..len + 1]));
}
out.push(Token::Num(int_literal(&rest[..len])?));
rest = &rest[len..];
} else if is_ident_start(c) {
// Not a macro: 0, as the C preprocessor counts it.
let len = rest.find(|c: char| !is_ident(c)).unwrap_or(rest.len());
out.push(Token::Num(0));
rest = &rest[len..];
} else {
let op = OPS
.iter()
.find(|op| rest.starts_with(**op))
.ok_or_else(|| format!("unexpected `{c}`"))?;
out.push(Token::Op(op));
rest = &rest[op.len()..];
}
rest = rest.trim_start();
}
Ok(out)
}
/// Decimal, `0x` hexadecimal or leading-zero octal, with an optional `u` suffix.
fn int_literal(text: &str) -> Result<i64, String> {
let digits = text.strip_suffix(['u', 'U']).unwrap_or(text);
let parsed = if let Some(hex) = digits.strip_prefix("0x").or(digits.strip_prefix("0X")) {
i64::from_str_radix(hex, 16)
} else if digits.len() > 1 && digits.starts_with('0') {
i64::from_str_radix(&digits[1..], 8)
} else {
digits.parse()
};
parsed.map_err(|_| format!("`{text}` is not an integer"))
}
struct Parser<'a> {
tokens: &'a [Token],
at: usize,
}
/// Binary operators from the loosest to the tightest (GLSL §5.1 / §3.3).
const LEVELS: [&[&str]; 10] = [
&["||"],
&["&&"],
&["|"],
&["^"],
&["&"],
&["==", "!="],
&["<", ">", "<=", ">="],
&["<<", ">>"],
&["+", "-"],
&["*", "/", "%"],
];
impl Parser<'_> {
fn next(&mut self) -> Option<Token> {
let t = self.tokens.get(self.at).copied();
self.at += 1;
t
}
fn binary(&mut self, level: usize) -> Result<i64, String> {
if level == LEVELS.len() {
return self.unary();
}
let mut left = self.binary(level + 1)?;
while let Some(&Token::Op(op)) = self.tokens.get(self.at) {
if !LEVELS[level].contains(&op) {
break;
}
self.at += 1;
let right = self.binary(level + 1)?;
left = apply(op, left, right)?;
}
Ok(left)
}
fn unary(&mut self) -> Result<i64, String> {
match self.next() {
Some(Token::Num(n)) => Ok(n),
Some(Token::Op("(")) => {
let value = self.binary(0)?;
match self.next() {
Some(Token::Op(")")) => Ok(value),
_ => Err("missing `)`".into()),
}
}
Some(Token::Op("-")) => Ok(self.unary()?.wrapping_neg()),
Some(Token::Op("+")) => self.unary(),
Some(Token::Op("!")) => Ok((self.unary()? == 0) as i64),
Some(Token::Op("~")) => Ok(!self.unary()?),
Some(t) => Err(format!("unexpected `{}`", t.text())),
None => Err("the condition ends too early".into()),
}
}
}
fn apply(op: &str, a: i64, b: i64) -> Result<i64, String> {
Ok(match op {
"||" => (a != 0 || b != 0) as i64,
"&&" => (a != 0 && b != 0) as i64,
"|" => a | b,
"^" => a ^ b,
"&" => a & b,
"==" => (a == b) as i64,
"!=" => (a != b) as i64,
"<" => (a < b) as i64,
">" => (a > b) as i64,
"<=" => (a <= b) as i64,
">=" => (a >= b) as i64,
"<<" => a.wrapping_shl(b as u32),
">>" => a.wrapping_shr(b as u32),
"+" => a.wrapping_add(b),
"-" => a.wrapping_sub(b),
"*" => a.wrapping_mul(b),
"/" | "%" if b == 0 => return Err("division by zero".into()),
"/" => a.wrapping_div(b),
_ => a.wrapping_rem(b),
})
}
#[cfg(test)]
mod tests {
use super::*;
fn eval(expr: &str) -> Result<i64, String> {
let mut m = Macros::standard();
m.define("SHADOW_RES", "2048");
m.define("WAVES", "");
m.define("QUALITY", "SHADOW_RES / 1024");
eval_condition(expr, &m)
}
#[test]
fn precedence_and_operators_follow_c() {
assert_eq!(eval("1 + 2 * 3"), Ok(7));
assert_eq!(eval("(1 + 2) * 3"), Ok(9));
assert_eq!(eval("1 << 2 + 1"), Ok(8));
assert_eq!(eval("6 & 3 == 3"), Ok(0));
assert_eq!(eval("1 || 0 && 0"), Ok(1));
assert_eq!(eval("-7 / 2"), Ok(-3));
assert_eq!(eval("-7 % 2"), Ok(-1));
assert_eq!(eval("!0 + ~0"), Ok(0));
assert_eq!(eval("0x10 + 010 + 3u"), Ok(27));
}
#[test]
fn macros_and_defined() {
assert_eq!(eval("MC_VERSION >= 11300"), Ok(1));
assert_eq!(eval("MC_VERSION <= 12111"), Ok(0));
assert_eq!(eval("QUALITY == 2"), Ok(1));
assert_eq!(eval("defined WAVES && defined(SHADOW_RES)"), Ok(1));
assert_eq!(eval("!defined NOPE"), Ok(1));
assert_eq!(eval("NOPE == 0"), Ok(1));
assert_eq!(eval("undefinedFeature"), Ok(0));
}
#[test]
fn bad_conditions_say_why() {
assert!(eval("1 / 0").is_err());
assert!(eval("SHADOW_RES == 0.5").is_err());
assert!(eval("(1 + 2").is_err());
assert!(eval("1 2").is_err());
assert!(eval("").is_err());
assert!(eval("defined").is_err());
assert!(eval("1 @ 2").is_err());
}
}
+14
View File
@@ -4,12 +4,26 @@
//!
//! - [`Pack`]: a pack as a folder or a zip, with paths relative to its `shaders` folder (spec 1.1).
//! - [`discover`]: the packs in a `shaderpacks` folder.
//! - [`Properties`]: `shaders.properties` and the id files, with their conditional directives (3.1, 1.5).
//! - [`Lang`]: the pack's translated names (8.4).
//! - [`preprocess`]: a stage file through `#include` and the GLSL preprocessor, lines mapped back (1.4, 1.5).
//! - [`translate_composite`]: a composite-style program into naga modules and its interface (1.6, 2.2).
//! - [`Macros`] and [`eval_condition`]: the macros packs test, and the `#if` arithmetic (1.5).
mod condition;
mod discover;
mod lang;
mod macros;
mod pack;
mod preprocess;
mod properties;
mod translate;
pub use condition::eval_condition;
pub use discover::{Found, discover};
pub use lang::Lang;
pub use macros::{MC_VERSION, Macros, RENDER_STAGES};
pub use pack::{Pack, PackError};
pub use preprocess::{Diagnostic, Origin, Preprocessed, preprocess};
pub use properties::{Entry, Note, Properties};
pub use translate::{Error, Interface, Program, Sampler, Target, Uniform, translate_composite};
+317
View File
@@ -0,0 +1,317 @@
//! Object-like macros: the ones packs test in `#if` lines and use inside property values
//! (spec 1.5, 1.5.1). The GLSL preprocessor's function-like macros are not needed here.
use std::collections::BTreeMap;
/// `MC_VERSION` of the version Horizon plays: 26.3 → 260300.
pub const MC_VERSION: u32 = mc_version(horizon_core::version::VERSION_ID);
/// The documented rule, `major * 10000 + minor * 100 + patch` (spec 1.5.1, OQ-1): `1.21` → 12100,
/// `1.7.10` → 10710. Parts that are not digits count as 0.
pub const fn mc_version(id: &str) -> u32 {
let bytes = id.as_bytes();
let (mut parts, mut part, mut i) = ([0u32; 3], 0, 0);
while i < bytes.len() && part < 3 {
match bytes[i] {
b'.' => part += 1,
b @ b'0'..=b'9' => parts[part] = parts[part] * 10 + (b - b'0') as u32,
_ => break,
}
i += 1;
}
parts[0] * 10000 + parts[1] * 100 + parts[2]
}
#[derive(Clone, Debug, PartialEq)]
struct Macro {
/// `None` for an object-like macro; the parameter names of a function-like one.
params: Option<Vec<String>>,
body: String,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Macros(BTreeMap<String, Macro>);
/// `MC_RENDER_STAGE_*` in the order the stages draw (spec 5.4); each is its index (OQ-7).
pub const RENDER_STAGES: [&str; 24] = [
"NONE",
"SKY",
"SUNSET",
"CUSTOM_SKY",
"SUN",
"MOON",
"STARS",
"VOID",
"TERRAIN_SOLID",
"TERRAIN_CUTOUT_MIPPED",
"TERRAIN_CUTOUT",
"ENTITIES",
"BLOCK_ENTITIES",
"DESTROY",
"OUTLINE",
"DEBUG",
"HAND_SOLID",
"TERRAIN_TRANSLUCENT",
"TRIPWIRE",
"PARTICLES",
"CLOUDS",
"RAIN_SNOW",
"WORLD_BORDER",
"HAND_TRANSLUCENT",
];
impl Macros {
/// The standard macros that need no renderer (spec 1.5.1): `MC_VERSION`, this system's
/// `MC_OS_*`, the GL and GLSL versions Horizon's translation accepts, the render stages and
/// the host settings at their defaults. The GPU vendor and family arrive with the renderer.
pub fn standard() -> Self {
let mut m = Self::default();
m.define("MC_VERSION", MC_VERSION.to_string());
let os = match std::env::consts::OS {
"windows" => "MC_OS_WINDOWS",
"macos" => "MC_OS_MAC",
"linux" => "MC_OS_LINUX",
_ => "MC_OS_OTHER",
};
m.define(os, "");
m.define("MC_GL_VERSION", "460");
m.define("MC_GLSL_VERSION", "460");
for (i, stage) in RENDER_STAGES.iter().enumerate() {
m.define(format!("MC_RENDER_STAGE_{stage}"), i.to_string());
}
m.define("MC_RENDER_QUALITY", "1.0");
m.define("MC_SHADOW_QUALITY", "1.0");
m.define("MC_HAND_DEPTH", "0.125");
m.define("MAX_COLOR_BUFFERS", "16");
m
}
pub fn define(&mut self, name: impl Into<String>, value: impl Into<String>) {
let body = value.into();
self.0.insert(name.into(), Macro { params: None, body });
}
pub fn define_function(&mut self, name: &str, params: Vec<String>, body: &str) {
let body = body.to_owned();
self.0.insert(
name.to_owned(),
Macro {
params: Some(params),
body,
},
);
}
pub fn undef(&mut self, name: &str) {
self.0.remove(name);
}
pub fn is_defined(&self, name: &str) -> bool {
self.0.contains_key(name)
}
/// The value of an object-like macro.
pub fn get(&self, name: &str) -> Option<&str> {
self.0
.get(name)
.filter(|m| m.params.is_none())
.map(|m| m.body.as_str())
}
/// `text` with every macro expanded, repeatedly; a macro is not expanded again inside its own
/// expansion, so `A` defined as `A + 1` stops. Function-like macros take their arguments
/// (expanded first, raw beside `##`, which pastes) when a `(` follows their name.
pub fn expand(&self, text: &str) -> String {
self.expand_hiding(text, &mut Vec::new())
}
fn expand_hiding<'a>(&'a self, text: &str, hidden: &mut Vec<&'a str>) -> String {
let mut out = String::with_capacity(text.len());
let tokens = lex(text);
let mut i = 0;
while i < tokens.len() {
let word = tokens[i];
i += 1;
let Some((name, m)) = self.0.get_key_value(word) else {
out.push_str(word);
continue;
};
if hidden.contains(&name.as_str()) {
out.push_str(word);
continue;
}
let body = match &m.params {
None => m.body.clone(),
Some(params) => {
let Some((args, next)) = arguments(&tokens, i) else {
out.push_str(word);
continue;
};
i = next;
self.substitute(params, &args, &m.body, hidden)
}
};
hidden.push(name);
out.push_str(&self.expand_hiding(&body, hidden));
hidden.pop();
}
out
}
/// A function-like macro's body with its parameters replaced.
fn substitute<'a>(
&'a self,
params: &[String],
args: &[String],
body: &str,
hidden: &mut Vec<&'a str>,
) -> String {
let tokens = lex(body);
let pasted = |at: usize, step: isize| {
let mut j = at as isize + step;
while (0..tokens.len() as isize).contains(&j) && tokens[j as usize].trim().is_empty() {
j += step;
}
(0..tokens.len() as isize).contains(&j) && tokens[j as usize] == "##"
};
let mut out = String::new();
for (at, token) in tokens.iter().enumerate() {
match params.iter().position(|p| p == token) {
Some(k) => {
let arg = args.get(k).map_or("", |a| a.trim());
if pasted(at, -1) || pasted(at, 1) {
out.push_str(arg);
} else {
out.push_str(&self.expand_hiding(arg, hidden));
}
}
None => out.push_str(token),
}
}
// `a ## b` → `ab`.
let mut joined = String::with_capacity(out.len());
for (k, part) in out.split("##").enumerate() {
joined.push_str(if k == 0 { part.trim_end() } else { part.trim() });
}
if out.contains("##") { joined } else { out }
}
}
/// The arguments of a call whose `(` is the next non-blank token from `i`, split at top-level
/// commas, and the index after its `)`. `None` when no `(` follows or it never closes.
fn arguments(tokens: &[&str], mut i: usize) -> Option<(Vec<String>, usize)> {
while tokens.get(i)?.trim().is_empty() {
i += 1;
}
if tokens[i] != "(" {
return None;
}
let (mut depth, mut args, mut from) = (0, Vec::new(), i + 1);
for (j, t) in tokens.iter().enumerate().skip(i + 1) {
match *t {
"(" => depth += 1,
")" if depth == 0 => {
args.push(tokens[from..j].concat());
return Some((args, j + 1));
}
")" => depth -= 1,
"," if depth == 0 => {
args.push(tokens[from..j].concat());
from = j + 1;
}
_ => {}
}
}
None
}
/// Preprocessing tokens: identifiers, numbers (with their suffixes and exponents), runs of
/// white space, `##`, and single characters. Concatenated they give back `text`.
pub(crate) fn lex(text: &str) -> Vec<&str> {
let bytes = text.as_bytes();
let mut out = Vec::new();
let mut i = 0;
while i < bytes.len() {
let c = bytes[i] as char;
let start = i;
i += 1;
if is_ident_start(c) {
while i < bytes.len() && is_ident(bytes[i] as char) {
i += 1;
}
} else if c.is_ascii_digit() || (c == '.' && bytes.get(i).is_some_and(u8::is_ascii_digit)) {
while i < bytes.len() {
let d = bytes[i] as char;
let exp_sign = (d == '+' || d == '-') && matches!(bytes[i - 1], b'e' | b'E');
if !(is_ident(d) || d == '.' || exp_sign) {
break;
}
i += 1;
}
} else if c.is_whitespace() {
while i < bytes.len() && (bytes[i] as char).is_whitespace() {
i += 1;
}
} else if c == '#' && bytes.get(i) == Some(&b'#') {
i += 1;
} else {
while !text.is_char_boundary(i) {
i += 1;
}
}
out.push(&text[start..i]);
}
out
}
pub(crate) fn is_ident_start(c: char) -> bool {
c.is_ascii_alphabetic() || c == '_'
}
pub(crate) fn is_ident(c: char) -> bool {
c.is_ascii_alphanumeric() || c == '_'
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn versions_follow_the_documented_rule() {
assert_eq!(mc_version("1.21"), 12100);
assert_eq!(mc_version("1.21.0"), 12100);
assert_eq!(mc_version("1.7.10"), 10710);
assert_eq!(mc_version("26.3"), 260300);
assert_eq!(MC_VERSION, 260300);
assert_eq!(Macros::standard().get("MC_VERSION"), Some("260300"));
}
#[test]
fn expansion_replaces_whole_words_and_stops_on_recursion() {
let mut m = Macros::default();
m.define("RES", "SIZE");
m.define("SIZE", "512");
m.define("LOOP", "LOOP + 1");
assert_eq!(m.expand("RES RES 0.5"), "512 512 0.5");
assert_eq!(
m.expand("RESOLUTION colortex4 LOOP"),
"RESOLUTION colortex4 LOOP + 1"
);
m.define("e5", "x");
assert_eq!(m.expand("1e5"), "1e5");
}
#[test]
fn function_like_macros_take_arguments_and_paste() {
let mut m = Macros::default();
m.define("N", "4");
m.define_function("SQ", vec!["x".into()], "((x) * (x))");
m.define_function("CAT", vec!["a".into(), "b".into()], "a ## b");
m.define_function("PI", vec![], "3.14");
assert_eq!(m.expand("SQ(N + 1)"), "((4 + 1) * (4 + 1))");
assert_eq!(m.expand("SQ(min(N, 2))"), "((min(4, 2)) * (min(4, 2)))");
assert_eq!(m.expand("CAT(N, 2) CAT(col, ortex4)"), "N2 colortex4");
assert_eq!(m.expand("PI() SQ SQ(1"), "3.14 SQ SQ(1");
assert_eq!(m.get("SQ"), None);
}
}
+1 -1
View File
@@ -170,7 +170,7 @@ impl Pack {
}
/// `a/./b/../c` → `a/c`; `None` for a path that climbs out of `shaders`.
fn normalize(path: &str) -> Option<String> {
pub(crate) fn normalize(path: &str) -> Option<String> {
let mut parts: Vec<&str> = Vec::new();
for part in path.split(['/', '\\']) {
match part {
+374
View File
@@ -0,0 +1,374 @@
//! The GLSL preprocessor a program's stage file goes through before translation (spec 1.4,
//! 1.5, 1.6): comments removed, `#include` expanded, conditionals evaluated, macros expanded.
//! Every output line remembers the pack file and line it came from, so errors found later point
//! at the pack's own code.
//!
//! `#include` is expanded where it stands in live code only: an include inside a false `#if`
//! is not read, so it can't fail (OQ-14). Ten levels deep at most, which also stops a file that
//! includes itself. `#version` and `#extension` are taken out of the code and kept apart: the
//! translation writes its own header.
use std::collections::HashMap;
use crate::Macros;
use crate::Pack;
use crate::condition::{Conditionals, split_directive};
use crate::pack::normalize;
const MAX_DEPTH: usize = 10;
/// Where a line came from: an index into [`Preprocessed::files`] and a line number from 1.
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct Origin {
pub file: u16,
pub line: u32,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Diagnostic {
pub origin: Origin,
pub message: String,
}
#[derive(Debug, Default)]
pub struct Preprocessed {
/// The code, one line per entry of `lines`.
pub code: String,
pub lines: Vec<Origin>,
/// The pack paths of the files read, the stage file first.
pub files: Vec<String>,
/// `#version` of the stage file: the number and the profile word, if any.
pub version: Option<(u32, Option<String>)>,
/// `#extension` lines of live code, as written after `#extension`.
pub extensions: Vec<String>,
/// The text of every block comment in live code (`DRAWBUFFERS:`, buffer formats).
pub comments: Vec<(Origin, String)>,
pub diagnostics: Vec<Diagnostic>,
}
impl Preprocessed {
/// `file:line` of an output line (from 0), for messages.
pub fn locate(&self, line: usize) -> String {
let o = self.lines.get(line).copied().unwrap_or_default();
let file = self.files.get(o.file as usize).map_or("?", String::as_str);
format!("{file}:{}", o.line)
}
}
/// Runs the preprocessor on the stage file `path` of `pack` with `macros` defined.
pub fn preprocess(pack: &Pack, path: &str, macros: &Macros) -> Preprocessed {
let mut run = Run {
pack,
macros: macros.clone(),
conditionals: Conditionals::default(),
out: Preprocessed::default(),
code: Vec::new(),
texts: HashMap::new(),
};
run.file(path, 0, Origin::default());
if run.conditionals.open() {
run.diagnose(Origin::default(), "`#if` without `#endif`");
}
run.out.code = run.code.join("\n");
run.out
}
struct Run<'p> {
pack: &'p Pack,
macros: Macros,
conditionals: Conditionals,
out: Preprocessed,
code: Vec<String>,
/// Files already read: a guarded header is included many times.
texts: HashMap<String, Option<String>>,
}
impl Run<'_> {
fn file(&mut self, path: &str, depth: usize, from: Origin) {
let cached = self
.texts
.entry(path.to_owned())
.or_insert_with(|| self.pack.read_text(path));
let Some(text) = cached.clone() else {
return self.diagnose(from, &format!("can't find `{path}`"));
};
let file = match self.out.files.iter().position(|f| f == path) {
Some(i) => i,
None => {
self.out.files.push(path.to_owned());
self.out.files.len() - 1
}
} as u16;
let (text, comments) = strip_comments(&text);
let lines: Vec<&str> = text.split('\n').map(|l| l.trim_end_matches('\r')).collect();
let mut comments = comments.into_iter().peekable();
let mut n = 0;
while n < lines.len() {
let first = n;
let mut line = lines[n].to_owned();
while line.ends_with('\\') && n + 1 < lines.len() {
line.pop();
n += 1;
line.push_str(lines[n]);
}
n += 1;
let origin = Origin {
file,
line: first as u32 + 1,
};
self.line(&line, origin, path, depth);
for k in first + 1..n {
self.emit(
String::new(),
Origin {
file,
line: k as u32 + 1,
},
);
}
while let Some((_, comment)) = comments.next_if(|(at, _)| *at < n) {
if self.conditionals.active() {
self.out.comments.push((origin, comment));
}
}
}
}
fn line(&mut self, line: &str, origin: Origin, path: &str, depth: usize) {
let Some(after) = line.trim_start().strip_prefix('#') else {
let code = if self.conditionals.active() {
self.macros.define("__LINE__", origin.line.to_string());
self.macros.expand(line)
} else {
String::new()
};
return self.emit(code, origin);
};
self.emit(String::new(), origin);
let Some((word, rest)) = split_directive(after) else {
return self.diagnose(origin, &format!("unknown directive `#{}`", after.trim()));
};
match self.conditionals.apply(word, rest, &self.macros) {
Err(e) => return self.diagnose(origin, &e),
Ok(true) => return,
Ok(false) if !self.conditionals.active() => return,
Ok(false) => {}
}
match word {
"define" => self.define(rest, origin),
"undef" => self.macros.undef(rest),
"include" => {
let target = rest.trim_matches(|c| c == '"' || c == '<' || c == '>');
match resolve(path, target) {
_ if depth + 1 >= MAX_DEPTH => self.diagnose(
origin,
&format!("`{target}`: includes nest more than {MAX_DEPTH} deep"),
),
Some(file) => {
// The include line's own (empty) output line stays; the file follows.
self.file(&file, depth + 1, origin);
}
None => self.diagnose(origin, &format!("`{target}` is outside the pack")),
}
}
"version" if depth == 0 && self.out.version.is_none() => {
let mut words = rest.split_whitespace();
let number = words.next().and_then(|v| v.parse().ok()).unwrap_or(110);
self.out.version = Some((number, words.next().map(str::to_owned)));
self.macros.define("__VERSION__", number.to_string());
}
"extension" => self.out.extensions.push(rest.to_owned()),
"error" => self.diagnose(origin, &format!("#error {rest}")),
"version" | "pragma" | "line" | "" => {}
_ => self.diagnose(origin, &format!("unknown directive `#{word}`")),
}
}
/// `#define NAME body` or `#define NAME(a, b) body` (no space before the `(`).
fn define(&mut self, rest: &str, origin: Origin) {
let len = rest
.find(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
.unwrap_or(rest.len());
let (name, tail) = rest.split_at(len);
if name.is_empty() {
return self.diagnose(origin, "`#define` without a name");
}
if let Some(params) = tail.strip_prefix('(') {
let Some((params, body)) = params.split_once(')') else {
return self.diagnose(origin, &format!("`#define {name}(` without `)`"));
};
let params = params
.split(',')
.map(|p| p.trim().to_owned())
.filter(|p| !p.is_empty())
.collect();
self.macros.define_function(name, params, body.trim());
} else {
self.macros.define(name, tail.trim());
}
}
fn emit(&mut self, code: String, origin: Origin) {
self.code.push(code);
self.out.lines.push(origin);
}
fn diagnose(&mut self, origin: Origin, message: &str) {
self.out.diagnostics.push(Diagnostic {
origin,
message: message.to_owned(),
});
}
}
/// An `#include` path: from the `shaders` folder with a leading `/`, else from the including
/// file's folder (spec 1.4). `None` when it climbs out of the pack.
fn resolve(from_file: &str, target: &str) -> Option<String> {
match target.strip_prefix('/') {
Some(absolute) => normalize(absolute),
None => {
let dir = from_file.rsplit_once('/').map_or("", |(dir, _)| dir);
normalize(&format!("{dir}/{target}"))
}
}
}
/// `text` with comments replaced by spaces (new lines kept, so line numbers hold), and the body of
/// each block comment with the index of the line it starts on.
fn strip_comments(text: &str) -> (String, Vec<(usize, String)>) {
let mut out = String::with_capacity(text.len());
let mut comments = Vec::new();
let (mut rest, mut line) = (text, 0);
while let Some(at) = rest.find('/') {
let (before, from) = rest.split_at(at);
out.push_str(before);
line += before.matches('\n').count();
if let Some(body) = from.strip_prefix("//") {
let end = body.find('\n').unwrap_or(body.len());
rest = &body[end..];
} else if let Some(body) = from.strip_prefix("/*") {
let end = body.find("*/").unwrap_or(body.len());
let comment = &body[..end];
comments.push((line, comment.to_owned()));
let newlines = comment.matches('\n').count();
line += newlines;
out.push(' ');
out.extend(std::iter::repeat_n('\n', newlines));
rest = body.get(end + 2..).unwrap_or("");
} else {
out.push('/');
rest = &from[1..];
}
}
out.push_str(rest);
(out, comments)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::pack::tests::zip;
fn run(files: &[(&str, &str)]) -> Preprocessed {
let files: Vec<(String, &str)> = files
.iter()
.map(|(p, t)| (format!("shaders/{p}"), *t))
.collect();
let files: Vec<(&str, &str)> = files.iter().map(|(p, t)| (p.as_str(), *t)).collect();
let pack = Pack::from_zip_bytes("P.zip", zip(&files)).unwrap();
preprocess(&pack, "program/final.fsh", &Macros::standard())
}
fn code_lines(p: &Preprocessed) -> Vec<(&str, String)> {
p.code
.lines()
.enumerate()
.filter(|(_, l)| !l.trim().is_empty())
.map(|(i, l)| (l.trim(), p.locate(i)))
.collect()
}
#[test]
fn includes_resolve_and_lines_map_back() {
let p = run(&[
(
"program/final.fsh",
"#version 330 compatibility\n#include \"/lib/a.glsl\"\nvoid main() {}\n",
),
(
"lib/a.glsl",
"#ifndef A\n#define A\n#include \"b.glsl\"\nfloat a;\n#endif\n",
),
("lib/b.glsl", "// b\nfloat b;\n#include \"a.glsl\"\n"),
]);
assert_eq!(p.version, Some((330, Some("compatibility".into()))));
assert_eq!(
code_lines(&p),
[
("float b;", "lib/b.glsl:2".into()),
("float a;", "lib/a.glsl:4".into()),
("void main() {}", "program/final.fsh:3".into()),
]
);
assert!(p.diagnostics.is_empty(), "{:?}", p.diagnostics);
}
#[test]
fn missing_and_runaway_includes_are_diagnosed_where_they_stand() {
let p = run(&[(
"program/final.fsh",
"#ifdef NOPE\n#include \"gone.glsl\"\n#endif\n#include \"gone.glsl\"\n#include \"final.fsh\"\n",
)]);
let messages: Vec<String> = p
.diagnostics
.iter()
.map(|d| format!("{}: {}", d.origin.line, d.message))
.collect();
assert_eq!(messages[0], "4: can't find `program/gone.glsl`");
assert!(
messages
.last()
.unwrap()
.ends_with("includes nest more than 10 deep")
);
assert!(
run(&[("program/final.fsh", "#include \"../../x\"")]).diagnostics[0]
.message
.contains("outside")
);
}
#[test]
fn macros_expand_like_the_glsl_preprocessor() {
let p = run(&[(
"program/final.fsh",
"#define SQ(x) ((x) * (x))\n#define CAT(a, b) a ## b\n#define N 4\n#define \\\n LONG 1\n\
float v = SQ(N + 1) + CAT(N, 2) + LONG;\nint l = __LINE__;\n#if MC_VERSION > 12100 && defined MC_RENDER_STAGE_SKY\nint ok;\n#else\nint bad;\n#endif\n",
)]);
let lines: Vec<&str> = code_lines(&p).into_iter().map(|(l, _)| l).collect();
assert_eq!(
lines,
[
"float v = ((4 + 1) * (4 + 1)) + N2 + 1;",
"int l = 7;",
"int ok;"
]
);
assert_eq!(p.lines.len(), p.code.split('\n').count());
}
#[test]
fn comments_extensions_and_errors() {
let p = run(&[(
"program/final.fsh",
"#version 120\n#extension GL_ARB_shader_texture_lod : enable\n/* DRAWBUFFERS:04 */\n#if 0\n/* RENDERTARGETS: 1 */\n#error dead\n#endif\n\
/* multi\nline */ float x; // gone\n#error live\n#bogus\n",
)]);
assert_eq!(p.extensions, ["GL_ARB_shader_texture_lod : enable"]);
let comments: Vec<&str> = p.comments.iter().map(|(_, c)| c.trim()).collect();
assert_eq!(comments, ["DRAWBUFFERS:04", "multi\nline"]);
assert_eq!(code_lines(&p), [("float x;", "program/final.fsh:9".into())]);
let messages: Vec<&str> = p.diagnostics.iter().map(|d| d.message.as_str()).collect();
assert_eq!(messages, ["#error live", "unknown directive `#bogus`"]);
}
}
+229
View File
@@ -0,0 +1,229 @@
//! `.properties` files: `shaders.properties` (spec 3.1) and the id files `block.properties`,
//! `item.properties`, `entity.properties` (4.1). `key=value` lines, `#` comments, a trailing `\`
//! continuing a value on the next line, and conditional directives evaluated with the
//! standard and option macros (1.5 item 2). `#define` and `#undef` work in every one of them:
//! a public pack defines a macro in its `shaders.properties` and tests it further down.
//! No other escape is read: values are names, numbers and expressions. A later duplicate key
//! replaces the earlier one, with a note.
use crate::Macros;
use crate::condition::{Conditionals, split_directive};
/// One `key=value`, with the line it starts on (from 1).
#[derive(Clone, Debug, PartialEq)]
pub struct Entry {
pub key: String,
pub value: String,
pub line: u32,
}
/// Something wrong with a line, for the pack's log.
#[derive(Clone, Debug, PartialEq)]
pub struct Note {
pub line: u32,
pub message: String,
}
#[derive(Debug, Default, PartialEq)]
pub struct Properties {
/// In file order; a key appears once (a later duplicate replaces the earlier one).
pub entries: Vec<Entry>,
pub notes: Vec<Note>,
}
impl Properties {
/// Reads `text`; `#define`/`#undef` change the macros for the lines after them only.
pub fn parse(text: &str, macros: &Macros) -> Self {
let mut out = Self::default();
let mut macros = macros.clone();
let mut conditionals = Conditionals::default();
let mut lines = text.lines().zip(1u32..);
while let Some((raw, number)) = lines.next() {
let line = raw.trim();
let on = conditionals.active();
if let Some(directive) = line.strip_prefix('#') {
if let Some((word, rest)) = split_directive(directive) {
out.directive(word, rest, number, &mut conditionals, &mut macros);
}
continue;
}
// Continuation lines belong to their entry even inside a skipped branch.
let mut logical = line.to_owned();
while ends_with_continuation(&logical) {
logical.pop();
match lines.next() {
Some((next, _)) => logical.push_str(next.trim()),
None => break,
}
}
if !on || logical.is_empty() {
continue;
}
match logical.split_once('=') {
Some((key, value)) if !key.trim().is_empty() => out.insert(Entry {
key: key.trim().to_owned(),
value: value.trim().to_owned(),
line: number,
}),
_ => out.note(number, "not a key=value line"),
}
}
if conditionals.open() {
out.note(text.lines().count() as u32, "`#if` without `#endif`");
}
out
}
pub fn get(&self, key: &str) -> Option<&str> {
self.entry(key).map(|e| e.value.as_str())
}
pub fn entry(&self, key: &str) -> Option<&Entry> {
self.entries.iter().find(|e| e.key == key)
}
fn insert(&mut self, entry: Entry) {
if let Some(at) = self.entries.iter().position(|e| e.key == entry.key) {
let first = self.entries.remove(at).line;
self.note(
entry.line,
&format!(
"`{}` is set again (first on line {first}); this one counts",
entry.key
),
);
}
self.entries.push(entry);
}
fn note(&mut self, line: u32, message: &str) {
self.notes.push(Note {
line,
message: message.to_owned(),
});
}
/// A line starting with `#` whose first word may be a directive; any other word makes it a
/// comment.
fn directive(
&mut self,
word: &str,
rest: &str,
number: u32,
conditionals: &mut Conditionals,
macros: &mut Macros,
) {
match conditionals.apply(word, rest, macros) {
Err(e) => self.note(number, &e),
Ok(true) => {}
Ok(false) if conditionals.active() => match word {
"define" => {
let (name, value) = rest.split_once(char::is_whitespace).unwrap_or((rest, ""));
macros.define(name, value.trim());
}
"undef" => macros.undef(rest),
_ => {}
},
Ok(false) => {}
}
}
}
/// An odd number of trailing backslashes: the last one joins the next line (`\\` is a literal).
fn ends_with_continuation(line: &str) -> bool {
line.bytes().rev().take_while(|&b| b == b'\\').count() % 2 == 1
}
#[cfg(test)]
mod tests {
use super::*;
fn read(text: &str) -> Properties {
let mut m = Macros::standard();
m.define("SHADOWS", "");
m.define("QUALITY", "2");
Properties::parse(text, &m)
}
fn pairs(p: &Properties) -> Vec<(&str, &str)> {
p.entries
.iter()
.map(|e| (e.key.as_str(), e.value.as_str()))
.collect()
}
#[test]
fn keys_values_and_comments() {
let p =
read("# a comment\n\n sun = false\n#commented.key=1\nscreen=A B [C]\r\nbad line\n");
assert_eq!(pairs(&p), [("sun", "false"), ("screen", "A B [C]")]);
assert_eq!(p.entry("screen").unwrap().line, 5);
assert_eq!(
p.notes,
[Note {
line: 6,
message: "not a key=value line".into()
}]
);
}
#[test]
fn continuation_lines_join() {
let p = read("screen=A B \\\n C D\\\n E\nsliders=X\nliteral=a\\\\\nnext=1");
assert_eq!(
pairs(&p),
[
("screen", "A B C DE"),
("sliders", "X"),
("literal", "a\\\\"),
("next", "1")
]
);
assert_eq!(p.entry("sliders").unwrap().line, 4);
}
#[test]
fn conditionals_pick_one_branch() {
let p = read(
"#if MC_VERSION >= 11300\nnew=1\n#else\nold=1\n#endif\n\
#ifdef SHADOWS\n#if QUALITY == 1\nq=1\n#elif QUALITY == 2\nq=2\n#elif QUALITY >= 2\nq=3\n#endif\n#endif\n\
#ifndef SHADOWS\nnone=1\n#elif 0\nnever=1\n#else\nfallback=1\n#endif\n# ifdef SHADOWS // trailing comment\nspaced=1\n#endif",
);
assert_eq!(
pairs(&p),
[("new", "1"), ("q", "2"), ("fallback", "1"), ("spaced", "1")]
);
assert!(p.notes.is_empty(), "{:?}", p.notes);
}
#[test]
fn comment_lines_are_not_directives() {
let p = read("#ifdefault=1\n#iffy\n#endiff\n#else-where\nkey=1");
assert_eq!(pairs(&p), [("key", "1")]);
assert!(p.notes.is_empty(), "{:?}", p.notes);
}
#[test]
fn defines_apply_to_the_lines_after_them() {
let text = "#ifdef MC_OS_NONE\n#define WATER 0\n#endif\n#define WATER 8\n\
#if WATER == 8\nblock.8=minecraft:water\n#endif\n#undef WATER\n#ifdef WATER\nno=1\n#endif";
let p = Properties::parse(text, &Macros::default());
assert_eq!(pairs(&p), [("block.8", "minecraft:water")]);
assert!(p.notes.is_empty(), "{:?}", p.notes);
}
#[test]
fn duplicate_keys_keep_the_last() {
let p = read("a=1\nb=2\na=3");
assert_eq!(pairs(&p), [("b", "2"), ("a", "3")]);
assert_eq!(p.notes[0].line, 3);
}
#[test]
fn broken_conditionals_are_noted() {
let p = read("#endif\n#else\n#if 1 +\nx=1\n#endif\n#if 1\n#else\n#else\n");
let lines: Vec<u32> = p.notes.iter().map(|n| n.line).collect();
assert_eq!(lines, [1, 2, 3, 8, 8]);
assert!(p.entries.is_empty());
}
}
+436
View File
@@ -0,0 +1,436 @@
//! A program's preprocessed GLSL (any version from 110 to 460, compatibility or core) turned
//! into the Vulkan-flavoured GLSL 450 naga reads, then into validated naga modules (spec 1.6,
//! 2.2). The rewrite works on preprocessing tokens and never adds or removes a line of the pack's
//! code, so naga's line numbers map back through the preprocessor to the pack's files.
//!
//! What it does:
//! - Loose uniforms go into one uniform block shared by both stages (`bool` stored as `int`).
//! - Each sampler is split into a texture and a sampler object, combined again where it is used
//! (`sampler2D(t, s)`), or passed as two arguments to the pack's own functions.
//! - The compatibility built-ins (`gl_Vertex`, `gl_MultiTexCoordN`, `gl_*Matrix`, `ftransform()`,
//! `gl_FragData`, `gl_FragColor`, `gl_TexCoord`, `texture2D`...) become declarations of ours,
//! named with the reserved prefix `hz_`; varyings get locations by name across the stages.
//! - The pack's `main` becomes `hz_main`, called by a `main` of ours that sets the inputs and
//! adapts GL's conventions (spec 5.5): clip y is negated for every target but the window, so
//! memory rows are GL's rows and `gl_FragCoord` needs no change, and clip z goes from GL's
//! [-w, w] to [0, w], so stored depth is GL's window depth.
//!
//! Composite-style programs draw one quad from (0,0) to (1,1) with an identity model-view and an
//! orthographic projection onto it, the texture coordinate equal to the position (OQ-6).
use std::collections::BTreeMap;
use std::fmt::Write;
use naga::ShaderStage;
use naga::front::glsl::{Frontend, Options};
use naga::valid::{Capabilities, ValidationFlags, Validator};
use crate::Preprocessed;
mod rewrite;
mod scan;
use scan::Stage;
/// What a program's stages need from the game.
#[derive(Debug, Default, PartialEq)]
pub struct Interface {
/// The uniform block at binding 0: its size and each value's name, GLSL type and offset.
/// Ours come first: `hz_target` (the target's width, height, 1/width, 1/height), and the
/// fixed-function fog the compatibility `gl_Fog` reads (OQ-28): `hz_fog_color` (vanilla's fog
/// colour) and `hz_fog` (density, start, end, 1 / (end - start)).
pub uniform_size: u32,
pub uniforms: Vec<Uniform>,
/// Each sampler: texture at `binding`, sampler object at `binding + 1`.
pub samplers: Vec<Sampler>,
/// The fragment stage's output locations.
pub outputs: Vec<u32>,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Uniform {
pub name: String,
/// The type the pack declared (`float`, `ivec2`, `mat4`; `bool` is stored as an `int`).
pub ty: String,
pub offset: u32,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Sampler {
pub name: String,
/// The type the pack declared: `sampler2D`, `sampler2DShadow`, `usampler2D`, `sampler3D`...
pub ty: String,
pub binding: u32,
}
/// A program that failed, at a pack file and line.
#[derive(Clone, Debug, PartialEq)]
pub struct Error {
pub file: String,
pub line: u32,
pub message: String,
}
pub struct Program {
pub vertex: naga::Module,
pub fragment: naga::Module,
pub interface: Interface,
}
/// Where the program draws, which decides the conventions of 5.5.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Target {
/// Pack buffers: rows stored as GL stores them.
Buffers,
/// The window (`final`).
Window,
}
/// Translates a composite-style program (`composite`, `deferred`, `final`...).
pub fn translate_composite(
vertex: &Preprocessed,
fragment: &Preprocessed,
target: Target,
) -> Result<Program, Vec<Error>> {
let mut v = Stage::scan(vertex, ShaderStage::Vertex);
let mut f = Stage::scan(fragment, ShaderStage::Fragment);
let mut errors: Vec<Error> = v.errors.drain(..).chain(f.errors.drain(..)).collect();
// One uniform block and one set of samplers for both stages.
let mut uniforms: BTreeMap<String, String> = BTreeMap::new();
let mut samplers: BTreeMap<String, String> = BTreeMap::new();
for stage in [&v, &f] {
for (name, ty, at) in &stage.uniforms {
if let Some(old) = uniforms.insert(name.clone(), ty.clone())
&& &old != ty
{
errors.push(stage.error(
*at,
&format!("`{name}` is `{old}` in one stage and `{ty}` in the other"),
));
}
}
for (name, ty) in &stage.samplers {
samplers.insert(name.clone(), ty.clone());
}
}
let samplers: Vec<Sampler> = samplers
.into_iter()
.enumerate()
.map(|(i, (name, ty))| Sampler {
name,
ty,
binding: 1 + 2 * i as u32,
})
.collect();
// Varyings by name: the vertex outputs, then fragment inputs the vertex stage lacks.
let mut locations: BTreeMap<String, (u32, String)> = BTreeMap::new();
for var in v.outs.iter().chain(&f.ins) {
let next = locations.len() as u32;
locations
.entry(var.name.clone())
.or_insert((next, var.decl.clone()));
}
let header = |stage: &Stage| {
let mut h = String::from("#version 450\n");
h.push_str("layout(set = 0, binding = 0) uniform HzUniforms { vec4 hz_target; vec4 hz_fog_color; vec4 hz_fog;");
for (name, ty) in &uniforms {
match ty.as_str() {
"bool" => write!(h, " int hz_bool_{name};"),
_ => match ty.split_once('[') {
Some((base, size)) => write!(h, " {base} {name}[{size};"),
None => write!(h, " {ty} {name};"),
},
}
.unwrap();
}
h.push_str(" };\n");
for s in &samplers {
let (texture, sampler) = split_sampler(&s.ty);
writeln!(
h,
"layout(set = 0, binding = {}) uniform {texture} {}; layout(set = 0, binding = {}) uniform {sampler} hz_s_{};",
s.binding, s.name, s.binding + 1, s.name
)
.unwrap();
}
let direction = if stage.kind == ShaderStage::Vertex {
"out"
} else {
"in"
};
for (name, (location, decl)) in &locations {
writeln!(
h,
"layout(location = {location}) {direction} {}",
decl.replace("@", name)
)
.unwrap();
}
h.push_str(&stage.prelude());
h
};
let flip = if target == Target::Buffers { "-" } else { "" };
let fog =
" hz_Fog = hz_FogParameters(hz_fog_color, hz_fog.x, hz_fog.y, hz_fog.z, hz_fog.w);\n";
let mut vertex_main = String::from(
"\nvoid main() {\n vec2 hz_p = hz_quad[gl_VertexIndex];\n hz_Vertex = vec4(hz_p, 0.0, 1.0);\n \
hz_MultiTexCoord0 = hz_Vertex;\n hz_Color = vec4(1.0);\n hz_Normal = vec3(0.0, 0.0, 1.0);\n",
);
for (name, value) in &v.attributes {
writeln!(vertex_main, " {name} = {value};").unwrap();
}
vertex_main.push_str(fog);
for name in &v.inits {
writeln!(vertex_main, " {name} = hz_init_{name}();").unwrap();
}
write!(
vertex_main,
" hz_main();\n gl_Position = vec4(gl_Position.x, {flip}gl_Position.y, (gl_Position.z + gl_Position.w) * 0.5, gl_Position.w);\n}}\n"
)
.unwrap();
let mut fragment_main = String::from("\nvoid main() {\n");
fragment_main.push_str(fog);
if target == Target::Window && f.frag_coord {
fragment_main.push_str(" hz_FragCoord = vec4(gl_FragCoord.x, hz_target.y - gl_FragCoord.y, gl_FragCoord.zw);\n");
}
for name in &f.inits {
writeln!(fragment_main, " {name} = hz_init_{name}();").unwrap();
}
fragment_main.push_str(" hz_main();\n}\n");
let v_text = format!(
"{}{}{vertex_main}",
header(&v),
v.rewrite(&samplers, &uniforms)
);
let f_text = format!(
"{}{}{fragment_main}",
header(&f),
f.rewrite(&samplers, &uniforms)
);
if !errors.is_empty() {
return Err(errors);
}
let vertex_module = compile(&v, &v_text, header(&v).lines().count());
let fragment_module = compile(&f, &f_text, header(&f).lines().count());
let (vertex, fragment) = match (vertex_module, fragment_module) {
(Ok(v), Ok(f)) => (v, f),
(a, b) => return Err(a.err().into_iter().chain(b.err()).flatten().collect()),
};
let (uniform_size, offsets) = block_layout(&fragment);
let uniforms = offsets
.into_iter()
.filter_map(|(member, offset)| {
let name = member
.strip_prefix("hz_bool_")
.unwrap_or(&member)
.to_owned();
let ty = if name.starts_with("hz_") {
"vec4".to_owned()
} else {
uniforms.get(&name)?.clone()
};
Some(Uniform { name, ty, offset })
})
.collect();
let outputs = f.outputs.iter().copied().collect();
Ok(Program {
vertex,
fragment,
interface: Interface {
uniform_size,
uniforms,
samplers,
outputs,
},
})
}
/// `sampler2DShadow` → (`texture2D`, `samplerShadow`); `isampler3D` → (`itexture3D`, `sampler`).
pub(super) fn split_sampler(ty: &str) -> (String, &'static str) {
let shadow = ty.ends_with("Shadow");
let base = ty
.trim_end_matches("Shadow")
.replacen("sampler", "texture", 1);
(base, if shadow { "samplerShadow" } else { "sampler" })
}
/// Parses and validates one stage; errors at the pack's files and lines.
fn compile(stage: &Stage, text: &str, header_lines: usize) -> Result<naga::Module, Vec<Error>> {
let at = |span: naga::Span, message: String| {
let start = span.to_range().map_or(0, |r| r.start);
let line = text[..start.min(text.len())].matches('\n').count();
match line
.checked_sub(header_lines)
.and_then(|l| stage.pp.lines.get(l))
{
Some(o) => Error {
file: stage
.pp
.files
.get(o.file as usize)
.cloned()
.unwrap_or_default(),
line: o.line,
message,
},
None => Error {
file: "<horizon>".into(),
line: line as u32 + 1,
message,
},
}
};
let module = Frontend::default()
.parse(&Options::from(stage.kind), text)
.map_err(|e| {
e.errors
.into_iter()
.map(|e| at(e.meta, e.kind.to_string()))
.collect::<Vec<_>>()
})?;
Validator::new(ValidationFlags::all(), Capabilities::all())
.validate(&module)
.map_err(|e| {
let span = e.spans().next().map_or(naga::Span::default(), |(s, _)| *s);
// The validator nests its reasons: say them all.
let inner = e.into_inner();
let mut message = inner.to_string();
let mut source = std::error::Error::source(&inner);
while let Some(s) = source {
write!(message, ": {s}").unwrap();
source = s.source();
}
vec![at(span, message)]
})?;
Ok(module)
}
/// The uniform block's size and member offsets, from naga's layout.
fn block_layout(module: &naga::Module) -> (u32, Vec<(String, u32)>) {
for (_, var) in module.global_variables.iter() {
if var.space != naga::AddressSpace::Uniform {
continue;
}
if let naga::TypeInner::Struct { members, span } = &module.types[var.ty].inner {
let offsets = members
.iter()
.filter_map(|m| Some((m.name.clone()?, m.offset)))
.collect();
return (*span, offsets);
}
}
(16, vec![("hz_target".into(), 0)])
}
#[cfg(test)]
mod tests {
use super::*;
use crate::pack::tests::zip;
use crate::{Macros, Pack, preprocess};
fn program(files: &[(&str, &str)], target: Target) -> Result<Program, Vec<Error>> {
let files: Vec<(String, &str)> = files
.iter()
.map(|(p, t)| (format!("shaders/{p}"), *t))
.collect();
let files: Vec<(&str, &str)> = files.iter().map(|(p, t)| (p.as_str(), *t)).collect();
let pack = Pack::from_zip_bytes("P.zip", zip(&files)).unwrap();
let m = Macros::standard();
translate_composite(
&preprocess(&pack, "composite.vsh", &m),
&preprocess(&pack, "composite.fsh", &m),
target,
)
}
const VSH_120: &str = "#version 120\nvarying vec2 texcoord;\nvoid main() {\n gl_Position = ftransform();\n \
texcoord = (gl_TextureMatrix[0] * gl_MultiTexCoord0).xy;\n}\n";
#[test]
fn compatibility_programs_translate() {
let fsh = "#version 120\nuniform sampler2D colortex0;\nuniform sampler2D depthtex0;\nuniform float viewWidth;\n\
uniform bool hideGUI;\nuniform int frameCounter;\nvarying vec2 texcoord;\n\
vec3 tint(sampler2D s, vec2 c) { return texture2D(s, c).rgb; }\nvoid main() {\n \
vec3 c = tint(colortex0, texcoord) * texture2D(depthtex0, texcoord).r;\n if (hideGUI) c *= 0.5;\n \
/* DRAWBUFFERS:04 */\n gl_FragData[0] = vec4(c, 1.0);\n gl_FragData[1] = vec4(frameCounter / viewWidth);\n}\n";
let p = program(
&[("composite.vsh", VSH_120), ("composite.fsh", fsh)],
Target::Buffers,
)
.unwrap();
let i = &p.interface;
let samplers: Vec<(&str, u32)> = i
.samplers
.iter()
.map(|s| (s.name.as_str(), s.binding))
.collect();
assert_eq!(samplers, [("colortex0", 1), ("depthtex0", 3)]);
let uniforms: Vec<(&str, &str)> = i
.uniforms
.iter()
.map(|u| (u.name.as_str(), u.ty.as_str()))
.collect();
assert_eq!(
uniforms,
[
("hz_target", "vec4"),
("hz_fog_color", "vec4"),
("hz_fog", "vec4"),
("frameCounter", "int"),
("hideGUI", "bool"),
("viewWidth", "float")
]
);
assert_eq!(i.uniforms[0].offset, 0);
assert!(i.uniform_size >= 28);
assert_eq!(i.outputs, [0, 1]);
}
#[test]
fn core_programs_translate() {
let vsh = "#version 330 core\nin vec3 vaPosition;\nin vec2 vaUV0;\nuniform mat4 modelViewMatrix, projectionMatrix;\n\
out vec2 uv;\nflat out int flatId;\nvoid main() {\n \
gl_Position = projectionMatrix * modelViewMatrix * vec4(vaPosition, 1.0);\n uv = vaUV0;\n flatId = 1;\n}\n";
let fsh = "#version 330 core\nin vec2 uv;\nuniform sampler2D colortex0;\nuniform usampler2D colortex5;\n\
layout(location = 2) out vec4 color;\nvoid main() {\n \
if (texture(colortex0, uv).a < 0.1) discard;\n color = texture(colortex0, uv) + float(texelFetch(colortex5, ivec2(gl_FragCoord.xy), 0).r);\n}\n";
let p = program(
&[("composite.vsh", vsh), ("composite.fsh", fsh)],
Target::Window,
)
.unwrap();
assert_eq!(p.interface.outputs, [2]);
assert!(
p.interface
.uniforms
.iter()
.any(|u| u.name == "projectionMatrix" && u.ty == "mat4")
);
}
#[test]
fn errors_point_at_the_pack_file_and_line() {
let fsh =
"#version 120\n#include \"/lib/bad.glsl\"\nvoid main() { gl_FragColor = vec4(f()); }\n";
let bad = "// fine\nfloat f() {\n return undefinedThing;\n}\n";
let errors = program(
&[
("composite.vsh", VSH_120),
("composite.fsh", fsh),
("lib/bad.glsl", bad),
],
Target::Buffers,
)
.err()
.unwrap();
assert_eq!(
(errors[0].file.as_str(), errors[0].line),
("lib/bad.glsl", 3),
"{errors:?}"
);
}
}
@@ -0,0 +1,257 @@
//! The second pass: the stage's code written again for naga, line for line, and the
//! declarations of ours it needs.
use std::collections::{BTreeMap, HashMap};
use std::fmt::Write;
use naga::ShaderStage;
use super::scan::{LATER_KEYWORDS, Stage, UNIFORM_STATE, VERTEX_INPUTS, is_ident, significant};
use super::{Sampler, split_sampler};
impl Stage<'_> {
/// Declarations of ours this stage uses (after the uniform block, samplers and varyings).
pub(super) fn prelude(&self) -> String {
let mut p = String::from(
"float hz_modf(float x, out float i) { i = trunc(x); return x - i; }\n\
vec2 hz_modf(vec2 x, out vec2 i) { i = trunc(x); return x - i; }\n\
vec3 hz_modf(vec3 x, out vec3 i) { i = trunc(x); return x - i; }\n\
vec4 hz_modf(vec4 x, out vec4 i) { i = trunc(x); return x - i; }\n\
struct hz_FogParameters { vec4 color; float density; float start; float end; float scale; };\n\
hz_FogParameters hz_Fog;\n\
const mat4 hz_ModelViewMatrix = mat4(1.0); const mat4 hz_ProjectionMatrix = mat4(2.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, -1.0, -1.0, 0.0, 1.0);\n\
const mat4 hz_ModelViewProjectionMatrix = hz_ProjectionMatrix; const mat4 hz_ModelViewMatrixInverse = mat4(1.0); const mat3 hz_NormalMatrix = mat3(1.0);\n\
const mat4 hz_ProjectionMatrixInverse = mat4(0.5, 0.0, 0.0, 0.0, 0.0, 0.5, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.5, 0.5, 0.0, 1.0);\n\
const mat4 hz_TextureMatrix[8] = mat4[8](mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0), mat4(1.0));\n",
);
if self.kind == ShaderStage::Vertex {
p.push_str(
"const vec2 hz_quad[6] = vec2[6](vec2(0.0, 0.0), vec2(1.0, 0.0), vec2(1.0, 1.0), vec2(0.0, 0.0), vec2(1.0, 1.0), vec2(0.0, 1.0));\n\
vec4 hz_Vertex; vec4 hz_Color; vec4 hz_SecondaryColor; vec3 hz_Normal; float hz_FogCoord;\n\
vec4 hz_MultiTexCoord0; vec4 hz_MultiTexCoord1; vec4 hz_MultiTexCoord2; vec4 hz_MultiTexCoord3;\n\
vec4 hz_MultiTexCoord4; vec4 hz_MultiTexCoord5; vec4 hz_MultiTexCoord6; vec4 hz_MultiTexCoord7;\n",
);
} else {
for n in &self.outputs {
if !self.frag_outs.iter().any(|(at, _)| at == n) {
writeln!(p, "layout(location = {n}) out vec4 hz_out{n};").unwrap();
}
}
for (_, decl) in &self.frag_outs {
writeln!(p, "{decl}").unwrap();
}
p.push_str("vec4 hz_FragCoord;\n");
}
p
}
/// The pack's code with the rewrites above; the same number of lines.
pub(super) fn rewrite(
&self,
samplers: &[Sampler],
uniforms: &BTreeMap<String, String>,
) -> String {
let removed = |i: usize| self.removed.iter().any(|r| r.contains(&i));
let sampler_ty: HashMap<&str, &str> = samplers
.iter()
.map(|s| (s.name.as_str(), s.ty.as_str()))
.collect();
let old = self.pp.version.as_ref().is_none_or(|(v, _)| *v < 400);
let fragment = self.kind == ShaderStage::Fragment;
let mut out = String::with_capacity(self.pp.code.len() + 1024);
// Per open bracket: the function called, and the closing text to add.
let mut calls: Vec<(&str, &str, usize)> = Vec::new();
// Sampler parameters of the function being written: name → type.
let mut params: HashMap<String, String> = HashMap::new();
let mut depth = 0;
let mut skip_until = 0;
let next = |i: usize| (i + 1..self.tokens.len()).find(|&j| significant(self.tokens[j]));
let prev = |i: usize| (0..i).rev().find(|&j| significant(self.tokens[j]));
for (i, &t) in self.tokens.iter().enumerate() {
if i < skip_until {
continue;
}
if let Some(text) = self.replaced.get(&i) {
out.push_str(text);
continue;
}
if removed(i) {
out.extend(t.chars().filter(|&c| c == '\n'));
if t.contains(' ') && !t.contains('\n') {
out.push(' ');
}
continue;
}
let called = next(i).is_some_and(|j| self.tokens[j] == "(");
match t {
"{" => depth += 1,
"}" => {
depth -= 1;
if depth == 0 {
params.clear();
}
}
"(" => {
let callee = prev(i).map_or("", |j| self.tokens[j]);
let close = if matches!(callee, "shadow2D" | "shadow2DLod" | "shadow2DProj") {
")"
} else {
""
};
calls.push((callee, close, 0));
}
")" => {
if let Some((_, close, _)) = calls.pop() {
out.push_str(close);
}
}
"," => {
if let Some(c) = calls.last_mut() {
c.2 += 1;
}
}
_ => {}
}
// `T x[] = T[](a, b)`: naga gives unsized arrays no size, so the count is written in.
if t == "["
&& let Some(close) = next(i).filter(|&j| self.tokens[j] == "]")
{
if let Some(n) = self.array_length(close) {
write!(out, "[{n}]").unwrap();
skip_until = close + 1;
continue;
}
}
if !is_ident(t) {
out.push_str(t);
continue;
}
// A function definition's sampler parameters: `sampler2D s` → `texture2D s, sampler hz_s_s`.
if depth == 0 && t.contains("sampler") && calls.len() == 1 {
if let Some(n) = next(i).filter(|&n| is_ident(self.tokens[n])) {
let name = self.tokens[n];
let (texture, sampler) = split_sampler(t);
write!(out, "{texture} {name}, {sampler} hz_s_{name}").unwrap();
params.insert(name.to_owned(), t.to_owned());
skip_until = n + 1;
continue;
}
}
let sampler = params
.get(t)
.map(String::as_str)
.or_else(|| sampler_ty.get(t).copied());
if let Some(ty) = sampler {
let as_argument = calls.last().is_some_and(|(callee, _, arg)| {
self.sampler_functions
.get(*callee)
.is_some_and(|p| p.get(*arg) == Some(&true))
}) && prev(i)
.is_some_and(|j| matches!(self.tokens[j], "(" | ","));
if as_argument {
write!(out, "{t}, hz_s_{t}").unwrap();
} else {
write!(out, "{ty}({t}, hz_s_{t})").unwrap();
}
continue;
}
if uniforms.get(t).is_some_and(|ty| ty == "bool") {
write!(out, "(hz_bool_{t} != 0)").unwrap();
continue;
}
let replaced: Option<String> = match t {
"main" if depth == 0 && called => Some("hz_main".into()),
// GLSL's `modf`, which naga's frontend lacks.
"modf" if called => Some("hz_modf".into()),
"texture2D" | "texture1D" | "texture3D" | "textureCube" if called => {
Some("texture".into())
}
"texture2DLod" | "texture3DLod" | "texture2DLodARB" | "textureCubeLod"
if called =>
{
Some("textureLod".into())
}
"texture2DProj" if called => Some("textureProj".into()),
"texture2DGrad" | "texture2DGradARB" if called => Some("textureGrad".into()),
"shadow2D" if called => Some("vec4(texture".into()),
"shadow2DLod" if called => Some("vec4(textureLod".into()),
"shadow2DProj" if called => Some("vec4(textureProj".into()),
"ftransform" if called => {
// `ftransform()` → the product it stands for; its `()` stays as a no-op call... so skip it.
let close = next(i).and_then(next);
skip_until = close.map_or(i + 1, |c| c + 1);
Some("(hz_ModelViewProjectionMatrix * hz_Vertex)".into())
}
"gl_FragColor" => Some("hz_out0".into()),
"gl_FragData" | "gl_TexCoord" => match self.index_after(i) {
Some((n, close)) => {
skip_until = close + 1;
Some(match t {
"gl_FragData" => format!("hz_out{n}"),
_ => format!("hz_TexCoord{n}"),
})
}
None => None,
},
"gl_Color" if fragment => Some("hz_FrontColor".into()),
"gl_FragCoord" if !fragment => None,
"gl_FragCoord" => Some("hz_FragCoord".into()),
"gl_FrontColor" => Some("hz_FrontColor".into()),
"gl_Fog" => Some("hz_Fog".into()),
_ if UNIFORM_STATE.iter().any(|b| t.starts_with(b)) => {
Some(format!("hz_{}", &t[3..]))
}
_ if !fragment && VERTEX_INPUTS.iter().any(|b| t.starts_with(b)) => {
Some(format!("hz_{}", &t[3..]))
}
"gl_VertexID" => Some("gl_VertexIndex".into()),
"gl_InstanceID" => Some("gl_InstanceIndex".into()),
_ if old && LATER_KEYWORDS.contains(&t) => Some(format!("hz_kw_{t}")),
_ => None,
};
out.push_str(replaced.as_deref().unwrap_or(t));
}
out
}
}
impl Stage<'_> {
/// The element count of an unsized array whose `[]` ends at `close`: the arguments of the
/// constructor that follows (`[] = T[](...)` or `[](...)`).
pub(super) fn array_length(&self, close: usize) -> Option<usize> {
let mut j = close + 1;
let mut seen_eq = false;
while let Some(&t) = self.tokens.get(j) {
match t {
"(" => break,
"=" if !seen_eq => seen_eq = true,
";" | "," | ")" | "{" => return None,
_ => {}
}
j += 1;
}
let (mut depth, mut args, mut any) = (0, 1, false);
for &t in &self.tokens[j + 1..] {
match t {
"(" | "[" | "{" => depth += 1,
")" if depth == 0 => return any.then_some(args),
")" | "]" | "}" => depth -= 1,
"," if depth == 0 => args += 1,
_ if significant(t) => any = true,
_ => {}
}
}
None
}
}
/// What our vertex `main` gives a vertex input of a composite-style program.
pub(super) fn attribute_value(name: &str, ty: &str) -> String {
match name {
"vaPosition" => "vec3(hz_p, 0.0)".into(),
"vaUV0" => "hz_p".into(),
"vaColor" => "vec4(1.0)".into(),
"vaNormal" => "vec3(0.0, 0.0, 1.0)".into(),
_ => format!("{ty}(0)"),
}
}
@@ -0,0 +1,399 @@
//! The first pass over a stage's tokens: what it declares at global scope, which of its
//! functions take samplers, which outputs and built-ins it uses.
use std::collections::{BTreeSet, HashMap};
use std::fmt::Write;
use naga::ShaderStage;
use super::Error;
use super::rewrite::attribute_value;
use crate::Preprocessed;
use crate::macros::lex;
/// A varying: its name and its declaration with `@` where the name goes (`flat ivec2 @[2];`).
pub(super) struct Var {
pub(super) name: String,
pub(super) decl: String,
}
pub(super) struct Stage<'p> {
pub(super) kind: ShaderStage,
pub(super) pp: &'p Preprocessed,
pub(super) tokens: Vec<&'p str>,
/// Token ranges to blank out (declarations moved into the header).
pub(super) removed: Vec<std::ops::Range<usize>>,
/// Name, type with its array suffix, token of the type.
pub(super) uniforms: Vec<(String, String, usize)>,
pub(super) samplers: Vec<(String, String)>,
pub(super) ins: Vec<Var>,
pub(super) outs: Vec<Var>,
/// Vertex inputs, now private globals, and the value our `main` gives each.
pub(super) attributes: Vec<(String, String)>,
/// The pack's fragment outputs, declared again with their locations.
pub(super) frag_outs: Vec<(u32, String)>,
/// The pack's functions that take samplers: which parameters are samplers.
pub(super) sampler_functions: HashMap<String, Vec<bool>>,
pub(super) outputs: BTreeSet<u32>,
pub(super) frag_coord: bool,
/// Tokens written differently, by index.
pub(super) replaced: HashMap<usize, String>,
/// Globals whose initializer runs in our `main` (`g = hz_init_g();`), in order.
pub(super) inits: Vec<String>,
pub(super) errors: Vec<Error>,
}
pub(super) const STORAGE: [&str; 5] = ["uniform", "in", "out", "attribute", "varying"];
pub(super) const QUALIFIERS: [&str; 11] = [
"flat",
"smooth",
"noperspective",
"centroid",
"invariant",
"highp",
"mediump",
"lowp",
"precise",
"sample",
"const",
];
/// The compatibility profile's uniform state (with its `Inverse`/`Transpose` forms).
pub(super) const UNIFORM_STATE: [&str; 5] = [
"gl_ModelViewMatrix",
"gl_ProjectionMatrix",
"gl_ModelViewProjectionMatrix",
"gl_NormalMatrix",
"gl_TextureMatrix",
];
/// The compatibility profile's vertex inputs.
pub(super) const VERTEX_INPUTS: [&str; 6] = [
"gl_Vertex",
"gl_Color",
"gl_SecondaryColor",
"gl_Normal",
"gl_FogCoord",
"gl_MultiTexCoord",
];
/// Words that became keywords after GLSL 1.x and that old packs use as names.
pub(super) const LATER_KEYWORDS: [&str; 17] = [
"sample",
"buffer",
"shared",
"filter",
"input",
"output",
"common",
"partition",
"active",
"resource",
"patch",
"subroutine",
"coherent",
"volatile",
"restrict",
"readonly",
"writeonly",
];
pub(super) fn significant(t: &str) -> bool {
!t.trim().is_empty()
}
pub(super) fn is_ident(t: &str) -> bool {
t.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_')
}
impl<'p> Stage<'p> {
pub(super) fn scan(pp: &'p Preprocessed, kind: ShaderStage) -> Self {
let mut s = Stage {
kind,
pp,
tokens: lex(&pp.code),
removed: Vec::new(),
uniforms: Vec::new(),
samplers: Vec::new(),
ins: Vec::new(),
outs: Vec::new(),
attributes: Vec::new(),
frag_outs: Vec::new(),
sampler_functions: HashMap::new(),
outputs: BTreeSet::new(),
frag_coord: false,
replaced: HashMap::new(),
inits: Vec::new(),
errors: Vec::new(),
};
let (mut start, mut depth, mut parens) = (0, 0usize, 0usize);
for i in 0..s.tokens.len() {
match s.tokens[i] {
"(" => parens += 1,
")" => parens = parens.saturating_sub(1),
"{" if depth == 0 && parens == 0 => {
s.global(start, i, true);
depth = 1;
}
"{" => depth += 1,
"}" if depth > 0 => {
depth -= 1;
if depth == 0 {
start = i + 1;
}
}
";" if depth == 0 && parens == 0 => {
s.global(start, i, false);
start = i + 1;
}
"gl_FragData" | "gl_FragColor" if kind == ShaderStage::Fragment => {
let n = if s.tokens[i] == "gl_FragColor" {
Some(0)
} else {
s.index_after(i).map(|(n, _)| n)
};
match n {
Some(n) => _ = s.outputs.insert(n),
None => s
.errors
.push(s.error(i, "`gl_FragData` needs a constant index")),
}
}
"gl_FragCoord" => s.frag_coord = true,
"gl_TexCoord" => {
if let Some((n, _)) = s.index_after(i) {
s.builtin_varying(format!("hz_TexCoord{n}"));
}
}
"gl_FrontColor" => s.builtin_varying("hz_FrontColor".into()),
"gl_Color" if kind == ShaderStage::Fragment => {
s.builtin_varying("hz_FrontColor".into())
}
_ => {}
}
}
s
}
pub(super) fn builtin_varying(&mut self, name: String) {
let list = if self.kind == ShaderStage::Vertex {
&mut self.outs
} else {
&mut self.ins
};
if !list.iter().any(|v| v.name == name) {
list.push(Var {
name,
decl: "vec4 @;".into(),
});
}
}
/// `[ n ]` right after token `i`: the number and the index of the `]`.
pub(super) fn index_after(&self, i: usize) -> Option<(u32, usize)> {
let mut it = (i + 1..self.tokens.len()).filter(|&j| significant(self.tokens[j]));
let open = it.next()?;
let n = it.next()?;
let close = it.next()?;
(self.tokens[open] == "[" && self.tokens[close] == "]")
.then(|| self.tokens[n].parse().ok().map(|n| (n, close)))
.flatten()
}
pub(super) fn error(&self, token: usize, message: &str) -> Error {
let line: usize = self.tokens[..token.min(self.tokens.len())]
.iter()
.map(|t| t.matches('\n').count())
.sum();
let o = self.pp.lines.get(line).copied().unwrap_or_default();
Error {
file: self
.pp
.files
.get(o.file as usize)
.cloned()
.unwrap_or_default(),
line: o.line,
message: message.to_owned(),
}
}
/// A statement at global scope, tokens `from..to` (without its `;`, or before a `{`).
pub(super) fn global(&mut self, from: usize, to: usize, body: bool) {
let words: Vec<usize> = (from..to)
.filter(|&i| significant(self.tokens[i]))
.collect();
// A prototype: a `(` before any `=`, and no storage qualifier.
let paren = words
.iter()
.position(|&i| matches!(self.tokens[i], "(" | "="));
let prototype = paren.is_some_and(|p| self.tokens[words[p]] == "(")
&& !words.iter().any(|&i| STORAGE.contains(&self.tokens[i]));
if body || prototype {
return self.function(&words);
}
let mut k = 0;
let mut storage = None;
let mut location = None;
let mut interpolation = String::new();
while let Some(&i) = words.get(k) {
let w = self.tokens[i];
if w == "layout" {
// layout(location = n, ...)
let mut j = k + 1;
while let Some(&t) = words.get(j) {
j += 1;
if self.tokens[t] == "location" {
location = words
.get(j + 1)
.and_then(|&n| self.tokens[n].parse::<u32>().ok());
}
if self.tokens[t] == ")" {
break;
}
}
k = j;
continue;
}
if STORAGE.contains(&w) {
storage = Some(w);
} else if QUALIFIERS.contains(&w) {
if matches!(
w,
"flat" | "smooth" | "noperspective" | "centroid" | "sample"
) {
write!(interpolation, "{w} ").unwrap();
}
} else {
break;
}
k += 1;
}
let Some(storage) = storage else {
return self.initializer(&words, to);
};
let Some(&ty_at) = words.get(k) else { return };
let ty = self.tokens[ty_at].to_owned();
if words[k..].iter().any(|&i| self.tokens[i] == "{") {
return; // An interface block: left as the pack wrote it.
}
// Declarators: name [array] [= init], separated by commas.
let mut declarators: Vec<(String, String)> = Vec::new();
let mut j = k + 1;
while let Some(&n) = words.get(j) {
let name = self.tokens[n].to_owned();
let mut array = String::new();
j += 1;
let mut nesting = 0;
while let Some(&t) = words.get(j) {
match self.tokens[t] {
"," if nesting == 0 => break,
"(" | "[" => nesting += 1,
")" | "]" => nesting -= 1,
_ => {}
}
if self.tokens[t] == "=" && nesting == 0 {
// An initializer: uniforms are set by the game, so it is dropped.
while words.get(j).is_some_and(|&t| self.tokens[t] != ",") {
j += 1;
}
break;
}
array.push_str(self.tokens[t]);
j += 1;
}
j += 1;
declarators.push((name, array));
}
let fragment = self.kind == ShaderStage::Fragment;
let input = matches!(storage, "in" | "attribute") || (storage == "varying" && fragment);
self.removed.push(from..to + 1);
for (name, array) in declarators {
let decl = format!("{interpolation}{ty} @{array};");
match storage {
"uniform" if ty.contains("sampler") => self.samplers.push((name, ty.clone())),
"uniform" if ty.contains("image") => self
.errors
.push(self.error(ty_at, &format!("`{name}`: images are not supported yet"))),
"uniform" => self.uniforms.push((name, format!("{ty}{array}"), ty_at)),
_ if input && !fragment => {
let value = attribute_value(&name, &ty);
self.attributes.push((name, value));
self.removed.pop();
// A private global of ours now: `in`/`attribute` and qualifiers go.
for &w in &words[..k] {
self.removed.push(w..w + 1);
}
}
_ if input => self.ins.push(Var { name, decl }),
_ if fragment => {
let n = location.unwrap_or(self.outputs.len() as u32);
self.outputs.insert(n);
self.frag_outs
.push((n, format!("layout(location = {n}) out {ty} {name}{array};")));
}
_ => self.outs.push(Var { name, decl }),
}
}
}
/// A global `T name = a ? b : c;`: naga can't evaluate a conditional at global scope, so
/// the initializer becomes a function of ours on the same line (`T name; T hz_init_name() {
/// return a ? b : c; }`), called by our `main` before the pack's.
pub(super) fn initializer(&mut self, words: &[usize], to: usize) {
let text = |i: usize| self.tokens[i];
let Some(eq) = words.iter().position(|&i| text(i) == "=") else {
return;
};
if !words[eq..].iter().any(|&i| text(i) == "?")
|| words[..eq].iter().any(|&i| text(i) == ",")
{
return;
}
let Some(name_at) = words[..eq].iter().rposition(|&i| is_ident(text(i))) else {
return;
};
let name = text(words[name_at]).to_owned();
let array: String = words[name_at + 1..eq].iter().map(|&i| text(i)).collect();
let ty: Vec<&str> = words[..name_at]
.iter()
.map(|&i| text(i))
.filter(|&t| t != "const")
.collect();
let Some(ty) = ty.last() else { return };
for &i in &words[..name_at] {
if text(i) == "const" {
self.removed.push(i..i + 1);
}
}
self.replaced.insert(
words[eq],
format!("; {ty}{array} hz_init_{name}() {{ return"),
);
self.replaced.insert(to, "; }".into());
self.inits.push(name);
}
/// A function definition or prototype: remember which parameters are samplers.
pub(super) fn function(&mut self, words: &[usize]) {
let Some(open) = words.iter().position(|&i| self.tokens[i] == "(") else {
return;
};
let Some(&name) = open.checked_sub(1).and_then(|n| words.get(n)) else {
return;
};
let mut params = Vec::new();
let mut is_sampler = false;
for &i in &words[open + 1..] {
match self.tokens[i] {
"," | ")" => {
params.push(is_sampler);
is_sampler = false;
}
t if t.contains("sampler") => is_sampler = true,
_ => {}
}
}
if params.iter().any(|&p| p) {
self.sampler_functions
.insert(self.tokens[name].to_owned(), params);
}
}
}
+127 -6
View File
@@ -5,7 +5,11 @@
use std::path::Path;
use std::time::Instant;
use horizon_shaders::{Lang, Pack, discover};
use std::collections::BTreeMap;
use horizon_shaders::{
Lang, Macros, Pack, Properties, Target, discover, preprocess, translate_composite,
};
#[test]
#[ignore = "needs HORIZON_TEST_SHADER_PACKS"]
@@ -22,21 +26,138 @@ fn real_packs_read() {
for f in found.iter().filter(|f| f.problem.is_none()) {
let start = Instant::now();
let pack = Pack::open(&f.path).unwrap();
let properties = pack.read_text("shaders.properties");
let macros = Macros::standard();
let shaders = Properties::parse(
&pack.read_text("shaders.properties").unwrap_or_default(),
&macros,
);
let ids: Vec<Properties> = ["block", "item", "entity", "dimension"]
.iter()
.filter_map(|n| pack.read_text(&format!("{n}.properties")))
.map(|t| Properties::parse(&t, &macros))
.collect();
let lang = Lang::load(&pack, "en_us");
let elapsed = start.elapsed();
println!(
"{}: {} files at the root, shaders.properties {} bytes, lang {}, {:.2} ms",
"{}: shaders.properties {} keys ({} notes), {} id files {} keys ({} notes), lang {}, {:.2} ms",
f.name,
pack.list("").len(),
properties.as_ref().map_or(0, String::len),
shaders.entries.len(),
shaders.notes.len(),
ids.len(),
ids.iter().map(|p| p.entries.len()).sum::<usize>(),
ids.iter().map(|p| p.notes.len()).sum::<usize>(),
if lang == Lang::default() {
"none"
} else {
"read"
},
elapsed.as_secs_f64() * 1000.0
);
for n in &shaders.notes {
println!(" shaders.properties:{}: {}", n.line, n.message);
}
// Every stage file of the base and dimension folders through the preprocessor.
let start = Instant::now();
let (mut stages, mut lines, mut diagnostics) = (0, 0, Vec::new());
for dir in ["", "world0", "world-1", "world1"] {
for file in pack.list(dir) {
if [".vsh", ".fsh", ".gsh", ".csh"]
.iter()
.any(|e| file.ends_with(e))
{
let path = if dir.is_empty() {
file
} else {
format!("{dir}/{file}")
};
let p = preprocess(&pack, &path, &macros);
stages += 1;
lines += p.lines.len();
diagnostics.extend(p.diagnostics.iter().map(|d| {
let file = &p.files[d.origin.file as usize];
format!("{file}:{}: {}", d.origin.line, d.message)
}));
}
}
}
println!(
" {stages} stage files preprocessed, {lines} lines out, {} diagnostics, {:.1} ms",
diagnostics.len(),
start.elapsed().as_secs_f64() * 1000.0
);
assert!(properties.is_some(), "{}: no shaders.properties", f.name);
diagnostics.sort();
diagnostics.dedup();
for d in diagnostics.iter().take(8) {
println!(" {d}");
}
// Every composite-style program with both stages, translated and validated.
let start = Instant::now();
let (mut ok, mut failed, mut causes) = (0, 0, BTreeMap::<String, usize>::new());
for dir in ["", "world0", "world-1", "world1"] {
for file in pack.list(dir) {
let Some(name) = file.strip_suffix(".fsh") else {
continue;
};
let composite = [
"composite",
"deferred",
"final",
"prepare",
"begin",
"shadowcomp",
]
.iter()
.any(|p| {
name.strip_prefix(p)
.is_some_and(|n| n.chars().all(|c| c.is_ascii_digit()))
});
let path = |ext: &str| {
if dir.is_empty() {
format!("{name}.{ext}")
} else {
format!("{dir}/{name}.{ext}")
}
};
if !composite || !pack.exists(&path("vsh")) {
continue;
}
let target = if name == "final" {
Target::Window
} else {
Target::Buffers
};
let v = preprocess(&pack, &path("vsh"), &macros);
let fr = preprocess(&pack, &path("fsh"), &macros);
match translate_composite(&v, &fr, target) {
Ok(_) => ok += 1,
Err(errors) => {
failed += 1;
let first = &errors[0];
let cause: String = first.message.chars().take(90).collect();
*causes.entry(cause).or_default() += 1;
if std::env::var_os("SHOW_ERRORS").is_some() {
println!(
" {}: {}:{}: {}",
path("fsh"),
first.file,
first.line,
first.message
);
}
}
}
}
}
println!(
" composite-style programs: {ok} translate, {failed} fail, {:.0} ms",
start.elapsed().as_secs_f64() * 1000.0
);
let mut causes: Vec<_> = causes.into_iter().collect();
causes.sort_by_key(|c| std::cmp::Reverse(c.1));
for (cause, n) in causes.iter().take(6) {
println!(" {n} x {cause}");
}
assert!(!shaders.entries.is_empty(), "{}: no keys read", f.name);
read += 1;
}
assert!(read > 0, "no packs under {}", dir.display());