diff --git a/AGENTS.md b/AGENTS.md index 0656e40fc..a72c6723a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,7 @@ Game workspace (root `Cargo.toml`); `launcher/` and `crates/horizon-telemetry-se | `horizon-friends`, `horizon-relay` | Play with Friends: invite codes, encrypted tunnel, the relay ([docs/play-with-friends.md](docs/play-with-friends.md)) | | `horizon-mods`, `horizon-mod-sdk`, `cargo-horizon-mod` | Sandboxed WebAssembly mods: runtime, SDK + WIT, `cargo horizon-mod` ([docs/mods.md](docs/mods.md)) | | `horizon-port-pack` | Porting Java modpacks ([docs/modpacks.md](docs/modpacks.md)) | +| `horizon-shader-api` | Horizon's own WGSL shader packs ([docs/api/shaders.md](docs/api/shaders.md)): `horizon-shaders.toml`, the `.hshader` package; shared by the game, the launcher and the registry | | `horizon-shaders` | Shader packs (the `shaders/` folder format), rebuilt clean-room from a written spec of the format; no wgpu | | `horizon-theme` | Design tokens and fonts for the launcher, the game's own menus and the website | | `horizon-telemetry`, `horizon-telemetry-server`, `horizon-bench` | Opt-in data collection, its ingest server, the benchmark report format | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e6e959186..21951c083 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,6 +7,11 @@ welcome on [Discord](https://discord.gg/HcTeMZxe5E). This repository holds the game, the launcher and the website, plus the tools, tests and docs that build, check and release them. Anything else (videos, Discord and other community material) lives elsewhere. +**Where to start: [playhorizon.dev/hub](https://playhorizon.dev/hub).** It shows what is being done now and next +([ROADMAP.md](ROADMAP.md) in the repository), the work board, good first tasks, and the forms to report a bug or +suggest something. A pull request for a roadmap item carries its `roadmap:` label; ideas go to the suggestion +queue (the hub, or `/suggest` on Discord) rather than to issues, which are for bugs. + ## Build it You need Rust (stable), Python 3.10+, JDK 25, and on Linux the ALSA and udev headers (`libasound2-dev libudev-dev`). diff --git a/Cargo.lock b/Cargo.lock index 2d09d2d0e..70c3c831c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1437,6 +1437,19 @@ dependencies = [ "ureq", ] +[[package]] +name = "horizon-shader-api" +version = "0.1.0" +dependencies = [ + "bytemuck", + "naga", + "serde", + "serde_json", + "sha2", + "toml", + "zip", +] + [[package]] name = "horizon-shaders" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c6f2bc581..294e93aa9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] exclude = ["launcher", "crates/horizon-telemetry-server", "crates/horizon-mod-registry", "third_party/wgpu-hal"] -members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-shaders", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"] +members = ["crates/horizon-core", "crates/horizon-dualsense", "crates/horizon-threads", "crates/horizon-world", "crates/horizon-client", "crates/horizon-server-files", "crates/horizon-server", "crates/horizon-bots", "crates/horizon-assets", "crates/horizon-protocol", "crates/horizon-theme", "crates/horizon-prepare", "crates/horizon-telemetry", "crates/horizon-bench", "crates/horizon-friends", "crates/horizon-relay", "crates/horizon-mod-sdk", "crates/horizon-mods", "crates/horizon-mods/tests/guest", "crates/cargo-horizon-mod", "crates/horizon-port-pack", "crates/horizon-shaders", "crates/horizon-shader-api", "crates/horizon-release-sign", "crates/horizon-mod-sdk/examples/coords-hud", "xtask"] resolver = "2" [workspace.package] diff --git a/README.md b/README.md index 6dd192d5d..ceee29249 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Download · Documentation · Release notes · + Roadmap · Project status · Contributing

@@ -109,6 +110,10 @@ Everything prepared lands in the gitignored `.local/`. It is derived from the ga The client starts on the title screen. Command-line options (`--world`, `--seed`, `--creative`, `--render-distance`, `--dedicated`), controls, scripting through `horizon-ctl` and the architecture are described in [the native client guide](docs/native-client.md). The rest of the developer documentation is indexed in [docs/README.md](docs/README.md), including [the parity workflow](docs/parity-workflow.md). +## Roadmap + +What is being done now, what comes next and where you can help, in plain words and in technical detail: [ROADMAP.md](ROADMAP.md). [playhorizon.dev/hub](https://playhorizon.dev/hub) shows it with the work board, the journal of what shipped and how to help. + ## Contributing Issues and pull requests are welcome: see [CONTRIBUTING.md](CONTRIBUTING.md) for how we work (vanilla first, small pull requests, labels), or [the contributing guide](https://playhorizon.dev/docs/contributing) on the site. Questions and ideas: [Discord](https://discord.gg/HcTeMZxe5E). Before opening a pull request, run the self-check and paste the block it prints: diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 000000000..b433fad3a --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,341 @@ +# Horizon roadmap + + + +What is being done on Horizon, what comes next and where you can help. There are no dates: things are Now, Next or Later. Each item is said in plain words, with the technical picture folded under it. The same roadmap, with a Simple / Technical switch, the board, the journal and how to help, is on [playhorizon.dev/hub](https://playhorizon.dev/hub). Pull requests carry the `roadmap:` label of their item. + +## At a glance + +Horizon is Minecraft: Java Edition 26.3 rewritten from scratch in Rust, so it runs faster on the same computer. Single player, LAN, servers, resource packs and Horizon's own mods work today, on your own copy of the game. + +**Being done now** + +- **Steady frames at long render distances**: Making Horizon faster than vanilla on every computer, with smooth camera movement even at 32 chunks, by spreading the work better over the processor's cores. +- **Modpacks' world generation, natively**: Making the world-generation mods of popular modpacks (new structures, YUNG's dungeons and mineshafts, packs built on Lithostitched) generate their worlds in Horizon, natively and fast. +- **Playing on servers: everything shows as it should**: When you play on a server, other players, mobs, items and menus look and behave as in vanilla: skins, poses, item names, inventories and server networks. +- **Single player as a server: the last differences**: Single player now runs a small server in the background, as vanilla does. We are fixing what still behaves differently there; every fix also works for LAN and server players. +- **Horizon's mod registry**: One place for mods, modpacks and resource packs made for Horizon: browse them on the website and install them from the launcher in one click. + +**Next**: Microsoft sign-in (coming soon); CurseForge in the mod browser (coming soon); Fresh benchmarks against vanilla; Pistons push players and mobs; Horizon on Intel Macs. + +**How to help**: Play it and tell us what differs from vanilla (Discord or GitHub), run the benchmark on your computer, vote for what you want next, or pick one of the Help wanted items. + +**Developers**: Read CONTRIBUTING.md, pick a Help wanted item or a To do card, check vanilla's code for the behaviour, and paste the block `python3 tools/pr_ready.py` prints in your pull request. + +## Now + +### Performance + +#### Steady frames at long render distances + +Making Horizon faster than vanilla on every computer, with smooth camera movement even at 32 chunks, by spreading the work better over the processor's cores. + +
Technical + +Frame pacing at render distance 32 ([#678](https://github.com/ludovic111/horizon/issues/678)): long frames come from work competing with the render thread. In review: world generation workers build free structure starts instead of queueing them ([#1760](https://github.com/ludovic111/horizon/issues/1760)), the worker pool leaves one core as vanilla does ([#1778](https://github.com/ludovic111/horizon/issues/1778)), and the terrain vertex arena is read as storage where flat draws run ([#1742](https://github.com/ludovic111/horizon/issues/1742)). Still open: far terrain at 48 to 64 chunks ([#718](https://github.com/ludovic111/horizon/issues/718)) and CPU-bound fields of block entities ([#717](https://github.com/ludovic111/horizon/issues/717)). Every change shows before and after numbers from `tools/perf_bench.py` against vanilla 26.3 ([docs/benchmark.md](docs/benchmark.md)). + +
+ +#### Faster block lookups + +Speeding up the code that reads and writes the blocks of the world, which almost everything in the game goes through: generation, lighting, meshing and ticking. + +
Technical + +`PalettedContainer` and `PackedStorage` find an entry's word by multiply and shift instead of an integer division ([#1790](https://github.com/ludovic111/horizon/issues/1790), [#1794](https://github.com/ludovic111/horizon/issues/1794)); meshing looks up baked models by index, not by name ([#1747](https://github.com/ludovic111/horizon/issues/1747)). Each change keeps the paletted storage harnesses identical and shows its microbenchmark ([docs/performance/paletted-container-access-results.md](docs/performance/paletted-container-access-results.md)). + +
+ +### Modpacks + +#### Modpacks' world generation, natively + +Making the world-generation mods of popular modpacks (new structures, YUNG's dungeons and mineshafts, packs built on Lithostitched) generate their worlds in Horizon, natively and fast. + +
Technical + +`horizon-port-pack` carries a pack's data-driven content over; structure mods that need code get a native port: YUNG's worldgen as written ([#1746](https://github.com/ludovic111/horizon/issues/1746)), Lithostitched's limited and guaranteed elements with pool and processor modifiers ([#1777](https://github.com/ludovic111/horizon/issues/1777)), its processors as the game's own types ([#1765](https://github.com/ludovic111/horizon/issues/1765)). Proved by the structure blocks and layout harnesses ([#1753](https://github.com/ludovic111/horizon/issues/1753), [#1757](https://github.com/ludovic111/horizon/issues/1757)). Mods whose Java code does more than world generation don't run ([docs/modpacks.md](docs/modpacks.md)). + +
+ +### Servers and multiplayer + +#### Playing on servers: everything shows as it should + +When you play on a server, other players, mobs, items and menus look and behave as in vanilla: skins, poses, item names, inventories and server networks. + +
Technical + +Client-of-a-server fixes in `game/remote*.rs` and `horizon-protocol`: mobs' aggressive and item-use flags ([#1802](https://github.com/ludovic111/horizon/issues/1802), [#1798](https://github.com/ludovic111/horizon/issues/1798)), ghost items after a remote menu ([#1795](https://github.com/ludovic111/horizon/issues/1795)), skins ([#1652](https://github.com/ludovic111/horizon/issues/1652)), cookies and transfer packets for proxy networks ([#1650](https://github.com/ludovic111/horizon/issues/1650)), firework boosts ([#1671](https://github.com/ludovic111/horizon/issues/1671)), living-entity attributes and simulation distance ([#1731](https://github.com/ludovic111/horizon/issues/1731)), styled item names ([#1702](https://github.com/ludovic111/horizon/issues/1702)), primed TNT between updates ([#1710](https://github.com/ludovic111/horizon/issues/1710)). + +
+ +#### Server dialogs and login screens + +Servers that show dialogs, such as the register and login screens of AuthMe, will work in Horizon. + +
Technical + +26.3's dialog packets and model ([#1720](https://github.com/ludovic111/horizon/issues/1720)), the dialog screen with its buttons and actions ([#1734](https://github.com/ludovic111/horizon/issues/1734)), then text and boolean inputs ([#1735](https://github.com/ludovic111/horizon/issues/1735)). + +
+ +### Mods and API + +#### Horizon's mod registry + +One place for mods, modpacks and resource packs made for Horizon: browse them on the website and install them from the launcher in one click. + +
Technical + +Merged: the registry index (schema 1, `.hpack`, checks before install, [#1634](https://github.com/ludovic111/horizon/issues/1634)), `cargo horizon-mod check` ([#1642](https://github.com/ludovic111/horizon/issues/1642)) and the submission service (`crates/horizon-mod-registry`, [#1659](https://github.com/ludovic111/horizon/issues/1659)). In review: the website's /mods ([#1596](https://github.com/ludovic111/horizon/issues/1596)) and /mods/submit ([#1635](https://github.com/ludovic111/horizon/issues/1635)), and in the launcher the registry as a source ([#1636](https://github.com/ludovic111/horizon/issues/1636)), install, enable and takedowns ([#1637](https://github.com/ludovic111/horizon/issues/1637)), the Mods page ([#1638](https://github.com/ludovic111/horizon/issues/1638)) and modpacks as their own profiles ([#1640](https://github.com/ludovic111/horizon/issues/1640)). + +More: [docs/mods.md](docs/mods.md) + +
+ +### Gameplay parity + +#### Single player as a server: the last differences + +Single player now runs a small server in the background, as vanilla does. We are fixing what still behaves differently there; every fix also works for LAN and server players. + +
Technical + +Since [#1514](https://github.com/ludovic111/horizon/issues/1514) single player runs on the server thread (`game/integrated.rs`) and the window joins it as a client; `--in-frame` keeps the old path. The open `[server-thread]` issues list the gaps. In review: a tool that changes a block ends the click ([#1797](https://github.com/ludovic111/horizon/issues/1797)), potions keep their potion over the connection ([#1796](https://github.com/ludovic111/horizon/issues/1796)), mobs' aggression and arm swings ([#1802](https://github.com/ludovic111/horizon/issues/1802)). [docs/server-levels.md](docs/server-levels.md). + +
+ +#### Play in your language + +Item names, container titles and server messages follow the language you pick, as in vanilla. Today parts of the game stay in English. + +
Technical + +Translatable components resolved through the selected language for items, containers and server text ([#1806](https://github.com/ludovic111/horizon/issues/1806)), on top of the bundled languages the prepared resources carry ([#1594](https://github.com/ludovic111/horizon/issues/1594)). Left: the client's own screens ([#1672](https://github.com/ludovic111/horizon/issues/1672)) and the main menu with zh_cn ([#1774](https://github.com/ludovic111/horizon/issues/1774)). + +
+ +### Rendering + +#### Shader packs, written again from scratch + +Shader packs were removed and are being rebuilt from scratch. They come back, off by default, once the best-known packs draw properly. + +
Technical + +A clean-room rewrite in `horizon-shaderpack`, from the packs' public format documentation only: reading packs, the preprocessor and `.properties` files are merged ([#1584](https://github.com/ludovic111/horizon/issues/1584), [#1585](https://github.com/ludovic111/horizon/issues/1585), [#1586](https://github.com/ludovic111/horizon/issues/1586)); translating programs to WGSL through naga, buffers, uniforms and the terrain, entity, sky, weather and hand stages are in review ([#1698](https://github.com/ludovic111/horizon/issues/1698) to [#1801](https://github.com/ludovic111/horizon/issues/1801)). + +More: [#1561](https://github.com/ludovic111/horizon/issues/1561) · [#1771](https://github.com/ludovic111/horizon/issues/1771) · [#1786](https://github.com/ludovic111/horizon/issues/1786) + +
+ +### Horizon extras + +#### Discord Rich Presence + +Your Discord profile can show that you are playing Horizon, with a switch to turn it off. + +
Technical + +A Discord IPC client on its own thread ([#1787](https://github.com/ludovic111/horizon/issues/1787)) and the presence with its setting ([#1788](https://github.com/ludovic111/horizon/issues/1788)). Merged: it ships with the next release. + +
+ +### Code and tools + +#### A leaner codebase that builds faster + +Cleaning up Horizon's code: removing what nothing uses and making it quicker to build, so fixes and new features land faster. + +
Technical + +A review of the whole tree: dead code and stubs removed ([#1432](https://github.com/ludovic111/horizon/issues/1432), [#1658](https://github.com/ludovic111/horizon/issues/1658)), one probe binary per crate ([#1453](https://github.com/ludovic111/horizon/issues/1453), [#1454](https://github.com/ludovic111/horizon/issues/1454)), shared harness helpers ([#1448](https://github.com/ludovic111/horizon/issues/1448)), clippy clean on the whole tree ([#1707](https://github.com/ludovic111/horizon/issues/1707)). [docs/architecture.md](docs/architecture.md) maps each feature to its files and harnesses. + +
+ +## Next + +### Performance + +#### Fresh benchmarks against vanilla + +New public numbers comparing Horizon with vanilla on many kinds of computers, measured the same way every time, on the website's Benchmarks page. + +
Technical + +Redo the comparison after the current performance work ([#1073](https://github.com/ludovic111/horizon/issues/1073)); reliability, coverage and presentation ([#1346](https://github.com/ludovic111/horizon/issues/1346)); a scene must count as loaded only once its first chunk arrives ([#1749](https://github.com/ludovic111/horizon/issues/1749)). Players' reports come in through `--benchmark` and `horizon-launcher data benchmark` ([docs/benchmark.md](docs/benchmark.md)). + +
+ +### Modpacks + +#### CurseForge in the mod browser (coming soon) + +Coming soon: search and download CurseForge mods and modpacks from the launcher, as you can with Modrinth today. + +
Technical + +The mod browser lists CurseForge as coming soon ([docs/mod-browser.md](docs/mod-browser.md)): it needs CurseForge API access. Meanwhile packs import from a CurseForge zip or instance folder, and jars only CurseForge hosts come from the player's own instance ([docs/modpacks.md](docs/modpacks.md)). + +
+ +### Gameplay parity + +#### Pistons push players and mobs + +Pistons and slime blocks will push players, mobs and items as in vanilla. Today moving blocks pass through them. + +
Technical + +Port `PistonMovingBlockEntity.moveCollidedEntities` and `moveStuckEntities` (honey), the slime bounce and `Entity.limitPistonMovement` for `MoverType.PISTON`, in single player and for a server's players. Issues [#1496](https://github.com/ludovic111/horizon/issues/1496), [#1609](https://github.com/ludovic111/horizon/issues/1609), [#1608](https://github.com/ludovic111/horizon/issues/1608), [#1704](https://github.com/ludovic111/horizon/issues/1704), [#1758](https://github.com/ludovic111/horizon/issues/1758). + +
+ +### Launcher + +#### Microsoft sign-in (coming soon) + +Coming soon: sign in with your Microsoft account to play on online-mode servers. The code is ready; it waits for Microsoft's approval. + +
Technical + +Device-code sign-in in the launcher and online-mode login in the client ([docs/microsoft-account.md](docs/microsoft-account.md)). It is turned on by setting the app's client id once Minecraft API access is approved; until then the launcher shows "Coming soon". + +More: [#1270](https://github.com/ludovic111/horizon/issues/1270) + +
+ +#### Horizon on Intel Macs + +Builds of the game and the launcher for Macs with an Intel processor. + +
Technical + +x86_64 game and launcher builds, picked per processor ([#1171](https://github.com/ludovic111/horizon/issues/1171)). Before it ships it needs a run on a real Intel Mac: Rosetta on Apple Silicon only proves the CPU code. Requests: [#1161](https://github.com/ludovic111/horizon/issues/1161), [#1792](https://github.com/ludovic111/horizon/issues/1792). + +
+ +## Later + +### Gameplay parity + +#### Following new Minecraft versions + +When Mojang releases a new version of the game, Horizon follows it, and every system is checked against the new version again. + +
Technical + +The port pipeline detects a release, diffs its classes against 26.3 and writes a port report ([docs/version-ports.md](docs/version-ports.md)); the harnesses then run against the new version. + +
+ +### Horizon extras + +#### An optional AI teammate + +An AI teammate that plays beside you and that you can talk to. It is off unless you turn it on, and the No AI switch hides it completely. + +
Technical + +Draft pull requests: the companion ([#1068](https://github.com/ludovic111/horizon/issues/1068)), its safety modes and their evaluation ([#1069](https://github.com/ludovic111/horizon/issues/1069)), and a route that finishes the game ([#1070](https://github.com/ludovic111/horizon/issues/1070)). Local models first; it ships only once the safety evaluation passes. + +
+ +## Help wanted + +### Performance + +#### Benchmark your computer + +Run Horizon's built-in benchmark and share the report: every new computer makes the performance work more accurate. No coding needed. + +
Technical + +`horizon-launcher play --arg=--benchmark --arg=standard` writes a JSON report; `horizon-launcher data benchmark ` shows what would be sent and uploads it ([website/docs/performance.md](website/docs/performance.md)). Results from three installs with the same hardware appear on the Benchmarks page. Example: [#1711](https://github.com/ludovic111/horizon/issues/1711). + +
+ +### Gameplay parity + +#### End city banners keep their pattern + +Banners in End cities show as plain magenta instead of with their pattern. + +
Technical + +Compare the banner block entities the End city templates place with vanilla's (`EndCityPieces`, the templates' block entity NBT, `BannerBlockEntity`'s patterns), in `crates/horizon-world/src/worldgen/structure/`. Issue [#1438](https://github.com/ludovic111/horizon/issues/1438); possibly related: [#526](https://github.com/ludovic111/horizon/issues/526). + +
+ +#### Enchanted loot in End cities + +Armour and tools in End city chests should come enchanted, as in vanilla. + +
Technical + +`loot.rs` implements `enchant_with_levels`: find why the End city chest table's rolls come out bare, and compare them with vanilla's for a few seeds. Issues [#1437](https://github.com/ludovic111/horizon/issues/1437), [#1660](https://github.com/ludovic111/horizon/issues/1660). + +
+ +#### Cooldowns drawn on items + +After you throw an ender pearl, its slot should show the cooldown sweeping away, as in vanilla. + +
Technical + +Vanilla draws `ItemCooldowns`' percent as a white overlay over the slot (`GuiGraphicsExtractor.itemDecorations` and `itemCooldown`) in the hotbar and in inventories. Issue [#1763](https://github.com/ludovic111/horizon/issues/1763). + +
+ +#### Chat settings and team prefixes + +The Hidden and Commands Only chat settings should hide chat, and team prefixes should appear before names in chat. + +
Technical + +Vanilla checks `ChatVisiblity` before it delivers chat and system messages ([#1376](https://github.com/ludovic111/horizon/issues/1376)), and chat uses the team-formatted display name ([#1378](https://github.com/ludovic111/horizon/issues/1378)). Both issues have exact steps to reproduce. + +
+ +#### A pointing hand over buttons + +The mouse pointer should turn into a pointing hand over menu buttons, as in vanilla 26.3. + +
Technical + +26.3's `AbstractWidget` asks for a cursor from `CursorTypes` when it is hovered; winit's `CursorIcon::Pointer` gives the same shape. Issue [#1347](https://github.com/ludovic111/horizon/issues/1347). + +
+ +#### Ctrl+Backspace and scroll bars in menus + +Ctrl+Backspace should delete a whole word in the creative search, and every list's scroll bar should be draggable. + +
Technical + +`EditBox` deletes by word with Ctrl ([#1764](https://github.com/ludovic111/horizon/issues/1764)); the Game Rules and Statistics lists' scroll bars can't be dragged ([#1782](https://github.com/ludovic111/horizon/issues/1782), after [#978](https://github.com/ludovic111/horizon/issues/978)). + +
+ +#### Chickens flap while they fall + +Chickens should flap their wings while they fall, not stand still in the air. + +
Technical + +Vanilla's `Chicken.aiStep` updates `flap` and `flapSpeed` when the chicken isn't on the ground, and the model's wings follow them. Issue [#1706](https://github.com/ludovic111/horizon/issues/1706). + +
+ +## Recently shipped + +- **Server resource packs** (v0.2.48): Servers can send you their resource pack: Horizon asks, downloads it and applies it, like vanilla. [#1562](https://github.com/ludovic111/horizon/issues/1562) · [#1569](https://github.com/ludovic111/horizon/issues/1569) · [#1570](https://github.com/ludovic111/horizon/issues/1570) +- **Single player on its own server thread** (v0.2.39): Single player runs on a server thread, as vanilla does, so the world keeps ticking smoothly while the window draws. [#1514](https://github.com/ludovic111/horizon/issues/1514) · [docs/server-levels.md](docs/server-levels.md) +- **GPU occlusion culling on Macs** (v0.2.17): On Macs the GPU now skips terrain hidden behind hills and caves: 8 to 48% more frames per second at 32 chunks on an M4 Pro. [#1434](https://github.com/ludovic111/horizon/issues/1434) +- **Animated textures on the GPU** (v0.2.5): Water, lava and other animated blocks are now animated by the graphics card: 1.7 to 2.4 times better 1% lows on Intel graphics. [#1239](https://github.com/ludovic111/horizon/issues/1239) +- **Far Detail for integrated graphics** (v0.2.5): An optional Reduced setting for far terrain, suggested on integrated graphics, that saves 2 to 4 ms of GPU time per frame. [#1199](https://github.com/ludovic111/horizon/issues/1199) · [#1029](https://github.com/ludovic111/horizon/issues/1029) +- **The launcher's mod browser** (launcher@0.2.0): Search Modrinth for mods, modpacks and resource packs from the launcher; it says before installing what works in Horizon. [#1154](https://github.com/ludovic111/horizon/issues/1154) · [#1242](https://github.com/ludovic111/horizon/issues/1242) · [#1243](https://github.com/ludovic111/horizon/issues/1243) · [#1246](https://github.com/ludovic111/horizon/issues/1246) · [docs/mod-browser.md](docs/mod-browser.md) +- **A blocky, Minecraft-like launcher** (launcher@0.2.0): The launcher got a new blocky look in Horizon's purple, in dark and light. [#1269](https://github.com/ludovic111/horizon/issues/1269) · [docs/design-system.md](docs/design-system.md) +- **Dynamic lights** (v2026.10.22): An optional extra: torches you hold, drop or set on fire light up what is around them. [#1162](https://github.com/ludovic111/horizon/issues/1162) +- **Host a server from the launcher** (launcher-v0.15.0): The launcher's Servers page creates, starts, backs up and updates a Horizon server with buttons, and invites friends. [#1026](https://github.com/ludovic111/horizon/issues/1026) · [docs/hosting.md](docs/hosting.md) diff --git a/crates/cargo-horizon-mod/src/check.rs b/crates/cargo-horizon-mod/src/check.rs index 0294bec98..8a25d1a24 100644 --- a/crates/cargo-horizon-mod/src/check.rs +++ b/crates/cargo-horizon-mod/src/check.rs @@ -21,6 +21,8 @@ pub struct Report { pub sha256: String, pub size: u64, pub signed: bool, + /// A mod has code to run (a content mod may have none). + pub code: bool, /// A mod's component imports. pub imports: Vec, /// A mod's commands, each run once. @@ -49,6 +51,7 @@ impl Report { "sha256": self.sha256, "size": self.size, "signed": self.signed, + "code": self.code, "imports": self.imports, "commands": self.commands, "output": self.output, @@ -87,7 +90,9 @@ impl Report { for r in &f.requires { out += &format!(" needs {} {} ({})\n", r.id, r.version, &r.sha256[..12]); } - if self.kind == Kind::Mod { + if self.kind == Kind::Mod && !self.code { + out += " no code: content only, nothing to run\n"; + } else if self.kind == Kind::Mod { let commands = if self.commands.is_empty() { String::new() } else { @@ -142,6 +147,7 @@ pub fn check( sha256: index::sha256_file(path)?, size, signed, + code: false, imports: Vec::new(), commands: Vec::new(), output: Vec::new(), @@ -153,10 +159,14 @@ pub fn check( } /// Runs a mod in the sandbox, as the game does, from a scratch copy so its -/// storage lands in a folder removed afterwards. +/// storage lands in a folder removed afterwards. A content mod has no code to +/// run: `inspect` already read its content as the game does. fn trial_run(path: &Path, report: &mut Report) -> Result<(), String> { let package = Package::read(path)?; - let wasm = package.wasm.as_deref().ok_or("the .hmod has no mod.wasm")?; + let Some(wasm) = package.wasm.as_deref() else { + return Ok(()); + }; + report.code = true; report.imports = component_imports(wasm)?; package .manifest diff --git a/crates/cargo-horizon-mod/src/main.rs b/crates/cargo-horizon-mod/src/main.rs index 925633930..a4585f680 100644 --- a/crates/cargo-horizon-mod/src/main.rs +++ b/crates/cargo-horizon-mod/src/main.rs @@ -1,14 +1,16 @@ //! `cargo horizon-mod`: make Horizon mods in Rust (`docs/mods.md`). //! -//! - `new [--name "Name"] [--sdk-path ]`: a mod crate from the template. -//! - `build [--debug]`: the `.hmod` in `target/horizon-mod/`. +//! - `new [--template code|content] [--name "Name"] [--sdk-path ]`: +//! a mod from a template: a Rust crate, or a content mod (no code). +//! - `build [--debug]`: the `.hmod` in `target/horizon-mod/`, with the mod's +//! `assets/` and `data/`; a folder without `Cargo.toml` is content only. //! - `run [--game ] [-- ]`: build, then play with //! the mod; run `build` again and the game reloads it. //! - `check [--json]`: what the registry checks of a submission //! (`.hmod`, `.hpack`, resource pack `.zip`), with a trial run for a mod. //! - `registry build|review|check|list`: the registry repository's CI. use horizon_mods::{ - Manifest, Package, + Capability, Content, Manifest, Package, manifest::{MANIFEST, component_imports, componentize}, }; mod check; @@ -36,7 +38,7 @@ fn main() -> ExitCode { Some("run") => run(&args[1..]), Some("check") => check(&args[1..]), Some("registry") => registry::main(&args[1..]), - _ => Err("usage: cargo horizon-mod new | build [--debug] | run [--game ] [-- ] | check [--kind ] [--key --signature ] [--json] | registry build|review|check|list".into()), + _ => Err("usage: cargo horizon-mod new [--template code|content] | build [--debug] | run [--game ] [-- ] | check [--kind ] [--key --signature ] [--json] | registry build|review|check|list".into()), }; match result { Ok(()) => ExitCode::SUCCESS, @@ -90,19 +92,68 @@ fn new(args: &[String]) -> Result<(), String> { .replace("{name}", &name) .replace("{sdk}", &sdk) }; - let files = [ + let code = [ ("Cargo.toml", include_str!("../template/Cargo.toml.in")), (MANIFEST, include_str!("../template/horizon-mod.toml")), ("src/lib.rs", include_str!("../template/src/lib.rs")), (".gitignore", include_str!("../template/gitignore")), ]; - for (file, text) in files { + // A block copying a vanilla one, with its model, name, loot, recipe and tag. + let content = [ + ( + MANIFEST, + include_str!("../template/content/horizon-mod.toml"), + ), + ( + "data/{id}/horizon/block/glowing_amethyst.json", + include_str!("../template/content/block.json"), + ), + ( + "assets/{id}/blockstates/glowing_amethyst.json", + include_str!("../template/content/blockstate.json"), + ), + ( + "assets/{id}/models/block/glowing_amethyst.json", + include_str!("../template/content/model.json"), + ), + ( + "assets/{id}/items/glowing_amethyst.json", + include_str!("../template/content/item.json"), + ), + ( + "assets/{id}/lang/en_us.json", + include_str!("../template/content/lang.json"), + ), + ( + "data/{id}/loot_table/blocks/glowing_amethyst.json", + include_str!("../template/content/loot.json"), + ), + ( + "data/{id}/recipe/glowing_amethyst.json", + include_str!("../template/content/recipe.json"), + ), + ( + "data/minecraft/tags/block/mineable/pickaxe.json", + include_str!("../template/content/pickaxe.json"), + ), + (".gitignore", include_str!("../template/gitignore")), + ]; + let files: &[(&str, &str)] = match flag(args, "--template").unwrap_or("code") { + "code" => &code, + "content" => &content, + other => return Err(format!("no template {other}: code or content")), + }; + for &(file, text) in files { + let file = fill(file); + let file = file.as_str(); let to = dir.join(file); std::fs::create_dir_all(to.parent().expect("a file in the mod")) .map_err(|e| e.to_string())?; std::fs::write(&to, fill(text)).map_err(|e| format!("{}: {e}", to.display()))?; } - Manifest::parse(&std::fs::read_to_string(dir.join(MANIFEST)).map_err(|e| e.to_string())?)?; + let manifest = + Manifest::parse(&std::fs::read_to_string(dir.join(MANIFEST)).map_err(|e| e.to_string())?)?; + Content::read(dir, &manifest)?; println!( "Created {} (mod id {id}). Next: cd {path} && cargo horizon-mod run", dir.display() @@ -110,14 +161,53 @@ fn new(args: &[String]) -> Result<(), String> { Ok(()) } -/// Builds the crate in the current folder and packs `target/horizon-mod/.hmod`. +/// Builds the mod in the current folder and packs `target/horizon-mod/.hmod`: +/// the manifest, the crate's component (none without a `Cargo.toml`: a +/// content mod) and the files in `assets/` and `data/`. fn build(release: bool) -> Result { - let manifest_text = std::fs::read_to_string(MANIFEST) + build_in(Path::new("."), release) +} + +fn build_in(root: &Path, release: bool) -> Result { + let manifest_text = std::fs::read_to_string(root.join(MANIFEST)) .map_err(|e| format!("{MANIFEST}: {e} (run this in a mod's folder)"))?; let manifest = Manifest::parse(&manifest_text)?; + let (wasm, target_dir) = if root.join("Cargo.toml").exists() { + let (wasm, target) = compile(root, release)?; + // The game refuses a component importing capabilities the manifest lacks. + manifest.check_imports(component_imports(&wasm)?.iter().map(String::as_str))?; + (Some(wasm), target) + } else if manifest.capabilities.contains(&Capability::Registries) { + (None, root.join("target")) + } else { + return Err(format!( + "no Cargo.toml: only a content mod (capability registries in {MANIFEST}) has no code" + )); + }; + // The same checks as the game's, and the files in the order it lists them. + let content = Content::read(root, &manifest)?; + let mut reader = content.reader()?; + let mut files = Vec::with_capacity(content.files.len()); + for name in &content.files { + files.push((name.as_str(), reader.read(name)?)); + } + let files: Vec<(&str, &[u8])> = files.iter().map(|(n, b)| (*n, b.as_slice())).collect(); + let dir = target_dir.join("horizon-mod"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + let path = dir.join(format!("{}.hmod", manifest.id)); + // Write then rename, so a running game never reads half a file. + let partial = path.with_extension("hmod.partial"); + let bytes = Package::write_files(&manifest_text, wasm.as_deref(), &files)?; + std::fs::write(&partial, bytes).map_err(|e| e.to_string())?; + std::fs::rename(&partial, &path).map_err(|e| e.to_string())?; + Ok(path) +} + +/// Builds the crate for WebAssembly: its component, and Cargo's target folder. +fn compile(root: &Path, release: bool) -> Result<(Vec, PathBuf), String> { let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); let mut cmd = Command::new(cargo); - cmd.args([ + cmd.current_dir(root).args([ "build", "--lib", "--target", @@ -145,17 +235,8 @@ fn build(release: bool) -> Result { .rfind(|f| f.extension().is_some_and(|e| e == "wasm")) .ok_or("the build made no .wasm: is the crate a cdylib?")?; let wasm = componentize(&std::fs::read(&core).map_err(|e| e.to_string())?)?; - // The game refuses a component importing capabilities the manifest lacks. - manifest.check_imports(component_imports(&wasm)?.iter().map(String::as_str))?; - let target_dir = core.ancestors().nth(3).ok_or("unexpected target folder")?; - let dir = target_dir.join("horizon-mod"); - std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; - let path = dir.join(format!("{}.hmod", manifest.id)); - // Write then rename, so a running game never reads half a file. - let partial = path.with_extension("hmod.partial"); - std::fs::write(&partial, Package::write(&manifest_text, &wasm)?).map_err(|e| e.to_string())?; - std::fs::rename(&partial, &path).map_err(|e| e.to_string())?; - Ok(path) + let target = core.ancestors().nth(3).ok_or("unexpected target folder")?; + Ok((wasm, target.to_path_buf())) } /// `check `: the registry's checks of a submission, printed; fails @@ -237,4 +318,49 @@ mod tests { ); std::fs::remove_dir_all(&dir).unwrap(); } + + #[test] + fn the_content_template_builds_and_checks_with_no_code() { + let dir = std::env::temp_dir().join(format!("horizon-mod-content-{}", std::process::id())); + let path = dir.join("gems"); + let new_args = [ + path.display().to_string(), + "--template".into(), + "content".into(), + ]; + new(&new_args).unwrap(); + assert!(!path.join("Cargo.toml").exists()); + let hmod = build_in(&path, true).unwrap(); + assert_eq!(hmod, path.join("target/horizon-mod/gems.hmod")); + let package = Package::read(&hmod).unwrap(); + assert!(package.wasm.is_none()); + let content = Content::read(&hmod, &package.manifest).unwrap(); + assert!( + content + .files + .contains(&"data/gems/horizon/block/glowing_amethyst.json".into()) + ); + assert_eq!(content.files.len(), 8, "{:?}", content.files); + let report = check::check(&hmod, check::kind_of(&hmod).unwrap(), None, None).unwrap(); + assert!(!report.code); + assert!( + report.text().contains("no code: content only"), + "{}", + report.text() + ); + // Without registries, a mod needs code. + let manifest = std::fs::read_to_string(path.join(MANIFEST)).unwrap(); + let manifest = manifest.replace("[\"registries\"]", "[]"); + std::fs::write(path.join(MANIFEST), manifest).unwrap(); + assert!(build_in(&path, true).unwrap_err().contains("no Cargo.toml")); + assert!( + new(&[ + dir.join("x").display().to_string(), + "--template".into(), + "lua".into() + ]) + .is_err() + ); + std::fs::remove_dir_all(&dir).unwrap(); + } } diff --git a/crates/cargo-horizon-mod/template/content/block.json b/crates/cargo-horizon-mod/template/content/block.json new file mode 100644 index 000000000..ea68a1a27 --- /dev/null +++ b/crates/cargo-horizon-mod/template/content/block.json @@ -0,0 +1,4 @@ +{ + "template": "minecraft:amethyst_block", + "properties": { "hardness": 2.0, "light": 7 } +} diff --git a/crates/cargo-horizon-mod/template/content/blockstate.json b/crates/cargo-horizon-mod/template/content/blockstate.json new file mode 100644 index 000000000..ea472666e --- /dev/null +++ b/crates/cargo-horizon-mod/template/content/blockstate.json @@ -0,0 +1,5 @@ +{ + "variants": { + "": { "model": "{id}:block/glowing_amethyst" } + } +} diff --git a/crates/cargo-horizon-mod/template/content/horizon-mod.toml b/crates/cargo-horizon-mod/template/content/horizon-mod.toml new file mode 100644 index 000000000..4f133a905 --- /dev/null +++ b/crates/cargo-horizon-mod/template/content/horizon-mod.toml @@ -0,0 +1,10 @@ +id = "{id}" +name = "{name}" +version = "0.1.0" +description = "" +authors = [] +# A content mod: blocks and items from data/ and assets/, no code (API 0.2). +api = "0.2" +side = "both" +# registries: "May add blocks and items to your worlds". +capabilities = ["registries"] diff --git a/crates/cargo-horizon-mod/template/content/item.json b/crates/cargo-horizon-mod/template/content/item.json new file mode 100644 index 000000000..411bc4499 --- /dev/null +++ b/crates/cargo-horizon-mod/template/content/item.json @@ -0,0 +1,3 @@ +{ + "model": { "type": "minecraft:model", "model": "{id}:block/glowing_amethyst" } +} diff --git a/crates/cargo-horizon-mod/template/content/lang.json b/crates/cargo-horizon-mod/template/content/lang.json new file mode 100644 index 000000000..74baddab8 --- /dev/null +++ b/crates/cargo-horizon-mod/template/content/lang.json @@ -0,0 +1,3 @@ +{ + "block.{id}.glowing_amethyst": "Glowing Amethyst" +} diff --git a/crates/cargo-horizon-mod/template/content/loot.json b/crates/cargo-horizon-mod/template/content/loot.json new file mode 100644 index 000000000..898111f1b --- /dev/null +++ b/crates/cargo-horizon-mod/template/content/loot.json @@ -0,0 +1,11 @@ +{ + "type": "minecraft:block", + "pools": [ + { + "condition": { "type": "minecraft:survives_explosion" }, + "entries": [{ "type": "minecraft:item", "name": "{id}:glowing_amethyst" }], + "rolls": 1 + } + ], + "random_sequence": "{id}:blocks/glowing_amethyst" +} diff --git a/crates/cargo-horizon-mod/template/content/model.json b/crates/cargo-horizon-mod/template/content/model.json new file mode 100644 index 000000000..5710fecb1 --- /dev/null +++ b/crates/cargo-horizon-mod/template/content/model.json @@ -0,0 +1,4 @@ +{ + "parent": "minecraft:block/cube_all", + "textures": { "all": "minecraft:block/amethyst_block" } +} diff --git a/crates/cargo-horizon-mod/template/content/pickaxe.json b/crates/cargo-horizon-mod/template/content/pickaxe.json new file mode 100644 index 000000000..8725edebd --- /dev/null +++ b/crates/cargo-horizon-mod/template/content/pickaxe.json @@ -0,0 +1,3 @@ +{ + "values": ["{id}:glowing_amethyst"] +} diff --git a/crates/cargo-horizon-mod/template/content/recipe.json b/crates/cargo-horizon-mod/template/content/recipe.json new file mode 100644 index 000000000..5edb03d94 --- /dev/null +++ b/crates/cargo-horizon-mod/template/content/recipe.json @@ -0,0 +1,6 @@ +{ + "type": "minecraft:crafting_shapeless", + "category": "building", + "ingredients": ["minecraft:amethyst_block", "minecraft:glowstone_dust"], + "result": { "id": "{id}:glowing_amethyst" } +} diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index 699468651..7fcb3be89 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -12098,6 +12098,8 @@ enum PendingWorldEntry { Server { address: String, name: String, + /// A transfer's carried-over server cookies (`TransferState`). + cookies: Option, }, } @@ -12172,8 +12174,12 @@ impl Game<'_> { }) => { self.open_world(&folder, name.as_deref(), seed, creative, create.as_deref())?; } - Some(PendingWorldEntry::Server { address, name }) => { - self.connect_server(&address, &name)?; + Some(PendingWorldEntry::Server { + address, + name, + cookies, + }) => { + self.connect_server_with_cookies(&address, &name, cookies)?; } None => {} } diff --git a/crates/horizon-client/src/game/agent.rs b/crates/horizon-client/src/game/agent.rs index f6b68aa74..57c12da97 100644 --- a/crates/horizon-client/src/game/agent.rs +++ b/crates/horizon-client/src/game/agent.rs @@ -469,7 +469,8 @@ impl Game<'_> { "flying": p.flying, "fall_flying": p.fall_flying, "sneaking": self.keys.contains(&KeyCode::ShiftLeft), - "sprinting": self.keys.contains(&KeyCode::ControlLeft), + "sprinting": p.sprinting, + "fov_modifier": self.view.fov_modifier, "health": p.health.health(), "absorption": p.health.absorption(), // The hurt animation's ticks left (the camera's hurt tilt). diff --git a/crates/horizon-client/src/game/remote.rs b/crates/horizon-client/src/game/remote.rs index bc14b5263..620d145bf 100644 --- a/crates/horizon-client/src/game/remote.rs +++ b/crates/horizon-client/src/game/remote.rs @@ -193,6 +193,9 @@ pub(super) struct Remote { /// the oldest of them first waited (`poll_remote`). backlog: VecDeque, backlog_since: Option, + /// `ClientboundTransferPacket`'s target: when the connection ends the + /// game connects here, the stored cookies carried over (`TransferState`). + transfer: Option, } /// The time a frame spends on chunk packets before it leaves the rest of @@ -257,14 +260,25 @@ fn number_format(f: &play::NumberFormat) -> horizon_client::scoreboard::NumberFo impl Game<'_> { /// Leave the title for a server (ConnectScreen): an empty world fed by the connection. pub(super) fn connect_server(&mut self, address: &str, name: &str) -> Result<()> { + self.connect_server_with_cookies(address, name, None) + } + /// `connect_server` carrying a transfer's stored server cookies + /// (`TransferState`); None on an ordinary join. + pub(super) fn connect_server_with_cookies( + &mut self, + address: &str, + name: &str, + cookies: Option, + ) -> Result<()> { if self.defer_world_entry(super::PendingWorldEntry::Server { address: address.to_owned(), name: name.to_owned(), + cookies: cookies.clone(), })? { return Ok(()); } self.join_world_rest()?; - if let Err(error) = self.join_server(address, name, None) { + if let Err(error) = self.join_server(address, name, None, cookies) { log::warn!("Horizon: could not join server: {error}"); if self.title.is_none() { self.title = Some(horizon_client::title::Title::new( @@ -285,13 +299,14 @@ impl Game<'_> { pipe: horizon_protocol::connection::Pipe, name: &str, ) -> Result<()> { - self.join_server("local", name, Some(pipe)) + self.join_server("local", name, Some(pipe), None) } fn join_server( &mut self, address: &str, name: &str, local: Option, + cookies: Option, ) -> Result<()> { self.perf.world_opening(); let root = self.options.resources.clone(); @@ -323,7 +338,17 @@ impl Game<'_> { Some(pipe) => Session::connect_local(pipe, name, view, lang.clone()), None => { self.extras_connecting(address, name); - Session::connect(address, name, auth.clone(), view, lang.clone()) + match cookies { + Some(cookies) => Session::connect_transferred( + address, + name, + auth.clone(), + view, + lang.clone(), + cookies, + ), + None => Session::connect(address, name, auth.clone(), view, lang.clone()), + } } }; let player_type = super::perspective::player_type(&root); @@ -446,6 +471,7 @@ impl Game<'_> { chunk_center: (0, 0), backlog: VecDeque::new(), backlog_since: None, + transfer: None, }); // The level's TickRateManager follows the server's ticking packets. self.clock = horizon_client::clock::FixedStep::client(); @@ -867,6 +893,15 @@ impl Game<'_> { } } Event::Reconfigured(registries) => self.set_registries(registries), + // handleTransfer: "Cannot transfer to server from + // singleplayer" (no ServerData), so the world just closes. + Event::Transfer { host, port } => { + if let Some(r) = self.remote.as_mut() + && r.session.address != "local" + { + r.transfer = Some(format!("{host}:{port}")); + } + } Event::PackPush(p) => self.handle_pack_push(p), Event::PackPop(id) => self.handle_pack_pop(id), Event::DecodeError(e) => { @@ -882,6 +917,20 @@ impl Game<'_> { from_server, } => { log::info!("Horizon: disconnected: {reason}"); + // ClientboundTransferPacket left a target: connect there + // instead of showing the disconnect, the stored cookies + // carried over (ClientCommonPacketListenerImpl. + // handleTransfer → ConnectScreen.startConnecting). + let transfer = self.remote.as_mut().and_then(|r| { + r.transfer.take().map(|address| { + (address, r.session.name.clone(), r.session.cookies.clone()) + }) + }); + if let Some((address, name, cookies)) = transfer { + self.leave_server(None)?; + self.connect_server_with_cookies(&address, &name, Some(cookies))?; + return Ok(()); + } let retry = self.reconnect_after(&reason, from_server); self.leave_server(Some(reason))?; self.offer_reconnect(retry); @@ -1015,6 +1064,9 @@ impl Game<'_> { r.stepped.clear(); r.head_lerps.clear(); r.pending_items.clear(); + // The entity ids are the old world's; the new dimension's + // player entities re-map to their profiles on AddEntity. + r.level_players.clear(); // A new ClientLevel: a new chunk storage, centred on 0, 0. r.chunk_center = (0, 0); } @@ -4796,6 +4848,11 @@ impl Game<'_> { self.mount_menu_reset(); // The client's menu has no level access: what it holds is the server's. self.station = None; + // AbstractContainerMenu.removed: the server gives the grid and the + // carried stack back to the inventory and syncs it; the copies here + // are stale the moment the window closes. + self.craft_grid.fill(None); + self.carried = None; self.craft_size = 2; if let Some(key) = self.container.take() { // Server containers stay renderable but lose their copied slots. diff --git a/crates/horizon-client/src/game/startup_menu.rs b/crates/horizon-client/src/game/startup_menu.rs index 96af99bbc..fe568fe71 100644 --- a/crates/horizon-client/src/game/startup_menu.rs +++ b/crates/horizon-client/src/game/startup_menu.rs @@ -33,6 +33,7 @@ impl StartupMenu { Some(PendingWorldEntry::Server { address: address.clone(), name: options.player_name(), + cookies: None, }) } else { options @@ -125,6 +126,7 @@ impl StartupMenu { Action::Connect { address } => PendingWorldEntry::Server { address, name: player_name, + cookies: None, }, _ => return, }; @@ -762,7 +764,7 @@ mod tests { "Guest".into(), ); assert!( - matches!(menu.pending.take(), Some(PendingWorldEntry::Server { address, name }) + matches!(menu.pending.take(), Some(PendingWorldEntry::Server { address, name, .. }) if address == "localhost:25565" && name == "Guest") ); assert!(menu.pending.take().is_none(), "an entry is consumed once"); diff --git a/crates/horizon-client/src/hud.rs b/crates/horizon-client/src/hud.rs index 89d1b1c3c..617126c2e 100644 --- a/crates/horizon-client/src/hud.rs +++ b/crates/horizon-client/src/hud.rs @@ -78,7 +78,10 @@ pub const MAP_SLOTS: usize = 256; /// The atlas slot the minimap extra draws into. pub const MINIMAP_SPRITE: &str = "horizon/minimap"; /// Atlas slots for downloaded player skins (`skins/`), 64x64 each. -pub const SKIN_SLOTS: usize = 32; +/// Vanilla registers a texture per skin. Here a skin evicted from its slot +/// is not stored again, and its player falls back to the default skin, so +/// the slots must cover a busy server's player list. +pub const SKIN_SLOTS: usize = 128; /// Reused by the visible rows of the classic world selector. pub const WORLD_ICON_SLOTS: usize = 32; /// Atlas slots for the pack icons on the pack screen, one per icon drawn diff --git a/crates/horizon-client/src/lan/hub.rs b/crates/horizon-client/src/lan/hub.rs index c2a395899..dd0a32f22 100644 --- a/crates/horizon-client/src/lan/hub.rs +++ b/crates/horizon-client/src/lan/hub.rs @@ -1984,7 +1984,9 @@ impl Hub { S::PlayerInput(p) => { self.guests[i].input = p.flags; self.guests[i].sneaking = p.flags & 32 != 0; - self.guests[i].sprinting = p.flags & 64 != 0 && p.flags & 1 != 0; + // handlePlayerInput only records keys and updates crouching. + // Sprinting persists after the sprint key is released (or + // starts with a double tap); PlayerCommand changes it. } // OPEN_INVENTORY (the ridden mount's inventory) and // START_FALL_FLYING (checked against the guest's chest slot diff --git a/crates/horizon-client/src/net.rs b/crates/horizon-client/src/net.rs index 9cce9e1c6..18aec0bea 100644 --- a/crates/horizon-client/src/net.rs +++ b/crates/horizon-client/src/net.rs @@ -18,6 +18,7 @@ use horizon_protocol::{ text::{Language, plain, translatable}, }; use std::{ + collections::HashMap, sync::{ Arc, Mutex, atomic::{AtomicBool, Ordering}, @@ -27,6 +28,12 @@ use std::{ time::Duration, }; +/// `CommonListenerCookie.serverCookies`: cookies the server asked the client +/// to store (`ClientboundStoreCookiePacket`), answered to +/// `ClientboundCookieRequestPacket`. A `transfer` carries them to the next +/// connection (`TransferState`), so a proxy's backend switch keeps them. +pub type Cookies = Arc>>>; + /// What the network thread reports to the game. pub enum Event { /// Progress text for the connecting screen. @@ -48,6 +55,13 @@ pub enum Event { PackPush(cfg::ResourcePackPush), /// ClientboundResourcePackPopPacket: one pack, or every pack. PackPop(Option), + /// `ClientboundTransferPacket`: the server sent us to another address. + /// The connection ends right after; the game reconnects there with the + /// stored cookies. + Transfer { + host: String, + port: i32, + }, /// The connection ended. `from_server`: the server gave the reason (or /// the connection was refused for good), rather than the connection /// failing (auto-reconnect retries only failures and shutdowns). @@ -67,6 +81,8 @@ pub struct Session { pub address: String, pub name: String, pub counters: Arc>>>, + /// The cookies this server stored on us (`ClientboundStoreCookiePacket`). + pub cookies: Cookies, /// HORIZON_NET_DEBUG=1 logs every packet sent from the game thread. debug: bool, } @@ -79,7 +95,27 @@ impl Session { view_distance: i8, lang: Arc, ) -> Self { - Self::start(address, name, auth, view_distance, lang, None) + Self::start(address, name, auth, view_distance, lang, None, None) + } + /// `ConnectScreen.startConnecting` for a `transfer`: the cookies the + /// server stored carry to the new connection (`TransferState`). + pub fn connect_transferred( + address: &str, + name: &str, + auth: Option>, + view_distance: i8, + lang: Arc, + cookies: Cookies, + ) -> Self { + Self::start( + address, + name, + auth, + view_distance, + lang, + None, + Some(cookies), + ) } /// Join an integrated server through memory (`Connection.connectToLocalServer`). pub fn connect_local( @@ -88,7 +124,7 @@ impl Session { view_distance: i8, lang: Arc, ) -> Self { - Self::start("local", name, None, view_distance, lang, Some(pipe)) + Self::start("local", name, None, view_distance, lang, Some(pipe), None) } fn start( address: &str, @@ -97,12 +133,18 @@ impl Session { view_distance: i8, lang: Arc, local: Option, + cookies: Option, ) -> Self { let (send, events) = channel(); let writer = Arc::new(Mutex::new(None)); let configuring = Arc::new(AtomicBool::new(true)); let counters = Arc::new(Mutex::new(None)); - let (thread_writer, thread_counters) = (writer.clone(), counters.clone()); + // Connection.initiateServerboundPlayConnection: a transfer's + // handshake says so (ClientIntent.TRANSFER). + let transferred = cookies.is_some(); + let cookies = cookies.unwrap_or_default(); + let (thread_writer, thread_counters, thread_cookies) = + (writer.clone(), counters.clone(), cookies.clone()); let thread_configuring = configuring.clone(); let (address_owned, name_owned) = (address.to_owned(), name.to_owned()); let thread = std::thread::Builder::new() @@ -119,6 +161,8 @@ impl Session { &thread_writer, &thread_configuring, &thread_counters, + &thread_cookies, + transferred, ); let from_server = result.is_ok(); let reason = result.unwrap_or_else(|e| e); @@ -140,6 +184,7 @@ impl Session { address: address.to_owned(), name: name.to_owned(), counters, + cookies, debug, } } @@ -291,6 +336,8 @@ fn run( shared_writer: &Mutex>, configuring: &AtomicBool, shared_counters: &Mutex>>, + cookies: &Cookies, + transferred: bool, ) -> Result { let (host, port, mut c) = if let Some(pipe) = local { (address.to_owned(), 0, Connection::local(pipe)) @@ -321,7 +368,11 @@ fn run( protocol: PROTOCOL_VERSION, host: host.clone(), port, - intent: handshake::LOGIN, + intent: if transferred { + handshake::TRANSFER + } else { + handshake::LOGIN + }, }) .encode(), ) @@ -391,11 +442,14 @@ fn run( ) .map_err(e)?; } + // ClientHandshakePacketListenerImpl.handleRequestCookie: the + // cookies a transfer carried over (none on a plain join). login::Clientbound::CookieRequest(q) => { + let payload = cookies.lock().unwrap().get(&q.key).cloned(); c.send( &login::Serverbound::CookieResponse(login::CookieResponse { key: q.key, - payload: None, + payload, }) .encode(), ) @@ -419,13 +473,13 @@ fn run( .map_err(e)?; let (mut reader, writer) = c.split(); *shared_writer.lock().unwrap() = Some(writer); - let registries = configure(&mut reader, shared_writer, configuring, lang, send)?; + let registries = configure(&mut reader, shared_writer, configuring, lang, send, cookies)?; let _ = send.send(Event::Joined { registries: Arc::new(registries), uuid: profile.uuid, name: profile.name, }); - receive_play(&mut reader, shared_writer, configuring, lang, send) + receive_play(&mut reader, shared_writer, configuring, lang, send, cookies) } fn receive_play( @@ -434,6 +488,7 @@ fn receive_play( configuring: &AtomicBool, lang: &Language, send: &Sender, + cookies: &Cookies, ) -> Result { loop { let (id, payload) = reader.recv().map_err(|e| e.to_string())?; @@ -469,9 +524,31 @@ fn receive_play( let _ = w.send(&play::Serverbound::ConfigurationAcknowledged(Empty).encode()); configuring.store(true, Ordering::Relaxed); } - let registries = configure(reader, shared_writer, configuring, lang, send)?; + let registries = + configure(reader, shared_writer, configuring, lang, send, cookies)?; let _ = send.send(Event::Reconfigured(Arc::new(registries))); } + // ClientCommonPacketListenerImpl.handleRequestCookie: the stored + // cookie (or none) goes straight back. + play::Clientbound::CookieRequest(q) => { + let payload = cookies.lock().unwrap().get(&q.key).cloned(); + reply(play::Serverbound::CookieResponse(cfg::CookieResponse { + key: q.key, + payload, + })); + } + play::Clientbound::StoreCookie(p) => { + cookies.lock().unwrap().insert(p.key, p.payload); + } + // ClientCommonPacketListenerImpl.handleTransfer: the connection + // ends and the game connects to the other address. + play::Clientbound::Transfer(p) => { + let _ = send.send(Event::Transfer { + host: p.host, + port: p.port, + }); + return Ok(tr(lang, "disconnect.transfer")); + } play::Clientbound::Disconnect(d) => return Ok(plain(&d.reason, lang)), play::Clientbound::CustomPayload(p) if p.channel == "minecraft:brand" => { receive_brand(&p, send); @@ -493,6 +570,7 @@ fn configure( configuring: &AtomicBool, lang: &Language, send: &Sender, + cookies: &Cookies, ) -> Result { let mut registries = Dynamic::default(); let reply = |p: cfg::Serverbound| -> Result<(), String> { @@ -525,11 +603,22 @@ fn configure( let _ = send.send(Event::PackPop(p.id)); } cfg::Clientbound::CookieRequest(q) => { + let payload = cookies.lock().unwrap().get(&q.key).cloned(); reply(cfg::Serverbound::CookieResponse(cfg::CookieResponse { key: q.key, - payload: None, + payload, }))? } + cfg::Clientbound::StoreCookie(p) => { + cookies.lock().unwrap().insert(p.key, p.payload); + } + cfg::Clientbound::Transfer(p) => { + let _ = send.send(Event::Transfer { + host: p.host, + port: p.port, + }); + return Err(tr(lang, "disconnect.transfer")); + } cfg::Clientbound::Disconnect(d) => return Err(plain(&d.reason, lang)), cfg::Clientbound::FinishConfiguration(_) => { let mut w = writer.lock().unwrap(); @@ -659,6 +748,7 @@ mod tests { &AtomicBool::new(false), &Language::new(), &send, + &Cookies::default(), ) }); let mut server = Connection::local(server); @@ -708,6 +798,7 @@ mod tests { address: "local".into(), name: "Player".into(), counters: Arc::new(Mutex::new(None)), + cookies: Cookies::default(), debug: false, }; let (thread_writer, thread_configuring) = @@ -719,6 +810,7 @@ mod tests { &thread_configuring, &Language::new(), &send, + &Cookies::default(), ) }); let mut server = Connection::local(server); @@ -749,6 +841,68 @@ mod tests { assert!(thread.join().unwrap().is_err()); } + #[test] + fn play_answers_cookie_requests_from_stored_cookies_and_follows_transfers() { + let (client, server) = horizon_protocol::connection::Pipe::pair(); + let (mut reader, writer) = Connection::local(client).split(); + let writer = Arc::new(Mutex::new(Some(writer))); + let (send, events) = channel(); + let cookies = Cookies::default(); + let (thread_writer, thread_cookies) = (writer.clone(), cookies.clone()); + let thread = std::thread::spawn(move || { + receive_play( + &mut reader, + &thread_writer, + &AtomicBool::new(false), + &Language::new(), + &send, + &thread_cookies, + ) + }); + let mut server = Connection::local(server); + let ask = |server: &mut Connection, key: &str| { + let request = play::Clientbound::CookieRequest(cfg::CookieRequest { key: key.into() }); + server.send(&request.encode()).unwrap(); + let (id, body) = server.recv().unwrap(); + play::Serverbound::decode(id, &body).unwrap() + }; + // handleRequestCookie answers an unknown key with no payload. + assert_eq!( + ask(&mut server, "proxy:session"), + play::Serverbound::CookieResponse(cfg::CookieResponse { + key: "proxy:session".into(), + payload: None, + }) + ); + let store = play::Clientbound::StoreCookie(cfg::StoreCookie { + key: "proxy:session".into(), + payload: vec![1, 2, 3], + }); + server.send(&store.encode()).unwrap(); + assert_eq!( + ask(&mut server, "proxy:session"), + play::Serverbound::CookieResponse(cfg::CookieResponse { + key: "proxy:session".into(), + payload: Some(vec![1, 2, 3]), + }) + ); + let transfer = play::Clientbound::Transfer(cfg::Transfer { + host: "example.net".into(), + port: 25566, + }); + server.send(&transfer.encode()).unwrap(); + assert!(thread.join().unwrap().is_ok()); + assert!(matches!( + events.try_iter().last(), + Some(Event::Transfer { host, port: 25566 }) if host == "example.net" + )); + // The stored cookie stays for the next connection (TransferState). + assert_eq!( + cookies.lock().unwrap().get("proxy:session"), + Some(&vec![1, 2, 3]) + ); + } + #[test] fn server_brand_payload_is_decoded_and_invalid_or_unrelated_payloads_are_ignored() { let (send, events) = channel(); diff --git a/crates/horizon-client/tests/lan_bot.rs b/crates/horizon-client/tests/lan_bot.rs index 6dd1b8258..34e8265a1 100644 --- a/crates/horizon-client/tests/lan_bot.rs +++ b/crates/horizon-client/tests/lan_bot.rs @@ -278,6 +278,65 @@ fn run( handle.join().unwrap().unwrap() } +#[test] +fn sprint_commands_survive_keyboard_input_changes() { + let Some(resources) = resources() else { + return; + }; + let mut s = server(&resources); + let port = s.hub.lan.port; + let _bot = run(&mut s, move || connect(port, "Sprinter", true)); + let deadline = Instant::now() + Duration::from_secs(5); + while s.hub.guests.is_empty() { + s.tick(&[]); + assert!(Instant::now() < deadline, "guest did not join"); + thread::yield_now(); + } + let id = s.hub.guests[0].entity_id; + s.hub.guests[0].loaded = true; + s.hub.guests[0] + .inbox + .push_back(S::PlayerCommand(play::PlayerCommand { + id, + action: 1, + data: 0, + })); + s.tick(&[]); + assert!(s.hub.guests[0].sprinting); + // LocalPlayer keeps sprinting after Ctrl is released while W remains + // down. Double-tap sprint also sends forward without the sprint key. + for flags in [65, 1, 0, 33] { + s.hub.guests[0] + .inbox + .push_back(S::PlayerInput(play::PlayerInput { flags })); + s.tick(&[]); + let g = &s.hub.guests[0]; + assert!( + g.sprinting, + "keyboard input {flags} cleared the sprint flag" + ); + assert_eq!(g.input, flags); + assert_eq!(g.sneaking, flags & 32 != 0); + } + s.hub.guests[0] + .inbox + .push_back(S::PlayerCommand(play::PlayerCommand { + id, + action: 2, + data: 0, + })); + s.tick(&[]); + assert!(!s.hub.guests[0].sprinting); + s.hub.guests[0] + .inbox + .push_back(S::PlayerInput(play::PlayerInput { flags: 65 })); + s.tick(&[]); + assert!( + !s.hub.guests[0].sprinting, + "keyboard input started a sprint" + ); +} + #[test] fn status_and_join() { let Some(resources) = resources() else { diff --git a/crates/horizon-mod-registry/README.md b/crates/horizon-mod-registry/README.md index 936fe5acf..f7cdc68c5 100644 --- a/crates/horizon-mod-registry/README.md +++ b/crates/horizon-mod-registry/README.md @@ -40,5 +40,8 @@ REGISTRY_PUBLIC_URL=http://localhost:4173 REGISTRY_SESSION_KEY=... GITHUB_CLIENT | `REGISTRY_REPO` | Default `ludovic111/horizon-mods` | | `PORT` | Default 8080 | -On Railway it is its own service (`railway.json`, built with the `Dockerfile` from the repository root); the website's -Caddy sends `/api/mods/*` to it over Railway's private network. +On Railway it is the service `horizon-mod-registry` of the project `horizon-site`, next to the website: built from +`main` with this `Dockerfile` from the repository root, health check `/healthz`, no public domain and no volume. The +website's Caddy sends `/api/mods/*` to it over Railway's private network (`REGISTRY_UPSTREAM` on the website service: +`http://horizon-mod-registry.railway.internal:8080`). `railway.json` lists the same settings; Railway reads them from +the service, not from the file. diff --git a/crates/horizon-mod-registry/src/lib.rs b/crates/horizon-mod-registry/src/lib.rs index 9799bace8..72970017a 100644 --- a/crates/horizon-mod-registry/src/lib.rs +++ b/crates/horizon-mod-registry/src/lib.rs @@ -625,11 +625,16 @@ pub fn publish( .map(|r| format!("- needs {} {}", r.id, r.version)) .collect(); let body = format!( - "\nSubmitted by @{login} through [the form]({public}/mods/submit).\n\n\ + "\n🤖 Submitted through [{site}]({public}/mods/submit) by @{login}. The form opens the pull request with \ + the registry's token: neither Ludovic nor a reviewer wrote it.\n\n\ | | |\n|---|---|\n| Project | `{id}` ({kind}, {new}) |\n| Version | {version} for {game} |\n| File | `{file}`, {size} bytes, SHA-256 `{sha256}` |\n| Signed | {signed} |\n| Source | {source} |\n\n{lists}\n\n\ The checks below run the file's own checks again and, for a mod, a trial run in the sandbox. \ Reviewers: read the source and the checks' report before you approve.", public = config.public_url, + site = config + .public_url + .split_once("://") + .map_or(config.public_url.as_str(), |(_, host)| host), id = form.id, kind = kind.name(), new = if is_new { "new" } else { "update" }, diff --git a/crates/horizon-mod-registry/tests/submit.rs b/crates/horizon-mod-registry/tests/submit.rs index 77487089d..f586effac 100644 --- a/crates/horizon-mod-registry/tests/submit.rs +++ b/crates/horizon-mod-registry/tests/submit.rs @@ -303,7 +303,11 @@ async fn a_submission_becomes_a_pull_request() { assert!(message.contains("Co-authored-by: alice <7+alice@users.noreply.github.com>")); assert_eq!(replaced, &None); let body = m.pulls[0]["body"].as_str().unwrap(); - assert!(body.starts_with(""), "{body}"); + assert!( + body.starts_with("\n🤖 Submitted through "), + "{body}" + ); + assert!(body.contains("/mods/submit) by @alice."), "{body}"); assert!(body.contains("may draw text on your screen"), "{body}"); } // The same version again: its branch is there. diff --git a/crates/horizon-protocol/src/packets/play.rs b/crates/horizon-protocol/src/packets/play.rs index 26bb96316..67373cf4b 100644 --- a/crates/horizon-protocol/src/packets/play.rs +++ b/crates/horizon-protocol/src/packets/play.rs @@ -13,8 +13,9 @@ use crate::{ packets::{ Empty, configuration::{ - ClientInformation, CustomPayload, Disconnect, KeepAlive, Ping, ResourcePackPop, - ResourcePackPush, ResourcePackResponse, UpdateTags, + ClientInformation, CookieRequest, CookieResponse, CustomPayload, Disconnect, KeepAlive, + Ping, ResourcePackPop, ResourcePackPush, ResourcePackResponse, StoreCookie, Transfer, + UpdateTags, }, login::Property, }, @@ -1759,6 +1760,7 @@ packet_enum!(Clientbound, crate::ids::play::clientbound { ContainerSetContent(ContainerSetContent) = CONTAINER_SET_CONTENT, ContainerSetData(ContainerSetData) = CONTAINER_SET_DATA, ContainerSetSlot(ContainerSetSlot) = CONTAINER_SET_SLOT, + CookieRequest(CookieRequest) = COOKIE_REQUEST, CustomPayload(CustomPayload) = CUSTOM_PAYLOAD, Disconnect(Disconnect) = DISCONNECT, DisguisedChat(DisguisedChat) = DISGUISED_CHAT, @@ -1836,6 +1838,7 @@ packet_enum!(Clientbound, crate::ids::play::clientbound { SoundEntity(SoundEntity) = SOUND_ENTITY, StopSound(StopSound) = STOP_SOUND, StartConfiguration(Empty) = START_CONFIGURATION, + StoreCookie(StoreCookie) = STORE_COOKIE, SwingAnimation(SwingAnimation) = SWING_ANIMATION, SystemChat(SystemChat) = SYSTEM_CHAT, TabList(TabList) = TAB_LIST, @@ -1843,6 +1846,7 @@ packet_enum!(Clientbound, crate::ids::play::clientbound { TeleportEntity(TeleportEntity) = TELEPORT_ENTITY, TickingState(TickingState) = TICKING_STATE, TickingStep(TickingStep) = TICKING_STEP, + Transfer(Transfer) = TRANSFER, UpdateAdvancements(UpdateAdvancements) = UPDATE_ADVANCEMENTS, UpdateAttributes(UpdateAttributes) = UPDATE_ATTRIBUTES, LevelParticles(LevelParticles) = LEVEL_PARTICLES, @@ -2223,6 +2227,7 @@ packet_enum!(Serverbound, crate::ids::play::serverbound { ContainerButtonClick(ContainerButtonClick) = CONTAINER_BUTTON_CLICK, ContainerClick(ContainerClick) = CONTAINER_CLICK, ContainerClose(ContainerClose) = CONTAINER_CLOSE, + CookieResponse(CookieResponse) = COOKIE_RESPONSE, CustomPayload(CustomPayload) = CUSTOM_PAYLOAD, EditBook(EditBook) = EDIT_BOOK, Interact(Interact) = INTERACT, diff --git a/crates/horizon-shader-api/Cargo.toml b/crates/horizon-shader-api/Cargo.toml new file mode 100644 index 000000000..3d365cd83 --- /dev/null +++ b/crates/horizon-shader-api/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "horizon-shader-api" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version = "1.85" +description = "Horizon's own shader packs (WGSL, docs/shaders.md): the format, the stages, the check, and the horizon-shader tool" + +[dependencies] +bytemuck = { version = "1.24", features = ["derive"] } +# Parses and validates a pack's WGSL, and translates it as wgpu does for Vulkan, Metal and DirectX 12. +naga = { version = "30.0.1", features = ["wgsl-in", "spv-out", "msl-out", "hlsl-out"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +toml = "1.1.6" +zip = { version = "2", default-features = false, features = ["deflate"] } + +[lints] +workspace = true diff --git a/crates/horizon-shader-api/packs/golden_hour/README.md b/crates/horizon-shader-api/packs/golden_hour/README.md new file mode 100644 index 000000000..943bce85a --- /dev/null +++ b/crates/horizon-shader-api/packs/golden_hour/README.md @@ -0,0 +1,4 @@ +# Golden Hour + +Horizon's example shader pack (docs/shaders.md): a sky stage and two post passes, with five options. Check it with +`horizon-shader check crates/horizon-shader-api/packs/golden_hour`. diff --git a/crates/horizon-shader-api/packs/golden_hour/common.wgsl b/crates/horizon-shader-api/packs/golden_hour/common.wgsl new file mode 100644 index 000000000..687e76f4b --- /dev/null +++ b/crates/horizon-shader-api/packs/golden_hour/common.wgsl @@ -0,0 +1,13 @@ +fn luminance(c: vec3) -> f32 { + return dot(c, vec3(0.2126, 0.7152, 0.0722)); +} + +// 1 while the sun is up in the Overworld, fading out around the horizon. +fn daylight() -> f32 { + return select(0.0, smoothstep(-0.1, 0.1, frame.sun_direction.y), frame.dimension == 0); +} + +// 1 when the sun is on the horizon, 0 from a quarter of the way up. +fn sun_low() -> f32 { + return 1.0 - smoothstep(0.0, 0.35, abs(frame.sun_direction.y)); +} diff --git a/crates/horizon-shader-api/packs/golden_hour/horizon-shaders.toml b/crates/horizon-shader-api/packs/golden_hour/horizon-shaders.toml new file mode 100644 index 000000000..8b67cff3b --- /dev/null +++ b/crates/horizon-shader-api/packs/golden_hour/horizon-shaders.toml @@ -0,0 +1,69 @@ +schema = 1 +id = "golden_hour" +name = "Golden Hour" +version = "1.0.0" +api = "0.1" +description = "Warm light at sunrise and sunset, a soft glow around bright things and a gentle colour grade." +authors = ["Horizon"] +license = "MIT" +shared = ["common.wgsl"] + +[sky] +file = "sky.wgsl" +name = "Warm horizon" + +[[post]] +file = "post/glow.wgsl" +name = "Glow" + +[[post]] +file = "post/grade.wgsl" +name = "Colour grade" + +[[options]] +id = "sun_glow" +name = "Sunset glow" +description = "How warm the sky is around a low sun." +type = "float" +default = 0.6 +min = 0.0 +max = 1.0 +step = 0.05 + +[[options]] +id = "glow" +name = "Glow" +description = "The soft halo around bright things: the sun, lava, torches." +type = "float" +default = 0.35 +min = 0.0 +max = 1.0 +step = 0.05 + +[[options]] +id = "warmth" +name = "Warmth" +description = "How golden the light turns when the sun is low." +type = "float" +default = 0.4 +min = 0.0 +max = 1.0 +step = 0.05 + +[[options]] +id = "saturation" +name = "Saturation" +type = "float" +default = 1.1 +min = 0.0 +max = 2.0 +step = 0.05 + +[[options]] +id = "vignette" +name = "Vignette" +type = "bool" +default = true + +[budget] +gpu_ms = 1.0 diff --git a/crates/horizon-shader-api/packs/golden_hour/post/glow.wgsl b/crates/horizon-shader-api/packs/golden_hour/post/glow.wgsl new file mode 100644 index 000000000..4d523c9ef --- /dev/null +++ b/crates/horizon-shader-api/packs/golden_hour/post/glow.wgsl @@ -0,0 +1,22 @@ +// A soft halo around bright pixels: twelve taps on two rings, scaled with +// the picture's height so it looks the same at any resolution. +override glow: f32 = 0.35; + +fn bright(c: vec3) -> vec3 { + return c * smoothstep(0.7, 1.0, luminance(c)); +} + +fn post(input: PostInput) -> vec4 { + if glow <= 0.0 { + return input.color; + } + let step = frame.resolution.y / 720.0 / frame.resolution; + var sum = vec3(0.0); + for (var i = 0; i < 12; i++) { + let angle = f32(i) * 0.5235988; + let radius = select(3.0, 7.0, (i & 1) == 1); + let offset = vec2(cos(angle), sin(angle)) * radius * step; + sum += bright(textureSampleLevel(previous, linear_sampler, input.uv + offset, 0.0).rgb); + } + return vec4(input.color.rgb + sum * (glow / 6.0), input.color.a); +} diff --git a/crates/horizon-shader-api/packs/golden_hour/post/grade.wgsl b/crates/horizon-shader-api/packs/golden_hour/post/grade.wgsl new file mode 100644 index 000000000..809b0877e --- /dev/null +++ b/crates/horizon-shader-api/packs/golden_hour/post/grade.wgsl @@ -0,0 +1,19 @@ +// Golden light when the sun is low, a soft shoulder for the brightest +// colours (white stays white), saturation and a light vignette. +override warmth: f32 = 0.4; +override saturation: f32 = 1.1; +override vignette: bool = true; + +fn post(input: PostInput) -> vec4 { + var c = input.color.rgb; + let golden = warmth * mix(0.3, 1.0, sun_low()) * daylight() * (1.0 - frame.rain * 0.7); + c *= mix(vec3(1.0), vec3(1.08, 1.0, 0.86), golden); + c = c / (1.0 + 0.25 * c) * 1.25; + c = mix(vec3(luminance(c)), c, saturation); + if vignette { + let aspect = frame.resolution.x / frame.resolution.y; + let d = length((input.uv - 0.5) * vec2(aspect, 1.0)) / length(vec2(aspect, 1.0) * 0.5); + c *= 1.0 - smoothstep(0.55, 1.1, d) * 0.3; + } + return vec4(clamp(c, vec3(0.0), vec3(1.0)), input.color.a); +} diff --git a/crates/horizon-shader-api/packs/golden_hour/sky.wgsl b/crates/horizon-shader-api/packs/golden_hour/sky.wgsl new file mode 100644 index 000000000..dbf1e0ab5 --- /dev/null +++ b/crates/horizon-shader-api/packs/golden_hour/sky.wgsl @@ -0,0 +1,14 @@ +// Warms the sky around a low sun, and deepens the blue overhead by day. +override sun_glow: f32 = 0.6; + +fn sky(input: SkyInput) -> vec4 { + let d = input.direction; + let toward = max(dot(d, frame.sun_direction), 0.0); + let near_horizon = 1.0 - smoothstep(0.0, 0.4, abs(d.y)); + let day = daylight() * (1.0 - frame.rain); + let halo = pow(toward, 6.0) * 0.6 + near_horizon * toward * toward * 0.4; + let warm = vec3(1.0, 0.55, 0.25) * halo * sun_low() * day * sun_glow; + let zenith = smoothstep(0.1, 1.0, d.y) * day * 0.12; + let base = input.color.rgb * (1.0 - zenith * vec3(0.5, 0.25, 0.0)); + return vec4(base + warm, input.color.a); +} diff --git a/crates/horizon-shader-api/src/bin/horizon-shader.rs b/crates/horizon-shader-api/src/bin/horizon-shader.rs new file mode 100644 index 000000000..b6130282d --- /dev/null +++ b/crates/horizon-shader-api/src/bin/horizon-shader.rs @@ -0,0 +1,307 @@ +//! `horizon-shader`: make Horizon shader packs (docs/shaders.md). +//! +//! - `new [--template post|sky|material] [--name "Name"]`: a pack from a template. +//! - `check ... [--json]`: the check the game, the launcher and the registry run. +//! - `pack [--out ]`: check, then write the reproducible `.hshader`. +use horizon_shader_api::{EXTENSION, MANIFEST, Pack, Problem, check, package::sha256_hex}; +use serde::Serialize; +use std::{path::Path, process::ExitCode}; + +const USAGE: &str = "usage: horizon-shader new [--template post|sky|material] [--name \"Name\"] | check ... [--json] | pack [--out ]"; + +/// (template, its files as (name, text)). +const TEMPLATES: [(&str, [(&str, &str); 2]); 3] = [ + ( + "post", + [ + ( + MANIFEST, + include_str!("../../templates/post/horizon-shaders.toml"), + ), + ("post.wgsl", include_str!("../../templates/post/post.wgsl")), + ], + ), + ( + "sky", + [ + ( + MANIFEST, + include_str!("../../templates/sky/horizon-shaders.toml"), + ), + ("sky.wgsl", include_str!("../../templates/sky/sky.wgsl")), + ], + ), + ( + "material", + [ + ( + MANIFEST, + include_str!("../../templates/material/horizon-shaders.toml"), + ), + ( + "material.wgsl", + include_str!("../../templates/material/material.wgsl"), + ), + ], + ), +]; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + let result = match args.first().map(String::as_str) { + Some("new") => new(&args[1..]), + Some("check") => check_command(&args[1..]), + Some("pack") => pack(&args[1..]), + _ => Err(USAGE.into()), + }; + match result { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} + +/// The value after `flag` in `args`. +fn flag<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { + let i = args.iter().position(|a| a == flag)?; + args.get(i + 1).map(String::as_str) +} + +/// The arguments that are neither flags nor their values. +fn positional(args: &[String], flags: &[&str]) -> Vec { + let mut out = Vec::new(); + let mut it = args.iter(); + while let Some(a) = it.next() { + if flags.contains(&a.as_str()) { + it.next(); + } else if !a.starts_with("--") { + out.push(a.clone()); + } + } + out +} + +fn new(args: &[String]) -> Result<(), String> { + let [dir] = positional(args, &["--template", "--name"]) + .try_into() + .map_err(|_| USAGE.to_string())?; + let dir = Path::new(&dir); + if dir.exists() { + return Err(format!("{} already exists", dir.display())); + } + let template = flag(args, "--template").unwrap_or("post"); + let (_, files) = TEMPLATES + .iter() + .find(|(name, _)| *name == template) + .ok_or_else(|| format!("no template {template:?}: post, sky or material"))?; + let base = dir + .file_name() + .and_then(|n| n.to_str()) + .ok_or("the path needs a folder name")?; + let mut id: String = base + .chars() + .map(|c| match c { + 'a'..='z' | '0'..='9' => c, + 'A'..='Z' => c.to_ascii_lowercase(), + _ => '_', + }) + .collect(); + id.truncate(64); + let name = flag(args, "--name").map_or_else(|| base.replace(['-', '_'], " "), str::to_owned); + std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + for (file, text) in files { + let text = text + .replace("{{id}}", &id) + .replace("{{name}}", &name.replace('"', "'")); + let path = dir.join(file); + std::fs::write(&path, text).map_err(|e| format!("{}: {e}", path.display()))?; + } + println!( + "Made {} from the {template} template. Edit it, then: horizon-shader check {}", + dir.display(), + dir.display() + ); + Ok(()) +} + +#[derive(Serialize)] +struct StageJson { + stage: &'static str, + file: String, + label: String, + expressions: usize, + texture_reads: usize, +} + +#[derive(Serialize)] +struct ProblemJson { + file: String, + line: usize, + column: usize, + message: String, +} + +#[derive(Serialize)] +struct CheckJson { + path: String, + ok: bool, + id: Option, + version: Option, + api: Option, + /// The file's own SHA-256 (a `.hshader`), else the reproducible package's. + sha256: Option, + stages: Vec, + problems: Vec, +} + +/// Reads and checks the pack at `path`. +fn check_one(path: &Path) -> CheckJson { + let mut out = CheckJson { + path: path.display().to_string(), + ok: false, + id: None, + version: None, + api: None, + sha256: None, + stages: Vec::new(), + problems: Vec::new(), + }; + let problem = |message: String| ProblemJson { + file: path.display().to_string(), + line: 0, + column: 0, + message, + }; + let pack = match Pack::read(path) { + Ok(p) => p, + Err(e) => { + out.problems.push(problem(e)); + return out; + } + }; + out.id = Some(pack.manifest.id.clone()); + out.version = Some(pack.manifest.version.clone()); + out.api = Some(pack.manifest.api.clone()); + out.sha256 = if path.is_dir() { + pack.sha256().ok() + } else { + std::fs::read(path).ok().map(|b| sha256_hex(&b)) + }; + match check(&pack) { + Ok(report) => { + out.ok = true; + out.stages = report + .stages + .into_iter() + .map(|s| StageJson { + stage: s.kind.name(), + file: s.file, + label: s.label, + expressions: s.expressions, + texture_reads: s.texture_reads, + }) + .collect(); + } + Err(problems) => { + out.problems = problems + .into_iter() + .map( + |Problem { + file, + line, + column, + message, + }| ProblemJson { + file, + line, + column, + message, + }, + ) + .collect(); + } + } + out +} + +fn check_command(args: &[String]) -> Result<(), String> { + let paths = positional(args, &[]); + if paths.is_empty() { + return Err(USAGE.into()); + } + let results: Vec = paths.iter().map(|p| check_one(Path::new(p))).collect(); + let failed = results.iter().filter(|r| !r.ok).count(); + if args.iter().any(|a| a == "--json") { + let json = serde_json::to_string_pretty(&results).map_err(|e| e.to_string())?; + println!("{json}"); + } else { + for r in &results { + print_result(r); + } + } + match failed { + 0 => Ok(()), + n => Err(format!("{n} of {} packs failed the check", results.len())), + } +} + +fn print_result(r: &CheckJson) { + let what = match (&r.id, &r.version, &r.api) { + (Some(id), Some(v), Some(api)) => format!("{id} {v} (shader API {api})"), + _ => r.path.clone(), + }; + if !r.ok { + println!("{what}: FAILED"); + for p in &r.problems { + if p.line == 0 { + println!(" {}: {}", p.file, p.message); + } else { + println!(" {}:{}:{}: {}", p.file, p.line, p.column, p.message); + } + } + return; + } + println!("{what}: ok"); + for s in &r.stages { + println!( + " {:<9} {:<28} {:>6} expressions, {} texture reads", + s.stage, s.label, s.expressions, s.texture_reads + ); + } + if let Some(sha) = &r.sha256 { + println!(" sha256 {sha}"); + } +} + +fn pack(args: &[String]) -> Result<(), String> { + let [dir] = positional(args, &["--out"]) + .try_into() + .map_err(|_| USAGE.to_string())?; + let dir = Path::new(&dir) + .canonicalize() + .map_err(|e| format!("{dir}: {e}"))?; + if !dir.is_dir() { + return Err(format!("{} is not a pack folder", dir.display())); + } + let dir = dir.as_path(); + let pack = Pack::read(dir)?; + if let Err(problems) = check(&pack) { + for p in &problems { + eprintln!("{p}"); + } + return Err(format!("{} failed the check", dir.display())); + } + let bytes = pack.package()?; + let out = flag(args, "--out").map_or_else( + || { + let m = &pack.manifest; + dir.with_file_name(format!("{}-{}.{EXTENSION}", m.id, m.version)) + }, + Into::into, + ); + std::fs::write(&out, &bytes).map_err(|e| format!("{}: {e}", out.display()))?; + println!("Wrote {} (sha256 {})", out.display(), sha256_hex(&bytes)); + Ok(()) +} diff --git a/crates/horizon-shader-api/src/check.rs b/crates/horizon-shader-api/src/check.rs new file mode 100644 index 000000000..6c43718f7 --- /dev/null +++ b/crates/horizon-shader-api/src/check.rs @@ -0,0 +1,495 @@ +//! The check a pack passes before it runs: in the game when it loads (a pack +//! that fails keeps the last good one drawing), in the launcher before it +//! installs, in the registry's CI, and in `horizon-shader check`. +//! +//! Each stage is built as the renderer builds it, parsed and validated by +//! naga, checked against its signature and the API's rules (no bindings, no +//! entry points, no `horizon_` names), then translated to SPIR-V, Metal and +//! HLSL with its options at their defaults, minimums and maximums. +use crate::manifest::{OptionKind, PackOption}; +use crate::package::Pack; +use crate::stages::{Composed, StageKind, compose}; +use naga::valid::{Capabilities, ModuleInfo, ValidationFlags, Validator}; +use std::collections::HashSet; +use std::fmt; + +/// naga expressions a stage may hold, over all its functions. +pub const MAX_EXPRESSIONS: usize = 20_000; +/// Texture reads (samples, gathers and loads) a stage may write. +pub const MAX_TEXTURE_READS: usize = 64; + +/// What is wrong, and where: a file of the pack, with the line and column +/// when known (0 when not). +#[derive(Clone, Debug, PartialEq)] +pub struct Problem { + pub file: String, + pub line: usize, + pub column: usize, + pub message: String, +} + +impl fmt::Display for Problem { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.line == 0 { + write!(f, "{}: {}", self.file, self.message) + } else { + write!( + f, + "{}:{}:{}: {}", + self.file, self.line, self.column, self.message + ) + } + } +} + +fn problem(file: &str, message: impl Into) -> Problem { + Problem { + file: file.to_owned(), + line: 0, + column: 0, + message: message.into(), + } +} + +/// One stage that passed, ready for the renderer. +#[derive(Debug)] +pub struct CheckedStage { + pub kind: StageKind, + pub file: String, + /// The manifest's name for it, else its file. + pub label: String, + pub module: naga::Module, + pub info: ModuleInfo, + pub expressions: usize, + pub texture_reads: usize, +} + +#[derive(Debug)] +pub struct Report { + pub stages: Vec, +} + +/// Checks every stage of `pack`, and that each option is an `override` of +/// its type in some stage. Returns every problem found, not just the first. +pub fn check(pack: &Pack) -> Result> { + let m = &pack.manifest; + let stages = m + .sky + .iter() + .map(|s| (StageKind::Sky, s)) + .chain(m.material.iter().map(|s| (StageKind::Material, s))) + .chain(m.post.iter().map(|s| (StageKind::Post, s))); + let (mut checked, mut problems) = (Vec::new(), Vec::new()); + for (kind, stage) in stages { + match check_stage(pack, kind, &stage.file) { + Ok(mut c) => { + c.label = stage.name.clone().unwrap_or_else(|| stage.file.clone()); + checked.push(c); + } + Err(p) => problems.extend(p), + } + } + if problems.is_empty() { + for option in &m.options { + let used = checked.iter().any(|c| { + c.module + .overrides + .iter() + .any(|(_, o)| o.name.as_deref() == Some(&option.id)) + }); + if !used { + problems.push(problem( + crate::MANIFEST, + format!( + "option {:?}: no stage declares `override {}: {}`", + option.id, + option.id, + option.kind.wgsl_type() + ), + )); + } + } + } + if problems.is_empty() { + Ok(Report { stages: checked }) + } else { + Err(problems) + } +} + +/// Builds and checks one stage of `pack` from its file. +pub fn check_stage(pack: &Pack, kind: StageKind, file: &str) -> Result> { + // First without the entry points, so a missing or mistyped stage function + // is reported as such rather than as a call that doesn't resolve. + let mut own = compose(pack, kind, file); + let entries = own.parts.pop().expect("entries are the last part"); + own.text.truncate(entries.1.start); + let module = parse(&own)?; + let mut problems = signature(&module, kind, file); + problems.extend(api_rules(&module, kind, &own)); + problems.extend(options(&module, &pack.manifest.options, file)); + if !problems.is_empty() { + return Err(problems); + } + let composed = compose(pack, kind, file); + let module = parse(&composed)?; + let info = Validator::new(ValidationFlags::all(), Capabilities::default()) + .validate(&module) + .map_err(|e| { + let at = e.spans().next().and_then(|(span, _)| span.to_range()); + let mut message = e.as_inner().to_string(); + let mut source = std::error::Error::source(e.as_inner()); + while let Some(s) = source { + message += &format!(": {s}"); + source = s.source(); + } + vec![located(&composed, at.map(|r| r.start), message)] + })?; + let (expressions, texture_reads) = cost(&module); + let mut problems = Vec::new(); + if expressions > MAX_EXPRESSIONS { + problems.push(problem( + file, + format!("{expressions} expressions; a stage may hold {MAX_EXPRESSIONS}"), + )); + } + if texture_reads > MAX_TEXTURE_READS { + problems.push(problem( + file, + format!("{texture_reads} texture reads; a stage may write {MAX_TEXTURE_READS}"), + )); + } + problems.extend(translate(&module, &info, &pack.manifest.options, file)); + if !problems.is_empty() { + return Err(problems); + } + Ok(CheckedStage { + kind, + file: file.to_owned(), + label: file.to_owned(), + module, + info, + expressions, + texture_reads, + }) +} + +fn located(composed: &Composed, offset: Option, message: String) -> Problem { + match offset { + Some(at) => { + let (file, line, column) = composed.locate(at); + Problem { + file: file.to_owned(), + line, + column, + message, + } + } + None => problem(&composed.parts[composed.parts.len() - 2].0, message), + } +} + +fn parse(composed: &Composed) -> Result> { + naga::front::wgsl::parse_str(&composed.text).map_err(|e| { + let label = e.labels().next(); + let mut message = e.message().to_owned(); + if let Some((_, text)) = label.filter(|(_, t)| !t.is_empty() && !message.contains(*t)) { + message += &format!(" ({text})"); + } + let at = label.and_then(|(span, _)| span.to_range()).map(|r| r.start); + vec![located(composed, at, message)] + }) +} + +fn type_name(module: &naga::Module, ty: naga::Handle) -> String { + let t = &module.types[ty]; + if let Some(name) = &t.name { + return name.clone(); + } + match t.inner { + naga::TypeInner::Scalar(s) => scalar_name(s).to_owned(), + naga::TypeInner::Vector { size, scalar } => { + format!("vec{}<{}>", size as u8, scalar_name(scalar)) + } + _ => format!("{:?}", t.inner), + } +} + +fn scalar_name(s: naga::Scalar) -> &'static str { + match (s.kind, s.width) { + (naga::ScalarKind::Bool, _) => "bool", + (naga::ScalarKind::Float, 4) => "f32", + (naga::ScalarKind::Sint, 4) => "i32", + (naga::ScalarKind::Uint, 4) => "u32", + _ => "another scalar", + } +} + +fn signature(module: &naga::Module, kind: StageKind, file: &str) -> Vec { + let found = module + .functions + .iter() + .find(|(_, f)| f.name.as_deref() == Some(kind.function())); + let Some((_, f)) = found else { + return vec![problem(file, format!("defines no `{}`", kind.signature()))]; + }; + let args: Vec = f + .arguments + .iter() + .map(|a| type_name(module, a.ty)) + .collect(); + let result = f.result.as_ref().map(|r| type_name(module, r.ty)); + if args == [kind.input().name] && result.as_deref() == Some(kind.output()) { + return Vec::new(); + } + vec![problem( + file, + format!( + "`fn {}({}) -> {}` should be `{}`", + kind.function(), + args.join(", "), + result.as_deref().unwrap_or("()"), + kind.signature() + ), + )] +} + +/// The names of everything a module defines at its top level. +fn names(module: &naga::Module) -> HashSet { + let types = module.types.iter().filter_map(|(_, t)| t.name.clone()); + let functions = module.functions.iter().filter_map(|(_, f)| f.name.clone()); + let globals = module + .global_variables + .iter() + .filter_map(|(_, g)| g.name.clone()); + let constants = module.constants.iter().filter_map(|(_, c)| c.name.clone()); + let overrides = module.overrides.iter().filter_map(|(_, o)| o.name.clone()); + types + .chain(functions) + .chain(globals) + .chain(constants) + .chain(overrides) + .collect() +} + +/// A pack's files declare no bindings, no entry points and no `horizon_` +/// names: what the prelude doesn't define is the pack's. +fn api_rules(module: &naga::Module, kind: StageKind, own: &Composed) -> Vec { + let prelude = naga::front::wgsl::parse_str(&kind.prelude()).expect("the prelude parses"); + let ours = names(&prelude); + let file = &own.parts[own.parts.len() - 1].0; + let mut problems = Vec::new(); + for e in &module.entry_points { + problems.push(problem( + file, + format!( + "declares the entry point `{}`: a stage is a function, Horizon adds the entry points", + e.name + ), + )); + } + for (_, g) in module.global_variables.iter() { + let name = g.name.clone().unwrap_or_default(); + if g.binding.is_some() && !ours.contains(&name) { + problems.push(problem( + file, + format!( + "declares the binding `{name}`: a stage reads only the resources of its prelude" + ), + )); + } + } + let mut theirs: Vec = names(module) + .into_iter() + .filter(|n| !ours.contains(n) && n.starts_with("horizon_")) + .collect(); + theirs.sort(); + for name in theirs { + problems.push(problem( + file, + format!("`{name}`: names starting with horizon_ are Horizon's"), + )); + } + problems +} + +/// Each `override` is an option of its type, or has a default of its own. +fn options(module: &naga::Module, options: &[PackOption], file: &str) -> Vec { + let mut problems = Vec::new(); + for (_, o) in module.overrides.iter() { + let name = o.name.as_deref().unwrap_or("?"); + let ty = type_name(module, o.ty); + match options.iter().find(|opt| opt.id == name) { + Some(opt) if ty != opt.kind.wgsl_type() => problems.push(problem( + file, + format!( + "`override {name}: {ty}`: option {name:?} is a {}, so its override is `{}`", + kind_name(&opt.kind), + opt.kind.wgsl_type() + ), + )), + None if o.init.is_none() => problems.push(problem( + file, + format!("`override {name}` has no default and no [[options]] entry"), + )), + _ => {} + } + } + problems +} + +fn kind_name(kind: &OptionKind) -> &'static str { + match kind { + OptionKind::Bool { .. } => "bool", + OptionKind::Float { .. } => "float", + OptionKind::Int { .. } => "int", + OptionKind::Choice { .. } => "choice", + } +} + +fn cost(module: &naga::Module) -> (usize, usize) { + let functions = module + .functions + .iter() + .map(|(_, f)| f) + .chain(module.entry_points.iter().map(|e| &e.function)); + let (mut expressions, mut reads) = (0, 0); + for f in functions { + expressions += f.expressions.len(); + reads += f + .expressions + .iter() + .filter(|(_, e)| { + matches!( + e, + naga::Expression::ImageSample { .. } | naga::Expression::ImageLoad { .. } + ) + }) + .count(); + } + (expressions, reads) +} + +/// The option values a pack is translated with: its defaults, then every +/// option at its minimum, then at its maximum. +fn option_sets(options: &[PackOption]) -> [(&'static str, Vec<(String, f64)>); 3] { + let at = |pick: fn(&OptionKind) -> f64| { + options + .iter() + .map(|o| (o.id.clone(), pick(&o.kind))) + .collect() + }; + [ + ("defaults", at(OptionKind::default_constant)), + ( + "minimums", + at(|k| match *k { + OptionKind::Bool { .. } | OptionKind::Choice { .. } => 0.0, + OptionKind::Float { min, .. } => f64::from(min), + OptionKind::Int { min, .. } => f64::from(min), + }), + ), + ( + "maximums", + at(|k| match k { + OptionKind::Bool { .. } => 1.0, + OptionKind::Choice { values, .. } => (values.len() - 1) as f64, + OptionKind::Float { max, .. } => f64::from(*max), + OptionKind::Int { max, .. } => f64::from(*max), + }), + ), + ] +} + +/// Translates every entry point for Vulkan (SPIR-V), Metal and DirectX 12 +/// (HLSL) with each option set, as wgpu does when it builds the pipelines. +fn translate( + module: &naga::Module, + info: &ModuleInfo, + options: &[PackOption], + file: &str, +) -> Vec { + let present: HashSet<&str> = module + .overrides + .iter() + .filter_map(|(_, o)| o.name.as_deref()) + .collect(); + let mut problems = Vec::new(); + for (set, values) in option_sets(options) { + let mut constants = naga::back::PipelineConstants::default(); + for (id, value) in values + .into_iter() + .filter(|(id, _)| present.contains(id.as_str())) + { + constants.insert(id, value); + } + for e in &module.entry_points { + let fail = |backend: &str, err: String| { + problem( + file, + format!("with the options at their {set}, {backend} refuses it: {err}"), + ) + }; + let processed = naga::back::pipeline_constants::process_overrides( + module, + info, + Some((e.stage, &e.name)), + &constants, + ); + let (m, i) = match processed { + Ok(p) => p, + Err(err) => { + problems.push(fail("the override step", err.to_string())); + return problems; + } + }; + let spv = naga::back::spv::write_vec( + &m, + &i, + &naga::back::spv::Options::default(), + Some(&naga::back::spv::PipelineOptions { + shader_stage: e.stage, + entry_point: e.name.clone(), + }), + ); + let msl = naga::back::msl::write_string( + &m, + &i, + &naga::back::msl::Options { + lang_version: (2, 2), + ..Default::default() + }, + &naga::back::msl::PipelineOptions { + entry_point: Some((e.stage, e.name.clone())), + ..Default::default() + }, + ); + let mut hlsl = String::new(); + let hlsl_result = naga::back::hlsl::Writer::new( + &mut hlsl, + &naga::back::hlsl::Options::default(), + &naga::back::hlsl::PipelineOptions { + entry_point: Some((e.stage, e.name.clone())), + }, + ) + .write(&m, &i, None); + let errors = [ + ("Vulkan (SPIR-V)", spv.err().map(|e| e.to_string())), + ("Metal", msl.err().map(|e| e.to_string())), + ( + "DirectX 12 (HLSL)", + hlsl_result.err().map(|e| e.to_string()), + ), + ]; + for (backend, err) in errors { + if let Some(err) = err { + problems.push(fail(backend, err)); + } + } + if !problems.is_empty() { + return problems; + } + } + } + problems +} diff --git a/crates/horizon-shader-api/src/inputs.rs b/crates/horizon-shader-api/src/inputs.rs new file mode 100644 index 000000000..5bc82fd82 --- /dev/null +++ b/crates/horizon-shader-api/src/inputs.rs @@ -0,0 +1,218 @@ +//! What a pack's WGSL reads: the frame uniform every stage sees, which the +//! renderer fills and uploads as the Rust struct below, and the stage input +//! and output structs. The WGSL of each is generated from these definitions +//! (`Struct::wgsl`), and so is the reference (docs/api/shaders-reference.md). + +/// One field of a WGSL struct, for the prelude and the reference. +#[derive(Debug)] +pub struct Field { + pub name: &'static str, + pub ty: &'static str, + /// Its meaning, unit and range. + pub doc: &'static str, +} + +/// A struct a pack's WGSL can name. +#[derive(Debug)] +pub struct Struct { + pub name: &'static str, + pub doc: &'static str, + pub fields: &'static [Field], +} + +impl Struct { + pub fn wgsl(&self) -> String { + let mut out = format!("struct {} {{\n", self.name); + for f in self.fields { + out += &format!(" {}: {},\n", f.name, f.ty); + } + out + "}\n" + } +} + +/// The WGSL type of a Rust field the renderer uploads. +pub trait Wgsl { + const TYPE: &'static str; +} + +macro_rules! wgsl_types { + ($($rust:ty => $wgsl:literal),* $(,)?) => { + $(impl Wgsl for $rust { + const TYPE: &'static str = $wgsl; + })* + }; +} + +wgsl_types! { + f32 => "f32", + u32 => "u32", + i32 => "i32", + [f32; 2] => "vec2", + [f32; 3] => "vec3", + [f32; 4] => "vec4", + [[f32; 4]; 4] => "mat4x4", +} + +/// A uniform struct: the Rust type the renderer uploads, and its WGSL. +macro_rules! uniform { + ($(#[doc = $doc:literal])* $name:ident { + $($(#[doc = $fdoc:literal])* $field:ident: $ty:ty,)* + }) => { + $(#[doc = $doc])* + #[repr(C)] + #[derive(Clone, Copy, Debug, Default, PartialEq, bytemuck::Pod, bytemuck::Zeroable)] + pub struct $name { + $($(#[doc = $fdoc])* pub $field: $ty,)* + } + + impl $name { + pub const WGSL: Struct = Struct { + name: stringify!($name), + doc: concat!($($doc, "\n"),*), + fields: &[$(Field { + name: stringify!($field), + ty: <$ty as Wgsl>::TYPE, + doc: concat!($($fdoc),*), + }),*], + }; + /// Each field's byte offset, in `WGSL.fields`' order. + pub const OFFSETS: &'static [usize] = &[$(std::mem::offset_of!($name, $field)),*]; + } + }; +} + +uniform! { + /// The frame, as `frame` in every stage. Positions and directions are + /// in world axes relative to the camera (x east, y up, z south), so they + /// stay precise far from the world's origin; depth is reversed (1 near, + /// 0 at the far plane and in the sky). + Frame { + /// Camera-relative world to view space: the camera's rotation. + view: [[f32; 4]; 4], + /// View to clip space (reversed depth, wgpu's clip space: z 0..1). + projection: [[f32; 4]; 4], + /// View to camera-relative world space. + view_inverse: [[f32; 4]; 4], + /// Clip to view space. + projection_inverse: [[f32; 4]; 4], + /// The camera's world position, in blocks (about a millimetre of precision 10 000 blocks out). + camera_position: [f32; 3], + /// Seconds since the pack was loaded, wrapping to 0 every hour. + time: f32, + /// Unit vector toward the sun (below the horizon at night). + sun_direction: [f32; 3], + /// The time of day, 0..1: 0 sunrise, 0.25 noon, 0.5 sunset, 0.75 midnight. + day_time: f32, + /// Unit vector toward the moon. + moon_direction: [f32; 3], + /// Vanilla's GameTime: (game time % 24000 + partial tick) / 24000, 0..1. + game_time: f32, + /// Vanilla's sky colour this frame, 0..1. + sky_color: [f32; 3], + /// Rain strength, 0..1. + rain: f32, + /// Vanilla's fog colour this frame (also the clear colour), 0..1. + fog_color: [f32; 3], + /// Thunder strength, 0..1. + thunder: f32, + /// The size of the world's picture, in pixels (the resolution scale applied). + resolution: [f32; 2], + /// Block and sky light at the camera, 0..1 (level / 15). + eye_light: [f32; 2], + /// Where vanilla's fog starts, in blocks from the camera. + fog_start: f32, + /// Where vanilla's fog is full, in blocks. + fog_end: f32, + /// The render distance, in blocks. + render_distance: f32, + /// The far plane, in blocks. + far: f32, + /// Night vision strength, 0..1. + night_vision: f32, + /// Blindness strength, 0..1. + blindness: f32, + /// The darkness effect's strength, 0..1. + darkness: f32, + /// The Brightness option, 0..1 (Moody to Bright). + brightness: f32, + /// What the camera is in: 0 air, 1 water, 2 lava, 3 powder snow. + eye_in: u32, + /// 0 the Overworld, -1 the Nether, 1 the End, 2 any other dimension. + dimension: i32, + /// Frames drawn since the pack was loaded (wraps). + frame_index: u32, + /// 1 while F1 hides the HUD, else 0. + hide_gui: u32, + } +} + +macro_rules! fields { + ($($name:ident: $ty:literal = $doc:literal,)*) => { + &[$(Field { name: stringify!($name), ty: $ty, doc: $doc }),*] + }; +} + +/// The post stage's input: one pixel of the picture. +pub const POST_INPUT: Struct = Struct { + name: "PostInput", + doc: "One pixel of the picture, for `fn post(input: PostInput) -> vec4`.", + fields: fields! { + uv: "vec2" = "Where the pixel is, 0..1 from the top left.", + pixel: "vec2" = "The pixel's centre, in pixels from the top left.", + color: "vec4" = "The previous pass's colour here (the world's picture for the first pass).", + depth: "f32" = "The world's depth here, reversed: 0 is the sky.", + }, +}; + +/// The material stage's input: one point of a block's surface. +pub const MATERIAL_INPUT: Struct = Struct { + name: "MaterialInput", + doc: "One point of a block's surface, for `fn material(input: MaterialInput) -> Surface`.", + fields: fields! { + albedo: "vec4" = "The block texture's colour here (alpha 0..1).", + tint: "vec4" = "The vertex colour: biome tint, face shade and ambient occlusion.", + light: "vec2" = "Block and sky light, 0..1, smoothed over the face.", + lightmap: "vec3" = "Vanilla's light colour for that light this frame.", + position: "vec3" = "The point, camera-relative, in blocks.", + uv: "vec2" = "The texture coordinate inside the block atlas.", + translucent: "bool" = "Drawn in the translucent pass (water, stained glass, ice).", + }, +}; + +/// What the material stage returns. +pub const SURFACE: Struct = Struct { + name: "Surface", + doc: "What `material` returns. Vanilla's own is `albedo * tint * vec4(lightmap, 1.0)`; \ + Horizon then applies the alpha test (cutout and translucent) and fog.", + fields: fields! { + color: "vec4" = "The lit colour, 0..1, with its alpha.", + }, +}; + +/// The sky stage's input: one direction of the sky. +pub const SKY_INPUT: Struct = Struct { + name: "SkyInput", + doc: "One direction of the sky, for `fn sky(input: SkyInput) -> vec4`. The sun, moon, \ + stars and clouds are drawn over the result.", + fields: fields! { + direction: "vec3" = "Unit view ray, camera-relative world axes.", + color: "vec4" = "Vanilla's sky colour along the ray (sky, fog and sunrise).", + }, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn frame_fields_carry_their_docs() { + let wgsl = Frame::WGSL; + assert_eq!(wgsl.fields.len(), 28); + assert!(wgsl.fields.iter().all(|f| !f.doc.trim().is_empty())); + assert_eq!(std::mem::size_of::(), 4 * 64 + 9 * 16); + assert!( + wgsl.wgsl() + .contains(" camera_position: vec3,\n time: f32,\n") + ); + } +} diff --git a/crates/horizon-shader-api/src/lib.rs b/crates/horizon-shader-api/src/lib.rs new file mode 100644 index 000000000..ff676f64f --- /dev/null +++ b/crates/horizon-shader-api/src/lib.rs @@ -0,0 +1,17 @@ +//! Horizon's own shader packs (docs/api/shaders.md): WGSL written against +//! the renderer's stages, described by `horizon-shaders.toml` and shipped as +//! a folder or a `.hshader`. This crate reads and checks packs; the game, the +//! launcher and the registry all use it, so a pack is judged the same way +//! everywhere. +pub mod check; +pub mod inputs; +pub mod manifest; +pub mod package; +pub mod reference; +pub mod stages; + +pub use check::{Problem, Report, check}; +pub use inputs::Frame; +pub use manifest::{API, MANIFEST, Manifest, OptionKind, PackOption, SCHEMA, Stage}; +pub use package::{EXTENSION, Pack}; +pub use stages::{Composed, StageKind, compose}; diff --git a/crates/horizon-shader-api/src/manifest.rs b/crates/horizon-shader-api/src/manifest.rs new file mode 100644 index 000000000..d03a4f213 --- /dev/null +++ b/crates/horizon-shader-api/src/manifest.rs @@ -0,0 +1,365 @@ +//! `horizon-shaders.toml`, a shader pack's manifest (schema 1, docs/shaders.md). +use serde::Deserialize; + +/// The manifest's file name at the root of a pack. +pub const MANIFEST: &str = "horizon-shaders.toml"; +/// The manifest schema this build reads. +pub const SCHEMA: u32 = 1; +/// The shader API line this build serves: the stages, their signatures and +/// the inputs a pack's WGSL may read. +pub const API: &str = "0.1"; +/// Full-screen passes a pack may chain. +pub const MAX_POST_PASSES: usize = 8; +/// Options a pack may declare. +pub const MAX_OPTIONS: usize = 64; +/// The most GPU time a pack may declare for itself, in milliseconds. +pub const MAX_GPU_MS: f32 = 16.0; + +/// A checked manifest. +#[derive(Clone, Debug, PartialEq)] +pub struct Manifest { + /// `[a-z0-9_]{1,64}`: the pack's settings key and registry id. + pub id: String, + pub name: String, + pub version: String, + /// The API line the pack is written for. + pub api: String, + pub description: String, + pub authors: Vec, + pub license: Option, + /// WGSL files put before every stage's own file, in order. + pub shared: Vec, + pub sky: Option, + pub material: Option, + /// Full-screen passes, in the order they run. + pub post: Vec, + pub options: Vec, + pub budget: Budget, +} + +/// One stage: the WGSL file that defines its function. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct Stage { + pub file: String, + /// Shown in the GPU time overlay; the file name when absent. + #[serde(default)] + pub name: Option, +} + +/// An option the player can change: a WGSL `override` constant of the +/// same name, set when the pipelines are built. +#[derive(Clone, Debug, PartialEq)] +pub struct PackOption { + /// The `override`'s name in the pack's WGSL. + pub id: String, + pub name: String, + pub description: String, + pub kind: OptionKind, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum OptionKind { + /// `override : bool`. + Bool { default: bool }, + /// `override : f32`; the slider moves by `step`. + Float { + default: f32, + min: f32, + max: f32, + step: f32, + }, + /// `override : i32`. + Int { default: i32, min: i32, max: i32 }, + /// `override : u32`, the index of the chosen value. + Choice { values: Vec, default: u32 }, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Budget { + /// The GPU time the pack aims to add to a frame, in milliseconds; the + /// overlay shows the pack over budget past it. + pub gpu_ms: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Raw { + schema: u32, + id: String, + name: String, + version: String, + api: String, + #[serde(default)] + description: String, + #[serde(default)] + authors: Vec, + license: Option, + #[serde(default)] + shared: Vec, + sky: Option, + material: Option, + #[serde(default)] + post: Vec, + #[serde(default)] + options: Vec, + #[serde(default)] + budget: RawBudget, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawOption { + id: String, + name: String, + #[serde(default)] + description: String, + #[serde(rename = "type")] + kind: String, + default: toml::Value, + min: Option, + max: Option, + step: Option, + values: Option>, +} + +#[derive(Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct RawBudget { + gpu_ms: Option, +} + +impl Manifest { + pub fn parse(text: &str) -> Result { + let raw: Raw = toml::from_str(text).map_err(|e| format!("{MANIFEST}: {e}"))?; + let err = |e: String| format!("{MANIFEST}: {e}"); + if raw.schema != SCHEMA { + return Err(err(format!( + "schema {}; this Horizon reads {SCHEMA}", + raw.schema + ))); + } + if raw.api != API { + return Err(err(format!( + "written for shader API {}; this Horizon serves {API}", + raw.api + ))); + } + let id_ok = (1..=64).contains(&raw.id.len()) + && raw + .id + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'); + if !id_ok { + return Err(err(format!("id {:?} is not [a-z0-9_]{{1,64}}", raw.id))); + } + if raw.name.trim().is_empty() || raw.version.trim().is_empty() { + return Err(err("name and version can't be empty".into())); + } + if raw.sky.is_none() && raw.material.is_none() && raw.post.is_empty() { + return Err(err( + "no stage: give it a [sky], a [material] or a [[post]]".into() + )); + } + if raw.post.len() > MAX_POST_PASSES { + return Err(err(format!( + "{} post passes; a pack may chain {MAX_POST_PASSES}", + raw.post.len() + ))); + } + let stages = raw.sky.iter().chain(&raw.material).chain(&raw.post); + for file in raw.shared.iter().chain(stages.map(|s| &s.file)) { + if !crate::package::plain_path(file) || !file.ends_with(".wgsl") { + return Err(err(format!("{file:?} is not a plain path to a .wgsl file"))); + } + } + if raw.options.len() > MAX_OPTIONS { + return Err(err(format!( + "{} options; a pack may declare {MAX_OPTIONS}", + raw.options.len() + ))); + } + let mut options = Vec::with_capacity(raw.options.len()); + for (i, o) in raw.options.into_iter().enumerate() { + let id = o.id.clone(); + let option = o.check().map_err(|e| err(format!("option {id:?}: {e}")))?; + if options[..i].iter().any(|p: &PackOption| p.id == option.id) { + return Err(err(format!("option {id:?} is declared twice"))); + } + options.push(option); + } + let gpu_ms = match raw.budget.gpu_ms { + Some(ms) if !(ms > 0.0 && ms <= f64::from(MAX_GPU_MS)) => { + return Err(err(format!( + "budget.gpu_ms {ms} is not in (0, {MAX_GPU_MS}]" + ))); + } + ms => ms.map(|ms| ms as f32), + }; + Ok(Manifest { + id: raw.id, + name: raw.name, + version: raw.version, + api: raw.api, + description: raw.description, + authors: raw.authors, + license: raw.license, + shared: raw.shared, + sky: raw.sky, + material: raw.material, + post: raw.post, + options, + budget: Budget { gpu_ms }, + }) + } + + /// Every WGSL file the manifest names, shared files first, each once. + pub fn files(&self) -> Vec<&str> { + let mut files: Vec<&str> = Vec::new(); + let stages = self.sky.iter().chain(&self.material).chain(&self.post); + for file in self.shared.iter().chain(stages.map(|s| &s.file)) { + if !files.contains(&file.as_str()) { + files.push(file); + } + } + files + } +} + +/// A WGSL identifier the renderer leaves to packs: ASCII, not starting +/// with a digit, `_` or the renderer's own `horizon_` prefix. +pub fn pack_identifier(name: &str) -> bool { + let mut bytes = name.bytes(); + bytes.next().is_some_and(|b| b.is_ascii_alphabetic()) + && bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') + && name.len() <= 64 + && !name.starts_with("horizon_") +} + +impl RawOption { + fn check(self) -> Result { + if !pack_identifier(&self.id) { + return Err( + "the id must be a WGSL identifier: a letter, then letters, digits or _, up to 64, not starting with horizon_".into(), + ); + } + let int = |v: &toml::Value| { + v.as_integer() + .and_then(|i| i32::try_from(i).ok()) + .ok_or_else(|| format!("{v} is not a 32-bit integer")) + }; + let float = |v: &toml::Value| { + v.as_float() + .or_else(|| v.as_integer().map(|i| i as f64)) + .filter(|f| f.is_finite()) + .map(|f| f as f32) + .ok_or_else(|| format!("{v} is not a number")) + }; + let need = |v: Option, key: &str| v.ok_or_else(|| format!("needs {key}")); + let unused = |present: bool, key: &str| { + if present { + Err(format!("{key} doesn't apply to type = {:?}", self.kind)) + } else { + Ok(()) + } + }; + let kind = match self.kind.as_str() { + "bool" => { + unused(self.min.is_some() || self.max.is_some(), "min/max")?; + unused(self.step.is_some(), "step")?; + unused(self.values.is_some(), "values")?; + let default = self + .default + .as_bool() + .ok_or("default must be true or false")?; + OptionKind::Bool { default } + } + "float" => { + unused(self.values.is_some(), "values")?; + let (min, max) = ( + float(&need(self.min, "min")?)?, + float(&need(self.max, "max")?)?, + ); + let default = float(&self.default)?; + let step = self.step.map_or((max - min) / 100.0, |s| s as f32); + if !(min < max && (min..=max).contains(&default)) { + return Err(format!("needs min < max and default in [{min}, {max}]")); + } + if !(step > 0.0 && step <= max - min) { + return Err(format!("step {step} is not in (0, max - min]")); + } + OptionKind::Float { + default, + min, + max, + step, + } + } + "int" => { + unused(self.step.is_some(), "step")?; + unused(self.values.is_some(), "values")?; + let (min, max) = (int(&need(self.min, "min")?)?, int(&need(self.max, "max")?)?); + let default = int(&self.default)?; + if !(min < max && (min..=max).contains(&default)) { + return Err(format!("needs min < max and default in [{min}, {max}]")); + } + OptionKind::Int { default, min, max } + } + "choice" => { + unused(self.min.is_some() || self.max.is_some(), "min/max")?; + unused(self.step.is_some(), "step")?; + let values = self.values.ok_or("needs values")?; + if !(2..=32).contains(&values.len()) { + return Err("needs 2 to 32 values".into()); + } + if values + .iter() + .enumerate() + .any(|(i, v)| v.trim().is_empty() || values[..i].contains(v)) + { + return Err("values must be distinct and not empty".into()); + } + let default = self + .default + .as_str() + .and_then(|d| values.iter().position(|v| v == d)) + .ok_or("default must be one of the values")?; + OptionKind::Choice { + values, + default: default as u32, + } + } + other => { + return Err(format!("type {other:?} is not bool, float, int or choice")); + } + }; + Ok(PackOption { + id: self.id, + name: self.name, + description: self.description, + kind, + }) + } +} + +impl OptionKind { + /// The WGSL type of the option's `override`. + pub fn wgsl_type(&self) -> &'static str { + match self { + OptionKind::Bool { .. } => "bool", + OptionKind::Float { .. } => "f32", + OptionKind::Int { .. } => "i32", + OptionKind::Choice { .. } => "u32", + } + } + + /// The default as the pipeline constant wgpu takes. + pub fn default_constant(&self) -> f64 { + match *self { + OptionKind::Bool { default } => f64::from(u8::from(default)), + OptionKind::Float { default, .. } => f64::from(default), + OptionKind::Int { default, .. } => f64::from(default), + OptionKind::Choice { default, .. } => f64::from(default), + } + } +} diff --git a/crates/horizon-shader-api/src/package.rs b/crates/horizon-shader-api/src/package.rs new file mode 100644 index 000000000..e2e644a04 --- /dev/null +++ b/crates/horizon-shader-api/src/package.rs @@ -0,0 +1,190 @@ +//! A shader pack on disk: a folder holding `horizon-shaders.toml`, or the +//! same files zipped as a `.hshader` (what the registry and the launcher +//! carry). Both read into one [`Pack`]. +use crate::manifest::{MANIFEST, Manifest}; +use std::{ + collections::BTreeMap, + io::{Read, Write}, + path::Path, +}; + +/// A packaged pack's extension. +pub const EXTENSION: &str = "hshader"; +/// Bytes per WGSL file, per other file, and in all. +pub const MAX_WGSL: u64 = 256 << 10; +const MAX_OTHER: u64 = 1 << 20; +pub const MAX_TOTAL: u64 = 8 << 20; +/// Files in a `.hshader`. +const MAX_ENTRIES: usize = 512; +/// Files a pack may hold beside its manifest and WGSL. +const EXTRAS: [&str; 5] = [ + "README.md", + "LICENSE", + "LICENSE.md", + "LICENSE.txt", + "icon.png", +]; + +/// A pack read from a folder or a `.hshader`: its manifest and every file, +/// by path from the pack's root (`/`-separated). +#[derive(Debug)] +pub struct Pack { + pub manifest: Manifest, + pub files: BTreeMap>, +} + +/// A relative path made of plain parts: no root, `..`, `\`, empty or +/// hidden parts. +pub fn plain_path(path: &str) -> bool { + !path.is_empty() + && path.split('/').all(|part| { + !part.is_empty() + && !part.starts_with('.') + && part + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)) + }) +} + +impl Pack { + /// Reads `path`: a `.hshader` file, or a folder (its hidden files and + /// folders, such as `.git`, are skipped). + pub fn read(path: &Path) -> Result { + let mut files = BTreeMap::new(); + if path.is_dir() { + read_dir(path, "", &mut files)?; + } else { + let file = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut zip = zip::ZipArchive::new(file) + .map_err(|e| format!("{}: not a .{EXTENSION} (zip): {e}", path.display()))?; + if zip.len() > MAX_ENTRIES { + return Err(format!("over {MAX_ENTRIES} files in the package")); + } + for i in 0..zip.len() { + let mut entry = zip.by_index(i).map_err(|e| e.to_string())?; + if entry.is_dir() { + continue; + } + let name = entry.name().to_owned(); + let mut bytes = Vec::new(); + (&mut entry) + .take(MAX_OTHER.max(MAX_WGSL) + 1) + .read_to_end(&mut bytes) + .map_err(|e| format!("{name}: {e}"))?; + if files.insert(name.clone(), bytes).is_some() { + return Err(format!("{name} is in the package twice")); + } + } + } + Pack::from_files(files) + } + + /// Checks a pack's files: a manifest, the WGSL it names (as UTF-8), + /// nothing it doesn't use, and sizes within bounds. + pub fn from_files(files: BTreeMap>) -> Result { + let text = files + .get(MANIFEST) + .ok_or_else(|| format!("{MANIFEST}: missing at the pack's root"))?; + let manifest = Manifest::parse( + std::str::from_utf8(text).map_err(|_| format!("{MANIFEST}: not UTF-8"))?, + )?; + let used = manifest.files(); + let mut total = 0; + for (name, bytes) in &files { + if !plain_path(name) { + return Err(format!("{name:?} is not a plain path")); + } + let size = bytes.len() as u64; + total += size; + let max = if name.ends_with(".wgsl") { + if !used.contains(&name.as_str()) { + return Err(format!("{name} is not used by any stage or in shared")); + } + if std::str::from_utf8(bytes).is_err() { + return Err(format!("{name}: not UTF-8")); + } + MAX_WGSL + } else if name == MANIFEST || EXTRAS.contains(&name.as_str()) { + MAX_OTHER + } else { + return Err(format!( + "{name}: a pack holds only {MANIFEST}, .wgsl files and {}", + EXTRAS.join(", ") + )); + }; + if size > max { + return Err(format!("{name} is over {} KiB", max >> 10)); + } + } + if total > MAX_TOTAL { + return Err(format!("the pack is over {} MiB", MAX_TOTAL >> 20)); + } + if let Some(missing) = used.iter().find(|f| !files.contains_key(**f)) { + return Err(format!("{missing}: named in {MANIFEST} but missing")); + } + Ok(Pack { manifest, files }) + } + + /// A WGSL file's text (every file the manifest names is checked UTF-8). + pub fn source(&self, file: &str) -> Option<&str> { + self.files + .get(file) + .and_then(|b| std::str::from_utf8(b).ok()) + } + + /// The `.hshader` bytes: the files in path order, with fixed times and + /// permissions, so the same pack always gives the same bytes (and the + /// same SHA-256). + pub fn package(&self) -> Result, String> { + let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated) + .last_modified_time(zip::DateTime::default()) + .unix_permissions(0o644); + let err = |e: zip::result::ZipError| e.to_string(); + for (name, bytes) in &self.files { + zip.start_file(name.as_str(), options).map_err(err)?; + zip.write_all(bytes).map_err(|e| e.to_string())?; + } + Ok(zip.finish().map_err(err)?.into_inner()) + } + + /// The SHA-256 of [`Pack::package`], in lowercase hex: the pack's + /// identity in the registry. + pub fn sha256(&self) -> Result { + Ok(sha256_hex(&self.package()?)) + } +} + +/// SHA-256 in lowercase hex. +pub fn sha256_hex(bytes: &[u8]) -> String { + use sha2::Digest; + sha2::Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +fn read_dir(dir: &Path, prefix: &str, files: &mut BTreeMap>) -> Result<(), String> { + let entries = std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + for entry in entries { + let entry = entry.map_err(|e| format!("{}: {e}", dir.display()))?; + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with('.') { + continue; + } + let path = entry.path(); + let rel = format!("{prefix}{name}"); + if path.is_dir() { + read_dir(&path, &format!("{rel}/"), files)?; + } else { + let size = entry.metadata().map_err(|e| format!("{rel}: {e}"))?.len(); + if size > MAX_TOTAL { + return Err(format!("{rel} is over {} MiB", MAX_TOTAL >> 20)); + } + let bytes = std::fs::read(&path).map_err(|e| format!("{rel}: {e}"))?; + files.insert(rel, bytes); + } + } + Ok(()) +} diff --git a/crates/horizon-shader-api/src/reference.rs b/crates/horizon-shader-api/src/reference.rs new file mode 100644 index 000000000..df74b7633 --- /dev/null +++ b/crates/horizon-shader-api/src/reference.rs @@ -0,0 +1,97 @@ +//! docs/api/shaders-reference.md, generated from the stages and inputs this +//! crate defines (`tests/reference.rs` fails when the file drifts). +use crate::check::{MAX_EXPRESSIONS, MAX_TEXTURE_READS}; +use crate::inputs::{Frame, SURFACE, Struct}; +use crate::manifest::{API, MAX_GPU_MS, MAX_OPTIONS, MAX_POST_PASSES}; +use crate::package::{MAX_TOTAL, MAX_WGSL}; +use crate::stages::{GROUP, StageKind}; + +fn table(s: &Struct, offsets: Option<&[usize]>) -> String { + let mut out = format!("### `{}`\n\n{}\n\n", s.name, one_line(s.doc)); + out += if offsets.is_some() { + "| Field | Type | Offset | Meaning |\n| --- | --- | --- | --- |\n" + } else { + "| Field | Type | Meaning |\n| --- | --- | --- |\n" + }; + for (i, f) in s.fields.iter().enumerate() { + let offset = offsets.map_or(String::new(), |o| format!(" {} |", o[i])); + out += &format!( + "| `{}` | `{}` |{offset} {} |\n", + f.name, + f.ty, + one_line(f.doc) + ); + } + out + "\n" +} + +fn one_line(doc: &str) -> String { + doc.split_whitespace().collect::>().join(" ") +} + +pub fn markdown() -> String { + let mut out = format!( + "# Shader API reference ({API})\n\n\ + Generated from `crates/horizon-shader-api` (`HORIZON_WRITE_DOCS=1 cargo test -p horizon-shader-api --test reference`\n\ + rewrites it; the test fails when it drifts). The API's design and pack format: [shaders.md](shaders.md).\n\n\ + A stage is a function your WGSL defines, with the signature below; Horizon puts the prelude (these structs, the\n\ + resources and helpers) before your files and calls the function from its own pass. Your files declare no\n\ + bindings and no entry points, and no names starting with `horizon_`.\n\n\ + ## Stages\n\n\ + | Stage | In `horizon-shaders.toml` | Your function |\n| --- | --- | --- |\n" + ); + for kind in StageKind::ALL { + let key = match kind { + StageKind::Post => "[[post]]", + StageKind::Sky => "[sky]", + StageKind::Material => "[material]", + }; + out += &format!("| {} | `{key}` | `{}` |\n", kind.name(), kind.signature()); + } + out += "\n"; + for kind in StageKind::ALL { + out += &format!("## {}\n\n`{}`\n\n", kind.name(), kind.signature()); + out += &table(kind.input(), None); + if kind == StageKind::Material { + out += &table(&SURFACE, None); + } + out += &format!( + "Resources (bind group {GROUP}):\n\n| Binding | Declaration | Meaning |\n| --- | --- | --- |\n" + ); + for r in kind.resources() { + out += &format!("| {} | `{}` | {} |\n", r.binding, r.ty, r.doc); + } + out += "\n"; + if kind == StageKind::Post { + out += "Helpers:\n\n\ + - `fn depth_at(uv: vec2) -> f32`: the world's depth at `uv`.\n\ + - `fn view_position(uv: vec2, depth: f32) -> vec3`: the camera-relative position, in blocks, of\n \ + the point at `uv` and `depth`.\n\n"; + } + } + out += "## The frame\n\n"; + out += &table(&Frame::WGSL, Some(Frame::OFFSETS)); + out += &format!( + "## Options\n\n\ + Each `[[options]]` entry sets the WGSL `override` of the same name when the pipelines are built:\n\n\ + | `type` | Keys | The `override` |\n| --- | --- | --- |\n\ + | `bool` | `default` | `override : bool` |\n\ + | `float` | `default`, `min`, `max`, `step` | `override : f32` |\n\ + | `int` | `default`, `min`, `max` | `override : i32` |\n\ + | `choice` | `values`, `default` (one of them) | `override : u32`, the value's index |\n\n\ + ## Limits\n\n\ + - {MAX_POST_PASSES} post passes, {MAX_OPTIONS} options, `budget.gpu_ms` up to {MAX_GPU_MS}.\n\ + - {} KiB per WGSL file, {} MiB per pack.\n\ + - Per stage: {MAX_EXPRESSIONS} naga expressions over all its functions, {MAX_TEXTURE_READS} texture reads (samples,\n \ + gathers and loads, the prelude's included).\n\n\ + ## The check\n\n\ + A pack runs only after it passes `horizon_shader_api::check` (the game when it loads a pack, the launcher before it\n\ + installs one, the registry's CI): each stage is parsed and validated by naga, checked against its signature and\n\ + the rules above, and translated for Vulkan (SPIR-V), Metal and DirectX 12 (HLSL) with its options at their\n\ + defaults, minimums and maximums. Every option must be an `override` of its type in some stage; an `override`\n\ + that is no option needs a default.\n", + MAX_WGSL >> 10, + MAX_TOTAL >> 20 + ); + out +} diff --git a/crates/horizon-shader-api/src/stages.rs b/crates/horizon-shader-api/src/stages.rs new file mode 100644 index 000000000..12bd3e038 --- /dev/null +++ b/crates/horizon-shader-api/src/stages.rs @@ -0,0 +1,265 @@ +//! The stages a pack can fill, each a WGSL function with a fixed signature +//! that Horizon calls inside its own passes, and the WGSL put around a +//! pack's files to build a stage: the prelude before (the inputs and the +//! resources), the entry points after. +use crate::inputs::{Frame, MATERIAL_INPUT, POST_INPUT, SKY_INPUT, SURFACE, Struct}; +use crate::package::Pack; + +/// The bind group of a pack's resources; groups 0 to 2 are the renderer's. +pub const GROUP: u32 = 3; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StageKind { + /// The sky's colour along each view ray, before the sun, moon and stars. + Sky, + /// The colour of block surfaces (terrain), before the alpha test and fog. + Material, + /// A full-screen pass over the world's picture, under the hand and HUD. + Post, +} + +/// A resource a stage reads, declared by its prelude. +#[derive(Debug)] +pub struct Resource { + pub binding: u32, + pub name: &'static str, + pub ty: &'static str, + pub doc: &'static str, +} + +const FRAME: Resource = Resource { + binding: 0, + name: "frame", + ty: "var frame: Frame", + doc: "The frame (`Frame`).", +}; + +const POST_RESOURCES: [Resource; 6] = [ + FRAME, + Resource { + binding: 1, + name: "scene_color", + ty: "var scene_color: texture_2d", + doc: "The world's picture before any pass.", + }, + Resource { + binding: 2, + name: "scene_depth", + ty: "var scene_depth: texture_depth_2d", + doc: "The world's depth, reversed (1 near, 0 far and sky).", + }, + Resource { + binding: 3, + name: "previous", + ty: "var previous: texture_2d", + doc: "The previous pass's output (`scene_color` for the first pass).", + }, + Resource { + binding: 4, + name: "linear_sampler", + ty: "var linear_sampler: sampler", + doc: "Bilinear filtering, clamped to the edges.", + }, + Resource { + binding: 5, + name: "nearest_sampler", + ty: "var nearest_sampler: sampler", + doc: "Nearest texel, clamped to the edges.", + }, +]; + +/// Functions the post prelude gives packs. +const POST_HELPERS: &str = "\ +// The world's depth at `uv` (reversed: 0 is the sky). +fn depth_at(uv: vec2) -> f32 { + let size = vec2(textureDimensions(scene_depth)); + let texel = vec2(clamp(uv * size, vec2(0.0), size - 1.0)); + return textureLoad(scene_depth, texel, 0); +} +// The camera-relative world position, in blocks, of the point at `uv` and `depth`. +fn view_position(uv: vec2, depth: f32) -> vec3 { + let clip = vec4(uv.x * 2.0 - 1.0, 1.0 - uv.y * 2.0, depth, 1.0); + let view = frame.projection_inverse * clip; + return (frame.view_inverse * vec4(view.xyz / view.w, 0.0)).xyz; +} +"; + +/// The post stage's entry points, as the renderer builds them: a full-screen +/// triangle, and a fragment that calls the pack's `post` for each pixel. +const POST_ENTRIES: &str = "\ +struct HorizonPostOut { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} +@vertex fn horizon_vs_post(@builtin(vertex_index) i: u32) -> HorizonPostOut { + let p = vec2(f32((i << 1u) & 2u), f32(i & 2u)); + var out: HorizonPostOut; + out.position = vec4(p * 2.0 - 1.0, 0.0, 1.0); + out.uv = vec2(p.x, 1.0 - p.y); + return out; +} +@fragment fn horizon_fs_post(in: HorizonPostOut) -> @location(0) vec4 { + let color = textureSampleLevel(previous, linear_sampler, in.uv, 0.0); + return post(PostInput(in.uv, in.position.xy, color, depth_at(in.uv))); +} +"; + +/// Entry points with the material stage's inputs: the renderer calls +/// `material` from its own terrain shader in the same way. +const MATERIAL_ENTRIES: &str = "\ +struct HorizonMaterialIn { + @builtin(position) position: vec4, + @location(0) albedo: vec4, + @location(1) tint: vec4, + @location(2) light: vec2, + @location(3) lightmap: vec3, + @location(4) world: vec3, + @location(5) uv: vec2, + @location(6) @interpolate(flat) translucent: u32, +} +@fragment fn horizon_fs_material(in: HorizonMaterialIn) -> @location(0) vec4 { + let input = MaterialInput(in.albedo, in.tint, in.light, in.lightmap, in.world, in.uv, in.translucent != 0u); + return material(input).color; +} +"; + +/// Entry points with the sky stage's inputs: the renderer calls `sky` from +/// its own sky shader in the same way. +const SKY_ENTRIES: &str = "\ +struct HorizonSkyIn { + @builtin(position) position: vec4, + @location(0) direction: vec3, + @location(1) color: vec4, +} +@fragment fn horizon_fs_sky(in: HorizonSkyIn) -> @location(0) vec4 { + return sky(SkyInput(normalize(in.direction), in.color)); +} +"; + +impl StageKind { + pub const ALL: [StageKind; 3] = [StageKind::Sky, StageKind::Material, StageKind::Post]; + + /// The manifest's name for the stage (`[sky]`, `[material]`, `[[post]]`). + pub fn name(self) -> &'static str { + match self { + StageKind::Sky => "sky", + StageKind::Material => "material", + StageKind::Post => "post", + } + } + + /// The function a pack defines: it has the stage's name. + pub fn function(self) -> &'static str { + self.name() + } + + pub fn input(self) -> &'static Struct { + match self { + StageKind::Sky => &SKY_INPUT, + StageKind::Material => &MATERIAL_INPUT, + StageKind::Post => &POST_INPUT, + } + } + + /// The type the function returns. + pub fn output(self) -> &'static str { + match self { + StageKind::Material => SURFACE.name, + StageKind::Sky | StageKind::Post => "vec4", + } + } + + /// `fn (input: ) -> `. + pub fn signature(self) -> String { + format!( + "fn {}(input: {}) -> {}", + self.function(), + self.input().name, + self.output() + ) + } + + pub fn resources(self) -> &'static [Resource] { + match self { + StageKind::Post => &POST_RESOURCES, + StageKind::Sky | StageKind::Material => std::slice::from_ref(&FRAME), + } + } + + /// Everything the pack's files may use, declared before them. + pub fn prelude(self) -> String { + let mut out = + String::from("// Horizon shader API prelude (docs/api/shaders-reference.md)\n"); + out += &Frame::WGSL.wgsl(); + out += &self.input().wgsl(); + if self == StageKind::Material { + out += &SURFACE.wgsl(); + } + for r in self.resources() { + out += &format!("@group({GROUP}) @binding({}) {};\n", r.binding, r.ty); + } + if self == StageKind::Post { + out += POST_HELPERS; + } + out + } + + /// The entry points put after the pack's files. + pub fn entries(self) -> &'static str { + match self { + StageKind::Sky => SKY_ENTRIES, + StageKind::Material => MATERIAL_ENTRIES, + StageKind::Post => POST_ENTRIES, + } + } +} + +/// A stage's WGSL, and where each part of it came from. +#[derive(Debug)] +pub struct Composed { + pub text: String, + /// (the file, or `` / ``, its byte range in `text`). + pub parts: Vec<(String, std::ops::Range)>, +} + +impl Composed { + /// The file, line and column (1-based) of a byte offset in `text`. + pub fn locate(&self, offset: usize) -> (&str, usize, usize) { + let (file, range) = self + .parts + .iter() + .find(|(_, r)| r.contains(&offset)) + .or(self.parts.last()) + .expect("a composed stage has parts"); + let before = &self.text[range.start..offset.clamp(range.start, range.end)]; + let line = before.matches('\n').count() + 1; + let column = before.len() - before.rfind('\n').map_or(0, |i| i + 1) + 1; + (file, line, column) + } +} + +/// The WGSL of the stage `kind` whose own file is `file`: the prelude, the +/// pack's shared files in order, the file, then the entry points. +pub fn compose(pack: &Pack, kind: StageKind, file: &str) -> Composed { + let mut composed = Composed { + text: String::new(), + parts: Vec::new(), + }; + let mut push = |name: &str, text: &str| { + let start = composed.text.len(); + composed.text += text; + if !text.ends_with('\n') { + composed.text.push('\n'); + } + composed + .parts + .push((name.to_owned(), start..composed.text.len())); + }; + push("", &kind.prelude()); + for shared in pack.manifest.shared.iter().filter(|s| *s != file) { + push(shared, pack.source(shared).unwrap_or_default()); + } + push(file, pack.source(file).unwrap_or_default()); + push("", kind.entries()); + composed +} diff --git a/crates/horizon-shader-api/templates/material/horizon-shaders.toml b/crates/horizon-shader-api/templates/material/horizon-shaders.toml new file mode 100644 index 000000000..8cb7737e6 --- /dev/null +++ b/crates/horizon-shader-api/templates/material/horizon-shaders.toml @@ -0,0 +1,18 @@ +schema = 1 +id = "{{id}}" +name = "{{name}}" +version = "0.1.0" +api = "0.1" +description = "How block surfaces are lit." + +[material] +file = "material.wgsl" + +[[options]] +id = "contrast" +name = "Light contrast" +type = "float" +default = 1.2 +min = 0.5 +max = 2.0 +step = 0.05 diff --git a/crates/horizon-shader-api/templates/material/material.wgsl b/crates/horizon-shader-api/templates/material/material.wgsl new file mode 100644 index 000000000..ef9d89a2b --- /dev/null +++ b/crates/horizon-shader-api/templates/material/material.wgsl @@ -0,0 +1,9 @@ +// The colour of each point of a block's surface, before the alpha test and +// fog. Vanilla's own: albedo * tint * vec4(lightmap, 1.0). +// Reference: docs/api/shaders-reference.md. +override contrast: f32 = 1.2; + +fn material(input: MaterialInput) -> Surface { + let light = pow(input.lightmap, vec3(contrast)); + return Surface(input.albedo * input.tint * vec4(light, 1.0)); +} diff --git a/crates/horizon-shader-api/templates/post/horizon-shaders.toml b/crates/horizon-shader-api/templates/post/horizon-shaders.toml new file mode 100644 index 000000000..e135ee838 --- /dev/null +++ b/crates/horizon-shader-api/templates/post/horizon-shaders.toml @@ -0,0 +1,22 @@ +schema = 1 +id = "{{id}}" +name = "{{name}}" +version = "0.1.0" +api = "0.1" +description = "A post-processing pass over the world." + +[[post]] +file = "post.wgsl" +name = "Colour" + +[[options]] +id = "saturation" +name = "Saturation" +type = "float" +default = 1.15 +min = 0.0 +max = 2.0 +step = 0.05 + +[budget] +gpu_ms = 0.5 diff --git a/crates/horizon-shader-api/templates/post/post.wgsl b/crates/horizon-shader-api/templates/post/post.wgsl new file mode 100644 index 000000000..d4d179379 --- /dev/null +++ b/crates/horizon-shader-api/templates/post/post.wgsl @@ -0,0 +1,9 @@ +// Runs once per pixel of the world's picture, under the hand and the HUD. +// Reference: docs/api/shaders-reference.md (the inputs, `frame` and the helpers). +override saturation: f32 = 1.15; + +fn post(input: PostInput) -> vec4 { + let grey = dot(input.color.rgb, vec3(0.2126, 0.7152, 0.0722)); + let color = mix(vec3(grey), input.color.rgb, saturation); + return vec4(color, input.color.a); +} diff --git a/crates/horizon-shader-api/templates/sky/horizon-shaders.toml b/crates/horizon-shader-api/templates/sky/horizon-shaders.toml new file mode 100644 index 000000000..5939f25fd --- /dev/null +++ b/crates/horizon-shader-api/templates/sky/horizon-shaders.toml @@ -0,0 +1,18 @@ +schema = 1 +id = "{{id}}" +name = "{{name}}" +version = "0.1.0" +api = "0.1" +description = "A sky of its own." + +[sky] +file = "sky.wgsl" + +[[options]] +id = "glow" +name = "Sun glow" +type = "float" +default = 0.5 +min = 0.0 +max = 1.0 +step = 0.05 diff --git a/crates/horizon-shader-api/templates/sky/sky.wgsl b/crates/horizon-shader-api/templates/sky/sky.wgsl new file mode 100644 index 000000000..b5aaaa7ad --- /dev/null +++ b/crates/horizon-shader-api/templates/sky/sky.wgsl @@ -0,0 +1,9 @@ +// The sky's colour along each view ray; the sun, moon, stars and clouds are +// drawn over it. Reference: docs/api/shaders-reference.md. +override glow: f32 = 0.5; + +fn sky(input: SkyInput) -> vec4 { + let toward_sun = max(dot(input.direction, frame.sun_direction), 0.0); + let warm = vec3(1.0, 0.6, 0.3) * pow(toward_sun, 8.0) * glow; + return vec4(input.color.rgb + warm, input.color.a); +} diff --git a/crates/horizon-shader-api/tests/check.rs b/crates/horizon-shader-api/tests/check.rs new file mode 100644 index 000000000..4c6168a43 --- /dev/null +++ b/crates/horizon-shader-api/tests/check.rs @@ -0,0 +1,146 @@ +//! `check`: the sample pack passes, and each kind of mistake is reported +//! with the pack's own file and line. +use horizon_shader_api::{MANIFEST, Pack, Problem, check}; +use std::path::Path; + +fn sample() -> Pack { + Pack::read(Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/samples/schema-1" + ))) + .unwrap() +} + +/// The sample with `file` replaced by `text`, checked. +fn problems(file: &str, text: &str) -> Vec { + let mut files = sample().files; + files.insert(file.into(), text.as_bytes().to_vec()); + check(&Pack::from_files(files).unwrap()).expect_err("should fail") +} + +fn one(file: &str, text: &str, needle: &str) -> Problem { + let p = problems(file, text); + assert!( + p.iter().any(|p| p.message.contains(needle)), + "{needle:?} not in {p:#?}" + ); + p.into_iter().find(|p| p.message.contains(needle)).unwrap() +} + +const GRADE: &str = "post/grade.wgsl"; + +#[test] +fn the_sample_passes() { + let pack = sample(); + let report = check(&pack).unwrap(); + let labels: Vec<_> = report.stages.iter().map(|s| s.label.as_str()).collect(); + assert_eq!( + labels, + ["sky.wgsl", "Terrain", "Colour grade", "post/vignette.wgsl"] + ); + assert!(report.stages.iter().all(|s| s.expressions > 0)); + let post = &report.stages[2]; + assert!(post.texture_reads >= 2, "the entry point samples and loads"); + let sha = pack.sha256().unwrap(); + assert_eq!(sha.len(), 64); + assert_eq!(sha, sample().sha256().unwrap()); +} + +#[test] +fn a_syntax_error_points_at_its_line() { + let p = one( + GRADE, + "override saturation: f32 = 1.1;\noverride tonemap: u32 = 1u;\nfn post(input: PostInput) -> vec4 {\n return input.color +;\n}\n", + "expected", + ); + assert_eq!((p.file.as_str(), p.line), (GRADE, 4), "{p}"); + assert!(p.to_string().starts_with("post/grade.wgsl:4:"), "{p}"); +} + +#[test] +fn the_stage_function_must_have_its_signature() { + let header = "override saturation: f32 = 1.1;\noverride tonemap: u32 = 1u;\n"; + one( + GRADE, + &format!("{header}fn post(input: PostInput) -> vec3 {{ return input.color.rgb; }}\n"), + "`fn post(PostInput) -> vec3` should be `fn post(input: PostInput) -> vec4`", + ); + one( + GRADE, + &format!("{header}fn grade(input: PostInput) -> vec4 {{ return input.color; }}\n"), + "defines no `fn post(input: PostInput) -> vec4`", + ); +} + +#[test] +fn a_pack_declares_no_bindings_entry_points_or_horizon_names() { + let body = "override saturation: f32 = 1.1;\noverride tonemap: u32 = 1u;\nfn post(input: PostInput) -> vec4 { return input.color; }\n"; + one( + GRADE, + &format!("@group(3) @binding(9) var extra: texture_2d;\n{body}"), + "declares the binding `extra`", + ); + one( + GRADE, + &format!( + "{body}@fragment fn main() -> @location(0) vec4 {{ return vec4(1.0); }}\n" + ), + "declares the entry point `main`", + ); + one( + GRADE, + &format!("{body}fn horizon_mine() {{}}\n"), + "`horizon_mine`: names starting with horizon_", + ); +} + +#[test] +fn overrides_match_the_options() { + let post = "fn post(input: PostInput) -> vec4 { return input.color; }\n"; + one( + GRADE, + &format!("override saturation: i32 = 1;\noverride tonemap: u32 = 1u;\n{post}"), + "option \"saturation\" is a float, so its override is `f32`", + ); + one( + GRADE, + &format!( + "override saturation: f32 = 1.1;\noverride tonemap: u32 = 1u;\noverride extra: f32;\n{post}" + ), + "`override extra` has no default and no [[options]] entry", + ); + let p = one( + GRADE, + &format!("override saturation: f32 = 1.1;\n{post}"), + "option \"tonemap\": no stage declares `override tonemap: u32`", + ); + assert_eq!(p.file, MANIFEST); +} + +#[test] +fn validation_errors_point_into_the_pack() { + // A compute-only barrier in a fragment stage passes the parser, not the validator. + let p = one( + GRADE, + "override saturation: f32 = 1.1;\noverride tonemap: u32 = 1u;\n\ + fn post(input: PostInput) -> vec4 {\n \ + workgroupBarrier();\n \ + return input.color;\n}\n", + "forbidden at this stage", + ); + assert_eq!(p.file, GRADE, "{p}"); +} + +#[test] +fn texture_reads_have_a_budget() { + let reads: String = (0..70) + .map(|i| format!(" c += textureLoad(previous, vec2({i}, 0), 0);\n")) + .collect(); + one( + GRADE, + &format!( + "override saturation: f32 = 1.1;\noverride tonemap: u32 = 1u;\nfn post(input: PostInput) -> vec4 {{\n var c = input.color;\n{reads} return c;\n}}\n" + ), + "texture reads; a stage may write 64", + ); +} diff --git a/crates/horizon-shader-api/tests/reference.rs b/crates/horizon-shader-api/tests/reference.rs new file mode 100644 index 000000000..fc548590b --- /dev/null +++ b/crates/horizon-shader-api/tests/reference.rs @@ -0,0 +1,119 @@ +//! The stages' WGSL against the Rust definitions: the preludes compile, the +//! frame's WGSL layout is the uploaded struct's, and +//! docs/api/shaders-reference.md is up to date. Rewrite the reference with +//! `HORIZON_WRITE_DOCS=1 cargo test -p horizon-shader-api --test reference`. +use horizon_shader_api::{Frame, Pack, StageKind, compose, reference}; +use std::path::Path; + +const DOC: &str = "docs/api/shaders-reference.md"; + +fn root() -> &'static Path { + Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/../..")) +} + +fn validate(text: &str) -> naga::Module { + let module = naga::front::wgsl::parse_str(text) + .unwrap_or_else(|e| panic!("{}\n{text}", e.emit_to_string(text))); + naga::valid::Validator::new( + naga::valid::ValidationFlags::all(), + naga::valid::Capabilities::default(), + ) + .validate(&module) + .unwrap_or_else(|e| panic!("{}\n{text}", e.emit_to_string(text))); + module +} + +/// Vanilla's look, written as each stage. +fn vanilla(kind: StageKind) -> &'static str { + match kind { + StageKind::Sky => "fn sky(input: SkyInput) -> vec4 { return input.color; }\n", + StageKind::Material => { + "fn material(input: MaterialInput) -> Surface {\n \ + return Surface(input.albedo * input.tint * vec4(input.lightmap, 1.0));\n}\n" + } + StageKind::Post => "fn post(input: PostInput) -> vec4 { return input.color; }\n", + } +} + +#[test] +fn every_stage_compiles_with_vanilla_s_function() { + for kind in StageKind::ALL { + let text = format!("{}{}{}", kind.prelude(), vanilla(kind), kind.entries()); + let module = validate(&text); + assert!( + module + .entry_points + .iter() + .all(|e| e.name.starts_with("horizon_")), + "{kind:?}" + ); + } +} + +#[test] +fn the_frame_s_wgsl_layout_is_the_uploaded_struct_s() { + let text = format!("{}{}", StageKind::Sky.prelude(), vanilla(StageKind::Sky)); + let module = validate(&text); + let (_, ty) = module + .types + .iter() + .find(|(_, t)| t.name.as_deref() == Some("Frame")) + .expect("Frame in the prelude"); + let naga::TypeInner::Struct { members, span } = &ty.inner else { + panic!("Frame is a struct") + }; + assert_eq!(*span as usize, std::mem::size_of::()); + let wgsl: Vec<_> = members + .iter() + .map(|m| (m.name.clone().unwrap(), m.offset as usize)) + .collect(); + let rust: Vec<_> = Frame::WGSL + .fields + .iter() + .zip(Frame::OFFSETS) + .map(|(f, &o)| (f.name.to_owned(), o)) + .collect(); + assert_eq!(wgsl, rust); +} + +#[test] +fn the_sample_pack_s_stages_compile() { + let dir = root().join("crates/horizon-shader-api/tests/samples/schema-1"); + let pack = Pack::read(&dir).unwrap(); + let m = &pack.manifest; + let stages = [ + (StageKind::Sky, m.sky.iter().collect::>()), + (StageKind::Material, m.material.iter().collect()), + (StageKind::Post, m.post.iter().collect()), + ]; + for (kind, list) in stages { + for stage in list { + let composed = compose(&pack, kind, &stage.file); + validate(&composed.text); + // Errors point into the pack's own file. + let part = composed.parts.iter().find(|(f, _)| *f == stage.file); + let part = part.unwrap().1.clone(); + let needle = format!("fn {}(", kind.function()); + let at = part.start + composed.text[part].find(&needle).unwrap(); + let source = pack.source(&stage.file).unwrap(); + let line = source.lines().position(|l| l.starts_with(&needle)).unwrap() + 1; + assert_eq!(composed.locate(at), (stage.file.as_str(), line, 1)); + } + } +} + +#[test] +fn the_reference_is_up_to_date() { + let path = root().join(DOC); + let want = reference::markdown(); + if std::env::var_os("HORIZON_WRITE_DOCS").is_some() { + std::fs::write(&path, &want).unwrap(); + } + let have = std::fs::read_to_string(&path) + .unwrap_or_default() + .replace("\r\n", "\n"); + assert!( + have == want, + "{DOC} is out of date: HORIZON_WRITE_DOCS=1 cargo test -p horizon-shader-api --test reference" + ); +} diff --git a/crates/horizon-shader-api/tests/samples/schema-1/README.md b/crates/horizon-shader-api/tests/samples/schema-1/README.md new file mode 100644 index 000000000..a8ad4583b --- /dev/null +++ b/crates/horizon-shader-api/tests/samples/schema-1/README.md @@ -0,0 +1 @@ +The schema 1 sample read by `tests/schema.rs`; `tests/reference.rs` compiles its stages. diff --git a/crates/horizon-shader-api/tests/samples/schema-1/common.wgsl b/crates/horizon-shader-api/tests/samples/schema-1/common.wgsl new file mode 100644 index 000000000..8fef8baf9 --- /dev/null +++ b/crates/horizon-shader-api/tests/samples/schema-1/common.wgsl @@ -0,0 +1,3 @@ +fn luminance(c: vec3) -> f32 { + return dot(c, vec3(0.2126, 0.7152, 0.0722)); +} diff --git a/crates/horizon-shader-api/tests/samples/schema-1/horizon-shaders.toml b/crates/horizon-shader-api/tests/samples/schema-1/horizon-shaders.toml new file mode 100644 index 000000000..6e8a545a7 --- /dev/null +++ b/crates/horizon-shader-api/tests/samples/schema-1/horizon-shaders.toml @@ -0,0 +1,59 @@ +# Schema 1 of horizon-shaders.toml with every key. Readers of later schemas +# keep reading this file (docs/api/stability.md, "Data formats"). +schema = 1 +id = "sample_pack" +name = "Sample Pack" +version = "1.0.0" +api = "0.1" +description = "Every key of schema 1." +authors = ["Horizon"] +license = "MIT" +shared = ["common.wgsl"] + +[sky] +file = "sky.wgsl" + +[material] +file = "material.wgsl" +name = "Terrain" + +[[post]] +file = "post/grade.wgsl" +name = "Colour grade" + +[[post]] +file = "post/vignette.wgsl" + +[[options]] +id = "saturation" +name = "Saturation" +description = "How colourful the picture is." +type = "float" +default = 1.1 +min = 0.0 +max = 2.0 +step = 0.05 + +[[options]] +id = "vignette" +name = "Vignette" +type = "bool" +default = true + +[[options]] +id = "samples" +name = "Samples" +type = "int" +default = 4 +min = 1 +max = 16 + +[[options]] +id = "tonemap" +name = "Tone mapping" +type = "choice" +values = ["None", "Soft", "Filmic"] +default = "Soft" + +[budget] +gpu_ms = 1.5 diff --git a/crates/horizon-shader-api/tests/samples/schema-1/material.wgsl b/crates/horizon-shader-api/tests/samples/schema-1/material.wgsl new file mode 100644 index 000000000..830470a6b --- /dev/null +++ b/crates/horizon-shader-api/tests/samples/schema-1/material.wgsl @@ -0,0 +1,4 @@ +fn material(input: MaterialInput) -> Surface { + let lit = input.albedo * input.tint * vec4(input.lightmap, 1.0); + return Surface(vec4(lit.rgb * (0.9 + 0.1 * luminance(lit.rgb)), lit.a)); +} diff --git a/crates/horizon-shader-api/tests/samples/schema-1/post/grade.wgsl b/crates/horizon-shader-api/tests/samples/schema-1/post/grade.wgsl new file mode 100644 index 000000000..8daa1eab0 --- /dev/null +++ b/crates/horizon-shader-api/tests/samples/schema-1/post/grade.wgsl @@ -0,0 +1,8 @@ +override saturation: f32 = 1.1; +override tonemap: u32 = 1u; + +fn post(input: PostInput) -> vec4 { + let c = input.color.rgb; + let graded = mix(vec3(luminance(c)), c, saturation); + return vec4(select(graded, graded / (graded + 0.5) * 1.5, tonemap > 0u), input.color.a); +} diff --git a/crates/horizon-shader-api/tests/samples/schema-1/post/vignette.wgsl b/crates/horizon-shader-api/tests/samples/schema-1/post/vignette.wgsl new file mode 100644 index 000000000..84f7261cb --- /dev/null +++ b/crates/horizon-shader-api/tests/samples/schema-1/post/vignette.wgsl @@ -0,0 +1,8 @@ +override vignette: bool = true; +override samples: i32 = 4; + +fn post(input: PostInput) -> vec4 { + let d = distance(input.uv, vec2(0.5)); + let dark = select(1.0, 1.0 - smoothstep(0.4, 0.9, d) * 0.5, vignette); + return vec4(input.color.rgb * dark, input.color.a); +} diff --git a/crates/horizon-shader-api/tests/samples/schema-1/sky.wgsl b/crates/horizon-shader-api/tests/samples/schema-1/sky.wgsl new file mode 100644 index 000000000..55ca0ac1d --- /dev/null +++ b/crates/horizon-shader-api/tests/samples/schema-1/sky.wgsl @@ -0,0 +1,3 @@ +fn sky(input: SkyInput) -> vec4 { + return input.color; +} diff --git a/crates/horizon-shader-api/tests/schema.rs b/crates/horizon-shader-api/tests/schema.rs new file mode 100644 index 000000000..2176805ee --- /dev/null +++ b/crates/horizon-shader-api/tests/schema.rs @@ -0,0 +1,238 @@ +//! Schema 1 of `horizon-shaders.toml` and the `.hshader` package. +use horizon_shader_api::{MANIFEST, Manifest, OptionKind, Pack}; +use std::{collections::BTreeMap, path::Path}; + +fn sample_dir() -> &'static Path { + Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/samples/schema-1" + )) +} + +fn sample_text() -> String { + std::fs::read_to_string(sample_dir().join(MANIFEST)).unwrap() +} + +fn refused(text: &str) -> String { + Manifest::parse(text).expect_err("should be refused") +} + +#[test] +fn reads_every_key_of_schema_1() { + let m = Manifest::parse(&sample_text()).unwrap(); + assert_eq!( + ( + m.id.as_str(), + m.name.as_str(), + m.version.as_str(), + m.api.as_str() + ), + ("sample_pack", "Sample Pack", "1.0.0", "0.1") + ); + assert_eq!(m.authors, ["Horizon"]); + assert_eq!(m.license.as_deref(), Some("MIT")); + assert_eq!(m.sky.as_ref().unwrap().file, "sky.wgsl"); + assert_eq!( + m.material.as_ref().unwrap().name.as_deref(), + Some("Terrain") + ); + let post: Vec<_> = m.post.iter().map(|s| s.file.as_str()).collect(); + assert_eq!(post, ["post/grade.wgsl", "post/vignette.wgsl"]); + assert_eq!( + m.files(), + [ + "common.wgsl", + "sky.wgsl", + "material.wgsl", + "post/grade.wgsl", + "post/vignette.wgsl" + ] + ); + let kinds: Vec<_> = m.options.iter().map(|o| (o.id.as_str(), &o.kind)).collect(); + assert_eq!( + kinds, + [ + ( + "saturation", + &OptionKind::Float { + default: 1.1, + min: 0.0, + max: 2.0, + step: 0.05 + } + ), + ("vignette", &OptionKind::Bool { default: true }), + ( + "samples", + &OptionKind::Int { + default: 4, + min: 1, + max: 16 + } + ), + ( + "tonemap", + &OptionKind::Choice { + values: vec!["None".into(), "Soft".into(), "Filmic".into()], + default: 1 + } + ), + ] + ); + let types: Vec<_> = m.options.iter().map(|o| o.kind.wgsl_type()).collect(); + assert_eq!(types, ["f32", "bool", "i32", "u32"]); + let constants: Vec<_> = m + .options + .iter() + .map(|o| o.kind.default_constant()) + .collect(); + assert_eq!(constants, [f64::from(1.1f32), 1.0, 4.0, 1.0]); + assert_eq!(m.budget.gpu_ms, Some(1.5)); +} + +#[test] +fn the_smallest_manifest() { + let m = Manifest::parse( + "schema = 1\nid = \"p\"\nname = \"P\"\nversion = \"1\"\napi = \"0.1\"\n[[post]]\nfile = \"a.wgsl\"\n", + ) + .unwrap(); + assert!(m.sky.is_none() && m.options.is_empty() && m.budget.gpu_ms.is_none()); +} + +#[test] +fn refuses_what_this_build_does_not_read() { + let text = sample_text(); + let cases = [ + (text.replace("schema = 1", "schema = 2"), "schema 2"), + ( + text.replace("api = \"0.1\"", "api = \"0.2\""), + "shader API 0.2", + ), + ( + text.replace("\"sample_pack\"", "\"Sample\""), + "is not [a-z0-9_]", + ), + ( + text.replace("license = \"MIT\"", "licence = \"MIT\""), + "unknown field", + ), + ( + text.replace("gpu_ms = 1.5", "gpu_ms = 40.0"), + "budget.gpu_ms", + ), + ( + text.replace("\"common.wgsl\"", "\"../common.wgsl\""), + "not a plain path", + ), + ( + text.replace("\"sky.wgsl\"", "\"sky.glsl\""), + "not a plain path to a .wgsl", + ), + ( + text.replace("id = \"samples\"", "id = \"vignette\""), + "declared twice", + ), + ( + text.replace("id = \"samples\"", "id = \"horizon_samples\""), + "WGSL identifier", + ), + ( + text.replace("default = \"Soft\"", "default = \"Hard\""), + "one of the values", + ), + ( + text.replace("default = 1.1", "default = 3.0"), + "default in [0, 2]", + ), + ( + text.replace("default = 4", "default = 4.5"), + "not a 32-bit integer", + ), + ( + text.replace("type = \"bool\"", "type = \"color\""), + "is not bool", + ), + ( + text.replace("default = true", "default = true\nmin = 0"), + "doesn't apply", + ), + ]; + for (text, needle) in cases { + let e = refused(&text); + assert!(e.contains(needle), "{needle:?} not in {e:?}"); + assert!(e.starts_with(MANIFEST), "{e}"); + } + let no_stage = "schema = 1\nid = \"p\"\nname = \"P\"\nversion = \"1\"\napi = \"0.1\"\n"; + assert!(refused(no_stage).contains("no stage")); + let many = (0..9).map(|i| format!("[[post]]\nfile = \"p{i}.wgsl\"\n")); + assert!(refused(&format!("{no_stage}{}", many.collect::())).contains("9 post passes")); +} + +#[test] +fn reads_the_sample_folder_and_its_package_alike() { + let folder = Pack::read(sample_dir()).unwrap(); + assert_eq!(folder.files.len(), 7); + assert!( + folder + .source("post/grade.wgsl") + .unwrap() + .contains("fn post") + ); + let bytes = folder.package().unwrap(); + assert_eq!( + bytes, + folder.package().unwrap(), + "packaging is reproducible" + ); + let dir = std::env::temp_dir().join(format!("horizon-shader-api-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("sample_pack.hshader"); + std::fs::write(&path, &bytes).unwrap(); + let packaged = Pack::read(&path).unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + assert_eq!(packaged.files, folder.files); + assert_eq!(packaged.manifest, folder.manifest); +} + +#[test] +fn refuses_files_a_pack_does_not_use() { + let base = || -> BTreeMap> { Pack::read(sample_dir()).unwrap().files }; + let mut files = base(); + files.insert("old.wgsl".into(), b"fn f() {}".to_vec()); + assert!( + Pack::from_files(files) + .unwrap_err() + .contains("old.wgsl is not used") + ); + let mut files = base(); + files.insert("run.sh".into(), Vec::new()); + assert!( + Pack::from_files(files) + .unwrap_err() + .contains("run.sh: a pack holds only") + ); + let mut files = base(); + files.remove("sky.wgsl"); + assert!( + Pack::from_files(files) + .unwrap_err() + .contains("sky.wgsl: named in") + ); + let mut files = base(); + files.insert("sky.wgsl".into(), vec![0xff, 0xfe]); + assert!(Pack::from_files(files).unwrap_err().contains("not UTF-8")); + let mut files = base(); + files.insert("post/grade.wgsl".into(), vec![b' '; 300 << 10]); + assert!( + Pack::from_files(files) + .unwrap_err() + .contains("over 256 KiB") + ); + let mut files = base(); + files.remove(MANIFEST); + assert!( + Pack::from_files(files) + .unwrap_err() + .contains("missing at the pack's root") + ); +} diff --git a/crates/horizon-shader-api/tests/sdk.rs b/crates/horizon-shader-api/tests/sdk.rs new file mode 100644 index 000000000..ef69a88a0 --- /dev/null +++ b/crates/horizon-shader-api/tests/sdk.rs @@ -0,0 +1,92 @@ +//! The SDK: every template and the example pack pass the check, and the +//! `horizon-shader` tool makes, checks and packages a pack. +use horizon_shader_api::{Pack, check}; +use std::{path::Path, process::Command}; + +const TOOL: &str = env!("CARGO_BIN_EXE_horizon-shader"); + +fn scratch(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("horizon-shader-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn run(args: &[&str]) -> (bool, String) { + let out = Command::new(TOOL).args(args).output().unwrap(); + let text = + String::from_utf8_lossy(&out.stdout).into_owned() + &String::from_utf8_lossy(&out.stderr); + (out.status.success(), text) +} + +#[test] +fn the_example_pack_passes() { + let dir = Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/packs/golden_hour")); + let report = check(&Pack::read(dir).unwrap()).unwrap(); + let labels: Vec<_> = report.stages.iter().map(|s| s.label.as_str()).collect(); + assert_eq!(labels, ["Warm horizon", "Glow", "Colour grade"]); +} + +#[test] +fn every_template_makes_a_pack_that_passes() { + let root = scratch("templates"); + for template in ["post", "sky", "material"] { + let dir = root.join(format!("My-{template}")); + let dir_s = dir.to_str().unwrap(); + let (ok, text) = run(&["new", dir_s, "--template", template]); + assert!(ok, "{text}"); + let pack = Pack::read(&dir).unwrap(); + assert_eq!(pack.manifest.id, format!("my_{template}")); + assert_eq!(pack.manifest.name, format!("My {template}")); + let (ok, text) = run(&["check", dir_s]); + assert!(ok && text.contains(": ok"), "{text}"); + } + let (ok, text) = run(&[ + "new", + root.join("x").to_str().unwrap(), + "--template", + "water", + ]); + assert!(!ok && text.contains("no template \"water\""), "{text}"); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn packs_and_checks_the_package() { + let root = scratch("pack"); + let dir = root.join("glow"); + let dir_s = dir.to_str().unwrap(); + assert!(run(&["new", dir_s, "--name", "Glow"]).0); + let out = root.join("glow.hshader"); + let (ok, text) = run(&["pack", dir_s, "--out", out.to_str().unwrap()]); + assert!(ok, "{text}"); + let sha = text + .split("sha256 ") + .nth(1) + .unwrap() + .trim_end_matches([')', '\n']) + .to_owned(); + let (ok, json) = run(&["check", out.to_str().unwrap(), "--json"]); + assert!(ok, "{json}"); + let report: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(report[0]["ok"], true); + assert_eq!(report[0]["id"], "glow"); + assert_eq!(report[0]["sha256"], sha.as_str()); + assert_eq!(report[0]["stages"][0]["stage"], "post"); + + std::fs::write( + dir.join("post.wgsl"), + "fn post(input: PostInput) -> vec4 { return input.color }\n", + ) + .unwrap(); + let (ok, json) = run(&["check", dir_s, "--json"]); + assert!(!ok); + let report: serde_json::Value = + serde_json::from_str(json.split("\nerror:").next().unwrap()).unwrap(); + assert_eq!(report[0]["ok"], false); + assert_eq!(report[0]["problems"][0]["file"], "post.wgsl"); + assert_eq!(report[0]["problems"][0]["line"], 1); + let (ok, text) = run(&["pack", dir_s]); + assert!(!ok && text.contains("failed the check"), "{text}"); + std::fs::remove_dir_all(root).unwrap(); +} diff --git a/crates/horizon-telemetry-server/Cargo.lock b/crates/horizon-telemetry-server/Cargo.lock index e4e7d060e..9b5fc960d 100644 --- a/crates/horizon-telemetry-server/Cargo.lock +++ b/crates/horizon-telemetry-server/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "atomic-waker" version = "1.1.2" @@ -92,6 +98,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -131,6 +146,17 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "foldhash" version = "0.1.5" @@ -235,6 +261,7 @@ dependencies = [ "serde_json", "tokio", "tower", + "ureq", ] [[package]] @@ -473,6 +500,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.4" @@ -701,6 +738,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "slab" version = "0.4.12" @@ -851,10 +894,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ "base64", + "flate2", "log", "once_cell", "rustls", "rustls-pki-types", + "serde", + "serde_json", "url", "webpki-roots 0.26.11", ] @@ -1099,6 +1145,12 @@ dependencies = [ "syn", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/crates/horizon-telemetry-server/Cargo.toml b/crates/horizon-telemetry-server/Cargo.toml index 8dbb0b7fd..486648b5f 100644 --- a/crates/horizon-telemetry-server/Cargo.toml +++ b/crates/horizon-telemetry-server/Cargo.toml @@ -19,6 +19,8 @@ rusqlite = { version = "0.37", features = ["bundled"] } serde = { version = "1", features = ["derive"] } serde_json = "1" ruzstd = "0.9" +# Opens the hub's bug reports as GitHub issues (blocking, run off the async workers; rustls). +ureq = { version = "2.12", features = ["json"] } [dev-dependencies] # The uploader the launcher uses, against this server, end to end. diff --git a/crates/horizon-telemetry-server/README.md b/crates/horizon-telemetry-server/README.md index ed54024a1..a401a72b0 100644 --- a/crates/horizon-telemetry-server/README.md +++ b/crates/horizon-telemetry-server/README.md @@ -1,6 +1,6 @@ # horizon-telemetry-server -This server receives what players choose to send (performance summaries, crash and bug reports, benchmark uploads: [docs/telemetry.md](../../docs/telemetry.md)) and stores it in one SQLite file. It serves anonymous totals to the website. It also keeps the live player count from the games' anonymous "I'm playing" pings, in memory only. It is one small Rust process, about 7 MB of memory when idle. +This server receives what players choose to send (performance summaries, crash and bug reports, benchmark uploads: [docs/telemetry.md](../../docs/telemetry.md)) and stores it in one SQLite file. It serves anonymous totals to the website. It also keeps the live player count from the games' anonymous "I'm playing" pings, in memory only, and takes what players send from the hub (playhorizon.dev/hub): votes, bug reports and suggestions. It is one small Rust process, about 7 MB of memory when idle. cargo run --release # in this folder: its own workspace, like the launcher's PORT=8080 HORIZON_TELEMETRY_DB=/data/telemetry.sqlite3 horizon-telemetry-server @@ -17,8 +17,15 @@ It runs at **https://telemetry.playhorizon.dev** (`DEFAULT_ENDPOINT` in `crates/ | `GET /v1/benchmarks/summary` | Players' benchmark uploads grouped by GPU, CPU, render distance and game, for the website's Benchmarks page (below). Cached 10 minutes, same `Access-Control-Allow-Origin`. | | `POST /v1/live` | A game's "I'm playing" ping (JSON, at most 512 bytes). Returns 204. Returns 400 for a bad ping, 413 for one too large, 429 over the address's limit, 503 when it holds `LIVE_MAX_SESSIONS` already. | | `GET /v1/live` | How many play now, rounded: `playing`, `lan`, `versions`, `platforms`. Made again at most every 15 seconds, with the same `Access-Control-Allow-Origin`. | +| `POST /hub/vote` | `{"item": "", "voter": "", "up": true}`: a 👍 on a roadmap item, one per voter; `"up": false` takes it back. Returns `{"item", "votes"}`. 400 for a bad id, 429 over the address's limit. | +| `GET /hub/votes` | `{"votes": {"": n}}`. Made again at most every minute. | +| `POST /hub/report` | A bug report from the hub's form: `{"title", "did", "happened", "expected", "version"?, "system", "name"?}`. Opens a GitHub issue at once (labels `bug` and `from-website`, the `#horizon-bugs` template, "Reported on playhorizon.dev") and returns 201 `{"id", "issue": ""}`. If GitHub fails: 202 `{"id", "issue": null}`, and the upkeep tries again for a week. The same report twice in a day returns the first one's issue. 400 for an empty or too long field, 429 over the address's limit, 503 past the day's cap or without `GITHUB_TOKEN`. | +| `POST /hub/suggest` | `{"text", "item"?, "name"?}` into the suggestion queue. Returns 202 `{"id"}`. Never published: Kira relays it to the maintainers' private channel. | +| `GET /hub/suggestions?after=&limit=100` | The queue, oldest first, for Kira: `{"suggestions": [{"id", "text", "item"?, "name"?, "created_at"}]}`. Needs `Authorization: Bearer `: 401 without it, 404 when the server has none. | | `GET /healthz` | `ok` | +The hub's posts take JSON from the website (`OPTIONS` preflights answer with its origin). A form's hidden `website` field filled in (a bot) gets 202 and is dropped. + The totals are grouped per version, OS, GPU vendor and render distance. Each group has its sessions, install-days, hours played, the 14 frame-time buckets summed, and the mean p50 and p99 frame times. Groups with fewer than 5 install-days are left out. The totals also list crash counts per version. Benchmark uploads are published only through the summary below, never one by one. ## Benchmarks summary @@ -56,6 +63,8 @@ The live count is a table in memory: a session ID, when it last pinged, its vers The day stored is the day the server received an item. The sender's IP address is never written down. The per-address rate limit keeps addresses in memory for an hour at most. +The hub keeps votes as an item ID and the browser's random ID, and reports and suggestions as their text, for `HORIZON_TELEMETRY_HUB_KEEP_DAYS` (90). Its rate limits key their buckets by a hash of the address with a key drawn at each start, in memory. + Raw rows are deleted after `HORIZON_TELEMETRY_RETENTION_DAYS` (180). The daily totals stay; they hold no install ID. ## Settings @@ -73,6 +82,14 @@ All are environment variables: - `HORIZON_TELEMETRY_LIVE_PER_HOUR` per address, pings and reads (600, bursts of 60: ten players behind one address) - `HORIZON_TELEMETRY_LIVE_WINDOW_SECONDS` (120) - `HORIZON_TELEMETRY_LIVE_MAX_SESSIONS` (200 000) +- `HORIZON_TELEMETRY_HUB_PER_HOUR` votes and vote reads per address (600, bursts of 60: a page load is one read, and several players can share an address) +- `HORIZON_TELEMETRY_HUB_POSTS_PER_HOUR` reports and suggestions per address (20, bursts of 2) +- `HORIZON_TELEMETRY_HUB_REPORTS_PER_DAY` from everyone (30): each opens an issue +- `HORIZON_TELEMETRY_HUB_SUGGESTIONS_PER_DAY` from everyone (200) +- `HORIZON_TELEMETRY_HUB_KEEP_DAYS` (90) +- `HORIZON_TELEMETRY_HUB_TOKEN`: the token Kira reads the suggestions with (unset: `GET /hub/suggestions` is off) +- `HORIZON_TELEMETRY_GITHUB_TOKEN`: a fine-grained token with Issues read and write on the repository only (unset: `POST /hub/report` answers 503 and the form points to GitHub and Discord) +- `HORIZON_TELEMETRY_GITHUB_REPO` (`ludovic111/horizon`) ## Hosting diff --git a/crates/horizon-telemetry-server/src/github.rs b/crates/horizon-telemetry-server/src/github.rs new file mode 100644 index 000000000..a800442e7 --- /dev/null +++ b/crates/horizon-telemetry-server/src/github.rs @@ -0,0 +1,128 @@ +//! A website bug report as a GitHub issue: the text made inert (no mentions, no cross-references, no HTML), +//! written as the template Kira's `/report` and `#horizon-bugs` use, and the call that opens it. +use std::time::Duration; + +use serde_json::{Map, Value, json}; + +/// Opens an issue; its URL, or why not. The server's is [`Filer::github`]; tests give their own. +pub type Filer = dyn Fn(&Issue) -> Result + Send + Sync; + +pub struct Issue { + pub title: String, + pub body: String, + pub labels: Vec, +} + +/// Labels every website report gets. +pub const LABELS: [&str; 2] = ["bug", "from-website"]; + +/// Text people wrote, safe in an issue: no HTML, no `@` mentions, no `#12` or issue-URL cross-references, +/// no closing keywords, no code spans to break out of. The same rules as Kira's `neutralise`. +pub fn neutralise(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let chars: Vec = text.chars().collect(); + let mut i = 0; + while i < chars.len() { + let c = chars[i]; + let next = chars.get(i + 1).copied(); + let word_start = i == 0 || !chars[i - 1].is_alphanumeric(); + match c { + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '`' => out.push('\''), + // A zero-width space after `@` or `#` keeps the text and drops the mention or the link. + '@' | '#' if next.is_some_and(|n| n.is_alphanumeric()) => { + out.push(c); + out.push('\u{200b}'); + } + _ if word_start && closing_keyword(&chars[i..]) => { + out.push(c); + out.push('\u{200b}'); + } + _ => out.push(c), + } + i += 1; + } + // github.com/owner/repo/issues/12 links cross-reference the issue: shown, not linked. + out.replace("github.com/", "github.com\u{200b}/") +} + +fn closing_keyword(rest: &[char]) -> bool { + let word: String = rest + .iter() + .take_while(|c| c.is_alphabetic()) + .flat_map(|c| c.to_lowercase()) + .collect(); + [ + "close", "closes", "closed", "fix", "fixes", "fixed", "resolve", "resolves", "resolved", + ] + .contains(&word.as_str()) +} + +/// Text as a markdown quote, every line prefixed. +fn quote(text: &str) -> String { + text.lines() + .map(|l| format!("> {l}").trim_end().to_owned()) + .collect::>() + .join("\n") +} + +/// The issue for a report's fields (`hub::REPORT`'s names, already checked). +pub fn issue(fields: &Map) -> Issue { + let field = |name: &str| { + fields + .get(name) + .and_then(Value::as_str) + .map(neutralise) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "not given".into()) + }; + let title = neutralise(&field("title").replace(['\n', '\r'], " ")); + let text = [ + format!("**What I did:**\n{}", field("did")), + format!("**What happened:**\n{}", field("happened")), + format!("**What I expected:**\n{}", field("expected")), + format!("**Horizon version:** {}", field("version")), + format!("**OS, CPU, GPU:** {}", field("system")), + ] + .join("\n"); + let by = fields + .get("name") + .and_then(Value::as_str) + .map(|n| neutralise(n).replace(['*', '_', '~', '|', '[', ']'], "")) + .filter(|n| !n.is_empty()) + .map(|n| format!(" by **{n}**")) + .unwrap_or_default(); + let body = format!( + "{}\n\nReported on playhorizon.dev{by}\n\nOpened by the bug form on playhorizon.dev/hub. The reporter has this issue's link; they don't get GitHub notifications.", + quote(&text) + ); + Issue { + title, + body, + labels: LABELS.iter().map(|l| (*l).to_owned()).collect(), + } +} + +/// Opens issues in `repo` (`owner/name`) with `token` (a fine-grained token with Issues: write on that repo +/// only). Errors never carry the token or GitHub's answer body. +pub fn github(repo: String, token: String) -> Box { + Box::new(move |issue: &Issue| { + let response = ureq::post(&format!("https://api.github.com/repos/{repo}/issues")) + .set("Authorization", &format!("Bearer {token}")) + .set("Accept", "application/vnd.github+json") + .set("X-GitHub-Api-Version", "2022-11-28") + .set("User-Agent", "horizon-telemetry-server") + .timeout(Duration::from_secs(20)) + .send_json(json!({ "title": issue.title, "body": issue.body, "labels": issue.labels })); + match response { + Ok(response) => response + .into_json::() + .ok() + .and_then(|v| v["html_url"].as_str().map(str::to_owned)) + .ok_or_else(|| "GitHub's answer had no issue URL".into()), + Err(ureq::Error::Status(code, _)) => Err(format!("GitHub answered {code}")), + Err(ureq::Error::Transport(e)) => Err(format!("could not reach GitHub: {}", e.kind())), + } + }) +} diff --git a/crates/horizon-telemetry-server/src/hub.rs b/crates/horizon-telemetry-server/src/hub.rs new file mode 100644 index 000000000..eb92cd692 --- /dev/null +++ b/crates/horizon-telemetry-server/src/hub.rs @@ -0,0 +1,242 @@ +//! The hub's votes: a 👍 on a roadmap item at playhorizon.dev/hub, and the helpers the hub's other +//! endpoints (`intake`) share. roadmap/README.md documents them for Kira and the website. +//! +//! - `POST /hub/vote`: a 👍 on a roadmap item, one per browser (`voter`, a random id the page makes and +//! keeps); `"up": false` takes it back. Answers the item's count. +//! - `GET /hub/votes`: every item's count, cached a minute. +//! +//! Stored: an item id and the browser's random id per vote. Never an IP address: the rate limits key their +//! buckets by a hash of it with a key drawn at start, in memory. +use std::collections::HashMap; +use std::hash::BuildHasher; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use axum::extract::{Request, State}; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::Response; +use rusqlite::params; +use serde_json::{Map, Value, json}; + +use crate::store::Store; +use crate::{App, json_response, peer, take}; + +pub const SCHEMA: &str = " +CREATE TABLE IF NOT EXISTS hub_votes (item TEXT NOT NULL, voter TEXT NOT NULL, PRIMARY KEY (item, voter)) WITHOUT ROWID; +"; + +/// Items with votes, at most: a vote for a new id past this is refused (ids are only checked for shape). +const MAX_ITEMS: i64 = 1000; +/// The largest body accepted: a report's fields at their limits, as JSON, fit. +const MAX_BODY: usize = 24 * 1024; +const VOTES_FOR: Duration = Duration::from_secs(60); + +/// The hub's buckets: keyed by a keyed hash of the address, never the address itself. +pub type HashedBuckets = HashMap; + +/// A roadmap item id's shape: lowercase words joined by dashes (tools/roadmap.py), at most 48 characters. +pub fn item_id(id: &str) -> bool { + (1..=48).contains(&id.len()) + && !id.starts_with('-') + && !id.ends_with('-') + && !id.contains("--") + && id + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +/// A voter id's shape: what `crypto.randomUUID()` and similar make. +fn voter_id(id: &str) -> bool { + (16..=64).contains(&id.len()) + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} + +impl Store { + /// Adds or takes back a vote; the item's count after it. None when the item would be one too many. + pub fn vote(&mut self, item: &str, voter: &str, up: bool) -> rusqlite::Result> { + let tx = self.db.transaction()?; + if up { + let known = tx + .prepare("SELECT 1 FROM hub_votes WHERE item = ?1 LIMIT 1")? + .exists([item])?; + let items: i64 = + tx.query_row("SELECT count(DISTINCT item) FROM hub_votes", [], |r| { + r.get(0) + })?; + if !known && items >= MAX_ITEMS { + return Ok(None); + } + tx.execute( + "INSERT OR IGNORE INTO hub_votes (item, voter) VALUES (?1, ?2)", + params![item, voter], + )?; + } else { + tx.execute( + "DELETE FROM hub_votes WHERE item = ?1 AND voter = ?2", + params![item, voter], + )?; + } + let count = tx.query_row( + "SELECT count(*) FROM hub_votes WHERE item = ?1", + [item], + |r| r.get(0), + )?; + tx.commit()?; + Ok(Some(count)) + } + + /// Every item's count. + pub fn votes(&self) -> rusqlite::Result> { + let mut statement = self + .db + .prepare("SELECT item, count(*) FROM hub_votes GROUP BY item ORDER BY item")?; + let rows = + statement.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))?; + rows.map(|row| row.map(|(item, n)| (item, Value::from(n)))) + .collect() + } +} + +impl App { + pub(crate) fn hashed(&self, req: &Request) -> u64 { + self.hub_key.hash_one(self.client(req.headers(), peer(req))) + } + + /// Takes a token from the address's hub bucket (votes and reads of the counts). + fn allow_hub(&self, req: &Request) -> bool { + take( + &self.hub_buckets, + self.config.hub_per_hour, + self.hashed(req), + ) + } + + /// The hub's share of `maintain`: idle buckets are forgotten. + pub(crate) fn forget_hub_buckets(&self) { + for buckets in [&self.hub_buckets, &self.hub_post_buckets] { + buckets + .lock() + .unwrap_or_else(|e| e.into_inner()) + .retain(|_, (_, at)| at.elapsed() < Duration::from_secs(3600)); + } + } +} + +/// A JSON answer to the website: its origin may read it, nothing caches it. +pub(crate) fn answer(app: &App, code: StatusCode, body: Value) -> Response { + let mut response = json_response(app, body.to_string(), 0); + *response.status_mut() = code; + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + response +} + +pub(crate) fn refuse(app: &App, code: StatusCode, message: &str) -> Response { + answer(app, code, json!({ "error": message })) +} + +/// A browser's preflight for the JSON posts: the website may post them. +pub async fn preflight(State(app): State>) -> Response { + let mut response = answer(&app, StatusCode::NO_CONTENT, Value::Null); + *response.body_mut() = axum::body::Body::empty(); + let headers = response.headers_mut(); + headers.remove(header::CONTENT_TYPE); + headers.insert( + header::ACCESS_CONTROL_ALLOW_METHODS, + HeaderValue::from_static("POST"), + ); + headers.insert( + header::ACCESS_CONTROL_ALLOW_HEADERS, + HeaderValue::from_static("content-type"), + ); + headers.insert( + header::ACCESS_CONTROL_MAX_AGE, + HeaderValue::from_static("86400"), + ); + response +} + +pub(crate) async fn json_body(req: Request) -> Result { + let bytes = axum::body::to_bytes(req.into_body(), MAX_BODY) + .await + .map_err(|_| "too large")?; + serde_json::from_slice(&bytes).map_err(|_| "send JSON") +} + +/// `POST /hub/vote` `{"item", "voter", "up"}`: 200 `{"item", "votes"}`. +pub async fn vote(State(app): State>, req: Request) -> Response { + if !app.allow_hub(&req) { + return refuse(&app, StatusCode::TOO_MANY_REQUESTS, "too many requests"); + } + let body = match json_body(req).await { + Ok(body) => body, + Err(e) => return refuse(&app, StatusCode::BAD_REQUEST, e), + }; + let item = body["item"].as_str().unwrap_or(""); + let voter = body["voter"].as_str().unwrap_or(""); + if !item_id(item) || !voter_id(voter) { + return refuse(&app, StatusCode::BAD_REQUEST, "an item id and a voter id"); + } + let up = body["up"].as_bool().unwrap_or(true); + let counted = app.store().vote(item, voter, up); + match counted { + Ok(Some(votes)) => { + *app.votes_cache.lock().unwrap_or_else(|e| e.into_inner()) = None; + answer( + &app, + StatusCode::OK, + json!({ "item": item, "votes": votes }), + ) + } + Ok(None) => refuse(&app, StatusCode::BAD_REQUEST, "no such item"), + Err(e) => { + eprintln!("horizon-telemetry-server: could not store a vote: {e}"); + refuse(&app, StatusCode::SERVICE_UNAVAILABLE, "try again later") + } + } +} + +/// `GET /hub/votes`: `{"votes": {"": n}}`, made again at most every minute. +pub async fn votes(State(app): State>, req: Request) -> Response { + if !app.allow_hub(&req) { + return refuse(&app, StatusCode::TOO_MANY_REQUESTS, "too many requests"); + } + let json = { + let mut cache = app.votes_cache.lock().unwrap_or_else(|e| e.into_inner()); + match &*cache { + Some((at, json)) if at.elapsed() < VOTES_FOR => json.clone(), + _ => { + let counts = app.store().votes(); + match counts { + Ok(counts) => { + let json = json!({ "votes": counts }).to_string(); + *cache = Some((Instant::now(), json.clone())); + json + } + Err(e) => { + eprintln!("horizon-telemetry-server: could not read the votes: {e}"); + return refuse(&app, StatusCode::SERVICE_UNAVAILABLE, "try again later"); + } + } + } + } + }; + json_response(&app, json, VOTES_FOR.as_secs()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ids_are_checked_for_shape() { + assert!(item_id("smooth-frames") && item_id("v2")); + for bad in ["", "-a", "a-", "a--b", "Smooth", "a b", &"a".repeat(49)] { + assert!(!item_id(bad), "{bad}"); + } + assert!(voter_id("0b7c3c9e-3f0a-4d8e-9a39-4c1f5a6b7d21") && !voter_id("short")); + } +} diff --git a/crates/horizon-telemetry-server/src/intake.rs b/crates/horizon-telemetry-server/src/intake.rs new file mode 100644 index 000000000..59ce57141 --- /dev/null +++ b/crates/horizon-telemetry-server/src/intake.rs @@ -0,0 +1,370 @@ +//! What players send from the hub (playhorizon.dev/hub) for the maintainers: bug reports and suggestions. +//! roadmap/README.md documents it for Kira and the website. +//! +//! - `POST /hub/report`: a bug report, opened at once as a GitHub issue (labels `bug`, `from-website`) in +//! the template Kira's `/report` uses (`github`). Kept and tried again by the upkeep if GitHub fails. +//! - `POST /hub/suggest`: a suggestion, into the queue. Never published: Kira relays it to the +//! maintainers' private channel. +//! - `GET /hub/suggestions?after=`: the queue for Kira, with the `HORIZON_TELEMETRY_HUB_TOKEN` bearer. +//! +//! Stored: reports and suggestions as their text, for `hub_keep_days`. Never an IP address. +use std::sync::Arc; + +use axum::extract::{Query, Request, State}; +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use rusqlite::params; +use serde::Deserialize; +use serde_json::{Map, Value, json}; + +use crate::hub::{answer, item_id, json_body, refuse}; +use crate::store::Store; +use crate::{App, github, take, utc_now}; + +pub const SCHEMA: &str = " +CREATE TABLE IF NOT EXISTS hub_queue ( + id INTEGER PRIMARY KEY, kind TEXT NOT NULL, day INTEGER NOT NULL, created TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'new', issue TEXT NOT NULL DEFAULT '', body TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS hub_queue_kind ON hub_queue (kind, state, id); +"; + +/// Days the upkeep keeps trying to open a report's issue. +const RETRY_DAYS: u32 = 7; + +/// A field's rules: its name, whether it is needed, its length in characters, whether it may hold lines. +struct Field(&'static str, bool, std::ops::RangeInclusive, bool); + +/// A bug report: the `#horizon-bugs` template's fields. +const REPORT: &[Field] = &[ + Field("title", true, 5..=120, false), + Field("did", true, 3..=2000, true), + Field("happened", true, 3..=2000, true), + Field("expected", true, 3..=2000, true), + Field("version", false, 0..=60, false), + Field("system", true, 3..=200, false), + Field("name", false, 0..=64, false), +]; +const SUGGESTION: &[Field] = &[ + Field("text", true, 10..=1000, true), + Field("item", false, 0..=48, false), + Field("name", false, 0..=64, false), +]; + +/// Text as stored: no control characters but line breaks (where allowed), trimmed. +fn clean(text: &str, lines: bool) -> String { + let text = text.replace("\r\n", "\n"); + let kept: String = text + .chars() + .map(|c| if !lines && c == '\n' { ' ' } else { c }) + .filter(|c| !c.is_control() || (lines && *c == '\n')) + .collect(); + kept.trim().to_owned() +} + +/// The request's fields checked against `rules`: the cleaned object, or what is wrong. +fn fields(body: &Value, rules: &[Field]) -> Result, String> { + let object = body.as_object().ok_or("send a JSON object")?; + let mut out = Map::new(); + for Field(name, needed, length, lines) in rules { + let text = match object.get(*name) { + None | Some(Value::Null) => String::new(), + Some(Value::String(s)) => clean(s, *lines), + Some(_) => return Err(format!("{name} is text")), + }; + if text.is_empty() && !needed { + continue; + } + if !length.contains(&text.chars().count()) { + return Err(format!( + "{name}: {} to {} characters", + length.start(), + length.end() + )); + } + out.insert((*name).to_owned(), Value::String(text)); + } + Ok(out) +} + +impl Store { + /// How many entries of `kind` came in on `day`. + pub fn queued_on(&self, kind: &str, day: u32) -> rusqlite::Result { + self.db.query_row( + "SELECT count(*) FROM hub_queue WHERE kind = ?1 AND day = ?2", + params![kind, day], + |r| r.get(0), + ) + } + + /// Queues an entry: its id, and its issue when the same text already came in since `since` and was + /// opened (a double click, a form sent twice). + pub fn enqueue( + &mut self, + kind: &str, + day: u32, + since: u32, + body: &Map, + ) -> rusqlite::Result<(i64, Option)> { + let body = Value::Object(body.clone()).to_string(); + let same = self + .db + .prepare( + "SELECT id, issue FROM hub_queue WHERE kind = ?1 AND body = ?2 AND day >= ?3 LIMIT 1", + )? + .query_row(params![kind, body, since], |r| { + Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)) + }); + match same { + Ok((id, issue)) => return Ok((id, Some(issue).filter(|i| !i.is_empty()))), + Err(rusqlite::Error::QueryReturnedNoRows) => {} + Err(e) => return Err(e), + } + self.db.execute( + "INSERT INTO hub_queue (kind, day, created, body) VALUES (?1, ?2, ?3, ?4)", + params![kind, day, utc_now(), body], + )?; + Ok((self.db.last_insert_rowid(), None)) + } + + /// Suggestions after `after`, oldest first: `{id, text, item?, name?, created_at}`. + pub fn suggestions(&self, after: i64, limit: u32) -> rusqlite::Result> { + let mut statement = self.db.prepare( + "SELECT id, created, body FROM hub_queue WHERE kind = 'suggestion' AND id > ?1 ORDER BY id LIMIT ?2", + )?; + let rows = statement.query_map(params![after, limit], |r| { + let mut entry: Map = + serde_json::from_str(&r.get::<_, String>(2)?).unwrap_or_default(); + entry.insert("id".into(), r.get::<_, i64>(0)?.into()); + entry.insert("created_at".into(), r.get::<_, String>(1)?.into()); + Ok(Value::Object(entry)) + })?; + rows.collect() + } + + /// Reports whose issue isn't open yet, from `since` on: id and fields. + pub fn unfiled(&self, since: u32) -> rusqlite::Result)>> { + let mut statement = self.db.prepare( + "SELECT id, body FROM hub_queue WHERE kind = 'report' AND state = 'new' AND day >= ?1 ORDER BY id", + )?; + let rows = statement.query_map([since], |r| { + Ok(( + r.get::<_, i64>(0)?, + serde_json::from_str(&r.get::<_, String>(1)?).unwrap_or_default(), + )) + })?; + rows.collect() + } + + /// Marks a report as opened: `issue` is its URL. + pub fn filed(&mut self, id: i64, issue: &str) -> rusqlite::Result<()> { + self.db.execute( + "UPDATE hub_queue SET state = 'filed', issue = ?2 WHERE id = ?1", + params![id, issue], + )?; + Ok(()) + } + + /// Reports from before `retry_from` that never got an issue are given up; entries before `oldest_day` + /// go. + pub fn expire_queue(&mut self, oldest_day: u32, retry_from: u32) -> rusqlite::Result { + self.db.execute( + "UPDATE hub_queue SET state = 'failed' WHERE kind = 'report' AND state = 'new' AND day < ?1", + [retry_from], + )?; + self.db + .execute("DELETE FROM hub_queue WHERE day < ?1", [oldest_day]) + } +} + +impl App { + /// The same for reports and suggestions, a much smaller bucket. + fn allow_post(&self, req: &Request) -> bool { + take( + &self.hub_post_buckets, + self.config.hub_posts_per_hour, + self.hashed(req), + ) + } + + /// Opens the reports whose issue isn't open yet (the upkeep calls it off the async workers; it blocks). + /// Stops at the first failure: GitHub is down or the token is wrong, the next round tries again. + pub fn file_pending(&self) { + let Some(filer) = &self.filer else { return }; + let since = (self.today)().saturating_sub(RETRY_DAYS); + let pending = self.store().unfiled(since).unwrap_or_default(); + for (id, fields) in pending { + match filer(&github::issue(&fields)) { + Ok(url) => self.mark_filed(id, &url), + Err(e) => { + eprintln!("horizon-telemetry-server: report {id} still not opened: {e}"); + return; + } + } + } + } + + fn mark_filed(&self, id: i64, url: &str) { + let marked = self.store().filed(id, url); + if let Err(e) = marked { + eprintln!("horizon-telemetry-server: report {id} opened as {url}, not marked: {e}"); + } + } + + /// The intake's share of `maintain`: entries past `hub_keep_days` go, reports never opened after + /// [`RETRY_DAYS`] are given up. + pub(crate) fn expire_intake(&self) -> rusqlite::Result { + let today = (self.today)(); + self.store().expire_queue( + today.saturating_sub(self.config.hub_keep_days), + today.saturating_sub(RETRY_DAYS), + ) + } +} + +/// What `receive` keeps: the entry's id, its issue if already opened, its fields. +type Received = (i64, Option, Map); + +/// Checks a report or a suggestion and queues it. +async fn receive( + app: &App, + req: Request, + kind: &str, + rules: &[Field], + per_day: u32, +) -> Result> { + let no = |code: StatusCode, message: &str| Box::new(refuse(app, code, message)); + if !app.allow_post(&req) { + return Err(no(StatusCode::TOO_MANY_REQUESTS, "too many requests")); + } + let body = json_body(req) + .await + .map_err(|e| no(StatusCode::BAD_REQUEST, e))?; + // The form's hidden field, which people leave empty and form-filling bots don't: thanked, dropped. + if body["website"].as_str().is_some_and(|s| !s.is_empty()) { + let thanks = answer(app, StatusCode::ACCEPTED, json!({ "id": 0 })); + return Err(Box::new(thanks)); + } + let entry = fields(&body, rules).map_err(|e| no(StatusCode::BAD_REQUEST, &e))?; + if let Some(item) = entry.get("item").and_then(Value::as_str) + && !item_id(item) + { + return Err(no(StatusCode::BAD_REQUEST, "item: a roadmap item id")); + } + let today = (app.today)(); + let mut store = app.store(); + if store.queued_on(kind, today).unwrap_or(u32::MAX) >= per_day { + let full = "too many today: please try again tomorrow"; + return Err(no(StatusCode::SERVICE_UNAVAILABLE, full)); + } + match store.enqueue(kind, today, today.saturating_sub(1), &entry) { + Ok((id, issue)) => Ok((id, issue, entry)), + Err(e) => { + eprintln!("horizon-telemetry-server: could not queue a {kind}: {e}"); + Err(no(StatusCode::SERVICE_UNAVAILABLE, "try again later")) + } + } +} + +/// `POST /hub/report` `{"title", "did", "happened", "expected", "version"?, "system", "name"?}`: 201 +/// `{"id", "issue"}` once the issue is open, 202 `{"id", "issue": null}` when GitHub failed and the upkeep +/// will try again; 503 when reports are off (no GitHub token). +pub async fn report(State(app): State>, req: Request) -> Response { + let Some(filer) = app.filer.clone() else { + return refuse( + &app, + StatusCode::SERVICE_UNAVAILABLE, + "reports are closed here: use GitHub or Discord", + ); + }; + let per_day = app.config.hub_reports_per_day; + let (id, issue, fields) = match receive(&app, req, "report", REPORT, per_day).await { + Ok(received) => received, + Err(response) => return *response, + }; + if let Some(issue) = issue { + return answer( + &app, + StatusCode::CREATED, + json!({ "id": id, "issue": issue }), + ); + } + let opened = tokio::task::spawn_blocking(move || filer(&github::issue(&fields))) + .await + .map_err(|e| e.to_string()) + .and_then(|opened| opened); + match opened { + Ok(url) => { + app.mark_filed(id, &url); + answer(&app, StatusCode::CREATED, json!({ "id": id, "issue": url })) + } + Err(e) => { + eprintln!("horizon-telemetry-server: report {id} not opened yet: {e}"); + answer( + &app, + StatusCode::ACCEPTED, + json!({ "id": id, "issue": null }), + ) + } + } +} + +/// `POST /hub/suggest` `{"text", "item"?, "name"?}`: 202 `{"id"}`. +pub async fn suggest(State(app): State>, req: Request) -> Response { + let per_day = app.config.hub_suggestions_per_day; + match receive(&app, req, "suggestion", SUGGESTION, per_day).await { + Ok((id, ..)) => answer(&app, StatusCode::ACCEPTED, json!({ "id": id })), + Err(response) => *response, + } +} + +#[derive(Deserialize)] +pub struct After { + after: Option, + limit: Option, +} + +/// `GET /hub/suggestions?after=&limit=100` with `Authorization: Bearer `: +/// `{"suggestions": [{"id", "text", "item"?, "name"?, "created_at"}]}`, oldest first. 404 when no token is +/// set, 401 for a wrong one. +pub async fn suggestions( + State(app): State>, + headers: HeaderMap, + Query(after): Query, +) -> Response { + let Some(token) = app.config.hub_token.as_deref() else { + return StatusCode::NOT_FOUND.into_response(); + }; + let sent = headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .unwrap_or(""); + if !same(sent.as_bytes(), token.as_bytes()) { + return StatusCode::UNAUTHORIZED.into_response(); + } + let limit = after.limit.unwrap_or(100).clamp(1, 500); + let listed = app.store().suggestions(after.after.unwrap_or(0), limit); + match listed { + Ok(list) => ( + [ + ( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ), + (header::CACHE_CONTROL, HeaderValue::from_static("no-store")), + ], + json!({ "suggestions": list }).to_string(), + ) + .into_response(), + Err(e) => { + eprintln!("horizon-telemetry-server: could not read the suggestions: {e}"); + StatusCode::SERVICE_UNAVAILABLE.into_response() + } + } +} + +/// Equal, in a time that doesn't depend on where they differ. +fn same(a: &[u8], b: &[u8]) -> bool { + a.len() == b.len() && a.iter().zip(b).fold(0, |diff, (x, y)| diff | (x ^ y)) == 0 +} diff --git a/crates/horizon-telemetry-server/src/lib.rs b/crates/horizon-telemetry-server/src/lib.rs index 1a8e750e6..3ab923969 100644 --- a/crates/horizon-telemetry-server/src/lib.rs +++ b/crates/horizon-telemetry-server/src/lib.rs @@ -7,6 +7,7 @@ //! least 3 installs only (`summary`), cached; the website's Benchmarks page reads it. //! - `POST /v1/live`: a game's anonymous "I'm playing" ping (`live`, in memory only). //! - `GET /v1/live`: how many play now, rounded, cached briefly. +//! - `/hub/…`: the hub's votes (`hub`), bug reports and suggestions (`intake`). //! - `GET /healthz`. //! //! Abuse limits: a token bucket per IP address (kept in memory for an hour, never stored), a @@ -15,6 +16,7 @@ //! rows expire after `retention_days`; the daily totals stay. One small process and one SQLite //! file: see `README.md` for running it. use std::collections::HashMap; +use std::hash::{Hash, RandomState}; use std::net::{IpAddr, SocketAddr}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -27,10 +29,14 @@ use axum::routing::{delete, get, post}; use horizon_telemetry::batch; pub mod classes; +pub mod github; +pub mod hub; +pub mod intake; pub mod live; pub mod store; pub mod summary; +use github::Filer; use horizon_telemetry::live::{MAX_PING_BYTES, Ping}; use live::Live; use store::Store; @@ -56,6 +62,22 @@ pub struct Config { pub live_window: Duration, /// Sessions the live count holds at most; a new one past that is refused. pub live_max_sessions: usize, + /// The hub's votes and reads of their counts an IP address may make an hour. + pub hub_per_hour: f64, + /// Bug reports and suggestions together an IP address may send an hour. + pub hub_posts_per_hour: f64, + /// Bug reports taken a day, from everyone: each one opens a GitHub issue. + pub hub_reports_per_day: u32, + /// Suggestions taken a day, from everyone. + pub hub_suggestions_per_day: u32, + /// Days reports and suggestions are kept (Kira has relayed them long before). + pub hub_keep_days: u32, + /// The bearer token `GET /hub/suggestions` wants (Kira's); unset, it answers 404. + pub hub_token: Option, + /// Opens the website's bug reports as issues; unset, `POST /hub/report` answers 503. + pub github_token: Option, + /// Where those issues go, `owner/name`. + pub github_repo: String, } impl Default for Config { @@ -70,6 +92,14 @@ impl Default for Config { live_per_hour: 600.0, live_window: horizon_telemetry::live::WINDOW, live_max_sessions: 200_000, + hub_per_hour: 600.0, + hub_posts_per_hour: 20.0, + hub_reports_per_day: 30, + hub_suggestions_per_day: 200, + hub_keep_days: 90, + hub_token: None, + github_token: None, + github_repo: "ludovic111/horizon".into(), } } } @@ -77,9 +107,16 @@ impl Default for Config { impl Config { /// The defaults, with `HORIZON_TELEMETRY_RETENTION_DAYS`, `…_MIN_INSTALL_DAYS`, /// `…_REQUESTS_PER_HOUR`, `…_ITEMS_PER_DAY`, `…_TRUST_PROXY` (1), `…_ALLOW_ORIGIN`, - /// `…_LIVE_PER_HOUR`, `…_LIVE_WINDOW_SECONDS` and `…_LIVE_MAX_SESSIONS`. + /// `…_LIVE_PER_HOUR`, `…_LIVE_WINDOW_SECONDS`, `…_LIVE_MAX_SESSIONS`, `…_HUB_PER_HOUR`, + /// `…_HUB_POSTS_PER_HOUR`, `…_HUB_REPORTS_PER_DAY`, `…_HUB_SUGGESTIONS_PER_DAY`, `…_HUB_KEEP_DAYS`, + /// `…_HUB_TOKEN`, `…_GITHUB_TOKEN` and `…_GITHUB_REPO`. Empty tokens count as unset. pub fn from_env() -> Self { let var = |k: &str| std::env::var(format!("HORIZON_TELEMETRY_{k}")).ok(); + let secret = |k: &str| { + var(k) + .map(|v| v.trim().to_owned()) + .filter(|v| !v.is_empty()) + }; let d = Self::default(); Self { retention_days: var("RETENTION_DAYS") @@ -105,6 +142,24 @@ impl Config { live_max_sessions: var("LIVE_MAX_SESSIONS") .and_then(|v| v.parse().ok()) .unwrap_or(d.live_max_sessions), + hub_per_hour: var("HUB_PER_HOUR") + .and_then(|v| v.parse().ok()) + .unwrap_or(d.hub_per_hour), + hub_posts_per_hour: var("HUB_POSTS_PER_HOUR") + .and_then(|v| v.parse().ok()) + .unwrap_or(d.hub_posts_per_hour), + hub_reports_per_day: var("HUB_REPORTS_PER_DAY") + .and_then(|v| v.parse().ok()) + .unwrap_or(d.hub_reports_per_day), + hub_suggestions_per_day: var("HUB_SUGGESTIONS_PER_DAY") + .and_then(|v| v.parse().ok()) + .unwrap_or(d.hub_suggestions_per_day), + hub_keep_days: var("HUB_KEEP_DAYS") + .and_then(|v| v.parse().ok()) + .unwrap_or(d.hub_keep_days), + hub_token: secret("HUB_TOKEN"), + github_token: secret("GITHUB_TOKEN"), + github_repo: var("GITHUB_REPO").unwrap_or(d.github_repo), } } } @@ -126,6 +181,16 @@ pub struct App { summary_cache: Mutex>, /// Today, in days since 1970 (a test moves it). today: Box u32 + Send + Sync>, + /// The key the hub's buckets hash addresses with, drawn at start and never stored. + hub_key: RandomState, + /// The hub's votes and vote counts, per hashed address. + hub_buckets: Mutex, + /// The hub's reports and suggestions, per hashed address. + hub_post_buckets: Mutex, + /// `GET /hub/votes`'s answer and when it was made. + votes_cache: Mutex>, + /// Opens bug reports as GitHub issues; None without a token. + filer: Option>, } type Buckets = HashMap; @@ -145,6 +210,20 @@ impl App { store: Store, config: Config, today: Box u32 + Send + Sync>, + ) -> Arc { + let filer = config + .github_token + .clone() + .map(|token| Arc::from(github::github(config.github_repo.clone(), token))); + Self::with_filer(store, config, today, filer) + } + + /// [`Self::with_clock`] with the hub's bug reports opened by `filer` (tests give their own). + pub fn with_filer( + store: Store, + config: Config, + today: Box u32 + Send + Sync>, + filer: Option>, ) -> Arc { let live = Live::new(config.live_window, config.live_max_sessions); Arc::new(Self { @@ -157,6 +236,11 @@ impl App { cache: Mutex::default(), summary_cache: Mutex::default(), today, + hub_key: RandomState::new(), + hub_buckets: Mutex::default(), + hub_post_buckets: Mutex::default(), + votes_cache: Mutex::default(), + filer, }) } @@ -217,10 +301,13 @@ impl App { } /// Daily upkeep: totals rebuilt, expired rows removed, idle rate-limit entries and live - /// sessions forgotten. + /// sessions forgotten, the hub's old reports and suggestions removed. Reports still waiting for + /// their issue are [`Self::file_pending`]'s. pub fn maintain(&self) -> rusqlite::Result { let oldest = (self.today)().saturating_sub(self.config.retention_days); let removed = self.store().expire(oldest)?; + self.expire_intake()?; + self.forget_hub_buckets(); for buckets in [&self.buckets, &self.live_buckets] { buckets .lock() @@ -235,14 +322,14 @@ impl App { } } -/// Takes a token for `ip` from its bucket in `buckets` (`per_hour`, bursts of a tenth of it); +/// Takes a token for `key` from its bucket in `buckets` (`per_hour`, bursts of a tenth of it); /// false when it has none left. -fn take(buckets: &Mutex, per_hour: f64, ip: IpAddr) -> bool { +fn take(buckets: &Mutex>, per_hour: f64, key: K) -> bool { let rate = per_hour / 3600.0; let burst = (per_hour / 10.0).max(1.0); let mut buckets = buckets.lock().unwrap_or_else(|e| e.into_inner()); let now = Instant::now(); - let (tokens, at) = buckets.entry(ip).or_insert((burst, now)); + let (tokens, at) = buckets.entry(key).or_insert((burst, now)); *tokens = (*tokens + now.duration_since(*at).as_secs_f64() * rate).min(burst); *at = now; if *tokens < 1.0 { @@ -482,6 +569,14 @@ pub fn router(app: Arc) -> Router { .route("/v1/installs/{id}", delete(forget)) .route("/v1/aggregate", get(aggregate)) .route("/v1/benchmarks/summary", get(benchmarks)) + .route("/hub/vote", post(hub::vote).options(hub::preflight)) + .route("/hub/votes", get(hub::votes)) + .route("/hub/report", post(intake::report).options(hub::preflight)) + .route( + "/hub/suggest", + post(intake::suggest).options(hub::preflight), + ) + .route("/hub/suggestions", get(intake::suggestions)) .route("/healthz", get(|| async { "ok" })) .with_state(app) } diff --git a/crates/horizon-telemetry-server/src/main.rs b/crates/horizon-telemetry-server/src/main.rs index 9b9c702c3..728846a0f 100644 --- a/crates/horizon-telemetry-server/src/main.rs +++ b/crates/horizon-telemetry-server/src/main.rs @@ -22,6 +22,9 @@ async fn main() { if let Err(e) = upkeep.maintain() { eprintln!("horizon-telemetry-server: upkeep failed: {e}"); } + // The hub's reports GitHub refused before: their issues, off the async workers. + let filing = upkeep.clone(); + let _ = tokio::task::spawn_blocking(move || filing.file_pending()).await; } }); let listener = tokio::net::TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], port))) diff --git a/crates/horizon-telemetry-server/src/store.rs b/crates/horizon-telemetry-server/src/store.rs index 02c8803ae..9286dd36a 100644 --- a/crates/horizon-telemetry-server/src/store.rs +++ b/crates/horizon-telemetry-server/src/store.rs @@ -123,13 +123,15 @@ pub fn gpu_vendor(gpu: &str) -> &'static str { } pub struct Store { - db: Connection, + pub(crate) db: Connection, } impl Store { pub fn open(path: &str) -> rusqlite::Result { let mut db = Connection::open(path)?; db.execute_batch(SCHEMA)?; + db.execute_batch(crate::hub::SCHEMA)?; + db.execute_batch(crate::intake::SCHEMA)?; migrate(&mut db)?; Ok(Self { db }) } diff --git a/crates/horizon-telemetry-server/tests/github.rs b/crates/horizon-telemetry-server/tests/github.rs new file mode 100644 index 000000000..81f480b58 --- /dev/null +++ b/crates/horizon-telemetry-server/tests/github.rs @@ -0,0 +1,41 @@ +//! A website bug report as the issue GitHub gets: the template, and people's text made inert. +use horizon_telemetry_server::github::{issue, neutralise}; +use serde_json::{Map, Value, json}; + +#[test] +fn people_text_mentions_and_references_nothing() { + let text = + neutralise("@octocat see #12 and https://github.com/a/b/issues/3 `x` Fixes it"); + assert!(!text.contains("@o") && !text.contains("#1") && !text.contains("github.com/")); + assert!(!text.contains('<') && !text.contains('`')); + assert!(text.contains("F\u{200b}ixes")); + // Words that only start like a keyword stay as they are. + assert_eq!(neutralise("closet fixture"), "closet fixture"); +} + +#[test] +fn a_report_reads_as_the_template() { + let fields: Map = serde_json::from_value(json!({ + "title": "Water is black\non AMD", "did": "Opened a world", "happened": "Black water", + "expected": "Blue water", "system": "Windows 11, RX 6600", "name": "steve_" + })) + .unwrap(); + let issue = issue(&fields); + assert_eq!(issue.title, "Water is black on AMD"); + assert!( + issue + .body + .starts_with("> **What I did:**\n> Opened a world\n> **What happened:**") + ); + assert!( + issue + .body + .contains("> **Horizon version:** not given\n> **OS, CPU, GPU:** Windows 11, RX 6600") + ); + assert!( + issue + .body + .contains("Reported on playhorizon.dev by **steve**") + ); + assert_eq!(issue.labels, ["bug", "from-website"]); +} diff --git a/crates/horizon-telemetry-server/tests/hub.rs b/crates/horizon-telemetry-server/tests/hub.rs new file mode 100644 index 000000000..056978212 --- /dev/null +++ b/crates/horizon-telemetry-server/tests/hub.rs @@ -0,0 +1,398 @@ +//! The hub's intake through its HTTP interface: votes, bug reports (opened as issues by a fake GitHub), +//! suggestions and the queue Kira reads. +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use horizon_telemetry_server::github::{Filer, Issue}; +use horizon_telemetry_server::store::Store; +use horizon_telemetry_server::{App, Config, router}; +use http_body_util::BodyExt; +use serde_json::{Value, json}; +use tower::ServiceExt; + +const DAY: u32 = 20_741; +const VOTER: &str = "0b7c3c9e-3f0a-4d8e-9a39-4c1f5a6b7d21"; + +/// The issues the fake GitHub opened, and whether it is up. +#[derive(Default)] +struct GitHub { + opened: Mutex>, + down: Mutex, +} + +struct Hub { + app: Arc, + github: Arc, + day: Arc, +} + +fn hub(config: Config) -> Hub { + let github = Arc::new(GitHub::default()); + let fake = github.clone(); + let filer: Arc = Arc::new(move |issue: &Issue| { + if *fake.down.lock().unwrap() { + return Err("GitHub answered 502".into()); + } + let mut opened = fake.opened.lock().unwrap(); + opened.push(Issue { + title: issue.title.clone(), + body: issue.body.clone(), + labels: issue.labels.clone(), + }); + Ok(format!( + "https://github.com/ludovic111/horizon/issues/{}", + 1000 + opened.len() + )) + }); + let day = Arc::new(AtomicU32::new(DAY)); + let clock = day.clone(); + let app = App::with_filer( + Store::open(":memory:").unwrap(), + config, + Box::new(move || clock.load(Ordering::SeqCst)), + Some(filer), + ); + Hub { app, github, day } +} + +fn config() -> Config { + Config { + hub_token: Some("kira-secret".into()), + hub_posts_per_hour: 1000.0, + ..Config::default() + } +} + +impl Hub { + async fn send(&self, req: Request) -> (StatusCode, Value) { + let res = router(self.app.clone()).oneshot(req).await.unwrap(); + let code = res.status(); + let body = res.into_body().collect().await.unwrap().to_bytes(); + (code, serde_json::from_slice(&body).unwrap_or(Value::Null)) + } + + async fn post(&self, path: &str, body: Value) -> (StatusCode, Value) { + self.send( + Request::post(path) + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ) + .await + } + + async fn get(&self, path: &str, token: Option<&str>) -> (StatusCode, Value) { + let mut req = Request::get(path); + if let Some(token) = token { + req = req.header("authorization", format!("Bearer {token}")); + } + self.send(req.body(Body::empty()).unwrap()).await + } +} + +fn report(title: &str) -> Value { + json!({ + "title": title, + "did": "Opened a new world\nWalked to the sea", + "happened": "The water is black", + "expected": "Blue water, as in vanilla", + "version": "v0.2.57", + "system": "Windows 11, Ryzen 5 5600, RX 6600", + "website": "", + }) +} + +#[tokio::test] +async fn votes_count_once_per_browser_and_can_be_taken_back() { + let hub = hub(config()); + let vote = |up: bool, voter: &str| json!({ "item": "smooth-frames", "voter": voter, "up": up }); + assert_eq!( + hub.post("/hub/vote", vote(true, VOTER)).await, + ( + StatusCode::OK, + json!({ "item": "smooth-frames", "votes": 1 }) + ) + ); + assert_eq!( + hub.post("/hub/vote", vote(true, VOTER)).await.1["votes"], + 1, + "a second vote changes nothing" + ); + let other = "f".repeat(32); + assert_eq!( + hub.post("/hub/vote", vote(true, &other)).await.1["votes"], + 2 + ); + assert_eq!( + hub.get("/hub/votes", None).await, + (StatusCode::OK, json!({ "votes": { "smooth-frames": 2 } })) + ); + assert_eq!( + hub.post("/hub/vote", vote(false, VOTER)).await.1["votes"], + 1 + ); + assert_eq!( + hub.get("/hub/votes", None).await.1["votes"]["smooth-frames"], + 1, + "a vote drops the cached counts" + ); + for bad in [ + json!({ "item": "Not An Id", "voter": VOTER }), + json!({ "item": "a", "voter": "short" }), + json!([]), + ] { + assert_eq!(hub.post("/hub/vote", bad).await.0, StatusCode::BAD_REQUEST); + } +} + +#[tokio::test] +async fn a_report_becomes_an_issue_in_the_bug_template() { + let hub = hub(config()); + let (code, body) = hub + .post("/hub/report", report("Water is black on AMD")) + .await; + assert_eq!(code, StatusCode::CREATED); + assert_eq!( + body["issue"], + "https://github.com/ludovic111/horizon/issues/1001" + ); + let issue = hub.github.opened.lock().unwrap().remove(0); + assert_eq!(issue.title, "Water is black on AMD"); + assert_eq!(issue.labels, ["bug", "from-website"]); + for line in [ + "> **What I did:**\n> Opened a new world\n> Walked to the sea", + "> **What happened:**\n> The water is black", + "> **What I expected:**\n> Blue water, as in vanilla", + "> **Horizon version:** v0.2.57", + "> **OS, CPU, GPU:** Windows 11, Ryzen 5 5600, RX 6600", + "Reported on playhorizon.dev", + ] { + assert!(issue.body.contains(line), "{line}\n---\n{}", issue.body); + } + // The same report again (a double click) answers the same issue and opens nothing. + let (code, again) = hub + .post("/hub/report", report("Water is black on AMD")) + .await; + assert_eq!( + (code, &again["issue"]), + (StatusCode::CREATED, &body["issue"]) + ); + assert!(hub.github.opened.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn reports_need_every_template_field_and_skip_bots() { + let hub = hub(config()); + for empty in ["title", "did", "happened", "expected", "system"] { + let mut body = report("Water is black on AMD"); + body[empty] = json!(" "); + let (code, answer) = hub.post("/hub/report", body).await; + assert_eq!(code, StatusCode::BAD_REQUEST, "{empty}"); + assert!(answer["error"].as_str().unwrap().starts_with(empty)); + } + let mut bot = report("Cheap watches"); + bot["website"] = json!("https://spam.example"); + assert_eq!( + hub.post("/hub/report", bot).await, + (StatusCode::ACCEPTED, json!({ "id": 0 })) + ); + assert!(hub.github.opened.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn a_report_github_refused_is_opened_by_the_upkeep() { + let hub = hub(config()); + *hub.github.down.lock().unwrap() = true; + let (code, body) = hub + .post("/hub/report", report("Crash when opening a chest")) + .await; + assert_eq!((code, &body["issue"]), (StatusCode::ACCEPTED, &Value::Null)); + hub.app.file_pending(); + assert_eq!(hub.github.opened.lock().unwrap().len(), 0); + *hub.github.down.lock().unwrap() = false; + hub.app.file_pending(); + assert_eq!(hub.github.opened.lock().unwrap().len(), 1); + hub.app.file_pending(); + assert_eq!(hub.github.opened.lock().unwrap().len(), 1, "opened once"); +} + +#[tokio::test] +async fn reports_are_closed_without_a_github_token() { + let app = App::with_clock( + Store::open(":memory:").unwrap(), + Config::default(), + Box::new(|| DAY), + ); + let res = router(app) + .oneshot( + Request::post("/hub/report") + .header("content-type", "application/json") + .body(Body::from(report("Water is black").to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::SERVICE_UNAVAILABLE); +} + +#[tokio::test] +async fn suggestions_wait_in_the_queue_for_kira() { + let hub = hub(config()); + let first = json!({ "text": " Add a waypoint mod\r\nto the\u{7} browser ", "item": "mod-registry", "name": "alex", "extra": 1 }); + let (code, body) = hub.post("/hub/suggest", first).await; + assert_eq!(code, StatusCode::ACCEPTED); + let first_id = body["id"].as_i64().unwrap(); + hub.post( + "/hub/suggest", + json!({ "text": "Bigger render distance on the Steam Deck" }), + ) + .await; + assert_eq!( + hub.post("/hub/suggest", json!({ "text": "too short" })) + .await + .0, + StatusCode::BAD_REQUEST + ); + assert_eq!( + hub.post( + "/hub/suggest", + json!({ "text": "A long enough idea", "item": "Bad Id" }) + ) + .await + .0, + StatusCode::BAD_REQUEST + ); + + assert_eq!( + hub.get("/hub/suggestions", None).await.0, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + hub.get("/hub/suggestions", Some("wrong-secret")).await.0, + StatusCode::UNAUTHORIZED + ); + let (code, all) = hub.get("/hub/suggestions", Some("kira-secret")).await; + assert_eq!(code, StatusCode::OK); + let list = all["suggestions"].as_array().unwrap(); + assert_eq!(list.len(), 2); + assert_eq!( + list[0]["text"], "Add a waypoint mod\nto the browser", + "trimmed, no control characters" + ); + assert!(list[0].get("extra").is_none()); + assert_eq!(list[0]["item"], "mod-registry"); + assert_eq!(list[0]["name"], "alex"); + assert!(list[0]["created_at"].as_str().unwrap().ends_with('Z')); + assert!(list[1].get("name").is_none()); + let (_, after) = hub + .get( + &format!("/hub/suggestions?after={first_id}"), + Some("kira-secret"), + ) + .await; + assert_eq!(after["suggestions"].as_array().unwrap().len(), 1); + + let closed = hub_without_token(); + assert_eq!( + closed.get("/hub/suggestions", Some("anything")).await.0, + StatusCode::NOT_FOUND + ); +} + +fn hub_without_token() -> Hub { + hub(Config::default()) +} + +#[tokio::test] +async fn posts_are_rate_limited_per_address_and_capped_per_day() { + let hub = hub(Config { + hub_posts_per_hour: 20.0, + hub_suggestions_per_day: 3, + ..config() + }); + // A burst of a tenth of the hourly rate: two, then refused. + let idea = |n: u32| json!({ "text": format!("Idea number {n}, long enough") }); + assert_eq!( + hub.post("/hub/suggest", idea(1)).await.0, + StatusCode::ACCEPTED + ); + assert_eq!( + hub.post("/hub/suggest", idea(2)).await.0, + StatusCode::ACCEPTED + ); + assert_eq!( + hub.post("/hub/suggest", idea(3)).await.0, + StatusCode::TOO_MANY_REQUESTS + ); + + let busy = self::hub(Config { + hub_posts_per_hour: 1000.0, + hub_suggestions_per_day: 3, + ..config() + }); + for n in 0..3 { + assert_eq!( + busy.post("/hub/suggest", idea(n)).await.0, + StatusCode::ACCEPTED + ); + } + assert_eq!( + busy.post("/hub/suggest", idea(9)).await.0, + StatusCode::SERVICE_UNAVAILABLE + ); + busy.day.store(DAY + 1, Ordering::SeqCst); + assert_eq!( + busy.post("/hub/suggest", idea(9)).await.0, + StatusCode::ACCEPTED, + "a new day" + ); +} + +#[tokio::test] +async fn old_entries_go_and_reports_never_opened_are_given_up() { + let hub = hub(config()); + *hub.github.down.lock().unwrap() = true; + let (_, body) = hub + .post("/hub/report", report("Crash when opening a chest")) + .await; + assert!(body["id"].as_i64().unwrap() > 0); + hub.post("/hub/suggest", json!({ "text": "An idea from long ago" })) + .await; + hub.day.store(DAY + 8, Ordering::SeqCst); + hub.app.maintain().unwrap(); + *hub.github.down.lock().unwrap() = false; + hub.app.file_pending(); + assert!( + hub.github.opened.lock().unwrap().is_empty(), + "past a week a report is given up" + ); + hub.day.store(DAY + 91, Ordering::SeqCst); + hub.app.maintain().unwrap(); + let (_, left) = hub.get("/hub/suggestions", Some("kira-secret")).await; + assert!(left["suggestions"].as_array().unwrap().is_empty()); +} + +#[tokio::test] +async fn the_website_may_post_from_the_browser() { + let hub = hub(config()); + let res = router(hub.app.clone()) + .oneshot( + Request::options("/hub/report") + .header("origin", "https://playhorizon.dev") + .header("access-control-request-method", "POST") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::NO_CONTENT); + let header = |name: &str| res.headers()[name].to_str().unwrap().to_owned(); + assert_eq!( + header("access-control-allow-origin"), + "https://playhorizon.dev" + ); + assert_eq!(header("access-control-allow-methods"), "POST"); + assert_eq!(header("access-control-allow-headers"), "content-type"); +} diff --git a/docs/README.md b/docs/README.md index 1450ba388..19117ab37 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Horizon documentation -To play, use [the launcher](../launcher/README.md); the project site is [playhorizon.dev](https://playhorizon.dev). These pages are for developers. +To play, use [the launcher](../launcher/README.md); the project site is [playhorizon.dev](https://playhorizon.dev). These pages are for developers. What is being done now and next, the work board and how to help: [the hub](https://playhorizon.dev/hub) ([ROADMAP.md](../ROADMAP.md), [roadmap/README.md](../roadmap/README.md) for keeping it current). ## Build and run @@ -13,6 +13,7 @@ To play, use [the launcher](../launcher/README.md); the project site is [playhor - [Graphics](graphics.md): the Simple and Advanced graphics settings, the graphics benchmark and what it measured. - [Benchmark](benchmark.md): Horizon against vanilla 26.3 and vanilla with performance mods, on the same machine, world, path and settings: what is measured, how it is kept fair, how to reproduce it. - [Extras](extras.md): optional zoom, minimap, FPS and keystroke overlays and auto-reconnect, all off by default. +- [Shader packs](shaders.md): Horizon's own WGSL shader packs, how to make and check one (`horizon-shader`). - [Mods](mods.md): Horizon's own mod system (Rust, sandboxed WebAssembly), how to make and install mods. - [Modpacks](modpacks.md): porting whole packs (ATM10 and smaller): import, inventory, carry-over, an agent per mod with a harness against the Java original, coverage, and how "10x" is measured. - [Launcher](../launcher/README.md): installs, updates and starts builds; its CLI and MCP server; releasing it. diff --git a/docs/api/README.md b/docs/api/README.md index ad49ff609..ef3649105 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -51,5 +51,6 @@ Every page has the same parts: **Overview** (what you can build), **Guide** (whe **Examples**, **Stability** (the level of each part, per [stability.md](stability.md)), **Changelog** and **Not there yet** (what is planned, so nobody builds on a guess). -The website's summary for players and authors is [playhorizon.dev/docs/apis](https://playhorizon.dev/docs/apis) +The website shows these pages at [playhorizon.dev/docs/api](https://playhorizon.dev/docs/api), built from this +folder, and a summary for players and authors at [playhorizon.dev/docs/apis](https://playhorizon.dev/docs/apis) (`website/docs/apis.md`). diff --git a/docs/api/mod-index.md b/docs/api/mod-index.md index beb6ec2c8..6b807d950 100644 --- a/docs/api/mod-index.md +++ b/docs/api/mod-index.md @@ -138,9 +138,24 @@ Unknown keys are refused. Its CI runs `cargo-horizon-mod` (`crates/cargo-horizon | `cargo horizon-mod check [--json]` | A submission's checks: its kind's (above), its signature with `--key` and `--signature`, and for a mod a trial run in the sandbox (40 ticks, 120 frames, a chat line, each command). | | `cargo horizon-mod registry check --files ` | `check` on each file `head` lists and `base` doesn't: a pull request's new files. | | `cargo horizon-mod registry list ` | ` ` per version: the files to download. | -| `cargo horizon-mod registry build --files --out index.json` | Reads every entry and its file (`/`), lists every problem, and writes the index with URLs `/-/`. | +| `cargo horizon-mod registry build --files --out index.json` | Reads every entry and its file (`/`), lists every problem, and writes the index with URLs `-/` (`--release`, ending in `/`, defaults to the repository's releases). | | `cargo horizon-mod registry review --author ` | A pull request's changes: a published file never changes and is never removed (take it down in `revoked.toml`), a kind never changes; changes by a non-owner and to owners or keys are pointed out for the reviewers. | +How the repository runs (its workflows are in `.github/workflows/`): + +- **A pull request** (`check.yml`) runs `registry review`, downloads each new file from the `staging` prerelease, runs + `registry check` on it and builds the index it would publish; the report is the run's summary. The tool is + `cargo-horizon-mod` from Horizon at the commit in the repository variable `HORIZON_REV`. The form's pull requests + are opened with the token of the account in the variable `REGISTRY_BOT`; for those, the author is the login in the + body's `` line. +- **Merging** needs the `check` run and a CODEOWNER's review. `publish.yml` then makes each new version a release + `-` and deploys `index.json` and `icons/` to GitHub Pages: + `https://ludovic111.github.io/horizon-mods/index.json`, served with `Access-Control-Allow-Origin: *`. +- **Weekly**, `cleanup.yml` deletes staged copies of published files after 7 days and of files no open pull request + lists after 30. +- **Taking a file down** is a pull request adding it to `revoked.toml`. Removing an entry outright (a test, a project + nobody installed) fails `registry review` on purpose and needs an administrator's merge. + Kinds open to submissions: `mod`, `modpack`, `resourcepack`. `shader` is in the schema and stays closed until Horizon's own shader pack format exists (Iris and OptiFine packs are for the Modrinth browser); its check will compile the pack in a headless Horizon. A new kind gets its own check before it opens. diff --git a/docs/api/mods.md b/docs/api/mods.md index bec03e84e..5c323a0e1 100644 --- a/docs/api/mods.md +++ b/docs/api/mods.md @@ -19,11 +19,14 @@ the player before the mod is turned on, and checked against the component's real | Capabilities | 5: `game-read`, `chat`, `hud`, `storage`, `registries` | | Manifest `api` lines served | 2: `0.1` (code), `0.2` (content) | | SDK | the `Mod` trait (5 methods), `Registration` (4 builders), `export_mod!`, log helpers | -| `cargo horizon-mod` | `new`, `build`, `run` | +| `cargo horizon-mod` | `new` (templates `code` and `content`), `build`, `run`, `check` | | Example mods | `coords-hud` (code), `ruby-content` (content) | ## Guide +The step-by-step guide, from installing the tools to publishing on the registry, is +[Make a mod](https://playhorizon.dev/docs/make-a-mod) on the website. In short: + 1. `rustup target add wasm32-unknown-unknown`, `cargo install --git https://github.com/ludovic111/horizon cargo-horizon-mod`, then `cargo horizon-mod new my-mod` ([Making a mod](../mods.md#making-a-mod)). 2. Implement `Mod` for your type, list the capabilities you import in `horizon-mod.toml` @@ -55,6 +58,7 @@ scripting runtime in the game. | Way in | For | Status | Template and example | | --- | --- | --- | --- | | Rust | most mods; the fastest | first-class: the SDK, `cargo horizon-mod` | `cargo horizon-mod new`, `coords-hud` | +| Data only | blocks and items with no code | first-class: content mods (API 0.2) | `cargo horizon-mod new --template content`, `ruby-content` | | Lua scripts | people who don't want a compiler | planned: a script mod whose code is a `.lua` file, run by Horizon's own interpreter mod | planned, with the same example as Rust | | C, C++, Zig | authors of those languages | to evaluate: bindings generated from the WIT, built without WASI | one each if it passes | | Go (TinyGo), AssemblyScript, JavaScript / TypeScript | authors of those languages | to evaluate: each needs its toolchain to emit a component without WASI imports, or an interpreter mod as for Lua | one each if it passes | diff --git a/docs/api/shaders-reference.md b/docs/api/shaders-reference.md new file mode 100644 index 000000000..f9682a1a4 --- /dev/null +++ b/docs/api/shaders-reference.md @@ -0,0 +1,162 @@ +# Shader API reference (0.1) + +Generated from `crates/horizon-shader-api` (`HORIZON_WRITE_DOCS=1 cargo test -p horizon-shader-api --test reference` +rewrites it; the test fails when it drifts). The API's design and pack format: [shaders.md](shaders.md). + +A stage is a function your WGSL defines, with the signature below; Horizon puts the prelude (these structs, the +resources and helpers) before your files and calls the function from its own pass. Your files declare no +bindings and no entry points, and no names starting with `horizon_`. + +## Stages + +| Stage | In `horizon-shaders.toml` | Your function | +| --- | --- | --- | +| sky | `[sky]` | `fn sky(input: SkyInput) -> vec4` | +| material | `[material]` | `fn material(input: MaterialInput) -> Surface` | +| post | `[[post]]` | `fn post(input: PostInput) -> vec4` | + +## sky + +`fn sky(input: SkyInput) -> vec4` + +### `SkyInput` + +One direction of the sky, for `fn sky(input: SkyInput) -> vec4`. The sun, moon, stars and clouds are drawn over the result. + +| Field | Type | Meaning | +| --- | --- | --- | +| `direction` | `vec3` | Unit view ray, camera-relative world axes. | +| `color` | `vec4` | Vanilla's sky colour along the ray (sky, fog and sunrise). | + +Resources (bind group 3): + +| Binding | Declaration | Meaning | +| --- | --- | --- | +| 0 | `var frame: Frame` | The frame (`Frame`). | + +## material + +`fn material(input: MaterialInput) -> Surface` + +### `MaterialInput` + +One point of a block's surface, for `fn material(input: MaterialInput) -> Surface`. + +| Field | Type | Meaning | +| --- | --- | --- | +| `albedo` | `vec4` | The block texture's colour here (alpha 0..1). | +| `tint` | `vec4` | The vertex colour: biome tint, face shade and ambient occlusion. | +| `light` | `vec2` | Block and sky light, 0..1, smoothed over the face. | +| `lightmap` | `vec3` | Vanilla's light colour for that light this frame. | +| `position` | `vec3` | The point, camera-relative, in blocks. | +| `uv` | `vec2` | The texture coordinate inside the block atlas. | +| `translucent` | `bool` | Drawn in the translucent pass (water, stained glass, ice). | + +### `Surface` + +What `material` returns. Vanilla's own is `albedo * tint * vec4(lightmap, 1.0)`; Horizon then applies the alpha test (cutout and translucent) and fog. + +| Field | Type | Meaning | +| --- | --- | --- | +| `color` | `vec4` | The lit colour, 0..1, with its alpha. | + +Resources (bind group 3): + +| Binding | Declaration | Meaning | +| --- | --- | --- | +| 0 | `var frame: Frame` | The frame (`Frame`). | + +## post + +`fn post(input: PostInput) -> vec4` + +### `PostInput` + +One pixel of the picture, for `fn post(input: PostInput) -> vec4`. + +| Field | Type | Meaning | +| --- | --- | --- | +| `uv` | `vec2` | Where the pixel is, 0..1 from the top left. | +| `pixel` | `vec2` | The pixel's centre, in pixels from the top left. | +| `color` | `vec4` | The previous pass's colour here (the world's picture for the first pass). | +| `depth` | `f32` | The world's depth here, reversed: 0 is the sky. | + +Resources (bind group 3): + +| Binding | Declaration | Meaning | +| --- | --- | --- | +| 0 | `var frame: Frame` | The frame (`Frame`). | +| 1 | `var scene_color: texture_2d` | The world's picture before any pass. | +| 2 | `var scene_depth: texture_depth_2d` | The world's depth, reversed (1 near, 0 far and sky). | +| 3 | `var previous: texture_2d` | The previous pass's output (`scene_color` for the first pass). | +| 4 | `var linear_sampler: sampler` | Bilinear filtering, clamped to the edges. | +| 5 | `var nearest_sampler: sampler` | Nearest texel, clamped to the edges. | + +Helpers: + +- `fn depth_at(uv: vec2) -> f32`: the world's depth at `uv`. +- `fn view_position(uv: vec2, depth: f32) -> vec3`: the camera-relative position, in blocks, of + the point at `uv` and `depth`. + +## The frame + +### `Frame` + +The frame, as `frame` in every stage. Positions and directions are in world axes relative to the camera (x east, y up, z south), so they stay precise far from the world's origin; depth is reversed (1 near, 0 at the far plane and in the sky). + +| Field | Type | Offset | Meaning | +| --- | --- | --- | --- | +| `view` | `mat4x4` | 0 | Camera-relative world to view space: the camera's rotation. | +| `projection` | `mat4x4` | 64 | View to clip space (reversed depth, wgpu's clip space: z 0..1). | +| `view_inverse` | `mat4x4` | 128 | View to camera-relative world space. | +| `projection_inverse` | `mat4x4` | 192 | Clip to view space. | +| `camera_position` | `vec3` | 256 | The camera's world position, in blocks (about a millimetre of precision 10 000 blocks out). | +| `time` | `f32` | 268 | Seconds since the pack was loaded, wrapping to 0 every hour. | +| `sun_direction` | `vec3` | 272 | Unit vector toward the sun (below the horizon at night). | +| `day_time` | `f32` | 284 | The time of day, 0..1: 0 sunrise, 0.25 noon, 0.5 sunset, 0.75 midnight. | +| `moon_direction` | `vec3` | 288 | Unit vector toward the moon. | +| `game_time` | `f32` | 300 | Vanilla's GameTime: (game time % 24000 + partial tick) / 24000, 0..1. | +| `sky_color` | `vec3` | 304 | Vanilla's sky colour this frame, 0..1. | +| `rain` | `f32` | 316 | Rain strength, 0..1. | +| `fog_color` | `vec3` | 320 | Vanilla's fog colour this frame (also the clear colour), 0..1. | +| `thunder` | `f32` | 332 | Thunder strength, 0..1. | +| `resolution` | `vec2` | 336 | The size of the world's picture, in pixels (the resolution scale applied). | +| `eye_light` | `vec2` | 344 | Block and sky light at the camera, 0..1 (level / 15). | +| `fog_start` | `f32` | 352 | Where vanilla's fog starts, in blocks from the camera. | +| `fog_end` | `f32` | 356 | Where vanilla's fog is full, in blocks. | +| `render_distance` | `f32` | 360 | The render distance, in blocks. | +| `far` | `f32` | 364 | The far plane, in blocks. | +| `night_vision` | `f32` | 368 | Night vision strength, 0..1. | +| `blindness` | `f32` | 372 | Blindness strength, 0..1. | +| `darkness` | `f32` | 376 | The darkness effect's strength, 0..1. | +| `brightness` | `f32` | 380 | The Brightness option, 0..1 (Moody to Bright). | +| `eye_in` | `u32` | 384 | What the camera is in: 0 air, 1 water, 2 lava, 3 powder snow. | +| `dimension` | `i32` | 388 | 0 the Overworld, -1 the Nether, 1 the End, 2 any other dimension. | +| `frame_index` | `u32` | 392 | Frames drawn since the pack was loaded (wraps). | +| `hide_gui` | `u32` | 396 | 1 while F1 hides the HUD, else 0. | + +## Options + +Each `[[options]]` entry sets the WGSL `override` of the same name when the pipelines are built: + +| `type` | Keys | The `override` | +| --- | --- | --- | +| `bool` | `default` | `override : bool` | +| `float` | `default`, `min`, `max`, `step` | `override : f32` | +| `int` | `default`, `min`, `max` | `override : i32` | +| `choice` | `values`, `default` (one of them) | `override : u32`, the value's index | + +## Limits + +- 8 post passes, 64 options, `budget.gpu_ms` up to 16. +- 256 KiB per WGSL file, 8 MiB per pack. +- Per stage: 20000 naga expressions over all its functions, 64 texture reads (samples, + gathers and loads, the prelude's included). + +## The check + +A pack runs only after it passes `horizon_shader_api::check` (the game when it loads a pack, the launcher before it +installs one, the registry's CI): each stage is parsed and validated by naga, checked against its signature and +the rules above, and translated for Vulkan (SPIR-V), Metal and DirectX 12 (HLSL) with its options at their +defaults, minimums and maximums. Every option must be an `override` of its type in some stage; an `override` +that is no option needs a default. diff --git a/docs/api/shaders.md b/docs/api/shaders.md index 10d1ae3de..8e9e6919c 100644 --- a/docs/api/shaders.md +++ b/docs/api/shaders.md @@ -34,10 +34,55 @@ how the world is fed to the GPU. benchmark ([benchmark-report.md](../benchmark-report.md)) records the pack and its cost, so "fast" stays measured. - **Off by default.** Without a pack the renderer is vanilla's look, and no pack code runs. +## The pack format (schema 1) + +A pack is a folder, or the same files zipped as a `.hshader`, with `horizon-shaders.toml` at its root +(crate `horizon-shader-api`; the sample with every key is +`crates/horizon-shader-api/tests/samples/schema-1/`): + +```toml +schema = 1 # the manifest's format; a reader refuses schemas newer than it knows +id = "soft_light" # [a-z0-9_]{1,64}: the settings key and the registry id +name = "Soft Light" +version = "1.0.0" +api = "0.1" # the shader API line the WGSL is written for +description = "..." # optional, like authors and license +shared = ["common.wgsl"] # optional: WGSL put before every stage's file, in order + +[sky] # each stage is optional, but a pack has at least one +file = "sky.wgsl" + +[material] +file = "material.wgsl" + +[[post]] # full-screen passes, in the order they run (8 at most) +file = "post/grade.wgsl" +name = "Colour grade" # optional: the GPU time overlay's label + +[[options]] # 64 at most; `id` is the WGSL `override` it sets +id = "saturation" +name = "Saturation" +type = "float" # bool, float (min, max, step), int (min, max), choice (values) +default = 1.1 +min = 0.0 +max = 2.0 +step = 0.05 + +[budget] +gpu_ms = 1.5 # optional: the GPU time the pack aims to add (0 to 16 ms) +``` + +An option's `override` has the type of its kind: `bool`, `f32`, `i32`, or `u32` for a choice (the index of the +value). The pack holds only the manifest, the `.wgsl` files it names (each used, UTF-8, 256 KiB at most), +`README.md`, `LICENSE` (`.md`, `.txt`) and `icon.png`, under plain paths, 8 MiB in all. A `.hshader` is built +the same way every time (files in path order, fixed dates), so a pack's SHA-256 depends only on its files. + ## Guide, reference and examples -None yet. When the first stage lands: a guide here, a generated reference (`shaders-reference.md`) and an example pack -in the repository that the renderer's tests load. +- [shaders-reference.md](shaders-reference.md): every stage's signature, input and output structs, resources and + helpers, the frame's fields with their units, option types and limits, generated from `crates/horizon-shader-api`. +- [The guide](../shaders.md): making a pack with `horizon-shader new`, `check` and `pack`; the templates and the + example pack (`crates/horizon-shader-api/packs/golden_hour`). ## Stability @@ -46,9 +91,13 @@ pack exist ([stability.md](stability.md)). ## Not there yet -Everything on this page. The order: the renderer hooks shared with the format loader, then the material and post +The stages and everything that runs a pack (the format above is read and checked). The order: the renderer hooks shared with the format loader, then the material and post stages with options and hot reload, then the overlay, lighting, sky and shadow stages. ## Changelog +- 2026-10-25: `horizon-shader` (`new`, `check`, `pack`), three templates, the example pack and the guide. +- 2026-10-25: the check (naga validation, signatures, options, budgets, SPIR-V/Metal/HLSL translation). +- 2026-10-25: the stages of API 0.1 (sky, material, post), their inputs and the generated reference. +- 2026-10-25: the pack format, schema 1: `horizon-shaders.toml` and the `.hshader` package. - 2026-10-24: the earlier shader pack code was removed; this API is designed from scratch. diff --git a/docs/launcher-commands.md b/docs/launcher-commands.md index 9dc4382f4..7c9562441 100644 --- a/docs/launcher-commands.md +++ b/docs/launcher-commands.md @@ -323,7 +323,7 @@ Search for packs Horizon loads. - `TEXT` - `--kind` (mod, resource, modpack) - `--everything`: Also what was made for other versions or runs only partly, each marked -- `--source` (modrinth, curseforge) +- `--source` (modrinth, curseforge, horizon) MCP: `mods_search`. @@ -335,7 +335,7 @@ Say how well a Java mod runs in Horizon: its jar is downloaded (checksum checked - `PROJECT` - `--everything`: Allow a version made for another Minecraft version (1.21 and later) -- `--source` (modrinth, curseforge) +- `--source` (modrinth, curseforge, horizon) MCP: `mods_check`. @@ -347,7 +347,7 @@ Install a pack and what it needs, by its id or slug (checksums checked); a Java - `PROJECT` - `--everything`: Allow a version made for another Minecraft version -- `--source` (modrinth, curseforge) +- `--source` (modrinth, curseforge, horizon) MCP: `mods_install`. @@ -372,6 +372,26 @@ Remove a pack the browser installed. MCP: `mods_remove`, needs `confirm: true`. +## `horizon-launcher mods enable` + +Turn on a mod the browser installed (horizon-mods.txt, as the game's Options > Mods does). + + horizon-launcher mods enable + +- `PROJECT` + +MCP: `mods_enable`. + +## `horizon-launcher mods disable` + +Turn off a mod the browser installed; it stays installed. + + horizon-launcher mods disable + +- `PROJECT` + +MCP: `mods_disable`. + ## `horizon-launcher servers list` The servers, their folder and whether they run. diff --git a/docs/mod-browser.md b/docs/mod-browser.md index 7a000cb6c..a803420c3 100644 --- a/docs/mod-browser.md +++ b/docs/mod-browser.md @@ -149,6 +149,32 @@ horizon-launcher mods remove faithful-32x ## Sources +**Horizon's registry.** Mods (`.hmod`) and resource packs made for Horizon, reviewed as pull requests in +[ludovic111/horizon-mods](https://github.com/ludovic111/horizon-mods) ([api/mod-index.md](api/mod-index.md)). The +launcher reads its `index.json` (`browser/horizon.rs`; `HORIZON_MOD_INDEX=` reads a test +registry instead, whose files may then be on disk) at most every ten minutes, and keeps the last copy in +`mod-browser/horizon-index.json`. Before a result is installed it shows whether its files are signed and, for a mod, +what it may do, in the words of the game's Options > Mods. A download goes to `mod-browser/downloads/` and is moved +into place only after `horizon_mods::index::check_file` passes (size, SHA-256, the author's signature when the project +has a key, the file's own manifest against the index). A mod becomes `horizon-mods/.hmod`, turned on; +`mods disable` and `mods enable` switch it in `horizon-mods.txt`. When the index takes a file down, the launcher turns +it off the next time it reads the index or starts the game, says why, and refuses to install it or turn it back on. +A Horizon modpack (`.hpack`) installs as its own profile (`browser/horizon/pack.rs`): `horizon-packs//` holds the +checked pack, `pack.json` and `home/`, the profile's game folder, where each of the pack's mods is downloaded and checked +like any registry mod and turned on or off as the pack says; its overrides (settings, resource packs, a mod's saved +settings) are copied in on the first install only, so the player's changes survive updates. A pack whose mod was taken +down installs nothing. It is listed on the Modpacks tab with **Play** and **Remove**, and plays like an imported pack +(`modpacks::apply_profile`). Shader packs are listed by the index but not installed by this launcher yet. + +```sh +horizon-launcher mods search --source horizon --kind mod coords # with whether it is signed and what it may do +horizon-launcher mods install --source horizon coords_hud +horizon-launcher mods disable coords_hud +horizon-launcher mods install --source horizon cozy # a modpack: its own profile +horizon-launcher modpack play cozy +``` + + **Modrinth.** The public API (`api.modrinth.com/v2`) needs no key. The launcher names itself in its user agent, as Modrinth asks. diff --git a/docs/mods.md b/docs/mods.md index 5fa6a290d..3a4473032 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -16,7 +16,7 @@ folder, loads no runtime and plays exactly like vanilla 26.3. | SDK for mod authors | `crates/horizon-mod-sdk` | 0.1 | | Example mod | `crates/horizon-mod-sdk/examples/coords-hud` | coordinates, facing and time HUD, `/where`, `/coords` | | Runtime | `crates/horizon-mods` | wasmtime 36 LTS, component model | -| Authoring tool | `crates/cargo-horizon-mod` | `cargo horizon-mod new / build / run / check`, and the registry's CI (`registry build / review / list`) | +| Authoring tool | `crates/cargo-horizon-mod` | `cargo horizon-mod new [--template code\|content] / build / run / check`, and the registry's CI (`registry build / review / list`) | | In the game | `crates/horizon-client/src/game/mods.rs`, Options > Mods | client mods | | Launcher mod browser (Modrinth, CurseForge with a key) | [mod-browser.md](mod-browser.md) | resource packs, modpacks | | Registry index (`index.json`, `.hpack`), signatures | `horizon_mods::index`, [api/mod-index.md](api/mod-index.md) | schema 1; launcher and website next | @@ -215,6 +215,7 @@ Benchmarks (`--benchmark`) never load mods. rustup target add wasm32-unknown-unknown cargo install --git https://github.com/ludovic111/horizon cargo-horizon-mod cargo horizon-mod new my-mod # Cargo.toml, horizon-mod.toml, src/lib.rs + # (--template content: a content mod, no code) cd my-mod cargo horizon-mod run # build, then start the game with it cargo horizon-mod build # again after an edit: the running game reloads it @@ -292,7 +293,10 @@ permission. ### Content mods (API 0.2) A mod with `api = "0.2"` and capability `registries` may carry `assets/` and -`data/` in the `.hmod`, and needs no `mod.wasm` if it has no code. It adds +`data/` in the `.hmod`, and needs no `mod.wasm` if it has no code +(`cargo horizon-mod new --template content` starts one; `build` in a folder +with no `Cargo.toml` packs the manifest, `assets/` and `data/`, and `check` +reads it without a trial run). It adds blocks and items by declaring them in `data//horizon/block/*.json` and `data//horizon/item/*.json`, each copying a vanilla template ([modpacks.md](modpacks.md#registries-02-data-driven-blocks-and-items)): @@ -378,6 +382,29 @@ none. "top": "carve", "bottom": "none", "rigid_junctions": true, "inflate": true } } ``` +## Publish your mod + +Horizon's registry lists mods, modpacks (`.hpack`) and resource packs made for Horizon, each reviewed before it is +published. Players find them at [playhorizon.dev/mods](https://playhorizon.dev/mods) and in the launcher (Mods, then +**Horizon** beside Modrinth), which checks every file before it installs it. + +1. Run `cargo horizon-mod check` on the file (above): the registry runs the same checks. +2. Put the source somewhere public, under a licence that lets players run it: reviewers read it, and the file must be + built from it. +3. Open [playhorizon.dev/mods/submit](https://playhorizon.dev/mods/submit), sign in with GitHub (it reads only your + public profile) and fill in the page: the file, its name and summary, licence, source address, and optionally a + minisign public key with the file's `.minisig`. +4. The form checks the file again and opens a pull request in + [ludovic111/horizon-mods](https://github.com/ludovic111/horizon-mods) that adds `mods/.toml`, with you as its + owner. That repository's checks run the file in the sandbox and post their report on the pull request; answer the + reviewers there. +5. A reviewer merges it: within minutes the file is a release of horizon-mods, and the project is on the site and in + the launcher. + +An update is the same form with a new version; only the project's `owners` can publish one, signed with its key if it +has one. A published file never changes. Without the form, open the pull request by hand (the repository's README +says how). The formats: [api/mod-index.md](api/mod-index.md). + ## Versions and compatibility - The API is the WIT package `horizon:mods@0.1.0`. New functions and @@ -420,12 +447,10 @@ none. format: a `.hmod.minisig` beside it); the index pins each project's public key, so an update signed by another key is refused. Unsigned local files stay allowed, with an "unsigned" mark on the Mods page. -- **Where mods come from.** Modrinth is where players already look and it - stores files with hashes, but its loader list is Modrinth's to extend: we ask - Modrinth to add a `horizon` loader. Until then (and as a fallback), the - launcher reads a small index of approved mods: a JSON file in a public Git - repository (id, versions, file URLs as GitHub release assets, SHA-256, - signing key). No server of our own is needed. +- **Where mods come from.** Horizon's registry ([Publish your mod](#publish-your-mod)): reviewed pull requests in + [ludovic111/horizon-mods](https://github.com/ludovic111/horizon-mods), files as its release assets, and an + `index.json` on GitHub Pages that the launcher and the website read. A file the registry takes down + (`revoked.toml`) is refused by the launcher and turned off where installed. - **The launcher.** A Mods tab per installation: browse and search the index, install, update, turn on and off (it writes the same `horizon-mods.txt`), see capabilities before installing. The launcher's CLI and MCP tools gain diff --git a/docs/shaders.md b/docs/shaders.md new file mode 100644 index 000000000..99a08fa9a --- /dev/null +++ b/docs/shaders.md @@ -0,0 +1,127 @@ +# Horizon shader packs + +Horizon has its own shader packs: WGSL written against stages of Horizon's renderer, checked before they run. This page +is the guide for pack authors. The design and the pack format are in [api/shaders.md](api/shaders.md); every stage's +signature, input, resource and limit is in the generated [api/shaders-reference.md](api/shaders-reference.md). + +Shader packs in the Iris/OptiFine format are a separate feature with its own loader; this page is only about +Horizon's own format. + +| Part | Where | State | +|---|---|---| +| Format, stages, check | `crates/horizon-shader-api` | API 0.1, experimental | +| Authoring tool | `horizon-shader` (same crate) | `new`, `check`, `pack` | +| Templates | `crates/horizon-shader-api/templates/` | post, sky, material | +| Example pack | `crates/horizon-shader-api/packs/golden_hour` | a warm sky and two post passes | +| In the game, the launcher, the registry | | not yet: the renderer hooks come next | + +## Quick start + +```sh +cargo install --path crates/horizon-shader-api # or: cargo run -p horizon-shader-api --bin horizon-shader -- +horizon-shader new my_pack --template post # also: --template sky, --template material; --name "My Pack" +horizon-shader check my_pack # what the game, the launcher and the registry will check +horizon-shader pack my_pack # check, then write my_pack-0.1.0.hshader beside the folder +``` + +`check` prints each stage with its cost, and the pack's SHA-256: + +``` +golden_hour 1.0.0 (shader API 0.1): ok + sky Warm horizon 99 expressions, 0 texture reads + post Glow 185 expressions, 3 texture reads + post Colour grade 201 expressions, 2 texture reads + sha256 fd92a294... +``` + +or every problem, at its file, line and column: + +``` +my_pack 0.1.0 (shader API 0.1): FAILED + post.wgsl:9:1: expected `;`, found "}" +``` + +`--json` prints the same for tools (`ok`, `id`, `version`, `api`, `sha256`, `stages`, `problems`). + +## A pack + +A folder (or the same files zipped as a `.hshader`) with `horizon-shaders.toml` and WGSL files: + +```toml +schema = 1 +id = "my_pack" +name = "My Pack" +version = "0.1.0" +api = "0.1" + +[[post]] +file = "post.wgsl" + +[[options]] +id = "saturation" +name = "Saturation" +type = "float" +default = 1.15 +min = 0.0 +max = 2.0 +``` + +Every key is in [api/shaders.md](api/shaders.md#the-pack-format-schema-1). + +## Stages + +A stage is one function your file defines. Horizon puts its prelude (the structs, `frame`, the stage's resources and +helpers) before your file and calls the function from its own pass, so the world is still culled, batched and meshed +by Horizon: a pack changes how things look, not how they are drawn. + +| Stage | Your function | What it does | +| --- | --- | --- | +| `[sky]` | `fn sky(input: SkyInput) -> vec4` | the sky's colour along each view ray; the sun, moon, stars and clouds are drawn over it | +| `[material]` | `fn material(input: MaterialInput) -> Surface` | the lit colour of each point of a block's surface, before the alpha test and fog | +| `[[post]]` | `fn post(input: PostInput) -> vec4` | a full-screen pass over the world's picture (up to 8, in order), under the hand and HUD | + +A pack has at least one stage; a stage it leaves out keeps vanilla's look. Each input says what vanilla would draw +(`input.color` in the sky and post stages; `albedo * tint * vec4(lightmap, 1.0)` for a material), so a pack can start +from vanilla and change only what it means to. + +Every stage can read `frame`: the camera's matrices and position, the sun and moon directions, the time of day, +vanilla's sky and fog colours, rain and thunder, fog distances, the light at the camera, effects and the dimension. +Post passes also read `scene_color`, `scene_depth`, `previous` (the last pass's output), two samplers and the helpers +`depth_at(uv)` and `view_position(uv, depth)`. + +Rules the check enforces: your files declare no bindings (`@group`/`@binding`), no entry points and no names starting +with `horizon_`. Code shared by several stages goes in files listed in `shared`, put before every stage. + +## Options + +Each `[[options]]` entry is a WGSL `override` of the same name, set when the game builds the pipeline: no text +substitution, no recompiling a different program per value. + +```wgsl +override saturation: f32 = 1.15; // type = "float"; bool → bool, int → i32, choice → u32 (the value's index) +``` + +The game shows the options on the pack's settings screen. Branches on an option are on a constant, so the GPU +skips the side that isn't taken. + +## What the check does + +The game runs the same check when it loads a pack (a pack that fails is refused with the message, and the last good +one keeps drawing), the launcher before it installs one, and the registry's CI before a pack is listed: + +1. The manifest and the package: known keys, valid ids, every file used, sizes. +2. Each stage built as the renderer builds it and validated by naga. +3. The stage function's signature, and the rules above. +4. Each `override` matches its option's type (or has a default); each option is used. +5. Budgets: 20 000 expressions and 64 texture reads per stage. +6. Every entry point translated for Vulkan (SPIR-V), Metal and DirectX 12 (HLSL), with the options at their + defaults, minimums and maximums. + +`[budget] gpu_ms` is the GPU time your pack aims to add to a frame; the game's overlay will show each pass's time +against it. + +## Next + +The renderer hooks (shared with the Iris/OptiFine loader), then the post stage in the game with hot reload and the +GPU time per pass, Options > Shader packs listing these packs, then the material and sky stages, and the registry's +`shader` kind with installs from the launcher. diff --git a/launcher/Cargo.lock b/launcher/Cargo.lock index 5155420e0..1e8a5aee2 100644 --- a/launcher/Cargo.lock +++ b/launcher/Cargo.lock @@ -2673,6 +2673,7 @@ dependencies = [ "horizon-bench", "horizon-core", "horizon-friends", + "horizon-mods", "horizon-port-pack", "horizon-prepare", "horizon-release-sign", @@ -2680,8 +2681,10 @@ dependencies = [ "horizon-theme", "image", "keyring", + "objc2-app-kit", "oo7", "pulldown-cmark", + "raw-window-handle", "self-replace", "serde", "serde_json", @@ -2698,8 +2701,10 @@ name = "horizon-mods" version = "0.1.0" dependencies = [ "log", + "minisign-verify", "serde", "serde_json", + "sha2 0.10.9", "toml 1.1.8+spec-1.1.0", "zip", ] @@ -3470,6 +3475,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "minisign-verify" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871285dc19d8d0ebe0eef3d0e99a205f2a71363b122632cbbfa4a6c370a960ce" + [[package]] name = "miniz_oxide" version = "0.8.9" diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index a402d0e28..a97f128b8 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -31,6 +31,8 @@ horizon-friends = { path = "../crates/horizon-friends" } horizon-bench = { path = "../crates/horizon-bench" } # Import modpack: the porting pipeline, run on the player's own files (docs/modpacks.md). horizon-port-pack = { path = "../crates/horizon-port-pack" } +# Horizon's mod registry: its index and the checks a download passes before install. +horizon-mods = { path = "../crates/horizon-mods", features = ["index"] } # Signed updates: archives are checked against the release's signed checksums (docs/security.md). horizon-release-sign = { path = "../crates/horizon-release-sign" } # Opt-in data collection: the consent, the outbox and its upload (docs/telemetry.md). @@ -48,6 +50,10 @@ oo7 = { version = "0.5", default-features = false, features = ["async-std", "nat futures-lite = "2" [target.'cfg(target_os = "macos")'.dependencies] keyring = { version = "3", features = ["apple-native"] } +# Painting the window's title bar dark or light (titlebar::apply_appearance): crates already in +# the lockfile through gpui and wgpu. +objc2-app-kit = { version = "0.3", default-features = false, features = ["NSAppearance", "NSView", "NSWindow"] } +raw-window-handle = "0.6" [target.'cfg(target_os = "windows")'.dependencies] keyring = { version = "3", features = ["windows-native"] } diff --git a/launcher/src/backend.rs b/launcher/src/backend.rs index 270f856a0..9d1eeb111 100644 --- a/launcher/src/backend.rs +++ b/launcher/src/backend.rs @@ -1636,6 +1636,7 @@ pub fn need_prepared(tag: &str) -> Result<(), String> { pub fn client(tag: &str) -> Result { need_prepared(tag)?; complete_packs(tag); + crate::browser::horizon::before_play(&game_home()); let mut cmd = client_command(tag); // A sign-in that can't be renewed (no network, expired) still lets the game start, offline. if let Err(e) = crate::account::attach_session(&mut cmd, &game_home()) { diff --git a/launcher/src/browser.rs b/launcher/src/browser.rs index f02638db5..7f81d5e37 100644 --- a/launcher/src/browser.rs +++ b/launcher/src/browser.rs @@ -8,6 +8,7 @@ pub mod curseforge; pub mod details; pub mod download; +pub mod horizon; pub mod images; pub mod installed; pub mod java; @@ -56,6 +57,8 @@ impl Kind { pub enum SourceId { Modrinth, CurseForge, + /// Horizon's own registry of reviewed mods, modpacks and resource packs ([`horizon`]). + Horizon, } impl SourceId { @@ -63,6 +66,7 @@ impl SourceId { match self { SourceId::Modrinth => "Modrinth", SourceId::CurseForge => "CurseForge", + SourceId::Horizon => "Horizon", } } } @@ -132,6 +136,8 @@ pub struct FileRef { pub name: String, pub size: u64, pub sha512: Option, + /// What Horizon's registry lists. + pub sha256: Option, pub sha1: Option, pub primary: bool, } @@ -139,11 +145,13 @@ pub struct FileRef { impl FileRef { /// The strongest checksum the source gives: what the download is checked against. pub fn checksum(&self) -> Option<(&'static str, &str)> { - match (&self.sha512, &self.sha1) { - (Some(h), _) => Some(("SHA-512", h)), - (None, Some(h)) => Some(("SHA-1", h)), - (None, None) => None, - } + [ + ("SHA-512", &self.sha512), + ("SHA-256", &self.sha256), + ("SHA-1", &self.sha1), + ] + .into_iter() + .find_map(|(algo, h)| Some((algo, h.as_deref()?))) } } @@ -230,10 +238,11 @@ pub trait Source: Send + Sync { /// What the launcher says wherever CurseForge would be. pub const CURSEFORGE_SOON: &str = "CurseForge support is coming soon."; -/// The sources this launcher uses: Modrinth. CurseForge support is coming soon: its API needs a -/// paid key, so no search or install goes to it; `curseforge` is kept for then. +/// The sources this launcher uses: Modrinth and Horizon's registry. CurseForge support is coming +/// soon: its API needs a paid key, so no search or install goes to it; `curseforge` is kept for +/// then. pub fn sources() -> Vec> { - vec![Box::new(modrinth::Modrinth)] + vec![Box::new(modrinth::Modrinth), Box::new(horizon::Registry)] } /// The source of that id, when this launcher can use it. @@ -243,6 +252,15 @@ pub fn source(id: SourceId) -> Option> { /// How well a project of `kind` made for `game_versions` and `loaders` runs in Horizon. pub fn support(kind: Kind, game_versions: &[String], loaders: &[String]) -> Support { + if loaders.iter().any(|l| l == horizon::LOADER) { + // Made for Horizon and reviewed: it runs when it is for the version Horizon plays. + return match newest_release(game_versions) + .filter(|_| !game_versions.iter().any(|v| v == GAME)) + { + Some(other) => Support::No(format!("Made for Horizon on {other}.")), + None => Support::Works, + }; + } match kind { Kind::ResourcePack => { if game_versions.iter().any(|v| v == GAME) { @@ -309,6 +327,12 @@ pub fn pick(kind: Kind, versions: &[Version], everything: bool) -> Option<&Versi let mut usable = versions .iter() .filter(|v| v.blocked.is_none() && v.file().is_some()); + if versions + .iter() + .any(|v| v.loaders.iter().any(|l| l == horizon::LOADER)) + { + return usable.find(|v| v.game_versions.iter().any(|g| g == GAME)); + } if kind == Kind::Mod { let java = |v: &&Version| { v.loaders diff --git a/launcher/src/browser/curseforge.rs b/launcher/src/browser/curseforge.rs index 4894fe5f0..e17a6b822 100644 --- a/launcher/src/browser/curseforge.rs +++ b/launcher/src/browser/curseforge.rs @@ -164,6 +164,7 @@ pub(super) fn parse_file(f: &Value) -> Option { name: name.clone(), size: f["fileLength"].as_u64().unwrap_or(0), sha512: None, + sha256: None, sha1, primary: true, }] diff --git a/launcher/src/browser/download.rs b/launcher/src/browser/download.rs index 66199fd07..593273245 100644 --- a/launcher/src/browser/download.rs +++ b/launcher/src/browser/download.rs @@ -1,16 +1,18 @@ //! Downloads that reach the game folder only when their checksum matches the source's. use super::{safe_name, FileRef}; use sha1::Sha1; -use sha2::{Digest, Sha512}; +use sha2::{Digest, Sha256, Sha512}; use std::fs; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; -/// Hosts a file may come from: Modrinth's and CurseForge's file servers. -const HOSTS: [&str; 3] = [ +/// Hosts a file may come from: Modrinth's and CurseForge's file servers, and the release assets +/// of Horizon's registry. +const HOSTS: [&str; 4] = [ "https://cdn.modrinth.com/", "https://edge.forgecdn.net/", "https://mediafilez.forgecdn.net/", + horizon_mods::index::FILE_HOST, ]; /// Bigger than any pack these sources serve (Modrinth caps uploads at 500 MiB). const MAX: u64 = 1 << 30; @@ -31,10 +33,15 @@ pub(super) fn agent() -> ureq::Agent { pub fn refuse(file: &FileRef) -> Option { if !HOSTS.iter().any(|h| file.url.starts_with(h)) { return Some(format!( - "{} isn't on Modrinth's or CurseForge's file servers.", + "{} isn't on Modrinth's, CurseForge's or Horizon's file servers.", file.url )); } + refuse_file(file) +} + +/// [`refuse`] without the host: the name, the checksum and the size. +fn refuse_file(file: &FileRef) -> Option { if safe_name(&file.name).is_none() { return Some(format!("{:?} isn't a plain file name.", file.name)); } @@ -50,26 +57,29 @@ pub fn refuse(file: &FileRef) -> Option { /// Hashes as they are written: the strongest one the file lists. enum Hasher { Sha512(Sha512), + Sha256(Sha256), Sha1(Sha1), } impl Hasher { fn for_file(file: &FileRef) -> Self { - if file.sha512.is_some() { - Hasher::Sha512(Sha512::new()) - } else { - Hasher::Sha1(Sha1::new()) + match file.checksum() { + Some(("SHA-512", _)) => Hasher::Sha512(Sha512::new()), + Some(("SHA-256", _)) => Hasher::Sha256(Sha256::new()), + _ => Hasher::Sha1(Sha1::new()), } } fn update(&mut self, bytes: &[u8]) { match self { Hasher::Sha512(h) => h.update(bytes), + Hasher::Sha256(h) => h.update(bytes), Hasher::Sha1(h) => h.update(bytes), } } fn hex(self) -> String { let bytes: Vec = match self { Hasher::Sha512(h) => h.finalize().to_vec(), + Hasher::Sha256(h) => h.finalize().to_vec(), Hasher::Sha1(h) => h.finalize().to_vec(), }; bytes.iter().map(|b| format!("{b:02x}")).collect() @@ -81,13 +91,26 @@ impl Hasher { /// `progress` gets the fraction done. pub fn save( file: &FileRef, - mut reader: impl Read, + reader: impl Read, folder: &Path, progress: &dyn Fn(f32), ) -> Result { if let Some(why) = refuse(file) { return Err(why); } + save_from_anywhere(file, reader, folder, progress) +} + +/// [`save`] for a file that comes from no host: a local test registry's. +pub(super) fn save_from_anywhere( + file: &FileRef, + mut reader: impl Read, + folder: &Path, + progress: &dyn Fn(f32), +) -> Result { + if let Some(why) = refuse_file(file) { + return Err(why); + } let (algo, want) = file.checksum().expect("refuse checks for one"); fs::create_dir_all(folder).map_err(|e| format!("{}: {e}", folder.display()))?; let target = folder.join(&file.name); diff --git a/launcher/src/browser/horizon.rs b/launcher/src/browser/horizon.rs new file mode 100644 index 000000000..66553ab50 --- /dev/null +++ b/launcher/src/browser/horizon.rs @@ -0,0 +1,562 @@ +//! Horizon's own registry (docs/api/mod-index.md): mods, modpacks and resource packs made for +//! Horizon, reviewed as pull requests in ludovic111/horizon-mods. One file lists them all, +//! `index.json`; every download is checked against it before it reaches the game folder +//! (`horizon_mods::index::check_file`: size, SHA-256, the author's signature, the file's own +//! manifest), and files the index says were taken down are refused and turned off. +use super::{ + details::{Details, Link}, + download, installed, java, FileRef, Kind, Project, Query, Source, SourceId, Version, GAME, +}; +use horizon_mods::index::{self, Index, Trust}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +/// Where the registry publishes its index (GitHub Pages of ludovic111/horizon-mods). +pub const INDEX_URL: &str = "https://ludovic111.github.io/horizon-mods/index.json"; +/// The registry's pages on the website, `/`. +const SITE: &str = "https://playhorizon.dev/mods"; +/// The report form of a project. +const REPORT: &str = "https://github.com/ludovic111/horizon-mods/issues/new?template=report.yml"; +/// The loader name the browser gives registry versions, so `support` and `pick` know them. +pub const LOADER: &str = "horizon"; +/// How long a fetched index is used before it is fetched again. +const FRESH: Duration = Duration::from_secs(600); +/// Larger than any index should be. +const MAX_INDEX: u64 = 16 << 20; +/// The last index fetched, kept in the game folder for when the registry can't be reached. +const CACHE: &str = "mod-browser/horizon-index.json"; + +/// Where the index comes from: `HORIZON_MOD_INDEX` (a file or an `https://` address, for a +/// test registry), else [`INDEX_URL`]. +fn location() -> String { + std::env::var("HORIZON_MOD_INDEX") + .ok() + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| INDEX_URL.to_owned()) +} + +/// True when the index is a local file: its files may then be local too (`file://`), for +/// trying a registry out before it is published. +pub fn local() -> bool { + !location().starts_with("https://") +} + +static LOADED: Mutex)>> = Mutex::new(None); + +/// The index read last in this run, without asking the network: for what must not wait. +pub fn known_index() -> Option> { + LOADED + .lock() + .unwrap() + .as_ref() + .map(|(_, index)| index.clone()) +} + +/// The index: the one fetched in the last ten minutes, else fetched again (and kept in the game +/// folder), else the copy kept last time. +pub fn index() -> Result, String> { + if let Some((at, index)) = LOADED.lock().unwrap().as_ref() { + if at.elapsed() < FRESH { + return Ok(index.clone()); + } + } + let cache = crate::backend::game_home().join(CACHE); + let index = match fetch_index() { + Ok((index, text)) => { + if let Some(dir) = cache.parent() { + let _ = fs::create_dir_all(dir); + } + let _ = fs::write(&cache, text); + index + } + Err(e) => { + let kept = fs::read_to_string(&cache).map_err(|_| e.clone())?; + match local() { + true => Index::parse_local(&kept), + false => Index::parse(&kept), + } + .map_err(|_| e)? + } + }; + let index = Arc::new(index); + *LOADED.lock().unwrap() = Some((Instant::now(), index.clone())); + Ok(index) +} + +fn fetch_index() -> Result<(Index, String), String> { + use std::io::Read; + let at = location(); + let text = if at.starts_with("https://") { + let response = download::agent().get(&at).call().map_err(|e| match e { + ureq::Error::Status(code, _) => format!("The mod registry answered {code}."), + ureq::Error::Transport(t) => format!("The mod registry can't be reached: {t}"), + })?; + let mut text = String::new(); + response + .into_reader() + .take(MAX_INDEX) + .read_to_string(&mut text) + .map_err(|e| e.to_string())?; + text + } else { + fs::read_to_string(&at).map_err(|e| format!("{at}: {e}"))? + }; + let index = match local() { + true => Index::parse_local(&text)?, + false => Index::parse(&text)?, + }; + Ok((index, text)) +} + +/// The browser's kind for a registry kind; None for one the launcher doesn't install yet +/// (shader packs, kinds newer than this launcher). +fn kind_of(kind: &index::Kind) -> Option { + match kind { + index::Kind::Mod => Some(Kind::Mod), + index::Kind::ResourcePack => Some(Kind::ResourcePack), + index::Kind::Modpack => Some(Kind::Modpack), + index::Kind::Shader | index::Kind::Other(_) => None, + } +} + +/// What the player reads about a registry project before installing it: whether its files are +/// signed, and for a mod what it may do, in the game's words (Options > Mods). +pub fn trust_and_capabilities(id: &str) -> Vec { + let Ok(index) = index() else { + return Vec::new(); + }; + let Some(p) = index.project(id) else { + return Vec::new(); + }; + let mut lines = vec![match &p.key { + Some(_) => "Signed by its author: the launcher checks the signature".to_owned(), + None => "Unsigned: the launcher checks its checksum only".to_owned(), + }]; + if let Some(v) = p.versions.first().filter(|_| p.kind == index::Kind::Mod) { + lines.extend( + v.capabilities + .iter() + .map(|c| format!("May {}", c.describe())), + ); + if v.capabilities.is_empty() { + lines.push("Asks for nothing beyond its own log".into()); + } + } + lines +} + +fn project_of(p: &index::Project) -> Option { + let mut game_versions: Vec = p.versions.iter().map(|v| v.game.clone()).collect(); + game_versions.dedup(); + Some(Project { + source: SourceId::Horizon, + id: p.id.clone(), + slug: p.id.clone(), + title: p.name.clone(), + summary: p.summary.clone(), + author: p.authors.join(", "), + downloads: 0, + page: format!("{SITE}/{}", p.id), + icon: p.icon.clone(), + license: p.license.clone(), + kind: kind_of(&p.kind)?, + game_versions, + loaders: vec![LOADER.into()], + }) +} + +fn version_of(index: &Index, p: &index::Project, v: &index::Version) -> Version { + Version { + id: v.version.clone(), + number: v.version.clone(), + published: v.published.clone(), + game_versions: vec![v.game.clone()], + loaders: vec![LOADER.into()], + files: vec![FileRef { + url: v.file.url.clone(), + name: v.file.name.clone(), + size: v.file.size, + sha512: None, + sha256: Some(v.file.sha256.clone()), + sha1: None, + primary: true, + }], + required: v.requires.iter().map(|r| r.id.clone()).collect(), + blocked: index + .revoked(&p.id, &v.file.sha256) + .map(|r| format!("taken down: {}", r.reason)), + } +} + +/// A GitHub login as GitHub allows it (letters, digits, single dashes, at most 39), so an index +/// entry can't turn a link into something else. +fn github_login(login: &str) -> bool { + (1..=39).contains(&login.len()) + && login + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') + && !login.starts_with('-') + && !login.ends_with('-') + && !login.contains("--") +} + +/// The words of a query found in a project's id, name, summary or authors. +fn matches(p: &index::Project, text: &str) -> bool { + let hay = format!("{} {} {} {}", p.id, p.name, p.summary, p.authors.join(" ")).to_lowercase(); + text.split_whitespace() + .all(|word| hay.contains(&word.to_lowercase())) +} + +/// The projects of the query's kind whose words match, for this game unless `everything`. +fn search(index: &Index, query: &Query) -> Vec { + index + .projects + .iter() + .filter(|p| kind_of(&p.kind) == Some(query.kind) && matches(p, &query.text)) + .filter(|p| query.everything || p.versions.iter().any(|v| v.game == GAME)) + .filter_map(project_of) + .skip(query.offset) + .take(query.limit) + .collect() +} + +/// The registry as a browser source. +pub struct Registry; + +impl Source for Registry { + fn id(&self) -> SourceId { + SourceId::Horizon + } + + fn search(&self, query: &Query) -> Result, String> { + Ok(search(&*index()?, query)) + } + + fn project(&self, id: &str) -> Result { + let index = index()?; + let p = index + .project(id) + .ok_or_else(|| format!("{id} isn't in Horizon's registry."))?; + project_of(p).ok_or_else(|| { + format!( + "{} is a {} in Horizon's registry: this launcher can't install those yet.", + p.name, + p.kind.name() + ) + }) + } + + fn versions(&self, project: &Project) -> Result, String> { + let index = index()?; + let p = index + .project(&project.id) + .ok_or_else(|| format!("{} isn't in Horizon's registry.", project.title))?; + Ok(p.versions + .iter() + .map(|v| version_of(&index, p, v)) + .collect()) + } + + fn details(&self, project: &Project) -> Result { + let index = index()?; + let p = index + .project(&project.id) + .ok_or_else(|| format!("{} isn't in Horizon's registry.", project.title))?; + let newest = p.versions.first(); + let mut body = p.description.clone(); + if let Some(v) = newest.filter(|v| !v.capabilities.is_empty()) { + body.push_str("\n\n## What it may do\n\n"); + for c in &v.capabilities { + body.push_str(&format!("- May {}\n", c.describe())); + } + } + if let Some(v) = newest.filter(|v| !v.requires.is_empty()) { + body.push_str( + "\n\n## Its mods\n\nInstalled with it into its own profile, each checked:\n\n", + ); + for r in &v.requires { + let found = index + .project(&r.id) + .and_then(|m| Some((m, m.versions.iter().find(|w| w.version == r.version)?))); + let Some((m, w)) = found else { continue }; + let may: Vec<&str> = w.capabilities.iter().map(|c| c.describe()).collect(); + let may = match may.is_empty() { + true => String::new(), + false => format!(": may {}", may.join(", ")), + }; + body.push_str(&format!("- {} {}{may}\n", m.name, w.version)); + } + } + // Who publishes it: the GitHub accounts the registry checked at submission. + let owners = p.owners.iter().filter(|o| github_login(o)).map(|o| Link { + label: format!("@{o}"), + url: format!("https://github.com/{o}"), + }); + Ok(Details { + body, + links: [ + Link { + label: "Source".into(), + url: p.source.clone(), + }, + Link { + label: "Page".into(), + url: project.page.clone(), + }, + Link { + label: "Report".into(), + url: format!("{REPORT}&title={}", p.id), + }, + ] + .into_iter() + .chain(owners) + .collect(), + authors: p.authors.clone(), + published: p + .versions + .last() + .map(|v| v.published.clone()) + .unwrap_or_default(), + updated: newest.map(|v| v.published.clone()).unwrap_or_default(), + ..Details::default() + }) + } + + fn fetch( + &self, + file: &FileRef, + folder: &Path, + progress: &dyn Fn(f32), + ) -> Result { + fetch(file, folder, local(), progress) + } +} + +/// Downloads a registry file, checked; `file://` files only from a local index. +fn fetch( + file: &FileRef, + folder: &Path, + local: bool, + progress: &dyn Fn(f32), +) -> Result { + match file.url.strip_prefix("file://").filter(|_| local) { + Some(path) => { + let path = local_path(path); + let reader = fs::File::open(path).map_err(|e| format!("{path}: {e}"))?; + download::save_from_anywhere(file, reader, folder, progress) + } + None => download::fetch(file, folder, progress), + } +} + +/// The path of a `file:///` URL: on Windows `/C:/x` is `C:/x`. +fn local_path(path: &str) -> &str { + match path.strip_prefix('/') { + Some(rest) if cfg!(windows) && rest.get(1..2) == Some(":") => rest, + _ => path, + } +} + +/// What the registry adds to an installed entry. +fn trust_label(trust: Trust) -> &'static str { + match trust { + Trust::Verified => "Verified", + Trust::Unsigned => "Unsigned", + } +} + +/// The game folder path a registry project's file goes to. +fn relative(kind: Kind, id: &str, file_name: &str) -> Result { + match kind { + Kind::Mod => Ok(format!("{}/{id}.hmod", java::MODS_DIR)), + Kind::ResourcePack => Ok(format!("resourcepacks/{file_name}")), + Kind::Modpack => Err("A modpack installs as its own profile (pack::install).".into()), + } +} + +/// Installs a registry project into the game folder `home`: its newest version for this game, +/// downloaded into `mod-browser/downloads/`, checked against the index, then moved into place. +/// A mod is `horizon-mods/.hmod`, turned on in `horizon-mods.txt`. +pub fn install( + project: &Project, + home: &Path, + progress: &dyn Fn(&str, f32), +) -> Result, String> { + install_from(&*index()?, local(), project, home, progress) +} + +fn install_from( + index: &Index, + local: bool, + project: &Project, + home: &Path, + progress: &dyn Fn(&str, f32), +) -> Result, String> { + let p = index + .project(&project.id) + .ok_or_else(|| format!("{} isn't in Horizon's registry.", project.title))?; + let v = newest(index, p)?; + install_version(index, local, p, v, home, progress).map(|e| vec![e]) +} + +/// A project's newest version for this game that wasn't taken down. +fn newest<'a>(index: &Index, p: &'a index::Project) -> Result<&'a index::Version, String> { + let mut taken_down = None; + for v in p.versions.iter().filter(|v| v.game == GAME) { + match index.revoked(&p.id, &v.file.sha256) { + None => return Ok(v), + Some(r) => { + taken_down.get_or_insert(r); + } + } + } + Err(match taken_down { + Some(r) => format!("{} was taken down: {}", p.name, r.reason), + None => format!("{} has no version for {GAME} to install.", p.name), + }) +} + +/// Installs version `v` of the registry's mod or resource pack `p` into the game folder `home`. +fn install_version( + index: &Index, + local: bool, + p: &index::Project, + v: &index::Version, + home: &Path, + progress: &dyn Fn(&str, f32), +) -> Result { + let project = project_of(p).ok_or_else(|| format!("{} can't be installed here.", p.name))?; + if let Some(r) = index.revoked(&p.id, &v.file.sha256) { + return Err(format!("{} was taken down: {}", p.name, r.reason)); + } + let relative = relative(project.kind, &p.id, &v.file.name)?; + let mut list = installed::Installed::load(home); + let ours = |e: &installed::Entry| e.source == SourceId::Horizon && e.project == p.id; + let old = list.entries.iter().find(|e| ours(e)).cloned(); + if old.as_ref().is_some_and(|o| o.version == v.version) { + return Ok(old.expect("checked")); + } + let target = home.join(&relative); + let taken_by = list.entries.iter().find(|e| !ours(e) && e.file == relative); + let folder_mod = home.join(java::MODS_DIR).join(&p.id); + if let Some(other) = taken_by { + return Err(format!( + "{relative} belongs to {}: remove it first.", + other.title + )); + } + if (target.exists() && old.as_ref().is_none_or(|o| o.file != relative)) || folder_mod.exists() { + return Err(format!( + "A mod with the id {} is already in {}: move it away first.", + p.id, + java::MODS_DIR + )); + } + let line = format!("{} {}", p.name, v.version); + let file = &version_of(index, p, v).files[0]; + let staging = home.join("mod-browser/downloads"); + let path = fetch(file, &staging, local, &|f| progress(&line, f))?; + let trust = index::check_file(index, p, v, &path).inspect_err(|_| { + let _ = fs::remove_file(&path); + })?; + if let Some(dir) = target.parent() { + fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + } + fs::rename(&path, &target).map_err(|e| format!("{}: {e}", target.display()))?; + if let Some(old) = old.as_ref().filter(|o| o.file != relative) { + let _ = fs::remove_file(home.join(&old.file)); + } + if project.kind == Kind::Mod { + java::set_enabled(home, &p.id, true)?; + } + let entry = installed::Entry { + source: SourceId::Horizon, + project: p.id.clone(), + slug: p.id.clone(), + title: p.name.clone(), + kind: project.kind, + version: v.version.clone(), + number: v.version.clone(), + file: relative, + checksum: format!("SHA-256:{}", v.file.sha256), + page: project.page.clone(), + dependency: false, + verdict: Some(trust_label(trust).into()), + skipped: Vec::new(), + namespaces: Vec::new(), + }; + list.entries.retain(|e| !ours(e)); + list.entries.push(entry.clone()); + list.save(home)?; + Ok(entry) +} + +/// The mod id of an installed mod (`horizon-mods/` or `horizon-mods/.hmod`). +pub fn mod_id(entry: &installed::Entry) -> Option<&str> { + let rest = entry.file.strip_prefix(java::MODS_DIR)?.strip_prefix('/')?; + Some(rest.strip_suffix(".hmod").unwrap_or(rest)) +} + +/// Turns the registry's files that were taken down off in the game folder `home`, and says +/// which. Their files stay, so the player sees them and can remove them. +pub fn turn_off_revoked(index: &Index, home: &Path) -> Vec { + let mut out = Vec::new(); + for e in installed::Installed::load(home).entries { + let hash = e.checksum.strip_prefix("SHA-256:").unwrap_or_default(); + let Some(r) = (e.source == SourceId::Horizon) + .then(|| index.revoked(&e.project, hash)) + .flatten() + else { + continue; + }; + if let (Kind::Mod, Some(id)) = (e.kind, mod_id(&e)) { + if java::is_enabled(home, id) && java::set_enabled(home, id, false).is_ok() { + out.push(format!( + "{} was taken down and is off: {}", + e.title, r.reason + )); + } + } else if e.kind == Kind::ResourcePack { + out.push(format!( + "{} was taken down: {} Remove it.", + e.title, r.reason + )); + } + } + out +} + +/// Before the game starts: registry files taken down are turned off, by the index kept last +/// time (no waiting on the network), and the index is fetched again in the background for the +/// next start. Nothing happens when nothing came from the registry. +pub fn before_play(home: &Path) { + let from_registry = installed::Installed::load(home) + .entries + .iter() + .any(|e| e.source == SourceId::Horizon); + if !from_registry { + return; + } + let kept = fs::read_to_string(crate::backend::game_home().join(CACHE)).ok(); + let parse = |t: String| match local() { + true => Index::parse_local(&t).ok(), + false => Index::parse(&t).ok(), + }; + if let Some(index) = kept.and_then(parse) { + for line in turn_off_revoked(&index, home) { + eprintln!("{line}"); + } + } + let home = home.to_owned(); + std::thread::spawn(move || { + if let Ok(index) = index() { + turn_off_revoked(&index, &home); + } + }); +} + +pub mod pack; +#[cfg(test)] +mod tests; diff --git a/launcher/src/browser/horizon/pack.rs b/launcher/src/browser/horizon/pack.rs new file mode 100644 index 000000000..3cb6d7c9b --- /dev/null +++ b/launcher/src/browser/horizon/pack.rs @@ -0,0 +1,304 @@ +//! Horizon modpacks from the registry (`.hpack`, docs/api/mod-index.md), each played as its own +//! profile: `horizon-packs//` in the game's home holds the pack's file, `pack.json` (what +//! was installed) and `home/`, the profile's game folder: its saves, its options, and the pack's +//! mods in `home/horizon-mods/`, each downloaded and checked against the index like any +//! registry mod, turned on or off as the pack says. The pack's overrides (settings, resource +//! packs, a mod's saved settings) are copied in once, when it is installed. +use super::{fetch, index, install_version, local, newest, Index}; +use crate::browser::{java, FileRef}; +use horizon_mods::{index as idx, modpack::Modpack}; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const RECORD: &str = "pack.json"; + +/// Where registry modpacks live. +pub fn packs_dir() -> PathBuf { + crate::backend::game_home().join("horizon-packs") +} + +/// What `pack.json` keeps of an installed pack. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Installed { + pub id: String, + pub name: String, + pub version: String, + pub sha256: String, + /// The pack's mods: `id version`. + pub mods: Vec, + #[serde(skip)] + pub work: PathBuf, +} + +/// The installed registry packs, by name. +pub fn list() -> Vec { + list_in(&packs_dir()) +} + +fn list_in(dir: &Path) -> Vec { + let mut packs: Vec = fs::read_dir(dir) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| read(&e.path())) + .collect(); + packs.sort_by_key(|p| p.name.to_lowercase()); + packs +} + +fn read(work: &Path) -> Option { + let text = fs::read_to_string(work.join(RECORD)).ok()?; + let mut pack: Installed = serde_json::from_str(&text).ok()?; + pack.work = work.to_path_buf(); + Some(pack) +} + +/// Whether `work` is a registry pack's folder. +pub fn is_pack(work: &Path) -> bool { + work.join(RECORD).is_file() +} + +/// The profile's game folder. +pub fn home(work: &Path) -> PathBuf { + work.join("home") +} + +/// Installs the registry modpack `id`: its newest version for this game, then each of its mods. +/// Installing it again updates it; its saves and options stay. +pub fn install(id: &str, progress: &dyn Fn(&str, f32)) -> Result { + install_in(&*index()?, local(), id, &packs_dir(), progress) +} + +fn install_in( + index: &Index, + local: bool, + id: &str, + dir: &Path, + progress: &dyn Fn(&str, f32), +) -> Result { + let p = index + .project(id) + .ok_or_else(|| format!("{id} isn't in Horizon's registry."))?; + if p.kind != idx::Kind::Modpack { + return Err(format!("{} is a {}, not a modpack.", p.name, p.kind.name())); + } + let v = newest(index, p)?; + let work = dir.join(&p.id); + let fresh = !work.exists(); + let result = install_version_in(index, local, p, v, &work, progress); + if result.is_err() && fresh { + let _ = fs::remove_dir_all(&work); + } + result +} + +fn install_version_in( + index: &Index, + local: bool, + p: &idx::Project, + v: &idx::Version, + work: &Path, + progress: &dyn Fn(&str, f32), +) -> Result { + let line = format!("{} {}", p.name, v.version); + let file = FileRef { + url: v.file.url.clone(), + name: v.file.name.clone(), + size: v.file.size, + sha512: None, + sha256: Some(v.file.sha256.clone()), + sha1: None, + primary: true, + }; + let path = fetch(&file, &work.join("downloads"), local, &|f| { + progress(&line, f) + })?; + idx::check_file(index, p, v, &path).inspect_err(|_| { + let _ = fs::remove_file(&path); + })?; + let pack = Modpack::read(&path)?; + let home = home(work); + fs::create_dir_all(&home).map_err(|e| format!("{}: {e}", home.display()))?; + let mut mods = Vec::new(); + for m in &pack.manifest.mods { + let found = index + .project(&m.id) + .and_then(|mp| Some((mp, mp.versions.iter().find(|w| w.version == m.version)?))); + let Some((mp, mv)) = found.filter(|(_, mv)| mv.file.sha256 == m.sha256) else { + return Err(format!( + "{} needs {} {}, which isn't in the registry.", + p.name, m.id, m.version + )); + }; + install_version(index, local, mp, mv, &home, progress)?; + java::set_enabled(&home, &m.id, m.on)?; + mods.push(format!("{} {}", m.id, m.version)); + } + // Settings once: the player's own changes to the profile stay on updates. + let first = !work.join(RECORD).exists(); + if first { + Modpack::extract_overrides(&path, &home)?; + } + let kept = work.join(&v.file.name); + fs::rename(&path, &kept).map_err(|e| format!("{}: {e}", kept.display()))?; + let record = Installed { + id: p.id.clone(), + name: p.name.clone(), + version: v.version.clone(), + sha256: v.file.sha256.clone(), + mods, + work: work.to_path_buf(), + }; + let text = serde_json::to_string_pretty(&record).map_err(|e| e.to_string())?; + fs::write(work.join(RECORD), text).map_err(|e| e.to_string())?; + Ok(record) +} + +/// Turns the game's command into the pack's profile: its own game folder (the player's +/// options.txt copied in when the pack brings none), its taken-down mods turned off first. +pub fn apply(cmd: &mut Command, work: &Path) -> Result<(), String> { + if !is_pack(work) { + return Err(format!("{} is not an installed modpack.", work.display())); + } + let home = home(work); + fs::create_dir_all(&home).map_err(|e| format!("{}: {e}", home.display()))?; + let options = home.join("options.txt"); + let own = crate::backend::game_home().join("options.txt"); + if !options.exists() && own.is_file() { + let _ = fs::copy(own, &options); + } + super::before_play(&home); + cmd.current_dir(&home).env("HORIZON_HOME", &home); + Ok(()) +} + +/// Removes an installed pack: its mods and the profile's saves. +pub fn remove(work: &Path) -> Result<(), String> { + if work.parent() != Some(packs_dir().as_path()) || !is_pack(work) { + return Err(format!("{} is not an installed modpack.", work.display())); + } + fs::remove_dir_all(work).map_err(|e| format!("{}: {e}", work.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + /// A local registry: the mod and a pack of it, as `file://` files. + fn registry(dir: &Path, on: bool, extra: &str) -> Index { + let hmod_src = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../crates/horizon-mods/tests/index/coords_hud-0.1.0.hmod"); + let hmod = dir.join("coords_hud-0.1.0.hmod"); + fs::copy(hmod_src, &hmod).unwrap(); + let hmod_sha = idx::sha256_file(&hmod).unwrap(); + let hpack = dir.join("cozy-1.0.0.hpack"); + let mut zip = zip::ZipWriter::new(fs::File::create(&hpack).unwrap()); + let o = zip::write::SimpleFileOptions::default(); + zip.start_file("horizon-modpack.toml", o).unwrap(); + write!( + zip, + "format = 1\nid = \"cozy\"\nname = \"Cozy\"\nversion = \"1.0.0\"\ngame = \"26.3\"\n\ + [[mods]]\nid = \"coords_hud\"\nversion = \"0.1.0\"\nsha256 = \"{hmod_sha}\"\non = {on}\n" + ) + .unwrap(); + zip.start_file("overrides/options.txt", o).unwrap(); + zip.write_all(b"renderDistance:8\n").unwrap(); + zip.finish().unwrap(); + let size = |p: &Path| fs::metadata(p).unwrap().len(); + // A `file:///` URL: forward slashes (a Windows path's backslashes would be escapes in + // the JSON) and a leading slash before a Windows drive. + let url = |p: &Path| { + let path = p.display().to_string().replace('\\', "/"); + if path.starts_with('/') { + path + } else { + format!("/{path}") + } + }; + let text = format!( + r#"{{"schema": 1, "projects": [ + {{"id": "coords_hud", "kind": "mod", "name": "Coordinates HUD", "summary": "", "license": "MIT", + "source": "https://x.test", "versions": [{{"version": "0.1.0", "api": "0.1", "side": "client", + "capabilities": ["game-read", "hud", "chat"], "game": "26.3", "file": {{"name": "coords_hud-0.1.0.hmod", + "url": "file://{}", "size": {}, "sha256": "{hmod_sha}"}}}}]}}, + {{"id": "cozy", "kind": "modpack", "name": "Cozy", "summary": "", "license": "MIT", + "source": "https://x.test", "versions": [{{"version": "1.0.0", "api": "1", "game": "26.3", + "requires": [{{"id": "coords_hud", "version": "0.1.0", "sha256": "{hmod_sha}"}}], + "file": {{"name": "cozy-1.0.0.hpack", "url": "file://{}", "size": {}, "sha256": "{}"}}}}]}} + ]{extra}}}"#, + url(&hmod), + size(&hmod), + url(&hpack), + size(&hpack), + idx::sha256_file(&hpack).unwrap() + ); + Index::parse_local(&text).unwrap() + } + + fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "horizon-launcher-hpack-{name}-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn a_pack_installs_as_a_profile_with_its_mods() { + let dir = scratch("install"); + let index = registry(&dir, false, ""); + let packs = dir.join("horizon-packs"); + let pack = install_in(&index, true, "cozy", &packs, &|_, _| {}).unwrap(); + assert_eq!(pack.mods, ["coords_hud 0.1.0"]); + let home = home(&packs.join("cozy")); + assert!(home.join("horizon-mods/coords_hud.hmod").is_file()); + assert!( + !java::is_enabled(&home, "coords_hud"), + "the pack turns it off" + ); + assert_eq!( + fs::read_to_string(home.join("options.txt")).unwrap(), + "renderDistance:8\n" + ); + assert_eq!( + list_in(&packs), + [Installed { + work: packs.join("cozy"), + ..pack + }] + ); + // Settings the player changed stay on an update. + fs::write(home.join("options.txt"), "renderDistance:12\n").unwrap(); + install_in(&index, true, "cozy", &packs, &|_, _| {}).unwrap(); + assert_eq!( + fs::read_to_string(home.join("options.txt")).unwrap(), + "renderDistance:12\n" + ); + // Not a modpack, and not there. + assert!(install_in(&index, true, "coords_hud", &packs, &|_, _| {}) + .unwrap_err() + .contains("not a modpack")); + assert!(install_in(&index, true, "nope", &packs, &|_, _| {}).is_err()); + fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn a_pack_with_a_taken_down_mod_installs_nothing() { + let dir = scratch("revoked"); + let index = registry( + &dir, + true, + r#", "revoked": [{"id": "coords_hud", "reason": "Test."}]"#, + ); + let packs = dir.join("horizon-packs"); + let e = install_in(&index, true, "cozy", &packs, &|_, _| {}).unwrap_err(); + assert!(e.contains("taken down: Test."), "{e}"); + assert!(!packs.join("cozy").exists(), "nothing left behind"); + fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/launcher/src/browser/horizon/tests.rs b/launcher/src/browser/horizon/tests.rs new file mode 100644 index 000000000..fb99462e1 --- /dev/null +++ b/launcher/src/browser/horizon/tests.rs @@ -0,0 +1,164 @@ +//! The registry source against the sample index of `horizon_mods::index` +//! (`crates/horizon-mods/tests/index/`), its files served from disk. +use super::*; + +fn fixtures() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../crates/horizon-mods/tests/index") +} + +/// The sample index with its files on disk, as a local test registry lists them. +fn sample() -> Index { + let text = fs::read_to_string(fixtures().join("index-v1.json")).unwrap(); + let mut index = Index::parse(&text).unwrap(); + for p in &mut index.projects { + for v in &mut p.versions { + let path = fixtures().join(&v.file.name); + v.file.url = format!("file://{}", path.canonicalize().unwrap_or(path).display()); + } + } + index +} + +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "horizon-launcher-registry-{name}-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir +} + +fn hud(index: &Index) -> Project { + project_of(index.project("coords_hud").unwrap()).unwrap() +} + +#[test] +fn searches_by_kind_and_words() { + let index = sample(); + let found = |text: &str, kind| -> Vec { + search(&index, &Query::new(text, kind)) + .into_iter() + .map(|p| p.id) + .collect() + }; + assert_eq!(found("", Kind::Mod), ["coords_hud"]); + assert_eq!(found("coordinates HUD", Kind::Mod), ["coords_hud"]); + assert!(found("zoom", Kind::Mod).is_empty()); + // Modpacks under Modpacks; a kind this launcher can't install yet (a world) never. + assert_eq!(found("", Kind::Modpack), ["cozy"]); + assert!(found("skyblock", Kind::ResourcePack).is_empty()); + let p = hud(&index); + assert_eq!(p.support(), super::super::Support::Works); + assert_eq!(p.page, "https://playhorizon.dev/mods/coords_hud"); +} + +#[test] +fn installs_checked_turns_on_and_removes() { + let index = sample(); + let home = scratch("install"); + let p = hud(&index); + let done = install_from(&index, true, &p, &home, &|_, _| {}).unwrap(); + assert_eq!(done[0].file, "horizon-mods/coords_hud.hmod"); + assert_eq!(done[0].verdict.as_deref(), Some("Verified")); + assert!(done[0].checksum.starts_with("SHA-256:e1135345")); + assert!(home.join("horizon-mods/coords_hud.hmod").is_file()); + let list = fs::read_to_string(home.join("horizon-mods.txt")).unwrap(); + assert_eq!(list, "mod.coords_hud:true\n"); + assert_eq!( + fs::read_dir(home.join("mod-browser/downloads")) + .unwrap() + .count(), + 0 + ); + // Installed already: nothing to do. + let again = install_from(&index, true, &p, &home, &|_, _| {}).unwrap(); + assert_eq!(again, done); + + let mut revoked = index.clone(); + revoked.revoked.push(index::Revoked { + id: "coords_hud".into(), + sha256: None, + reason: "Test.".into(), + date: String::new(), + }); + let said = turn_off_revoked(&revoked, &home); + assert_eq!(said, ["Coordinates HUD was taken down and is off: Test."]); + assert_eq!( + fs::read_to_string(home.join("horizon-mods.txt")).unwrap(), + "" + ); + + installed::remove(&home, "coords_hud").unwrap(); + assert!(!home.join("horizon-mods/coords_hud.hmod").exists()); + assert!(installed::Installed::load(&home).entries.is_empty()); + fs::remove_dir_all(&home).unwrap(); +} + +#[test] +fn refuses_a_changed_file_a_taken_id_and_local_files_from_the_real_registry() { + let index = sample(); + let p = hud(&index); + + let home = scratch("tampered"); + let mut tampered = index.clone(); + let cozy = tampered.projects[1].versions[0].file.url.clone(); + tampered.projects[0].versions[0].file.url = cozy; + let e = install_from(&tampered, true, &p, &home, &|_, _| {}).unwrap_err(); + assert!(e.contains("larger than its source says"), "{e}"); + assert!(!home.join("horizon-mods").exists()); + + let e = install_from(&index, false, &p, &home, &|_, _| {}).unwrap_err(); + assert!(e.contains("file servers"), "{e}"); + + fs::create_dir_all(home.join("horizon-mods/coords_hud")).unwrap(); + let e = install_from(&index, true, &p, &home, &|_, _| {}).unwrap_err(); + assert!(e.contains("already in horizon-mods"), "{e}"); + + let mut taken_down = index.clone(); + taken_down.revoked.push(index::Revoked { + id: "coords_hud".into(), + sha256: Some(index.projects[0].versions[0].file.sha256.clone()), + reason: "Test.".into(), + date: String::new(), + }); + let e = install_from(&taken_down, true, &p, &home, &|_, _| {}).unwrap_err(); + assert_eq!(e, "Coordinates HUD was taken down: Test."); + fs::remove_dir_all(&home).unwrap(); +} + +#[test] +fn registry_versions_run_when_made_for_this_game() { + use super::super::{pick, support, Support}; + let v = |game: &str| vec![game.to_owned()]; + let horizon = vec![LOADER.to_owned()]; + assert_eq!(support(Kind::Mod, &v(GAME), &horizon), Support::Works); + assert!(!support(Kind::Mod, &v("26.2"), &horizon).installable()); + let index = sample(); + let p = index.project("coords_hud").unwrap(); + let versions: Vec = p + .versions + .iter() + .map(|x| version_of(&index, p, x)) + .collect(); + assert_eq!(pick(Kind::Mod, &versions, false).unwrap().number, "0.1.0"); +} + +#[test] +fn owner_links_only_take_github_logins() { + for ok in ["Stink-O", "ludovic111", "a"] { + assert!(github_login(ok), "{ok}"); + } + for bad in [ + "", + "-x", + "x-", + "a--b", + "bad login", + "a/b", + "x?y=1", + &"a".repeat(40), + ] { + assert!(!github_login(bad), "{bad}"); + } +} diff --git a/launcher/src/browser/images.rs b/launcher/src/browser/images.rs index fee2f219e..f5029a7af 100644 --- a/launcher/src/browser/images.rs +++ b/launcher/src/browser/images.rs @@ -9,10 +9,12 @@ use std::io::Read; use std::path::{Path, PathBuf}; /// Hosts images may come from. -const HOSTS: [&str; 3] = [ +const HOSTS: [&str; 4] = [ "https://cdn.modrinth.com/", "https://media.forgecdn.net/", "https://edge.forgecdn.net/", + // Horizon's registry's icons (its GitHub Pages). + "https://ludovic111.github.io/horizon-mods/", ]; /// The most an image may weigh before decoding. pub const MAX_DOWNLOAD: u64 = 8 << 20; diff --git a/launcher/src/browser/installed.rs b/launcher/src/browser/installed.rs index 451bdd43e..c866dd26d 100644 --- a/launcher/src/browser/installed.rs +++ b/launcher/src/browser/installed.rs @@ -170,6 +170,9 @@ pub fn install( home: &Path, progress: &dyn Fn(&str, f32), ) -> Result, String> { + if project.source == SourceId::Horizon { + return super::horizon::install(project, home, progress); + } if project.kind == Kind::Mod { return super::java::install(source, project, everything, home, progress); } @@ -248,7 +251,7 @@ pub fn remove(home: &Path, name: &str) -> Result { .lookup(name) .cloned() .ok_or_else(|| format!("{name} wasn't installed by the browser."))?; - if entry.kind == Kind::Mod { + if entry.kind == Kind::Mod && entry.source != SourceId::Horizon { super::java::remove(home, &entry)?; } else { match fs::remove_file(home.join(&entry.file)) { @@ -257,6 +260,9 @@ pub fn remove(home: &Path, name: &str) -> Result { Err(e) => return Err(format!("{}: {e}", entry.file)), } } + if let Some(id) = super::horizon::mod_id(&entry).filter(|_| entry.source == SourceId::Horizon) { + super::java::set_enabled(home, id, false)?; + } installed .entries .retain(|e| !(e.source == entry.source && e.project == entry.project)); diff --git a/launcher/src/browser/installed/tests.rs b/launcher/src/browser/installed/tests.rs index e940ce0f1..cb2814c29 100644 --- a/launcher/src/browser/installed/tests.rs +++ b/launcher/src/browser/installed/tests.rs @@ -86,6 +86,7 @@ impl Fake { .map(|b| format!("{b:02x}")) .collect(), ), + sha256: None, sha1: None, primary: true, }; diff --git a/launcher/src/browser/java.rs b/launcher/src/browser/java.rs index 76ce43c95..4ac7ae084 100644 --- a/launcher/src/browser/java.rs +++ b/launcher/src/browser/java.rs @@ -110,6 +110,16 @@ pub fn check_project( Ok((version, check)) } +/// Whether mod `id` is on in `horizon-mods.txt`. +pub fn is_enabled(home: &Path, id: &str) -> bool { + let key = format!("mod.{id}"); + fs::read_to_string(home.join(MODS_LIST)) + .unwrap_or_default() + .lines() + .filter_map(|l| l.split_once(':')) + .any(|(k, v)| k.trim() == key && v.trim() == "true") +} + /// Turns mod `id` on (or off) in `horizon-mods.txt`, keeping the other lines. pub fn set_enabled(home: &Path, id: &str, on: bool) -> Result<(), String> { let path = home.join(MODS_LIST); @@ -326,6 +336,7 @@ mod tests { name: "a.jar".into(), size: 1, sha512: sha512.map(str::to_owned), + sha256: None, sha1: sha1.map(str::to_owned), primary: true, }; diff --git a/launcher/src/browser/modrinth.rs b/launcher/src/browser/modrinth.rs index 722549742..52e0aa7f0 100644 --- a/launcher/src/browser/modrinth.rs +++ b/launcher/src/browser/modrinth.rs @@ -139,6 +139,7 @@ pub(super) fn parse_version(v: &Value) -> Option { name: f["filename"].as_str()?.to_owned(), size: f["size"].as_u64().unwrap_or(0), sha512: f["hashes"]["sha512"].as_str().map(str::to_ascii_lowercase), + sha256: None, sha1: f["hashes"]["sha1"].as_str().map(str::to_ascii_lowercase), primary: f["primary"].as_bool().unwrap_or(false), }) diff --git a/launcher/src/browser/tests.rs b/launcher/src/browser/tests.rs index 6167d5bd9..50bfbae40 100644 --- a/launcher/src/browser/tests.rs +++ b/launcher/src/browser/tests.rs @@ -18,6 +18,7 @@ fn version(id: &str, games: &[&str], loaders: &[&str]) -> Version { name: format!("{id}.zip"), size: 3, sha512: Some("ab".into()), + sha256: None, sha1: None, primary: true, }], @@ -305,6 +306,7 @@ fn file_for(body: &[u8], sha512: bool) -> FileRef { name: "pack.zip".into(), size: body.len() as u64, sha512: sha512.then(|| hex(&sha2::Sha512::digest(body))), + sha256: None, sha1: (!sha512).then(|| hex(&sha1::Sha1::digest(body))), primary: true, } @@ -383,6 +385,6 @@ fn live_modrinth_search_pick_and_download() { #[test] fn curseforge_is_coming_soon_and_never_searched() { let ids: Vec = sources().iter().map(|s| s.id()).collect(); - assert_eq!(ids, [SourceId::Modrinth]); + assert_eq!(ids, [SourceId::Modrinth, SourceId::Horizon]); assert!(source(SourceId::CurseForge).is_none()); } diff --git a/launcher/src/cli.rs b/launcher/src/cli.rs index d104fe7eb..25bbe0bee 100644 --- a/launcher/src/cli.rs +++ b/launcher/src/cli.rs @@ -199,6 +199,10 @@ enum ModsCmd { }, /// Remove a pack the browser installed Remove { project: String }, + /// Turn on a mod the browser installed (horizon-mods.txt, as the game's Options > Mods does) + Enable { project: String }, + /// Turn off a mod the browser installed; it stays installed + Disable { project: String }, } #[derive(Clone, Copy, clap::ValueEnum)] @@ -213,6 +217,8 @@ enum SourceArg { Modrinth, /// CurseForge support is coming soon Curseforge, + /// Horizon's own registry: mods, modpacks and resource packs made for Horizon, reviewed + Horizon, } impl SourceArg { @@ -221,6 +227,7 @@ impl SourceArg { let id = match self { SourceArg::Modrinth => SourceId::Modrinth, SourceArg::Curseforge => SourceId::CurseForge, + SourceArg::Horizon => SourceId::Horizon, }; browser::source(id).ok_or_else(|| browser::CURSEFORGE_SOON.into()) } @@ -1064,11 +1071,23 @@ fn servers(action: ServersCmd, as_json: bool) -> Result<(), String> { } fn mods(action: ModsCmd, as_json: bool) -> Result<(), String> { - use horizon_launcher::browser::{installed, Kind, Query}; + use horizon_launcher::browser::{horizon, installed, Kind, Query, SourceId}; use horizon_launcher::modpacks; let home = backend::game_home(); let entry_json = |e: &installed::Entry| serde_json::to_value(e).unwrap_or_default(); let line = |label: &str, f: f32| progress(label.to_string())(f); + let from_registry = installed::Installed::load(&home) + .entries + .iter() + .any(|e| e.source == SourceId::Horizon); + if from_registry { + if let Ok(index) = horizon::index() { + for said in horizon::turn_off_revoked(&index, &home) { + eprintln!("{said}"); + } + } + } + let on = matches!(action, ModsCmd::Enable { .. }); match action { ModsCmd::List => { let list = installed::Installed::load(&home).entries; @@ -1114,11 +1133,19 @@ fn mods(action: ModsCmd, as_json: bool) -> Result<(), String> { "id": p.id, "slug": p.slug, "title": p.title, "summary": p.summary, "author": p.author, "downloads": p.downloads, "page": p.page, "license": p.license, "support": support.label(), "reason": support.reason(), + "about": if p.source == SourceId::Horizon { horizon::trust_and_capabilities(&p.id) } else { Vec::new() }, }) }).collect()); } else { for p in &found { let support = p.support(); + if p.source == SourceId::Horizon { + println!("{} ({}) {}\n {}", p.title, p.slug, p.author, p.summary); + for line in horizon::trust_and_capabilities(&p.id) { + println!(" {line}"); + } + continue; + } let unchecked = p.kind == Kind::Mod && support.installable(); println!( "{} ({}) {} · {} downloads\n {}", @@ -1187,6 +1214,23 @@ fn mods(action: ModsCmd, as_json: bool) -> Result<(), String> { } => { let source = source.get()?; let p = source.project(&project)?; + if p.kind == Kind::Modpack && p.source == SourceId::Horizon { + let pack = horizon::pack::install(&p.id, &line)?; + if as_json { + json( + serde_json::json!({"pack": pack.id, "version": pack.version, "mods": pack.mods, "home": horizon::pack::home(&pack.work)}), + ); + } else { + println!( + "{} {} is installed as its own profile, with {}.", + pack.name, + pack.version, + pack.mods.join(", ") + ); + println!("Play it: horizon-launcher modpack play {}", pack.id); + } + return Ok(()); + } if p.kind == Kind::Modpack { let folder = modpacks::ports_dir().join("downloads"); let file = @@ -1251,15 +1295,43 @@ fn mods(action: ModsCmd, as_json: bool) -> Result<(), String> { let e = installed::remove(&home, &project)?; println!("Removed {} ({}).", e.title, e.file); } + ModsCmd::Enable { project } | ModsCmd::Disable { project } => { + let e = installed::Installed::load(&home) + .lookup(&project) + .cloned() + .ok_or_else(|| format!("{project} wasn't installed by the browser."))?; + let id = horizon::mod_id(&e) + .filter(|_| e.kind == Kind::Mod) + .ok_or_else(|| format!("{} isn't a mod: only mods turn on and off.", e.title))?; + if on { + if let Some(r) = horizon::index() + .ok() + .filter(|_| e.source == SourceId::Horizon) + .and_then(|i| { + i.revoked(&e.project, e.checksum.trim_start_matches("SHA-256:")) + .cloned() + }) + { + return Err(format!("{} was taken down: {}", e.title, r.reason)); + } + } + horizon_launcher::browser::java::set_enabled(&home, id, on)?; + println!("{} is {}.", e.title, if on { "on" } else { "off" }); + } } Ok(()) } fn modpack(action: ModpackCmd, as_json: bool) -> Result<(), String> { + use horizon_launcher::browser::horizon; use horizon_launcher::modpacks::{self, Imported}; use horizon_port_pack::report::Status; let log = &|line: String| eprintln!("{line}"); let work = |pack: &str| { + let registry = horizon::pack::packs_dir().join(pack); + if horizon::pack::is_pack(®istry) { + return Ok(registry); + } let work = modpacks::ports_dir().join(pack); modpacks::load(&work).map(|_| work) }; @@ -1293,10 +1365,26 @@ fn modpack(action: ModpackCmd, as_json: bool) -> Result<(), String> { match action { ModpackCmd::List => { let packs = modpacks::list(); - if packs.is_empty() && !as_json { + let registry = horizon::pack::list(); + if packs.is_empty() && registry.is_empty() && !as_json { println!("No modpack imported yet: horizon-launcher modpack import "); } packs.iter().for_each(show); + for p in ®istry { + if as_json { + json( + serde_json::json!({"pack": p.id, "name": p.name, "version": p.version, "source": "horizon", "mods": p.mods}), + ); + } else { + println!( + "{} {} ({}, from Horizon's registry)\n {}", + p.name, + p.version, + p.id, + p.mods.join(", ") + ); + } + } } ModpackCmd::Import { path } => show(&modpacks::import(&path, log)?), ModpackCmd::Port { diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 0bcbc4011..afc8dde45 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -205,6 +205,8 @@ struct Launcher { /// The Modpacks page: the imported packs (read when the page first opens), the one shown, /// the report's filter, and the pipeline step running now with its log. packs: Vec, + /// Modpacks installed from Horizon's registry, each its own profile. + registry_packs: Vec, packs_loaded: Option, pack_open: Option, pack_filter: Option, @@ -265,6 +267,9 @@ impl Launcher { { settings.selected = installed.first().cloned(); } + // Pin the window's chrome (macOS title bar) to the choice before the theme reads + // `window.appearance()`, so both draw the same look. + titlebar::apply_appearance(window, settings.appearance); let mut this = Self { theme: Theme::for_appearance(settings.appearance, window.appearance()), page: Page::from_env(), @@ -330,6 +335,7 @@ impl Launcher { play_pressed: None, bench: perf::Bench::from_env(), packs: Vec::new(), + registry_packs: Vec::new(), packs_loaded: None, pack_open: None, pack_filter: None, @@ -2482,6 +2488,7 @@ impl Launcher { let _ = this.update(cx, |this, cx| { this.settings.appearance = choice; this.settings.save(); + titlebar::apply_appearance(window, choice); this.theme = Theme::for_appearance(choice, window.appearance()); cx.notify(); }); diff --git a/launcher/src/modpacks.rs b/launcher/src/modpacks.rs index 909c6cfae..acd8241a5 100644 --- a/launcher/src/modpacks.rs +++ b/launcher/src/modpacks.rs @@ -7,6 +7,7 @@ //! `home/`, the profile's own saves and options. Nothing is uploaded or shared: mod jars are //! often All Rights Reserved, so the ports stay on the player's machine. use crate::backend; +use crate::browser::horizon::pack as horizon_pack; use horizon_port_pack::{ fetch::{Fetched, How}, import::Pack, @@ -232,6 +233,9 @@ pub fn outcome(pack: &Imported) -> String { /// Removes an imported pack: its jars, ports and the profile's saves. pub fn remove(work: &Path) -> Result<(), String> { + if horizon_pack::is_pack(work) { + return horizon_pack::remove(work); + } if work.parent() != Some(ports_dir().as_path()) { return Err(format!("{} is not an imported pack", work.display())); } @@ -245,7 +249,11 @@ pub fn profile_home(work: &Path) -> PathBuf { /// Turns the game's command into this pack's profile: its own home (the player's options.txt /// copied in the first time, so keys and video settings carry over) and its carried-over mods. +/// A modpack from Horizon's registry (`horizon-packs//`) plays its own profile the same way. pub fn apply_profile(cmd: &mut Command, work: &Path) -> Result<(), String> { + if horizon_pack::is_pack(work) { + return horizon_pack::apply(cmd, work); + } // Only mods that load together: the game would refuse the later of two sharing a namespace. let (mods, _) = pipeline::loadable(&carried(work)); if mods.is_empty() { diff --git a/launcher/src/modpacks_page.rs b/launcher/src/modpacks_page.rs index 6346f8a0b..25df3966f 100644 --- a/launcher/src/modpacks_page.rs +++ b/launcher/src/modpacks_page.rs @@ -1,6 +1,7 @@ //! Imported modpacks, on the Mods page's Modpacks tab: import a pack, read its coverage report, //! port what this computer can and play it as its own profile (`horizon_launcher::modpacks`). use super::*; +use horizon_launcher::browser::horizon::pack as horizon_pack; use horizon_launcher::modpacks::{self, Imported}; use horizon_port_pack::report::Status; use std::path::PathBuf; @@ -130,6 +131,7 @@ impl Launcher { if self.packs_loaded.is_none() { self.mods.report_list.reset(); self.packs = modpacks::list(); + self.registry_packs = horizon_pack::list(); if self.pack_open.is_none() { self.pack_open = self.packs.first().map(|p| p.slug.clone()); } @@ -177,6 +179,12 @@ impl Launcher { if let Some(e) = &self.pack_error { page = page.child(notice(e.clone(), &t)); } + if !self.registry_packs.is_empty() { + page = page.child(self.registry_pack_rows(cx)); + } + if self.packs.is_empty() && !self.registry_packs.is_empty() { + return page; + } if self.packs.is_empty() { return page.child( text(theme::CAPTION) @@ -216,6 +224,78 @@ impl Launcher { } } + /// The modpacks installed from Horizon's registry: each plays its own profile. + fn registry_pack_rows(&self, cx: &mut Context) -> Div { + let t = self.theme; + let mut list = div().flex().flex_col(); + for (i, p) in self.registry_packs.iter().enumerate() { + let (play, remove) = (p.work.clone(), p.work.clone()); + let id = p.id.clone(); + let key = format!("horizon:{id}"); + let confirming = self.confirm_remove.as_deref() == Some(key.as_str()); + let right = div() + .flex() + .gap(px(8.)) + .child( + button(format!("play-pack-{id}"), "Play", Kind::Primary, &t).on_click( + cx.listener(move |this, _: &ClickEvent, _, cx| { + this.play_pack(play.clone()); + cx.notify(); + }), + ), + ) + .child( + button( + format!("remove-pack-{id}"), + if confirming { + "Remove for good" + } else { + "Remove" + }, + if confirming { + Kind::Secondary + } else { + Kind::Ghost + }, + &t, + ) + .when(confirming, |b| b.text_color(t.danger)) + .on_click(cx.listener( + move |this, _: &ClickEvent, _, cx| { + if this.confirm_remove.as_ref() == Some(&key) { + match horizon_pack::remove(&remove) { + Ok(()) => this.packs_loaded = None, + Err(e) => this.pack_error = Some(e), + } + this.confirm_remove = None; + } else { + this.confirm_remove = Some(key.clone()); + } + cx.notify(); + }, + )), + ); + let detail = format!( + "{} from Horizon's registry, its own profile: {}", + p.version, + p.mods.join(", ") + ); + list = list.child(index_row( + i + 1, + p.name.clone(), + Some( + text(theme::CAPTION) + .text_color(t.muted) + .child(detail) + .into_any_element(), + ), + Some(right.into_any_element()), + &t, + )); + } + list + } + fn pack_log_plate(&self, t: &Theme) -> Div { let start = self.pack_log.len().saturating_sub(LOG_SHOWN); plate(t) diff --git a/launcher/src/mods_page.rs b/launcher/src/mods_page.rs index 86e5e2e48..50434756c 100644 --- a/launcher/src/mods_page.rs +++ b/launcher/src/mods_page.rs @@ -1,11 +1,13 @@ -//! The Mods page: search Modrinth (CurseForge: coming soon) for Java mods, modpacks and resource -//! packs, see where each comes from and how well it runs (a Java mod's jar is -//! checked first), install it with its checksum checked, and update or remove what was installed -//! (`horizon_launcher::browser`). Imported modpacks are listed on its Modpacks tab. +//! The Mods page: search Horizon's own registry for mods made for Horizon, or Modrinth +//! (CurseForge: coming soon) for Java mods, modpacks and resource packs, see where each comes +//! from and how well it runs (a Java mod's jar is checked first; a Horizon mod says whether it is +//! signed and what it may do), install it with its checksum checked, turn Horizon mods on and +//! off, and update or remove what was installed (`horizon_launcher::browser`). Imported modpacks +//! are listed on its Modpacks tab. use super::servers_page::{type_key, Key}; use super::*; use horizon_launcher::browser::{ - self, installed, java, Kind as PackKind, Project, Query, SourceId, Support, + self, horizon, installed, java, Kind as PackKind, Project, Query, SourceId, Support, }; use horizon_launcher::modpacks; use horizon_port_pack::check::Check; @@ -20,7 +22,10 @@ const KINDS: [(PackKind, &str, &str); 3] = [ /// Java mods checked at once. const CHECKING: usize = 2; /// The sources to pick from; CurseForge is shown beside them as coming soon. -const SOURCES: [(SourceId, &str); 1] = [(SourceId::Modrinth, "Modrinth")]; +const SOURCES: [(SourceId, &str); 2] = [ + (SourceId::Modrinth, "Modrinth"), + (SourceId::Horizon, "Horizon"), +]; const PAGE: usize = 20; pub(super) enum Event { @@ -31,6 +36,10 @@ pub(super) enum Event { Updates(HashMap<(SourceId, String), String>), /// A Java mod's check: its jar read, or why it couldn't be. Checked((SourceId, String), Result, String>), + /// What a Horizon registry project says before install: signed or not, what it may do. + About(HashMap>), + /// Registry files taken down that the launcher turned off. + TakenDown(Vec), /// A project page's part, an image or a filter's values (`mods_project`). Loaded(super::mods_project::Loaded), } @@ -68,6 +77,8 @@ pub(super) struct Mods { confirm_remove: Option, /// Java mods' checks, by source and project. pub(super) checks: HashMap<(SourceId, String), Checking>, + /// Horizon registry projects' trust and capabilities, by id ([`horizon::trust_and_capabilities`]). + pub(super) about: HashMap>, /// The project page, filters and images. pub(super) browse: super::mods_project::Browse, pub(super) tx: Sender, @@ -94,7 +105,11 @@ impl Mods { } else { PackKind::Mod }, - source: SourceId::Modrinth, + // `HORIZON_LAUNCHER_SOURCE=horizon` opens on Horizon's registry, for screenshots. + source: match std::env::var("HORIZON_LAUNCHER_SOURCE").as_deref() { + Ok("horizon") => SourceId::Horizon, + _ => SourceId::Modrinth, + }, text: String::new(), focus: cx.focus_handle().tab_index(0).tab_stop(true), focused: false, @@ -115,6 +130,7 @@ impl Mods { updates: HashMap::new(), confirm_remove: None, checks: HashMap::new(), + about: HashMap::new(), browse: Default::default(), tx, rx, @@ -134,7 +150,7 @@ impl Mods { /// How well a result runs: a Java mod's from its check once it has one. pub(super) fn support(&self, p: &Project) -> Support { let metadata = p.support(); - if p.kind != PackKind::Mod || !metadata.installable() { + if p.kind != PackKind::Mod || p.source == SourceId::Horizon || !metadata.installable() { return metadata; } match self.checks.get(&(p.source, p.id.clone())) { @@ -231,6 +247,16 @@ impl Launcher { m.checks.insert(key, state); } Event::Loaded(l) => loaded.push(l), + Event::About(about) => { + // Rows grow by a line: measure them again (`cull`). + m.about.extend(about); + m.results_list.reset(); + } + Event::TakenDown(lines) => { + m.message = Some(Ok(lines.join("\n"))); + m.installed = installed::Installed::load(&backend::game_home()).entries; + m.installed_list.reset(); + } } } // `HORIZON_LAUNCHER_PROJECT=` (with `HORIZON_LAUNCHER_PAGE=mods`): opens that @@ -278,7 +304,63 @@ impl Launcher { let result = browser::source(source) .ok_or_else(|| format!("{} isn't set up.", source.label())) .and_then(|s| s.search(&query)); + let about: HashMap> = match (&result, source) { + (Ok(found), SourceId::Horizon) => found + .iter() + .map(|p| (p.id.clone(), horizon::trust_and_capabilities(&p.id))) + .collect(), + _ => HashMap::new(), + }; let _ = tx.send(Event::Found(search, offset, result)); + if !about.is_empty() { + let _ = tx.send(Event::About(about)); + } + }); + } + + /// Reads the registry's index again when something installed came from it, and turns off + /// what it took down since (`horizon::turn_off_revoked`). + fn mods_taken_down(&mut self) { + if !self + .mods + .installed + .iter() + .any(|e| e.source == SourceId::Horizon) + { + return; + } + let tx = self.mods.tx.clone(); + std::thread::spawn(move || { + let Ok(index) = horizon::index() else { + return; + }; + let lines = horizon::turn_off_revoked(&index, &backend::game_home()); + if !lines.is_empty() { + let _ = tx.send(Event::TakenDown(lines)); + } + }); + } + + /// Turns a Horizon mod the browser installed on or off, as the game's Options > Mods does. + fn mods_switch(&mut self, e: &installed::Entry) { + let home = backend::game_home(); + let Some(id) = horizon::mod_id(e) else { + return; + }; + let on = !java::is_enabled(&home, id); + let revoked = horizon::known_index().and_then(|index| { + let hash = e.checksum.trim_start_matches("SHA-256:"); + index.revoked(&e.project, hash).map(|r| r.reason.clone()) + }); + self.mods.message = Some(match revoked.filter(|_| on) { + Some(reason) => Err(format!("{} was taken down: {reason}", e.title)), + None => java::set_enabled(&home, id, on).map(|()| { + format!( + "{} is {}. The game reads this when it starts.", + e.title, + if on { "on" } else { "off" } + ) + }), }); } @@ -288,7 +370,8 @@ impl Launcher { let m = &mut self.mods; let todo: Vec = found .into_iter() - .filter(|p| p.kind == PackKind::Mod && p.support().installable()) + .filter(|p| p.kind == PackKind::Mod && p.source != SourceId::Horizon) + .filter(|p| p.support().installable()) .filter(|p| !m.checks.contains_key(&(p.source, p.id.clone()))) .collect(); if todo.is_empty() { @@ -385,6 +468,16 @@ impl Launcher { None => source.project(&id)?, }; modpack = p.kind == PackKind::Modpack; + if modpack && p.source == SourceId::Horizon { + let pack = horizon::pack::install(&p.id, &progress)?; + return Ok(format!( + "Installed {} {} as its own profile with {}, each checked. Play it \ + from the list above.", + pack.name, + pack.version, + pack.mods.join(", ") + )); + } if modpack { let folder = modpacks::ports_dir().join("downloads"); let file = installed::fetch_modpack(&*source, &p, &folder, &|f| { @@ -403,6 +496,13 @@ impl Launcher { .iter() .map(|e| format!("{} {}", e.title, e.number)) .collect(); + if p.source == SourceId::Horizon && p.kind == PackKind::Mod { + return Ok(format!( + "Installed {} into horizon-mods and turned it on, checksum checked \ + (Options > Mods in the game).", + names.join(", ") + )); + } if p.kind == PackKind::Mod { let skipped = done.last().map_or(0, |e| e.skipped.len()); return Ok(format!( @@ -497,14 +597,15 @@ impl Launcher { self.mods.loaded = true; self.mods.installed = installed::Installed::load(&backend::game_home()).entries; self.mods_check_updates(); + self.mods_taken_down(); self.mods_search(0); } let header = page_header( "Mods", - "Java mods, modpacks and resource packs. Horizon ports what a Java mod \ - carries (blocks, items, recipes, worldgen) and says before you install what runs. \ - Each file is checked against its source's checksum before it reaches your game \ - folder.", + "Mods made for Horizon from its own reviewed registry, and Java mods, modpacks and \ + resource packs from Modrinth: Horizon ports what a Java mod carries (blocks, items, \ + recipes, worldgen) and says before you install what runs. Each file is checked \ + against its source's checksum before it reaches your game folder.", &t, ); let this = cx.entity().downgrade(); @@ -543,7 +644,10 @@ impl Launcher { })); // CurseForge support is coming soon: shown, inert, never searched (like the Microsoft // account's "Coming soon"). - let source_note = div().flex().child(source_switch(&t)); + let source_note = div().flex().child(self.mods_source_switch(cx)); + // The registry lists only what was made for Horizon: no "everything", no Modrinth filters. + let registry = self.mods.source == SourceId::Horizon; + let filters = (!registry).then(|| self.mods_filters(cx)); let controls = div() .flex() .flex_col() @@ -555,10 +659,10 @@ impl Launcher { .items_center() .gap(px(12.)) .child(self.mods_field(cx)) - .child(toggle), + .when(!registry, |d| d.child(toggle)), ) .child(source_note) - .child(self.mods_filters(cx)); + .children(filters); // The kinds are inventory tabs on the panel that holds the open one. let body = blocks::panel(&t).flex().flex_col().gap(px(24.)).p(px(20.)); let page = |body: Div| { @@ -687,6 +791,30 @@ impl Launcher { } cx.notify(); })); + if e.source == SourceId::Horizon && e.kind == PackKind::Mod { + let on = horizon::mod_id(e) + .is_some_and(|id| java::is_enabled(&backend::game_home(), id)); + let entry = e.clone(); + right = right.child( + div() + .id(SharedString::from(format!("mods-switch-{}", e.project))) + .flex() + .items_center() + .gap(px(8.)) + .cursor_pointer() + // Its id sits under the row's own, so one name serves every row. + .child(switch("mods-switch-knob", on, &t)) + .child(text(theme::LABEL).text_color(t.ink_soft).child(if on { + "On" + } else { + "Off" + })) + .on_click(cx.listener(move |this, _: &ClickEvent, _, cx| { + this.mods_switch(&entry); + cx.notify(); + })), + ); + } right = right.child(remove); let detail = div() .flex() @@ -750,6 +878,8 @@ impl Launcher { if m.results.is_empty() { let line = if m.searching || !m.searched { "Searching…" + } else if m.source == SourceId::Horizon { + "Nothing in Horizon's registry matches yet." } else if m.everything { "Nothing found." } else { @@ -768,6 +898,7 @@ impl Launcher { let busy = m.busy.as_ref().map(|b| b.0.as_str()); let support = m.support(p); let checking = p.kind == PackKind::Mod + && p.source != SourceId::Horizon && matches!( m.checks.get(&(p.source, p.id.clone())), None | Some(Checking::Running) @@ -828,7 +959,9 @@ impl Launcher { if !p.author.is_empty() { by.push(format!("by {}", p.author)); } - by.push(format!("{} downloads", count(p.downloads))); + if p.source != SourceId::Horizon { + by.push(format!("{} downloads", count(p.downloads))); + } by.push(format!("from {}", p.source.label())); if !p.license.is_empty() { by.push(p.license.clone()); @@ -842,6 +975,12 @@ impl Launcher { support .reason() .map(|r| div().text_color(t.ink_soft).child(r.to_string())), + ) + .children( + m.about + .get(&p.id) + .filter(|_| p.source == SourceId::Horizon) + .map(|lines| div().text_color(t.ink_soft).child(lines.join(" · "))), ); let open = p.clone(); let name = div() @@ -884,9 +1023,30 @@ impl Launcher { } } -/// The source switch while Modrinth is the only source: Modrinth chosen, CurseForge shown as -/// coming soon and not clickable, like the Microsoft account's "Coming soon". -fn source_switch(t: &Theme) -> Stateful
{ +impl Launcher { + /// The source switch: Modrinth or Horizon's registry, CurseForge shown as coming soon and not + /// clickable, like the Microsoft account's "Coming soon". + fn mods_source_switch(&self, cx: &mut Context) -> Stateful
{ + let this = cx.entity().downgrade(); + source_switch(&self.theme, self.mods.source, move |source, _, cx| { + let _ = this.update(cx, |this, cx| { + if this.mods.source != source { + this.mods.source = source; + this.mods.browse.filters = Default::default(); + this.mods.browse.open = None; + this.mods_search(0); + } + cx.notify(); + }); + }) + } +} + +fn source_switch( + t: &Theme, + current: SourceId, + on_pick: impl Fn(SourceId, &mut Window, &mut App) + Clone + 'static, +) -> Stateful
{ let soon = div() .flex() .items_center() @@ -899,7 +1059,7 @@ fn source_switch(t: &Theme) -> Stateful
{ .text_color(t.muted) .child("CurseForge (coming soon)"), ); - segments("mods-source", &SOURCES, SourceId::Modrinth, t, |_, _, _| {}).child(soon) + segments("mods-source", &SOURCES, current, t, on_pick).child(soon) } #[cfg(test)] diff --git a/launcher/src/mods_project.rs b/launcher/src/mods_project.rs index ad75f58cd..97d23e0aa 100644 --- a/launcher/src/mods_project.rs +++ b/launcher/src/mods_project.rs @@ -423,6 +423,7 @@ impl Launcher { self.mods.checks.get(&(p.source, p.id.clone())), None | Some(Checking::Running) ) && p.kind == PackKind::Mod + && p.source != SourceId::Horizon && p.support().installable(); let mut action = div().flex().items_center().gap(px(8.)).child( caps(if checking { diff --git a/launcher/src/theme.rs b/launcher/src/theme.rs index 60b796709..9d495a598 100644 --- a/launcher/src/theme.rs +++ b/launcher/src/theme.rs @@ -74,22 +74,33 @@ fn c(hex: u32) -> Hsla { rgb(hex).into() } +/// Whether the launcher is pinned dark (`Some(true)`) or light, or follows the system (`None`): +/// `HORIZON_LAUNCHER_APPEARANCE=light` or `dark` (for screenshots and tests), else the player's +/// choice in Settings. +pub fn pinned_dark(choice: Appearance) -> Option { + match std::env::var("HORIZON_LAUNCHER_APPEARANCE").as_deref() { + Ok("light") => Some(false), + Ok("dark") => Some(true), + _ => match choice { + Appearance::Dark => Some(true), + Appearance::Light => Some(false), + Appearance::System => None, + }, + } +} + impl Theme { - /// The player's choice in Settings (dark unless they picked another), the system's when they - /// asked to follow it. `HORIZON_LAUNCHER_APPEARANCE=light` or `dark` overrides both (for - /// screenshots and tests). + /// The pinned look (`pinned_dark`: dark unless the player picked another), the system's when + /// they asked to follow it. pub fn for_appearance(choice: Appearance, system: WindowAppearance) -> Self { - match std::env::var("HORIZON_LAUNCHER_APPEARANCE").as_deref() { - Ok("light") => return Self::light(), - Ok("dark") => return Self::dark(), - _ => {} - } - match (choice, system) { - (Appearance::Light, _) => Self::light(), - (Appearance::System, WindowAppearance::Light | WindowAppearance::VibrantLight) => { - Self::light() - } - _ => Self::dark(), + let system_dark = !matches!( + system, + WindowAppearance::Light | WindowAppearance::VibrantLight + ); + if pinned_dark(choice).unwrap_or(system_dark) { + Self::dark() + } else { + Self::light() } } diff --git a/launcher/src/titlebar.rs b/launcher/src/titlebar.rs index 3c2a78ea4..a023f7d7d 100644 --- a/launcher/src/titlebar.rs +++ b/launcher/src/titlebar.rs @@ -1,6 +1,8 @@ -//! Custom Windows chrome. Other platforms keep their native window decorations. +//! Custom Windows chrome. Other platforms keep their native window decorations; on macOS, +//! `apply_appearance` repaints that chrome (the title bar) to match the chosen look. use crate::theme::Theme; use gpui::{Div, TitlebarOptions, Window}; +use horizon_launcher::backend::Appearance; /// Whether the launcher draws its own title bar (`render`) rather than the system. pub const DRAWN: bool = cfg!(target_os = "windows"); @@ -18,6 +20,44 @@ pub fn render(_theme: &Theme, _window: &Window) -> Option
{ None } +/// macOS keeps the window's appearance on the system setting, so a launcher set to Dark still +/// draws a light title bar on a light desktop. Pin the NSWindow's appearance to the choice +/// instead (`theme::pinned_dark`: `System` clears the pin, handing the chrome back to the +/// system). Elsewhere the chrome is ours (Windows) or the system's (Linux). +#[cfg(target_os = "macos")] +pub fn apply_appearance(window: &Window, choice: Appearance) { + use objc2_app_kit::{ + NSAppearance, NSAppearanceCustomization, NSAppearanceName, NSAppearanceNameAqua, + NSAppearanceNameDarkAqua, NSView, + }; + use raw_window_handle::{HasWindowHandle, RawWindowHandle}; + + // AppKit's name constants are extern statics: reading them is `unsafe` to rustc only. + let name: Option<&'static NSAppearanceName> = match crate::theme::pinned_dark(choice) { + Some(true) => Some(unsafe { NSAppearanceNameDarkAqua }), + Some(false) => Some(unsafe { NSAppearanceNameAqua }), + None => None, + }; + // The trait, not `Window::window_handle` (which is gpui's own `AnyWindowHandle`). + let Ok(handle) = HasWindowHandle::window_handle(window) else { + return; + }; + let RawWindowHandle::AppKit(appkit) = handle.as_raw() else { + return; + }; + // The handle is the window's content view; its `window` is the NSWindow itself. Both outlive + // this call. + let view = unsafe { appkit.ns_view.cast::().as_ref() }; + let Some(native) = view.window() else { + return; + }; + let appearance = name.and_then(NSAppearance::appearanceNamed); + native.setAppearance(appearance.as_deref()); +} + +#[cfg(not(target_os = "macos"))] +pub fn apply_appearance(_: &Window, _: Appearance) {} + #[cfg(target_os = "windows")] mod windows { use super::*; diff --git a/roadmap/README.md b/roadmap/README.md new file mode 100644 index 000000000..22cde1a7d --- /dev/null +++ b/roadmap/README.md @@ -0,0 +1,116 @@ +# The roadmap + +`roadmap.toml` is the one source of the roadmap. `tools/roadmap.py build` turns it into +[ROADMAP.md](../ROADMAP.md); the website's hub (playhorizon.dev/hub) and Kira read the same items. The work +board stays the task-level view (every issue and pull request); the roadmap is the overview, and each pull +request carries the `roadmap:` label of the item it moves. + +## An item + +```toml +[[item]] +id = "server-dialogs" # lowercase words joined by dashes; never renamed (votes and links use it) +title = "Server dialogs and login screens" +status = "now" # now | next | later | help-wanted | shipped +area = "servers" # one of the [[area]] ids, listed in the order of priority +focus = true # optional: one of the 3 to 5 Now items in the overview +ongoing = true # optional: a workstream that goes on after its pull requests merge +plain = """Servers that show dialogs ... will work in Horizon.""" # players' words, at most 240 characters +dev = """26.3's dialog packets and model (#1720), ...""" # crates, approach, what is left +links = ["#1720", "#1734", "docs/hosting.md"] # #N, a repository path, or https:// +shipped_in = "v0.2.58" # shipped items only: a game tag, launcher@x.y.z, or "website" +post = "..." # optional: the announcement when it ships, if `plain` doesn't read as one +announce = true # optional: when it ships, Kira also posts it on X (`post`, else `plain`) +``` + +- **No dates, ever**: Now, Next, Later. The check rejects month names, years, quarters and "next week". + "Coming soon" is the only phrase for something waiting on someone else (Microsoft sign-in, CurseForge). +- **Only what is decided.** Ideas stay in issues and in the suggestion queue until they are. +- **Only what is true in a release** for shipped items: a merged pull request isn't shipped until a release has it. +- `plain` is for players: what they will be able to do, not how. `dev` is for contributors: the code, the + vanilla classes, the pull requests, what is left. Both stay public: no machines, accounts or private notes. +- Help wanted items are good first contributions: a clear vanilla behaviour, an issue with steps, a place to + start in `dev`. + +## Keeping it current + +Whoever moves the work (the orchestrator on each heartbeat, or anyone in their pull request): + +1. `python3 tools/roadmap.py sync`: every item whose linked pull requests all went out in a release, whose + linked issues are all closed and that isn't `ongoing` becomes `shipped`, with that release in + `shipped_in`, on top of the shipped items. It rewrites `roadmap.toml` and ROADMAP.md; open a pull request + with the change (`skip-release`). +2. `python3 tools/roadmap.py check`: the file's rules, then GitHub's state of every link. It fails when a + link is not an issue or pull request, when a Now item has no open pull request and none merged in the last + 14 days (link the new work, or move it to Next or Later), and when every link of a Help wanted item is + closed. It also says which items `sync` will ship once a release has them. +3. When work starts, opens a pull request or is decided: add or move the item, add its links, then + `python3 tools/roadmap.py build`. Keep the overview's focus to the 3 to 5 biggest Now items. +4. `python3 tools/roadmap.py labels` (after the change merges): creates a `roadmap:` label for each new + item, updates their colour (by status) and description, and adds the label to the open pull requests an item + links. Label a new pull request for an item by hand or by linking it and running `labels` again. Labels of + removed items are listed, never deleted. +5. Once a week, write the week's summary in `roadmap/journal/-.md` (for example + `2026-43.md`): `# Title`, a paragraph or two for players, then `## Technical` and the same week for + contributors, with its pull requests. `build` puts it into website/data/roadmap.json; the hub's Journal tab + shows it among the releases, and `/data/journal.json` carries it to Kira's #updates and to X. +6. Suggestions from the website and Discord reach the maintainers' private channel through Kira (below). A good + one becomes an item in a normal pull request; none is published automatically. + +CI runs `tests/test_roadmap.py` (in the Python tests), which fails when ROADMAP.md is not what `build` writes +or when the file breaks a rule; `python3 tools/roadmap.py check --offline` runs the same rules without GitHub. + +## The journal + +The hub's Journal tab and `https://playhorizon.dev/data/journal.json` are built with each deploy of the website +(`website/src/hub.js`). The tab shows every game and launcher release with its notes' changes, each placed under +the roadmap item it moves (the pull request's `roadmap:` label, else the item that links it), the merged pull +requests no release has yet, and the weekly summaries above. + +`journal.json` is what Kira reads for `#updates` and its X posts (kira's `src/journal.js`): one flat entry per +merged pull request, release, shipped item and weekly summary, newest first, the latest 300. + +```json +{ "format": "horizon-journal", "version": 2, "generated": "…", "url": "https://playhorizon.dev/hub#journal", + "entries": [ + { "id": "pr-1788", "kind": "merge", "date": "…", "title": "the pull request's title", "url": "…", + "number": 1788, "item": "discord-presence", "item_title": "Discord Rich Presence" }, + { "id": "release-v0.2.57", "kind": "release", "date": "…", "title": "…", "plain": "one line", "url": "…", + "tag": "v0.2.57" }, + { "id": "shipped-server-resource-packs", "kind": "shipped", "date": "the release's", "title": "…", + "plain": "…", "url": "https://playhorizon.dev/hub#server-resource-packs", "item": "…", "item_title": "…", + "shipped_in": "v0.2.48", "post": "at most 240 characters", "announce": false }, + { "id": "weekly-2026-43", "kind": "weekly", "date": "the week's last second", "title": "…", "plain": "…", + "dev": "…", "url": "https://playhorizon.dev/hub#week-2026-43" } ] } +``` + +- An `id` never changes: readers post each one once. `merge` entries are the pull requests players see (no + `skip-release` label), plus any that moves a roadmap item; `item` is absent when none does. +- `announce: true` (the item's `announce` key) asks Kira to post the shipped item on X too, with `post`. +- `https://playhorizon.dev/data/roadmap.json` is `roadmap.toml` as JSON (`areas`, `items` with every key above, + `overview`, the weekly summaries); an item's page is `https://playhorizon.dev/hub#`. + +## The intake: votes, bug reports, suggestions + +The hub's forms and vote buttons talk to the telemetry server (`https://telemetry.playhorizon.dev`, +`crates/horizon-telemetry-server`, `src/hub.rs` and `src/intake.rs`; its README lists the limits and settings). +Kira uses the same endpoints, so a bug or an idea goes the same way from the site and from Discord. + +| Endpoint | Body | Answer | +| --- | --- | --- | +| `POST /hub/vote` | `{"item": "", "voter": "", "up": true}` | `{"item", "votes"}`; `"up": false` takes the vote back | +| `GET /hub/votes` | | `{"votes": {"": n}}`, a minute old at most | +| `POST /hub/report` | `{"title", "did", "happened", "expected", "version"?, "system", "name"?}` | 201 `{"id", "issue": ""}`; 202 `{"id", "issue": null}` when GitHub was down (opened later) | +| `POST /hub/suggest` | `{"text", "item"?, "name"?}` | 202 `{"id"}` | +| `GET /hub/suggestions?after=` | `Authorization: Bearer ` | `{"suggestions": [{"id", "text", "item"?, "name"?, "created_at"}]}`, oldest first | + +- A report becomes an issue at once, labels `bug` and `from-website`, in the `#horizon-bugs` template (What I did, + What happened, What I expected, Horizon version, OS, CPU, GPU) with "Reported on playhorizon.dev". Every field + but the version and the name is required; mentions, `#123` references and HTML in people's text are made inert. +- Suggestions are never published. Kira polls `/hub/suggestions` with the last id it relayed and posts each one + to `#roadmap-suggestions`. +- Errors are `{"error": "…"}`: 400 names the field (`"did: 3 to 2000 characters"`), 429 means too many from that + address, 503 a full day or reports turned off (no GitHub token on the server). +- The page shows the forms and the buttons only once `GET /hub/votes` answers. Until then it points to GitHub and + Discord. It keeps a random voter id and the items it voted for in `localStorage` (`hub:voter`, `hub:voted`). + diff --git a/roadmap/journal/2026-43.md b/roadmap/journal/2026-43.md new file mode 100644 index 000000000..388f5d544 --- /dev/null +++ b/roadmap/journal/2026-43.md @@ -0,0 +1,17 @@ +# Single player becomes a server, servers send their packs + +Single player now runs on its own server thread, as vanilla does, and a run of fixes made shields, tridents, +elytra, maps, fishing, signs and books work there and for every server player. Servers can send you their +resource packs, the launcher got its blocky new look, and the first pieces of Horizon's mod registry landed. +Shader packs were taken out, to come back written from scratch. + +## Technical + +The server thread became the default (#1514), after guests and server-thread players reached parity: totems +(#1463), death drops (#1464), signs and books (#1465, #1466), shields, tridents, crossbows and spyglasses +(#1476), elytra (#1475), maps (#1510), fishing (#1509), wind charges (#1508), strikes and mob effects (#1512), +the cursor stack on close (#1513). Server resource packs: the dedicated server offers them (#1562) and the +client downloads, asks and applies them (#1569, #1570). Mod registry: index schema 1 and `.hpack` (#1634), +`cargo horizon-mod check` (#1642), the submission service (#1659). Shader packs were removed (#1561) and the +clean-room rewrite started from the format's documentation (#1584 onward). Releases now ship in batches of five +pull requests, or six hours after the oldest (#1554). diff --git a/roadmap/roadmap.toml b/roadmap/roadmap.toml new file mode 100644 index 000000000..5ad4d8846 --- /dev/null +++ b/roadmap/roadmap.toml @@ -0,0 +1,482 @@ +# Horizon's roadmap: the one source of truth for ROADMAP.md, playhorizon.dev/roadmap and Kira's answers. +# Edit this file, then run `python3 tools/roadmap.py build`. How to keep it current: roadmap/README.md. +# +# No dates, ever: an item is Now, Next or Later. Each item says the same thing twice: `plain` for players +# (one or two sentences, at most 240 characters, so it fits a post when it ships) and `dev` for contributors +# (crates, approach, what is left). Links are "#123" (an issue or a pull request), a path in the repository, +# or an https URL. Only what is decided goes here; ideas wait in GitHub issues and the suggestion queue. +# `focus = true` puts a Now item in the overview (3 to 5 of them). `ongoing = true` marks a workstream that goes on +# after its linked pull requests merge: `sync` never ships it, so link its new pull requests as they open. + +[overview] +today_plain = """Horizon is Minecraft: Java Edition 26.3 rewritten from scratch in Rust, so it runs faster on the \ +same computer. Single player, LAN, servers, resource packs and Horizon's own mods work today, on your own copy of \ +the game.""" +today_dev = """A native Rust and wgpu reimplementation of Java 26.3: the same saves, rules and terrain, checked \ +against the real game by more than 300 differential harnesses (a Java oracle and a Rust probe on the same inputs). \ +Single player runs on its own server thread, like vanilla's integrated server.""" +help_plain = """Play it and tell us what differs from vanilla (Discord or GitHub), run the benchmark on your \ +computer, vote for what you want next, or pick one of the Help wanted items.""" +help_dev = """Read CONTRIBUTING.md, pick a Help wanted item or a To do card, check vanilla's code for the \ +behaviour, and paste the block `python3 tools/pr_ready.py` prints in your pull request.""" + +# Areas, in the project's order of priority: performance first, then modpacks and servers. +[[area]] +id = "performance" +name = "Performance" + +[[area]] +id = "modpacks" +name = "Modpacks" + +[[area]] +id = "servers" +name = "Servers and multiplayer" + +[[area]] +id = "mods" +name = "Mods and API" + +[[area]] +id = "gameplay" +name = "Gameplay parity" + +[[area]] +id = "rendering" +name = "Rendering" + +[[area]] +id = "launcher" +name = "Launcher" + +[[area]] +id = "extras" +name = "Horizon extras" + +[[area]] +id = "project" +name = "Code and tools" + +# ---------------------------------------------------------------------------------------------------- Now + +[[item]] +id = "smooth-frames" +title = "Steady frames at long render distances" +status = "now" +area = "performance" +focus = true +plain = """Making Horizon faster than vanilla on every computer, with smooth camera movement even at 32 chunks, \ +by spreading the work better over the processor's cores.""" +dev = """Frame pacing at render distance 32 (#678): long frames come from work competing with the render thread. \ +In review: world generation workers build free structure starts instead of queueing them (#1760), the worker pool \ +leaves one core as vanilla does (#1778), and the terrain vertex arena is read as storage where flat draws run \ +(#1742). Still open: far terrain at 48 to 64 chunks (#718) and CPU-bound fields of block entities (#717). Every \ +change shows before and after numbers from `tools/perf_bench.py` against vanilla 26.3 (docs/benchmark.md).""" +links = ["#678", "#1760", "#1778", "#1742", "#718", "#717", "docs/benchmark.md"] + +[[item]] +id = "block-storage" +title = "Faster block lookups" +status = "now" +area = "performance" +plain = """Speeding up the code that reads and writes the blocks of the world, which almost everything in the game \ +goes through: generation, lighting, meshing and ticking.""" +dev = """`PalettedContainer` and `PackedStorage` find an entry's word by multiply and shift instead of an integer \ +division (#1790, #1794); meshing looks up baked models by index, not by name (#1747). Each change keeps the paletted \ +storage harnesses identical and shows its microbenchmark (docs/performance/paletted-container-access-results.md).""" +links = ["#1790", "#1794", "#1747", "docs/performance/paletted-container-access-results.md"] + +[[item]] +id = "modpack-worldgen" +title = "Modpacks' world generation, natively" +status = "now" +area = "modpacks" +announce = true +focus = true +plain = """Making the world-generation mods of popular modpacks (new structures, YUNG's dungeons and mineshafts, \ +packs built on Lithostitched) generate their worlds in Horizon, natively and fast.""" +dev = """`horizon-port-pack` carries a pack's data-driven content over; structure mods that need code get a native \ +port: YUNG's worldgen as written (#1746), Lithostitched's limited and guaranteed elements with pool and processor \ +modifiers (#1777), its processors as the game's own types (#1765). Proved by the structure blocks and layout \ +harnesses (#1753, #1757). Mods whose Java code does more than world generation don't run (docs/modpacks.md).""" +links = ["#1746", "#1777", "#1765", "#1753", "#1757", "docs/modpacks.md"] + +[[item]] +id = "server-clients" +title = "Playing on servers: everything shows as it should" +status = "now" +ongoing = true +area = "servers" +focus = true +plain = """When you play on a server, other players, mobs, items and menus look and behave as in vanilla: skins, \ +poses, item names, inventories and server networks.""" +dev = """Client-of-a-server fixes in `game/remote*.rs` and `horizon-protocol`: mobs' aggressive and item-use flags \ +(#1802, #1798), ghost items after a remote menu (#1795), skins (#1652), cookies and transfer packets for proxy \ +networks (#1650), firework boosts (#1671), living-entity attributes and simulation distance (#1731), styled item \ +names (#1702), primed TNT between updates (#1710).""" +links = ["#1802", "#1798", "#1795", "#1652", "#1650", "#1671", "#1731", "#1702", "#1710"] + +[[item]] +id = "server-thread" +title = "Single player as a server: the last differences" +status = "now" +ongoing = true +area = "gameplay" +focus = true +plain = """Single player now runs a small server in the background, as vanilla does. We are fixing what still \ +behaves differently there; every fix also works for LAN and server players.""" +dev = """Since #1514 single player runs on the server thread (`game/integrated.rs`) and the window joins it as a \ +client; `--in-frame` keeps the old path. The open `[server-thread]` issues list the gaps. In review: a tool that \ +changes a block ends the click (#1797), potions keep their potion over the connection (#1796), mobs' aggression \ +and arm swings (#1802). docs/server-levels.md.""" +links = ["#1514", "#1797", "#1796", "#1802", "docs/server-levels.md"] + +[[item]] +id = "mod-registry" +title = "Horizon's mod registry" +status = "now" +area = "mods" +announce = true +focus = true +plain = """One place for mods, modpacks and resource packs made for Horizon: browse them on the website and \ +install them from the launcher in one click.""" +dev = """Merged: the registry index (schema 1, `.hpack`, checks before install, #1634), `cargo horizon-mod check` \ +(#1642) and the submission service (`crates/horizon-mod-registry`, #1659). In review: the website's /mods (#1596) \ +and /mods/submit (#1635), and in the launcher the registry as a source (#1636), install, enable and takedowns \ +(#1637), the Mods page (#1638) and modpacks as their own profiles (#1640).""" +links = ["#1634", "#1642", "#1659", "#1596", "#1635", "#1636", "#1637", "#1638", "#1640", "docs/mods.md"] + +[[item]] +id = "server-dialogs" +title = "Server dialogs and login screens" +status = "now" +area = "servers" +plain = """Servers that show dialogs, such as the register and login screens of AuthMe, will work in Horizon.""" +dev = """26.3's dialog packets and model (#1720), the dialog screen with its buttons and actions (#1734), then text \ +and boolean inputs (#1735).""" +links = ["#1720", "#1734", "#1735"] + +[[item]] +id = "languages" +title = "Play in your language" +status = "now" +area = "gameplay" +plain = """Item names, container titles and server messages follow the language you pick, as in vanilla. Today \ +parts of the game stay in English.""" +dev = """Translatable components resolved through the selected language for items, containers and server text \ +(#1806), on top of the bundled languages the prepared resources carry (#1594). Left: the client's own screens \ +(#1672) and the main menu with zh_cn (#1774).""" +links = ["#1806", "#1594", "#1672", "#1774"] + +[[item]] +id = "shader-packs" +title = "Shader packs, written again from scratch" +status = "now" +ongoing = true +area = "rendering" +plain = """Shader packs were removed and are being rebuilt from scratch. They come back, off by default, once the \ +best-known packs draw properly.""" +dev = """A clean-room rewrite in `horizon-shaderpack`, from the packs' public format documentation only: reading \ +packs, the preprocessor and `.properties` files are merged (#1584, #1585, #1586); translating programs to WGSL \ +through naga, buffers, uniforms and the terrain, entity, sky, weather and hand stages are in review (#1698 to \ +#1801).""" +links = ["#1561", "#1584", "#1586", "#1698", "#1771", "#1786", "#1801"] + +[[item]] +id = "discord-presence" +title = "Discord Rich Presence" +status = "now" +area = "extras" +announce = true +plain = """Your Discord profile can show that you are playing Horizon, with a switch to turn it off.""" +dev = """A Discord IPC client on its own thread (#1787) and the presence with its setting (#1788). Merged: it ships \ +with the next release.""" +links = ["#1787", "#1788"] + +[[item]] +id = "lean-code" +title = "A leaner codebase that builds faster" +status = "now" +ongoing = true +area = "project" +plain = """Cleaning up Horizon's code: removing what nothing uses and making it quicker to build, so fixes and new \ +features land faster.""" +dev = """A review of the whole tree: dead code and stubs removed (#1432, #1658), one probe binary per crate (#1453, \ +#1454), shared harness helpers (#1448), clippy clean on the whole tree (#1707). docs/architecture.md maps each \ +feature to its files and harnesses.""" +links = ["#1432", "#1658", "#1453", "#1454", "#1448", "#1707", "docs/architecture.md"] + +# ---------------------------------------------------------------------------------------------------- Next + +[[item]] +id = "microsoft-sign-in" +title = "Microsoft sign-in (coming soon)" +status = "next" +area = "launcher" +announce = true +plain = """Coming soon: sign in with your Microsoft account to play on online-mode servers. The code is ready; it \ +waits for Microsoft's approval.""" +dev = """Device-code sign-in in the launcher and online-mode login in the client (docs/microsoft-account.md). It is \ +turned on by setting the app's client id once Minecraft API access is approved; until then the launcher shows \ +"Coming soon".""" +links = ["docs/microsoft-account.md", "#1270"] + +[[item]] +id = "curseforge" +title = "CurseForge in the mod browser (coming soon)" +status = "next" +area = "modpacks" +announce = true +plain = """Coming soon: search and download CurseForge mods and modpacks from the launcher, as you can with \ +Modrinth today.""" +dev = """The mod browser lists CurseForge as coming soon (docs/mod-browser.md): it needs CurseForge API access. \ +Meanwhile packs import from a CurseForge zip or instance folder, and jars only CurseForge hosts come from the \ +player's own instance (docs/modpacks.md).""" +links = ["docs/mod-browser.md", "docs/modpacks.md"] + +[[item]] +id = "benchmarks" +title = "Fresh benchmarks against vanilla" +status = "next" +area = "performance" +plain = """New public numbers comparing Horizon with vanilla on many kinds of computers, measured the same way \ +every time, on the website's Benchmarks page.""" +dev = """Redo the comparison after the current performance work (#1073); reliability, coverage and presentation \ +(#1346); a scene must count as loaded only once its first chunk arrives (#1749). Players' reports come in through \ +`--benchmark` and `horizon-launcher data benchmark` (docs/benchmark.md).""" +links = ["#1073", "#1346", "#1749", "docs/benchmark.md"] + +[[item]] +id = "pistons-push" +title = "Pistons push players and mobs" +status = "next" +area = "gameplay" +plain = """Pistons and slime blocks will push players, mobs and items as in vanilla. Today moving blocks pass \ +through them.""" +dev = """Port `PistonMovingBlockEntity.moveCollidedEntities` and `moveStuckEntities` (honey), the slime bounce and \ +`Entity.limitPistonMovement` for `MoverType.PISTON`, in single player and for a server's players. Issues #1496, \ +#1609, #1608, #1704, #1758.""" +links = ["#1496", "#1609", "#1608", "#1704", "#1758"] + +[[item]] +id = "intel-mac" +title = "Horizon on Intel Macs" +status = "next" +area = "launcher" +plain = """Builds of the game and the launcher for Macs with an Intel processor.""" +dev = """x86_64 game and launcher builds, picked per processor (#1171). Before it ships it needs a run on a real \ +Intel Mac: Rosetta on Apple Silicon only proves the CPU code. Requests: #1161, #1792.""" +links = ["#1171", "#1161", "#1792"] + +# ---------------------------------------------------------------------------------------------------- Later + +[[item]] +id = "ai-companion" +title = "An optional AI teammate" +status = "later" +area = "extras" +plain = """An AI teammate that plays beside you and that you can talk to. It is off unless you turn it on, and the \ +No AI switch hides it completely.""" +dev = """Draft pull requests: the companion (#1068), its safety modes and their evaluation (#1069), and a route \ +that finishes the game (#1070). Local models first; it ships only once the safety evaluation passes.""" +links = ["#1068", "#1069", "#1070"] + +[[item]] +id = "new-versions" +title = "Following new Minecraft versions" +status = "later" +area = "gameplay" +plain = """When Mojang releases a new version of the game, Horizon follows it, and every system is checked against \ +the new version again.""" +dev = """The port pipeline detects a release, diffs its classes against 26.3 and writes a port report \ +(docs/version-ports.md); the harnesses then run against the new version.""" +links = ["docs/version-ports.md"] + +# ---------------------------------------------------------------------------------------------------- Help wanted + +[[item]] +id = "benchmark-your-pc" +title = "Benchmark your computer" +status = "help-wanted" +area = "performance" +plain = """Run Horizon's built-in benchmark and share the report: every new computer makes the performance work \ +more accurate. No coding needed.""" +dev = """`horizon-launcher play --arg=--benchmark --arg=standard` writes a JSON report; `horizon-launcher data \ +benchmark ` shows what would be sent and uploads it (website/docs/performance.md). Results from three \ +installs with the same hardware appear on the Benchmarks page. Example: #1711.""" +links = ["website/docs/performance.md", "#1711"] + +[[item]] +id = "end-city-banners" +title = "End city banners keep their pattern" +status = "help-wanted" +area = "gameplay" +plain = """Banners in End cities show as plain magenta instead of with their pattern.""" +dev = """Compare the banner block entities the End city templates place with vanilla's (`EndCityPieces`, the \ +templates' block entity NBT, `BannerBlockEntity`'s patterns), in `crates/horizon-world/src/worldgen/structure/`. \ +Issue #1438; possibly related: #526.""" +links = ["#1438", "#526"] + +[[item]] +id = "end-city-loot" +title = "Enchanted loot in End cities" +status = "help-wanted" +area = "gameplay" +plain = """Armour and tools in End city chests should come enchanted, as in vanilla.""" +dev = """`loot.rs` implements `enchant_with_levels`: find why the End city chest table's rolls come out bare, and \ +compare them with vanilla's for a few seeds. Issues #1437, #1660.""" +links = ["#1437", "#1660"] + +[[item]] +id = "item-cooldowns" +title = "Cooldowns drawn on items" +status = "help-wanted" +area = "gameplay" +plain = """After you throw an ender pearl, its slot should show the cooldown sweeping away, as in vanilla.""" +dev = """Vanilla draws `ItemCooldowns`' percent as a white overlay over the slot \ +(`GuiGraphicsExtractor.itemDecorations` and `itemCooldown`) in the hotbar and in inventories. Issue #1763.""" +links = ["#1763"] + +[[item]] +id = "chat-settings" +title = "Chat settings and team prefixes" +status = "help-wanted" +area = "gameplay" +plain = """The Hidden and Commands Only chat settings should hide chat, and team prefixes should appear before \ +names in chat.""" +dev = """Vanilla checks `ChatVisiblity` before it delivers chat and system messages (#1376), and chat uses the \ +team-formatted display name (#1378). Both issues have exact steps to reproduce.""" +links = ["#1376", "#1378"] + +[[item]] +id = "pointer-hand" +title = "A pointing hand over buttons" +status = "help-wanted" +area = "gameplay" +plain = """The mouse pointer should turn into a pointing hand over menu buttons, as in vanilla 26.3.""" +dev = """26.3's `AbstractWidget` asks for a cursor from `CursorTypes` when it is hovered; winit's \ +`CursorIcon::Pointer` gives the same shape. Issue #1347.""" +links = ["#1347"] + +[[item]] +id = "text-field-keys" +title = "Ctrl+Backspace and scroll bars in menus" +status = "help-wanted" +area = "gameplay" +plain = """Ctrl+Backspace should delete a whole word in the creative search, and every list's scroll bar should be \ +draggable.""" +dev = """`EditBox` deletes by word with Ctrl (#1764); the Game Rules and Statistics lists' scroll bars can't be \ +dragged (#1782, after #978).""" +links = ["#1764", "#1782"] + +[[item]] +id = "chicken-flap" +title = "Chickens flap while they fall" +status = "help-wanted" +area = "gameplay" +plain = """Chickens should flap their wings while they fall, not stand still in the air.""" +dev = """Vanilla's `Chicken.aiStep` updates `flap` and `flapSpeed` when the chicken isn't on the ground, and the \ +model's wings follow them. Issue #1706.""" +links = ["#1706"] + +# ---------------------------------------------------------------------------------------------------- Shipped +# Newest first: `tools/roadmap.py sync` adds each newly shipped item on top. + +[[item]] +id = "server-resource-packs" +title = "Server resource packs" +status = "shipped" +shipped_in = "v0.2.48" +area = "servers" +plain = """Servers can send you their resource pack: Horizon asks, downloads it and applies it, like vanilla.""" +dev = """The dedicated server offers server.properties' pack (#1562); the client's pack manager, downloads, prompt \ +and Server Resource Packs setting (#1569, #1570).""" +links = ["#1562", "#1569", "#1570"] + +[[item]] +id = "server-thread-default" +title = "Single player on its own server thread" +status = "shipped" +shipped_in = "v0.2.39" +area = "gameplay" +plain = """Single player runs on a server thread, as vanilla does, so the world keeps ticking smoothly while the \ +window draws.""" +dev = """`game/integrated.rs` runs a second `Game` with one level per dimension; the window joins it through \ +memory like a LAN guest (#1514).""" +links = ["#1514", "docs/server-levels.md"] + +[[item]] +id = "metal-occlusion" +title = "GPU occlusion culling on Macs" +status = "shipped" +shipped_in = "v0.2.17" +area = "performance" +plain = """On Macs the GPU now skips terrain hidden behind hills and caves: 8 to 48% more frames per second at 32 \ +chunks on an M4 Pro.""" +dev = """GPU occlusion culling on Metal (#1434).""" +links = ["#1434"] + +[[item]] +id = "gpu-animated-textures" +title = "Animated textures on the GPU" +status = "shipped" +shipped_in = "v0.2.5" +area = "performance" +plain = """Water, lava and other animated blocks are now animated by the graphics card: 1.7 to 2.4 times better 1% \ +lows on Intel graphics.""" +dev = """Frames uploaded once, no per-tick CPU blend or copy (#1239).""" +links = ["#1239"] + +[[item]] +id = "far-detail" +title = "Far Detail for integrated graphics" +status = "shipped" +shipped_in = "v0.2.5" +area = "performance" +plain = """An optional Reduced setting for far terrain, suggested on integrated graphics, that saves 2 to 4 ms of GPU \ +time per frame.""" +dev = """Opt-in reduced far terrain (#1199), on far-terrain chunk summaries built on the workers (#1029).""" +links = ["#1199", "#1029"] + +[[item]] +id = "mod-browser" +title = "The launcher's mod browser" +status = "shipped" +shipped_in = "launcher@0.2.0" +area = "modpacks" +plain = """Search Modrinth for mods, modpacks and resource packs from the launcher; it says before installing what \ +works in Horizon.""" +dev = """The Mods page (#1154), Java mods checked before install (#1242), project pages, filters and galleries \ +(#1243), modpacks ported in the same step (#1246). docs/mod-browser.md.""" +links = ["#1154", "#1242", "#1243", "#1246", "docs/mod-browser.md"] + +[[item]] +id = "launcher-look" +title = "A blocky, Minecraft-like launcher" +status = "shipped" +shipped_in = "launcher@0.2.0" +area = "launcher" +plain = """The launcher got a new blocky look in Horizon's purple, in dark and light.""" +dev = """The Blocks look (#1269), from `horizon-theme` (docs/design-system.md).""" +links = ["#1269", "docs/design-system.md"] + +[[item]] +id = "dynamic-lights" +title = "Dynamic lights" +status = "shipped" +shipped_in = "v2026.10.22" +area = "extras" +plain = """An optional extra: torches you hold, drop or set on fire light up what is around them.""" +dev = """Held, dropped and burning light sources (#1162), in Horizon Extras, off by default.""" +links = ["#1162"] + +[[item]] +id = "servers-page" +title = "Host a server from the launcher" +status = "shipped" +shipped_in = "launcher-v0.15.0" +area = "servers" +plain = """The launcher's Servers page creates, starts, backs up and updates a Horizon server with buttons, and \ +invites friends.""" +dev = """The launcher's Servers page (#1026) over `horizon-server` (docs/hosting.md).""" +links = ["#1026", "docs/hosting.md"] diff --git a/tests/test_roadmap.py b/tests/test_roadmap.py new file mode 100644 index 000000000..6c7b14e31 --- /dev/null +++ b/tests/test_roadmap.py @@ -0,0 +1,239 @@ +import copy +import datetime +import json +import tomllib +import unittest +from unittest import mock + +from tools import roadmap + +SAMPLE = '''[overview] +today_plain = "a" +today_dev = "b" +help_plain = "c" +help_dev = "d" + +[[area]] +id = "performance" +name = "Performance" + +# ---- Now + +[[item]] +id = "fast" +title = "Fast" +status = "now" +area = "performance" +focus = true +plain = "Faster." +dev = """Multiply, \\ +not divide (#10).""" +links = ["#10"] + +[[item]] +id = "faster" +title = "Faster" +status = "now" +area = "performance" +focus = true +plain = "Even faster." +dev = "More (#11)." +links = ["#11", "#12"] + +[[item]] +id = "fastest" +title = "Fastest" +status = "now" +area = "performance" +focus = true +ongoing = true +plain = "The fastest." +dev = "Most (#13)." +links = ["#13"] + +# ---- Shipped + +[[item]] +id = "old" +title = "Old" +status = "shipped" +shipped_in = "v0.2.1" +area = "performance" +plain = "Done." +dev = "Done (#1)." +links = ["#1"] +''' + + +def pull(state, merged_at=None, paths=("crates/horizon-client/src/game.rs",)): + return {"kind": "PullRequest", "state": state, "merged_at": merged_at, "merge_commit": "abc" if merged_at else None, + "paths": list(paths)} + + +def issue(state): + return {"kind": "Issue", "state": state, "merged_at": None, "merge_commit": None, "paths": []} + + +class RepositoryRoadmap(unittest.TestCase): + def test_the_roadmap_keeps_its_rules(self): + self.assertEqual(roadmap.validate(roadmap.load()), []) + + def test_generated_files_match_the_toml(self): + for path, text in roadmap.outputs(roadmap.load()).items(): + self.assertEqual(path.read_text(), text, f"{path.name} is out of date: run `python3 tools/roadmap.py build`") + + def test_site_data_has_every_item_in_one_line_text(self): + data = json.loads(roadmap.site_data(tomllib.loads(SAMPLE))) + self.assertEqual([item["id"] for item in data["items"]], ["fast", "faster", "fastest", "old"]) + self.assertEqual(data["items"][0]["dev"], "Multiply, not divide (#10).") + self.assertEqual(data["areas"], [{"id": "performance", "name": "Performance"}]) + + +class Rules(unittest.TestCase): + def setUp(self): + self.data = tomllib.loads(SAMPLE) + + def problems(self, **changes): + data = copy.deepcopy(self.data) + data["item"][0].update(changes) + return roadmap.validate(data) + + def test_the_sample_is_valid(self): + self.assertEqual(roadmap.validate(self.data), []) + + def test_unknown_status_area_and_key(self): + self.assertTrue(any("status" in p for p in self.problems(status="soon"))) + self.assertTrue(any("area" in p for p in self.problems(area="nowhere"))) + self.assertTrue(any("unknown key" in p for p in self.problems(eta="soon"))) + + def test_plain_fits_a_post(self): + self.assertTrue(any("at most 240" in p for p in self.problems(plain="x" * 241))) + self.assertTrue(any("post" in p for p in self.problems(post="x" * 241))) + + def test_shipped_needs_its_release(self): + self.assertTrue(any("shipped_in" in p for p in self.problems(status="shipped", focus=False))) + self.assertTrue(any("only shipped" in p for p in self.problems(shipped_in="v0.2.3"))) + self.assertTrue(any("not a release" in p for p in self.problems(status="shipped", focus=False, + shipped_in="soon"))) + + def test_links_are_issues_urls_or_repository_paths(self): + self.assertEqual(self.problems(links=["#4", "https://playhorizon.dev", "docs/benchmark.md"]), []) + self.assertTrue(any("link" in p for p in self.problems(links=["docs/nothing-here.md"]))) + self.assertTrue(any("link" in p for p in self.problems(links=["http://example.com"]))) + + def test_ids_are_free_anchors_of_the_hub(self): + for taken in ("roadmap", "now", "h-next", "tab-board"): + self.assertTrue(any("taken by the hub page" in p for p in self.problems(id=taken)), taken) + + def test_flags_are_true_or_false(self): + self.assertEqual(self.problems(announce=True), []) + self.assertTrue(any("announce is true or false" in p for p in self.problems(announce="yes"))) + + def test_three_to_five_items_in_focus(self): + self.assertTrue(any("focus" in p for p in self.problems(focus=False))) + + def test_no_dates(self): + for text in ("Ready in March.", "Before 2027.", "In Q3.", "Next week.", "In 3 weeks.", "By the end of it."): + self.assertTrue(roadmap.dated(text), text) + for text in ("Coming soon.", "Pull request #2026.", "Java 26.3.", "Mobs march on.", "It may land."): + self.assertIsNone(roadmap.dated(text), text) + self.assertTrue(any("date" in p for p in self.problems(plain="Done in October."))) + + +class Render(unittest.TestCase): + def test_sections_links_and_the_overview(self): + text = roadmap.render(tomllib.loads(SAMPLE)) + self.assertIn("## Now", text) + self.assertIn("## Recently shipped", text) + self.assertNotIn("## Later", text) + self.assertIn("- **Fast**: Faster.", text) + self.assertIn("Multiply, not divide ([#10](https://github.com/ludovic111/horizon/issues/10)).", text) + # A link the technical text already gives isn't repeated; the others are listed after it. + self.assertIn("More: [#12](https://github.com/ludovic111/horizon/issues/12)", text) + self.assertIn("- **Old** (v0.2.1): Done.", text) + + +class GitHubState(unittest.TestCase): + now = datetime.datetime(2030, 1, 20, tzinfo=datetime.timezone.utc) + + def test_a_now_item_without_live_pull_requests_is_stale(self): + data = tomllib.loads(SAMPLE) + states = {1: pull("MERGED", "2029-12-01T00:00:00Z"), 10: pull("OPEN"), + 11: pull("MERGED", "2029-11-01T00:00:00Z"), 12: issue("OPEN"), + 13: pull("MERGED", "2030-01-19T00:00:00Z")} + problems, notes = roadmap.online_problems(data, states, self.now) + self.assertEqual(len(problems), 1) + self.assertIn("faster: stale", problems[0]) + self.assertTrue(any("fastest: ongoing" in note for note in notes)) + + def test_missing_numbers_are_reported(self): + problems, _ = roadmap.online_problems(tomllib.loads(SAMPLE), {}, self.now) + self.assertTrue(any("#10 is neither" in p for p in problems)) + + +class Journal(unittest.TestCase): + def test_weekly_summaries_newest_first_with_their_monday(self): + import tempfile + from pathlib import Path + with tempfile.TemporaryDirectory() as folder: + folder = Path(folder) + (folder / "2026-43.md").write_text("# A week\n\nPlain\nwords.\n\n## Technical\n\nDev (#1).\n") + (folder / "2026-42.md").write_text("# Older\n\nPlain.\n\n## Technical\n\nDev.\n") + (folder / "week-44.md").write_text("# Bad name\n") + (folder / "2026-45.md").write_text("# No technical part\n\nPlain.\n") + entries, problems = roadmap.weeks(folder) + self.assertEqual([e["week"] for e in entries], ["2026-43", "2026-42"]) + self.assertEqual(entries[0], {"week": "2026-43", "date": "2026-10-19", "title": "A week", + "plain": "Plain words.", "dev": "Dev (#1)."}) + self.assertEqual(len(problems), 2) + + +class Labels(unittest.TestCase): + def test_creates_updates_and_adds_labels(self): + data = tomllib.loads(SAMPLE) + existing = {"roadmap:fast": {"color": "5B3FD6", "description": "Roadmap, now: Fast"}, + "roadmap:faster": {"color": "000000", "description": "old"}, + "roadmap:gone": {"color": "000000", "description": ""}} + states = {10: dict(pull("OPEN"), labels=["roadmap:fast"]), 11: dict(pull("OPEN"), labels=[]), + 13: dict(pull("MERGED", "2030-01-01T00:00:00Z"), labels=[])} + changes, stale = roadmap.label_changes(data, existing, states) + self.assertEqual([(action, target) for action, target, _ in changes], + [("update", "roadmap:faster"), ("add", 11), ("create", "roadmap:fastest"), + ("create", "roadmap:old")]) + self.assertEqual(changes[0][2]["color"], "5b3fd6") + self.assertEqual(stale, ["roadmap:gone"]) + + +class Sync(unittest.TestCase): + def test_family_of_a_change(self): + self.assertEqual(roadmap.family(["launcher/src/main.rs"]), "launcher") + self.assertEqual(roadmap.family(["website/src/main.js"]), "website") + self.assertEqual(roadmap.family(["launcher/src/main.rs", "crates/x/src/lib.rs"]), "game") + + def test_merged_and_released_items_move_to_the_top_of_shipped(self): + data = tomllib.loads(SAMPLE) + states = {10: pull("MERGED", "2030-01-02T00:00:00Z"), 11: pull("MERGED", "2030-01-03T00:00:00Z"), + 12: issue("OPEN"), 13: pull("MERGED", "2030-01-03T00:00:00Z")} + with mock.patch.object(roadmap, "first_release", return_value="v0.2.9") as release: + text, log = roadmap.sync(data, SAMPLE, states) + release.assert_called_once_with("abc", "game") + # "faster" keeps an open issue and "fastest" is ongoing: only "fast" ships. + self.assertEqual(log, ["fast: shipped in v0.2.9"]) + new = tomllib.loads(text) + self.assertEqual([item["id"] for item in new["item"]], ["faster", "fastest", "fast", "old"]) + fast = new["item"][2] + self.assertEqual((fast["status"], fast["shipped_in"]), ("shipped", "v0.2.9")) + self.assertNotIn("focus", fast) + self.assertEqual(fast["dev"], "Multiply, not divide (#10).") + self.assertIn("# ---- Shipped\n\n[[item]]\nid = \"fast\"", text) + + def test_merged_but_unreleased_items_wait(self): + states = {10: pull("MERGED", "2030-01-02T00:00:00Z")} + with mock.patch.object(roadmap, "first_release", return_value=None): + text, log = roadmap.sync(tomllib.loads(SAMPLE), SAMPLE, states) + self.assertEqual(text, SAMPLE) + self.assertEqual(log, ["fast: merged, waiting for a release"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/lan_e2e.py b/tools/lan_e2e.py index f294a49fb..f79e07033 100755 --- a/tools/lan_e2e.py +++ b/tools/lan_e2e.py @@ -6302,5 +6302,51 @@ CHECKS += ['boat-dismount-pose'] CHECK_FNS['boat-dismount-pose'] = check_boat_dismount_pose +def check_sprint_fov(host, guest, out): + """Releasing Ctrl must preserve a sprint until W is released, then reset FOV. + + PlayerInput reports keys; PlayerCommand starts/stops the sprint. Confusing + them clears the client's sprint flag while leaving its speed modifier on. + """ + seen = {} + for game in (host, guest): + game('command', command='gamemode creative') + game('command', command='effect clear @s') + game.raw('options-set', 'fovEffectScale', '1.0') + x, y, z = map(math.floor, pos_of(game)) + game('command', command=f'fill {x - 2} {y} {z - 2} {x + 2} {y + 3} {z + 20} minecraft:air') + game('command', command=f'fill {x - 2} {y - 1} {z - 2} {x + 2} {y - 1} {z + 20} minecraft:stone') + game('command', command=f'tp @s {x + 0.5} {y} {z + 0.5} 0 0') + game.wait(20) + try: + game.raw('key', 'w', 'down') + game.raw('key', 'ctrl', 'down') + game.wait(10) + started = game('state')['player'] + game.raw('key', 'ctrl', 'up') + game.wait(10) + released_ctrl = game('state')['player'] + game.raw('key', 'w', 'up') + game.wait(20) + stopped = game('state')['player'] + finally: + game.raw('key', 'ctrl', 'up') + game.raw('key', 'w', 'up') + seen[game.name] = states = { + label: {k: p[k] for k in ('sprinting', 'fov_modifier')} + for label, p in [('started', started), ('released_ctrl', released_ctrl), ('stopped', stopped)] + } + for label in ('started', 'released_ctrl'): + assert states[label]['sprinting'], f'{game.name} {label}: {states}' + assert abs(states[label]['fov_modifier'] - 1.15) < 0.002, f'{game.name} {label}: {states}' + assert not stopped['sprinting'], f'{game.name}: {states}' + assert abs(stopped['fov_modifier'] - 1.0) < 0.002, f'{game.name}: {states}' + return seen + + +CHECKS += ['sprint-fov'] +CHECK_FNS['sprint-fov'] = check_sprint_fov + + if __name__ == '__main__': main() diff --git a/tools/pr_ready.py b/tools/pr_ready.py index a944e2116..505de28c1 100755 --- a/tools/pr_ready.py +++ b/tools/pr_ready.py @@ -108,7 +108,7 @@ def touched_crates(paths, packages): def area(path): """The part of the project a path belongs to; None for what goes with any change (docs, tests, harnesses).""" - if is_doc(path) or path.startswith(("tests/", "mods/", "docs/")) or path == "research-lock.json" or \ + if is_doc(path) or path.startswith(("tests/", "mods/", "docs/", "roadmap/")) or path == "research-lock.json" or \ (path.startswith("tools/") and not path.startswith("tools/ci/")): return None if path.startswith(("launcher/", "crates/horizon-prepare/")): @@ -288,7 +288,9 @@ def plan(args, base, changed, crates_of, libraries, dependencies): """The checks for these changed files, in the order they run.""" rust = any(workspace_rust(path) for path in changed) launcher = any(launcher_rust(path) for path in changed) - python = any(path.endswith(".py") or path.startswith(("tests/", "tools/")) for path in changed) + # The Python tests also hold ROADMAP.md to roadmap/roadmap.toml (tests/test_roadmap.py). + python = any(path.endswith(".py") or path.startswith(("tests/", "tools/", "roadmap/")) or path == "ROADMAP.md" + for path in changed) crates = touched_crates(changed, crates_of) if rust or TOKENS in changed else [] # An unrecognised Rust path must not silently skip validation. if rust and crates == []: diff --git a/tools/roadmap.py b/tools/roadmap.py new file mode 100644 index 000000000..4bc439e4d --- /dev/null +++ b/tools/roadmap.py @@ -0,0 +1,612 @@ +#!/usr/bin/env python3 +"""Horizon's roadmap: roadmap/roadmap.toml is the source, ROADMAP.md is generated from it. + + python3 tools/roadmap.py build write ROADMAP.md and website/data/roadmap.json + python3 tools/roadmap.py build --check fail when either is out of date (CI) + python3 tools/roadmap.py check --offline the file's own rules (CI) + python3 tools/roadmap.py check the same, then GitHub: links resolve, no stale Now item + python3 tools/roadmap.py sync [--dry-run] items whose pull requests all merged become shipped, + with the release they went out in + python3 tools/roadmap.py labels [--dry-run] a `roadmap:` label per item on GitHub, and on the open + pull requests the item links + +The online commands read GitHub with GITHUB_TOKEN, GH_TOKEN or `gh auth token`. Maintenance: roadmap/README.md. +""" + +import argparse +import datetime +import json +import os +import re +import subprocess +import sys +import tomllib +import urllib.parse +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SOURCE = ROOT / "roadmap" / "roadmap.toml" +OUTPUT = ROOT / "ROADMAP.md" +# What the website's hub reads (website/build.mjs), and serves at /data/roadmap.json for Kira and others. +SITE_DATA = ROOT / "website" / "data" / "roadmap.json" +# The journal's weekly summaries, one file per ISO week: roadmap/journal/2026-43.md (roadmap/README.md). +JOURNAL = ROOT / "roadmap" / "journal" +WEEK = re.compile(r"(\d{4})-(\d{2})\.md$") +REPO = "ludovic111/horizon" +SITE = "https://playhorizon.dev/hub" + +STATUSES = { + "now": "Now", + "next": "Next", + "later": "Later", + "help-wanted": "Help wanted", + "shipped": "Recently shipped", +} +PLAIN_MAX = 240 # one post's worth, so a shipped item can be announced as written +TITLE_MAX = 70 +FOCUS_RANGE = (3, 5) +STALE_DAYS = 14 # a Now item needs an open pull request, or one merged this recently +LABEL_PREFIX = "roadmap:" +# GitHub label colours by status, from the design system's palette. +LABEL_COLOURS = {"now": "5b3fd6", "next": "9a86ff", "later": "c5c7d1", "help-wanted": "1d7a48", "shipped": "646875"} +ITEM_KEYS = {"id", "title", "status", "area", "plain", "dev", "links", "focus", "ongoing", "shipped_in", "post", + "announce"} + +ISSUE = re.compile(r"#(\d+)$") +ID = re.compile(r"[a-z0-9]+(-[a-z0-9]+)*$") +# An item is the anchor playhorizon.dev/hub#: it can't take the name of a tab, a section or a heading. +RESERVED = {"main", "overview", "roadmap", "board", "journal", "contribute", "report", "suggest", "contributors", + "hub-data", *STATUSES} +RESERVED_PREFIXES = ("h-", "tab-", "col-", "week-", "journal-") +RELEASE = re.compile(r"(v\d[\w.]*|launcher@\d[\w.]*|launcher-v\d[\w.]*|website)$") +MONTHS = ("January|February|March|April|June|July|August|September|October|November|December") +# Kira's rule: no dates and no promises of when. "Coming soon" is the one allowed phrase. +DATE = re.compile( + rf"\b(?:{MONTHS})\b|\bMay \d|(? TITLE_MAX: + problems.append(f"{where}: the title is longer than {TITLE_MAX} characters") + for key in ("plain", "post"): + if len(item.get(key, "")) > PLAIN_MAX: + problems.append(f"{where}: {key} is {len(item[key])} characters, at most {PLAIN_MAX}") + for key in ("title", "plain", "dev", "post"): + found = dated(item.get(key, "")) + if found: + problems.append(f"{where}: {key} names a date or a deadline ({found!r}); the roadmap has none") + shipped_in = item.get("shipped_in") + if status == "shipped": + if not shipped_in: + problems.append(f"{where}: a shipped item needs shipped_in (a release tag, or \"website\")") + elif not RELEASE.match(shipped_in): + problems.append(f"{where}: shipped_in {shipped_in!r} is not a release tag or \"website\"") + elif shipped_in: + problems.append(f"{where}: only shipped items have shipped_in") + for key in ("focus", "ongoing", "announce"): + if key in item and not isinstance(item[key], bool): + problems.append(f"{where}: {key} is true or false") + if item.get("ongoing") and status not in ("now", "next", "later"): + problems.append(f"{where}: only Now, Next and Later items are ongoing") + if item.get("focus") and status != "now": + problems.append(f"{where}: only Now items can be in focus") + links = item.get("links", []) + if not isinstance(links, list): + problems.append(f"{where}: links is a list") + links = [] + if not links and status != "later": + problems.append(f"{where}: needs at least one link (an issue, a pull request or a doc)") + for link in links: + if ISSUE.match(link) or link.startswith("https://"): + continue + if link.startswith(("/", "http:")) or ".." in link or not (root / link).exists(): + problems.append(f"{where}: link {link!r} is neither #N, https:// nor a path in the repository") + problems += weeks()[1] if root == ROOT else [] + focus = sum(1 for item in items if item.get("focus")) + low, high = FOCUS_RANGE + if not low <= focus <= high: + problems.append(f"focus: {focus} items are in focus, the overview wants {low} to {high}") + return problems + + +def areas_in_order(data): + return [(area["id"], area["name"]) for area in data.get("area", [])] + + +def by_area(data, status): + """The items of a status, grouped by area in the areas' order, each group in the file's order.""" + items = [item for item in data.get("item", []) if item.get("status") == status] + return [(name, [item for item in items if item["area"] == key]) + for key, name in areas_in_order(data) if any(item["area"] == key for item in items)] + + +def linkify(text, links=()): + """Markdown links for the text's #N, and for the item's own paths and URLs where the text names them.""" + text = re.sub(r"(?", + "", + "What is being done on Horizon, what comes next and where you can help. There are no dates: things are" + " Now, Next or Later. Each item is said in plain words, with the technical picture folded under it." + f" The same roadmap, with a Simple / Technical switch, the board, the journal and how to help, is on" + f" [playhorizon.dev/hub]({SITE}). Pull requests carry the `roadmap:` label of their item.", + "", + "## At a glance", + "", + overview["today_plain"].strip(), + "", + "**Being done now**", + "", + ] + out += [f"- **{item['title']}**: {item['plain'].strip()}" for item in items if item.get("focus")] + nexts = [item["title"] for item in items if item.get("status") == "next"] + if nexts: + out += ["", "**Next**: " + "; ".join(nexts) + "."] + out += ["", f"**How to help**: {overview['help_plain'].strip()}", "", + f"**Developers**: {overview['help_dev'].strip()}"] + for status, heading in STATUSES.items(): + groups = by_area(data, status) + if not groups: + continue + out += ["", f"## {heading}"] + if status == "shipped": + # Newest first: the file's order, where `sync` puts each newly shipped item on top. + out += [""] + [f"- **{item['title']}** ({item['shipped_in']}): {item['plain'].strip()} " + f"{link_list(item['links'])}" for item in items if item.get("status") == status] + continue + for area, group in groups: + out += ["", f"### {area}"] + for item in group: + out += ["", f"#### {item['title']}", "", item["plain"].strip(), "", + "
Technical", "", linkify(item["dev"].strip(), item.get("links", []))] + more = [link for link in item.get("links", []) if not re.search(rf"{re.escape(link)}\b", item["dev"])] + if more: + out += ["", "More: " + link_list(more)] + out += ["", "
"] + return "\n".join(out) + "\n" + + +def weeks(directory=JOURNAL): + """The weekly summaries, newest first: {"week", "date" (its Monday), "title", "plain", "dev"}, and problems.""" + entries, problems = [], [] + for path in sorted(directory.glob("*.md"), reverse=True) if directory.exists() else []: + name = WEEK.match(path.name) + text = path.read_text() + parts = re.match(r"# (.+)\n\n(.+?)\n\n## Technical\n\n(.+)", text.strip() + "\n", re.S) + if not name or not 1 <= int(name[2]) <= 53: + problems.append(f"journal/{path.name}: name it by its ISO week, like 2026-43.md") + continue + if not parts: + problems.append(f"journal/{path.name}: needs '# Title', the plain text, then '## Technical' and its text") + continue + monday = datetime.date.fromisocalendar(int(name[1]), int(name[2]), 1) + entries.append({"week": path.stem, "date": monday.isoformat(), "title": parts[1].strip(), + "plain": " ".join(parts[2].split()), "dev": " ".join(parts[3].split())}) + return entries, problems + + +def site_data(data): + """website/data/roadmap.json: the items as the hub and Kira read them, in the file's order.""" + keys = ("id", "title", "status", "area", "plain", "dev", "links", "focus", "ongoing", "shipped_in", "post", + "announce") + out = { + "format": "horizon-roadmap", + "version": 1, + "source": "roadmap/roadmap.toml", + "overview": {key: " ".join(value.split()) for key, value in data["overview"].items()}, + "statuses": [{"id": key, "name": name} for key, name in STATUSES.items()], + "areas": [{"id": key, "name": name} for key, name in areas_in_order(data)], + "items": [{key: (" ".join(item[key].split()) if isinstance(item[key], str) else item[key]) + for key in keys if key in item} for item in data.get("item", [])], + "journal": weeks()[0], + } + return json.dumps(out, indent=1, ensure_ascii=False) + "\n" + + +def outputs(data): + """Every generated file and its content.""" + return {OUTPUT: render(data), SITE_DATA: site_data(data)} + + +# --- GitHub --------------------------------------------------------------------------------------------------- + +def token(): + for name in ("GITHUB_TOKEN", "GH_TOKEN"): + if os.environ.get(name): + return os.environ[name] + try: + return subprocess.run(["gh", "auth", "token"], capture_output=True, text=True, check=True).stdout.strip() + except (OSError, subprocess.CalledProcessError): + sys.exit("roadmap: reading GitHub needs GITHUB_TOKEN, GH_TOKEN or `gh auth login` (or use --offline)") + + +def numbers_in(data): + return sorted({int(ISSUE.match(link)[1]) for item in data.get("item", []) + for link in item.get("links", []) if ISSUE.match(link)}) + + +def github_states(numbers, auth): + """{number: {"kind": "Issue" | "PullRequest", "state", "merged_at", "merge_commit", "paths"}} for those + that exist, in one GraphQL request per hundred numbers.""" + owner, name = REPO.split("/") + states = {} + for start in range(0, len(numbers), 100): + fields = "\n".join( + f"n{n}: issueOrPullRequest(number: {n}) {{ __typename " + "... on Issue { state } " + "... on PullRequest { state mergedAt mergeCommit { oid } files(first: 100) { nodes { path } } " + "labels(first: 50) { nodes { name } } } }" + for n in numbers[start:start + 100]) + query = f'query {{ repository(owner: "{owner}", name: "{name}") {{ {fields} }} }}' + request = urllib.request.Request( + "https://api.github.com/graphql", data=json.dumps({"query": query}).encode(), + headers={"Authorization": f"bearer {auth}", "User-Agent": "horizon-roadmap"}) + with urllib.request.urlopen(request, timeout=30) as response: + answer = json.load(response) + for key, node in (answer.get("data", {}).get("repository") or {}).items(): + if node: + states[int(key[1:])] = { + "kind": node["__typename"], + "state": node["state"], + "merged_at": node.get("mergedAt"), + "merge_commit": (node.get("mergeCommit") or {}).get("oid"), + "paths": [file["path"] for file in ((node.get("files") or {}).get("nodes") or [])], + "labels": [label["name"] for label in ((node.get("labels") or {}).get("nodes") or [])], + } + return states + + +def pull_requests(item, states): + return [states[n] for n in (int(ISSUE.match(link)[1]) for link in item.get("links", []) if ISSUE.match(link)) + if n in states and states[n]["kind"] == "PullRequest"] + + +def open_issues(item, states): + return [n for n in (int(ISSUE.match(link)[1]) for link in item.get("links", []) if ISSUE.match(link)) + if n in states and states[n]["kind"] == "Issue" and states[n]["state"] == "OPEN"] + + +def online_problems(data, states, now): + """What only GitHub can tell: links that don't exist, Now items nobody works on, Help wanted items already + done, items `sync` would ship.""" + problems, notes = [], [] + recent = now - datetime.timedelta(days=STALE_DAYS) + for item in data.get("item", []): + where = f"item {item['id']}" + for link in item.get("links", []): + match = ISSUE.match(link) + if match and int(match[1]) not in states: + problems.append(f"{where}: {link} is neither an issue nor a pull request of {REPO}") + pulls = pull_requests(item, states) + status = item.get("status") + if status == "now": + alive = [p for p in pulls if p["state"] == "OPEN" or + (p["merged_at"] and datetime.datetime.fromisoformat(p["merged_at"]) >= recent)] + if not alive: + problems.append(f"{where}: stale: Now, but no open pull request and none merged in the last " + f"{STALE_DAYS} days (link the work, or move it to Next or Later)") + if status == "help-wanted": + issues = [states[int(ISSUE.match(link)[1])] for link in item.get("links", []) + if ISSUE.match(link) and int(ISSUE.match(link)[1]) in states] + if issues and all(i["state"] != "OPEN" for i in issues): + problems.append(f"{where}: every linked issue and pull request is closed: ship or drop it") + if status != "shipped" and pulls and all(p["state"] == "MERGED" for p in pulls): + if item.get("ongoing"): + notes.append(f"{where}: ongoing, and all its pull requests merged: link the new ones") + elif not open_issues(item, states): + notes.append(f"{where}: all its pull requests merged: `python3 tools/roadmap.py sync` ships it " + "once a release has them") + return problems, notes + + +# --- sync ----------------------------------------------------------------------------------------------------- + +def family(paths): + """Which releases carry a change: the launcher's, the website (deployed from main) or the game's.""" + if paths and all(path.startswith("launcher/") for path in paths): + return "launcher" + if paths and all(path.startswith("website/") for path in paths): + return "website" + return "game" + + +def first_release(commit, kind): + """The oldest release tag of this kind that contains the commit, or None while none does.""" + if kind == "website": + return "website" + tags = subprocess.run(["git", "tag", "--contains", commit, "--sort=creatordate"], cwd=ROOT, + capture_output=True, text=True).stdout.split() + wanted = (lambda tag: tag.startswith(("launcher@", "launcher-v"))) if kind == "launcher" else \ + (lambda tag: re.match(r"v\d", tag)) + return next((tag for tag in tags if wanted(tag)), None) + + +def release_of(pulls): + """The release that shipped the last of these merged pull requests, or None.""" + kinds = {family(p["paths"]) for p in pulls} + kind = "game" if "game" in kinds or len(kinds) > 1 else kinds.pop() + latest = max(pulls, key=lambda p: p["merged_at"]) + return first_release(latest["merge_commit"], kind) if latest["merge_commit"] else None + + +def item_span(lines, ident): + """[start, end) of the item's lines: its [[item]] header to its last key, without the comments that follow.""" + starts = [i for i, line in enumerate(lines) if line.strip() == "[[item]]"] + for position, start in enumerate(starts): + end = starts[position + 1] if position + 1 < len(starts) else len(lines) + if not any(re.match(rf'id\s*=\s*"{re.escape(ident)}"\s*$', line) for line in lines[start:end]): + continue + while end > start and (not lines[end - 1].strip() or re.match(r"#(\s|-|$)", lines[end - 1])): + end -= 1 + return start, end + raise KeyError(ident) + + +def mark_shipped(text, ident, release): + """The TOML with this item shipped in `release`, moved to the top of the shipped items (newest first).""" + lines = text.split("\n") + start, end = item_span(lines, ident) + block = [line for line in lines[start:end] if not line.startswith("shipped_in")] + block = [f'status = "shipped"\nshipped_in = "{release}"' if re.match(r"status\s*=", line) else line + for line in block] + block = [line for line in block if not re.match(r"focus\s*=", line)] + rest = lines[:start] + lines[end:] + # Drop the blank line the block leaves behind. + if start < len(rest) and start > 0 and not rest[start - 1].strip() and not rest[start].strip(): + del rest[start] + shipped = [i for i, line in enumerate(rest) if re.match(r'status\s*=\s*"shipped"', line)] + if shipped: + header = max(i for i in range(shipped[0] + 1) if rest[i].strip() == "[[item]]") + rest[header:header] = block + [""] + else: + rest += [""] + block + return "\n".join(rest) + + +def sync(data, text, states): + """Ship every item whose linked pull requests all went out in a release and whose linked issues are all + closed, unless it is ongoing. Returns the new text and what happened.""" + log = [] + for item in data.get("item", []): + if item.get("status") == "shipped": + continue + pulls = pull_requests(item, states) + # An ongoing item, or one with an open issue, still has work left whatever its pull requests say. + if item.get("ongoing") or open_issues(item, states): + continue + if not pulls or any(p["state"] != "MERGED" for p in pulls): + continue + release = release_of(pulls) + if release is None: + log.append(f"{item['id']}: merged, waiting for a release") + continue + text = mark_shipped(text, item["id"], release) + log.append(f"{item['id']}: shipped in {release}") + return text, log + + +# --- labels --------------------------------------------------------------------------------------------------- + +def label_of(item): + """The GitHub label of an item: its name, colour and description (GitHub allows 100 characters).""" + description = f"Roadmap, {STATUSES[item['status']].lower()}: {item['title']}" + return {"name": LABEL_PREFIX + item["id"], "color": LABEL_COLOURS[item["status"]], "description": description[:100]} + + +def label_changes(data, existing, states): + """What `labels` does: [(action, label or number, payload)] for labels to create or update, and open pull + requests linked from an item that lack its label. existing: {name: {"color", "description"}}; states as + github_states, plus each pull request's "labels".""" + changes = [] + for item in data.get("item", []): + label = label_of(item) + have = existing.get(label["name"]) + if have is None: + changes.append(("create", label["name"], label)) + elif (have.get("color", "").lower(), have.get("description") or "") != (label["color"], label["description"]): + changes.append(("update", label["name"], label)) + for link in item.get("links", []): + match = ISSUE.match(link) + state = states.get(int(match[1])) if match else None + if state and state["kind"] == "PullRequest" and state["state"] == "OPEN" and \ + label["name"] not in state.get("labels", []): + changes.append(("add", int(match[1]), label["name"])) + stale = sorted(set(existing) - {LABEL_PREFIX + item["id"] for item in data.get("item", [])}) + return changes, [name for name in stale if name.startswith(LABEL_PREFIX)] + + +def github_rest(method, path, auth, body=None): + request = urllib.request.Request( + f"https://api.github.com{path}", method=method, data=None if body is None else json.dumps(body).encode(), + headers={"Authorization": f"bearer {auth}", "Accept": "application/vnd.github+json", + "User-Agent": "horizon-roadmap"}) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) if response.status != 204 else None + + +def existing_labels(auth): + labels, page = {}, 1 + while True: + batch = github_rest("GET", f"/repos/{REPO}/labels?per_page=100&page={page}", auth) + labels.update({label["name"]: label for label in batch}) + if len(batch) < 100: + return labels + page += 1 + + +# --- commands ------------------------------------------------------------------------------------------------- + +def report(problems): + for problem in problems: + print(f"roadmap: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +def command_build(args): + data = load() + problems = validate(data) + if problems: + return report(problems) + files = outputs(data) + if args.check: + stale = [path for path, text in files.items() if not path.exists() or path.read_text() != text] + return report([f"{path.relative_to(ROOT)} is out of date: run `python3 tools/roadmap.py build`" + for path in stale]) + for path, text in files.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + print(f"roadmap: wrote {path.relative_to(ROOT)}") + return 0 + + +def command_check(args): + data = load() + problems = validate(data) + if not problems and not args.offline: + states = github_states(numbers_in(data), token()) + online, notes = online_problems(data, states, datetime.datetime.now(datetime.timezone.utc)) + problems += online + for note in notes: + print(f"roadmap: {note}") + if not problems: + print(f"roadmap: {len(data.get('item', []))} items, all good") + return report(problems) + + +def command_sync(args): + data = load() + problems = validate(data) + if problems: + return report(problems) + subprocess.run(["git", "fetch", "--quiet", "--tags", "origin"], cwd=ROOT, check=False) + text, log = sync(data, SOURCE.read_text(), github_states(numbers_in(data), token())) + for line in log: + print(f"roadmap: {line}") + if args.dry_run or text == SOURCE.read_text(): + return 0 + new = tomllib.loads(text) + if len(new.get("item", [])) != len(data.get("item", [])) or validate(new): + return report(["sync produced an invalid file; nothing written"] + validate(new)) + SOURCE.write_text(text) + for path, content in outputs(new).items(): + path.write_text(content) + print("roadmap: wrote roadmap/roadmap.toml, ROADMAP.md and website/data/roadmap.json") + return 0 + + +def command_labels(args): + data = load() + problems = validate(data) + if problems: + return report(problems) + auth = token() + changes, stale = label_changes(data, existing_labels(auth), github_states(numbers_in(data), auth)) + for action, target, payload in changes: + print(f"roadmap: {action} {target}" + (f" ({payload})" if action == "add" else "")) + if args.dry_run: + continue + if action == "create": + github_rest("POST", f"/repos/{REPO}/labels", auth, payload) + elif action == "update": + github_rest("PATCH", f"/repos/{REPO}/labels/{urllib.parse.quote(target, safe='')}", auth, + {"color": payload["color"], "description": payload["description"]}) + else: + github_rest("POST", f"/repos/{REPO}/issues/{target}/labels", auth, {"labels": [payload]}) + for name in stale: + print(f"roadmap: {name} names no item any more: rename or delete it by hand (it stays on its pull requests)") + if not changes: + print("roadmap: labels up to date") + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + build = commands.add_parser("build", help="write ROADMAP.md") + build.add_argument("--check", action="store_true", help="fail when ROADMAP.md is out of date") + check = commands.add_parser("check", help="the roadmap's rules, and GitHub's state of its links") + check.add_argument("--offline", action="store_true", help="skip GitHub") + sync_command = commands.add_parser("sync", help="ship the items whose pull requests all merged") + sync_command.add_argument("--dry-run", action="store_true", help="say what would change, write nothing") + labels = commands.add_parser("labels", help="a roadmap: label per item, on its linked open pull requests") + labels.add_argument("--dry-run", action="store_true", help="say what would change, change nothing") + args = parser.parse_args(argv) + return {"build": command_build, "check": command_check, "sync": command_sync, + "labels": command_labels}[args.command](args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/website/Caddyfile b/website/Caddyfile index 1571566ad..88390ffdc 100644 --- a/website/Caddyfile +++ b/website/Caddyfile @@ -20,7 +20,7 @@ Referrer-Policy strict-origin-when-cross-origin X-Frame-Options DENY Permissions-Policy "interest-cohort=(), browsing-topics=(), camera=(), microphone=(), geolocation=()" - Content-Security-Policy "default-src 'self'; script-src 'self' {$CSP_SCRIPT}; style-src 'self' {$CSP_STYLE}; img-src 'self' data: https://avatars.githubusercontent.com; font-src 'self'; connect-src 'self' https://api.github.com https://raw.githubusercontent.com https://telemetry.playhorizon.dev; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" + Content-Security-Policy "default-src 'self'; script-src 'self' {$CSP_SCRIPT}; style-src 'self' {$CSP_STYLE}; img-src 'self' data: https://avatars.githubusercontent.com https://ludovic111.github.io; font-src 'self'; connect-src 'self' https://api.github.com https://raw.githubusercontent.com https://telemetry.playhorizon.dev https://ludovic111.github.io; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" -Server } @assets path /assets/* @@ -33,10 +33,33 @@ rewrite /privacy /privacy.html rewrite /releases /releases.html rewrite /benchmarks /benchmarks.html + rewrite /hub /hub.html + # The roadmap is the hub's Roadmap tab. + redir /roadmap /hub#roadmap permanent @doc path_regexp doc ^/docs/([a-z-]+)$ rewrite @doc /docs/{re.doc.1}.html - @pages path / /index.html /404.html /privacy.html /releases /releases.html /benchmarks /benchmarks.html /benchmarks.json /docs /docs/* /sitemap.xml + # The API portal: /docs/api is its index, /docs/api/ its pages. + redir /docs/api/ /docs/api permanent + @apidoc path_regexp apidoc ^/docs/api/([a-z-]+)$ + rewrite @apidoc /docs/api/{re.apidoc.1}.html + # The registry's submission service (crates/horizon-mod-registry), over Railway's private network. Unset, the + # form says it isn't open. + handle /api/mods/* { + reverse_proxy {$REGISTRY_UPSTREAM:127.0.0.1:9} + } + # The registry: /mods, /mods/submit and /mods/ baked at build time, else the list page, whose script shows a project + # published since. + rewrite /mods /mods.html + @mod { + path_regexp ^/mods/[a-z0-9_]+$ + file {path}.html /mods.html + } + rewrite @mod {file_match.relative} + @pages path / /index.html /404.html /privacy.html /releases /releases.html /benchmarks /benchmarks.html /benchmarks.json /hub /hub.html /data/* /docs /docs/* /mods /mods.html /mods/* /sitemap.xml header @pages Cache-Control "no-cache" + # The hub's data for Kira and anyone else: readable from any origin. + @data path /data/* + header @data Access-Control-Allow-Origin "*" @legal path /LEGAL.md /fonts/*.txt header @legal Content-Type "text/plain; charset=utf-8" diff --git a/website/Dockerfile b/website/Dockerfile index ea9cc9afc..b075f9b58 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /site COPY build.mjs ./ COPY src ./src COPY docs ./docs +COPY data ./data COPY public ./public COPY LEGAL.md ./ ARG GITHUB_TOKEN diff --git a/website/README.md b/website/README.md index 9d1f8dc8d..ccf54b59b 100644 --- a/website/README.md +++ b/website/README.md @@ -25,12 +25,15 @@ src/404.html the not-found page src/privacy.html what the launcher can send when players allow it, retention, deletion (/privacy) src/releases.html every release, newest first, with what it adds and fixes, its notes and downloads (/releases) src/benchmarks.html Horizon against vanilla 26.3 on our machines, the hardware picker, players' results (/benchmarks) +src/hub.html the hub (/hub, /roadmap redirects to its Roadmap tab): Overview, Roadmap, Kanban, Journal, Contribute, a Simple / Technical switch +src/mods.html the registry's list (/mods) and a page per project (/mods/), one template src/docs.html the template of every documentation page (/docs/...), with its left navigation src/tokens.css the design tokens, generated from crates/horizon-theme (do not edit) src/style.css every style, on top of the tokens src/main.js the home page: download buttons, recent releases, links, contributors, nav, tabs, lightbox, motion src/chrome.js the light/dark switch, the phone menu and GitHub fetching, shared by the pages src/releases-page.js the Releases page: the channel filter and the refresh from GitHub +src/mods.js the registry's markup (list, project page, submit form), shared by build and page; mods-page.js: the filter, the search, the refresh; mods-submit.js: the form's sign-in and sending src/benchmarks-page.js the Benchmarks page: players' results from the telemetry server, the picker src/docs-page.js the Docs pages: the shared theme switch and phone menu src/bench.js the Benchmarks tables, shared by build and page @@ -39,6 +42,9 @@ src/estimate.js estimates for unmeasured hardware from the nearest measured m src/markdown.js a small Markdown renderer (escapes everything), shared by build and page src/releases.js the home page's "Recently added" and the Releases page's markup, shared by build and page src/contributors.js the contributors' filtering and markup, shared by build and page +src/hub.js the hub's roadmap, board and journal from data/roadmap.json, GitHub's issues and the releases, shared by build and page; /data/journal.json +src/hub-page.js the hub page: tabs by #hash, the Simple / Technical switch (localStorage), the area filter, live states +data/roadmap.json the roadmap, generated by tools/roadmap.py from roadmap/roadmap.toml (do not edit); served at /data/roadmap.json public/ fonts, screenshots, icons, og.png, robots.txt, benchmarks.json: copied as is (sitemap.xml is written by the build) data/benchmarks/ our raw benchmark reports (tools/perf_bench.py) and fleet.json, the list scripts/benchmarks.mjs reads scripts/benchmarks.mjs benchmark reports -> public/benchmarks.json (ours, and players' groups of 3+ installs) @@ -52,14 +58,15 @@ shots/ (gitignored) full-page screenshots at 360–1920 px, a 1250× ## What comes from where -- **Built in** (`build.mjs`): the parity table, its score, harness count and audit date, +- **Built in** (`build.mjs`): Horizon's registry (`/mods`, one page per project) from the index ludovic111/horizon-mods + publishes (`HORIZON_MOD_INDEX=` builds from a test index; docs/api/mod-index.md); the parity table, its score, harness count and audit date, all from `README.md`; the documentation pages (`/docs`) from `website/docs/`; the Discord and Sponsor links from `launcher/links.json`; the latest releases and the contributors (`/repos/ludovic111/horizon/contributors`, bots left out, the owner first), when GitHub answers; without an answer the build still succeeds and the Contributors block and its menu entries stay hidden until the page loads them. Locally these come from the checkout; on Railway, which builds only this folder, from `main` on GitHub. The build fails if the README has no parity table, so a broken README never deploys. -- **Fetched by the page**: the releases (the download button picks the newest +- **Fetched by the page**: the registry's index (a project merged after the build shows at once), the releases (the download button picks the newest `launcher-v*` release's asset for the visitor's OS, the home page lists the newest game releases, and the Releases page reloads its list when GitHub has a newer release), players' benchmark results (`GET https://telemetry.playhorizon.dev/v1/benchmarks/summary`, 3-second timeout), the contributors (avatars come from `avatars.githubusercontent.com`), @@ -74,8 +81,13 @@ mark merged-but-unreleased things "next release". `{{HARNESSES}}` and `{{PERCENT the README. A line holding only `![caption](/media/...)` becomes a figure (the file must be in `public/media/`); links to other pages are `/docs/#heading`. The build checks every link between pages and every image, and fails with the list when one is broken. `/server` -redirects to `/docs/hosting`. The developer docs in the repository's `docs/` stay as they are and are linked from -the Contributing page. +redirects to `/docs/hosting`. + +The API portal is the repository's `docs/api/`, not a copy: `nav.json` lists its README as `api` (`/docs/api`) and +its pages as `api/` (`/docs/api/`), read from the checkout or from `main` on GitHub like the README. +Links between its pages become the site's, every other relative link goes to the file on GitHub, and the same link +check applies. Markdown tables render as tables. The other developer docs in `docs/` stay on GitHub and are linked +from the Contributing page. ## Run it locally @@ -111,7 +123,9 @@ a Caddy stage serves `dist/` on `$PORT` with compression, cache headers, a stric Content-Security-Policy and the 404 page. Its watch patterns also redeploy when the README or `launcher/links.json` change on main. No variables are needed; an optional `GITHUB_TOKEN` (read-only, public repos) lets the build bake the releases even when -GitHub's anonymous rate limit is hit. +GitHub's anonymous rate limit is hit. `REGISTRY_UPSTREAM` +(`horizon-mod-registry.railway.internal:8080`) sends `/api/mods/*` to the registry's submission +service (`crates/horizon-mod-registry`); unset, `/mods/submit` says the form isn't open. ## Rules the site keeps diff --git a/website/build.mjs b/website/build.mjs index 8b0954054..b979f83f8 100644 --- a/website/build.mjs +++ b/website/build.mjs @@ -3,12 +3,13 @@ // docs/nav.json), plus public/ and fingerprinted CSS and JS. No dependencies. // // Railway builds only website/, so the README and links.json come from the repo when they -// are next to it (a local checkout) and from main on GitHub otherwise. The documentation lives in website/. The latest releases and +// are next to it (a local checkout) and from main on GitHub otherwise. The documentation lives in website/, except the +// API portal (docs/api/), read the same way as the README. The latest releases and // the contributors are baked in when GitHub answers, so the page reads well before (or without) JavaScript. import { readFile, writeFile, mkdir, rm, cp } from "node:fs/promises"; import { existsSync } from "node:fs"; import { createHash } from "node:crypto"; -import { join, dirname } from "node:path"; +import { join, dirname, posix } from "node:path"; import { fileURLToPath } from "node:url"; import { markdown, inline, section, escape } from "./src/markdown.js"; import { recentHtml, releasesPageHtml } from "./src/releases.js"; @@ -16,6 +17,8 @@ import { machinesHtml, playersHtml } from "./src/bench.js"; import { CPUS, GPUS, singleThread } from "./src/hardware.js"; import { isGame, isLauncher, launcherVersion, newestFirst } from "./src/versions.js"; import { contributorsFrom, contributorsHtml } from "./src/contributors.js"; +import { areaFilterHtml, boardFilterHtml, boardFrom, boardHtml, creditFrom, creditHtml, helpWantedHtml, journalFrom, journalHtml, journalJson, overviewHtml, roadmapHtml, slimIssue, suggestItemsHtml } from "./src/hub.js"; +import { INDEX_URL, KINDS, SUBMIT, listHtml, projectHtml, readIndex, submitHtml } from "./src/mods.js"; const site = dirname(fileURLToPath(import.meta.url)); const dist = join(site, "dist"); @@ -41,6 +44,21 @@ try { console.warn(`links.json: ${e.message}; using the built-in links`); } +// Horizon's registry (/mods): the index its repository publishes, or HORIZON_MOD_INDEX (a file or an address) to +// try a test registry. Without an answer the pages are built empty and load the index themselves. +let modProjects = []; +let modIndex = null; +try { + const from = process.env.HORIZON_MOD_INDEX || INDEX_URL; + const text = from.startsWith("https://") + ? await fetch(from, { signal: AbortSignal.timeout(15000) }).then((r) => (r.ok ? r.text() : Promise.reject(new Error(`${r.status}`)))) + : await readFile(from, "utf8"); + modIndex = JSON.parse(text); + modProjects = readIndex(modIndex) ?? (console.warn("mod index: a schema this site doesn't read"), []); +} catch (e) { + console.warn(`mod index: ${e.message}; the pages will load it themselves`); +} + // An optional token (a Railway variable, or `gh auth token` locally) lifts the rate limit. const githubHeaders = { Accept: "application/vnd.github+json", ...(process.env.GITHUB_TOKEN ? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` } : {}) }; // Every release (the Releases page lists them all), a page of 100 at a time. @@ -77,6 +95,59 @@ try { console.warn(`contributors: ${e.message}; the page will load them itself`); } +// The hub's roadmap (data/roadmap.json, generated from roadmap/roadmap.toml by tools/roadmap.py) and the live state +// of the issues and pull requests its items link: the open ones and those closed lately, then (with a token) the +// older ones in one GraphQL request. Without GitHub the hub shows the links without their state. +const roadmap = JSON.parse(await readFile(join(site, "data", "roadmap.json"), "utf8")); +const linked = new Set(roadmap.items.flatMap((i) => (i.links || []).map((l) => Number((l.match(/^#(\d+)$/) || [])[1])).filter(Boolean))); +let hubLive = []; +try { + const urls = [1, 2, 3].map((p) => `https://api.github.com/repos/${REPO}/issues?state=open&per_page=100&page=${p}`); + urls.push(`https://api.github.com/repos/${REPO}/issues?state=closed&sort=updated&direction=desc&per_page=100`); + for (const url of urls) { + const res = await fetch(url, { headers: githubHeaders, signal: AbortSignal.timeout(15000) }); + if (!res.ok) throw new Error(`${res.status} from GitHub`); + hubLive.push(...(await res.json()).map(slimIssue)); + } + const missing = [...linked].filter((n) => !hubLive.some((i) => i.n === n)); + if (missing.length && process.env.GITHUB_TOKEN) { + const fields = missing.map((n) => `n${n}: issueOrPullRequest(number: ${n}) { ... on Issue { number title state stateReason url } ... on PullRequest { number title state isDraft mergedAt url } }`).join(" "); + const res = await fetch("https://api.github.com/graphql", { + method: "POST", + headers: githubHeaders, + body: JSON.stringify({ query: `query { repository(owner: "ludovic111", name: "horizon") { ${fields} } }` }), + signal: AbortSignal.timeout(15000), + }); + const nodes = Object.values((await res.json()).data?.repository || {}).filter(Boolean); + hubLive.push( + ...nodes.map((x) => + slimIssue({ + number: x.number, title: x.title, html_url: x.url, draft: x.isDraft, + state: x.state === "OPEN" ? "open" : "closed", state_reason: x.stateReason === "NOT_PLANNED" ? "not_planned" : null, + pull_request: "isDraft" in x ? { merged_at: x.mergedAt } : undefined, + }), + ), + ); + } +} catch (e) { + console.warn(`hub: ${e.message}; the page will load the states itself`); +} +const hubLinked = new Map(hubLive.filter((i) => linked.has(i.n)).map((i) => [i.n, i])); +const board = boardFrom(hubLive, roadmap); +const journal = journalFrom(releases, hubLive, roadmap); + +// The hub's server (crates/horizon-telemetry-server: votes, bug reports, suggestions), and Kira's list of the +// contributors who linked their Discord account and agreed to be credited (kira's /contributors.json). +const hubApi = process.env.HUB_API_URL || "https://telemetry.playhorizon.dev"; +let kiraCredit = {}; +try { + const res = await fetch(process.env.KIRA_CONTRIBUTORS_URL || "https://kira-production-500e.up.railway.app/contributors.json", { signal: AbortSignal.timeout(10000) }); + if (!res.ok) throw new Error(`${res.status} from Kira`); + kiraCredit = await res.json(); +} catch (e) { + console.warn(`hub: contributors' Discord names: ${e.message}`); +} + // --- The parity table ------------------------------------------------------------------ const status = section(readme, "Project status") ?? ""; @@ -170,7 +241,7 @@ await writeFile(join(dist, "favicon.ico"), Buffer.concat([ico, png32])); const names = {}; const rewrite = (code) => code.replace(/"\.\/(\w+\.js)"/g, (all, file) => (names[file] ? `"./${names[file]}"` : all)); // Dependencies before what imports them. -const modules = ["markdown.js", "versions.js", "chrome.js", "hardware.js", "estimate.js", "releases.js", "contributors.js", "bench.js", "main.js", "releases-page.js", "benchmarks-page.js", "docs-page.js"]; +const modules = ["markdown.js", "versions.js", "chrome.js", "hardware.js", "estimate.js", "releases.js", "contributors.js", "bench.js", "hub.js", "main.js", "releases-page.js", "benchmarks-page.js", "docs-page.js", "hub-page.js", "mods.js", "mods-page.js", "mods-submit.js"]; for (const file of modules) { const code = rewrite(await readFile(join(site, "src", file), "utf8")); names[file] = `${file.replace(".js", "")}.${fingerprint(code)}.js`; @@ -203,7 +274,7 @@ const unavailable = `

The release notes could not be loaded here. // One top bar and phone menu for Home, Releases, Benchmarks and every Docs page. const navSections = [["/#about", "About"], ["/#screenshots", "In game"], ["/#parity", "Parity"], ["/#launcher", "Launcher"], ["/#recent", "New", "Recently added"], ["/#community", "Community"]]; -const navPages = [["/releases", "Releases"], ["/benchmarks", "Benchmarks"], ["/docs/", "Docs", "Documentation"]]; +const navPages = [["/hub", "Hub"], ["/mods", "Mods"], ["/releases", "Releases"], ["/benchmarks", "Benchmarks"], ["/docs/", "Docs", "Documentation"]]; const navLink = ([href, label, menuLabel], current, menu = false) => `${menu && menuLabel ? menuLabel : label}`; const siteNav = (current) => `