From 8c5d1ed86bcafe5a3fce108c7d22407f579ffc99 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:08:15 +0200 Subject: [PATCH] Agent control: the widget tree of every screen, click and set by id `ui` lists every widget the last frame drew (title menus, pause, options in both menu styles, inventories and menus) with an id, its kind, label, bounds and value; `click ID` clicks one through the same window events as the mouse, `set ID VALUE` moves a slider along its track. Ids come from labels (FOV: 70 is fov), world/server/language rows carry their names, Horizon switches, sliders and segments their row's caption, and inventory slots are slot-ID with their stacks. After a click, the next widget command waits for a redrawn frame. horizon-ctl and its MCP gain ui, click and set, and launch --offscreen. tools/agent_control_e2e.py gains a ui scenario and runs offscreen on macOS; both scenarios now handle Singleplayer opening Create New World when there are no worlds. Co-Authored-By: Claude Opus 5.5 --- .../src/bin/horizon-ctl/commands.rs | 14 +- crates/horizon-client/src/game.rs | 4 + crates/horizon-client/src/game/agent.rs | 6 +- crates/horizon-client/src/game/control.rs | 29 ++- crates/horizon-client/src/game/pad_menu.rs | 2 +- .../src/game/station_control.rs | 2 +- crates/horizon-client/src/game/ui_control.rs | 206 ++++++++++++++++++ crates/horizon-client/src/multiplayer.rs | 13 +- crates/horizon-client/src/options_ui.rs | 9 +- crates/horizon-client/src/title.rs | 9 +- crates/horizon-client/src/ui.rs | 14 +- crates/horizon-client/src/ui_theme.rs | 69 ++++-- crates/horizon-client/src/widgets.rs | 182 +++++++++++++++- docs/native-client.md | 12 + tools/agent_control_e2e.py | 94 +++++++- 15 files changed, 604 insertions(+), 61 deletions(-) create mode 100644 crates/horizon-client/src/game/ui_control.rs diff --git a/crates/horizon-client/src/bin/horizon-ctl/commands.rs b/crates/horizon-client/src/bin/horizon-ctl/commands.rs index c6beb4ab8..f85105859 100644 --- a/crates/horizon-client/src/bin/horizon-ctl/commands.rs +++ b/crates/horizon-client/src/bin/horizon-ctl/commands.rs @@ -145,7 +145,8 @@ pub const COMMANDS: &[Cmd] = &[ ["horizon-ctl launch --world Survival --seed 42 --wait", "horizon-ctl launch --world Flat --creative --render-distance 8"], &[flag("world", Ty::Str, "world folder to open or create"), flag("seed", Ty::Str, "seed for a new world"), flag("creative", Ty::Bool, "creative mode for a new world"), flag("version", Ty::Str, "an installed launcher version tag such as v2026.10.13"), flag("game", Ty::Str, "path to a horizon-client binary"), - flag("wait", Ty::Bool, "also wait until the world is ready (default in MCP)")], + flag("wait", Ty::Bool, "also wait until the world is ready (default in MCP)"), + flag("offscreen", Ty::Bool, "render without showing a window (screenshots still work): locked or headless sessions")], mcp: true, local: true), cmd!("wait-ready", Session, "wait-ready [SECONDS]", "wait until the player has spawned and no chunks are pending", "Polls status every 250 ms; replies with the status, or an error after SECONDS (default 120).", @@ -266,6 +267,17 @@ pub const COMMANDS: &[Cmd] = &[ ["horizon-ctl click-slot 36", "horizon-ctl click-slot 100 right", "horizon-ctl click-slot 200 shift"], &[p("slot", Ty::Int, "slot id"), opt("right", Ty::Bool, "right-click"), opt("shift", Ty::Bool, "shift-click")], mcp: true, local: false), cmd!("close", Screens, "close", "close the open menu and return to the game", "", ["horizon-ctl close"], &[], mcp: true, local: false), + cmd!("ui", Screens, "ui", "every widget of the open screen: id, kind, label, bounds, value", + "Kinds: button, slider (value 0..1), field, row (worlds, servers, packs, languages), tab (selected), slot (slot-ID, with its stack). Ids come from labels (\"FOV: 70\" is fov, a repeat gets -2) or kind-N without one, and stay put when a value changes. Reflects the last drawn frame (frame counts them).", + ["horizon-ctl ui"], &[], mcp: true, local: false), + cmd!("click", Screens, "click ID [right] [shift]", "click a widget of ui by its id, as the mouse would", + "A field takes focus (then type); slot-ID clicks a slot like click-slot.", + ["horizon-ctl click done", "horizon-ctl click video-settings", "horizon-ctl click row-0", "horizon-ctl click slot-0 shift"], + &[p("id", Ty::Str, "widget id from ui"), opt("right", Ty::Bool, "right-click"), opt("shift", Ty::Bool, "shift-click (slots)")], mcp: true, local: false), + cmd!("set", Screens, "set ID VALUE", "move a slider of ui to VALUE (0..1), as a click on its track", + "The option behind it rounds to its own steps; read the new label with ui.", + ["horizon-ctl set fov 0.5", "horizon-ctl set render-distance 0"], + &[p("id", Ty::Str, "slider id from ui"), p("value", Ty::Num, "0..1 along the track")], mcp: true, local: false), cmd!("click-button", Screens, "click-button LABEL", "click a button of the open screen or title menu by its label", "Case-insensitive; a unique part of the label is enough. screen lists the buttons.", ["horizon-ctl click-button Singleplayer", "horizon-ctl click-button \"Create New World\"", "horizon-ctl click-button respawn"], diff --git a/crates/horizon-client/src/game.rs b/crates/horizon-client/src/game.rs index c58dcefe0..1d824424e 100644 --- a/crates/horizon-client/src/game.rs +++ b/crates/horizon-client/src/game.rs @@ -118,6 +118,7 @@ mod totem; mod trading; mod trial_spawner; mod trident; +mod ui_control; mod villagers; mod wandering_trader; mod wind; @@ -945,6 +946,8 @@ struct Game<'a> { flight: Option, /// The running agent action from the control socket (game/agent.rs). agent: agent::Agent, + /// Widget commands waiting for the frame after a click (game/ui_control.rs). + ui_wait: ui_control::UiWait, /// The overworld's last `sky_darken` and its inputs (time, rain, thunder). sky_darken_memo: std::cell::Cell>, persist: persist::EntityMemory, @@ -9809,6 +9812,7 @@ pub fn run() -> Result<()> { pending_screenshot: None, pending_bench: None, flight: None, + ui_wait: Default::default(), agent: agent::Agent::default(), view_frustum: None, pending_uploads: Vec::new(), diff --git a/crates/horizon-client/src/game/agent.rs b/crates/horizon-client/src/game/agent.rs index 243351af6..18d768fbc 100644 --- a/crates/horizon-client/src/game/agent.rs +++ b/crates/horizon-client/src/game/agent.rs @@ -314,7 +314,7 @@ impl Game<'_> { "loaded": self.world.chunks.contains_key(&(pos[0] >> 4, pos[2] >> 4)), }) } - fn stack_value(&self, s: &ItemStack) -> Value { + pub(super) fn stack_value(&self, s: &ItemStack) -> Value { let mut v = json!({ "item": &*s.item, "count": s.count }); if s.damage > 0 { v["damage"] = json!(s.damage); @@ -356,7 +356,7 @@ impl Game<'_> { } Ok(()) } - fn screen_label(&self) -> &'static str { + pub(super) fn screen_label(&self) -> &'static str { if self.title.is_some() { return "title"; } @@ -782,7 +782,7 @@ impl Game<'_> { } /// Synthesize the pointer moving to GUI pixel (x, y) and a click there. - fn synth_click( + pub(super) fn synth_click( &mut self, event_loop: &ActiveEventLoop, x: f32, diff --git a/crates/horizon-client/src/game/control.rs b/crates/horizon-client/src/game/control.rs index af981b798..726b095b9 100644 --- a/crates/horizon-client/src/game/control.rs +++ b/crates/horizon-client/src/game/control.rs @@ -97,15 +97,22 @@ impl Game<'_> { } horizon_client::widgets::end_frame(self.frames); self.advance_agent(); - while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) { - let reply = match self.control_command(&request.line, &request.reply, event_loop) { - Ok(Some(value)) => value, - // Deferred: the command keeps its own reply channel. - Ok(None) => continue, - Err(e) => json!({ "error": e.to_string() }), - }; - let _ = request.reply.send(reply.to_string()); + for request in self.ui_wait.ready() { + self.answer(request, event_loop); } + while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) { + self.answer(request, event_loop); + } + } + + fn answer(&mut self, request: horizon_client::control::Request, event_loop: &ActiveEventLoop) { + let reply = match self.control_command(&request.line, &request.reply, event_loop) { + Ok(Some(value)) => value, + // Deferred: the command keeps its own reply channel. + Ok(None) => return, + Err(e) => json!({ "error": e.to_string() }), + }; + let _ = request.reply.send(reply.to_string()); } fn control_command( @@ -125,6 +132,11 @@ impl Game<'_> { if let Some(value) = self.block_command(&words)? { return Ok(Some(value)); } + match self.ui_command(&words, line, reply, event_loop)? { + super::agent::Handled::Reply(value) => return Ok(Some(value)), + super::agent::Handled::Deferred => return Ok(None), + super::agent::Handled::No => {} + } if let Some(value) = self.lan_control(&words)? { return Ok(Some(value)); } @@ -146,6 +158,7 @@ impl Game<'_> { .chain(super::render_control::HELP) .chain(super::sculk::HELP) .chain(super::agent::HELP) + .chain(super::ui_control::HELP) .chain(super::pad::HELP) .collect::>() }))), "setblock" | "fill" | "particle" | "signedit" => self.render_command(&words), diff --git a/crates/horizon-client/src/game/pad_menu.rs b/crates/horizon-client/src/game/pad_menu.rs index 46a6d66b1..e80aa2b32 100644 --- a/crates/horizon-client/src/game/pad_menu.rs +++ b/crates/horizon-client/src/game/pad_menu.rs @@ -101,7 +101,7 @@ impl Game<'_> { || self.book.is_some() } - fn gui_scale(&self) -> Option { + pub(super) fn gui_scale(&self) -> Option { self.renderer .as_ref() .map(|r| HudBatch::new([r.config.width, r.config.height])) diff --git a/crates/horizon-client/src/game/station_control.rs b/crates/horizon-client/src/game/station_control.rs index 19ae9c672..680939a4f 100644 --- a/crates/horizon-client/src/game/station_control.rs +++ b/crates/horizon-client/src/game/station_control.rs @@ -56,7 +56,7 @@ impl Game<'_> { } self.cursor = saved; } - fn gui_origin(&self) -> Result<(f32, f32)> { + pub(super) fn gui_origin(&self) -> Result<(f32, f32)> { let size = self .renderer .as_ref() diff --git a/crates/horizon-client/src/game/ui_control.rs b/crates/horizon-client/src/game/ui_control.rs new file mode 100644 index 000000000..a20598a60 --- /dev/null +++ b/crates/horizon-client/src/game/ui_control.rs @@ -0,0 +1,206 @@ +//! The widget tree of the open screen for agents (`ui`, `click ID`, `set ID +//! VALUE`; docs/agent-control.md): what the last frame drew, with ids, and +//! clicks that go through the same window events as the mouse. +use super::{Game, Screen, agent::Handled}; +use horizon_client::{ + Result, + control::Request, + widgets::{self, Kind, Node}, +}; +use serde_json::{Value, json}; +use std::{ + sync::mpsc::Sender, + time::{Duration, Instant}, +}; +use winit::{event::MouseButton, event_loop::ActiveEventLoop}; + +pub(super) const HELP: &[&str] = &[ + "ui every widget of the open screen: id, kind, label, bounds, value", + "click ID [right] [shift] click a widget of ui by its id (slot-N: as click-slot N)", + "set ID VALUE move a slider to VALUE (0..1), as a click on its track", +]; + +/// After a click, `ui`, `click` and `set` wait for a frame drawn since, so +/// they never read the screen the click left. +#[derive(Default)] +pub(super) struct UiWait { + /// The widget generation at the last click, and when it happened. + after: Option<(u64, Instant)>, + waiting: Vec, +} + +impl UiWait { + fn stale(&self) -> bool { + self.after.is_some_and(|(generation, at)| { + generation == widgets::generation() && at.elapsed() < Duration::from_secs(1) + }) + } + /// The commands that can run now. + pub(super) fn ready(&mut self) -> Vec { + if self.stale() { + return Vec::new(); + } + self.after = None; + std::mem::take(&mut self.waiting) + } + fn clicked(&mut self) { + self.after = Some((widgets::generation(), Instant::now())); + } +} + +impl Game<'_> { + /// The open screen's widgets: those the last frame drew, then the slots + /// of an open inventory or menu (`slot-ID`, the ids of `click-slot`). + fn ui_nodes(&self) -> Vec { + let size = self.gui_scale().map(|b| [b.width, b.height]); + let mut nodes = widgets::tree(&widgets::last_labelled(), &widgets::last_targets(), size); + if self.screen == Screen::Inventory + && self.title.is_none() + && let Ok((ox, oy)) = self.gui_origin() + { + nodes.extend(self.slots().into_iter().map(|i| { + let (x, y) = self.slot_position(i); + Node { + id: format!("slot-{i}"), + kind: Kind::Slot, + label: None, + rect: [ox + x, oy + y, 16.0, 16.0], + enabled: true, + } + })); + } + nodes + } + + fn node_value(&mut self, node: &Node) -> Value { + let mut v = json!({ + "id": node.id, + "kind": node.kind.name(), + "bounds": node.rect, + "enabled": node.enabled, + }); + if let Some(label) = &node.label { + v["label"] = json!(label); + } + match node.kind { + Kind::Slider { value, .. } => v["value"] = json!(value), + Kind::Tab(selected) => v["selected"] = json!(selected), + Kind::Slot => { + if let Some(slot) = node.id.strip_prefix("slot-").and_then(|s| s.parse().ok()) { + let stack = if slot == 200 { + self.craft_result() + } else { + self.slot_mut(slot).clone() + }; + v["stack"] = stack.map_or(Value::Null, |s| self.stack_value(&s)); + } + } + _ => {} + } + v + } + + fn find_node(&self, id: &str) -> Result { + let nodes = self.ui_nodes(); + nodes.iter().find(|n| n.id == id).cloned().ok_or_else(|| { + let ids: Vec<&str> = nodes.iter().map(|n| n.id.as_str()).collect(); + format!( + "no widget {id:?} on the {} screen; widgets: {ids:?}", + self.screen_label() + ) + .into() + }) + } + + /// `ui`, `click` and `set`; `Handled::No` for other commands. + pub(super) fn ui_command( + &mut self, + words: &[&str], + line: &str, + reply: &Sender, + event_loop: &ActiveEventLoop, + ) -> Result { + if !matches!(words[0], "ui" | "click" | "set") { + return Ok(Handled::No); + } + if self.ui_wait.stale() { + self.ui_wait.waiting.push(Request { + line: line.to_owned(), + reply: reply.clone(), + }); + return Ok(Handled::Deferred); + } + let id = || words.get(1).copied().ok_or("missing the widget id; see ui"); + match words[0] { + "ui" => { + let nodes = self.ui_nodes(); + let widgets: Vec = nodes.iter().map(|n| self.node_value(n)).collect(); + let size = self + .gui_scale() + .map(|b| [b.width.round(), b.height.round()]); + Ok(Handled::Reply(json!({ + "screen": self.screen_label(), + "gui_size": size, + "frame": widgets::generation(), + "widgets": widgets, + }))) + } + "click" => { + let node = self.find_node(id()?)?; + if !node.enabled { + return Err(format!("widget {:?} is disabled", node.id).into()); + } + let right = words.contains(&"right"); + if node.kind == Kind::Slot { + let slot = node.id.trim_start_matches("slot-"); + let mut click = vec!["click-slot", slot]; + click.extend(words.iter().filter(|w| matches!(**w, "right" | "shift"))); + self.station_command(&click)?; + } else { + let (x, y) = ( + node.rect[0] + node.rect[2] / 2.0, + node.rect[1] + node.rect[3] / 2.0, + ); + let button = if right { + MouseButton::Right + } else { + MouseButton::Left + }; + self.synth_click(event_loop, x, y, button)?; + } + self.ui_wait.clicked(); + Ok(Handled::Reply( + json!({ "ok": true, "clicked": node.id, "screen": self.screen_label() }), + )) + } + "set" => { + let node = self.find_node(id()?)?; + let value: f32 = words + .get(2) + .ok_or("set ID VALUE: missing the value")? + .parse() + .map_err(|_| "set ID VALUE: the value is a number from 0 to 1")?; + let Kind::Slider { track, .. } = node.kind else { + return Err(format!( + "widget {:?} is a {}; set moves sliders (click a button, type into a field)", + node.id, + node.kind.name() + ) + .into()); + }; + if !node.enabled { + return Err(format!("widget {:?} is disabled", node.id).into()); + } + // AbstractSliderButton.setValueFromMouse: (x - track start) / track width. + let x = track[0] + value.clamp(0.0, 1.0) * track[1]; + let y = node.rect[1] + node.rect[3] / 2.0; + self.synth_click(event_loop, x, y, MouseButton::Left)?; + self.ui_wait.clicked(); + Ok(Handled::Reply( + json!({ "ok": true, "set": node.id, "screen": self.screen_label() }), + )) + } + _ => Ok(Handled::No), + } + } +} diff --git a/crates/horizon-client/src/multiplayer.rs b/crates/horizon-client/src/multiplayer.rs index a13bc7063..0c81f6da2 100644 --- a/crates/horizon-client/src/multiplayer.rs +++ b/crates/horizon-client/src/multiplayer.rs @@ -455,10 +455,15 @@ impl Screen { && y + ROW_HEIGHT / 2.0 > top && y + ROW_HEIGHT / 2.0 < bottom { - crate::widgets::target_as( - [left, y, ROW_WIDTH, ROW_HEIGHT], - crate::widgets::Kind::Row, - ); + let rect = [left, y, ROW_WIDTH, ROW_HEIGHT]; + // The control socket's widget tree names the row. + let name = match row { + Row::Server(k) => servers[k].name.as_str(), + Row::Lan(k) => games[k].motd.as_str(), + Row::LanHeader => "", + }; + crate::widgets::record_as(name, rect, true, crate::widgets::Kind::Row); + crate::widgets::target_as(rect, crate::widgets::Kind::Row); } // extractSelection: a white outline round black (the list // has focus once a row is clicked). diff --git a/crates/horizon-client/src/options_ui.rs b/crates/horizon-client/src/options_ui.rs index 8d67dcc04..1e5c623be 100644 --- a/crates/horizon-client/src/options_ui.rs +++ b/crates/horizon-client/src/options_ui.rs @@ -1605,7 +1605,14 @@ impl OptionsUi { WKind::LanguageRow { label, selected, .. } => { - crate::widgets::target_as([wd.x, wd.y, wd.w, wd.h], crate::widgets::Kind::Row); + let rect = [wd.x, wd.y, wd.w, wd.h]; + crate::widgets::record_as( + &label.plain(), + rect, + true, + crate::widgets::Kind::Row, + ); + crate::widgets::target_as(rect, crate::widgets::Kind::Row); if *selected { batch.fill(atlas, wd.x, wd.y, wd.w, wd.h, u32::MAX); batch.fill( diff --git a/crates/horizon-client/src/title.rs b/crates/horizon-client/src/title.rs index c4e568df5..a514ed4bf 100644 --- a/crates/horizon-client/src/title.rs +++ b/crates/horizon-client/src/title.rs @@ -826,10 +826,15 @@ impl Title { } let mid = y - 2.0 + ROW_HEIGHT / 2.0; if mid > top && mid < bottom { - crate::widgets::target_as( - [x - 2.0, y - 2.0, ROW_WIDTH, ROW_HEIGHT], + let rect = [x - 2.0, y - 2.0, ROW_WIDTH, ROW_HEIGHT]; + // The control socket's widget tree names the row. + crate::widgets::record_as( + &world.name, + rect, + true, crate::widgets::Kind::Row, ); + crate::widgets::target_as(rect, crate::widgets::Kind::Row); } // extractSelection: white once the list has focus. if *selected == Some(i) { diff --git a/crates/horizon-client/src/ui.rs b/crates/horizon-client/src/ui.rs index 05d3b2ad6..dc9ce588d 100644 --- a/crates/horizon-client/src/ui.rs +++ b/crates/horizon-client/src/ui.rs @@ -283,15 +283,15 @@ pub fn slider( hover: bool, active: bool, ) { + let kind = crate::widgets::Kind::Slider { + value, + track: [x + 4.0, w - 8.0], + }; + // The control socket's widget tree (`ui`): its label and value. + crate::widgets::record_as(&label.plain(), [x, y, w, 20.0], active, kind); if active { // A controller's D-pad can focus it (Horizon's controller support). - crate::widgets::target_as( - [x, y, w, 20.0], - crate::widgets::Kind::Slider { - value, - track: [x + 4.0, w - 8.0], - }, - ); + crate::widgets::target_as([x, y, w, 20.0], kind); } let track = if hover && active { "gui/sprites/widget/slider_highlighted" diff --git a/crates/horizon-client/src/ui_theme.rs b/crates/horizon-client/src/ui_theme.rs index 74343efe7..1c41add9d 100644 --- a/crates/horizon-client/src/ui_theme.rs +++ b/crates/horizon-client/src/ui_theme.rs @@ -458,6 +458,9 @@ pub struct Ui<'a> { /// The number the next settings row shows (`01`, `02`…); `page_header` /// starts it again. pub row_index: usize, + /// The caption of the settings row drawn last: the control socket's + /// label for the switch, slider or segments drawn on it. + row_label: String, /// Secondary text (indices, inactive items, captions): `FAINT`, or `MUTED` over the home /// page's lighter veil (`horizon_theme::HOME_VEIL`). pub muted: u32, @@ -491,6 +494,7 @@ impl<'a> Ui<'a> { opacity: 1.0, offset: (0.0, 0.0), row_index: 0, + row_label: String::new(), muted: color::FAINT, } } @@ -953,6 +957,25 @@ impl<'a> Ui<'a> { self.mark(22.0, 26.0, 18.0); self.text_line(Face::Medium, BODY, 18.0, 48.0, 26.0, color::TEXT, title); } + /// A widget's label for the control socket (`click-button`, `ui`), in GUI pixels. + fn record_label( + &self, + label: &str, + [x, y, w, h]: [f32; 4], + enabled: bool, + kind: crate::widgets::Kind, + ) { + if self.real { + let k = self.density / self.scale; + let (ox, oy) = self.offset; + crate::widgets::record_as( + label, + [(x + ox) * k, (y + oy) * k, w * k, h * k], + enabled, + kind, + ); + } + } /// The launcher's `button`: a pill 32 tall (or `CONTROL_LARGE` with /// `LARGE` text), medium weight, a hover layer easing in over `QUICK`, /// and the press state. Returns the hover. @@ -964,12 +987,7 @@ impl<'a> Ui<'a> { kind: Kind, enabled: bool, ) -> f32 { - if self.real { - // The control socket's button list, in GUI pixels. - let k = self.density / self.scale; - let (ox, oy) = self.offset; - crate::widgets::record(label, [(x + ox) * k, (y + oy) * k, w * k, h * k], enabled); - } + self.record_label(label, [x, y, w, h], enabled, crate::widgets::Kind::Button); let hover = self.interact(id, x, y, w, h, enabled); let saved = self.opacity; if !enabled { @@ -1043,6 +1061,12 @@ impl<'a> Ui<'a> { for (i, label) in labels.iter().enumerate() { let iy = y + i as f32 * STEP; let active = i == selected; + self.record_label( + label, + [x, iy, w, CONTROL], + true, + crate::widgets::Kind::Tab(active), + ); let hover = self.interact_as( base + i as u64, [x, iy, w, CONTROL], @@ -1089,6 +1113,13 @@ impl<'a> Ui<'a> { let mut sx = x + 2.0; for (i, (label, sw)) in labels.iter().zip(&widths).enumerate() { let active = i == selected; + let name = format!("{} {label}", self.row_label); + self.record_label( + &name, + [sx, y + 2.0, *sw, SEGMENT], + true, + crate::widgets::Kind::Tab(active), + ); let hover = self.interact(base + i as u64, sx, y + 2.0, *sw, SEGMENT, true); if active { self.rounded(sx, y + 2.0, *sw, SEGMENT, PILL, color::TEXT); @@ -1117,7 +1148,12 @@ impl<'a> Ui<'a> { pub fn switch(&mut self, id: u64, right: f32, cy: f32, on: bool) { let (w, h) = (36.0, 20.0); let (x, y) = (right - w, (cy - h / 2.0).round()); - self.interact(id, x - 4.0, y - 4.0, w + 8.0, h + 8.0, true); + let rect = [x - 4.0, y - 4.0, w + 8.0, h + 8.0]; + if !self.row_label.is_empty() { + let label = format!("{}: {}", self.row_label, if on { "on" } else { "off" }); + self.record_label(&label, rect, true, crate::widgets::Kind::Button); + } + self.interact_as(id, rect, true, crate::widgets::Kind::Button); let t = self.tween(id ^ 0x5717c4, if on { 1.0 } else { 0.0 }, move_()); self.rounded( x, @@ -1142,6 +1178,7 @@ impl<'a> Ui<'a> { /// the value column as before the numbers came; the index takes its /// share from it. pub fn row(&mut self, x: f32, y: f32, label: &str, value: &str, value_w: f32) -> f32 { + self.row_label = label.to_owned(); let line = (BODY * 1.45).round(); let h = 12.0 + line.max(CONTROL) + 12.0; let ty = y + (h - line) / 2.0; @@ -1173,6 +1210,7 @@ impl<'a> Ui<'a> { description: &str, description_w: f32, ) -> f32 { + self.row_label = label.to_owned(); let h = 12.0 + CONTROL + 12.0; let top = y + 8.0; self.index(x, top); @@ -1218,15 +1256,14 @@ impl<'a> Ui<'a> { /// Dragging is the screen's job (see [`slider_value`]). pub fn slider(&mut self, id: u64, x: f32, cy: f32, w: f32, value: f32) { let v = value.clamp(0.0, 1.0); - let hover = self.interact_as( - id, - [x - 8.0, cy - 12.0, w + 16.0, 24.0], - true, - crate::widgets::Kind::Slider { - value: v, - track: [x, w], - }, - ); + let kind = crate::widgets::Kind::Slider { + value: v, + track: [x, w], + }; + let rect = [x - 8.0, cy - 12.0, w + 16.0, 24.0]; + let label = self.row_label.clone(); + self.record_label(&label, rect, true, kind); + let hover = self.interact_as(id, rect, true, kind); self.rounded(x, cy - 1.0, w, 2.0, 1.0, color::BORDER); self.rounded(x, cy - 1.0, w * v, 2.0, 1.0, color::TEXT); let r = 7.0 + hover; diff --git a/crates/horizon-client/src/widgets.rs b/crates/horizon-client/src/widgets.rs index b04b86abe..8bd94612d 100644 --- a/crates/horizon-client/src/widgets.rs +++ b/crates/horizon-client/src/widgets.rs @@ -1,9 +1,10 @@ -//! The labelled buttons drawn in the last frame, for the control socket: a -//! script clicks a button by its label (`horizon-ctl click-button Respawn`) -//! whatever screen or menu style drew it. +//! The labelled widgets and focus targets drawn in the last frame, for the +//! control socket: a script clicks a button by its label (`horizon-ctl +//! click-button Respawn`) or any widget by its id in [`tree`] (`horizon-ctl +//! click done`), whatever screen or menu style drew it. use std::cell::RefCell; -/// A button in GUI pixels (the coordinates screens take input in). +/// A labelled widget in GUI pixels (the coordinates screens take input in). #[derive(Clone, Debug)] pub struct Widget { pub label: String, @@ -12,6 +13,8 @@ pub struct Widget { pub w: f32, pub h: f32, pub enabled: bool, + /// Buttons for [`record`]; sliders and tabs carry their labels too. + pub kind: Kind, } /// What a controller does with a focus target (docs/controllers.md, "Focus @@ -106,7 +109,12 @@ pub fn generation() -> u64 { } /// Called by the button drawers. -pub fn record(label: &str, [x, y, w, h]: [f32; 4], enabled: bool) { +pub fn record(label: &str, rect: [f32; 4], enabled: bool) { + record_as(label, rect, enabled, Kind::Button); +} + +/// The label of a widget of another kind (a slider's "FOV: 70", a tab). +pub fn record_as(label: &str, [x, y, w, h]: [f32; 4], enabled: bool, kind: Kind) { let label = label.trim(); if label.is_empty() { return; @@ -120,6 +128,7 @@ pub fn record(label: &str, [x, y, w, h]: [f32; 4], enabled: bool) { w, h, enabled, + kind, }); } }); @@ -142,9 +151,118 @@ pub fn end_frame(frames: u64) { /// The buttons of the last drawn frame, top-most last. pub fn last() -> Vec { + WIDGETS.with_borrow(|(_, last, _)| { + last.iter() + .filter(|w| w.kind == Kind::Button) + .cloned() + .collect() + }) +} + +/// Every labelled widget of the last drawn frame, sliders and tabs included. +pub fn last_labelled() -> Vec { WIDGETS.with_borrow(|(_, last, _)| last.clone()) } +impl Kind { + /// The kind's name in the control socket's `ui` reply. + pub fn name(self) -> &'static str { + match self { + Kind::Button => "button", + Kind::Slider { .. } => "slider", + Kind::Field => "field", + Kind::Row => "row", + Kind::Slot => "slot", + Kind::Tab(_) => "tab", + } + } +} + +/// A widget of the `ui` tree: a labelled widget and its focus target merged. +#[derive(Clone, Debug, PartialEq)] +pub struct Node { + /// The label as a slug up to its `:` (`FOV: 70` is `fov`), `-2`, `-3` + /// for repeats; `-` in draw order without a label. + pub id: String, + pub kind: Kind, + pub label: Option, + pub rect: [f32; 4], + pub enabled: bool, +} + +/// `Video Settings...` → `video-settings`, `FOV: 70` → `fov`. +pub fn slug(label: &str) -> String { + let name = label.split(':').next().unwrap_or(label); + let mut out = String::new(); + for c in name.chars().flat_map(char::to_lowercase) { + if c.is_alphanumeric() { + out.push(c); + } else if !out.is_empty() && !out.ends_with('-') { + out.push('-'); + } + } + out.trim_end_matches('-').to_owned() +} + +/// The labelled widgets and the focus targets of a frame as one list with +/// ids: a target drawn at a labelled widget's rectangle is that widget +/// (its kind wins, a slider's value, unless the label says it's a tab: +/// Horizon's segments); other targets follow, unlabelled. With the GUI +/// size, what lies off screen (layout probes) is left out before naming. +pub fn tree(widgets: &[Widget], targets: &[Target], screen: Option<[f32; 2]>) -> Vec { + let same = |a: [f32; 4], b: [f32; 4]| a.iter().zip(b).all(|(p, q)| (p - q).abs() < 0.5); + let mut nodes: Vec = widgets + .iter() + .map(|w| Node { + id: String::new(), + kind: w.kind, + label: Some(w.label.clone()), + rect: [w.x, w.y, w.w, w.h], + enabled: w.enabled, + }) + .collect(); + for t in targets { + if let Some(node) = nodes.iter_mut().find(|n| same(n.rect, t.rect)) { + if !matches!(node.kind, Kind::Tab(_)) { + node.kind = t.kind; + } + } else { + nodes.push(Node { + id: String::new(), + kind: t.kind, + label: None, + rect: t.rect, + enabled: true, + }); + } + } + if let Some([w, h]) = screen { + nodes.retain(|n| { + let [x, y, nw, nh] = n.rect; + x >= -0.5 && y >= -0.5 && x + nw <= w + 0.5 && y + nh <= h + 0.5 + }); + } + let mut seen = std::collections::HashMap::::new(); + for node in &mut nodes { + let base = node + .label + .as_deref() + .map(slug) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| node.kind.name().to_owned()); + let n = seen.entry(base.clone()).or_default(); + *n += 1; + node.id = if node.label.is_none() { + format!("{base}-{}", *n - 1) + } else if *n > 1 { + format!("{base}-{n}") + } else { + base + }; + } + nodes +} + /// The button `label` names: an exact match (case-insensitive), else the only /// button whose label contains it. The top-most wins among equals. pub fn find(widgets: &[Widget], label: &str) -> Result { @@ -185,6 +303,7 @@ mod tests { w: 10.0, h: 10.0, enabled: true, + kind: Kind::Button, } } @@ -197,6 +316,59 @@ mod tests { assert!(find(&list, "options").is_err()); } + #[test] + fn tree_merges_labels_with_targets_and_names_them() { + let slider = Kind::Slider { + value: 0.25, + track: [4.0, 142.0], + }; + let mut fov = w("FOV: 70"); + fov.kind = slider; + let mut done = w("Done"); + done.y = 30.0; + let mut done2 = w("Done"); + done2.y = 60.0; + let widgets = [fov, w("Video Settings..."), done, done2]; + let targets = [ + Target { + rect: [0.0, 0.0, 10.0, 10.0], + kind: slider, + }, + Target { + rect: [0.0, 90.0, 10.0, 10.0], + kind: Kind::Row, + }, + Target { + rect: [0.0, 100.0, 10.0, 10.0], + kind: Kind::Row, + }, + ]; + let mut probe = w("Done"); + probe.x = -10_000.0; + let widgets = [&[probe][..], &widgets].concat(); + let nodes = tree(&widgets, &targets, Some([400.0, 200.0])); + let ids: Vec<&str> = nodes.iter().map(|n| n.id.as_str()).collect(); + assert_eq!( + ids, + ["fov", "video-settings", "done", "done-2", "row-0", "row-1"] + ); + assert_eq!(nodes[0].kind, slider, "the target's kind carries the value"); + assert_eq!(nodes[4].label, None); + assert_eq!(slug("Music & Sounds..."), "music-sounds"); + assert_eq!(slug("Difficulty: Normal"), "difficulty"); + assert_eq!(slug("Économie"), "économie"); + } + + #[test] + fn last_lists_buttons_only() { + end_frame(100); + record("Back", [0.0, 0.0, 5.0, 5.0], true); + record_as("FOV: 70", [0.0, 9.0, 5.0, 5.0], true, Kind::Tab(true)); + end_frame(101); + assert_eq!(last().len(), 1); + assert_eq!(last_labelled().len(), 2); + } + #[test] fn frames_rotate_only_when_drawn() { record("Respawn", [1.0, 2.0, 3.0, 4.0], true); diff --git a/docs/native-client.md b/docs/native-client.md index e51bd8e40..7f0b3239a 100644 --- a/docs/native-client.md +++ b/docs/native-client.md @@ -96,6 +96,8 @@ $C inventory # hotbar, inventory, armor, offhand $C open-inventory # craft planks in the 2x2 grid: $C click-slot 0; $C click-slot 100 right; $C click-slot 0; $C click-slot 200 shift; $C close $C place 2 65 -2 --item birch_planks # {"changed": true, "block": {...}} +$C pause; $C ui # the pause menu's widgets: ids, kinds, labels, bounds +$C click options; $C set fov 0.75; $C click done # widgets by id $C screenshot /tmp/view.png # PNG of the next presented frame $C quit # save and exit ``` @@ -116,6 +118,16 @@ game accepts, test hooks included. `look-at` a point, block face or entity, `attack [ID]`, `break X Y Z`, `place X Y Z [--item ITEM]`, `interact X Y Z` (doors, chests, crafting tables), `use-item [--hold S]` (eat, drink, throw), `select SLOT|ITEM`, `drop`. +- **Widgets:** `ui` lists every widget of the open screen (title menus, pause, + options pages in both menu styles, inventories and menus): an id, its kind + (`button`, `slider` with its 0..1 `value`, `field`, `row`, `tab` with + `selected`, `slot` with its stack), the label and its bounds. Ids come from + labels (`FOV: 70` is `fov`, `Video Settings...` is `video-settings`, a + world row is the world's name; a repeat gets `-2`), else `row-0`, `field-0`, + and slots are `slot-ID`. `click ID [right] [shift]` clicks one as the mouse + would and `set ID VALUE` moves a slider to 0..1; after a click, the next + `ui`, `click` or `set` waits for the screen to be redrawn. Design and plans: + [agent-control.md](agent-control.md). - **Screens:** `open-inventory`, `click-slot SLOT [right] [shift]` (0-8 hotbar, 9-35 inventory, 100+ crafting grid, 200 result, 300+ containers, 500-503 armor, 504 offhand, 600+ stations), `close`, `click-button LABEL` (any diff --git a/tools/agent_control_e2e.py b/tools/agent_control_e2e.py index 5d56398bc..0294863c8 100644 --- a/tools/agent_control_e2e.py +++ b/tools/agent_control_e2e.py @@ -1,16 +1,19 @@ #!/usr/bin/env python3 -"""End-to-end check of agent control: play Horizon on a virtual screen with horizon-ctl. +"""End-to-end check of agent control: play Horizon with horizon-ctl. A script creates a world through the title menus, finds a tree it can walk to, breaks a log, checks the inventory, crafts planks in the inventory screen, places one and reads it back. It runs through the CLI (one horizon-ctl process per command) and through `horizon-ctl mcp` with a -small JSON-RPC client, and saves a screenshot of each run. +small JSON-RPC client, and saves a screenshot of each run. The ui scenario drives the menus by +widget id instead: the title screen, the options (a slider set by id) and the inventory's slots. python3 tools/agent_control_e2e.py # both modes python3 tools/agent_control_e2e.py --mode mcp # one of them python3 tools/agent_control_e2e.py --bin-dir target/fast --out /tmp/agent-e2e + python3 tools/agent_control_e2e.py --scenario ui # menus by widget id only (a minute) -Needs vscreen (Linux desktop), the prepared resources in .local/horizon-client and a build of +Needs vscreen on Linux (the desktop); on macOS the game renders offscreen, so it also runs in a +locked session or over SSH. It needs the prepared resources in .local/horizon-client and a build of horizon-client and horizon-ctl (built with the fast profile unless --bin-dir is given). Games started here have no audio (HORIZON_NO_AUDIO=1). """ @@ -35,17 +38,20 @@ POSITIONAL = { 'goto': ['x', 'y', 'z'], 'break': ['x', 'y', 'z'], 'place': ['x', 'y', 'z'], 'block': ['x', 'y', 'z'], 'find_block': ['block', 'radius'], 'entities': ['radius', 'type'], 'click_slot': ['slot'], 'click_button': ['label'], 'type': ['text'], 'wait': ['ticks'], 'wait_ready': ['seconds'], - 'command': ['command'], 'look_at_block': ['x', 'y', 'z'], + 'command': ['command'], 'look_at_block': ['x', 'y', 'z'], 'click': ['id'], 'set': ['id', 'value'], + 'give': ['item', 'count'], } # Booleans the game takes as bare words. -BARE = {'click_slot': ['right', 'shift']} +BARE = {'click_slot': ['right', 'shift'], 'click': ['right', 'shift']} class Failed(Exception): pass -def vscreen_env(screen): +def game_env(screen): + if sys.platform == 'darwin': + return dict(os.environ, HORIZON_NO_AUDIO='1') env = dict(os.environ, VSCREEN=screen) subprocess.run(['vscreen', 'up'], env=env, check=False, capture_output=True) text = subprocess.run(['vscreen', 'env'], env=env, capture_output=True, text=True, check=True).stdout @@ -166,7 +172,8 @@ def scenario(d, out): assert screen['screen'] == 'title', screen assert any(b['label'] == 'Singleplayer' for b in screen['buttons']), screen d('click_button', label='Singleplayer') - d('click_button', label='Create New World') + if d('screen')['menu'].get('mode') != 'create-world': # with no worlds, Singleplayer opens it itself + d('click_button', label='Create New World') d('type', field='name', replace=True, text='Agent World') d('type', field='seed', text='5') # spawn near a walkable oak created = d('click_button', label='Create New World') @@ -243,18 +250,79 @@ def scenario(d, out): return {'tree': base, 'log': log_id, 'placed': target} -def run(mode, ctl, out, resources): +def widgets(d): + """The ui tree as {id: widget}.""" + return {w['id']: w for w in d('ui')['widgets']} + + +def ui_scenario(d, out): + # 1. Create a world through the title menus, by widget id. + tree = widgets(d) + assert tree['singleplayer']['kind'] == 'button', tree + d('click', id='singleplayer') + if 'game' not in widgets(d): # with no worlds, Singleplayer opens Create New World itself + d('click', id='create-new-world') + tree = widgets(d) + assert tree['game']['kind'] == 'button' and any(w['kind'] == 'field' for w in tree.values()), tree + d('type', field='name', replace=True, text='UI World') + created = d('click', id='create-new-world') + assert created['screen'] == 'playing', created + d('wait_ready', seconds=400) + + # 2. Options from the pause menu: a slider set by id moves its value and label. + d('pause') + d('click', id='options') + fov = widgets(d)['fov'] + assert fov['kind'] == 'slider' and abs(fov['value'] - 0.5) < 0.01, fov + d('set', id='fov', value=1) + moved = widgets(d)['fov'] + assert moved['value'] > 0.99 and moved['label'] != fov['label'], moved + d('set', id='fov', value=0.5) + try: + d('click', id='no-such-widget') + raise AssertionError('clicking an unknown id succeeded') + except Failed as e: + assert 'no widget' in str(e), e + try: + d('set', id='done', value=0.5) + raise AssertionError('set accepted a button') + except Failed as e: + assert 'set moves sliders' in str(e), e + d('click', id='done') + assert d('click', id='back-to-game')['screen'] == 'playing' + + # 3. The inventory's slots are widgets too, with their stacks. + d('give', item='oak_log', count=2) + d('open_inventory') + tree = widgets(d) + log = next(w for w in tree.values() if w['kind'] == 'slot' and (w.get('stack') or {}).get('item') == 'minecraft:oak_log') + assert log['stack']['count'] == 2 and 'slot-200' in tree, log + d('click', id=log['id']) + d('click', id='slot-100', right=True) + d('click', id=log['id']) + assert widgets(d)['slot-200']['stack']['item'] == 'minecraft:oak_planks' + d('close') + shot = d('screenshot', path=str(out / f'{d.name}-ui.png')) + if '_image' in shot: + (out / f'{d.name}-ui.png').write_bytes(shot['_image']) + return {'widgets': len(tree)} + + +SCENARIOS = {'survival': scenario, 'ui': ui_scenario} + + +def run(mode, ctl, out, resources, which='survival'): work = Path(tempfile.mkdtemp(prefix=f'hzac-{mode}-')) (work / '.local').mkdir() (work / '.local/horizon-client').mkdir() os.symlink(resources, work / '.local/horizon-client/26.3') socket = work / 'c.sock' - env = vscreen_env(os.environ.get('VSCREEN', 'agent-control')) + env = game_env(os.environ.get('VSCREEN', 'agent-control')) driver = (Cli if mode == 'cli' else Mcp)(ctl, socket, work, env) started = time.time() try: - driver('launch', wait=False) - summary = scenario(driver, out) + driver('launch', wait=False, offscreen=sys.platform == 'darwin') + summary = SCENARIOS[which](driver, out) print(f'[{mode}] PASS in {time.time() - started:.0f} s: {summary}') return True except (Failed, AssertionError, StopIteration) as e: @@ -276,6 +344,7 @@ def run(mode, ctl, out, resources): def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--mode', choices=['cli', 'mcp', 'both'], default='both') + parser.add_argument('--scenario', choices=[*SCENARIOS, 'all'], default='all') parser.add_argument('--bin-dir', help='prebuilt horizon-client and horizon-ctl (default: build with the fast profile)') parser.add_argument('--out', default=str(ROOT / '.local/agent-control-e2e'), help='screenshots go here') args = parser.parse_args() @@ -293,7 +362,8 @@ def main(): out = Path(args.out) out.mkdir(parents=True, exist_ok=True) modes = ['cli', 'mcp'] if args.mode == 'both' else [args.mode] - results = {m: run(m, bins / 'horizon-ctl', out, resources) for m in modes} + scenarios = list(SCENARIOS) if args.scenario == 'all' else [args.scenario] + results = {f'{m}/{s}': run(m, bins / 'horizon-ctl', out, resources, s) for m in modes for s in scenarios} print(json.dumps(results)) sys.exit(0 if all(results.values()) else 1)