Merge pull request #349 from ludovic111/feat/agent-ui-tree

Agent control: the widget tree of every screen, click and set by id
This commit is contained in:
Ludovic Marie
2026-10-15 17:33:58 +02:00
committed by GitHub
15 changed files with 604 additions and 61 deletions
@@ -145,7 +145,8 @@ pub const COMMANDS: &[Cmd] = &[
["horizon-ctl launch --world Survival --seed 42 --wait", "horizon-ctl launch --world Flat --creative --render-distance 8"],
&[flag("world", Ty::Str, "world folder to open or create"), flag("seed", Ty::Str, "seed for a new world"), flag("creative", Ty::Bool, "creative mode for a new world"),
flag("version", Ty::Str, "an installed launcher version tag such as v2026.10.13"), flag("game", Ty::Str, "path to a horizon-client binary"),
flag("wait", Ty::Bool, "also wait until the world is ready (default in MCP)")],
flag("wait", Ty::Bool, "also wait until the world is ready (default in MCP)"),
flag("offscreen", Ty::Bool, "render without showing a window (screenshots still work): locked or headless sessions")],
mcp: true, local: true),
cmd!("wait-ready", Session, "wait-ready [SECONDS]", "wait until the player has spawned and no chunks are pending",
"Polls status every 250 ms; replies with the status, or an error after SECONDS (default 120).",
@@ -266,6 +267,17 @@ pub const COMMANDS: &[Cmd] = &[
["horizon-ctl click-slot 36", "horizon-ctl click-slot 100 right", "horizon-ctl click-slot 200 shift"],
&[p("slot", Ty::Int, "slot id"), opt("right", Ty::Bool, "right-click"), opt("shift", Ty::Bool, "shift-click")], mcp: true, local: false),
cmd!("close", Screens, "close", "close the open menu and return to the game", "", ["horizon-ctl close"], &[], mcp: true, local: false),
cmd!("ui", Screens, "ui", "every widget of the open screen: id, kind, label, bounds, value",
"Kinds: button, slider (value 0..1), field, row (worlds, servers, packs, languages), tab (selected), slot (slot-ID, with its stack). Ids come from labels (\"FOV: 70\" is fov, a repeat gets -2) or kind-N without one, and stay put when a value changes. Reflects the last drawn frame (frame counts them).",
["horizon-ctl ui"], &[], mcp: true, local: false),
cmd!("click", Screens, "click ID [right] [shift]", "click a widget of ui by its id, as the mouse would",
"A field takes focus (then type); slot-ID clicks a slot like click-slot.",
["horizon-ctl click done", "horizon-ctl click video-settings", "horizon-ctl click row-0", "horizon-ctl click slot-0 shift"],
&[p("id", Ty::Str, "widget id from ui"), opt("right", Ty::Bool, "right-click"), opt("shift", Ty::Bool, "shift-click (slots)")], mcp: true, local: false),
cmd!("set", Screens, "set ID VALUE", "move a slider of ui to VALUE (0..1), as a click on its track",
"The option behind it rounds to its own steps; read the new label with ui.",
["horizon-ctl set fov 0.5", "horizon-ctl set render-distance 0"],
&[p("id", Ty::Str, "slider id from ui"), p("value", Ty::Num, "0..1 along the track")], mcp: true, local: false),
cmd!("click-button", Screens, "click-button LABEL", "click a button of the open screen or title menu by its label",
"Case-insensitive; a unique part of the label is enough. screen lists the buttons.",
["horizon-ctl click-button Singleplayer", "horizon-ctl click-button \"Create New World\"", "horizon-ctl click-button respawn"],
+4
View File
@@ -118,6 +118,7 @@ mod totem;
mod trading;
mod trial_spawner;
mod trident;
mod ui_control;
mod villagers;
mod wandering_trader;
mod wind;
@@ -946,6 +947,8 @@ struct Game<'a> {
flight: Option<control::Flight>,
/// The running agent action from the control socket (game/agent.rs).
agent: agent::Agent,
/// Widget commands waiting for the frame after a click (game/ui_control.rs).
ui_wait: ui_control::UiWait,
/// The overworld's last `sky_darken` and its inputs (time, rain, thunder).
sky_darken_memo: std::cell::Cell<Option<((i64, u32, u32), i32)>>,
persist: persist::EntityMemory,
@@ -9809,6 +9812,7 @@ pub fn run() -> Result<()> {
pending_screenshot: None,
pending_bench: None,
flight: None,
ui_wait: Default::default(),
agent: agent::Agent::default(),
view_frustum: None,
pending_uploads: Vec::new(),
+3 -3
View File
@@ -314,7 +314,7 @@ impl Game<'_> {
"loaded": self.world.chunks.contains_key(&(pos[0] >> 4, pos[2] >> 4)),
})
}
fn stack_value(&self, s: &ItemStack) -> Value {
pub(super) fn stack_value(&self, s: &ItemStack) -> Value {
let mut v = json!({ "item": &*s.item, "count": s.count });
if s.damage > 0 {
v["damage"] = json!(s.damage);
@@ -356,7 +356,7 @@ impl Game<'_> {
}
Ok(())
}
fn screen_label(&self) -> &'static str {
pub(super) fn screen_label(&self) -> &'static str {
if self.title.is_some() {
return "title";
}
@@ -782,7 +782,7 @@ impl Game<'_> {
}
/// Synthesize the pointer moving to GUI pixel (x, y) and a click there.
fn synth_click(
pub(super) fn synth_click(
&mut self,
event_loop: &ActiveEventLoop,
x: f32,
+21 -8
View File
@@ -97,15 +97,22 @@ impl Game<'_> {
}
horizon_client::widgets::end_frame(self.frames);
self.advance_agent();
while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) {
let reply = match self.control_command(&request.line, &request.reply, event_loop) {
Ok(Some(value)) => value,
// Deferred: the command keeps its own reply channel.
Ok(None) => continue,
Err(e) => json!({ "error": e.to_string() }),
};
let _ = request.reply.send(reply.to_string());
for request in self.ui_wait.ready() {
self.answer(request, event_loop);
}
while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) {
self.answer(request, event_loop);
}
}
fn answer(&mut self, request: horizon_client::control::Request, event_loop: &ActiveEventLoop) {
let reply = match self.control_command(&request.line, &request.reply, event_loop) {
Ok(Some(value)) => value,
// Deferred: the command keeps its own reply channel.
Ok(None) => return,
Err(e) => json!({ "error": e.to_string() }),
};
let _ = request.reply.send(reply.to_string());
}
fn control_command(
@@ -125,6 +132,11 @@ impl Game<'_> {
if let Some(value) = self.block_command(&words)? {
return Ok(Some(value));
}
match self.ui_command(&words, line, reply, event_loop)? {
super::agent::Handled::Reply(value) => return Ok(Some(value)),
super::agent::Handled::Deferred => return Ok(None),
super::agent::Handled::No => {}
}
if let Some(value) = self.lan_control(&words)? {
return Ok(Some(value));
}
@@ -146,6 +158,7 @@ impl Game<'_> {
.chain(super::render_control::HELP)
.chain(super::sculk::HELP)
.chain(super::agent::HELP)
.chain(super::ui_control::HELP)
.chain(super::pad::HELP)
.collect::<Vec<_>>() }))),
"setblock" | "fill" | "particle" | "signedit" => self.render_command(&words),
+1 -1
View File
@@ -101,7 +101,7 @@ impl Game<'_> {
|| self.book.is_some()
}
fn gui_scale(&self) -> Option<HudBatch> {
pub(super) fn gui_scale(&self) -> Option<HudBatch> {
self.renderer
.as_ref()
.map(|r| HudBatch::new([r.config.width, r.config.height]))
@@ -56,7 +56,7 @@ impl Game<'_> {
}
self.cursor = saved;
}
fn gui_origin(&self) -> Result<(f32, f32)> {
pub(super) fn gui_origin(&self) -> Result<(f32, f32)> {
let size = self
.renderer
.as_ref()
@@ -0,0 +1,206 @@
//! The widget tree of the open screen for agents (`ui`, `click ID`, `set ID
//! VALUE`; docs/agent-control.md): what the last frame drew, with ids, and
//! clicks that go through the same window events as the mouse.
use super::{Game, Screen, agent::Handled};
use horizon_client::{
Result,
control::Request,
widgets::{self, Kind, Node},
};
use serde_json::{Value, json};
use std::{
sync::mpsc::Sender,
time::{Duration, Instant},
};
use winit::{event::MouseButton, event_loop::ActiveEventLoop};
pub(super) const HELP: &[&str] = &[
"ui every widget of the open screen: id, kind, label, bounds, value",
"click ID [right] [shift] click a widget of ui by its id (slot-N: as click-slot N)",
"set ID VALUE move a slider to VALUE (0..1), as a click on its track",
];
/// After a click, `ui`, `click` and `set` wait for a frame drawn since, so
/// they never read the screen the click left.
#[derive(Default)]
pub(super) struct UiWait {
/// The widget generation at the last click, and when it happened.
after: Option<(u64, Instant)>,
waiting: Vec<Request>,
}
impl UiWait {
fn stale(&self) -> bool {
self.after.is_some_and(|(generation, at)| {
generation == widgets::generation() && at.elapsed() < Duration::from_secs(1)
})
}
/// The commands that can run now.
pub(super) fn ready(&mut self) -> Vec<Request> {
if self.stale() {
return Vec::new();
}
self.after = None;
std::mem::take(&mut self.waiting)
}
fn clicked(&mut self) {
self.after = Some((widgets::generation(), Instant::now()));
}
}
impl Game<'_> {
/// The open screen's widgets: those the last frame drew, then the slots
/// of an open inventory or menu (`slot-ID`, the ids of `click-slot`).
fn ui_nodes(&self) -> Vec<Node> {
let size = self.gui_scale().map(|b| [b.width, b.height]);
let mut nodes = widgets::tree(&widgets::last_labelled(), &widgets::last_targets(), size);
if self.screen == Screen::Inventory
&& self.title.is_none()
&& let Ok((ox, oy)) = self.gui_origin()
{
nodes.extend(self.slots().into_iter().map(|i| {
let (x, y) = self.slot_position(i);
Node {
id: format!("slot-{i}"),
kind: Kind::Slot,
label: None,
rect: [ox + x, oy + y, 16.0, 16.0],
enabled: true,
}
}));
}
nodes
}
fn node_value(&mut self, node: &Node) -> Value {
let mut v = json!({
"id": node.id,
"kind": node.kind.name(),
"bounds": node.rect,
"enabled": node.enabled,
});
if let Some(label) = &node.label {
v["label"] = json!(label);
}
match node.kind {
Kind::Slider { value, .. } => v["value"] = json!(value),
Kind::Tab(selected) => v["selected"] = json!(selected),
Kind::Slot => {
if let Some(slot) = node.id.strip_prefix("slot-").and_then(|s| s.parse().ok()) {
let stack = if slot == 200 {
self.craft_result()
} else {
self.slot_mut(slot).clone()
};
v["stack"] = stack.map_or(Value::Null, |s| self.stack_value(&s));
}
}
_ => {}
}
v
}
fn find_node(&self, id: &str) -> Result<Node> {
let nodes = self.ui_nodes();
nodes.iter().find(|n| n.id == id).cloned().ok_or_else(|| {
let ids: Vec<&str> = nodes.iter().map(|n| n.id.as_str()).collect();
format!(
"no widget {id:?} on the {} screen; widgets: {ids:?}",
self.screen_label()
)
.into()
})
}
/// `ui`, `click` and `set`; `Handled::No` for other commands.
pub(super) fn ui_command(
&mut self,
words: &[&str],
line: &str,
reply: &Sender<String>,
event_loop: &ActiveEventLoop,
) -> Result<Handled> {
if !matches!(words[0], "ui" | "click" | "set") {
return Ok(Handled::No);
}
if self.ui_wait.stale() {
self.ui_wait.waiting.push(Request {
line: line.to_owned(),
reply: reply.clone(),
});
return Ok(Handled::Deferred);
}
let id = || words.get(1).copied().ok_or("missing the widget id; see ui");
match words[0] {
"ui" => {
let nodes = self.ui_nodes();
let widgets: Vec<Value> = nodes.iter().map(|n| self.node_value(n)).collect();
let size = self
.gui_scale()
.map(|b| [b.width.round(), b.height.round()]);
Ok(Handled::Reply(json!({
"screen": self.screen_label(),
"gui_size": size,
"frame": widgets::generation(),
"widgets": widgets,
})))
}
"click" => {
let node = self.find_node(id()?)?;
if !node.enabled {
return Err(format!("widget {:?} is disabled", node.id).into());
}
let right = words.contains(&"right");
if node.kind == Kind::Slot {
let slot = node.id.trim_start_matches("slot-");
let mut click = vec!["click-slot", slot];
click.extend(words.iter().filter(|w| matches!(**w, "right" | "shift")));
self.station_command(&click)?;
} else {
let (x, y) = (
node.rect[0] + node.rect[2] / 2.0,
node.rect[1] + node.rect[3] / 2.0,
);
let button = if right {
MouseButton::Right
} else {
MouseButton::Left
};
self.synth_click(event_loop, x, y, button)?;
}
self.ui_wait.clicked();
Ok(Handled::Reply(
json!({ "ok": true, "clicked": node.id, "screen": self.screen_label() }),
))
}
"set" => {
let node = self.find_node(id()?)?;
let value: f32 = words
.get(2)
.ok_or("set ID VALUE: missing the value")?
.parse()
.map_err(|_| "set ID VALUE: the value is a number from 0 to 1")?;
let Kind::Slider { track, .. } = node.kind else {
return Err(format!(
"widget {:?} is a {}; set moves sliders (click a button, type into a field)",
node.id,
node.kind.name()
)
.into());
};
if !node.enabled {
return Err(format!("widget {:?} is disabled", node.id).into());
}
// AbstractSliderButton.setValueFromMouse: (x - track start) / track width.
let x = track[0] + value.clamp(0.0, 1.0) * track[1];
let y = node.rect[1] + node.rect[3] / 2.0;
self.synth_click(event_loop, x, y, MouseButton::Left)?;
self.ui_wait.clicked();
Ok(Handled::Reply(
json!({ "ok": true, "set": node.id, "screen": self.screen_label() }),
))
}
_ => Ok(Handled::No),
}
}
}
+9 -4
View File
@@ -455,10 +455,15 @@ impl Screen {
&& y + ROW_HEIGHT / 2.0 > top
&& y + ROW_HEIGHT / 2.0 < bottom
{
crate::widgets::target_as(
[left, y, ROW_WIDTH, ROW_HEIGHT],
crate::widgets::Kind::Row,
);
let rect = [left, y, ROW_WIDTH, ROW_HEIGHT];
// The control socket's widget tree names the row.
let name = match row {
Row::Server(k) => servers[k].name.as_str(),
Row::Lan(k) => games[k].motd.as_str(),
Row::LanHeader => "",
};
crate::widgets::record_as(name, rect, true, crate::widgets::Kind::Row);
crate::widgets::target_as(rect, crate::widgets::Kind::Row);
}
// extractSelection: a white outline round black (the list
// has focus once a row is clicked).
+8 -1
View File
@@ -1605,7 +1605,14 @@ impl OptionsUi {
WKind::LanguageRow {
label, selected, ..
} => {
crate::widgets::target_as([wd.x, wd.y, wd.w, wd.h], crate::widgets::Kind::Row);
let rect = [wd.x, wd.y, wd.w, wd.h];
crate::widgets::record_as(
&label.plain(),
rect,
true,
crate::widgets::Kind::Row,
);
crate::widgets::target_as(rect, crate::widgets::Kind::Row);
if *selected {
batch.fill(atlas, wd.x, wd.y, wd.w, wd.h, u32::MAX);
batch.fill(
+7 -2
View File
@@ -826,10 +826,15 @@ impl Title {
}
let mid = y - 2.0 + ROW_HEIGHT / 2.0;
if mid > top && mid < bottom {
crate::widgets::target_as(
[x - 2.0, y - 2.0, ROW_WIDTH, ROW_HEIGHT],
let rect = [x - 2.0, y - 2.0, ROW_WIDTH, ROW_HEIGHT];
// The control socket's widget tree names the row.
crate::widgets::record_as(
&world.name,
rect,
true,
crate::widgets::Kind::Row,
);
crate::widgets::target_as(rect, crate::widgets::Kind::Row);
}
// extractSelection: white once the list has focus.
if *selected == Some(i) {
+7 -7
View File
@@ -283,15 +283,15 @@ pub fn slider(
hover: bool,
active: bool,
) {
let kind = crate::widgets::Kind::Slider {
value,
track: [x + 4.0, w - 8.0],
};
// The control socket's widget tree (`ui`): its label and value.
crate::widgets::record_as(&label.plain(), [x, y, w, 20.0], active, kind);
if active {
// A controller's D-pad can focus it (Horizon's controller support).
crate::widgets::target_as(
[x, y, w, 20.0],
crate::widgets::Kind::Slider {
value,
track: [x + 4.0, w - 8.0],
},
);
crate::widgets::target_as([x, y, w, 20.0], kind);
}
let track = if hover && active {
"gui/sprites/widget/slider_highlighted"
+53 -16
View File
@@ -458,6 +458,9 @@ pub struct Ui<'a> {
/// The number the next settings row shows (`01`, `02`…); `page_header`
/// starts it again.
pub row_index: usize,
/// The caption of the settings row drawn last: the control socket's
/// label for the switch, slider or segments drawn on it.
row_label: String,
/// Secondary text (indices, inactive items, captions): `FAINT`, or `MUTED` over the home
/// page's lighter veil (`horizon_theme::HOME_VEIL`).
pub muted: u32,
@@ -491,6 +494,7 @@ impl<'a> Ui<'a> {
opacity: 1.0,
offset: (0.0, 0.0),
row_index: 0,
row_label: String::new(),
muted: color::FAINT,
}
}
@@ -953,6 +957,25 @@ impl<'a> Ui<'a> {
self.mark(22.0, 26.0, 18.0);
self.text_line(Face::Medium, BODY, 18.0, 48.0, 26.0, color::TEXT, title);
}
/// A widget's label for the control socket (`click-button`, `ui`), in GUI pixels.
fn record_label(
&self,
label: &str,
[x, y, w, h]: [f32; 4],
enabled: bool,
kind: crate::widgets::Kind,
) {
if self.real {
let k = self.density / self.scale;
let (ox, oy) = self.offset;
crate::widgets::record_as(
label,
[(x + ox) * k, (y + oy) * k, w * k, h * k],
enabled,
kind,
);
}
}
/// The launcher's `button`: a pill 32 tall (or `CONTROL_LARGE` with
/// `LARGE` text), medium weight, a hover layer easing in over `QUICK`,
/// and the press state. Returns the hover.
@@ -964,12 +987,7 @@ impl<'a> Ui<'a> {
kind: Kind,
enabled: bool,
) -> f32 {
if self.real {
// The control socket's button list, in GUI pixels.
let k = self.density / self.scale;
let (ox, oy) = self.offset;
crate::widgets::record(label, [(x + ox) * k, (y + oy) * k, w * k, h * k], enabled);
}
self.record_label(label, [x, y, w, h], enabled, crate::widgets::Kind::Button);
let hover = self.interact(id, x, y, w, h, enabled);
let saved = self.opacity;
if !enabled {
@@ -1043,6 +1061,12 @@ impl<'a> Ui<'a> {
for (i, label) in labels.iter().enumerate() {
let iy = y + i as f32 * STEP;
let active = i == selected;
self.record_label(
label,
[x, iy, w, CONTROL],
true,
crate::widgets::Kind::Tab(active),
);
let hover = self.interact_as(
base + i as u64,
[x, iy, w, CONTROL],
@@ -1089,6 +1113,13 @@ impl<'a> Ui<'a> {
let mut sx = x + 2.0;
for (i, (label, sw)) in labels.iter().zip(&widths).enumerate() {
let active = i == selected;
let name = format!("{} {label}", self.row_label);
self.record_label(
&name,
[sx, y + 2.0, *sw, SEGMENT],
true,
crate::widgets::Kind::Tab(active),
);
let hover = self.interact(base + i as u64, sx, y + 2.0, *sw, SEGMENT, true);
if active {
self.rounded(sx, y + 2.0, *sw, SEGMENT, PILL, color::TEXT);
@@ -1117,7 +1148,12 @@ impl<'a> Ui<'a> {
pub fn switch(&mut self, id: u64, right: f32, cy: f32, on: bool) {
let (w, h) = (36.0, 20.0);
let (x, y) = (right - w, (cy - h / 2.0).round());
self.interact(id, x - 4.0, y - 4.0, w + 8.0, h + 8.0, true);
let rect = [x - 4.0, y - 4.0, w + 8.0, h + 8.0];
if !self.row_label.is_empty() {
let label = format!("{}: {}", self.row_label, if on { "on" } else { "off" });
self.record_label(&label, rect, true, crate::widgets::Kind::Button);
}
self.interact_as(id, rect, true, crate::widgets::Kind::Button);
let t = self.tween(id ^ 0x5717c4, if on { 1.0 } else { 0.0 }, move_());
self.rounded(
x,
@@ -1142,6 +1178,7 @@ impl<'a> Ui<'a> {
/// the value column as before the numbers came; the index takes its
/// share from it.
pub fn row(&mut self, x: f32, y: f32, label: &str, value: &str, value_w: f32) -> f32 {
self.row_label = label.to_owned();
let line = (BODY * 1.45).round();
let h = 12.0 + line.max(CONTROL) + 12.0;
let ty = y + (h - line) / 2.0;
@@ -1173,6 +1210,7 @@ impl<'a> Ui<'a> {
description: &str,
description_w: f32,
) -> f32 {
self.row_label = label.to_owned();
let h = 12.0 + CONTROL + 12.0;
let top = y + 8.0;
self.index(x, top);
@@ -1218,15 +1256,14 @@ impl<'a> Ui<'a> {
/// Dragging is the screen's job (see [`slider_value`]).
pub fn slider(&mut self, id: u64, x: f32, cy: f32, w: f32, value: f32) {
let v = value.clamp(0.0, 1.0);
let hover = self.interact_as(
id,
[x - 8.0, cy - 12.0, w + 16.0, 24.0],
true,
crate::widgets::Kind::Slider {
value: v,
track: [x, w],
},
);
let kind = crate::widgets::Kind::Slider {
value: v,
track: [x, w],
};
let rect = [x - 8.0, cy - 12.0, w + 16.0, 24.0];
let label = self.row_label.clone();
self.record_label(&label, rect, true, kind);
let hover = self.interact_as(id, rect, true, kind);
self.rounded(x, cy - 1.0, w, 2.0, 1.0, color::BORDER);
self.rounded(x, cy - 1.0, w * v, 2.0, 1.0, color::TEXT);
let r = 7.0 + hover;
+177 -5
View File
@@ -1,9 +1,10 @@
//! The labelled buttons drawn in the last frame, for the control socket: a
//! script clicks a button by its label (`horizon-ctl click-button Respawn`)
//! whatever screen or menu style drew it.
//! The labelled widgets and focus targets drawn in the last frame, for the
//! control socket: a script clicks a button by its label (`horizon-ctl
//! click-button Respawn`) or any widget by its id in [`tree`] (`horizon-ctl
//! click done`), whatever screen or menu style drew it.
use std::cell::RefCell;
/// A button in GUI pixels (the coordinates screens take input in).
/// A labelled widget in GUI pixels (the coordinates screens take input in).
#[derive(Clone, Debug)]
pub struct Widget {
pub label: String,
@@ -12,6 +13,8 @@ pub struct Widget {
pub w: f32,
pub h: f32,
pub enabled: bool,
/// Buttons for [`record`]; sliders and tabs carry their labels too.
pub kind: Kind,
}
/// What a controller does with a focus target (docs/controllers.md, "Focus
@@ -106,7 +109,12 @@ pub fn generation() -> u64 {
}
/// Called by the button drawers.
pub fn record(label: &str, [x, y, w, h]: [f32; 4], enabled: bool) {
pub fn record(label: &str, rect: [f32; 4], enabled: bool) {
record_as(label, rect, enabled, Kind::Button);
}
/// The label of a widget of another kind (a slider's "FOV: 70", a tab).
pub fn record_as(label: &str, [x, y, w, h]: [f32; 4], enabled: bool, kind: Kind) {
let label = label.trim();
if label.is_empty() {
return;
@@ -120,6 +128,7 @@ pub fn record(label: &str, [x, y, w, h]: [f32; 4], enabled: bool) {
w,
h,
enabled,
kind,
});
}
});
@@ -142,9 +151,118 @@ pub fn end_frame(frames: u64) {
/// The buttons of the last drawn frame, top-most last.
pub fn last() -> Vec<Widget> {
WIDGETS.with_borrow(|(_, last, _)| {
last.iter()
.filter(|w| w.kind == Kind::Button)
.cloned()
.collect()
})
}
/// Every labelled widget of the last drawn frame, sliders and tabs included.
pub fn last_labelled() -> Vec<Widget> {
WIDGETS.with_borrow(|(_, last, _)| last.clone())
}
impl Kind {
/// The kind's name in the control socket's `ui` reply.
pub fn name(self) -> &'static str {
match self {
Kind::Button => "button",
Kind::Slider { .. } => "slider",
Kind::Field => "field",
Kind::Row => "row",
Kind::Slot => "slot",
Kind::Tab(_) => "tab",
}
}
}
/// A widget of the `ui` tree: a labelled widget and its focus target merged.
#[derive(Clone, Debug, PartialEq)]
pub struct Node {
/// The label as a slug up to its `:` (`FOV: 70` is `fov`), `-2`, `-3`
/// for repeats; `<kind>-<n>` in draw order without a label.
pub id: String,
pub kind: Kind,
pub label: Option<String>,
pub rect: [f32; 4],
pub enabled: bool,
}
/// `Video Settings...` → `video-settings`, `FOV: 70` → `fov`.
pub fn slug(label: &str) -> String {
let name = label.split(':').next().unwrap_or(label);
let mut out = String::new();
for c in name.chars().flat_map(char::to_lowercase) {
if c.is_alphanumeric() {
out.push(c);
} else if !out.is_empty() && !out.ends_with('-') {
out.push('-');
}
}
out.trim_end_matches('-').to_owned()
}
/// The labelled widgets and the focus targets of a frame as one list with
/// ids: a target drawn at a labelled widget's rectangle is that widget
/// (its kind wins, a slider's value, unless the label says it's a tab:
/// Horizon's segments); other targets follow, unlabelled. With the GUI
/// size, what lies off screen (layout probes) is left out before naming.
pub fn tree(widgets: &[Widget], targets: &[Target], screen: Option<[f32; 2]>) -> Vec<Node> {
let same = |a: [f32; 4], b: [f32; 4]| a.iter().zip(b).all(|(p, q)| (p - q).abs() < 0.5);
let mut nodes: Vec<Node> = widgets
.iter()
.map(|w| Node {
id: String::new(),
kind: w.kind,
label: Some(w.label.clone()),
rect: [w.x, w.y, w.w, w.h],
enabled: w.enabled,
})
.collect();
for t in targets {
if let Some(node) = nodes.iter_mut().find(|n| same(n.rect, t.rect)) {
if !matches!(node.kind, Kind::Tab(_)) {
node.kind = t.kind;
}
} else {
nodes.push(Node {
id: String::new(),
kind: t.kind,
label: None,
rect: t.rect,
enabled: true,
});
}
}
if let Some([w, h]) = screen {
nodes.retain(|n| {
let [x, y, nw, nh] = n.rect;
x >= -0.5 && y >= -0.5 && x + nw <= w + 0.5 && y + nh <= h + 0.5
});
}
let mut seen = std::collections::HashMap::<String, usize>::new();
for node in &mut nodes {
let base = node
.label
.as_deref()
.map(slug)
.filter(|s| !s.is_empty())
.unwrap_or_else(|| node.kind.name().to_owned());
let n = seen.entry(base.clone()).or_default();
*n += 1;
node.id = if node.label.is_none() {
format!("{base}-{}", *n - 1)
} else if *n > 1 {
format!("{base}-{n}")
} else {
base
};
}
nodes
}
/// The button `label` names: an exact match (case-insensitive), else the only
/// button whose label contains it. The top-most wins among equals.
pub fn find(widgets: &[Widget], label: &str) -> Result<Widget, String> {
@@ -185,6 +303,7 @@ mod tests {
w: 10.0,
h: 10.0,
enabled: true,
kind: Kind::Button,
}
}
@@ -197,6 +316,59 @@ mod tests {
assert!(find(&list, "options").is_err());
}
#[test]
fn tree_merges_labels_with_targets_and_names_them() {
let slider = Kind::Slider {
value: 0.25,
track: [4.0, 142.0],
};
let mut fov = w("FOV: 70");
fov.kind = slider;
let mut done = w("Done");
done.y = 30.0;
let mut done2 = w("Done");
done2.y = 60.0;
let widgets = [fov, w("Video Settings..."), done, done2];
let targets = [
Target {
rect: [0.0, 0.0, 10.0, 10.0],
kind: slider,
},
Target {
rect: [0.0, 90.0, 10.0, 10.0],
kind: Kind::Row,
},
Target {
rect: [0.0, 100.0, 10.0, 10.0],
kind: Kind::Row,
},
];
let mut probe = w("Done");
probe.x = -10_000.0;
let widgets = [&[probe][..], &widgets].concat();
let nodes = tree(&widgets, &targets, Some([400.0, 200.0]));
let ids: Vec<&str> = nodes.iter().map(|n| n.id.as_str()).collect();
assert_eq!(
ids,
["fov", "video-settings", "done", "done-2", "row-0", "row-1"]
);
assert_eq!(nodes[0].kind, slider, "the target's kind carries the value");
assert_eq!(nodes[4].label, None);
assert_eq!(slug("Music & Sounds..."), "music-sounds");
assert_eq!(slug("Difficulty: Normal"), "difficulty");
assert_eq!(slug("Économie"), "économie");
}
#[test]
fn last_lists_buttons_only() {
end_frame(100);
record("Back", [0.0, 0.0, 5.0, 5.0], true);
record_as("FOV: 70", [0.0, 9.0, 5.0, 5.0], true, Kind::Tab(true));
end_frame(101);
assert_eq!(last().len(), 1);
assert_eq!(last_labelled().len(), 2);
}
#[test]
fn frames_rotate_only_when_drawn() {
record("Respawn", [1.0, 2.0, 3.0, 4.0], true);
+12
View File
@@ -96,6 +96,8 @@ $C inventory # hotbar, inventory, armor, offhand
$C open-inventory # craft planks in the 2x2 grid:
$C click-slot 0; $C click-slot 100 right; $C click-slot 0; $C click-slot 200 shift; $C close
$C place 2 65 -2 --item birch_planks # {"changed": true, "block": {...}}
$C pause; $C ui # the pause menu's widgets: ids, kinds, labels, bounds
$C click options; $C set fov 0.75; $C click done # widgets by id
$C screenshot /tmp/view.png # PNG of the next presented frame
$C quit # save and exit
```
@@ -116,6 +118,16 @@ game accepts, test hooks included.
`look-at` a point, block face or entity, `attack [ID]`, `break X Y Z`,
`place X Y Z [--item ITEM]`, `interact X Y Z` (doors, chests, crafting
tables), `use-item [--hold S]` (eat, drink, throw), `select SLOT|ITEM`, `drop`.
- **Widgets:** `ui` lists every widget of the open screen (title menus, pause,
options pages in both menu styles, inventories and menus): an id, its kind
(`button`, `slider` with its 0..1 `value`, `field`, `row`, `tab` with
`selected`, `slot` with its stack), the label and its bounds. Ids come from
labels (`FOV: 70` is `fov`, `Video Settings...` is `video-settings`, a
world row is the world's name; a repeat gets `-2`), else `row-0`, `field-0`,
and slots are `slot-ID`. `click ID [right] [shift]` clicks one as the mouse
would and `set ID VALUE` moves a slider to 0..1; after a click, the next
`ui`, `click` or `set` waits for the screen to be redrawn. Design and plans:
[agent-control.md](agent-control.md).
- **Screens:** `open-inventory`, `click-slot SLOT [right] [shift]` (0-8 hotbar,
9-35 inventory, 100+ crafting grid, 200 result, 300+ containers, 500-503
armor, 504 offhand, 600+ stations), `close`, `click-button LABEL` (any
+82 -12
View File
@@ -1,16 +1,19 @@
#!/usr/bin/env python3
"""End-to-end check of agent control: play Horizon on a virtual screen with horizon-ctl.
"""End-to-end check of agent control: play Horizon with horizon-ctl.
A script creates a world through the title menus, finds a tree it can walk to, breaks a log,
checks the inventory, crafts planks in the inventory screen, places one and reads it back. It
runs through the CLI (one horizon-ctl process per command) and through `horizon-ctl mcp` with a
small JSON-RPC client, and saves a screenshot of each run.
small JSON-RPC client, and saves a screenshot of each run. The ui scenario drives the menus by
widget id instead: the title screen, the options (a slider set by id) and the inventory's slots.
python3 tools/agent_control_e2e.py # both modes
python3 tools/agent_control_e2e.py --mode mcp # one of them
python3 tools/agent_control_e2e.py --bin-dir target/fast --out /tmp/agent-e2e
python3 tools/agent_control_e2e.py --scenario ui # menus by widget id only (a minute)
Needs vscreen (Linux desktop), the prepared resources in .local/horizon-client and a build of
Needs vscreen on Linux (the desktop); on macOS the game renders offscreen, so it also runs in a
locked session or over SSH. It needs the prepared resources in .local/horizon-client and a build of
horizon-client and horizon-ctl (built with the fast profile unless --bin-dir is given). Games
started here have no audio (HORIZON_NO_AUDIO=1).
"""
@@ -35,17 +38,20 @@ POSITIONAL = {
'goto': ['x', 'y', 'z'], 'break': ['x', 'y', 'z'], 'place': ['x', 'y', 'z'], 'block': ['x', 'y', 'z'],
'find_block': ['block', 'radius'], 'entities': ['radius', 'type'], 'click_slot': ['slot'],
'click_button': ['label'], 'type': ['text'], 'wait': ['ticks'], 'wait_ready': ['seconds'],
'command': ['command'], 'look_at_block': ['x', 'y', 'z'],
'command': ['command'], 'look_at_block': ['x', 'y', 'z'], 'click': ['id'], 'set': ['id', 'value'],
'give': ['item', 'count'],
}
# Booleans the game takes as bare words.
BARE = {'click_slot': ['right', 'shift']}
BARE = {'click_slot': ['right', 'shift'], 'click': ['right', 'shift']}
class Failed(Exception):
pass
def vscreen_env(screen):
def game_env(screen):
if sys.platform == 'darwin':
return dict(os.environ, HORIZON_NO_AUDIO='1')
env = dict(os.environ, VSCREEN=screen)
subprocess.run(['vscreen', 'up'], env=env, check=False, capture_output=True)
text = subprocess.run(['vscreen', 'env'], env=env, capture_output=True, text=True, check=True).stdout
@@ -166,7 +172,8 @@ def scenario(d, out):
assert screen['screen'] == 'title', screen
assert any(b['label'] == 'Singleplayer' for b in screen['buttons']), screen
d('click_button', label='Singleplayer')
d('click_button', label='Create New World')
if d('screen')['menu'].get('mode') != 'create-world': # with no worlds, Singleplayer opens it itself
d('click_button', label='Create New World')
d('type', field='name', replace=True, text='Agent World')
d('type', field='seed', text='5') # spawn near a walkable oak
created = d('click_button', label='Create New World')
@@ -243,18 +250,79 @@ def scenario(d, out):
return {'tree': base, 'log': log_id, 'placed': target}
def run(mode, ctl, out, resources):
def widgets(d):
"""The ui tree as {id: widget}."""
return {w['id']: w for w in d('ui')['widgets']}
def ui_scenario(d, out):
# 1. Create a world through the title menus, by widget id.
tree = widgets(d)
assert tree['singleplayer']['kind'] == 'button', tree
d('click', id='singleplayer')
if 'game' not in widgets(d): # with no worlds, Singleplayer opens Create New World itself
d('click', id='create-new-world')
tree = widgets(d)
assert tree['game']['kind'] == 'button' and any(w['kind'] == 'field' for w in tree.values()), tree
d('type', field='name', replace=True, text='UI World')
created = d('click', id='create-new-world')
assert created['screen'] == 'playing', created
d('wait_ready', seconds=400)
# 2. Options from the pause menu: a slider set by id moves its value and label.
d('pause')
d('click', id='options')
fov = widgets(d)['fov']
assert fov['kind'] == 'slider' and abs(fov['value'] - 0.5) < 0.01, fov
d('set', id='fov', value=1)
moved = widgets(d)['fov']
assert moved['value'] > 0.99 and moved['label'] != fov['label'], moved
d('set', id='fov', value=0.5)
try:
d('click', id='no-such-widget')
raise AssertionError('clicking an unknown id succeeded')
except Failed as e:
assert 'no widget' in str(e), e
try:
d('set', id='done', value=0.5)
raise AssertionError('set accepted a button')
except Failed as e:
assert 'set moves sliders' in str(e), e
d('click', id='done')
assert d('click', id='back-to-game')['screen'] == 'playing'
# 3. The inventory's slots are widgets too, with their stacks.
d('give', item='oak_log', count=2)
d('open_inventory')
tree = widgets(d)
log = next(w for w in tree.values() if w['kind'] == 'slot' and (w.get('stack') or {}).get('item') == 'minecraft:oak_log')
assert log['stack']['count'] == 2 and 'slot-200' in tree, log
d('click', id=log['id'])
d('click', id='slot-100', right=True)
d('click', id=log['id'])
assert widgets(d)['slot-200']['stack']['item'] == 'minecraft:oak_planks'
d('close')
shot = d('screenshot', path=str(out / f'{d.name}-ui.png'))
if '_image' in shot:
(out / f'{d.name}-ui.png').write_bytes(shot['_image'])
return {'widgets': len(tree)}
SCENARIOS = {'survival': scenario, 'ui': ui_scenario}
def run(mode, ctl, out, resources, which='survival'):
work = Path(tempfile.mkdtemp(prefix=f'hzac-{mode}-'))
(work / '.local').mkdir()
(work / '.local/horizon-client').mkdir()
os.symlink(resources, work / '.local/horizon-client/26.3')
socket = work / 'c.sock'
env = vscreen_env(os.environ.get('VSCREEN', 'agent-control'))
env = game_env(os.environ.get('VSCREEN', 'agent-control'))
driver = (Cli if mode == 'cli' else Mcp)(ctl, socket, work, env)
started = time.time()
try:
driver('launch', wait=False)
summary = scenario(driver, out)
driver('launch', wait=False, offscreen=sys.platform == 'darwin')
summary = SCENARIOS[which](driver, out)
print(f'[{mode}] PASS in {time.time() - started:.0f} s: {summary}')
return True
except (Failed, AssertionError, StopIteration) as e:
@@ -276,6 +344,7 @@ def run(mode, ctl, out, resources):
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--mode', choices=['cli', 'mcp', 'both'], default='both')
parser.add_argument('--scenario', choices=[*SCENARIOS, 'all'], default='all')
parser.add_argument('--bin-dir', help='prebuilt horizon-client and horizon-ctl (default: build with the fast profile)')
parser.add_argument('--out', default=str(ROOT / '.local/agent-control-e2e'), help='screenshots go here')
args = parser.parse_args()
@@ -293,7 +362,8 @@ def main():
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
modes = ['cli', 'mcp'] if args.mode == 'both' else [args.mode]
results = {m: run(m, bins / 'horizon-ctl', out, resources) for m in modes}
scenarios = list(SCENARIOS) if args.scenario == 'all' else [args.scenario]
results = {f'{m}/{s}': run(m, bins / 'horizon-ctl', out, resources, s) for m in modes for s in scenarios}
print(json.dumps(results))
sys.exit(0 if all(results.values()) else 1)