Agent control: observe and act from the control socket

game/agent.rs adds commands an agent can play with: state, looking-at,
entities, block, blocks, find-block, inventory, screen and messages read the
game as JSON; move, jump, goto (A* over standable blocks), look-at, attack,
break, place, interact, use-item, select, drop, open-inventory, click-button,
click-at, command, respawn, pause, wait and stop act like a player. Actions
that take time run as a task advanced every frame and reply when done, with
what happened (break reports drops and pickups, place the block placed).

Buttons are clicked by label: the classic and Horizon button drawers record
what they draw each frame (widgets.rs), and a click is a synthesized pointer
move and press, so it behaves exactly like the mouse on every screen.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-13 21:57:09 +02:00
parent 2df60cd388
commit 7bb3347edd
8 changed files with 1866 additions and 0 deletions
+4
View File
@@ -2,6 +2,7 @@
//! block interaction, inventory and HUD.
mod advancements;
mod advancements_screen;
mod agent;
mod allays;
mod armor_stands;
mod backdrop;
@@ -806,6 +807,8 @@ struct Game<'a> {
view_frustum: Option<horizon_client::gfx::Frustum>,
/// A scripted flight from the control socket (`fly`).
flight: Option<control::Flight>,
/// The running agent action from the control socket (game/agent.rs).
agent: agent::Agent,
/// The overworld's last `sky_darken` and its inputs (time, rain, thunder).
sky_darken_memo: std::cell::Cell<Option<((i64, u32, u32), i32)>>,
persist: persist::EntityMemory,
@@ -8655,6 +8658,7 @@ pub fn run() -> Result<()> {
pending_screenshot: None,
pending_bench: None,
flight: None,
agent: agent::Agent::default(),
view_frustum: None,
pending_uploads: Vec::new(),
frame_graph: Default::default(),
File diff suppressed because it is too large Load Diff
@@ -87,6 +87,8 @@ impl Game<'_> {
self.pending_bench = Some((deadline, reply));
}
}
horizon_client::widgets::end_frame(self.frames);
self.advance_agent();
while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) {
let reply = match self.control_command(&request.line, &request.reply, event_loop) {
Ok(Some(value)) => value,
@@ -118,6 +120,11 @@ impl Game<'_> {
if let Some(value) = self.lan_control(&words)? {
return Ok(Some(value));
}
match self.agent_command(&words, line, reply, event_loop)? {
super::agent::Handled::Reply(value) => return Ok(Some(value)),
super::agent::Handled::Deferred => return Ok(None),
super::agent::Handled::No => {}
}
match words[0] {
"help" => Ok(Some(json!({ "commands": HELP
.iter()
@@ -126,6 +133,7 @@ impl Game<'_> {
.chain(super::lan::HELP)
.chain(super::render_control::HELP)
.chain(super::sculk::HELP)
.chain(super::agent::HELP)
.collect::<Vec<_>>() }))),
"setblock" | "fill" | "particle" | "signedit" => self.render_command(&words),
"sculk" => Ok(Some(self.sculk_command(&words)?)),
+1
View File
@@ -128,6 +128,7 @@ pub mod title;
pub mod ui;
pub mod ui_theme;
pub mod vanillagen;
pub mod widgets;
pub mod window;
pub mod world;
pub mod worldborder;
+1
View File
@@ -408,6 +408,7 @@ impl Title {
),
}
for b in self.buttons(w, h) {
crate::widgets::record(&b.label, [b.x, b.y, b.w, 20.0], b.enabled);
let hover = b.enabled
&& cursor.0 >= b.x
&& cursor.0 < b.x + b.w
+1
View File
@@ -247,6 +247,7 @@ pub fn button(
active: bool,
hover: bool,
) {
crate::widgets::record(&label.plain(), [x, y, w, h], active);
let sprite = if !active {
"gui/sprites/widget/button_disabled"
} else if hover {
+6
View File
@@ -845,6 +845,12 @@ impl<'a> Ui<'a> {
kind: Kind,
enabled: bool,
) -> f32 {
if self.real {
// The control socket's button list, in GUI pixels.
let k = self.density / self.scale;
let (ox, oy) = self.offset;
crate::widgets::record(label, [(x + ox) * k, (y + oy) * k, w * k, h * k], enabled);
}
let hover = self.interact(id, x, y, w, h, enabled);
let saved = self.opacity;
if !enabled {
+120
View File
@@ -0,0 +1,120 @@
//! The labelled buttons drawn in the last frame, for the control socket: a
//! script clicks a button by its label (`horizon-ctl click-button Respawn`)
//! whatever screen or menu style drew it.
use std::cell::RefCell;
/// A button in GUI pixels (the coordinates screens take input in).
#[derive(Clone, Debug)]
pub struct Widget {
pub label: String,
pub x: f32,
pub y: f32,
pub w: f32,
pub h: f32,
pub enabled: bool,
}
thread_local! {
/// Buttons drawn since the last `end_frame`, and the last frame's.
static WIDGETS: RefCell<(Vec<Widget>, Vec<Widget>, u64)> = const { RefCell::new((Vec::new(), Vec::new(), 0)) };
}
/// Called by the button drawers.
pub fn record(label: &str, [x, y, w, h]: [f32; 4], enabled: bool) {
let label = label.trim();
if label.is_empty() {
return;
}
WIDGETS.with_borrow_mut(|(current, _, _)| {
if current.len() < 512 {
current.push(Widget {
label: label.to_owned(),
x,
y,
w,
h,
enabled,
});
}
});
}
/// Once per event loop turn with the count of rendered frames: when a new
/// frame was drawn since the last call, its buttons become the last frame's.
pub fn end_frame(frames: u64) {
WIDGETS.with_borrow_mut(|(current, last, seen)| {
if *seen != frames {
*seen = frames;
*last = std::mem::take(current);
}
});
}
/// The buttons of the last drawn frame, top-most last.
pub fn last() -> Vec<Widget> {
WIDGETS.with_borrow(|(_, last, _)| last.clone())
}
/// The button `label` names: an exact match (case-insensitive), else the only
/// button whose label contains it. The top-most wins among equals.
pub fn find(widgets: &[Widget], label: &str) -> Result<Widget, String> {
let want = label.trim().to_lowercase();
let exact: Vec<&Widget> = widgets
.iter()
.filter(|w| w.label.to_lowercase() == want)
.collect();
if let Some(w) = exact.last() {
return Ok((*w).clone());
}
let partial: Vec<&Widget> = widgets
.iter()
.filter(|w| w.label.to_lowercase().contains(&want))
.collect();
match partial.as_slice() {
[w] => Ok((*w).clone()),
[] => Err(format!(
"no button labelled {label:?}; buttons: {:?}",
widgets.iter().map(|w| w.label.as_str()).collect::<Vec<_>>()
)),
many => Err(format!(
"{label:?} matches several buttons: {:?}",
many.iter().map(|w| w.label.as_str()).collect::<Vec<_>>()
)),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn w(label: &str) -> Widget {
Widget {
label: label.into(),
x: 0.0,
y: 0.0,
w: 10.0,
h: 10.0,
enabled: true,
}
}
#[test]
fn finds_exact_then_unique_partial() {
let list = [w("Play Selected World"), w("Create New World"), w("Back")];
assert_eq!(find(&list, "back").unwrap().label, "Back");
assert_eq!(find(&list, "create").unwrap().label, "Create New World");
assert!(find(&list, "world").is_err());
assert!(find(&list, "options").is_err());
}
#[test]
fn frames_rotate_only_when_drawn() {
record("Respawn", [1.0, 2.0, 3.0, 4.0], true);
end_frame(1);
assert_eq!(last().len(), 1);
end_frame(1);
assert_eq!(last().len(), 1);
end_frame(2);
assert!(last().is_empty());
}
}