Agent control: wait for a state, and the log of what happened

wait-for screen NAME|widget ID|loaded|chat TEXT|event KIND replies once
the game gets there (--timeout, default 30 s, ends in an error of kind
timeout; --since SEQ counts chat and events from an earlier point), so
scripts and agents stop polling. events [--since SEQ] lists what
happened, oldest first, numbered: screens, chat lines, deaths, respawns,
items gained or lost, dimensions, worlds opened and closed. The state is
compared once per event loop turn and only under --control, so watching
never changes the game. respawn now says when the death screen's button
is not active yet instead of clicking it to no effect.

horizon-ctl and its MCP gain both; tools/agent_control_e2e.py gains an
events scenario.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ludovic Marie
2026-10-15 14:51:30 +02:00
parent 1a067b3d9e
commit 0892f944c0
9 changed files with 418 additions and 4 deletions
@@ -148,6 +148,13 @@ pub const COMMANDS: &[Cmd] = &[
flag("wait", Ty::Bool, "also wait until the world is ready (default in MCP)"),
flag("offscreen", Ty::Bool, "render without showing a window (screenshots still work): locked or headless sessions")],
mcp: true, local: true),
cmd!("wait-for", Session, "wait-for screen NAME|widget ID|loaded|chat TEXT|event KIND [--timeout S] [--since SEQ]", "wait until the game reaches a state, then reply",
"screen takes the names state shows (playing, inventory, paused, dead, title...); widget an id from ui; loaded means a world is open, the player is in it and no chunk is pending; chat a new line containing TEXT; event a kind from events (death, respawn, item, screen...). chat and event count from --since SEQ (take last from events before acting), else from now. A timeout (default 30 s) is an error with kind timeout.",
["horizon-ctl wait-for screen inventory", "horizon-ctl wait-for loaded --timeout 120", "horizon-ctl wait-for chat joined the game", "horizon-ctl wait-for event death --timeout 600"],
&[flag("timeout", Ty::Num, "seconds (default 30)"), flag("since", Ty::Int, "chat and event: count from this event seq"), p("condition", Ty::Enum(&["screen", "widget", "loaded", "chat", "event"]), "what to wait for"), opt("value", Ty::Rest, "the screen name, widget id, chat text or event kind")], mcp: true, local: false),
cmd!("events", Observe, "events [--since SEQ]", "what happened since SEQ, oldest first: screens, chat, deaths, respawns, items gained or lost, dimensions, worlds",
"Every event has seq and tick; pass the reply's last as --since next time. The game keeps the newest 256.",
["horizon-ctl events", "horizon-ctl events --since 42"], &[flag("since", Ty::Int, "only events after this seq")], mcp: true, local: false),
cmd!("wait-ready", Session, "wait-ready [SECONDS]", "wait until the player has spawned and no chunks are pending",
"Polls status every 250 ms; replies with the status, or an error after SECONDS (default 120).",
["horizon-ctl wait-ready 300"], &[opt("seconds", Ty::Num, "how long to wait (default 120)")], mcp: true, local: true),
+3
View File
@@ -99,6 +99,8 @@ pub struct Chat {
pub options: ChatOptions,
/// Messages printed to stdout as `[CHAT]` (vanilla logs them).
pub log: bool,
/// Messages added since the start, for the control socket's events.
pub received: u64,
}
impl Chat {
pub fn lines_per_page(&self, focused: bool) -> usize {
@@ -130,6 +132,7 @@ impl Chat {
self.add_lines(atlas, &message, focused);
self.messages.insert(0, message);
self.messages.truncate(100);
self.received += 1;
}
fn add_lines(&mut self, atlas: &HudAtlas, message: &Message, focused: bool) {
let width = self.max_width();
+4
View File
@@ -34,6 +34,7 @@ mod dispenser;
mod dragon;
mod enchant_effects;
mod end;
mod events;
mod explosion;
mod extras;
mod fangs;
@@ -948,6 +949,8 @@ struct Game<'a> {
agent: agent::Agent,
/// Widget commands waiting for the frame after a click (game/ui_control.rs).
ui_wait: ui_control::UiWait,
/// What happened, for `events` and `wait-for` (game/events.rs).
events: events::Events,
/// The overworld's last `sky_darken` and its inputs (time, rain, thunder).
sky_darken_memo: std::cell::Cell<Option<((i64, u32, u32), i32)>>,
persist: persist::EntityMemory,
@@ -9813,6 +9816,7 @@ pub fn run() -> Result<()> {
pending_bench: None,
flight: None,
ui_wait: Default::default(),
events: Default::default(),
agent: agent::Agent::default(),
view_frustum: None,
pending_uploads: Vec::new(),
+8 -1
View File
@@ -382,7 +382,7 @@ impl Game<'_> {
Screen::WorldOptions => "world-options",
}
}
fn inventory_counts(&self) -> BTreeMap<String, i64> {
pub(super) fn inventory_counts(&self) -> BTreeMap<String, i64> {
counts(
self.inventory
.iter()
@@ -1397,6 +1397,13 @@ impl Game<'_> {
let label =
horizon_client::text::Text::translate("deathScreen.respawn", &[]).plain();
let w = widgets::find(&widgets::last(), &label)?;
if !w.enabled {
// DeathScreen enables its buttons after 20 ticks.
return Err(
"the Respawn button is not active yet; try again in a second (wait 20)"
.into(),
);
}
self.synth_click(
event_loop,
w.x + w.w / 2.0,
@@ -103,6 +103,9 @@ impl Game<'_> {
while let Some(request) = self.control.as_ref().and_then(|c| c.try_recv()) {
self.answer(request, event_loop);
}
if self.control.is_some() {
self.watch_events();
}
}
fn answer(&mut self, request: horizon_client::control::Request, event_loop: &ActiveEventLoop) {
@@ -137,6 +140,9 @@ impl Game<'_> {
super::agent::Handled::Deferred => return Ok(None),
super::agent::Handled::No => {}
}
if let Some(value) = self.events_command(&words, reply)? {
return Ok(value);
}
if let Some(value) = self.options_control(&words, line)? {
return Ok(Some(value));
}
@@ -163,6 +169,7 @@ impl Game<'_> {
.chain(super::agent::HELP)
.chain(super::ui_control::HELP)
.chain(super::options::CONTROL_HELP)
.chain(super::events::HELP)
.chain(super::pad::HELP)
.collect::<Vec<_>>() }))),
"setblock" | "fill" | "particle" | "signedit" => self.render_command(&words),
+338
View File
@@ -0,0 +1,338 @@
//! What happened in the game, for agents (`events`, `wait-for`; docs/agent-control.md): the
//! state is compared once per event loop turn, after the game ran, so watching never changes
//! what the game does.
use super::Game;
use horizon_client::Result;
use serde_json::{Value, json};
use std::{
collections::{BTreeMap, VecDeque},
sync::mpsc::Sender,
time::{Duration, Instant},
};
pub(super) const HELP: &[&str] = &[
"events [--since SEQ] what happened (screen, chat, death, respawn, item, dimension, world), oldest first",
"wait-for screen NAME|widget ID|loaded|chat TEXT|event KIND [--timeout S] [--since SEQ] reply once it holds (default 30 s; chat and event count from SEQ, else from now)",
];
/// Events kept for `events`.
const KEPT: usize = 256;
/// What the last turn saw.
#[derive(PartialEq)]
struct Snapshot {
screen: &'static str,
world: Option<String>,
dimension: String,
chat: u64,
counts: BTreeMap<String, i64>,
}
enum Condition {
Screen(String),
Widget(String),
Loaded,
Chat(String),
Event(String),
}
struct Wait {
condition: Condition,
/// The event sequence number when the wait started.
since: u64,
started: Instant,
timeout: Duration,
reply: Sender<String>,
}
#[derive(Default)]
pub(super) struct Events {
/// The sequence number of the newest event (the first is 1).
seq: u64,
log: VecDeque<Value>,
last: Option<Snapshot>,
waits: Vec<Wait>,
}
impl Events {
fn push(&mut self, tick: u64, mut event: Value) {
self.seq += 1;
event["seq"] = self.seq.into();
event["tick"] = tick.into();
if self.log.len() == KEPT {
self.log.pop_front();
}
self.log.push_back(event);
}
fn since(&self, seq: u64) -> impl Iterator<Item = &Value> {
self.log
.iter()
.filter(move |e| e["seq"].as_u64() > Some(seq))
}
}
/// The events between two snapshots.
fn diff(before: &Snapshot, now: &Snapshot, chat: &[String]) -> Vec<Value> {
let mut out = Vec::new();
if before.world != now.world {
if let Some(name) = &before.world {
out.push(json!({ "event": "world", "state": "closed", "name": name }));
}
if let Some(name) = &now.world {
out.push(json!({ "event": "world", "state": "opened", "name": name }));
}
}
if before.dimension != now.dimension && now.world.is_some() && before.world == now.world {
out.push(json!({ "event": "dimension", "from": before.dimension, "to": now.dimension }));
}
if before.screen != now.screen {
out.push(json!({ "event": "screen", "from": before.screen, "to": now.screen }));
if now.screen == "dead" {
out.push(json!({ "event": "death" }));
} else if before.screen == "dead" {
out.push(json!({ "event": "respawn" }));
}
}
// Oldest first; `chat` is newest first.
for text in chat.iter().rev() {
out.push(json!({ "event": "chat", "text": text }));
}
if before.world == now.world {
let names = before.counts.keys().chain(now.counts.keys());
let mut seen = std::collections::BTreeSet::new();
for item in names.filter(|n| seen.insert(*n)) {
let delta = now.counts.get(item).unwrap_or(&0) - before.counts.get(item).unwrap_or(&0);
if delta != 0 {
out.push(json!({ "event": "item", "item": item, "delta": delta }));
}
}
}
out
}
/// The condition, the timeout and `--since`.
fn parse_wait(words: &[&str]) -> Result<(Condition, Duration, Option<u64>)> {
let mut rest = Vec::new();
let mut timeout = 30.0;
let mut since = None;
let mut i = 0;
while i < words.len() {
if words[i] == "--timeout" {
timeout = words
.get(i + 1)
.and_then(|t| t.parse::<f64>().ok())
.ok_or("--timeout takes seconds")?;
i += 2;
} else if words[i] == "--since" {
since = Some(
words
.get(i + 1)
.and_then(|t| t.parse::<u64>().ok())
.ok_or("--since takes an event seq")?,
);
i += 2;
} else {
rest.push(words[i]);
i += 1;
}
}
let usage =
"wait-for screen NAME|widget ID|loaded|chat TEXT|event KIND [--timeout S] [--since SEQ]";
let text = rest.get(1..).map(|r| r.join(" ")).unwrap_or_default();
let need = |text: String| {
if text.is_empty() {
Err(usage)
} else {
Ok(text)
}
};
let condition = match rest.first().copied() {
Some("screen") => Condition::Screen(need(text)?),
Some("widget") => Condition::Widget(need(text)?),
Some("loaded") => Condition::Loaded,
Some("chat") => Condition::Chat(need(text)?),
Some("event") => Condition::Event(need(text)?),
_ => return Err(usage.into()),
};
Ok((
condition,
Duration::from_secs_f64(timeout.clamp(0.0, 3600.0)),
since,
))
}
impl Game<'_> {
fn snapshot(&self) -> Snapshot {
let world = self.title.is_none().then(|| self.level.name.clone());
Snapshot {
screen: self.screen_label(),
dimension: self.dim.id().to_string(),
chat: self.ui.chat.received,
counts: if world.is_some() {
self.inventory_counts()
} else {
BTreeMap::new()
},
world,
}
}
fn loaded(&self) -> bool {
self.title.is_none()
&& self.player.is_some()
&& self.streaming.pending() + self.pending_uploads.len() == 0
}
/// Once per event loop turn: record what changed, answer the waits that hold.
pub(super) fn watch_events(&mut self) {
let now = self.snapshot();
if let Some(before) = &self.events.last
&& *before != now
{
let new_chat = (now.chat - before.chat).min(100) as usize;
let lines = self.ui.chat.recent_lines(new_chat);
for event in diff(before, &now, &lines) {
self.events.push(self.ticks, event);
}
}
self.events.last = Some(now);
if self.events.waits.is_empty() {
return;
}
let waits = std::mem::take(&mut self.events.waits);
for wait in waits {
let done = match &wait.condition {
Condition::Screen(name) => self.screen_label() == name,
Condition::Widget(id) => self.ui_nodes().iter().any(|n| &n.id == id),
Condition::Loaded => self.loaded(),
Condition::Chat(text) => self.events.since(wait.since).any(|e| {
e["event"] == "chat" && e["text"].as_str().is_some_and(|t| t.contains(text))
}),
Condition::Event(kind) => {
self.events.since(wait.since).any(|e| e["event"] == **kind)
}
};
let waited = wait.started.elapsed();
let reply = if done {
json!({ "ok": true, "waited_seconds": waited.as_secs_f64(), "screen": self.screen_label() })
} else if waited >= wait.timeout {
json!({
"error": format!(
"timed out after {} s; the screen is {}",
wait.timeout.as_secs_f64(),
self.screen_label()
),
"kind": "timeout",
})
} else {
self.events.waits.push(wait);
continue;
};
let _ = wait.reply.send(reply.to_string());
}
}
/// `events` and `wait-for`; None for other commands, Some(None) when deferred.
pub(super) fn events_command(
&mut self,
words: &[&str],
reply: &Sender<String>,
) -> Result<Option<Option<Value>>> {
match words[0] {
"events" => {
let since = match words.get(1..) {
Some(["--since", n]) => n.parse().map_err(|_| "--since takes a number")?,
Some([]) | None => 0,
_ => return Err("events [--since SEQ]".into()),
};
let events: Vec<&Value> = self.events.since(since).collect();
Ok(Some(Some(
json!({ "events": events, "last": self.events.seq }),
)))
}
"wait-for" => {
let (condition, timeout, since) = parse_wait(&words[1..])?;
self.events.waits.push(Wait {
condition,
since: since.unwrap_or(self.events.seq),
started: Instant::now(),
timeout,
reply: reply.clone(),
});
// Answered at the end of this turn if it already holds.
Ok(Some(None))
}
_ => Ok(None),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn snap(screen: &'static str, chat: u64, logs: i64) -> Snapshot {
Snapshot {
screen,
world: Some("w".into()),
dimension: "minecraft:overworld".into(),
chat,
counts: BTreeMap::from([("minecraft:oak_log".to_owned(), logs)]),
}
}
#[test]
fn diffs_name_what_changed() {
let events = diff(
&snap("playing", 3, 1),
&snap("dead", 5, 0),
&["newest".into(), "older".into()],
);
let kinds: Vec<&str> = events
.iter()
.map(|e| e["event"].as_str().unwrap())
.collect();
assert_eq!(kinds, ["screen", "death", "chat", "chat", "item"]);
assert_eq!(events[2]["text"], "older", "chat oldest first");
assert_eq!(events[4]["delta"], -1);
let back = diff(&snap("dead", 5, 0), &snap("playing", 5, 0), &[]);
assert_eq!(back[1]["event"], "respawn");
let mut closed = snap("title", 5, 0);
closed.world = None;
closed.counts.clear();
let left = diff(&snap("playing", 5, 4), &closed, &[]);
assert_eq!(
left[0],
json!({ "event": "world", "state": "closed", "name": "w" })
);
assert!(
left.iter().all(|e| e["event"] != "item"),
"leaving drops no items"
);
}
#[test]
fn keeps_the_newest_events_numbered() {
let mut e = Events::default();
for i in 0..300 {
e.push(i, json!({ "event": "chat" }));
}
assert_eq!(e.log.len(), KEPT);
assert_eq!(e.since(295).count(), 5);
assert_eq!(e.since(0).next().unwrap()["seq"], 45);
}
#[test]
fn parses_conditions_and_timeouts() {
let (c, t, since) =
parse_wait(&["chat", "hello", "--since", "7", "there", "--timeout", "5"]).unwrap();
assert!(matches!(c, Condition::Chat(ref s) if s == "hello there"));
assert_eq!((t, since), (Duration::from_secs(5), Some(7)));
assert!(matches!(
parse_wait(&["loaded"]).unwrap().0,
Condition::Loaded
));
assert!(parse_wait(&["screen"]).is_err());
assert!(parse_wait(&["soon"]).is_err());
}
}
+1 -1
View File
@@ -52,7 +52,7 @@ impl UiWait {
impl Game<'_> {
/// The open screen's widgets: those the last frame drew, then the slots
/// of an open inventory or menu (`slot-ID`, the ids of `click-slot`).
fn ui_nodes(&self) -> Vec<Node> {
pub(super) fn ui_nodes(&self) -> Vec<Node> {
let size = self.gui_scale().map(|b| [b.width, b.height]);
let mut nodes = widgets::tree(&widgets::last_labelled(), &widgets::last_targets(), size);
if self.screen == Screen::Inventory
+7
View File
@@ -134,6 +134,13 @@ game accepts, test hooks included.
`key.keyboard.w`); `options-set KEY VALUE` sets one, applies it at once and
saves `options.txt` as the options screen does. A value the option refuses
is an error saying what it accepts, not vanilla's silent reset.
- **Waits and events:** `wait-for screen NAME|widget ID|loaded|chat TEXT|event
KIND [--timeout S] [--since SEQ]` replies once the game gets there (a timeout
is an error with `"kind": "timeout"`), so scripts don't poll. `events [--since
SEQ]` lists what happened, oldest first, each with a `seq` and the `tick`:
`screen` (from, to), `chat` (text), `death`, `respawn`, `item` (item, delta),
`dimension`, `world` (opened, closed). Take `last` from `events` before an
action whose event you then `wait-for --since` it, so a fast one isn't missed.
- **Screens:** `open-inventory`, `click-slot SLOT [right] [shift]` (0-8 hotbar,
9-35 inventory, 100+ crafting grid, 200 result, 300+ containers, 500-503
armor, 504 offhand, 600+ stations), `close`, `click-button LABEL` (any
+43 -2
View File
@@ -6,7 +6,8 @@ checks the inventory, crafts planks in the inventory screen, places one and read
runs through the CLI (one horizon-ctl process per command) and through `horizon-ctl mcp` with a
small JSON-RPC client, and saves a screenshot of each run. The ui scenario drives the menus by
widget id instead: the title screen, the options (a slider set by id) and the inventory's slots;
the options scenario reads and sets options.txt values by key.
the options scenario reads and sets options.txt values by key; the events scenario waits for
states (wait-for) and reads the event log.
python3 tools/agent_control_e2e.py # both modes
python3 tools/agent_control_e2e.py --mode mcp # one of them
@@ -41,6 +42,7 @@ POSITIONAL = {
'click_button': ['label'], 'type': ['text'], 'wait': ['ticks'], 'wait_ready': ['seconds'],
'command': ['command'], 'look_at_block': ['x', 'y', 'z'], 'click': ['id'], 'set': ['id', 'value'],
'give': ['item', 'count'], 'options_get': ['keys'], 'options_set': ['key', 'value'],
'wait_for': ['condition', 'value'],
}
# Booleans the game takes as bare words.
BARE = {'click_slot': ['right', 'shift'], 'click': ['right', 'shift']}
@@ -337,7 +339,46 @@ def options_scenario(d, out):
return {'options': len(options)}
SCENARIOS = {'survival': scenario, 'ui': ui_scenario, 'options': options_scenario}
def events_scenario(d, out):
# Waits instead of polling, and the event log of what happened meanwhile.
start = d('events')['last']
d('click', id='singleplayer')
if 'game' not in widgets(d):
d('click', id='create-new-world')
d('type', field='name', replace=True, text='Events')
d('click', id='create-new-world')
d('wait_for', condition='loaded', timeout=400)
seq = d('events')['last'] # an event can happen before the wait starts
d('give', item='oak_log', count=3)
d('wait_for', condition='event', value='item', timeout=5, since=seq)
seq = d('events')['last']
d('command', command='say hello agents')
d('wait_for', condition='chat', value='hello agents', timeout=5, since=seq)
d('open_inventory')
d('wait_for', condition='screen', value='inventory', timeout=5)
d('close')
seq = d('events')['last']
d('command', command='kill @s')
d('wait_for', condition='event', value='death', timeout=10, since=seq)
d('wait', ticks=25) # the death screen's buttons wake after 20 ticks
d('respawn')
d('wait_for', condition='event', value='respawn', timeout=10, since=seq)
try:
d('wait_for', condition='screen', value='advancements', timeout=0.5)
raise AssertionError('the wait did not time out')
except Failed as e:
assert 'timed out' in str(e) and 'timeout' in str(e), e
events = d('events', since=start)['events']
kinds = [e['event'] for e in events]
for kind in ['world', 'screen', 'item', 'chat', 'death', 'respawn']:
assert kind in kinds, f'no {kind} event in {kinds}'
logs = next(e for e in events if e['event'] == 'item' and e['item'] == 'minecraft:oak_log')
assert logs['delta'] == 3, logs
assert all(b['seq'] > a['seq'] for a, b in zip(events, events[1:]))
return {'events': len(events)}
SCENARIOS = {'survival': scenario, 'ui': ui_scenario, 'options': options_scenario, 'events': events_scenario}
def run(mode, ctl, out, resources, which='survival'):