Merge pull request #2114 from ludovic111/fix/harness-await-flakes

🤖 Agent: harness-only change (skip-release). The old harnesses fail under server stalls, the new ones pass; 20 affected harnesses pass locally, and the two changed ones rerun after merging main.
This commit is contained in:
Ludovic Marie
2026-10-26 12:53:58 +01:00
committed by GitHub
5 changed files with 225 additions and 86 deletions
@@ -5,9 +5,12 @@ import io.netty.buffer.Unpooled;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.FileDescriptor;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.net.Socket;
@@ -76,7 +79,8 @@ import net.minecraft.world.phys.Vec3;
* {@code step LABEL}, {@code join NAME}, {@code cmd NAME COMMAND...} (waits
* for its feedback), {@code pos NAME X Y Z YAW PITCH GROUND} (one client tick
* at that place, on the ground or not), {@code sprint NAME on|off}, {@code
* attack NAME TARGET} (ServerboundAttackPacket then the swing; TARGET is a
* sync NAME} (a round trip through the server thread; "synced NAME" on stdout when
* it is back), {@code attack NAME TARGET} (ServerboundAttackPacket then the swing; TARGET is a
* player's name or an entity type's path, the last one added), {@code use
* NAME} (ServerboundUseItemPacket, main hand) and {@code end}. Each player
* records the ClientboundSetEntityMotionPacket it receives, its own sprint
@@ -88,6 +92,8 @@ public final class KnockbackClient {
private static final Map<UUID, String> NAMES = new LinkedHashMap<>();
public static void main(String[] args) throws Exception {
// Bootstrap sends System.out to the logger: `synced` lines go to the real stdout.
PrintStream stdout = new PrintStream(new FileOutputStream(FileDescriptor.out), true, StandardCharsets.UTF_8);
ParityData.init();
String host = args[0];
int port = Integer.parseInt(args[1]);
@@ -116,6 +122,13 @@ public final class KnockbackClient {
case "attack" -> players.get(w[1]).attack(w[2]);
case "use" -> players.get(w[1]).send(new ServerboundUseItemPacket(InteractionHand.MAIN_HAND, 0, 0, 0));
case "punch" -> players.get(w[1]).send(net.minecraft.network.protocol.game.ServerboundPunchPacket.INSTANCE);
case "sync" -> {
try {
players.get(w[1]).sync();
} finally {
stdout.println("synced " + w[1]);
}
}
case "mark" -> players.get(w[1]).mark();
case "await" -> players.get(w[1]).await();
case "end" -> {
@@ -157,6 +170,7 @@ public final class KnockbackClient {
private final Map<String, Integer> last = new java.util.concurrent.ConcurrentHashMap<>();
private final Object replyLock = new Object();
private int replies;
private int syncs;
/** The motion packets about this player it was sent, and how many there were at the last `mark`. */
private int pushes;
private int marked;
@@ -181,6 +195,24 @@ public final class KnockbackClient {
}
}
/** A round trip through the server thread, its reply left out of the transcript: every packet this player
* was sent before the reply has been recorded when it returns. */
void sync() throws Exception {
synchronized (replyLock) {
int before = syncs;
send(new ServerboundChatCommandPacket("time query gametime"));
long until = System.nanoTime() + 30_000_000_000L;
while (syncs == before) {
long left = (until - System.nanoTime()) / 1_000_000L;
if (left <= 0) {
record("error no reply to sync");
return;
}
replyLock.wait(left);
}
}
}
void mark() {
synchronized (replyLock) {
marked = pushes;
@@ -327,6 +359,13 @@ public final class KnockbackClient {
for (var value : d.packedItems()) {
if (value.id() == 0) record("sprinting " + ((((Byte) value.value()) & 8) != 0));
}
} else if (p instanceof ClientboundSystemChatPacket c
&& c.content().getContents() instanceof net.minecraft.network.chat.contents.TranslatableContents t
&& t.getKey().equals("commands.time.query.gametime")) {
synchronized (replyLock) {
syncs++;
replyLock.notifyAll();
}
} else if (p instanceof ClientboundSystemChatPacket c) {
record("chat " + c.content().getString());
synchronized (replyLock) {
+14 -1
View File
@@ -206,6 +206,17 @@ def game_time(server, timeout=300):
raise AssertionError(f'{server.log_path.name}: no game time in {timeout:.0f} s')
def wait_ticks(server, count, timeout=300):
"""Waits until the server's game clock has advanced `count` ticks: a server that stalls under load still gets
as many ticks as an idle one, where a sleep gives it fewer. (Not while ticking is frozen.)"""
deadline = time.time() + timeout
until = game_time(server, timeout) + count
while game_time(server, max(1.0, deadline - time.time())) < until:
if time.time() > deadline:
raise AssertionError(f'{server.log_path.name}: the game clock did not advance {count} ticks in {timeout} s')
time.sleep(0.2)
def legacy_ping(port):
"""The reply to a 1.6 client's 0xFE 0x01 0xFA MC|PingHost query (LegacyQueryHandler)."""
host = 'localhost'
@@ -333,7 +344,9 @@ def run_side(label, command, directory, client):
joined.join()
if failure:
raise failure[0]
time.sleep(1)
# The client has gone; `list` must find no one, once the server has handled its leaving (a busy machine
# took longer than a second).
server.wait_for(rf'{PLAYER} left the game', timeout=60)
server.console('list')
server.console('say console says hi')
server.wait_for(r'\[Server\] console says hi', timeout=30)
+67 -35
View File
@@ -23,15 +23,14 @@ import platform
import shutil
import subprocess
import sys
import time
try:
from . import lab, test_rust_parity as base
from .test_dedicated_server_parity import Server, free_port
from .test_dedicated_server_parity import Server, free_port, wait_ticks
except ImportError:
import lab
import test_rust_parity as base
from test_dedicated_server_parity import Server, free_port
from test_dedicated_server_parity import Server, free_port, wait_ticks
NAME = 'locator-bar-parity'
PROPERTIES = {
@@ -82,7 +81,7 @@ SCRIPT = [
('creative', [], ['cmd Bob gamemode creative', 'cmd Bob tp @s 6.5 100 0.5']),
('bob-leaves', [], ['quit Bob']),
]
STEP_SECONDS = 2.0
STEP_TICKS = 40
# Vanilla's server moves a player who just joined on their own first ticks (a tiny step and back), which tracks
# them where they spawned; Horizon does not simulate its guests. These steps compare what the bar shows at
# their end (Bob has moved by then) rather than every packet.
@@ -147,6 +146,47 @@ def server_dir(directory, world=None):
return port
class Client:
"""A LocatorBarClient process driven by a script on its stdin. A step ends by the server's clock, not the wall's:
every script line has run (`sync`), the server has ticked `ticks` more times, and each client holds what the
server sent until then (a last `sync`). A server that stalls under load still gets all its ticks in the step,
where a fixed sleep filed a stalled server's packets under the next step."""
def __init__(self, command, port, out, log):
self.log = log
self.process = subprocess.Popen([*map(str, command), '127.0.0.1', str(port), str(out)], cwd=lab.ROOT,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True)
def tell(self, line):
self.process.stdin.write(line + '\n')
self.process.stdin.flush()
def sync(self, names):
"""Every line told so far has run for these joined players, and they have received what the server sent
before."""
for name in names:
self.tell(f'sync {name}')
for name in names:
while (line := self.process.stdout.readline()) != f'synced {name}\n':
if not line:
raise AssertionError(f'the client exited (see {self.log.name})')
self.log.write(line)
def wait(self, server, names, ticks):
self.sync(names)
wait_ticks(server, ticks)
self.sync(names)
def end(self):
self.tell('end')
self.log.write(self.process.communicate(timeout=60)[0])
def kill(self):
if self.process.poll() is None:
self.process.kill()
def generate_world(server_command, build):
"""The flat world both servers load, made once by vanilla: players appear at its spawn itself
(respawn_radius 0), on the ground at 0 -60 0, and the chunks around it are saved."""
@@ -176,28 +216,26 @@ def run_side(label, server_command, client_command, build, world):
try:
server.wait_for(r'Done \(')
with (build / f'{label}-client.log').open('w') as log:
client = subprocess.Popen([*map(str, client_command), '127.0.0.1', str(port), str(out)],
cwd=lab.ROOT, stdin=subprocess.PIPE, stdout=log, stderr=subprocess.STDOUT,
text=True)
client = Client(client_command, port, out, log)
joined = []
try:
for step, console, actions in SCRIPT:
client.stdin.write(f'step {step}\n')
client.stdin.flush()
client.tell(f'step {step}')
for command in console:
server.console(command)
for action in actions:
client.stdin.write(action + '\n')
client.stdin.flush()
if action.startswith('join'):
client.tell(action)
verb, *rest = action.split()
if verb == 'join':
# Joining takes the world's first chunks: wait for the server to place the player.
server.wait_for(rf'{action.split()[1]} joined the game', timeout=120)
time.sleep(STEP_SECONDS)
client.stdin.write('end\n')
client.stdin.flush()
client.wait(timeout=60)
server.wait_for(rf'{rest[0]} joined the game', timeout=120)
joined.append(rest[0])
elif verb == 'quit':
joined.remove(rest[0])
client.wait(server, joined, STEP_TICKS)
client.end()
finally:
if client.poll() is None:
client.kill()
client.kill()
finally:
server.stop()
if not out.exists():
@@ -234,10 +272,11 @@ def sides(name):
def one_player(name, script, shown, check, scope, step_seconds=4.0):
"""A harness of one player, Alice: on each server she joins, then runs `script`, a list of (step, client
actions) `step_seconds` apart. The lines `shown(transcript)` keeps must match vanilla's exactly, and
actions) that each last `step_seconds` of the server's time (20 ticks a second). The lines `shown(transcript)` keeps must match vanilla's exactly, and
`check(vanilla's shown lines, vanilla's transcript)` returns why vanilla's run is no proof (the script did
not run), or None."""
lock, build, vanilla_server, horizon_server, client = sides(name)
ticks = round(step_seconds * 20)
def run(label, server_command, world):
directory = build / label
@@ -248,27 +287,20 @@ def one_player(name, script, shown, check, scope, step_seconds=4.0):
try:
server.wait_for(r'Done \(')
with (build / f'{label}-client.log').open('w') as log:
process = subprocess.Popen([*map(str, client), '127.0.0.1', str(port), str(out)], cwd=lab.ROOT,
stdin=subprocess.PIPE, stdout=log, stderr=subprocess.STDOUT, text=True)
def tell(line):
process.stdin.write(line + '\n')
process.stdin.flush()
session = Client(client, port, out, log)
try:
for line in ['players on', 'state on', 'step joins', 'join Alice']:
tell(line)
session.tell(line)
server.wait_for(r'Alice joined the game', timeout=120)
time.sleep(step_seconds)
session.wait(server, ['Alice'], ticks)
for step, actions in script:
tell(f'step {step}')
session.tell(f'step {step}')
for action in actions:
tell(action)
time.sleep(step_seconds)
tell('end')
process.wait(timeout=60)
session.tell(action)
session.wait(server, ['Alice'], ticks)
session.end()
finally:
if process.poll() is None:
process.kill()
session.kill()
finally:
server.stop()
if not out.exists():
+20 -28
View File
@@ -16,17 +16,16 @@ import platform
import shutil
import subprocess
import sys
import time
try:
from . import lab, test_rust_parity as base
from .test_dedicated_server_parity import Server
from .test_locator_bar_parity import generate_world, server_dir
from .test_locator_bar_parity import Client, generate_world, server_dir
except ImportError:
import lab
import test_rust_parity as base
from test_dedicated_server_parity import Server
from test_locator_bar_parity import generate_world, server_dir
from test_locator_bar_parity import Client, generate_world, server_dir
NAME = 'server-dimension-players-parity'
NETHER, END, OVERWORLD = 'minecraft:the_nether', 'minecraft:the_end', 'minecraft:overworld'
@@ -47,7 +46,8 @@ SCRIPT = [
('bob-retarget-portal', ['cmd Bob setblock 0 101 3 minecraft:end_portal', 'cmd Bob tp @s 0.5 101 3.5']),
('bob-retarget-back', [f'cmd Bob execute in {OVERWORLD} run tp @s 2.5 100 0.5']),
]
STEP_SECONDS = 5.0
PLAYERS = ('Alice', 'Bob')
STEP_TICKS = 100
def run_side(label, server_command, client_command, build, world):
@@ -59,38 +59,30 @@ def run_side(label, server_command, client_command, build, world):
try:
server.wait_for(r'Done \(')
with (build / f'{label}-client.log').open('w') as log:
client = subprocess.Popen([*map(str, client_command), '127.0.0.1', str(port), str(out)],
cwd=lab.ROOT, stdin=subprocess.PIPE, stdout=log, stderr=subprocess.STDOUT,
text=True)
def tell(line):
client.stdin.write(line + '\n')
client.stdin.flush()
client = Client(client_command, port, out, log)
try:
tell('players on')
tell('step joins')
for name, x in (('Alice', 0.5), ('Bob', 2.5)):
tell(f'join {name}')
client.tell('players on')
client.tell('step joins')
for name, x in zip(PLAYERS, (0.5, 2.5)):
client.tell(f'join {name}')
server.wait_for(rf'{name} joined the game', timeout=120)
tell(f'fly {name} on')
tell(f'cmd {name} tp @s {x} 100 0.5')
time.sleep(STEP_SECONDS)
client.tell(f'fly {name} on')
client.tell(f'cmd {name} tp @s {x} 100 0.5')
client.wait(server, PLAYERS, STEP_TICKS)
for step, actions in SCRIPT:
tell(f'step {step}')
client.tell(f'step {step}')
for action in actions:
tell(action)
client.tell(action)
# Standing still, a client still sends its position every 20 ticks
# (LocalPlayer.sendPosition): the server then tracks to it what it has sent the
# chunks of since (ChunkMap.move).
time.sleep(STEP_SECONDS / 2)
tell('still Alice')
tell('still Bob')
time.sleep(STEP_SECONDS / 2)
tell('end')
client.wait(timeout=60)
client.wait(server, PLAYERS, STEP_TICKS // 2)
for name in PLAYERS:
client.tell(f'still {name}')
client.wait(server, PLAYERS, STEP_TICKS // 2)
client.end()
finally:
if client.poll() is None:
client.kill()
client.kill()
finally:
server.stop()
if not out.exists():
+84 -21
View File
@@ -18,6 +18,11 @@ players made of vanilla's protocol code, joins them both and runs one script on
travelled a tick (the mob's hit, then ServerPlayer's travel, then ServerEntity.sendChanges). It is night, so
that no zombie burns (a burning mob is in its hurt cooldown).
The script waits on the server, not on the wall: `ticks N` is N ticks of its game clock (with a `sync` round trip
before and after, so each client holds what the server sent), `tickstep` is one `/tick step` that has run, `frozen`
a frozen game that has stopped, and `hit` a hit that has landed (a frozen server still ticks its players, so a hit
lands in Alice's next tick). A server that stalls under load gets the same ticks as an idle one.
Each case is compared on its own. KNOWN lists the cases Horizon still gets wrong and why; one that matches
again is reported, and should leave the list.
"""
@@ -32,15 +37,16 @@ import time
try:
from . import lab, test_rust_parity as base
from .test_dedicated_server_parity import Server, free_port
from .test_dedicated_server_parity import Server, free_port, game_time, wait_ticks
from .test_locator_bar_parity import generate_world, ops_json
except ImportError:
import lab
import test_rust_parity as base
from test_dedicated_server_parity import Server, free_port
from test_dedicated_server_parity import Server, free_port, game_time, wait_ticks
from test_locator_bar_parity import generate_world, ops_json
NAME = 'server-knockback-parity'
PLAYERS = ('Alice', 'Bob')
PROPERTIES = {
'online-mode': 'false',
'white-list': 'false',
@@ -82,18 +88,17 @@ def mob_case(kind, weapon=SWORD, sprint=False, crit=False, air=False):
f'cmd Alice effect give {mob} minecraft:slowness 100000 127 true']
lines = ['cmd Alice kill @e[type=!player]', 'cmd Alice gamemode creative', 'cmd Bob gamemode creative',
f'cmd Alice item replace entity @s weapon.mainhand with {weapon}', 'cmd Alice tp Bob 2.5 -60 12.5',
*stand('Alice', A, -90), 'sleep 1.2']
*stand('Alice', A, -90), 'ticks 24']
# In the air: frozen before its first tick, so it has not landed.
lines += ['cmd Alice tick freeze', *summon] if air else [*summon, 'sleep 1.5', 'cmd Alice tick freeze']
lines += ['sleep 0.3', f'cmd Alice data get entity {mob} Pos']
lines += ['cmd Alice tick freeze', 'frozen', *summon] if air else [*summon, 'ticks 30', 'cmd Alice tick freeze', 'frozen']
lines += [f'cmd Alice data get entity {mob} Pos']
if sprint:
lines.append('sprint Alice on')
if crit:
lines += falling(0.8)
lines += ['sleep 0.2', f'attack Alice {kind}', 'sleep 0.2', f'cmd Alice data get entity {mob} Motion']
lines += [f'hit {kind}', f'cmd Alice data get entity {mob} Motion']
for _ in range(20):
lines += ['cmd Alice tick step 1', 'sleep 0.25', f'cmd Alice data get entity {mob} Pos',
f'cmd Alice data get entity {mob} Motion']
lines += ['tickstep', f'cmd Alice data get entity {mob} Pos', f'cmd Alice data get entity {mob} Motion']
lines += ['cmd Alice tick unfreeze', 'cmd Alice kill @e[type=!player]']
if sprint:
lines.append('sprint Alice off')
@@ -114,7 +119,7 @@ def pvp_case(weapon=SWORD, sprint=False, crit=False, shield=False, armour=False,
if shield:
# A shield blocks from five ticks after it is raised.
lines += ['cmd Bob item replace entity @s weapon.mainhand with minecraft:shield', 'use Bob']
lines += ['sleep 1.5', *stand('Bob', V, 90, ticks=1)]
lines += ['ticks 30', *stand('Bob', V, 90, ticks=1)]
if sprint:
lines.append('sprint Alice on')
if crit:
@@ -124,10 +129,12 @@ def pvp_case(weapon=SWORD, sprint=False, crit=False, shield=False, armour=False,
if cooldown:
# A swing at the air first: the hit is weak, and the stronger one four ticks later lands inside Bob's hurt
# cooldown (LivingEntity.hurtServer: it hurts by the difference, with no knockback and no motion packet).
lines += ['sleep 0.2', 'punch Alice', 'attack Alice Bob']
lines += ['sleep 0.2', 'attack Alice Bob', 'sleep 0.3']
lines += ['ticks 4', 'punch Alice', 'attack Alice Bob']
# The gap between those two hits is an upper bound on the server's ticks, so a stalled server (which gets
# fewer in a wall's 0.2 s) still lands the second inside the cooldown; the rest is to have the server's ticks.
lines += ['sleep 0.2' if cooldown else 'ticks 4', 'attack Alice Bob', 'ticks 6']
if twice:
lines += [*stand('Bob', V, 90, ticks=20), 'sleep 1.5', 'attack Alice Bob', 'sleep 0.3']
lines += [*stand('Bob', V, 90, ticks=20), 'ticks 30', 'attack Alice Bob', 'ticks 6']
if sprint:
lines.append('sprint Alice off')
lines += stand('Alice', A, -90)
@@ -144,7 +151,7 @@ def mob_hits_case(kind):
hit = ['mark Bob', f'cmd Alice summon minecraft:{kind} {V[0] + 1} {V[1]} {V[2]} {{Rotation:[90f,0f]}}',
f'cmd Alice effect give {mob} minecraft:slowness 100000 127 true', 'await Bob', f'cmd Alice kill {mob}']
return ['cmd Bob clear @s', 'cmd Bob effect give @s minecraft:resistance 1000 4 true', 'cmd Alice gamemode creative',
*stand('Bob', V, 90), *hit, 'sleep 2', *stand('Bob', V, 90), *hit, 'sleep 1',
*stand('Bob', V, 90), *hit, 'ticks 40', *stand('Bob', V, 90), *hit, 'ticks 20',
'cmd Alice kill @e[type=!player]', 'cmd Alice gamemode survival']
@@ -173,7 +180,20 @@ CASES = {
}
SETUP = ['cmd Alice gamerule spawn_mobs false', 'cmd Alice gamerule spawn_monsters false',
'cmd Alice kill @e[type=!player]', 'cmd Alice gamerule advance_time false', 'cmd Alice time set midnight',
f'cmd Alice tp Alice {A[0]} {A[1]} {A[2]} -90 0', f'cmd Alice tp Bob {V[0]} {V[1]} {V[2]} 90 0', 'sleep 2']
f'cmd Alice tp Alice {A[0]} {A[1]} {A[2]} -90 0', f'cmd Alice tp Bob {V[0]} {V[1]} {V[2]} 90 0', 'ticks 40']
def server_data(server, selector, path, timeout=120):
"""An entity's data, asked from the console (its reply stays in the server's log, out of the clients')."""
deadline = time.time() + timeout
start = len(server.lines)
server.console(f'data get entity {selector} {path}')
while time.time() < deadline:
for line in server.lines[start:]:
if 'has the following entity data' in line:
return line.split('data: ', 1)[-1]
time.sleep(0.02)
raise AssertionError(f'{server.log_path.name}: no entity data in {timeout} s')
def server_dir(directory, world):
@@ -201,29 +221,72 @@ def run_side(label, server_command, client_command, build, world):
server.wait_for(r'Done \(')
with (build / f'{label}-client.log').open('w') as log:
client = subprocess.Popen([*map(str, client_command), '127.0.0.1', str(port), str(out)],
cwd=lab.ROOT, stdin=subprocess.PIPE, stdout=log, stderr=subprocess.STDOUT,
text=True)
cwd=lab.ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True)
def sync():
"""Every script line sent so far has run, and each client holds what its server sent before."""
for name in PLAYERS:
client.stdin.write(f'sync {name}\n')
client.stdin.flush()
for name in PLAYERS:
while (reply := client.stdout.readline()) != f'synced {name}\n':
if not reply:
raise AssertionError(f'{label}: the client exited (see {build}/{label}-client.log)')
log.write(reply)
def send(line):
if line.startswith('sleep '):
"""A script line for the client, or one of ours: the waits are on the server's game clock, not the
wall's, so a server that stalls under load still gets the ticks each case needs."""
word = line.split()[0]
if word == 'sleep':
time.sleep(float(line.split()[1]))
elif word == 'ticks':
sync()
wait_ticks(server, int(line.split()[1]))
sync()
elif word == 'frozen':
# The tick in flight when ticking was frozen has finished: the clock reads the same twice.
sync()
while game_time(server) != game_time(server):
pass
elif word == 'hit':
# Alice hits the mob. A frozen server still ticks its players, and the hit lands in Alice's
# next tick, not when the packet arrives: the mob's motion says when it has.
sync()
mob = f'@e[type=minecraft:{line.split()[1]},limit=1]'
before = server_data(server, mob, 'Motion')
client.stdin.write(f'attack Alice {line.split()[1]}\n')
client.stdin.flush()
while server_data(server, mob, 'Motion') == before:
time.sleep(0.05)
sync()
elif word == 'tickstep':
# `/tick step 1` answers at once and the tick runs after: the clock says when it has.
sync()
before = game_time(server)
client.stdin.write('cmd Alice tick step 1\n')
client.stdin.flush()
while game_time(server) == before:
time.sleep(0.05)
sync()
else:
client.stdin.write(line + '\n')
client.stdin.flush()
try:
for name in ('Alice', 'Bob'):
for name in PLAYERS:
send(f'join {name}')
server.wait_for(rf'{name} joined the game', timeout=120)
time.sleep(2)
wait_ticks(server, 40)
for line in SETUP:
send(line)
for case, lines in CASES.items():
send(f'step {case}')
for line in lines:
send(line)
time.sleep(0.5)
send('ticks 10')
send('end')
client.wait(timeout=60)
log.write(client.communicate(timeout=60)[0])
finally:
if client.poll() is None:
client.kill()