server_bench: compare each bot count's work across servers, and flag runs that aren't equal

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_server_bench.py

@@ -0,0 +1,32 @@
+import unittest
+
+from tools.server_bench import work
+
+
+def run(server, bots, sent, new, waited=0.0, behaviour="mixed"):
+    return {"server": server, "behaviour": behaviour, "bots": bots, "chunks_per_second": sent,
+            "unique_per_second": new, "waited_ticks_per_second": waited}
+
+
+class ServerBenchWorkTests(unittest.TestCase):
+    def test_shares_of_the_busiest_server(self):
+        [row] = work([run("horizon", 10, 98.2, 36.5), run("vanilla", 10, 24.3, 19.1, 12.0)])
+        self.assertFalse(row["equal"])
+        self.assertEqual(row["servers"]["horizon"]["chunks_per_second"], {"value": 98.2, "share": 1.0})
+        self.assertEqual(row["servers"]["vanilla"]["chunks_per_second"]["share"], 0.247)
+        self.assertEqual(row["servers"]["vanilla"]["waited_ticks_per_second"], 12.0)
+
+    def test_equal_within_ten_percent_or_a_chunk(self):
+        [row] = work([run("horizon", 25, 236, 28.9), run("vanilla", 25, 252, 28.9)])
+        self.assertTrue(row["equal"])
+        # One bot: 0.3 against 1.1 new chunks a second is within a chunk.
+        [row] = work([run("horizon", 1, 9.9, 0.3), run("vanilla", 1, 9.9, 1.1)])
+        self.assertTrue(row["equal"])
+
+    def test_only_counts_run_on_several_servers(self):
+        failed = dict(run("vanilla", 50, 0, 0), error="watchdog")
+        self.assertEqual(work([run("horizon", 50, 450, 10), failed, run("horizon", 1, 9, 0.3)]), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/server_bench.py

@@ -19,7 +19,9 @@ The bots can run on another machine, so they don't take the server's CPU: `--bot
 `--bots-bin` naming horizon-bots there and `--connect` the address it reaches this machine at. `--nice 19`
 runs every server at the lowest priority (a shared machine). The explore bots fly as fast as each server
 sends them chunks, so a faster server gets more terrain to generate; `--explore-speed` slows them to a rate
-both servers keep up with, for equal work.
+both servers keep up with, for equal work. After the runs, the work table compares the servers at each bot
+count: chunks sent and new chunks generated a second, against the busiest server's. A run whose work is more
+than 10 % apart isn't a fair comparison of its tick times.
 
 The vanilla side runs the pinned server jar from tools/lab.py prepare (your own copy). It needs Java 25. The
 Horizon side runs target/release/horizon-server with the engine beside it; build both first:
@@ -306,6 +308,38 @@ def run_once(kind, bots, behaviour, args, out):
     return result
 
 
+# How far apart two servers' chunks a second (sent, and new) may be and still count as the same work: 10 %,
+# or a chunk a second when there are few.
+EQUAL_WORK = 0.10
+
+
+def work(runs):
+    """Each pass and bot count that ran on more than one server: every server's chunks sent and new chunks a
+    second as a share of the busiest one's, the ticks a second its bots stood waiting for their own chunk,
+    and whether the work was equal (within EQUAL_WORK on both)."""
+    groups = {}
+    for r in runs:
+        if 'error' not in r:
+            groups.setdefault((r['behaviour'], r['bots']), []).append(r)
+    rows = []
+    for (behaviour, bots), group in groups.items():
+        if len(group) < 2:
+            continue
+        row = {'behaviour': behaviour, 'bots': bots, 'servers': {}, 'equal': True}
+        for key in ('chunks_per_second', 'unique_per_second'):
+            top = max(r.get(key) or 0 for r in group)
+            for r in group:
+                value = r.get(key) or 0
+                share = round(value / top, 3) if top else 1.0
+                row['servers'].setdefault(r['server'], {})[key] = {'value': value, 'share': share}
+                if top - value > max(EQUAL_WORK * top, 1.0):
+                    row['equal'] = False
+        for r in group:
+            row['servers'][r['server']]['waited_ticks_per_second'] = r.get('waited_ticks_per_second')
+        rows.append(row)
+    return rows
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
     parser.add_argument('--servers', default='horizon,vanilla')
@@ -346,6 +380,7 @@ def main():
         'java': subprocess.run(['java', '--version'], capture_output=True, text=True).stdout.splitlines()[:1],
         'settings': {**vars(args), 'out': str(args.out), 'properties': PROPERTIES},
         'runs': runs,
+        'work': work(runs),
     }
     path = args.out or out / 'report.json'
     path.parent.mkdir(parents=True, exist_ok=True)
@@ -356,11 +391,20 @@ def main():
         if 'error' in r:
             print(f'{r["server"]:8} {r["behaviour"]:8} {r["bots"]:>4}  failed: {r["error"].splitlines()[0]}')
             continue
-        per_bot = r['chunks_per_bot'].get(r['behaviour'])
+        # A mixed pass: every bot's.
+        per_bot = r['chunks_per_bot'].get(r['behaviour']) or round((r['chunks_total'] or 0) / r['bots'])
         print(f'{r["server"]:8} {r["behaviour"]:8} {r["bots"]:>4} {r["start_seconds"]:>8} {r["mspt"]["average"]!s:>6} '
               f'{r["mspt"]["p95"]!s:>6} {r["mspt"]["p99"]!s:>6} {r["rss_mb"]["mean"]!s:>7} {r["cpu_percent"]!s:>6} '
               f'{r["chunks_per_second"]!s:>8} {per_bot!s:>6} {r["resent"]!s:>6} {r["pulled_back"]!s:>6} '
               f'{r["bots_joined"]!s:>6}')
+    if report['work']:
+        print(f'\nWork, against the busiest server (chunks a second sent, new; ticks a second bots waited):')
+        for row in report['work']:
+            cells = '  '.join(
+                f'{server} {w["chunks_per_second"]["value"]} ({w["chunks_per_second"]["share"]:.0%}), '
+                f'{w["unique_per_second"]["value"]} ({w["unique_per_second"]["share"]:.0%}), '
+                f'{w["waited_ticks_per_second"]}' for server, w in row['servers'].items())
+            print(f'{row["behaviour"]:8} {row["bots"]:>4}  {"equal" if row["equal"] else "NOT EQUAL"}  {cells}')
     print(f'\nWrote {path}')