diff --git a/tests/test_server_bench.py b/tests/test_server_bench.py new file mode 100644 index 000000000..f392297bc --- /dev/null +++ b/tests/test_server_bench.py @@ -0,0 +1,32 @@ +import unittest + +from tools.server_bench import work + + +def run(server, bots, sent, new, waited=0.0, behaviour="mixed"): + return {"server": server, "behaviour": behaviour, "bots": bots, "chunks_per_second": sent, + "unique_per_second": new, "waited_ticks_per_second": waited} + + +class ServerBenchWorkTests(unittest.TestCase): + def test_shares_of_the_busiest_server(self): + [row] = work([run("horizon", 10, 98.2, 36.5), run("vanilla", 10, 24.3, 19.1, 12.0)]) + self.assertFalse(row["equal"]) + self.assertEqual(row["servers"]["horizon"]["chunks_per_second"], {"value": 98.2, "share": 1.0}) + self.assertEqual(row["servers"]["vanilla"]["chunks_per_second"]["share"], 0.247) + self.assertEqual(row["servers"]["vanilla"]["waited_ticks_per_second"], 12.0) + + def test_equal_within_ten_percent_or_a_chunk(self): + [row] = work([run("horizon", 25, 236, 28.9), run("vanilla", 25, 252, 28.9)]) + self.assertTrue(row["equal"]) + # One bot: 0.3 against 1.1 new chunks a second is within a chunk. + [row] = work([run("horizon", 1, 9.9, 0.3), run("vanilla", 1, 9.9, 1.1)]) + self.assertTrue(row["equal"]) + + def test_only_counts_run_on_several_servers(self): + failed = dict(run("vanilla", 50, 0, 0), error="watchdog") + self.assertEqual(work([run("horizon", 50, 450, 10), failed, run("horizon", 1, 9, 0.3)]), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/server_bench.py b/tools/server_bench.py index c79d6aab1..0ba99617a 100755 --- a/tools/server_bench.py +++ b/tools/server_bench.py @@ -19,7 +19,9 @@ The bots can run on another machine, so they don't take the server's CPU: `--bot `--bots-bin` naming horizon-bots there and `--connect` the address it reaches this machine at. `--nice 19` runs every server at the lowest priority (a shared machine). The explore bots fly as fast as each server sends them chunks, so a faster server gets more terrain to generate; `--explore-speed` slows them to a rate -both servers keep up with, for equal work. +both servers keep up with, for equal work. After the runs, the work table compares the servers at each bot +count: chunks sent and new chunks generated a second, against the busiest server's. A run whose work is more +than 10 % apart isn't a fair comparison of its tick times. The vanilla side runs the pinned server jar from tools/lab.py prepare (your own copy). It needs Java 25. The Horizon side runs target/release/horizon-server with the engine beside it; build both first: @@ -306,6 +308,38 @@ def run_once(kind, bots, behaviour, args, out): return result +# How far apart two servers' chunks a second (sent, and new) may be and still count as the same work: 10 %, +# or a chunk a second when there are few. +EQUAL_WORK = 0.10 + + +def work(runs): + """Each pass and bot count that ran on more than one server: every server's chunks sent and new chunks a + second as a share of the busiest one's, the ticks a second its bots stood waiting for their own chunk, + and whether the work was equal (within EQUAL_WORK on both).""" + groups = {} + for r in runs: + if 'error' not in r: + groups.setdefault((r['behaviour'], r['bots']), []).append(r) + rows = [] + for (behaviour, bots), group in groups.items(): + if len(group) < 2: + continue + row = {'behaviour': behaviour, 'bots': bots, 'servers': {}, 'equal': True} + for key in ('chunks_per_second', 'unique_per_second'): + top = max(r.get(key) or 0 for r in group) + for r in group: + value = r.get(key) or 0 + share = round(value / top, 3) if top else 1.0 + row['servers'].setdefault(r['server'], {})[key] = {'value': value, 'share': share} + if top - value > max(EQUAL_WORK * top, 1.0): + row['equal'] = False + for r in group: + row['servers'][r['server']]['waited_ticks_per_second'] = r.get('waited_ticks_per_second') + rows.append(row) + return rows + + def main(): parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) parser.add_argument('--servers', default='horizon,vanilla') @@ -346,6 +380,7 @@ def main(): 'java': subprocess.run(['java', '--version'], capture_output=True, text=True).stdout.splitlines()[:1], 'settings': {**vars(args), 'out': str(args.out), 'properties': PROPERTIES}, 'runs': runs, + 'work': work(runs), } path = args.out or out / 'report.json' path.parent.mkdir(parents=True, exist_ok=True) @@ -356,11 +391,20 @@ def main(): if 'error' in r: print(f'{r["server"]:8} {r["behaviour"]:8} {r["bots"]:>4} failed: {r["error"].splitlines()[0]}') continue - per_bot = r['chunks_per_bot'].get(r['behaviour']) + # A mixed pass: every bot's. + per_bot = r['chunks_per_bot'].get(r['behaviour']) or round((r['chunks_total'] or 0) / r['bots']) print(f'{r["server"]:8} {r["behaviour"]:8} {r["bots"]:>4} {r["start_seconds"]:>8} {r["mspt"]["average"]!s:>6} ' f'{r["mspt"]["p95"]!s:>6} {r["mspt"]["p99"]!s:>6} {r["rss_mb"]["mean"]!s:>7} {r["cpu_percent"]!s:>6} ' f'{r["chunks_per_second"]!s:>8} {per_bot!s:>6} {r["resent"]!s:>6} {r["pulled_back"]!s:>6} ' f'{r["bots_joined"]!s:>6}') + if report['work']: + print(f'\nWork, against the busiest server (chunks a second sent, new; ticks a second bots waited):') + for row in report['work']: + cells = ' '.join( + f'{server} {w["chunks_per_second"]["value"]} ({w["chunks_per_second"]["share"]:.0%}), ' + f'{w["unique_per_second"]["value"]} ({w["unique_per_second"]["share"]:.0%}), ' + f'{w["waited_ticks_per_second"]}' for server, w in row['servers'].items()) + print(f'{row["behaviour"]:8} {row["bots"]:>4} {"equal" if row["equal"] else "NOT EQUAL"} {cells}') print(f'\nWrote {path}')