diff --git a/tools/parity_shard.py b/tools/parity_shard.py new file mode 100644 index 000000000..249235ffb --- /dev/null +++ b/tools/parity_shard.py @@ -0,0 +1,21 @@ +"""A sharded harness's share of its cases. + +A harness that declares PARITY_SHARDS = n runs as n entries in tools/parity.py, which sets PARITY_SHARD=i/n for +each. Outside the runner PARITY_SHARD is unset and a harness runs everything. Kept out of lab.py: every harness +imports lab.py, so a change there reruns them all. +""" +import os + + +def shard(items): + """Every n-th item from the i-th, or all of them outside the runner.""" + spec = os.environ.get("PARITY_SHARD") + if not spec: + return items + index, count = map(int, spec.split("/")) + return items[index - 1::count] + + +def label(): + """A directory name for this share ("2-4", or "all"): shards run at once and must not share outputs.""" + return os.environ.get("PARITY_SHARD", "all").replace("/", "-") diff --git a/tools/test_datafix_parity.py b/tools/test_datafix_parity.py index a2d11009c..4c189983f 100644 --- a/tools/test_datafix_parity.py +++ b/tools/test_datafix_parity.py @@ -33,6 +33,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import anvil_dump # noqa: E402 +import parity_shard # noqa: E402 ROOT = Path(__file__).resolve().parent.parent LOCAL = ROOT / '.local' / 'datafix' @@ -63,7 +64,7 @@ def java_classpath(): import test_model_baking_parity as baking lock, jars, _ = base.verify_artifacts() libraries = baking.client_libraries() - build = LOCAL / 'build' / os.environ.get('PARITY_SHARD', 'all').replace('/', '-') # shards compile apart + build = LOCAL / 'build' / parity_shard.label() # shards compile apart build.mkdir(parents=True, exist_ok=True) classes, dependencies, _ = harness.compile_oracle(build, 'VanillaDataFixOracle', jars, libraries, lock['minecraft']['java_major']) @@ -78,15 +79,6 @@ def build_rust(): return Path(meta['target_directory']) / 'release' / 'examples' / 'datafix_world' -def shard(items): - """This run's share of items when tools/parity.py runs the harness in parts (PARITY_SHARD=i/n).""" - spec = os.environ.get('PARITY_SHARD') - if not spec: - return items - index, count = map(int, spec.split('/')) - return items[index - 1::count] - - def run_vanilla(classpath, world, out, target=None): # The oracle replaces OUT itself: left in place, an unchanged run's outputs can be reused. log = out.parent / (out.name + '.log') @@ -226,7 +218,7 @@ def main(): band = args.band if band and not args.versions: # worlds in the layout of before 1.16, saved before the target worlds = [w for w in worlds if json.loads((world_path(w) / 'datafix-source.json').read_text())['data_version'] < band] - versions = shard(args.versions or worlds) + versions = parity_shard.shard(args.versions or worlds) classpath = java_classpath() binary = None if args.vanilla_only else Path(args.binary) if args.binary else build_rust() total = 0 diff --git a/tools/test_world_spawn_parity.py b/tools/test_world_spawn_parity.py index 9b495c3dc..4247a0ee7 100644 --- a/tools/test_world_spawn_parity.py +++ b/tools/test_world_spawn_parity.py @@ -29,6 +29,10 @@ Seeds: the player report's seed (1891623539983467248: Horizon spawned at -48 -48, vanilla at 152 145), the chunk generation harness's seeds and --random seeds from a fixed generator. +tools/parity.py runs it in PARITY_SHARDS parts, each creating the worlds of +every PARITY_SHARDS-th seed: each seed is a vanilla and a Horizon server start, +~15 s, and 60 seeds in one run were the longest harness (~25 min). + usage: test_world_spawn_parity.py [--random N] [--jobs N] [--origins-only] """ import argparse @@ -43,16 +47,18 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path try: - from . import anvil_dump, lab, oracle_harness as oh, test_rust_parity as base + from . import anvil_dump, lab, oracle_harness as oh, parity_shard, test_rust_parity as base from .test_dedicated_server_parity import Server, free_port except ImportError: import anvil_dump import lab import oracle_harness as oh + import parity_shard import test_rust_parity as base from test_dedicated_server_parity import Server, free_port NAME = 'world-spawn-parity' +PARITY_SHARDS = 4 REPORT = 1891623539983467248 SEEDS = [REPORT, 12345, -7046029254386353131, 987654321987, 4242, -1, 0x5DEECE66D, 20260607, 777, 0] RESOURCES = lab.RESOURCES @@ -107,7 +113,7 @@ def main(): parser.add_argument('--seed', type=int, action='append', help='only these seeds (repeatable)') args = parser.parse_args() lock, jars, libraries = base.verify_artifacts() - build = lab.LOCAL / NAME + build = lab.LOCAL / NAME / parity_shard.label() # shards run at once build.mkdir(parents=True, exist_ok=True) dependencies = os.pathsep.join(map(str, [jars['server'], *libraries])) classes = build / 'classes' @@ -116,7 +122,7 @@ def main(): '-d', classes, lab.ROOT / 'mods/horizon/src/test/java/dev/horizonlab/parity/VanillaWorldSpawnOracle.java'], build, 'compile-java') oh.run_logged(['cargo', 'build', '--release', '-p', 'horizon-world', '--example', 'spawn_origin'], build, 'compile-probe') - all_seeds = args.seed or seeds(args.random) + all_seeds = parity_shard.shard(args.seed or seeds(args.random)) started = time.time() vanilla, rust = origins(build, classes, dependencies, all_seeds)