horizon-airdef/tools/test_command_parse_parity.py

#!/usr/bin/env python3
"""Compare Horizon's command parser with vanilla's real Brigadier command tree.

Java (VanillaCommandParseOracle) builds vanilla's `Commands` dispatcher with
the real registries and data-pack loaded tags, exports the whole tree, and
parses a corpus of command strings. The corpus is generated by walking that
tree (every command and subcommand with valid and invalid sample values per
argument type) plus hand-written edge cases for selectors, NBT paths,
coordinates, ranges, resource locations and block/item predicates. For every
input the parse success, the exact error text and cursor, the parsed node
names and the completion suggestions at the end of the input must match.
"""
import argparse
import json
import os
from pathlib import Path
import platform
import subprocess
import sys

try:
    from . import lab, test_rust_parity as base
    from . import command_parse_corpus as corpus_module
except ImportError:
    import lab
    import test_rust_parity as base
    import command_parse_corpus as corpus_module


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--prepare', action='store_true')
    parser.add_argument('--permission', type=int, nargs='*', default=[4, 2, 0])
    parser.add_argument('--limit', type=int, default=0, help='print at most this many mismatches per class')
    parser.add_argument('--allow-gaps', action='store_true', help='report mismatches without failing')
    args = parser.parse_args()
    build = lab.LOCAL / 'command-parse-parity'
    build.mkdir(parents=True, exist_ok=True)
    report_path = build / 'report.json'
    report_path.unlink(missing_ok=True)
    if args.prepare:
        lab.prepare()
    lock, jars, libraries = base.verify_artifacts()
    oracle = lab.ROOT / 'mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCommandParseOracle.java'
    classes = build / 'classes'
    classes.mkdir(exist_ok=True)
    dependency_path = os.pathsep.join(map(str, libraries))

    def run(command, name):
        with (build / (name + '.log')).open('w', encoding='utf-8') as log:
            result = subprocess.run(list(map(str, command)), cwd=lab.ROOT, stdout=log, stderr=subprocess.STDOUT)
        if result.returncode:
            tail = (build / (name + '.log')).read_text(errors='replace')[-2500:]
            sys.exit(f'{name} failed ({result.returncode}):\n{tail}')

    run(['javac', '--release', lock['minecraft']['java_major'], '-proc:none', '-cp', str(jars['server']) + os.pathsep + dependency_path, '-d', classes, oracle], 'compile-java')

    def java(*arguments):
        run(['java', '-Xmx1G', '-cp', os.pathsep.join([str(classes), str(jars['server']), dependency_path]), 'dev.horizonlab.parity.VanillaCommandParseOracle', *arguments], 'java-' + arguments[0])

    java('registries', str(build))
    java('tree', str(build))
    tree = json.loads((build / 'tree.json').read_text())
    registries = json.loads((build / 'registries.json').read_text())
    corpus = corpus_module.corpus(tree, registries)
    (build / 'corpus.txt').write_text(''.join(line + '\n' for line in corpus), encoding='utf-8')
    run(['cargo', 'build', '--release', '-p', 'horizon-client', '--example', 'command_parse_fixture'], 'compile-rust')
    metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--no-deps', '--format-version', '1'], cwd=lab.ROOT, text=True))
    binary = Path(metadata['target_directory']) / 'release/examples' / ('command_parse_fixture.exe' if os.name == 'nt' else 'command_parse_fixture')
    resources = lab.LOCAL / 'horizon-client/26.3'
    total_failures = 0
    summary = []
    for permission in args.permission:
        level_dir = build / f'perm{permission}'
        level_dir.mkdir(exist_ok=True)
        (level_dir / 'registries.json').write_bytes((build / 'registries.json').read_bytes())
        stamp = level_dir / 'vanilla.stamp'
        digest = lab.digest(build / 'corpus.txt', 'sha256') + lab.digest(oracle, 'sha256')
        if not (stamp.exists() and stamp.read_text() == digest and (level_dir / 'vanilla.jsonl').exists()):
            java('run', str(level_dir), str(build / 'corpus.txt'), str(permission))
            stamp.write_text(digest)
        run([binary, resources, build / 'registries.json', build / 'corpus.txt', str(permission), level_dir / 'rust.jsonl'], f'rust{permission}')
        failures = compare(level_dir / 'vanilla.jsonl', level_dir / 'rust.jsonl', args.limit)
        count = sum(1 for _ in (level_dir / 'vanilla.jsonl').open())
        print(f'permission {permission}: {count - failures}/{count} commands match', flush=True)
        summary.append({'permission': permission, 'commands': count, 'mismatches': failures})
        total_failures += failures
    report = {'schema': 1, 'minecraft': lock['minecraft']['version'], 'platform': platform.platform(), 'results': summary,
              'scope': 'Parse success, error text and cursor, parsed node names and end-of-input suggestions for a tree-walked corpus.'}
    lab.write_json(report_path, report)
    if total_failures and not args.allow_gaps:
        sys.exit(f'{total_failures} command parse mismatches; see {build}')
    print(f'Command parse parity report: {report_path}')


def compare(vanilla_path, rust_path, limit):
    vanilla = [json.loads(line) for line in vanilla_path.open(encoding='utf-8')]
    rust = [json.loads(line) for line in rust_path.open(encoding='utf-8')]
    if len(vanilla) != len(rust):
        raise SystemExit(f'line count differs: vanilla {len(vanilla)} rust {len(rust)}')
    shown = {}
    failures = 0
    mismatches = []
    for v, r in zip(vanilla, rust):
        assert v['in'] == r['in'], (v['in'], r['in'])
        kind = None
        if 'crash' in v:
            continue  # oracle could not evaluate (missing server state)
        # Node names are internal to each implementation; only success matters.
        vp = {k: val for k, val in v['parse'].items() if k != 'nodes'}
        rp = {k: val for k, val in r['parse'].items() if k != 'nodes'}
        if vp != rp:
            kind = 'parse'
        elif v['sugg'] != r['sugg']:
            kind = 'suggestions'
        if kind:
            failures += 1
            mismatches.append({'kind': kind, 'in': v['in'], 'vanilla': v['parse'] if kind == 'parse' else v['sugg'], 'rust': r['parse'] if kind == 'parse' else r['sugg']})
    out = vanilla_path.with_name('mismatches.jsonl')
    out.write_text(''.join(json.dumps(m, ensure_ascii=False) + '\n' for m in mismatches), encoding='utf-8')
    for m in mismatches:
        n = shown.get(m['kind'], 0)
        if limit and n >= limit:
            continue
        shown[m['kind']] = n + 1
        print(m['kind'], repr(m['in']), '\n  vanilla', json.dumps(m['vanilla'], ensure_ascii=False)[:300], '\n  rust   ', json.dumps(m['rust'], ensure_ascii=False)[:300])
    return failures


if __name__ == '__main__':
    main()