diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..59560e262 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,79 @@ +name: nightly audit +# Every vanilla harness, from scratch, on main, at 03:00 Paris time on the Mac mini's runner (it holds the +# maintainer's own Minecraft files). The resources are prepared again first when the preparer or its exporters +# changed (tools/ci/vanilla-data.sh). A harness that passed in the previous audit and fails now gets an issue +# labelled parity-regression (one per harness); its issue closes when it passes again +# (tools/ci/parity_issues.py). The bugs session picks those up. See docs/release-flow.md. +# gh workflow run "nightly audit" # now +# gh workflow run "nightly audit" -f dry_run=true # the audit, without touching issues +on: + schedule: + # GitHub's schedules are in UTC: both, and the first step keeps the one that is 03:00 in Paris. + - cron: '7 1 * * *' + - cron: '7 2 * * *' + workflow_dispatch: + inputs: + dry_run: + description: Run the audit, but only print what it would do with issues + type: boolean + default: false +permissions: + contents: read + issues: write +concurrency: + group: nightly-audit + cancel-in-progress: false +jobs: + audit: + runs-on: [self-hosted, horizon-game] + timeout-minutes: 150 + steps: + - name: 03:00 in Paris + id: when + run: | + hour=$(TZ=Europe/Paris date +%H) + if [[ "${{ github.event_name }}" == schedule && "$hour" != 03 ]]; then + echo "It is ${hour}:00 in Paris: the other schedule runs the audit." + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@v4 + if: steps.when.outputs.run == 'true' + with: + fetch-depth: 0 + persist-credentials: false + # Keeps .local/ (parity state, probe builds, the previous audit) and target/. + clean: false + - uses: dtolnay/rust-toolchain@stable + if: steps.when.outputs.run == 'true' + - name: Tools from the runner + if: steps.when.outputs.run == 'true' + run: bash tools/ci/runner-env.sh + - name: Vanilla data, prepared again if the preparer changed + if: steps.when.outputs.run == 'true' + run: bash tools/ci/vanilla-data.sh + - name: Every harness + id: audit + if: steps.when.outputs.run == 'true' + env: + PARITY_HOSTS: '' + run: | + echo "since=$(date +%Y-%m-%dT%H:%M:%S)" >> "$GITHUB_OUTPUT" + set +e + python3 tools/parity.py --all --timeout 1800 2>&1 | tee "$RUNNER_TEMP/audit.log" + echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + { echo '```'; tail -40 "$RUNNER_TEMP/audit.log"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + - name: Issues for the harnesses that regressed + if: steps.when.outputs.run == 'true' && always() && steps.audit.outputs.since + env: + GH_TOKEN: ${{ github.token }} + run: python3 tools/ci/parity_issues.py --since "${{ steps.audit.outputs.since }}" ${{ inputs.dry_run && '--dry-run' || '' }} | tee -a "$GITHUB_STEP_SUMMARY" + - name: Logs of the failed harnesses + if: steps.when.outputs.run == 'true' && steps.audit.outputs.exit != '0' + uses: actions/upload-artifact@v4 + with: + name: nightly-parity-logs + path: .local/parity-runner/logs/ + if-no-files-found: ignore + retention-days: 14 diff --git a/docs/release-flow.md b/docs/release-flow.md index 78f084252..a9bfc166d 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -211,3 +211,19 @@ see the previous one as the newest; launchers that already updated keep running. With several game releases a day, the newest launcher release soon falls out of the newest 100 releases the launcher and the website read: both then look the newest `launcher-v` tags up one by one (`git/matching-refs/tags/launcher-v`, then `releases/tags/`). + +## Nightly audit + +`.github/workflows/nightly.yml` runs every harness from scratch (`tools/parity.py --all`) on main at 03:00 Paris +time, on the Mac mini's runner, after preparing the resources again if the preparer or its exporters changed. +`tools/ci/parity_issues.py` then compares the results with the previous audit +(`.local/parity-runner/audit.json` in the runner's checkout): + +- a harness that passed last night and fails now gets an issue labelled `parity-regression`, titled + `Parity regression: `, with the commit, the run and the end of its log (one issue per harness: an open + one gets a comment instead); +- a harness with an open issue that passes again gets its issue closed. + +The bugs session picks the issues up. The audit's logs are the run's `nightly-parity-logs` artifact; harnesses +listed `linux_only` are skipped there (see above). By hand: `gh workflow run "nightly audit"` (add +`-f dry_run=true` to leave issues alone). diff --git a/tests/test_parity_issues.py b/tests/test_parity_issues.py new file mode 100644 index 000000000..1c202c272 --- /dev/null +++ b/tests/test_parity_issues.py @@ -0,0 +1,27 @@ +import unittest + +from tools.ci import parity_issues as issues + + +class ParityIssuesTests(unittest.TestCase): + def test_results_merge_shards_and_skip_earlier_runs(self): + state = {"harnesses": { + "food": {"status": "pass", "finished": "2026-10-15T03:10:00"}, + "datafix#1": {"status": "pass", "finished": "2026-10-15T03:20:00"}, + "datafix#2": {"status": "fail", "finished": "2026-10-15T03:21:00"}, + "light": {"status": "fail", "finished": "2026-10-14T03:30:00"}, + "copied": {"status": "pass", "finished": "2026-10-15T03:30:00", "shared": True}, + }} + self.assertEqual(issues.results(state, "2026-10-15T03:00:00"), {"food": "pass", "datafix": "fail"}) + + def test_plan_opens_comments_and_closes(self): + now = {"a": "fail", "b": "fail", "c": "pass", "d": "fail", "e": "pass"} + previous = {"a": "pass", "b": "fail", "c": "fail", "e": "pass"} + self.assertEqual(issues.plan(now, previous, {"b": 7, "c": 8}), + [("open", "a"), ("comment", "b"), ("close", "c")]) + # A harness that never passed in an audit (d) gets no issue; one that keeps passing (e) nothing. + self.assertEqual(issues.plan(now, {}, {}), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ci/parity_issues.py b/tools/ci/parity_issues.py new file mode 100644 index 000000000..f8fb47345 --- /dev/null +++ b/tools/ci/parity_issues.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Turns the nightly audit's results into GitHub issues (.github/workflows/nightly.yml). + + python3 tools/ci/parity_issues.py --since 2026-10-15T03:00:00 [--dry-run] + +Reads tools/parity.py's state (.local/parity-runner/state.json) for the harnesses that finished since --since, +shards merged into their harness, and compares them with the previous audit (.local/parity-runner/audit.json): + +- a harness that fails now and passed in the previous audit gets one open issue labelled parity-regression, + titled "Parity regression: " (an open one is commented on instead of opening another); +- a harness with an open issue that still fails gets a comment on it; +- a harness with an open issue that passes again gets its issue closed. + +The first audit only records the results. Needs the gh CLI; --dry-run prints what it would do. +""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] +STATE = ROOT / ".local" / "parity-runner" / "state.json" +AUDIT = ROOT / ".local" / "parity-runner" / "audit.json" +LOGS = ROOT / ".local" / "parity-runner" / "logs" +LABEL = "parity-regression" +TITLE = "Parity regression: " + + +def results(state, since): + """{harness: "pass" | "fail"} of the harnesses that finished since `since`; a harness fails if a shard does.""" + found = {} + for name, record in state.get("harnesses", {}).items(): + if record.get("finished", "") < since or record.get("shared"): + continue + harness = name.split("#")[0] + found[harness] = "fail" if record["status"] != "pass" or found.get(harness) == "fail" else "pass" + return found + + +def plan(now, previous, open_issues): + """The actions to take: (kind, harness) with kind open, comment or close. open_issues: {harness: number}.""" + actions = [] + for harness, status in sorted(now.items()): + if status == "fail" and harness in open_issues: + actions.append(("comment", harness)) + elif status == "fail" and previous.get(harness) == "pass": + actions.append(("open", harness)) + elif status == "pass" and harness in open_issues: + actions.append(("close", harness)) + return actions + + +def log_tail(harness, lines=60): + logs = sorted(LOGS.glob(f"{harness}*.log")) + text = "\n".join(path.read_text(errors="replace") for path in logs) + return "\n".join(text.splitlines()[-lines:]) + + +def gh(*arguments, dry_run=False): + if dry_run: + print("would run: gh " + " ".join(arguments[:4]) + " ...") + return "" + return subprocess.run(["gh", *arguments], cwd=ROOT, check=True, capture_output=True, text=True).stdout + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--since", required=True, help="the audit's start (the state's local time format)") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + now = results(json.loads(STATE.read_text()), args.since) + previous = json.loads(AUDIT.read_text()) if AUDIT.exists() else None + listed = json.loads(gh("issue", "list", "--label", LABEL, "--state", "open", "--limit", "200", + "--json", "number,title") or "[]") + open_issues = {issue["title"][len(TITLE):]: issue["number"] for issue in listed if issue["title"].startswith(TITLE)} + commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip() + run = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{os.environ.get('GITHUB_REPOSITORY', '')}" \ + f"/actions/runs/{os.environ.get('GITHUB_RUN_ID', '')}" + failing = sorted(harness for harness, status in now.items() if status == "fail") + print(f"{len(now)} harnesses ran: {len(now) - len(failing)} passed, {len(failing)} failed" + + (f" ({', '.join(failing)})" if failing else "")) + if previous is None: + print("First audit: results recorded, no issues opened.") + actions = plan(now, {}, open_issues) + else: + actions = plan(now, previous, open_issues) + for kind, harness in actions: + details = (f"Nightly audit of `{commit[:12]}` ({run}).\n\n" + f"Reproduce: `python3 tools/parity.py {harness}`\n\n
End of the log\n\n" + f"```\n{log_tail(harness)}\n```\n
\n") + print(f"{kind}: {harness}") + if kind == "open": + gh("issue", "create", "--label", LABEL, "--title", TITLE + harness, "--body", + f"`{harness}` passed in the previous nightly audit and fails now.\n\n" + details, dry_run=args.dry_run) + elif kind == "comment": + gh("issue", "comment", str(open_issues[harness]), "--body", "Still failing.\n\n" + details, dry_run=args.dry_run) + else: + gh("issue", "close", str(open_issues[harness]), "--comment", + f"Passes again in the nightly audit of `{commit[:12]}` ({run}).", dry_run=args.dry_run) + if not args.dry_run: + merged = dict(previous or {}) + merged.update(now) + AUDIT.write_text(json.dumps(merged, indent=1, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())