diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index 1fc31d8cb..9f678b43b 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -26,35 +26,46 @@ jobs: steps: - uses: actions/checkout@v4 with: - # The commit the event names: a fork's later pushes need another run-parity label. + # The commit the event names: a fork's later pushes need another run-parity label. The next step merges + # main into it. ref: ${{ inputs.head }} fetch-depth: 0 persist-credentials: false # Keeps .local/ (parity state, probe builds, oracle results) and target/ between runs. clean: false + - name: The head merged into main (what merging it would land) + id: merge + env: + BASE_SHA: ${{ inputs.base }} + HEAD_SHA: ${{ inputs.head }} + run: | + # Harness fixes and game fixes that reached main since the branch was made then apply here too: a head + # tested alone failed harnesses already fixed on main (server_knockback before #2114, mob_ai's heap before + # #2270, save_roundtrip before #2212). Selection then compares with this main, so it sees only the pull + # request's own changes. A head that conflicts with main is tested alone. + set -euo pipefail + git merge --abort 2>/dev/null || true + git fetch -q --no-tags origin +refs/heads/main:refs/remotes/origin/main + main=$(git rev-parse origin/main) + if git -c user.name=parity -c user.email=parity@localhost merge -q --no-edit --no-ff "$main" >/dev/null 2>&1; then + echo "base=$main" >> "$GITHUB_OUTPUT" + echo "Tested: head ${HEAD_SHA:0:9} merged into main ${main:0:9}" | tee -a "$GITHUB_STEP_SUMMARY" + else + git merge --abort + echo "base=$BASE_SHA" >> "$GITHUB_OUTPUT" + echo "::warning::head ${HEAD_SHA:0:9} conflicts with main ${main:0:9}: tested alone, without main's fixes" + echo "Tested: head ${HEAD_SHA:0:9} alone (it conflicts with main ${main:0:9})" | tee -a "$GITHUB_STEP_SUMMARY" + fi - uses: dtolnay/rust-toolchain@stable - name: Tools from the runner run: bash tools/ci/runner-env.sh - name: Vanilla data, and resources prepared by this preparer - env: - BASE_SHA: ${{ inputs.base }} - run: | - # main's script: one written before a fix to it left a real .local/horizon-client without sounds.json on - # the runner, and every later job there failed `subtitle`. A branch that changes the script runs its own. - set -euo pipefail - if git rev-parse --verify -q origin/main >/dev/null \ - && git diff --quiet "$(git merge-base "$BASE_SHA" HEAD)" HEAD -- tools/ci/vanilla-data.sh; then - git show origin/main:tools/ci/vanilla-data.sh > "$RUNNER_TEMP/vanilla-data.sh" - bash "$RUNNER_TEMP/vanilla-data.sh" - else - bash tools/ci/vanilla-data.sh - fi + run: bash tools/ci/vanilla-data.sh - name: The prepared output is the one the preparer's OUTPUT_VERSION says env: - BASE_SHA: ${{ inputs.base }} + BASE_SHA: ${{ steps.merge.outputs.base }} run: | - # The job runs main's workflow on the head's files: a head made before OUTPUT_VERSION (#1947) has neither - # the tool nor the number, so there is nothing to check. + # A head made before OUTPUT_VERSION (#1947) that conflicts with main has neither the tool nor the number. if [ ! -f tools/prepare_output.py ] || ! grep -q OUTPUT_VERSION crates/horizon-prepare/src/lib.rs; then echo "This head's preparer has no OUTPUT_VERSION: nothing to check" exit 0 @@ -65,47 +76,8 @@ jobs: - name: The harnesses this change can affect env: PARITY_HOSTS: '' - BASE_SHA: ${{ inputs.base }} - run: | - # main's selection (tools/parity.py --changed --list with main's parity.py and parity_reach.py), so a - # branch made before a better selection landed gets it too; the head's own runner then runs those - # harnesses. A branch that changes the selection itself selects with its own code. - set -euo pipefail - own() { python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400; } # world_spawn takes ~1480 s on a busy runner - fork_point=$(git merge-base "$BASE_SHA" HEAD) - if ! git rev-parse --verify -q origin/main >/dev/null || ! git diff --quiet "$fork_point" HEAD -- tools/parity.py tools/parity_reach.py; then - own - exit - fi - had_reach=$(git ls-files tools/parity_reach.py) - git show origin/main:tools/parity.py > tools/parity.py - git show origin/main:tools/parity_reach.py > tools/parity_reach.py - status=0 - python3 tools/parity.py --changed "$BASE_SHA" --list > .local/parity-selection.txt || status=$? - git checkout -- tools/parity.py - if [ -n "$had_reach" ]; then git checkout -- tools/parity_reach.py; else rm -f tools/parity_reach.py; fi - cat .local/parity-selection.txt - if [ "$status" != 0 ]; then - echo "main's selection failed: the head's own selection runs" - own - exit - fi - names=$(sed '$d' .local/parity-selection.txt | grep -E '^[A-Za-z0-9_#]+$' || true) - # The summary line says how many would run: a count that doesn't match the names read (a format change) - # must not let the job pass untested. - expected=$(tail -1 .local/parity-selection.txt | sed -nE 's/^([0-9]+) of [0-9]+ harnesses would run$/\1/p') - count=$(printf '%s' "$names" | grep -c . || true) - if [ -z "$expected" ] || [ "$expected" != "$count" ]; then - echo "main's selection listed $count names, its summary says '${expected:-?}': the head's own selection runs" - own - exit - fi - if [ "$count" = 0 ]; then - echo "Nothing to run." - exit 0 - fi - # shellcheck disable=SC2086 - python3 tools/parity.py $names --timeout 2400 + BASE_SHA: ${{ steps.merge.outputs.base }} + run: python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400 # world_spawn takes ~1480 s on a busy runner - name: Logs of the failed harnesses if: failure() uses: actions/upload-artifact@v4 diff --git a/docs/parity-workflow.md b/docs/parity-workflow.md index 27505d9de..c0cf3aec5 100644 --- a/docs/parity-workflow.md +++ b/docs/parity-workflow.md @@ -133,6 +133,13 @@ when named and in the nightly audit (`--all`). LAN writes `results.json` with the host mode and each result beside its screenshots and logs. A skipped case is recorded explicitly and does not count as passing coverage. +`known_flaky` in `tools/parity_hosts.json` lists harnesses that fail now and then for a +cause not fixed yet, each with its reason. One that fails runs once more from +scratch, and a pass then counts. `parity.py` prints a `RETRIED` line for it +in its summary and in the CI job's summary (PR parity and parity-main), so the +flake stays visible. No other harness gets this retry. Fix the cause, then take +the harness off the list. + The game-binary fixture audit keeps these complementary direct rule tests: - `command_harness.rs`: command feedback, styled spans and result callbacks, diff --git a/docs/release-flow.md b/docs/release-flow.md index ce7934824..b0d435c29 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -32,6 +32,12 @@ copy is: the Mac mini, his Linux desktop and playhorizon (below). Their runners between runs (`clean: false`), so the probes build incrementally into `.local/parity-target` and earlier passes stand. +The job tests the pull request's head merged into main's current tip, which is what merging it would land: harness +and game fixes that reached main after the branch was made apply to it too, and an old branch runs main's runner +scripts and selection unless it changes them. The base for the selection is that main commit, so only the pull +request's own changes count. The job summary names the main commit. A head that conflicts with main is tested +alone, with a warning. The check is still reported on the head. + 1. `tools/ci/runner-env.sh` puts the machine's Python 3.13 and the JDK `research-lock.json` pins on the PATH (Homebrew and `java_home` on the Mac; on Linux what `linux-runner.sh` pinned, else nixpkgs), and sccache when the machine has it. @@ -50,9 +56,6 @@ stand. modules, Java oracles and every Rust source their probes compile) changed since the merge base, plus any harness nothing is recorded about yet, and runs those that have no pass for their current inputs. A change to `Cargo.lock` or `research-lock.json` affects every harness. Each harness has 25 minutes, the job 45. - The selection runs with main's `tools/parity.py` and `tools/parity_reach.py` (`--changed --list`), so a - branch made before a better selection landed gets it too, and the head's own `parity.py` then runs the named - harnesses. A branch that changes either file selects with its own code. Failed harnesses' logs are uploaded as the `parity-logs` artifact. diff --git a/tests/test_parity_runner.py b/tests/test_parity_runner.py index 518f23ba4..49a88f0b1 100644 --- a/tests/test_parity_runner.py +++ b/tests/test_parity_runner.py @@ -290,9 +290,6 @@ class OracleReuseTests(unittest.TestCase): self.assertEqual(self.output.read_text(), "b") -if __name__ == "__main__": - unittest.main() - class LargeOutputTests(unittest.TestCase): def test_a_pass_clears_only_its_own_directory(self): @@ -323,3 +320,84 @@ class LargeOutputTests(unittest.TestCase): listing.write_text(f"{own / 'server.txt'}\n{own / 'client.txt'}\n") parity.clear_large_outputs("state_provider#2", trace) self.assertEqual(sorted(path.name for path in own.iterdir()), ["report.json", "server.txt"]) + + +class FakeExecutor: + """Runs nothing: each start of a harness takes the next status from its script, a list.""" + name, dead, max_peak = "fake", False, None + + def __init__(self): + self.started, self.finished = [], [] + + def free(self): + return 1 + + def accepts(self, name): + return True + + def fits(self, expected, only_here=True): + return True + + def start(self, label, script, reuse, expected, shard=None): + self.started.append((label, reuse)) + self.finished.append((label, {"status": script.pop(0), "seconds": 1.0, "dependencies": []})) + return True + + def poll(self): + finished, self.finished = self.finished, [] + return finished + + +class KnownFlakyTests(unittest.TestCase): + def run_harnesses(self, scripts, flaky, rounds=None): + executor = FakeExecutor() + state = {"harnesses": {name: {"status": "pass", "oracle_reuse": True} for name in scripts}} + with tempfile.TemporaryDirectory() as tmp, mock.patch.multiple(parity, STATE_DIR=Path(tmp), SHARED=Path(tmp)), \ + mock.patch.object(parity, "save_state"), mock.patch.object(parity, "time") as clock: + clock.monotonic.return_value = 0.0 + results, _ = parity.run(scripts, state, None, "env", [executor], True, rounds, flaky) + return results, executor.started + + def test_a_known_flaky_failure_runs_once_more_from_scratch_and_says_so(self): + results, started = self.run_harnesses({"dedicated_server": ["fail", "pass"]}, {"dedicated_server": "timer"}) + self.assertEqual(results["dedicated_server"]["status"], "pass") + self.assertEqual(results["dedicated_server"]["retried"], "known flaky") + # The first run reused oracle results; a known flaky harness doesn't lose that by flaking. + self.assertTrue(results["dedicated_server"]["oracle_reuse"]) + self.assertEqual(started, [("dedicated_server", True), ("dedicated_server", False)]) + + def test_only_once(self): + results, started = self.run_harnesses({"dedicated_server": ["fail", "fail"]}, {"dedicated_server": "timer"}) + self.assertEqual(results["dedicated_server"]["status"], "fail") + self.assertEqual(len(started), 2) + + def test_other_harnesses_keep_only_the_reused_oracle_retry(self): + results, started = self.run_harnesses({"food": ["fail", "pass"]}, {"dedicated_server": "timer"}) + self.assertEqual((results["food"]["status"], results["food"]["retried"]), ("pass", "reused oracle results")) + self.assertEqual(started, [("food", True), ("food", False)]) + self.assertFalse(results["food"]["oracle_reuse"]) + + def test_repeated_runs_get_no_retry(self): + results, started = self.run_harnesses({"dedicated_server": ["pass", "fail"]}, {"dedicated_server": "timer"}, + rounds={"dedicated_server": 2}) + self.assertEqual((results["dedicated_server"]["status"], len(started)), ("fail", 2)) + + def test_retries_reach_the_job_summary(self): + with tempfile.TemporaryDirectory() as tmp: + summary = Path(tmp) / "summary.md" + with mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(summary)}), \ + mock.patch("builtins.print"): + parity.report_retries({"dedicated_server": {"status": "pass", "retried": "known flaky"}, + "food": {"status": "pass"}}, {"dedicated_server": "timer"}) + text = summary.read_text() + self.assertIn("RETRIED dedicated_server: failed, then passed on its one retry (known flaky: timer", text) + self.assertNotIn("food", text) + + def test_the_known_flaky_list_has_a_reason_per_harness(self): + flaky = parity.known_flaky(parity.host_config()) + self.assertTrue(all(isinstance(reason, str) and len(reason) > 20 for reason in flaky.values())) + self.assertTrue(set(flaky) <= {parity.base_name(name) for name in parity.harnesses()}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/parity.py b/tools/parity.py index dbff8c146..a5f7f2db1 100644 --- a/tools/parity.py +++ b/tools/parity.py @@ -26,6 +26,11 @@ change costs only the Rust side. A harness that fails while reusing oracle results is rerun fresh, and stops reusing them if it then passes. --all and --fresh never reuse them. +A harness listed in parity_hosts.json's "known_flaky" (each with its reason) +that fails runs once more from scratch instead, and a pass then counts. Every +retry is printed as a RETRIED line in the summary and in the CI job's summary. +Fix the cause, then take the harness off the list: nothing else gets this retry. + Harness probes build into .local/parity-target with release optimisations but without LTO and with 16 codegen units (`PROFILE` below), which builds in seconds instead of minutes. Rust does not let those settings change results: @@ -1014,11 +1019,26 @@ def harness_of(label): return label.split("~")[0] -def run(selected, state, hasher, environment_digest, executors, reuse_oracles, rounds=None): +def keep_first_log(label): + """Move a harness's log aside before its retry writes a new one; returns where it went.""" + log = STATE_DIR / "logs" / f"{label}.log" + kept = log.with_name(f"{label}.first-run.log") + try: + log.replace(kept) + except OSError: + return log # a remote host's run: its log stays on that machine + return kept + + +def run(selected, state, hasher, environment_digest, executors, reuse_oracles, rounds=None, known_flaky=None): """Run the selected harnesses on the executors. rounds: how many times each must run (default once); its runs - must agree, and a harness that both passes and fails is reported as flaky and recorded as failing.""" + must agree, and a harness that both passes and fails is reported as flaky and recorded as failing. + known_flaky ({name: reason}, tools/parity_hosts.json): a harness listed there that fails its single round runs + once more, from scratch; its record keeps `retried` (the first run's log is kept beside its own).""" records = state["harnesses"] rounds = rounds or {} + known_flaky = known_flaky or {} + second_chance = set() # known flaky harnesses that already failed once in this run # Longest first, so the slow harnesses don't start last; a harness's later rounds after its first. queue = sorted((name if index == 1 else f"{name}~{index}" for name in selected for index in range(1, rounds.get(name, 1) + 1)), key=lambda label: (-records.get(harness_of(label), {}).get("seconds", 1e9), label)) @@ -1041,7 +1061,9 @@ def run(selected, state, hasher, environment_digest, executors, reuse_oracles, r if pick is None: break name = harness_of(pick) - reuse = reuse_oracles and records.get(name, {}).get("oracle_reuse", True) and pick not in retried + # A retry runs the oracle again too: it may be the side that failed (mob_ai's heap). + reuse = reuse_oracles and records.get(name, {}).get("oracle_reuse", True) and pick not in retried \ + and name not in second_chance if not executor.start(pick, selected[name], reuse, records.get(name, {}).get("peak_bytes", DEFAULT_MEMORY), SHARDS.get(name)): break @@ -1066,9 +1088,12 @@ def run(selected, state, hasher, environment_digest, executors, reuse_oracles, r queue.insert(0, label) continue passed = record["status"] == "pass" - if not passed and reused and label == name and records.get(name, {}).get("status") != "fail": + flaky_retry = rounds.get(name, 1) == 1 and base_name(name) in known_flaky + if not passed and reused and label == name and records.get(name, {}).get("status") != "fail" \ + and not flaky_retry: # A new failure might come from a reused oracle result: run it again from scratch. Later rounds reuse - # what the first just validated, so their failures count (a flaky harness). + # what the first just validated, so their failures count (a flaky harness). A known flaky harness + # gets its own retry below instead, which keeps reusing oracle results afterwards. retried.add(label) queue.insert(0, label) print(f"retry {label} without reused oracle results", flush=True) @@ -1086,8 +1111,19 @@ def run(selected, state, hasher, environment_digest, executors, reuse_oracles, r if len(done[name]) < rounds.get(name, 1): continue runs = done.pop(name) + if runs[0]["status"] != "pass" and flaky_retry and name not in second_chance: + second_chance.add(name) + first_log = keep_first_log(label) + print(f"RETRIED {name}: failed, and it is a known flaky harness ({known_flaky[base_name(name)]}): " + f"running it once more (first run's log: {first_log})", flush=True) + queue.insert(0, label) + continue record = dict(runs[0], status="pass" if all(run["status"] == "pass" for run in runs) else "fail", rounds=len(runs)) + if name in second_chance: + record["retried"] = "known flaky" + elif label in retried: + record["retried"] = "reused oracle results" if len({run["status"] for run in runs}) > 1: record["flaky"] = [run["status"] for run in runs] print(f"FLAKY {name}: {', '.join(record['flaky'])} in {len(runs)} runs", flush=True) @@ -1309,12 +1345,36 @@ def platform_skips(config): return dict(config.get("linux_only", {})) +def known_flaky(config): + """Harnesses tools/parity_hosts.json lists as failing now and then for a cause not fixed yet: one that fails runs + once more, and passing then counts, reported as RETRIED. {name: reason}""" + return dict(config.get("known_flaky", {})) + + def named_only(config): """Harnesses tools/parity_hosts.json keeps out of every automatic selection (--changed, --failed, the default, --flake-check): they run only when named or with --all, the nightly audit. {name: reason}""" return dict(config.get("named_only", {})) +def report_retries(results, flaky): + """Say loudly which harnesses needed their one retry (a known flaky one, or one that failed with reused oracle + results), here and in the CI job's summary: a pass on the retry counts, so this is where flakes stay visible.""" + def why(name, record): + if record["retried"] == "known flaky": + return (f"known flaky: {flaky.get(base_name(name), '?')}; first run's log: " + f"{STATE_DIR / 'logs' / (name + '.first-run.log')}") + return "its first run reused oracle results; it no longer reuses them" if record["status"] == "pass" else \ + "its first run reused oracle results" + lines = [f"RETRIED {name}: failed, then {'passed' if record['status'] == 'pass' else 'failed again'} on its one " + f"retry ({why(name, record)})" for name, record in sorted(results.items()) if record.get("retried")] + for line in lines: + print(line, flush=True) + if lines and os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write("### Harnesses that failed once, then ran again\n" + "".join(f"- {line}\n" for line in lines)) + + def finished_run(failed): """Record that this run completed, for tools/gate.py: a crashed runner never writes it.""" STATE_DIR.mkdir(parents=True, exist_ok=True) @@ -1463,7 +1523,8 @@ def main(): reuse_oracles = not (args.fresh or args.all) executors = [*remotes, LocalExecutor(jobs, int(memory_available() * 0.75), args.timeout, REMOTE_LOCAL_PEAK if remotes and args.jobs is None else None)] - results, elapsed = run(selected, state, hasher, environment_key(hasher), executors, reuse_oracles, rounds) + results, elapsed = run(selected, state, hasher, environment_key(hasher), executors, reuse_oracles, rounds, + known_flaky(config)) finally: for executor in remotes: executor.stop() @@ -1471,6 +1532,7 @@ def main(): failed = sorted(name for name, record in results.items() if record["status"] != "pass") finished_run(failed) print(f"\n{len(results) - len(failed)} passed, {len(failed)} failed in {elapsed / 60:.1f} min.") + report_retries(results, known_flaky(config)) for index, name in enumerate(failed): log = STATE_DIR / 'logs' / (name + '.log') print(f" {name}: {log}") diff --git a/tools/parity_hosts.json b/tools/parity_hosts.json index b56106e4d..42b9a46fd 100644 --- a/tools/parity_hosts.json +++ b/tools/parity_hosts.json @@ -42,5 +42,10 @@ "linux_only": {}, "named_only": { "lan_integration": "the whole lan_e2e suite: ~40 min a run on a loaded runner, and single checks flake under load; run it by name, or tools/lan_e2e.py" + }, + "known_flaky": { + "dedicated_server": "its Java client moves to the next step on a 1.5 s timer, not on the server's reply: under load a respawn's add_entity/update_attributes or entity_event 19 lands in the wrong step (failed, then passed at the same head, 10-26)", + "remote_fireball": "samples the fireball on the wall clock, so a slow host sees it in too few samples (desktop runner, 3 times on 10-26)", + "world_reopen": "counted 4 minecarts instead of 3 once on a loaded Mac mini runner, then passed at the same head (10-26)" } }