diff --git a/.github/workflows/parity-main.yml b/.github/workflows/parity-main.yml index c47005ab0..70fb33796 100644 --- a/.github/workflows/parity-main.yml +++ b/.github/workflows/parity-main.yml @@ -48,6 +48,9 @@ jobs: GH_TOKEN: ${{ github.token }} run: echo "sha=$(python3 tools/parity_main.py base)" >> "$GITHUB_OUTPUT" - name: The harnesses the merges since then can affect + id: harnesses + # A failure here is judged by the next step: the run passes when every failed harness passes again at HEAD. + continue-on-error: true env: PARITY_HOSTS: '' BASE_SHA: ${{ steps.base.outputs.sha }} @@ -55,8 +58,8 @@ jobs: echo "Merges since ${BASE_SHA}:" git log --first-parent --format=' %h %s' "$BASE_SHA..HEAD" python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400 - - name: The merge that broke them - if: failure() + - name: A flake, or the merge that broke them + if: steps.harnesses.outcome == 'failure' env: GH_TOKEN: ${{ github.token }} PARITY_HOSTS: '' @@ -65,7 +68,7 @@ jobs: rm -rf .local/parity-main-logs && cp -R .local/parity-runner/logs .local/parity-main-logs 2>/dev/null || true python3 tools/parity_main.py bisect "$BASE_SHA" --comment - name: Logs of the failed harnesses - if: failure() + if: steps.harnesses.outcome == 'failure' uses: actions/upload-artifact@v4 with: name: parity-main-logs diff --git a/tests/test_parity_main.py b/tests/test_parity_main.py index aa58375a6..46e3d0466 100644 --- a/tests/test_parity_main.py +++ b/tests/test_parity_main.py @@ -45,7 +45,7 @@ class ParityMainTests(unittest.TestCase): last_run.parent.mkdir(parents=True) last_run.write_text('{"failed": ["music"]}') real_run = parity_main.subprocess.run - for broken_from, blamed in ((None, None), (0, None), (2, "2")): + for broken_from, blamed, code in ((None, None, 0), (0, None, 1), (2, "2", 1)): comments = [] # git runs for real in the scratch repo; a `gh pr comment` is only recorded, never sent. fake_run = lambda command, **kwargs: (comments.append(command) if command[0] == "gh" @@ -55,7 +55,8 @@ class ParityMainTests(unittest.TestCase): and commits.index(git("rev-parse", "HEAD")) >= broken_from), \ mock.patch.object(parity_main.subprocess, "run", fake_run), \ mock.patch("builtins.print") as printed: - parity_main.bisect(commits[0], 60, comment=True) + self.assertEqual(parity_main.bisect(commits[0], 60, comment=True), code, + "0 only when the failures pass again at HEAD") if blamed: self.assertEqual([command[3] for command in comments], [blamed]) else: diff --git a/tools/parity_main.py b/tools/parity_main.py index 76fa49070..f61c9b028 100644 --- a/tools/parity_main.py +++ b/tools/parity_main.py @@ -78,34 +78,35 @@ def bisect(since, timeout, comment): failed = json.loads(LAST_RUN.read_text(encoding="utf-8")).get("failed") or [] if not failed: print("No failed harness recorded: nothing to bisect.") - return 0 + return 1 head = git("rev-parse", "HEAD").stdout.strip() commits = git("rev-list", "--first-parent", "--reverse", f"{since}..{head}").stdout.split() if not commits: print("No merge since the base.") - return 0 + return 1 def fails(commit): print(f"--- {commit[:9]} {git('log', '-1', '--format=%s', commit).stdout.strip()}", flush=True) git("checkout", "-q", "--detach", commit) return run_harnesses(failed, timeout) - def blame_nobody(why): + def blame_nobody(why, code): line = f"parity-main: {', '.join(failed)} {why}: no merge is named." print(line) if os.environ.get("GITHUB_STEP_SUMMARY"): with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: summary.write(f"### {line}\n") - return 0 + return code # A merge is named only when the harnesses fail again at HEAD and pass at the base: otherwise the search would # end on the newest merge (a flake, or a runner missing files, passes everywhere now) or the oldest one (the - # runner fails at every commit) and blame it for nothing. + # runner fails at every commit) and blame it for nothing. The exit code is the run's verdict: 0 only when every + # failed harness passes when run again at HEAD (the commit is good, so releases may follow it), 1 otherwise. try: if not fails(head): - return blame_nobody(f"pass when run again at {head[:9]}: a flake or the runner") + return blame_nobody(f"pass when run again at {head[:9]}: a flake, counted as a pass", 0) if fails(since): - return blame_nobody(f"fail at the base {since[:9]} too: the runner or a flake") + return blame_nobody(f"fail at the base {since[:9]} too: the runner or a flake", 1) culprit = first_bad(commits, fails) if len(commits) > 1 else commits[0] finally: git("checkout", "-q", "--detach", head) @@ -122,7 +123,7 @@ def bisect(since, timeout, comment): body = (f"🤖 Agent: parity-main found that this merge breaks {', '.join(f'`{name}`' for name in failed)} on main " f"(bisected over {len(commits)} merges: {url}). Releases wait until parity-main passes again.") subprocess.run(["gh", "pr", "comment", number.group(1), "--repo", REPO, "--body", body]) - return 0 + return 1 def main():