parity-main: a failure that passes again at HEAD counts as a pass

Twice today (runs 40301239508, 40311712302) mob_ai failed in parity-main, passed when bisect ran it again at
the same HEAD, and the run still ended red, so no game release could follow main for three hours. The
harness step now continues on error and the bisect step gives the verdict: 0 when every failed harness
passes again at HEAD, 1 when they fail at the base too or a merge is named.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/parity-main.yml

@@ -48,6 +48,9 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: echo "sha=$(python3 tools/parity_main.py base)" >> "$GITHUB_OUTPUT"
       - name: The harnesses the merges since then can affect
+        id: harnesses
+        # A failure here is judged by the next step: the run passes when every failed harness passes again at HEAD.
+        continue-on-error: true
         env:
           PARITY_HOSTS: ''
           BASE_SHA: ${{ steps.base.outputs.sha }}
@@ -55,8 +58,8 @@ jobs:
           echo "Merges since ${BASE_SHA}:"
           git log --first-parent --format='  %h %s' "$BASE_SHA..HEAD"
           python3 tools/parity.py --changed "$BASE_SHA" --timeout 2400
-      - name: The merge that broke them
-        if: failure()
+      - name: A flake, or the merge that broke them
+        if: steps.harnesses.outcome == 'failure'
         env:
           GH_TOKEN: ${{ github.token }}
           PARITY_HOSTS: ''
@@ -65,7 +68,7 @@ jobs:
           rm -rf .local/parity-main-logs && cp -R .local/parity-runner/logs .local/parity-main-logs 2>/dev/null || true
           python3 tools/parity_main.py bisect "$BASE_SHA" --comment
       - name: Logs of the failed harnesses
-        if: failure()
+        if: steps.harnesses.outcome == 'failure'
         uses: actions/upload-artifact@v4
         with:
           name: parity-main-logs

tests/test_parity_main.py

@@ -45,7 +45,7 @@ class ParityMainTests(unittest.TestCase):
             last_run.parent.mkdir(parents=True)
             last_run.write_text('{"failed": ["music"]}')
             real_run = parity_main.subprocess.run
-            for broken_from, blamed in ((None, None), (0, None), (2, "2")):
+            for broken_from, blamed, code in ((None, None, 0), (0, None, 1), (2, "2", 1)):
                 comments = []
                 # git runs for real in the scratch repo; a `gh pr comment` is only recorded, never sent.
                 fake_run = lambda command, **kwargs: (comments.append(command) if command[0] == "gh"
@@ -55,7 +55,8 @@ class ParityMainTests(unittest.TestCase):
                                        and commits.index(git("rev-parse", "HEAD")) >= broken_from), \
                      mock.patch.object(parity_main.subprocess, "run", fake_run), \
                      mock.patch("builtins.print") as printed:
-                    parity_main.bisect(commits[0], 60, comment=True)
+                    self.assertEqual(parity_main.bisect(commits[0], 60, comment=True), code,
+                                     "0 only when the failures pass again at HEAD")
                 if blamed:
                     self.assertEqual([command[3] for command in comments], [blamed])
                 else:

tools/parity_main.py

@@ -78,34 +78,35 @@ def bisect(since, timeout, comment):
         failed = json.loads(LAST_RUN.read_text(encoding="utf-8")).get("failed") or []
     if not failed:
         print("No failed harness recorded: nothing to bisect.")
-        return 0
+        return 1
     head = git("rev-parse", "HEAD").stdout.strip()
     commits = git("rev-list", "--first-parent", "--reverse", f"{since}..{head}").stdout.split()
     if not commits:
         print("No merge since the base.")
-        return 0
+        return 1
 
     def fails(commit):
         print(f"--- {commit[:9]} {git('log', '-1', '--format=%s', commit).stdout.strip()}", flush=True)
         git("checkout", "-q", "--detach", commit)
         return run_harnesses(failed, timeout)
 
-    def blame_nobody(why):
+    def blame_nobody(why, code):
         line = f"parity-main: {', '.join(failed)} {why}: no merge is named."
         print(line)
         if os.environ.get("GITHUB_STEP_SUMMARY"):
             with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary:
                 summary.write(f"### {line}\n")
-        return 0
+        return code
 
     # A merge is named only when the harnesses fail again at HEAD and pass at the base: otherwise the search would
     # end on the newest merge (a flake, or a runner missing files, passes everywhere now) or the oldest one (the
-    # runner fails at every commit) and blame it for nothing.
+    # runner fails at every commit) and blame it for nothing. The exit code is the run's verdict: 0 only when every
+    # failed harness passes when run again at HEAD (the commit is good, so releases may follow it), 1 otherwise.
     try:
         if not fails(head):
-            return blame_nobody(f"pass when run again at {head[:9]}: a flake or the runner")
+            return blame_nobody(f"pass when run again at {head[:9]}: a flake, counted as a pass", 0)
         if fails(since):
-            return blame_nobody(f"fail at the base {since[:9]} too: the runner or a flake")
+            return blame_nobody(f"fail at the base {since[:9]} too: the runner or a flake", 1)
         culprit = first_bad(commits, fails) if len(commits) > 1 else commits[0]
     finally:
         git("checkout", "-q", "--detach", head)
@@ -122,7 +123,7 @@ def bisect(since, timeout, comment):
         body = (f"🤖 Agent: parity-main found that this merge breaks {', '.join(f'`{name}`' for name in failed)} on main "
                 f"(bisected over {len(commits)} merges: {url}). Releases wait until parity-main passes again.")
         subprocess.run(["gh", "pr", "comment", number.group(1), "--repo", REPO, "--body", body])
-    return 0
+    return 1
 
 
 def main():