Merge pull request #2858 from ludovic111/feat/nightly-set
🤖 Agent: Merged by Ludovic's orchestrator after review (nightly-2: the /nightly set library; tools/ci/ is hand-merged).
This commit is contained in:
@@ -0,0 +1,266 @@
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from tools.ci import nightly_set as ns
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NOW = "2026-10-28T12:00:00+00:00"
|
||||
PATTERNS = ns.guarded_patterns((ROOT / ".github/CODEOWNERS").read_text())
|
||||
|
||||
|
||||
def pr(n, head="h", state="open", files=("crates/horizon-world/src/lib.rs",), base="main"):
|
||||
return ns.PullRequest(n, state, f"{head}{n}", f"Fix {n}", f"dev{n}", base, list(files))
|
||||
|
||||
|
||||
def lookup(*prs):
|
||||
known = {p.number: p for p in prs}
|
||||
return lambda n: known.get(n)
|
||||
|
||||
|
||||
def run(state, text, login="kacpherek", *prs):
|
||||
return ns.apply(state, ns.parse(text), login, NOW, lookup(*prs), PATTERNS)
|
||||
|
||||
|
||||
class ParseTests(unittest.TestCase):
|
||||
def test_commands(self):
|
||||
self.assertEqual(ns.parse("/nightly 2640 #2642, 2640"), ns.Command("set", [2640, 2642]))
|
||||
self.assertEqual(ns.parse("/nightly add 2650"), ns.Command("add", [2650]))
|
||||
self.assertEqual(ns.parse("/Nightly REMOVE 2640"), ns.Command("remove", [2640]))
|
||||
self.assertEqual(ns.parse("/nightly clear"), ns.Command("clear", []))
|
||||
self.assertEqual(ns.parse("/nightly status"), ns.Command("status", []))
|
||||
self.assertEqual(ns.parse("/nightly"), ns.Command("status", []))
|
||||
self.assertEqual(ns.parse("/nightly dry-run 1 2"), ns.Command("dry-run", [1, 2]))
|
||||
self.assertEqual(ns.parse("\n /nightly 7\nthanks!"), ns.Command("set", [7]))
|
||||
|
||||
def test_other_comments_are_not_commands(self):
|
||||
for body in ("", "LGTM", "please /nightly 2640", "/nightlyx 1", "> /nightly 1"):
|
||||
self.assertIsNone(ns.parse(body), body)
|
||||
|
||||
def test_wrong_commands_are_answered(self):
|
||||
for body in ("/nightly please", "/nightly add", "/nightly clear 5", "/nightly 1 `rm -rf`",
|
||||
"/nightly " + " ".join(str(n) for n in range(1, 32))):
|
||||
with self.assertRaises(ns.Refused, msg=body):
|
||||
ns.parse(body)
|
||||
|
||||
def test_answers_never_quote_markdown_or_mentions(self):
|
||||
with self.assertRaises(ns.Refused) as e:
|
||||
ns.parse("/nightly @ludovic111`x`")
|
||||
self.assertNotIn("@", str(e.exception).split("isn't")[0])
|
||||
self.assertNotIn("`x`", str(e.exception))
|
||||
|
||||
|
||||
class AllowTests(unittest.TestCase):
|
||||
LIST = "# trusted\nkacpherek # added by Ludovic\nrdbtCVS\n\nnot a login!\n"
|
||||
|
||||
def test_listed_reviewers_and_ludovic(self):
|
||||
reviewers = ns.read_reviewers(self.LIST)
|
||||
ns.authorize("kacpherek", reviewers)
|
||||
ns.authorize("RDBTcvs", reviewers)
|
||||
ns.authorize("ludovic111", reviewers)
|
||||
for login in ("levzzz5154", "someone", "not"):
|
||||
with self.assertRaises(ns.Refused):
|
||||
ns.authorize(login, reviewers)
|
||||
|
||||
def test_paused_stops_reviewers_not_ludovic(self):
|
||||
reviewers = ns.read_reviewers(self.LIST + "paused # Ludovic, 2026-10-29\n")
|
||||
self.assertTrue(reviewers.paused)
|
||||
with self.assertRaises(ns.Refused):
|
||||
ns.authorize("kacpherek", reviewers)
|
||||
ns.authorize("ludovic111", reviewers)
|
||||
|
||||
class RefusalTests(unittest.TestCase):
|
||||
def test_what_codeowners_guards_is_refused(self):
|
||||
for path in (".github/workflows/x.yml", "tools/ci/nightly.py", "crates/horizon-release-sign/src/lib.rs",
|
||||
"crates/horizon-server/src/update.rs", "launcher/src/backend.rs", "launcher/links.json",
|
||||
"launcher/src/signed.rs", "tools/release_train.py"):
|
||||
self.assertTrue(ns.guarded(path, PATTERNS), path)
|
||||
for path in ("crates/horizon-client/src/game.rs", "launcher/src/main.rs", "tools/ci.py", "docs/x.md",
|
||||
"crates/horizon-server/src/main.rs"):
|
||||
self.assertFalse(ns.guarded(path, PATTERNS), path)
|
||||
self.assertTrue(ns.guarded("tools/ci/x.py", ns.guarded_patterns(""))) # even without CODEOWNERS
|
||||
|
||||
def test_a_set_with_a_refused_pull_request_changes_nothing(self):
|
||||
state = {"prs": []}
|
||||
bad = [pr(2, files=["tools/ci/nightly.py"]), pr(3, state="closed"), pr(4, state="merged"), pr(5, base="x")]
|
||||
for p in bad:
|
||||
with self.assertRaises(ns.Refused):
|
||||
run(state, f"/nightly 1 {p.number}", "kacpherek", pr(1), p)
|
||||
with self.assertRaises(ns.Refused):
|
||||
run(state, "/nightly 1 9", "kacpherek", pr(1)) # 9 doesn't exist
|
||||
|
||||
def test_at_most_thirty(self):
|
||||
state = {"prs": [ns.entry(pr(n), "a", NOW) for n in range(1, 31)]}
|
||||
with self.assertRaises(ns.Refused):
|
||||
run(state, "/nightly add 31", "kacpherek", pr(31))
|
||||
|
||||
|
||||
class SetTests(unittest.TestCase):
|
||||
def test_set_add_remove_clear(self):
|
||||
r = run({}, "/nightly 1 2", "kacpherek", pr(1), pr(2))
|
||||
self.assertTrue(r.build)
|
||||
self.assertEqual([e["number"] for e in r.state["prs"]], [1, 2])
|
||||
self.assertEqual(r.state["prs"][0]["asked_by"], "kacpherek")
|
||||
# add takes the newest head again and keeps its place
|
||||
r = run(r.state, "/nightly add 1 3", "rdbtCVS", pr(1, head="new"), pr(3))
|
||||
self.assertEqual([(e["number"], e["head"]) for e in r.state["prs"]], [(1, "new1"), (2, "h2"), (3, "h3")])
|
||||
r = run(r.state, "/nightly remove 2", "rdbtCVS")
|
||||
self.assertEqual([e["number"] for e in r.state["prs"]], [1, 3])
|
||||
r = run(r.state, "/nightly clear", "kacpherek")
|
||||
self.assertEqual(r.state["prs"], [])
|
||||
self.assertTrue(r.build)
|
||||
|
||||
def test_status_and_dry_run_change_nothing(self):
|
||||
state = run({}, "/nightly 1", "kacpherek", pr(1)).state
|
||||
status = run(state, "/nightly status", "levzzz5154")
|
||||
self.assertIs(status.state, state)
|
||||
self.assertFalse(status.build)
|
||||
self.assertIn("#1 Fix 1", status.answer)
|
||||
dry = run(state, "/nightly dry-run 2 3", "kacpherek", pr(2), pr(3))
|
||||
self.assertIs(dry.state, state)
|
||||
self.assertEqual([e["number"] for e in dry.dry_run], [2, 3])
|
||||
|
||||
def test_ludovics_remove_wins(self):
|
||||
state = run({}, "/nightly 1 2", "kacpherek", pr(1), pr(2)).state
|
||||
state = run(state, "/nightly remove 2", "ludovic111").state
|
||||
self.assertEqual(state["blocked"], [2])
|
||||
for text in ("/nightly add 2", "/nightly 1 2", "/nightly dry-run 2"):
|
||||
with self.assertRaises(ns.Refused):
|
||||
run(state, text, "kacpherek", pr(1), pr(2))
|
||||
state = run(state, "/nightly add 2", "ludovic111", pr(2)).state
|
||||
self.assertEqual(state["blocked"], [])
|
||||
self.assertEqual([e["number"] for e in run(state, "/nightly add 2", "kacpherek", pr(2)).state["prs"]], [1, 2])
|
||||
|
||||
def test_titles_never_mention_anyone(self):
|
||||
p = pr(1)
|
||||
p.title = "Fix @everyone\nsecond line"
|
||||
answer = run({}, "/nightly 1", "kacpherek", p).answer
|
||||
self.assertNotIn("@everyone", answer)
|
||||
self.assertNotIn("second line\n", answer)
|
||||
|
||||
def test_builds_are_numbered_per_day(self):
|
||||
self.assertEqual(ns.next_build("2026-10-28", []), "nightly/build-2026-10-28-1")
|
||||
self.assertEqual(ns.next_build("2026-10-28", ["nightly/build-2026-10-28-1", "nightly/build-2026-10-28-4"]),
|
||||
"nightly/build-2026-10-28-5")
|
||||
|
||||
|
||||
def sh(repo, *args, env=None):
|
||||
return subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True, text=True,
|
||||
env={**os.environ, **(env or {})}).stdout.strip()
|
||||
|
||||
|
||||
class AssemblyTests(unittest.TestCase):
|
||||
"""A fixture repository: main and three pull requests, two of them touching the same line."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.repo = self.tmp.name
|
||||
ident = {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t",
|
||||
"GIT_COMMITTER_EMAIL": "t@t", "GIT_AUTHOR_DATE": "2026-10-28T10:00:00Z",
|
||||
"GIT_COMMITTER_DATE": "2026-10-28T10:00:00Z"}
|
||||
self.env = ident
|
||||
sh(self.repo, "init", "-q", "-b", "main")
|
||||
self.write({"a.txt": "a\n", "b.txt": "b\n", "c.txt": "c\n"})
|
||||
self.main = self.commit("main")
|
||||
self.p1 = self.branch("p1", {"a.txt": "a from p1\n"})
|
||||
self.p2 = self.branch("p2", {"b.txt": "b from p2\n", ".gitattributes": "* filter=evil\n"})
|
||||
self.p3 = self.branch("p3", {"a.txt": "a from p3\n"})
|
||||
sh(self.repo, "checkout", "-q", "main")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def write(self, files):
|
||||
for name, text in files.items():
|
||||
Path(self.repo, name).write_text(text)
|
||||
|
||||
def commit(self, message):
|
||||
sh(self.repo, "add", "-A")
|
||||
sh(self.repo, "commit", "-q", "-m", message, env=self.env)
|
||||
return sh(self.repo, "rev-parse", "HEAD")
|
||||
|
||||
def branch(self, name, files):
|
||||
sh(self.repo, "checkout", "-q", "-b", name, self.main)
|
||||
self.write(files)
|
||||
return self.commit(name)
|
||||
|
||||
def test_clean_merges_in_order_and_the_same_inputs_give_the_same_commit(self):
|
||||
built = ns.assemble(self.main, [(1, self.p1), (2, self.p2)], cwd=self.repo)
|
||||
self.assertEqual(sh(self.repo, "show", f"{built}:a.txt"), "a from p1")
|
||||
self.assertEqual(sh(self.repo, "show", f"{built}:b.txt"), "b from p2")
|
||||
self.assertEqual(sh(self.repo, "rev-list", "--count", f"{self.main}..{built}", "--first-parent"), "2")
|
||||
self.assertEqual(ns.assemble(self.main, [(1, self.p1), (2, self.p2)], cwd=self.repo), built)
|
||||
# nothing was checked out: the working tree is still main's
|
||||
self.assertEqual(Path(self.repo, "a.txt").read_text(), "a\n")
|
||||
self.assertFalse(Path(self.repo, ".gitattributes").exists())
|
||||
|
||||
def test_two_pull_requests_that_conflict_are_named(self):
|
||||
with self.assertRaises(ns.Conflict) as e:
|
||||
ns.assemble(self.main, [(1, self.p1), (2, self.p2), (3, self.p3)], cwd=self.repo)
|
||||
self.assertEqual((e.exception.number, e.exception.other, e.exception.files), (3, 1, ["a.txt"]))
|
||||
self.assertIn("#1 and #3 conflict in `a.txt`", e.exception.text())
|
||||
|
||||
def test_a_conflict_with_main_says_so(self):
|
||||
sh(self.repo, "checkout", "-q", "main")
|
||||
self.write({"a.txt": "a from main\n"})
|
||||
main = self.commit("main moves")
|
||||
with self.assertRaises(ns.Conflict) as e:
|
||||
ns.assemble(main, [(1, self.p1)], cwd=self.repo)
|
||||
self.assertIsNone(e.exception.other)
|
||||
self.assertIn("conflicts with main", e.exception.text())
|
||||
|
||||
def test_changed_files_finds_guarded_paths(self):
|
||||
self.assertEqual(sorted(ns.changed_files(self.main, self.p2, cwd=self.repo)), [".gitattributes", "b.txt"])
|
||||
|
||||
|
||||
class StateTests(unittest.TestCase):
|
||||
"""The nightly-state branch on a local bare repository standing in for origin."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.origin, self.repo = Path(self.tmp.name, "origin.git"), Path(self.tmp.name, "work")
|
||||
subprocess.run(["git", "init", "-q", "--bare", str(self.origin)], check=True)
|
||||
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
|
||||
sh(self.repo, "remote", "add", "origin", str(self.origin))
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_first_save_then_load(self):
|
||||
self.assertEqual(ns.load_state(str(self.repo)), ({}, "", None))
|
||||
ns.update_state(lambda st, b: ({"prs": [1]}, b + "{}\n", "first"), "t", cwd=str(self.repo))
|
||||
state, builds, head = ns.load_state(str(self.repo))
|
||||
self.assertEqual((state, builds), ({"prs": [1]}, "{}\n"))
|
||||
self.assertIn("first", sh(self.repo, "log", "-1", "--format=%s", head))
|
||||
|
||||
def test_a_run_that_pushed_first_is_kept(self):
|
||||
ns.update_state(lambda st, b: ({"prs": [1]}, b, "one"), "t", cwd=str(self.repo))
|
||||
other = Path(self.tmp.name, "other")
|
||||
subprocess.run(["git", "init", "-q", str(other)], check=True)
|
||||
sh(other, "remote", "add", "origin", str(self.origin))
|
||||
seen = []
|
||||
|
||||
def add_two(st, b):
|
||||
seen.append(list(st.get("prs", [])))
|
||||
if len(seen) == 1: # meanwhile another run saves first
|
||||
ns.update_state(lambda s2, b2: ({"prs": s2["prs"] + [3]}, b2, "other"), "t", cwd=str(other))
|
||||
return {"prs": st["prs"] + [2]}, b, "two"
|
||||
final = ns.update_state(add_two, "t", cwd=str(self.repo))
|
||||
self.assertEqual(seen, [[1], [1, 3]])
|
||||
self.assertEqual(final["prs"], [1, 3, 2])
|
||||
self.assertEqual(ns.load_state(str(self.repo))[0]["prs"], [1, 3, 2])
|
||||
|
||||
def test_builds_today_lists_the_days_branches(self):
|
||||
sh(self.repo, "commit", "-q", "--allow-empty", "-m", "x", env={
|
||||
"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"})
|
||||
head = sh(self.repo, "rev-parse", "HEAD")
|
||||
for name in ("nightly/build-2026-10-28-1", "nightly/build-2026-10-28-2", "nightly/build-2026-10-27-1"):
|
||||
self.assertTrue(ns.push(head, name, "t", cwd=str(self.repo)))
|
||||
self.assertEqual(sorted(ns.builds_today("2026-10-28", cwd=str(self.repo))),
|
||||
["nightly/build-2026-10-28-1", "nightly/build-2026-10-28-2"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,528 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Trusted reviewers pick the pull requests a Nightly holds (docs/release-flow.md "Nightly channel").
|
||||
|
||||
nightly_set.py command one `/nightly …` comment (nightly-command.yml): checks who asked, updates the set,
|
||||
starts the build and answers in the thread
|
||||
|
||||
A comment by Ludovic or by a login in .github/nightly-reviewers.txt, on any issue or pull request:
|
||||
|
||||
/nightly 2640 2642 the set becomes these pull requests, merged onto main in this order; builds now
|
||||
/nightly add 2650 adds one, or takes its newest head again; builds now
|
||||
/nightly remove 2640 drops one; builds now. Ludovic's remove also keeps reviewers from adding it back
|
||||
/nightly clear empties the set: the Nightly is the proven union or main again; builds now
|
||||
/nightly status what the set holds
|
||||
/nightly dry-run 2640 builds main + these and signs with a throwaway key; publishes nothing, the set stays
|
||||
|
||||
The set (set.json on the nightly-state branch) keeps each pull request at the head it had when someone asked for
|
||||
it; the scheduled nightly takes their newest heads and drops the merged and closed ones (nightly.py pick).
|
||||
builds.jsonl there records every build's set, for the stats. Nothing here runs a pull request's code: its commits
|
||||
are fetched and merged with `git merge-tree --write-tree`, which writes git objects and never a working tree, so no
|
||||
file of theirs reaches the disk and no hook, filter or script of theirs runs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
OWNER = "ludovic111"
|
||||
REVIEWERS = ".github/nightly-reviewers.txt"
|
||||
CODEOWNERS = ".github/CODEOWNERS"
|
||||
# What CODEOWNERS guards today, refused even if the file moved: what builds, signs and checks releases.
|
||||
ALWAYS_GUARDED = ["/.github/", "/tools/ci/", "/crates/horizon-release-sign/", "/crates/horizon-server/src/update.rs",
|
||||
"/launcher/src/backend.rs", "/launcher/links.json"]
|
||||
STATE_BRANCH = "nightly-state"
|
||||
BUILD_PREFIX = "nightly/build-"
|
||||
MAX_PRS = 30
|
||||
MAX_BUILDS_PER_DAY = 6
|
||||
BOT = ("github-actions[bot]", "41898283+github-actions[bot]@users.noreply.github.com")
|
||||
# Git never runs anything of the merged commits; these make sure nothing of the runner's either.
|
||||
SAFE_GIT = ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", "-c", "merge.renormalize=false"]
|
||||
NUMBER = re.compile(r"^#?(\d{1,6}),?$")
|
||||
LOGIN = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$")
|
||||
VERBS = ("add", "remove", "clear", "status", "dry-run")
|
||||
USAGE = ("Use `/nightly 2640 2642`, `/nightly add 2650`, `/nightly remove 2640`, `/nightly clear`, "
|
||||
"`/nightly status` or `/nightly dry-run 2640`.")
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
"""A command that changes nothing; its text is the answer."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Command:
|
||||
verb: str # set, add, remove, clear, status, dry-run
|
||||
prs: list[int] = field(default_factory=list)
|
||||
|
||||
def text(self) -> str:
|
||||
words = ["/nightly"] + ([] if self.verb == "set" else [self.verb]) + [str(n) for n in self.prs]
|
||||
return " ".join(words)
|
||||
|
||||
|
||||
def safe(word: str) -> str:
|
||||
"""A word of a comment, fit to quote back: no markdown, no mentions."""
|
||||
return re.sub(r"[^\w#:.-]", "", word)[:40] or "?"
|
||||
|
||||
|
||||
def parse(body: str) -> Command | None:
|
||||
"""The command on a comment's first line; None when it isn't one. Raises Refused when it is one, but wrong."""
|
||||
line = next((line.strip() for line in body.splitlines() if line.strip()), "")
|
||||
words = line.split()
|
||||
if not words or words[0].lower() != "/nightly":
|
||||
return None
|
||||
rest = words[1:]
|
||||
verb = "status" if not rest else "set"
|
||||
if rest and rest[0].lower() in VERBS:
|
||||
verb = rest.pop(0).lower()
|
||||
prs: list[int] = []
|
||||
for word in rest:
|
||||
m = NUMBER.match(word)
|
||||
if not m:
|
||||
raise Refused(f"`{safe(word)}` isn't a pull request number. {USAGE}")
|
||||
if (n := int(m.group(1))) not in prs:
|
||||
prs.append(n)
|
||||
if verb in ("clear", "status") and prs:
|
||||
raise Refused(f"`/nightly {verb}` takes no numbers. {USAGE}")
|
||||
if verb not in ("clear", "status") and not prs:
|
||||
raise Refused(f"`/nightly {'' if verb == 'set' else verb + ' '}` needs pull request numbers. {USAGE}")
|
||||
if len(prs) > MAX_PRS:
|
||||
raise Refused(f"A Nightly holds at most {MAX_PRS} pull requests; that's {len(prs)}.")
|
||||
return Command(verb, prs)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Reviewers:
|
||||
logins: set[str] # lowercase
|
||||
paused: bool
|
||||
|
||||
|
||||
def read_reviewers(text: str) -> Reviewers:
|
||||
"""The list file: one login per line, `#` comments, and a `paused` line that stops every reviewer."""
|
||||
logins, paused = set(), False
|
||||
for raw in text.splitlines():
|
||||
line = raw.split("#", 1)[0].strip()
|
||||
if not line:
|
||||
continue
|
||||
words = line.split()
|
||||
if len(words) != 1: # not a login: ignored rather than half-read
|
||||
continue
|
||||
word = words[0]
|
||||
if word.lower() == "paused":
|
||||
paused = True
|
||||
elif LOGIN.match(word):
|
||||
logins.add(word.lower())
|
||||
return Reviewers(logins, paused)
|
||||
|
||||
|
||||
def is_owner(login: str) -> bool:
|
||||
return login.lower() == OWNER
|
||||
|
||||
|
||||
def authorize(login: str, reviewers: Reviewers) -> None:
|
||||
"""Ludovic always; a listed reviewer unless the list is paused; nobody else."""
|
||||
if is_owner(login):
|
||||
return
|
||||
if login.lower() not in reviewers.logins:
|
||||
raise Refused(f"Only Ludovic and the trusted reviewers listed in `{REVIEWERS}` can pick the Nightly.")
|
||||
if reviewers.paused:
|
||||
raise Refused(f"`/nightly` is paused: Ludovic stopped it in `{REVIEWERS}`.")
|
||||
|
||||
|
||||
def guarded_patterns(codeowners: str) -> list[str]:
|
||||
"""The paths CODEOWNERS gives to Ludovic, plus ALWAYS_GUARDED."""
|
||||
found = [line.split()[0] for line in codeowners.splitlines() if line.strip().startswith("/")]
|
||||
return sorted(set(found + ALWAYS_GUARDED))
|
||||
|
||||
|
||||
def guarded(path: str, patterns: list[str]) -> bool:
|
||||
p = "/" + path.lstrip("/")
|
||||
return any(p.startswith(pat) if pat.endswith("/") else p == pat for pat in patterns)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PullRequest:
|
||||
number: int
|
||||
state: str # open, closed, merged
|
||||
head: str
|
||||
title: str
|
||||
author: str
|
||||
base: str
|
||||
files: list[str]
|
||||
|
||||
|
||||
def check(pr: PullRequest | None, number: int, patterns: list[str]) -> str | None:
|
||||
"""Why `pr` can't go in a Nightly, or None."""
|
||||
if pr is None:
|
||||
return f"#{number} isn't a pull request here"
|
||||
if pr.state != "open":
|
||||
return f"#{number} is {pr.state}"
|
||||
if pr.base != "main":
|
||||
return f"#{number} targets `{safe(pr.base)}`, not main"
|
||||
bad = [f for f in pr.files if guarded(f, patterns)]
|
||||
if bad:
|
||||
more = f" and {len(bad) - 1} more" if len(bad) > 1 else ""
|
||||
return f"#{number} changes `{bad[0]}`{more}, which only Ludovic merges (CODEOWNERS)"
|
||||
return None
|
||||
|
||||
|
||||
def entry(pr: PullRequest, asked_by: str, now: str) -> dict:
|
||||
return {"number": pr.number, "head": pr.head, "title": pr.title, "author": pr.author, "asked_by": asked_by,
|
||||
"asked_at": now}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Result:
|
||||
state: dict
|
||||
answer: str
|
||||
build: bool
|
||||
dry_run: list[dict] | None = None # the set a dry run builds; the saved set stays
|
||||
|
||||
|
||||
def plain(text: str) -> str:
|
||||
"""A title fit for a comment or the notes: no mention, no markdown that spans lines."""
|
||||
return text.replace("@", "@\u200b").replace("\n", " ").strip()[:200]
|
||||
|
||||
|
||||
def describe(prs: list[dict], heading: str = "The set") -> str:
|
||||
if not prs:
|
||||
return "The set is empty: the Nightly is the newest proven union, or main."
|
||||
lines = [f"{heading} ({len(prs)}), merged onto main in this order:"]
|
||||
lines += [f"- #{e['number']} {plain(e['title'])} (by {e['author']}, at {e['head'][:7]}, asked by "
|
||||
f"{e['asked_by']})" for e in prs]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def apply(state: dict, cmd: Command, login: str, now: str, lookup: Callable[[int], PullRequest | None],
|
||||
patterns: list[str]) -> Result:
|
||||
"""What `cmd` by `login` makes of `state` (set.json). Pure but for `lookup`; raises Refused."""
|
||||
st = copy.deepcopy(state)
|
||||
prs: list[dict] = st.setdefault("prs", [])
|
||||
blocked: list[int] = st.setdefault("blocked", [])
|
||||
owner = is_owner(login)
|
||||
if cmd.verb == "status":
|
||||
extra = f"\nOnly Ludovic can add these back: {', '.join(f'#{n}' for n in blocked)}." if blocked else ""
|
||||
return Result(state, describe(prs) + extra, False)
|
||||
if cmd.verb == "clear":
|
||||
st["prs"] = []
|
||||
elif cmd.verb == "remove":
|
||||
absent = [n for n in cmd.prs if n not in [e["number"] for e in prs]]
|
||||
if absent and len(absent) == len(cmd.prs) and not owner:
|
||||
raise Refused(f"{', '.join(f'#{n}' for n in absent)} isn't in the set. " + describe(prs))
|
||||
st["prs"] = [e for e in prs if e["number"] not in cmd.prs]
|
||||
if owner:
|
||||
st["blocked"] = blocked + [n for n in cmd.prs if n not in blocked]
|
||||
else: # set, add, dry-run
|
||||
if not owner and (held := [n for n in cmd.prs if n in blocked]):
|
||||
raise Refused(f"Ludovic took {', '.join(f'#{n}' for n in held)} out of the Nightly; only he can put "
|
||||
"it back.")
|
||||
problems, asked = [], []
|
||||
for n in cmd.prs:
|
||||
pr = lookup(n)
|
||||
if why := check(pr, n, patterns):
|
||||
problems.append(why)
|
||||
else:
|
||||
asked.append(entry(pr, login, now))
|
||||
if problems:
|
||||
raise Refused("Nothing changed: " + "; ".join(problems) + ".")
|
||||
if cmd.verb == "add":
|
||||
fresh = {e["number"]: e for e in asked}
|
||||
new = [fresh.pop(e["number"], e) for e in prs] + list(fresh.values()) # keeps its place, newest head
|
||||
else:
|
||||
new = asked
|
||||
if len(new) > MAX_PRS:
|
||||
raise Refused(f"A Nightly holds at most {MAX_PRS} pull requests; that would be {len(new)}.")
|
||||
if cmd.verb == "dry-run":
|
||||
return Result(state, describe(new, "Dry run, the saved set stays as it is. It builds"), True,
|
||||
dry_run=new)
|
||||
if owner:
|
||||
st["blocked"] = [n for n in blocked if n not in cmd.prs]
|
||||
st["prs"] = new
|
||||
return Result(st, describe(st["prs"]), True)
|
||||
|
||||
|
||||
def log_command(state: dict, cmd: Command, login: str, now: str, issue: str, url: str) -> dict:
|
||||
st = copy.deepcopy(state)
|
||||
st["log"] = (st.get("log", []) + [{"at": now, "by": login, "command": cmd.text(), "issue": issue,
|
||||
"url": url}])[-50:]
|
||||
return st
|
||||
|
||||
|
||||
# --- git -------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def git(*args: str, cwd: str | None = None, env: dict | None = None, check: bool = True,
|
||||
input: str | None = None) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["git", *SAFE_GIT, *args], cwd=cwd, env={**os.environ, **(env or {})}, check=check,
|
||||
capture_output=True, text=True, input=input)
|
||||
|
||||
|
||||
def out(*args: str, cwd: str | None = None, env: dict | None = None) -> str:
|
||||
return git(*args, cwd=cwd, env=env).stdout.strip()
|
||||
|
||||
|
||||
def identity(date: str) -> dict:
|
||||
"""The bot's name, and a fixed date so the same inputs give the same commit."""
|
||||
name, email = BOT
|
||||
return {"GIT_AUTHOR_NAME": name, "GIT_AUTHOR_EMAIL": email, "GIT_COMMITTER_NAME": name,
|
||||
"GIT_COMMITTER_EMAIL": email, "GIT_AUTHOR_DATE": date, "GIT_COMMITTER_DATE": date}
|
||||
|
||||
|
||||
def merge_tree(a: str, b: str, cwd: str | None = None) -> tuple[str | None, list[str]]:
|
||||
"""The merged tree of commits a and b, or None and the conflicted paths. Objects only, no working tree."""
|
||||
r = git("merge-tree", "--write-tree", "--name-only", "--no-messages", a, b, cwd=cwd, check=False)
|
||||
if r.returncode == 0:
|
||||
return r.stdout.split()[0], []
|
||||
if r.returncode == 1:
|
||||
return None, [line for line in r.stdout.splitlines()[1:] if line.strip()]
|
||||
raise RuntimeError(f"git merge-tree {a[:9]} {b[:9]}: {r.stderr.strip()}")
|
||||
|
||||
|
||||
def commit_tree(tree: str, parents: list[str], message: str, date: str, cwd: str | None = None) -> str:
|
||||
args = ["commit-tree", tree]
|
||||
for p in parents:
|
||||
args += ["-p", p]
|
||||
return out(*args, "-m", message, cwd=cwd, env=identity(date))
|
||||
|
||||
|
||||
@dataclass
|
||||
class Conflict(Exception):
|
||||
number: int
|
||||
other: int | None # the earlier pull request it conflicts with; None: main; 0: only with several together
|
||||
files: list[str]
|
||||
|
||||
def text(self) -> str:
|
||||
where = ", ".join(f"`{f}`" for f in self.files[:8]) + (" and more" if len(self.files) > 8 else "")
|
||||
if self.other is None:
|
||||
return (f"#{self.number} conflicts with main in {where}: it needs main merged into it first.")
|
||||
if self.other == 0:
|
||||
return (f"#{self.number} conflicts with the pull requests before it, taken together, in {where}.")
|
||||
return (f"#{self.other} and #{self.number} conflict in {where}. Make an integration pull request that "
|
||||
"combines them (in your fork) and pick that one instead.")
|
||||
|
||||
|
||||
def changed_files(base: str, head: str, cwd: str | None = None) -> list[str]:
|
||||
"""What `head` changes since it left `base`, renames' old paths included."""
|
||||
since = out("merge-base", base, head, cwd=cwd)
|
||||
names = out("diff", "--name-status", "--no-renames", since, head, cwd=cwd)
|
||||
return [line.split("\t", 1)[1] for line in names.splitlines() if "\t" in line]
|
||||
|
||||
|
||||
def culprit(main: str, earlier: list[tuple[int, str]], head: str, date: str, cwd: str | None) -> int | None:
|
||||
if merge_tree(main, head, cwd)[0] is None:
|
||||
return None
|
||||
for n, h in earlier:
|
||||
tree, _ = merge_tree(main, h, cwd)
|
||||
if tree is None:
|
||||
continue
|
||||
if merge_tree(commit_tree(tree, [main, h], "probe", date, cwd), head, cwd)[0] is None:
|
||||
return n
|
||||
return 0
|
||||
|
||||
|
||||
def assemble(main: str, heads: list[tuple[int, str]], cwd: str | None = None) -> str:
|
||||
"""main, then each pull request's head merged in order: the commit a Nightly builds. Raises Conflict."""
|
||||
date = out("show", "-s", "--format=%cI", main, cwd=cwd)
|
||||
current = main
|
||||
for i, (n, head) in enumerate(heads):
|
||||
tree, files = merge_tree(current, head, cwd)
|
||||
if tree is None:
|
||||
raise Conflict(n, culprit(main, heads[:i], head, date, cwd), files)
|
||||
current = commit_tree(tree, [current, head], f"Nightly: merge #{n} at {head[:12]}", date, cwd)
|
||||
return current
|
||||
|
||||
|
||||
def fetch_heads(heads: list[tuple[int, str]], cwd: str | None = None) -> list[int]:
|
||||
"""Fetches each head from origin; returns the numbers whose head is gone (force-pushed away and collected)."""
|
||||
shas = [h for _, h in heads]
|
||||
if shas and git("fetch", "--quiet", "--no-tags", "origin", *shas, cwd=cwd, check=False).returncode == 0:
|
||||
return []
|
||||
gone = []
|
||||
for n, h in heads:
|
||||
if git("cat-file", "-e", f"{h}^{{commit}}", cwd=cwd, check=False).returncode == 0:
|
||||
continue
|
||||
git("fetch", "--quiet", "--no-tags", "origin", h, cwd=cwd, check=False)
|
||||
if git("cat-file", "-e", f"{h}^{{commit}}", cwd=cwd, check=False).returncode != 0:
|
||||
gone.append(n)
|
||||
return gone
|
||||
|
||||
|
||||
def auth(token: str) -> list[str]:
|
||||
"""A `git -c` for pushing with the job's token, never written to a file or a log."""
|
||||
import base64
|
||||
basic = base64.b64encode(f"x-access-token:{token}".encode()).decode()
|
||||
os.environ.setdefault("GIT_TERMINAL_PROMPT", "0")
|
||||
return ["-c", f"http.https://github.com/.extraheader=AUTHORIZATION: basic {basic}"]
|
||||
|
||||
|
||||
def push(sha: str, ref: str, token: str, cwd: str | None = None, force: bool = False) -> bool:
|
||||
spec = f"{'+' if force else ''}{sha}:refs/heads/{ref}"
|
||||
return git(*auth(token), "push", "--quiet", "--no-verify", "origin", spec, cwd=cwd, check=False).returncode == 0
|
||||
|
||||
|
||||
def builds_today(today: str, cwd: str | None = None) -> list[str]:
|
||||
listed = out("ls-remote", "origin", f"refs/heads/{BUILD_PREFIX}{today}-*", cwd=cwd)
|
||||
return [line.split("refs/heads/", 1)[1] for line in listed.splitlines() if "refs/heads/" in line]
|
||||
|
||||
|
||||
def next_build(today: str, existing: list[str]) -> str:
|
||||
taken = [int(m.group(1)) for b in existing if (m := re.search(r"-(\d+)$", b))]
|
||||
return f"{BUILD_PREFIX}{today}-{max(taken, default=0) + 1}"
|
||||
|
||||
|
||||
# --- the state branch ------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_state(cwd: str | None = None) -> tuple[dict, str, str | None]:
|
||||
"""set.json, builds.jsonl and the branch's head (None before the first command)."""
|
||||
if git("fetch", "--quiet", "--no-tags", "origin", f"+refs/heads/{STATE_BRANCH}:refs/nightly/state", cwd=cwd,
|
||||
check=False).returncode != 0:
|
||||
return {}, "", None
|
||||
head = out("rev-parse", "refs/nightly/state", cwd=cwd)
|
||||
state = json.loads(git("show", f"{head}:set.json", cwd=cwd, check=False).stdout or "{}")
|
||||
builds = git("show", f"{head}:builds.jsonl", cwd=cwd, check=False).stdout
|
||||
return state, builds, head
|
||||
|
||||
|
||||
def write_state(state: dict, builds: str, parent: str | None, message: str, cwd: str | None = None) -> str:
|
||||
"""A commit of the state branch, made in a private index: the checkout isn't touched."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
env = {"GIT_INDEX_FILE": str(Path(tmp) / "index")}
|
||||
for name, text in (("set.json", json.dumps(state, indent=2) + "\n"), ("builds.jsonl", builds),
|
||||
("README.md", STATE_README)):
|
||||
blob = git("hash-object", "-w", "--stdin", cwd=cwd, input=text).stdout.strip()
|
||||
git("update-index", "--add", "--cacheinfo", f"100644,{blob},{name}", cwd=cwd, env=env)
|
||||
tree = out("write-tree", cwd=cwd, env=env)
|
||||
now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds")
|
||||
return commit_tree(tree, [parent] if parent else [], message, now, cwd)
|
||||
|
||||
|
||||
STATE_README = """# nightly-state
|
||||
|
||||
Written by tools/ci/nightly_set.py and tools/ci/nightly.py, never by hand. `set.json`: the pull requests the trusted
|
||||
reviewers picked for the Nightly (docs/release-flow.md "Nightly channel"), each at the head it was asked at.
|
||||
`builds.jsonl`: one line per Nightly build with its set.
|
||||
"""
|
||||
|
||||
|
||||
def update_state(change: Callable[[dict, str], tuple[dict, str, str] | None], token: str,
|
||||
cwd: str | None = None, tries: int = 5):
|
||||
"""Applies change(state, builds) -> (state, builds, message), or None for no change, and pushes it; when another
|
||||
run pushed first, reads the branch again and applies it again. Returns the last state."""
|
||||
for _ in range(tries):
|
||||
state, builds, head = load_state(cwd)
|
||||
changed = change(state, builds)
|
||||
if changed is None:
|
||||
return state
|
||||
new, new_builds, message = changed
|
||||
if push(write_state(new, new_builds, head, message, cwd), STATE_BRANCH, token, cwd):
|
||||
return new
|
||||
time.sleep(2)
|
||||
raise RuntimeError(f"could not update {STATE_BRANCH} after {tries} tries")
|
||||
|
||||
|
||||
# --- GitHub ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def gh(*args: str, check: bool = True) -> str:
|
||||
r = subprocess.run(["gh", *args], capture_output=True, text=True)
|
||||
if check and r.returncode != 0:
|
||||
raise RuntimeError(f"gh {args[0]} {args[1] if len(args) > 1 else ''}: {r.stderr.strip()[:300]}")
|
||||
return r.stdout.strip() if r.returncode == 0 else ""
|
||||
|
||||
|
||||
def lookup(repo: str) -> Callable[[int], PullRequest | None]:
|
||||
def get(n: int) -> PullRequest | None:
|
||||
raw = gh("api", f"repos/{repo}/pulls/{n}", check=False)
|
||||
if not raw:
|
||||
return None
|
||||
p = json.loads(raw)
|
||||
files = gh("api", "--paginate", f"repos/{repo}/pulls/{n}/files", "--jq",
|
||||
".[] | .filename, (.previous_filename // empty)").splitlines()
|
||||
state = "merged" if p.get("merged_at") else p["state"]
|
||||
return PullRequest(n, state, p["head"]["sha"], p["title"], p["user"]["login"], p["base"]["ref"], files)
|
||||
return get
|
||||
|
||||
|
||||
def answer(repo: str, issue: str, text: str) -> None:
|
||||
gh("api", f"repos/{repo}/issues/{issue}/comments", "-f", f"body=🤖 Nightly: {text}")
|
||||
|
||||
|
||||
def dispatch(repo: str, issue: str, request: str, dry_run: list[dict] | None) -> str:
|
||||
"""Starts `nightly build` for this request; returns its run's URL once GitHub lists it."""
|
||||
args = ["workflow", "run", "nightly-build.yml", "--repo", repo, "--ref", "main", "-f", f"issue={issue}",
|
||||
"-f", f"request={request}", "-f", "force=true"]
|
||||
if dry_run is not None:
|
||||
args += ["-f", "dry_run=true", "-f", "prs=" + " ".join(f"{e['number']}@{e['head']}" for e in dry_run)]
|
||||
gh(*args)
|
||||
for _ in range(15):
|
||||
time.sleep(4)
|
||||
runs = json.loads(gh("run", "list", "--repo", repo, "--workflow", "nightly-build.yml", "--event",
|
||||
"workflow_dispatch", "--limit", "20", "--json", "displayTitle,url", check=False) or "[]")
|
||||
for r in runs:
|
||||
if request in r["displayTitle"]:
|
||||
return r["url"]
|
||||
return f"https://github.com/{repo}/actions/workflows/nightly-build.yml"
|
||||
|
||||
|
||||
def command() -> None:
|
||||
"""nightly-command.yml: the comment arrives in env vars, never in a shell line."""
|
||||
repo, issue = os.environ["GITHUB_REPOSITORY"], os.environ["ISSUE"]
|
||||
login, body, url = os.environ["AUTHOR"], os.environ.get("BODY", ""), os.environ.get("COMMENT_URL", "")
|
||||
token = os.environ["GH_TOKEN"]
|
||||
try:
|
||||
cmd = parse(body)
|
||||
except Refused as e:
|
||||
answer(repo, issue, str(e))
|
||||
return
|
||||
if cmd is None:
|
||||
return
|
||||
now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds")
|
||||
try:
|
||||
reviewers = Path(REVIEWERS).read_text() if Path(REVIEWERS).exists() else ""
|
||||
authorize(login, read_reviewers(reviewers))
|
||||
patterns = guarded_patterns(Path(CODEOWNERS).read_text() if Path(CODEOWNERS).exists() else "")
|
||||
if cmd.verb != "status":
|
||||
today = now[:10]
|
||||
done = builds_today(today)
|
||||
if len(done) >= MAX_BUILDS_PER_DAY:
|
||||
raise Refused(f"{len(done)} Nightly builds today already, the most a day takes: the next scheduled "
|
||||
"one (20:17 UTC) builds the set.")
|
||||
results: list[Result] = []
|
||||
|
||||
def change(state: dict, builds: str):
|
||||
result = apply(state, cmd, login, now, lookup(repo), patterns)
|
||||
results.append(result)
|
||||
if result.state is state: # status, dry run: nothing to save
|
||||
return None
|
||||
return log_command(result.state, cmd, login, now, issue, url), builds, f"{cmd.text()} by {login}"
|
||||
|
||||
update_state(change, token)
|
||||
result = results[-1]
|
||||
except Refused as e:
|
||||
answer(repo, issue, str(e))
|
||||
return
|
||||
text = result.answer
|
||||
if result.build:
|
||||
run = dispatch(repo, issue, f"{issue}-{int(time.time())}", result.dry_run)
|
||||
text += f"\n\nBuilding: {run}. I'll answer here when it's {'done' if result.dry_run else 'published'}."
|
||||
answer(repo, issue, text)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if sys.argv[1:] == ["command"]:
|
||||
command()
|
||||
else:
|
||||
sys.exit(__doc__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user