From 5ded675a7d5bb1d0f01b8a8e6e68143a8a3f0f45 Mon Sep 17 00:00:00 2001 From: Ludovic Marie <136255825+ludovic111@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:16:48 +0100 Subject: [PATCH] Nightly: publish the exact source, and tag its commit for good The nightly release names its commit, and now also attaches horizon-source.tar.gz (git archive of that commit, made from git objects in pick and signed with the builds) and tags the commit nightly-source/-. The dated release and the nightly/build-* branch are pruned after 7 days; the tag is not, so native mods pinned to a Nightly's commit keep building. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly-build.yml | 21 +++++++++++++- docs/release-flow.md | 6 ++++ tests/test_nightly.py | 24 ++++++++++++++++ tools/ci/nightly.py | 43 ++++++++++++++++++++++++++--- 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 3fa440bd1..d8c4c0b79 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -25,6 +25,9 @@ run-name: nightly build${{ inputs.request && format(' (request {0})', inputs.req # names the channel ("horizon release nightly"), so a nightly's signature can't pass for a release's. # - `publish` verifies the signature with main's horizon-release-sign, checks the smoke test, and only then moves # the tag. Like sign, it never executes what build produced. +# - The source ships with the build: `pick` writes horizon-source.tar.gz with `git archive` (git objects again, nothing +# run), `sign` signs it with the builds, and `publish` attaches it and tags the commit nightly-source/- +# for good (the dated release and the build branch go after 7 days; native mods pin the commit: docs/native-mods.md). # gh workflow run "nightly build" # now (skips when the pick is the same commit as last night's) # gh workflow run "nightly build" -f force=true # build and publish even if unchanged # gh workflow run "nightly build" -f dry_run=true # build and sign with a throwaway key, publish nothing @@ -76,6 +79,7 @@ jobs: date: ${{ steps.pick.outputs.date }} version: ${{ steps.pick.outputs.version }} nightly: ${{ steps.pick.outputs.nightly }} + source_tag: ${{ steps.pick.outputs.source_tag }} dry_run: ${{ steps.mode.outputs.dry_run }} steps: - uses: actions/checkout@v4 @@ -103,6 +107,18 @@ jobs: GH_TOKEN: ${{ github.token }} run: | { echo "### nightly/build-* branches"; python3 tools/ci/nightly.py prune-builds --days 7; } | tee -a "$GITHUB_STEP_SUMMARY" + - name: The source of the build (horizon-source.tar.gz) + if: steps.pick.outputs.build == 'true' + env: + SHA: ${{ steps.pick.outputs.sha }} + run: git archive --format=tar.gz --prefix="horizon-${SHA:0:7}/" -o horizon-source.tar.gz "$SHA" + - uses: actions/upload-artifact@v4 + if: steps.pick.outputs.build == 'true' + with: + name: horizon-source + path: horizon-source.tar.gz + if-no-files-found: error + retention-days: 3 - uses: actions/upload-artifact@v4 with: name: nightly-notes @@ -144,7 +160,7 @@ jobs: EPHEMERAL: ${{ needs.pick.outputs.dry_run == 'true' && 'yes' || '' }} run: | bash tools/ci/sign_release.sh nightly dist/horizon-linux-x86_64.tar.gz dist/horizon-windows-x64.zip \ - dist/horizon-macos-arm64.zip dist/horizon-server-linux-x86_64.tar.gz + dist/horizon-macos-arm64.zip dist/horizon-server-linux-x86_64.tar.gz dist/horizon-source.tar.gz - uses: actions/upload-artifact@v4 with: name: signature @@ -187,6 +203,7 @@ jobs: GH_TOKEN: ${{ github.token }} SHA: ${{ needs.pick.outputs.sha }} DATE: ${{ needs.pick.outputs.date }} + SOURCE_TAG: ${{ needs.pick.outputs.source_tag }} run: | set -euo pipefail python3 tools/ci/nightly.py rotate @@ -198,7 +215,9 @@ jobs: "dist/horizon-windows-x64.zip#Nightly game files: Windows (horizon-windows-x64.zip)" \ "dist/horizon-macos-arm64.zip#Nightly game files: macOS (horizon-macos-arm64.zip)" \ "dist/horizon-server-linux-x86_64.tar.gz#Nightly dedicated server for Linux (horizon-server-linux-x86_64.tar.gz)" \ + "dist/horizon-source.tar.gz#Source of this Nightly, commit ${SHA:0:7} (horizon-source.tar.gz; kept for good as the tag $SOURCE_TAG)" \ dist/SHA256SUMS "${signature[@]}" dist/smoke-test.json + python3 tools/ci/nightly.py keep-source "$SHA" "$DATE" python3 tools/ci/nightly.py prune --days 7 - name: Dry run, the nightly that would be published if: needs.pick.outputs.dry_run == 'true' diff --git a/docs/release-flow.md b/docs/release-flow.md index e5201d089..ec7157c45 100644 --- a/docs/release-flow.md +++ b/docs/release-flow.md @@ -449,6 +449,12 @@ than main's head and holds something main lacks, otherwise main (`tools/ci/night new publishes nothing. A Nightly can include pull requests not reviewed yet: that is what it is for, playing them before the gate. +Every Nightly ships its exact source. Its notes name the commit; `horizon-source.tar.gz` on the release is that +commit's `git archive`, signed with the builds in `SHA256SUMS`; and `nightly.py keep-source` tags the commit +`nightly-source/-`. That tag is never pruned, unlike the `nightly-` release and the +`nightly/build-*` branch (7 days each), because native mods pin a Nightly's commit and the launcher downloads that +commit's source to build them ([native-mods.md](native-mods.md)). + ### Trusted reviewers One list, `.github/trusted-reviewers.txt`, governs two things for the people on it (kacpherek, rdbtCVS and diff --git a/tests/test_nightly.py b/tests/test_nightly.py index 567b3d86e..9af593eac 100644 --- a/tests/test_nightly.py +++ b/tests/test_nightly.py @@ -62,6 +62,19 @@ class NightlyTests(unittest.TestCase): self.assertIn("- Fix pistons (#2476)", text) self.assertIn("Everything in it is on main.", nightly.notes("2026-10-28", ref("main", "m" * 40, 0), [])) + def test_notes_name_the_commit_and_its_permanent_source_tag(self): + sha = "0c0206714abc" + "0" * 28 + text = nightly.notes("2026-10-28", ref("union/a", sha, 1), []) + self.assertIn(f"Source: commit {sha}, kept for good as the tag `nightly-source/2026-10-28-0c02067`", text) + main = nightly.Ref("main", "m" * 40, None) + self.assertIn("`nightly-source/2026-10-28-bbbbbbb`", + nightly.set_notes("2026-10-28", nightly.Ref("b", "b" * 40, None), main, [], [])) + + def test_the_source_tag_outlives_every_prune(self): + tag = nightly.source_tag("2026-10-20", "a" * 40) + self.assertEqual(nightly.stale([{"tagName": tag, "isPrerelease": True}], dt.date(2026, 11, 28), 7), []) + self.assertIsNone(nightly.DATED.match(tag)) + class ScheduledTests(unittest.TestCase): @@ -149,6 +162,17 @@ class PruneBuildsTests(unittest.TestCase): "feature/nightly/build-2026-10-01-1": "g", "nightly/build-2026-10-02-1": "n"} self.assertEqual(nightly.stale_builds(branches, set(), self.TODAY, 7), []) + def test_the_source_is_archived_from_git_signed_attached_and_tagged(self): + build = (WORKFLOWS / "nightly-build.yml").read_text() + pick = build.split("\n build:", 1)[0] + self.assertIn('git archive --format=tar.gz --prefix="horizon-${SHA:0:7}/" -o horizon-source.tar.gz "$SHA"', + pick) + sign = build.split("\n sign:", 1)[1].split("\n publish:", 1)[0] + self.assertIn("dist/horizon-source.tar.gz", sign.split("sign_release.sh nightly", 1)[1]) + publish = build.split("\n publish:", 1)[1].split("\n answer:", 1)[0] + self.assertIn('"dist/horizon-source.tar.gz#Source of this Nightly', publish) + self.assertIn('nightly.py keep-source "$SHA" "$DATE"', publish) + def test_the_workflow_prunes_on_schedule_only(self): build = (WORKFLOWS / "nightly-build.yml").read_text() step = build.split("Old nightly/build-* branches deleted", 1)[1].split("- uses:", 1)[0] diff --git a/tools/ci/nightly.py b/tools/ci/nightly.py index e04b99ddd..4fb6436f9 100644 --- a/tools/ci/nightly.py +++ b/tools/ci/nightly.py @@ -6,6 +6,7 @@ nightly.py prune [--days 7] deletes the nightly- pre-releases and tags older than that, nothing else nightly.py prune-builds [--days 7] deletes the nightly/build-- branches older than that, but the newest and any a published nightly points to (Ludovic, 2026-10-27) + nightly.py keep-source tags the nightly's commit nightly-source/-, for good A nightly builds main plus the pull requests the trusted reviewers picked (the set: tools/ci/nightly_set.py), merged in their order and pushed as nightly/build--. With no set it builds the newest union branch on origin @@ -13,6 +14,10 @@ in their order and pushed as nightly/build--. With no set it builds the newer than main's head. It is published as the `nightly` pre-release; the previous one stays a week as nightly-. Runs from main's checkout with `gh`; it reads and merges the branches with git and never runs anything from them. + +Each nightly ships its exact source: horizon-source.tar.gz (`git archive` of the commit, signed with the builds) on +the release, and the tag nightly-source/-, which no prune deletes. Native mods pin a nightly's commit +(docs/native-mods.md), so that commit must stay downloadable after its release and build branch are gone. """ from __future__ import annotations @@ -33,6 +38,7 @@ else: # run as tools/ci/nightly.py: its folder is on the path DATED = re.compile(r"^nightly-(\d{4}-\d{2}-\d{2})$") TITLE_DATE = re.compile(r"Nightly (\d{4}-\d{2}-\d{2})") +SOURCE_TAG = "nightly-source/" @dataclass @@ -84,10 +90,20 @@ OPT_IN = ("**Opt-in, not a release.** A Nightly can include pull requests not re "as they are. Report bugs with the version the game shows (\"Nightly · \").") +def source_tag(today: str, sha: str) -> str: + """The permanent tag of a nightly's commit: neither `prune` (nightly- only) nor `prune-builds` touches it.""" + return f"{SOURCE_TAG}{today}-{sha[:7]}" + + +def source_line(today: str, sha: str) -> str: + return (f"Source: commit {sha}, kept for good as the tag `{source_tag(today, sha)}`; horizon-source.tar.gz is " + "its tree. Native mods built for this Nightly pin that commit.") + + def set_notes(today: str, chosen: Ref, main: Ref, prs: list[dict], left_out: list[str]) -> str: """The notes of a build of main plus the reviewers' set.""" lines = [f"Nightly {today} · {chosen.sha[:7]}: main ({main.sha[:7]}) with {len(prs)} pull requests merged in, " - f"built from `{chosen.name}` ({chosen.sha}).", "", OPT_IN, "", + f"built from `{chosen.name}` ({chosen.sha}).", "", source_line(today, chosen.sha), "", OPT_IN, "", f"### Pull requests in this Nightly ({len(prs)})", ""] lines += [f"- {ns.plain(e['title'])} (#{e['number']}) by {e['author']}, at {e['head'][:7]}, picked by " f"{e['asked_by']}" for e in prs] @@ -97,7 +113,8 @@ def set_notes(today: str, chosen: Ref, main: Ref, prs: list[dict], left_out: lis def notes(today: str, chosen: Ref, held: list[dict], left_out: list[str] | None = None) -> str: - lines = [f"Nightly {today} · {chosen.sha[:7]}: built from `{chosen.name}` ({chosen.sha}).", "", OPT_IN, ""] + lines = [f"Nightly {today} · {chosen.sha[:7]}: built from `{chosen.name}` ({chosen.sha}).", "", + source_line(today, chosen.sha), "", OPT_IN, ""] if left_out: lines += [f"The reviewers' set had nothing left to build: {'; '.join(ns.plain(w) for w in left_out)}.", ""] if held: @@ -184,7 +201,7 @@ def build_set(main: Ref, prs: list[dict], left_out: list[str], today: str, force with open("nightly-notes.md", "w") as f: f.write(set_notes(today, chosen, main, kept, left_out)) output(build=str(build).lower(), sha=sha, short=sha[:7], source=name, date=today, version=f"nightly-{today}", - nightly=f"{today} {sha[:7]}") + nightly=f"{today} {sha[:7]}", source_tag=source_tag(today, sha)) print(f"{'build' if build else 'skip (unchanged since the last nightly)'}: {name} {sha}, main {main.sha[:7]} + " + ", ".join(f"#{e['number']}" for e in kept)) @@ -233,7 +250,7 @@ def build_union(main: Ref, today: str, force: bool, left_out: list[str] | None = with open("nightly-notes.md", "w") as f: f.write(notes(today, chosen, held, left_out)) output(build=str(build).lower(), sha=chosen.sha, short=chosen.sha[:7], source=chosen.name, date=today, - version=f"nightly-{today}", nightly=f"{today} {chosen.sha[:7]}") + version=f"nightly-{today}", nightly=f"{today} {chosen.sha[:7]}", source_tag=source_tag(today, chosen.sha)) print(f"{'build' if build else 'skip (unchanged since the last nightly)'}: {chosen.name} {chosen.sha}, " f"{len(held)} open pull requests beyond main") @@ -309,6 +326,19 @@ def prune_builds(days: int) -> None: f"nightlies' and the last {days} days')") +def keep_source(sha: str, today: str) -> None: + """Tags the published nightly's commit for good (source_tag); a rerun of the same build finds it there.""" + tag = source_tag(today, sha) + there = remote_refs(f"refs/tags/{tag}").get(tag) + if there == sha: + print(f"`{tag}` already names {sha[:7]}") + return + if there: + raise SystemExit(f"`{tag}` names {there[:7]}, not {sha[:7]}") + gh("api", f"repos/{os.environ['GITHUB_REPOSITORY']}/git/refs", "-f", f"ref=refs/tags/{tag}", "-f", f"sha={sha}") + print(f"tagged {sha[:7]} `{tag}`") + + def main() -> None: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="cmd", required=True) @@ -316,6 +346,9 @@ def main() -> None: sub.add_parser("rotate") sub.add_parser("prune").add_argument("--days", type=int, default=7) sub.add_parser("prune-builds").add_argument("--days", type=int, default=7) + keep = sub.add_parser("keep-source") + keep.add_argument("sha") + keep.add_argument("date") args = ap.parse_args() if args.cmd == "pick": pick(args.force) @@ -323,6 +356,8 @@ def main() -> None: rotate() elif args.cmd == "prune-builds": prune_builds(args.days) + elif args.cmd == "keep-source": + keep_source(args.sha, args.date) else: prune(args.days)