From e660df461c14453da94eff4ed57ae0e405ec8791 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 17 Aug 2026 10:30:28 +0100 Subject: [PATCH 01/10] feat(ci): gate release PRs on the signed hash list, and cut the tag after merge Two workflows around the existing release ritual (a reviewed development -> master PR carrying the version bump): release-gate runs on PRs targeting master and judges the merge preview: the version moves forward numerically, the ZelBack tree hash is in the validly signed document central serves (verified against the published keys, so a stale deploy answering with the unsigned array is a red run), and helpers/hashes.json carries the hash for old fluxbench's fallback. Read-only, no secrets. Enforcement comes from branch protection listing the check as required. release-tag runs on master pushes that change the version: re-verifies the merged tree against the signed document (polling up to 15 minutes for the divergent-merge case), samples ten live nodes' /flux/hashlist -- advisory until the fleet serves it -- then pushes the annotated tag, publishes the GitHub Release, and records the tag on the provenance row in fluxhashes. The annotation happens here because the tag is pushed with GITHUB_TOKEN, whose pushes do not trigger workflows, so the publish workflow's tag path cannot fire for CI-cut tags. Idempotent on re-run once the tag exists. Every run block was executed verbatim against local origins under Linux: 27 cases covering version comparisons (including 8.17.9 -> 8.17.10 numerically), signature rejection of the unsigned catch-all body and of a wrong-key document, the poll loop converging on a later attempt, the advisory sample staying advisory, tag idempotency, and provenance annotation exactly once with attribution preserved. That harness caught one real bug: NEW_HASH was written to GITHUB_ENV but not exported, so the inline node scripts in the same step would have read undefined and failed every release. Co-Authored-By: Claude Fable 5 --- .github/workflows/release-gate.yml | 105 +++++++++++++++ .github/workflows/release-tag.yml | 199 +++++++++++++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 .github/workflows/release-gate.yml create mode 100644 .github/workflows/release-tag.yml diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 0000000000..02031bbcc2 --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,105 @@ +name: release-gate + +# Required status check on release PRs (development -> master). Verifies, before a release can +# merge, that what master will become is already approved: the version moves forward, the tree's +# hash is in the signed document central serves, and the old-fluxbench fallback list carries it. +# +# This check is the gate the human sees; release-tag.yml re-verifies the merged tree before +# cutting the tag and GitHub Release. Enforcement comes from branch protection listing this check +# as required -- the workflow itself has no power to block a merge. +# +# Deliberately strict: central not serving a validly signed document is a red run, not a skip. A +# gate that passes when its source is missing is not a gate. +# +# Read-only: no secrets, no pushes. + +on: + pull_request: + branches: [master] + +permissions: + contents: read + +concurrency: + group: release-gate-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + gate: + runs-on: ubuntu-22.04 + steps: + # The default pull_request checkout is the merge preview -- the tree master will actually + # hold after the merge -- which is the thing worth judging, not the PR head alone. + - uses: actions/checkout@v3 + + - name: Version moves forward + run: | + git fetch --quiet --depth 1 origin master + BASE=$(git show origin/master:package.json | jq -r .version) + CANDIDATE=$(jq -r .version package.json) + echo "master: ${BASE} candidate: ${CANDIDATE}" + # Strictly newer, compared numerically per part. String inequality would call 8.17.10 + # older than 8.17.9. + node -e ' + const [base, head] = process.argv.slice(1).map((v) => v.split(".").map(Number)); + const delta = head[0] - base[0] || head[1] - base[1] || head[2] - base[2]; + if (!(delta > 0)) { + throw new Error("the candidate version does not move past master -- bump package.json in the release PR"); + } + ' "$BASE" "$CANDIDATE" + + - name: Tree hash is in the signed document central serves + run: | + NEW_HASH=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}') + echo "tree hash: ${NEW_HASH}" + # exported for the node script below; GITHUB_ENV only reaches subsequent steps + export NEW_HASH + echo "NEW_HASH=${NEW_HASH}" >> "$GITHUB_ENV" + + curl -sS -m 30 -o hashlist-signed.json https://hashes.runonflux.io/hashlist + + # Verify the signature the way a consumer does, against the published public keys + # (RunOnFlux/fluxhashes SIGNING.md), so a stale deploy answering every path with the + # unsigned array -- or anything else that is not the signed document -- is a red run. + # + # Single-quoted, with the hash read from the environment: inside a double-quoted shell + # string the ${...} below would be the shell's, not node's. + node -e ' + const crypto = require("crypto"); + const fs = require("fs"); + const PINNED_PUBLIC_KEYS = [ + "3023cb5e01dc22257ac5c31c4d12106cd0d58fa2005f867b3fdc5d303f6446ec", + "fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8", + ]; + const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); + const document = JSON.parse(fs.readFileSync("hashlist-signed.json", "utf8")); + const payload = Buffer.from(document.payload_b64, "base64"); + const signature = Buffer.from(document.sig_b64, "base64"); + const verified = PINNED_PUBLIC_KEYS.some((hex) => crypto.verify( + null, + payload, + crypto.createPublicKey({ + key: Buffer.concat([SPKI_ED25519_PREFIX, Buffer.from(hex, "hex")]), + format: "der", + type: "spki", + }), + signature, + )); + if (!verified) { + throw new Error("central is not serving a validly signed hash list"); + } + const { seq, hashes } = JSON.parse(payload.toString("utf8")); + if (!hashes.includes(process.env.NEW_HASH)) { + throw new Error(`the tree hash is not in the signed document (seq ${seq}) -- wait for the signing run to cover the release candidate`); + } + console.log(`signed document seq ${seq} carries the tree hash`); + ' + + # Old fluxbench falls back to helpers/hashes.json via GitHub raw when central is unreachable. + # The entry rides the release PR instead of a commit straight to master. This step retires + # with old fluxbench. + - name: Fallback list carries the hash + run: | + jq -e --arg h "$NEW_HASH" 'index($h) != null' helpers/hashes.json > /dev/null \ + || { echo "helpers/hashes.json does not contain ${NEW_HASH} -- add it in the release PR"; exit 1; } + echo "helpers/hashes.json carries the tree hash" diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml new file mode 100644 index 0000000000..2e76cdfa0e --- /dev/null +++ b/.github/workflows/release-tag.yml @@ -0,0 +1,199 @@ +name: release-tag + +# Runs when a release lands on master: re-verifies the merged tree against the signed document, +# takes an advisory look at live-node distribution, then cuts the tag and the GitHub Release and +# records the tag on the provenance row in fluxhashes. +# +# The pre-merge gate (release-gate.yml) judged the merge preview; this judges what master actually +# holds, which is the authoritative check -- it covers the only divergence branch protection +# cannot. The tag and Release are outputs of a verified release: nothing in the fleet follows +# them, the fleet follows master. +# +# A red run here means the release is stuck loudly with no tag -- the fleet is already on the +# verified merge tree, so nothing is half-made. + +on: + push: + branches: [master] + +permissions: + contents: write + +concurrency: + group: release-tag + cancel-in-progress: false + +jobs: + tag: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + # A push that does not change the version is not a release. HEAD^1 is master's previous tip + # whether the push was a merge or a plain commit. + - name: Did the version change? + id: version + run: | + CURRENT=$(jq -r .version package.json) + PREVIOUS=$(git show 'HEAD^1:package.json' 2>/dev/null | jq -r .version || echo "") + echo "previous: ${PREVIOUS:-none} current: ${CURRENT}" + if [ "$CURRENT" = "$PREVIOUS" ]; then + echo "version unchanged, not a release" + echo "release=false" >> "$GITHUB_OUTPUT" + else + echo "release=true" >> "$GITHUB_OUTPUT" + echo "version=${CURRENT}" >> "$GITHUB_OUTPUT" + fi + + # Idempotent: a re-run after the tag exists has nothing left to do. + - name: Tag already cut? + id: existing + if: steps.version.outputs.release == 'true' + run: | + if git ls-remote --exit-code --tags origin "refs/tags/v${{ steps.version.outputs.version }}" > /dev/null; then + echo "tag v${{ steps.version.outputs.version }} already exists" + echo "done=true" >> "$GITHUB_OUTPUT" + else + echo "done=false" >> "$GITHUB_OUTPUT" + fi + + # The merge tree normally equals the PR head tree the gate already saw signed, so this + # passes on the first poll. The loop covers the divergent case: the on-push publish and the + # signing run pick the new tree up within minutes, and this waits for them rather than + # failing a release over a race. + - name: Merged tree is in the signed document central serves + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + run: | + NEW_HASH=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}') + echo "merged tree hash: ${NEW_HASH}" + # exported for the node script below; GITHUB_ENV only reaches subsequent steps + export NEW_HASH + echo "NEW_HASH=${NEW_HASH}" >> "$GITHUB_ENV" + + for attempt in $(seq 1 30); do + curl -sS -m 30 -o hashlist-signed.json https://hashes.runonflux.io/hashlist || true + if node -e ' + const crypto = require("crypto"); + const fs = require("fs"); + const PINNED_PUBLIC_KEYS = [ + "3023cb5e01dc22257ac5c31c4d12106cd0d58fa2005f867b3fdc5d303f6446ec", + "fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8", + ]; + const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); + const document = JSON.parse(fs.readFileSync("hashlist-signed.json", "utf8")); + const payload = Buffer.from(document.payload_b64, "base64"); + const signature = Buffer.from(document.sig_b64, "base64"); + const verified = PINNED_PUBLIC_KEYS.some((hex) => crypto.verify( + null, + payload, + crypto.createPublicKey({ + key: Buffer.concat([SPKI_ED25519_PREFIX, Buffer.from(hex, "hex")]), + format: "der", + type: "spki", + }), + signature, + )); + if (!verified) throw new Error("not a validly signed document"); + const { seq, hashes } = JSON.parse(payload.toString("utf8")); + if (!hashes.includes(process.env.NEW_HASH)) throw new Error(`not in seq ${seq}`); + console.log(`signed document seq ${seq} carries the merged tree hash`); + ' 2> poll-error.txt; then + exit 0 + fi + echo "attempt ${attempt}: $(cat poll-error.txt) -- retrying in 30s" + sleep 30 + done + + echo "the signed document never covered the merged tree; no tag is cut" + exit 1 + + # Advisory until the fleet serves /flux/hashlist: results are logged, never fatal. Once the + # FluxOS side is deployed, enforcement is turning the failure at the end into exit 1. + - name: Live-node sample (advisory) + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + run: | + curl -sS -m 60 -o nodelist.json https://api.runonflux.io/daemon/viewdeterministiczelnodelist || { echo "ADVISORY: node list unavailable"; exit 0; } + jq -r '.data[].ip | select(. != null and . != "")' nodelist.json | shuf -n 10 > sample.txt || true + SERVING=0 + while read -r ADDR; do + HOST=${ADDR%:*}; PORT=${ADDR##*:}; [ "$HOST" = "$PORT" ] && PORT=16127 + BODY=$(curl -sS -m 5 "http://${HOST}:${PORT}/flux/hashlist" 2>/dev/null || true) + PAYLOAD=$(printf '%s' "$BODY" | jq -r 'try (.payload_b64 // .data.payload_b64) // empty' 2>/dev/null || true) + if [ -n "$PAYLOAD" ] && printf '%s' "$PAYLOAD" | base64 -d 2>/dev/null | grep -q "$NEW_HASH"; then + SERVING=$((SERVING + 1)) + fi + done < sample.txt + echo "ADVISORY: $(wc -l < sample.txt | tr -d ' ') nodes sampled, ${SERVING} serving a document with the merged tree hash" + + - name: Cut the tag and the GitHub Release + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + env: + GH_TOKEN: ${{ github.token }} + run: | + VERSION=${{ steps.version.outputs.version }} + git config user.email runonfluxbot@gmail.com + git config user.name runonfluxbot + git tag -a "v${VERSION}" -m "FluxOS v${VERSION}" + git push --quiet origin "refs/tags/v${VERSION}" + gh release create "v${VERSION}" --title "FluxOS v${VERSION}" --generate-notes --verify-tag + echo "released v${VERSION}" + + # nodejs.yml annotates provenance rows when a tag push triggers it -- but pushes made with + # GITHUB_TOKEN do not trigger workflows, and the tag above is one. So the annotation happens + # here, same record, same rule: rows are only ever created or given their missing tag. + - name: Record the tag on the provenance row + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + env: + API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} + TAG_NAME: v${{ steps.version.outputs.version }} + run: | + git clone --quiet "https://x-access-token:${API_TOKEN_GITHUB}@github.com/RunOnFlux/fluxhashes.git" fluxhashes + cd fluxhashes + git config user.email runonfluxbot@gmail.com + git config user.name runonfluxbot + + for attempt in 1 2 3; do + git fetch --quiet origin master + git reset --quiet --hard origin/master + + node -e ' + const fs = require("fs"); + const FILE = "src/hashes/provenance.json"; + const hash = process.env.NEW_HASH; + const record = fs.existsSync(FILE) + ? JSON.parse(fs.readFileSync(FILE, "utf8")) + : { hashes: {} }; + if (!record.hashes) record.hashes = {}; + const row = record.hashes[hash]; + if (!row) { + record.hashes[hash] = { + published: new Date().toISOString().slice(0, 10), + commit: process.env.GITHUB_SHA, + branch: null, + tag: process.env.TAG_NAME, + }; + } else if (!row.tag) { + row.tag = process.env.TAG_NAME; + } + fs.writeFileSync(FILE, `${JSON.stringify(record, null, 2)}\n`); + ' + + git add src/hashes/provenance.json + if git diff --cached --quiet -- src/hashes/provenance.json; then + echo "provenance already carries the tag" + exit 0 + fi + + git commit --quiet -m "Tag ${TAG_NAME} on ${NEW_HASH}" -- src/hashes/provenance.json + + if git push --quiet; then + echo "provenance row tagged ${TAG_NAME}" + exit 0 + fi + echo "push raced with another publish, retrying" + done + + echo "could not record the tag after 3 attempts" + exit 1 From d732608bf71b984dca310e36cfacc98004a5b8b4 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 24 Aug 2026 10:59:04 +0100 Subject: [PATCH 02/10] feat(ci): the tag annotation becomes a dispatch; the second write path to fluxhashes dies The provenance-annotation step held the other PAT write path into fluxhashes, with its own clone, retry loop, and copy of the row discipline. Under the single-writer design the signer owns the record: this now dispatches "commit X, tag vN" and the signer resolves the tag against its own view of the flux repository and annotates the row itself. The merged tree hash rides along as the claimed_hash tripwire. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SFrS7Q3JwuPj6Yr4vALnwp --- .github/workflows/release-tag.yml | 66 ++++++------------------------- 1 file changed, 12 insertions(+), 54 deletions(-) diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 2e76cdfa0e..449bc9e556 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -140,60 +140,18 @@ jobs: gh release create "v${VERSION}" --title "FluxOS v${VERSION}" --generate-notes --verify-tag echo "released v${VERSION}" - # nodejs.yml annotates provenance rows when a tag push triggers it -- but pushes made with - # GITHUB_TOKEN do not trigger workflows, and the tag above is one. So the annotation happens - # here, same record, same rule: rows are only ever created or given their missing tag. - - name: Record the tag on the provenance row + # The tag above was pushed with GITHUB_TOKEN, which triggers no workflows -- so the signer + # would only pick it up on its next run's ls-remote delta. This dispatch makes the + # annotation immediate; the signer resolves the tag against its own view of this repository + # and annotates the provenance row itself. Nothing here writes to fluxhashes. + - name: Request the tag annotation if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' env: - API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} - TAG_NAME: v${{ steps.version.outputs.version }} + GH_TOKEN: ${{ secrets.FLUXHASHES_DISPATCH_TOKEN }} run: | - git clone --quiet "https://x-access-token:${API_TOKEN_GITHUB}@github.com/RunOnFlux/fluxhashes.git" fluxhashes - cd fluxhashes - git config user.email runonfluxbot@gmail.com - git config user.name runonfluxbot - - for attempt in 1 2 3; do - git fetch --quiet origin master - git reset --quiet --hard origin/master - - node -e ' - const fs = require("fs"); - const FILE = "src/hashes/provenance.json"; - const hash = process.env.NEW_HASH; - const record = fs.existsSync(FILE) - ? JSON.parse(fs.readFileSync(FILE, "utf8")) - : { hashes: {} }; - if (!record.hashes) record.hashes = {}; - const row = record.hashes[hash]; - if (!row) { - record.hashes[hash] = { - published: new Date().toISOString().slice(0, 10), - commit: process.env.GITHUB_SHA, - branch: null, - tag: process.env.TAG_NAME, - }; - } else if (!row.tag) { - row.tag = process.env.TAG_NAME; - } - fs.writeFileSync(FILE, `${JSON.stringify(record, null, 2)}\n`); - ' - - git add src/hashes/provenance.json - if git diff --cached --quiet -- src/hashes/provenance.json; then - echo "provenance already carries the tag" - exit 0 - fi - - git commit --quiet -m "Tag ${TAG_NAME} on ${NEW_HASH}" -- src/hashes/provenance.json - - if git push --quiet; then - echo "provenance row tagged ${TAG_NAME}" - exit 0 - fi - echo "push raced with another publish, retrying" - done - - echo "could not record the tag after 3 attempts" - exit 1 + gh api -X POST repos/RunOnFlux/fluxhashes/actions/workflows/sign-hashlist.yml/dispatches \ + -f ref=master \ + -f "inputs[commit]=${GITHUB_SHA}" \ + -f "inputs[ref]=v${{ steps.version.outputs.version }}" \ + -f "inputs[ref_type]=tag" \ + -f "inputs[claimed_hash]=${NEW_HASH}" From f7d953baa3d40696f35bc4c98d916edc5ce9c95a Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 24 Aug 2026 12:25:11 +0100 Subject: [PATCH 03/10] chore: pin the regenerated key 1 in the gate workflows Matches fluxhashes feature/sign-hashlist: key 1 regenerated 2026-08-24 into the environment-scoped secret; key 2 unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SFrS7Q3JwuPj6Yr4vALnwp --- .github/workflows/release-gate.yml | 2 +- .github/workflows/release-tag.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 02031bbcc2..6141eac44f 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -68,7 +68,7 @@ jobs: const crypto = require("crypto"); const fs = require("fs"); const PINNED_PUBLIC_KEYS = [ - "3023cb5e01dc22257ac5c31c4d12106cd0d58fa2005f867b3fdc5d303f6446ec", + "14837066068b258bfbd0749702056f7065361af44aed48761834744391cbbaaa", "fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8", ]; const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 449bc9e556..b0b7a953b0 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -78,7 +78,7 @@ jobs: const crypto = require("crypto"); const fs = require("fs"); const PINNED_PUBLIC_KEYS = [ - "3023cb5e01dc22257ac5c31c4d12106cd0d58fa2005f867b3fdc5d303f6446ec", + "14837066068b258bfbd0749702056f7065361af44aed48761834744391cbbaaa", "fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8", ]; const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); From 462c3434322c2298a3afd431f8bfc59db9a8b061 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 24 Aug 2026 12:47:50 +0100 Subject: [PATCH 04/10] feat(ci): the tag-annotation dispatch mints a GitHub App token Same credential as nodejs.yml's dispatch: the flux-hashlist-dispatch app, Actions on fluxhashes only, short-lived token per run, nothing to renew. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SFrS7Q3JwuPj6Yr4vALnwp --- .github/workflows/release-tag.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index b0b7a953b0..112dd43a7f 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -144,10 +144,19 @@ jobs: # would only pick it up on its next run's ls-remote delta. This dispatch makes the # annotation immediate; the signer resolves the tag against its own view of this repository # and annotates the provenance row itself. Nothing here writes to fluxhashes. + - name: Mint the dispatch token + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + id: dispatch-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.FLUXHASHES_APP_ID }} + private-key: ${{ secrets.FLUXHASHES_APP_KEY }} + owner: RunOnFlux + repositories: fluxhashes - name: Request the tag annotation if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' env: - GH_TOKEN: ${{ secrets.FLUXHASHES_DISPATCH_TOKEN }} + GH_TOKEN: ${{ steps.dispatch-token.outputs.token }} run: | gh api -X POST repos/RunOnFlux/fluxhashes/actions/workflows/sign-hashlist.yml/dispatches \ -f ref=master \ From e815f327998b9593bd79914c7cad32c700531437 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 25 Aug 2026 12:56:00 +0100 Subject: [PATCH 05/10] fix(ci): name the missing directory instead of blaming the signed document Both workflows derive the tree hash with the same unguarded pipeline the publish path used: a missing ZelBack makes find error while the pipeline's status stays awk's, and the result is d41d8cd98f00b204e9800998ecf8427e, the md5 of an empty stream. Neither is unsafe today. Both use the value only to ask "is this hash in the signed document", the signer now refuses to list the empty hash, and it is not among the 224 entries currently published -- so the answer is no and the gate goes red or the tag is not cut. The outcome is already correct. What is wrong is the diagnosis. The failure reads "not in the signed document", which sends the reader to the signing chain, the published list and hashes.runonflux.io when the actual cause is that ZelBack is not there. On the day someone renames that directory all three copies of this pipeline fire at once; two now say what happened and this was the one that would not. Guarded the same way as the other two, so a future reader does not have to work out whether the omission here was deliberate. Verified by extracting each step's script verbatim from the workflow and running it against three fixtures: absent ZelBack exits 1 on find's own error, empty ZelBack exits 1 on the guard, a real tree passes through unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HQKpZbuxYqrWeErU7vKgoA --- .github/workflows/release-gate.yml | 10 ++++++++++ .github/workflows/release-tag.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 6141eac44f..b4dd4956d5 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -50,7 +50,17 @@ jobs: - name: Tree hash is in the signed document central serves run: | + set -eo pipefail NEW_HASH=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}') + # d41d8... is the md5 of an empty stream, produced whenever nothing was hashed: ZelBack + # absent (pipefail catches that) or present holding no regular files (find exits 0 and + # emits nothing, which pipefail cannot see). The signer refuses to list that value, so + # the membership check below would fail anyway -- but it would fail as "not in the signed + # document", sending the reader to the signing chain instead of the missing directory. + if [ "$NEW_HASH" = d41d8cd98f00b204e9800998ecf8427e ]; then + echo 'nothing was hashed -- this checkout has no ZelBack files' + exit 1 + fi echo "tree hash: ${NEW_HASH}" # exported for the node script below; GITHUB_ENV only reaches subsequent steps export NEW_HASH diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 112dd43a7f..52fc82e268 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -66,7 +66,17 @@ jobs: - name: Merged tree is in the signed document central serves if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' run: | + set -eo pipefail NEW_HASH=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}') + # d41d8... is the md5 of an empty stream, produced whenever nothing was hashed: ZelBack + # absent (pipefail catches that) or present holding no regular files (find exits 0 and + # emits nothing, which pipefail cannot see). The signer refuses to list that value, so + # the membership check below would fail anyway -- but it would fail as "not in the signed + # document", sending the reader to the signing chain instead of the missing directory. + if [ "$NEW_HASH" = d41d8cd98f00b204e9800998ecf8427e ]; then + echo 'nothing was hashed -- this checkout has no ZelBack files' + exit 1 + fi echo "merged tree hash: ${NEW_HASH}" # exported for the node script below; GITHUB_ENV only reaches subsequent steps export NEW_HASH From 2b4466a7c2d85910fe224f22657265a999d4efc8 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 25 Aug 2026 13:21:50 +0100 Subject: [PATCH 06/10] chore(ci): move the actions to the current majors actions/checkout v3 -> v7, actions/create-github-app-token v1 -> v3. checkout v3 was three majors behind and runs on a deprecated Node. v7 also refuses to fetch fork pull request code under pull_request_target and workflow_run -- neither workflow uses those triggers, so nothing changes today, but the dangerous shape now fails closed if one is ever added. The app-token majors do not reach us: v2 removed the underscore input spellings and release-tag uses the hyphenated ones; v3 removed custom proxy handling and raises the self-hosted runner floor, and this runs GitHub-hosted with no proxy. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HQKpZbuxYqrWeErU7vKgoA --- .github/workflows/release-gate.yml | 2 +- .github/workflows/release-tag.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index b4dd4956d5..bbac841a84 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -30,7 +30,7 @@ jobs: steps: # The default pull_request checkout is the merge preview -- the tree master will actually # hold after the merge -- which is the thing worth judging, not the PR head alone. - - uses: actions/checkout@v3 + - uses: actions/checkout@v7 - name: Version moves forward run: | diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 52fc82e268..c8114b195e 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -27,7 +27,7 @@ jobs: tag: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v7 with: fetch-depth: 2 @@ -157,7 +157,7 @@ jobs: - name: Mint the dispatch token if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' id: dispatch-token - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@v3 with: app-id: ${{ secrets.FLUXHASHES_APP_ID }} private-key: ${{ secrets.FLUXHASHES_APP_KEY }} From f2128bafb81182e9dd6dab32c76a492cb6791604 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 1 Sep 2026 12:44:15 +0100 Subject: [PATCH 07/10] fix(ci): a tag carrying the release version is not proof of a re-run The idempotency check matched the tag by name alone, so a tag holding the release version but pointing at another commit -- one cut by hand, or one master was later rolled back behind -- was read as "this workflow already ran". Every remaining step was skipped, including the merged-tree verification that is the only check covering what master actually holds, and the job reported success with no tag cut and nothing verified. The tag is now resolved to a commit and compared to the one being released. A match is a genuine re-run and still skips; anything else stops the job and names both commits. Both tag forms are read, because an annotated tag advertises the commit through a peeled ref and a lightweight tag does not -- and every tag in this repository today is lightweight, so reading one form finds neither reliably. ls-remote answers empty when nothing matches, so pipefail is what separates "no such tag" from a transport failure. tests/ci/release-tag.sh runs the step as it ships, extracted from the workflow rather than retyped, across both tag forms and a dead remote; it also asserts the rest of the job still reads the output this step writes, since an output that is never written compares equal to nothing and silently disables the release. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/release-tag.yml | 32 +++- tests/ci/release-tag.sh | 297 ++++++++++++++++++++++++++++++ 2 files changed, 325 insertions(+), 4 deletions(-) create mode 100755 tests/ci/release-tag.sh diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index c8114b195e..f075fc96f2 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -11,6 +11,10 @@ name: release-tag # # A red run here means the release is stuck loudly with no tag -- the fleet is already on the # verified merge tree, so nothing is half-made. +# +# Tests: tests/ci/release-tag.sh -- runs offline, no secrets. It extracts steps from this file +# verbatim, so renaming one breaks it loudly rather than silently skipping it. Run it after any +# change here. on: push: @@ -47,16 +51,36 @@ jobs: echo "version=${CURRENT}" >> "$GITHUB_OUTPUT" fi - # Idempotent: a re-run after the tag exists has nothing left to do. + # Idempotent on a re-run of THIS commit, loud on anything else. A tag carrying the release + # version but pointing somewhere else is not a re-run -- it is a hand-cut tag, or a master + # that was rolled back behind one. Treating that as "already done" would skip the merged-tree + # verification below, which is the only check covering what master actually holds, and report + # the release green with no tag and nothing verified. - name: Tag already cut? id: existing if: steps.version.outputs.release == 'true' + env: + VERSION: ${{ steps.version.outputs.version }} run: | - if git ls-remote --exit-code --tags origin "refs/tags/v${{ steps.version.outputs.version }}" > /dev/null; then - echo "tag v${{ steps.version.outputs.version }} already exists" + set -eo pipefail + # An annotated tag advertises a peeled ^{} ref holding the commit; a lightweight tag has + # no ^{} line and points at the commit directly. Everything this workflow cuts is + # annotated; every tag cut by hand is lightweight, and at the time of writing all 312 + # tags in this repository are lightweight -- so reading only one form finds neither + # reliably. ls-remote exits 0 with no output when nothing matches, so an empty answer + # means "no such tag" while a genuine transport failure dies here on pipefail. + PEELED=$(git ls-remote --tags origin "refs/tags/v${VERSION}^{}" | cut -f1) + DIRECT=$(git ls-remote --tags origin "refs/tags/v${VERSION}" | cut -f1) + EXISTING=${PEELED:-$DIRECT} + if [ -z "$EXISTING" ]; then + echo "done=false" >> "$GITHUB_OUTPUT" + elif [ "$EXISTING" = "$GITHUB_SHA" ]; then + echo "tag v${VERSION} already points at this commit -- re-run, nothing to do" echo "done=true" >> "$GITHUB_OUTPUT" else - echo "done=false" >> "$GITHUB_OUTPUT" + echo "tag v${VERSION} already exists at ${EXISTING}, which is not this commit (${GITHUB_SHA})" + echo "resolve the tag before re-running: this release has not been verified against the signed document" + exit 1 fi # The merge tree normally equals the PR head tree the gate already saw signed, so this diff --git a/tests/ci/release-tag.sh b/tests/ci/release-tag.sh new file mode 100755 index 0000000000..85b52dfbee --- /dev/null +++ b/tests/ci/release-tag.sh @@ -0,0 +1,297 @@ +#!/usr/bin/env bash +# +# Tests for .github/workflows/release-tag.yml. +# +# Covers the "Tag already cut?" step, which decides whether a release that has just landed on +# master still needs verifying and tagging. Getting that decision wrong in the permissive +# direction is silent: the job skips the merged-tree verification -- the only check covering what +# master actually holds -- cuts no tag, and reports green. +# +# The step under test is EXTRACTED VERBATIM from the workflow, never retyped, so what runs here is +# what ships. It carries no ${{ }} (the version arrives through env:), so it runs as a plain +# script against a local bare repository standing in for origin. +# +# Two kinds of check: +# * behaviour -- the step's own decisions, eight cases +# * contract -- that the rest of the job still reads the output the way the step writes it, +# which is where a plausible-looking change does its damage +# +# Then every check is mutation-tested: five defects are reintroduced one at a time and the suite +# must go red for each. A suite that stays green under a mutation is not testing what it claims. +# +# Verified separately against real GitHub on 2026-09-01, because a local bare repo cannot prove it: +# annotated tags advertise a peeled refs/tags/^{} entry over https and the pattern form +# matches it (github.com/git/git, tag v2.39.2); lightweight tags advertise no such entry. At that +# date RunOnFlux/flux held 312 tags and not one was annotated -- which is why reading only the +# peeled form finds none of the tags cut by hand. +# +# No network, no secrets. Everything lives under a mktemp -d removed on exit. +# +# Usage: tests/ci/release-tag.sh [path-to-repo] + +set -uo pipefail + +REPO=${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)} +WORKFLOW="$REPO/.github/workflows/release-tag.yml" + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +PASS=0 +FAIL=0 +FAILED_CASES=() + +# ---------------------------------------------------------------- extract the step, verbatim +extract_step() { + python3 - "$WORKFLOW" "$1" <<'PY' +import sys, yaml +doc = yaml.safe_load(open(sys.argv[1])) +steps = [s for s in doc['jobs']['tag']['steps'] if s.get('name') == 'Tag already cut?'] +assert len(steps) == 1, f"expected one 'Tag already cut?' step, found {len(steps)}" +run = steps[0]['run'] +assert '${{' not in run, "the step interpolates ${{ }} into the shell -- it cannot be run verbatim" +open(sys.argv[2], 'w').write(run) +PY +} + +# ---------------------------------------------------------------- the world the step runs against +# A bare repo as origin, holding two commits, plus a clone whose origin points at it. +build_world() { + rm -rf "$WORK/origin.git" "$WORK/clone" + mkdir -p "$WORK/src" + ( + cd "$WORK/src" && rm -rf .git ./* + git init -q . && git config user.email t@t && git config user.name t + echo one > f && git add f && git commit -qm one + echo two > f && git add f && git commit -qm two + ) + git clone -q --bare "$WORK/src" "$WORK/origin.git" + git clone -q "$WORK/origin.git" "$WORK/clone" + HEAD_SHA=$(git -C "$WORK/clone" rev-parse HEAD) + OTHER_SHA=$(git -C "$WORK/clone" rev-parse 'HEAD^') +} + +tag_on_origin() { # + local name=$1 sha=$2 kind=$3 + if [ "$kind" = annotated ]; then + git -C "$WORK/src" tag -a "$name" -m "annotated $name" "$sha" -f + else + git -C "$WORK/src" tag "$name" "$sha" -f + fi + git -C "$WORK/src" push -q --force "$WORK/origin.git" "refs/tags/$name" +} + +# ---------------------------------------------------------------- reporting +ok() { PASS=$((PASS + 1)); printf ' ok %s\n' "$1"; } +bad() { FAIL=$((FAIL + 1)); FAILED_CASES+=("$1"); printf ' FAIL %s\n' "$1"; shift; printf ' %s\n' "$@"; } + +# ---------------------------------------------------------------- one behaviour case +# run_case