feat(ci): gate release PRs on the signed hash list, and cut the tag after merge

.github/workflows/release-gate.yml

@@ -0,0 +1,161 @@
+name: release-gate
+
+# Required status check on release PRs (development -> master). Verifies, before a release can
+# merge, that what master will become is already approved: the version moves forward, the tree's
+# hash is in the signed document central serves, and the old-fluxbench fallback list carries it.
+#
+# This check is the gate the human sees; release-tag.yml re-verifies the merged tree before
+# cutting the tag and GitHub Release. Enforcement comes from branch protection listing this check
+# as required -- the workflow itself has no power to block a merge.
+#
+# Deliberately strict: central not serving a validly signed document is a red run, not a skip. A
+# gate that passes when its source is missing is not a gate.
+#
+# Read-only: no secrets, no pushes.
+#
+# Tests: tests/ci/release-workflows.sh -- runs offline, no secrets. It extracts steps from this
+# file verbatim, so renaming one breaks it loudly rather than silently skipping it.
+
+on:
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release-gate-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  gate:
+    runs-on: ubuntu-22.04
+    steps:
+      # The default pull_request checkout is the merge preview -- the tree master will actually
+      # hold after the merge -- which is the thing worth judging, not the PR head alone.
+      - uses: actions/checkout@v7
+
+      - name: Version moves forward
+        run: |
+          git fetch --quiet --depth 1 origin master
+          BASE=$(git show origin/master:package.json | jq -r .version)
+          CANDIDATE=$(jq -r .version package.json)
+          echo "master: ${BASE}  candidate: ${CANDIDATE}"
+          # Strictly newer, compared numerically per part. String inequality would call 8.17.10
+          # older than 8.17.9.
+          #
+          # The shape is checked before anything is compared. Splitting on dots and comparing
+          # whatever falls out lets a version the parser cannot read be judged by whichever part
+          # happens to differ: "8.18" has no third part and passes, "8.18.0-rc1" passes on its
+          # minor while "8.17.2-rc1" is refused on a NaN. Refuse what cannot be read instead.
+          node -e '
+            const parse = (label, value) => {
+              const parts = /^(\d+)\.(\d+)\.(\d+)$/.exec(value);
+              if (!parts) {
+                throw new Error(`the ${label} version is not three numeric parts: ${value}`);
+              }
+              return parts.slice(1).map(Number);
+            };
+            const base = parse("master", process.argv[1]);
+            const head = parse("candidate", process.argv[2]);
+            const delta = head[0] - base[0] || head[1] - base[1] || head[2] - base[2];
+            if (!(delta > 0)) {
+              throw new Error("the candidate version does not move past master -- bump package.json in the release PR");
+            }
+          ' "$BASE" "$CANDIDATE"
+
+      # The tree judged below is the merge preview, which is a ref nowhere. Its hash is in the
+      # signed document only because some branch tip already holds the same ZelBack subtree --
+      # normally this PR's head, because master carries no code of its own: every commit landing
+      # directly on master has been version metadata or operational data under helpers/, outside
+      # the hashed tree.
+      #
+      # When master DOES carry ZelBack changes the head lacks, the merged tree exists on no branch,
+      # no signing run can ever cover it, and the check below fails telling the reader to wait for
+      # a signing run that will never come. Refuse here instead, and name the thing that fixes it.
+      #
+      # Deliberately compares the ZelBack subtree, not the whole tree: master diverging under
+      # helpers/ is routine and cannot change the hash.
+      - name: The merged ZelBack is a tree the signer can have seen
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          set -eo pipefail
+          git fetch --quiet --depth 1 origin "refs/pull/${PR_NUMBER}/head"
+          PREVIEW=$(git rev-parse 'HEAD:ZelBack')
+          HEAD_TREE=$(git rev-parse 'FETCH_HEAD:ZelBack')
+          if [ "$PREVIEW" != "$HEAD_TREE" ]; then
+            echo "the merge preview's ZelBack is ${PREVIEW}, this PR head's is ${HEAD_TREE}."
+            echo "master carries ZelBack changes that are not on ${GITHUB_HEAD_REF}, so the merged"
+            echo "tree exists on no branch and no signing run can ever cover it."
+            echo "fix: merge master into ${GITHUB_HEAD_REF} and re-run -- its tip will then carry the"
+            echo "merged tree and the signer will pick it up. Waiting will not help."
+            exit 1
+          fi
+          echo "the merged ZelBack is this PR head's ZelBack (${PREVIEW})"
+
+      - name: Tree hash is in the signed document central serves
+        run: |
+          set -eo pipefail
+          NEW_HASH=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}')
+          # d41d8... is the md5 of an empty stream, produced whenever nothing was hashed: ZelBack
+          # absent (pipefail catches that) or present holding no regular files (find exits 0 and
+          # emits nothing, which pipefail cannot see). The signer refuses to list that value, so
+          # the membership check below would fail anyway -- but it would fail as "not in the signed
+          # document", sending the reader to the signing chain instead of the missing directory.
+          if [ "$NEW_HASH" = d41d8cd98f00b204e9800998ecf8427e ]; then
+            echo 'nothing was hashed -- this checkout has no ZelBack files'
+            exit 1
+          fi
+          echo "tree hash: ${NEW_HASH}"
+          # exported for the node script below; GITHUB_ENV only reaches subsequent steps
+          export NEW_HASH
+          echo "NEW_HASH=${NEW_HASH}" >> "$GITHUB_ENV"
+
+          curl -sS -m 30 -o hashlist-signed.json https://hashes.runonflux.io/hashlist
+
+          # Verify the signature the way a consumer does, against the published public keys
+          # (RunOnFlux/fluxhashes SIGNING.md), so a stale deploy answering every path with the
+          # unsigned array -- or anything else that is not the signed document -- is a red run.
+          #
+          # Single-quoted, with the hash read from the environment: inside a double-quoted shell
+          # string the ${...} below would be the shell's, not node's.
+          node -e '
+            const crypto = require("crypto");
+            const fs = require("fs");
+            const PINNED_PUBLIC_KEYS = [
+              "14837066068b258bfbd0749702056f7065361af44aed48761834744391cbbaaa",
+              "fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8",
+            ];
+            const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex");
+            const document = JSON.parse(fs.readFileSync("hashlist-signed.json", "utf8"));
+            const payload = Buffer.from(document.payload_b64, "base64");
+            const signature = Buffer.from(document.sig_b64, "base64");
+            const verified = PINNED_PUBLIC_KEYS.some((hex) => crypto.verify(
+              null,
+              payload,
+              crypto.createPublicKey({
+                key: Buffer.concat([SPKI_ED25519_PREFIX, Buffer.from(hex, "hex")]),
+                format: "der",
+                type: "spki",
+              }),
+              signature,
+            ));
+            if (!verified) {
+              throw new Error("central is not serving a validly signed hash list");
+            }
+            const { seq, hashes } = JSON.parse(payload.toString("utf8"));
+            if (!hashes.includes(process.env.NEW_HASH)) {
+              throw new Error(`the tree hash is not in the signed document (seq ${seq}) -- wait for the signing run to cover the release candidate`);
+            }
+            console.log(`signed document seq ${seq} carries the tree hash`);
+          '
+
+      # Old fluxbench falls back to helpers/hashes.json via GitHub raw when central is unreachable.
+      # The entry rides the release PR instead of a commit straight to master. This step retires
+      # with old fluxbench.
+      - name: Fallback list carries the hash
+        run: |
+          jq -e --arg h "$NEW_HASH" 'index($h) != null' helpers/hashes.json > /dev/null \
+            || { echo "helpers/hashes.json does not contain ${NEW_HASH} -- add it in the release PR"; exit 1; }
+          echo "helpers/hashes.json carries the tree hash"

.github/workflows/release-tag.yml

@@ -0,0 +1,207 @@
+name: release-tag
+
+# Runs when a release lands on master: re-verifies the merged tree against the signed document,
+# takes an advisory look at live-node distribution, then cuts the tag and the GitHub Release and
+# records the tag on the provenance row in fluxhashes.
+#
+# The pre-merge gate (release-gate.yml) judged the merge preview; this judges what master actually
+# holds, which is the authoritative check -- it covers the only divergence branch protection
+# cannot. The tag and Release are outputs of a verified release: nothing in the fleet follows
+# them, the fleet follows master.
+#
+# A red run here means the release is stuck loudly with no tag -- the fleet is already on the
+# verified merge tree, so nothing is half-made.
+#
+# Tests: tests/ci/release-workflows.sh -- runs offline, no secrets. It extracts steps from this file
+# verbatim, so renaming one breaks it loudly rather than silently skipping it. Run it after any
+# change here.
+
+on:
+  push:
+    branches: [master]
+
+permissions:
+  contents: write
+
+concurrency:
+  group: release-tag
+  cancel-in-progress: false
+
+jobs:
+  tag:
+    # Only the real repository. A fork pushing to its own master has neither our credentials
+    # nor a tree on the signed list, so every run there is noise. Matches nodejs.yml.
+    if: github.repository == 'RunOnFlux/flux'
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 2
+
+      # A push that does not change the version is not a release. HEAD^1 is master's previous tip
+      # whether the push was a merge or a plain commit.
+      - name: Did the version change?
+        id: version
+        run: |
+          CURRENT=$(jq -r .version package.json)
+          PREVIOUS=$(git show 'HEAD^1:package.json' 2>/dev/null | jq -r .version || echo "")
+          echo "previous: ${PREVIOUS:-none}  current: ${CURRENT}"
+          if [ "$CURRENT" = "$PREVIOUS" ]; then
+            echo "version unchanged, not a release"
+            echo "release=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "release=true" >> "$GITHUB_OUTPUT"
+            echo "version=${CURRENT}" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Idempotent on a re-run of THIS commit, loud on anything else. A tag carrying the release
+      # version but pointing somewhere else is not a re-run -- it is a hand-cut tag, or a master
+      # that was rolled back behind one. Treating that as "already done" would skip the merged-tree
+      # verification below, which is the only check covering what master actually holds, and report
+      # the release green with no tag and nothing verified.
+      - name: Tag already cut?
+        id: existing
+        if: steps.version.outputs.release == 'true'
+        env:
+          VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          set -eo pipefail
+          # An annotated tag advertises a peeled ^{} ref holding the commit; a lightweight tag has
+          # no ^{} line and points at the commit directly. Everything this workflow cuts is
+          # annotated; every tag cut by hand is lightweight, and at the time of writing all 312
+          # tags in this repository are lightweight -- so reading only one form finds neither
+          # reliably. ls-remote exits 0 with no output when nothing matches, so an empty answer
+          # means "no such tag" while a genuine transport failure dies here on pipefail.
+          PEELED=$(git ls-remote --tags origin "refs/tags/v${VERSION}^{}" | cut -f1)
+          DIRECT=$(git ls-remote --tags origin "refs/tags/v${VERSION}" | cut -f1)
+          EXISTING=${PEELED:-$DIRECT}
+          if [ -z "$EXISTING" ]; then
+            echo "done=false" >> "$GITHUB_OUTPUT"
+          elif [ "$EXISTING" = "$GITHUB_SHA" ]; then
+            echo "tag v${VERSION} already points at this commit -- re-run, nothing to do"
+            echo "done=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "tag v${VERSION} already exists at ${EXISTING}, which is not this commit (${GITHUB_SHA})"
+            echo "resolve the tag before re-running: this release has not been verified against the signed document"
+            exit 1
+          fi
+
+      # The merge tree normally equals the PR head tree the gate already saw signed, so this
+      # passes on the first poll. The loop covers the divergent case: the on-push publish and the
+      # signing run pick the new tree up within minutes, and this waits for them rather than
+      # failing a release over a race.
+      - name: Merged tree is in the signed document central serves
+        if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false'
+        run: |
+          set -eo pipefail
+          NEW_HASH=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}')
+          # d41d8... is the md5 of an empty stream, produced whenever nothing was hashed: ZelBack
+          # absent (pipefail catches that) or present holding no regular files (find exits 0 and
+          # emits nothing, which pipefail cannot see). The signer refuses to list that value, so
+          # the membership check below would fail anyway -- but it would fail as "not in the signed
+          # document", sending the reader to the signing chain instead of the missing directory.
+          if [ "$NEW_HASH" = d41d8cd98f00b204e9800998ecf8427e ]; then
+            echo 'nothing was hashed -- this checkout has no ZelBack files'
+            exit 1
+          fi
+          echo "merged tree hash: ${NEW_HASH}"
+          # exported for the node script below; GITHUB_ENV only reaches subsequent steps
+          export NEW_HASH
+          echo "NEW_HASH=${NEW_HASH}" >> "$GITHUB_ENV"
+
+          for attempt in $(seq 1 30); do
+            curl -sS -m 30 -o hashlist-signed.json https://hashes.runonflux.io/hashlist || true
+            if node -e '
+              const crypto = require("crypto");
+              const fs = require("fs");
+              const PINNED_PUBLIC_KEYS = [
+                "14837066068b258bfbd0749702056f7065361af44aed48761834744391cbbaaa",
+                "fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8",
+              ];
+              const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex");
+              const document = JSON.parse(fs.readFileSync("hashlist-signed.json", "utf8"));
+              const payload = Buffer.from(document.payload_b64, "base64");
+              const signature = Buffer.from(document.sig_b64, "base64");
+              const verified = PINNED_PUBLIC_KEYS.some((hex) => crypto.verify(
+                null,
+                payload,
+                crypto.createPublicKey({
+                  key: Buffer.concat([SPKI_ED25519_PREFIX, Buffer.from(hex, "hex")]),
+                  format: "der",
+                  type: "spki",
+                }),
+                signature,
+              ));
+              if (!verified) throw new Error("not a validly signed document");
+              const { seq, hashes } = JSON.parse(payload.toString("utf8"));
+              if (!hashes.includes(process.env.NEW_HASH)) throw new Error(`not in seq ${seq}`);
+              console.log(`signed document seq ${seq} carries the merged tree hash`);
+            ' 2> poll-error.txt; then
+              exit 0
+            fi
+            # the message line, not the 15-line trace, times thirty attempts. Node prints the
+            # file marker first, so the first line is never the message; head -1 is the fallback
+            # for a failure that is not a Node error at all, so the log cannot go silent.
+            echo "attempt ${attempt}: $(grep -m1 -E '^[A-Za-z]*Error: ' poll-error.txt || head -1 poll-error.txt) -- retrying in 30s"
+            sleep 30
+          done
+
+          echo "the signed document never covered the merged tree; no tag is cut"
+          exit 1
+
+      # Advisory until the fleet serves /flux/hashlist: results are logged, never fatal. Once the
+      # FluxOS side is deployed, enforcement is turning the failure at the end into exit 1.
+      - name: Live-node sample (advisory)
+        if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false'
+        run: |
+          curl -sS -m 60 -o nodelist.json https://api.runonflux.io/daemon/viewdeterministiczelnodelist || { echo "ADVISORY: node list unavailable"; exit 0; }
+          jq -r '.data[].ip | select(. != null and . != "")' nodelist.json | shuf -n 10 > sample.txt || true
+          SERVING=0
+          while read -r ADDR; do
+            HOST=${ADDR%:*}; PORT=${ADDR##*:}; [ "$HOST" = "$PORT" ] && PORT=16127
+            BODY=$(curl -sS -m 5 "http://${HOST}:${PORT}/flux/hashlist" 2>/dev/null || true)
+            PAYLOAD=$(printf '%s' "$BODY" | jq -r 'try (.payload_b64 // .data.payload_b64) // empty' 2>/dev/null || true)
+            if [ -n "$PAYLOAD" ] && printf '%s' "$PAYLOAD" | base64 -d 2>/dev/null | grep -q "$NEW_HASH"; then
+              SERVING=$((SERVING + 1))
+            fi
+          done < sample.txt
+          echo "ADVISORY: $(wc -l < sample.txt | tr -d ' ') nodes sampled, ${SERVING} serving a document with the merged tree hash"
+
+      - name: Cut the tag and the GitHub Release
+        if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          git config user.email runonfluxbot@gmail.com
+          git config user.name runonfluxbot
+          git tag -a "v${VERSION}" -m "FluxOS v${VERSION}"
+          git push --quiet origin "refs/tags/v${VERSION}"
+          gh release create "v${VERSION}" --title "FluxOS v${VERSION}" --generate-notes --verify-tag
+          echo "released v${VERSION}"
+
+      # The tag above was pushed with GITHUB_TOKEN, which triggers no workflows -- so the signer
+      # would only pick it up on its next run's ls-remote delta. This dispatch makes the
+      # annotation immediate; the signer resolves the tag against its own view of this repository
+      # and annotates the provenance row itself. Nothing here writes to fluxhashes.
+      - name: Mint the dispatch token
+        if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false'
+        id: dispatch-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.FLUXHASHES_APP_ID }}
+          private-key: ${{ secrets.FLUXHASHES_APP_KEY }}
+          owner: RunOnFlux
+          repositories: fluxhashes
+      - name: Request the tag annotation
+        if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false'
+        env:
+          GH_TOKEN: ${{ steps.dispatch-token.outputs.token }}
+          VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          gh api -X POST repos/RunOnFlux/fluxhashes/actions/workflows/sign-hashlist.yml/dispatches \
+            -f ref=master \
+            -f "inputs[commit]=${GITHUB_SHA}" \
+            -f "inputs[ref]=v${VERSION}" \
+            -f "inputs[ref_type]=tag" \
+            -f "inputs[claimed_hash]=${NEW_HASH}"