Release 8.18.0: app ownership and tenant isolation, placement diversity, and the first gated release

.eslintignore

@@ -5,3 +5,8 @@
 /ZelApps/
 /docs/
 /dev/
+# ESM, and these rules are written for the CommonJS backend: an ESM import must
+# carry its .js extension, which import/extensions is configured to forbid. It
+# needs a config of its own rather than this one, so it is out of scope here
+# instead of silently wrong.
+/test-infra/

.eslintrc.js

@@ -15,7 +15,7 @@ module.exports = {
   extends: [
     'eslint:recommended',
   ],
-  plugins: [],
+  plugins: ['import'],
   rules: {
     'max-len': [
       'error',
@@ -25,6 +25,9 @@ module.exports = {
         ignoreTrailingComments: true,
       },
     ],
+    // Two spaces, and enforced: the repo carried no indent rule at all, so a
+    // mis-indented block read as intentional to every reader and to CI alike.
+    indent: ['error', 2, { SwitchCase: 1 }],
     'no-console': 'off',
     'linebreak-style': [
       'error',

.github/workflows/integration-harness.yml

@@ -64,10 +64,10 @@ jobs:
           echo "leftover harness containers: $(docker ps -aq --filter label=flux-e2e-run | wc -l)"
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v7
         with:
           node-version: 20
 
@@ -77,6 +77,12 @@ jobs:
 
       - name: Build harness images
         run: |
+          # The fixtures' app binary. It is gitignored and checkout runs
+          # git clean, so it is NEVER present here - eight suites push it, and
+          # suite 63 needs it built from THIS branch's test-app.c (a cached one
+          # has no BURN_CPU, so the container idles and 63's throttle wait dies
+          # at 150s looking like a broken throttler rather than a missing file).
+          bash test-infra/test-app/build.sh
           docker compose -f test-infra/docker-compose.yml -p flux-e2e build \
             fluxos-01 daemon-stub syncthing-stub fdm-stub peer-stub external-http-stub
           docker pull mongo:8
@@ -113,7 +119,7 @@ jobs:
 
       - name: Upload logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: e2e-logs
           path: ${{ runner.temp }}/e2e-logs

.github/workflows/nodejs.yml

@@ -2,7 +2,71 @@ name: Node CI
 
 on: [push]
 
+# Cross-repo pushes and dispatches use per-run GitHub App tokens minted below; the run's own token
+# needs read only.
+permissions:
+  contents: read
+
 jobs:
+  # Publication is a request, not a write: the fluxhashes signer fetches this commit itself,
+  # derives the tree hash from the bytes it fetched, and publishes list, signed document and
+  # provenance in one commit. The credential can start, cancel and re-run workflows on fluxhashes
+  # and delete their logs; it cannot write to the repository, and it sends pointers rather than hash
+  # values. That bounds what the credential does, but it does not make the list unreachable through
+  # it: a fork network shares one object store, so a dispatched commit may be any commit ever pushed
+  # to this repository or to a public fork of it, and the signer derives what it is pointed at. The
+  # credential is held by whoever can land a workflow change on any branch here, who can already get
+  # a hash listed by pushing -- the same trust boundary, not a defence against it.
+  # NEW_HASH rides along as a tripwire the signer checks against its own computation -- a mismatch
+  # is a red signing run, never a listed value. Forks publish nothing.
+  #
+  # Its own job, deliberately: the hash describes a checkout nothing else has touched, and a
+  # publication failure (fluxhashes unavailable, token expired) stays a red publish job instead of
+  # blocking the test suite.
+  publish:
+    runs-on: ubuntu-22.04
+    if: github.repository == 'RunOnFlux/flux'
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v7
+      - name: Check Hash
+        run: |
+          set -eo pipefail
+          newhash=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}')
+          # d41d8... is the md5 of an empty stream, which the pipeline yields whenever nothing was
+          # hashed: ZelBack absent (pipefail catches that) or present holding no regular files
+          # (find exits 0 and emits nothing, which pipefail cannot see). It is a well-formed hash
+          # meaning "a node whose ZelBack holds no files is genuine FluxOS". The signer refuses to
+          # list it, but a claim it cannot match aborts the signing run rather than this one --
+          # so fail here, in the repository that produced it.
+          if [ "$newhash" = d41d8cd98f00b204e9800998ecf8427e ]; then
+            echo 'nothing was hashed -- this checkout has no ZelBack files'
+            exit 1
+          fi
+          echo $newhash
+          echo NEW_HASH=$newhash >> $GITHUB_ENV
+      # The credential is a GitHub App scoped to fluxhashes with Actions permission only: the
+      # workflow mints a short-lived installation token per run, so there is no long-lived token
+      # anywhere and nothing to renew.
+      - name: Mint the dispatch token
+        id: dispatch-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.FLUXHASHES_APP_ID }}
+          private-key: ${{ secrets.FLUXHASHES_APP_KEY }}
+          owner: RunOnFlux
+          repositories: fluxhashes
+      - name: Request hash publication
+        env:
+          GH_TOKEN: ${{ steps.dispatch-token.outputs.token }}
+        run: |
+          gh api -X POST repos/RunOnFlux/fluxhashes/actions/workflows/sign-hashlist.yml/dispatches \
+            -f ref=master \
+            -f "inputs[commit]=${GITHUB_SHA}" \
+            -f "inputs[ref]=${GITHUB_REF_NAME}" \
+            -f "inputs[ref_type]=${GITHUB_REF_TYPE}" \
+            -f "inputs[claimed_hash]=${NEW_HASH}"
+
   build:
     runs-on: ${{ matrix.os }}
 
@@ -19,44 +83,11 @@ jobs:
           - 27017:27017
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v7
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v7
         with:
           node-version: ${{ matrix.node-version }}
-      - name: Check Hash
-        run: |
-          newhash=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}')
-          echo $newhash
-          echo NEW_HASH=$newhash >> $GITHUB_ENV
-      - name: Get current hashes
-        run: |
-          mkdir hashes
-          wget 'https://raw.githubusercontent.com/RunOnFlux/fluxhashes/master/src/hashes/hashes.js' -P hashes
-          hashfile=`cat hashes/hashes.js | sed "s/return/_/gi" | sed "s/\n/_/gi"`
-          echo HASH_FILE=$hashfile >> $GITHUB_ENV
-      - name: Show hashes
-        run: |
-          echo $HASH_FILE
-          echo $NEW_HASH
-      - name: Patch hashes
-        if: ${{ !contains(env.HASH_FILE, env.NEW_HASH) }}
-        run: |
-          newhash=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}')
-          sed -i "s/  ];/    '$newhash',\n  ];/gi" hashes/hashes.js
-          tail -n 200 hashes/hashes.js
-      - name: Push hashes to fluxhashes
-        if: ${{ !contains(env.HASH_FILE, env.NEW_HASH) }}
-        uses: cpina/github-action-push-to-another-repository@main
-        env:
-          API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
-        with:
-          source-directory: "hashes"
-          destination-github-username: "RunOnFlux"
-          destination-repository-name: "fluxhashes"
-          user-email: runonfluxbot@gmail.com
-          target-branch: master
-          target-directory: src/hashes/
       - name: install flux and flux benchmark daemons
         run: |
           echo 'deb https://apt.runonflux.io/ '$(lsb_release -cs)' main' | sudo tee /etc/apt/sources.list.d/flux.list
@@ -93,14 +124,43 @@ jobs:
           CI: true
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
-      - name: Push docs to other repo # Push services directory from RunOnFlux/flux to RunOnFlux/fluxjsdocs repo to build JSDocs separately.
-        uses: cpina/github-action-push-to-another-repository@main
-        env:
-          API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
+      # Only development publishes the docs. Without the ref condition every branch build races for
+      # fluxjsdocs master, so the published JSDocs are whichever branch happened to build last --
+      # an unmerged feature branch as often as not.
+      - name: Mint the docs token
+        if: github.repository == 'RunOnFlux/flux' && github.ref == 'refs/heads/development'
+        id: docs-token
+        uses: actions/create-github-app-token@v3
         with:
-          source-directory: "ZelBack/src/services"
-          destination-github-username: "RunOnFlux"
-          destination-repository-name: "fluxjsdocs"
-          user-email: runonfluxbot@gmail.com
-          target-branch: master
-          target-directory: services/
+          app-id: ${{ secrets.FLUXJSDOCS_APP_ID }}
+          private-key: ${{ secrets.FLUXJSDOCS_APP_KEY }}
+          owner: RunOnFlux
+          repositories: fluxjsdocs
+      # Git directly, rather than a third-party action. The action this replaces was referenced by
+      # a personal repository's default branch, so every build ran whatever happened to be on it at
+      # that moment -- and was handed a token that can write to fluxjsdocs. Nothing outside Flux
+      # touches the credential now. fluxjsdocs is public, so the clone is anonymous and the token
+      # authenticates only the push, which also keeps it out of the clone's stored config.
+      - name: Push docs to fluxjsdocs # Publishes ZelBack/src/services so JSDocs builds separately.
+        if: github.repository == 'RunOnFlux/flux' && github.ref == 'refs/heads/development'
+        env:
+          GH_TOKEN: ${{ steps.docs-token.outputs.token }}
+        run: |
+          set -eo pipefail
+          CLONE=$(mktemp -d)
+          git clone --quiet --depth 1 --single-branch --branch master \
+            https://github.com/RunOnFlux/fluxjsdocs.git "$CLONE"
+          # Replace rather than merge, so a file deleted here disappears there too.
+          rm -rf "$CLONE/services"
+          mkdir -p "$CLONE/services"
+          cp -a ZelBack/src/services/. "$CLONE/services/"
+          git -C "$CLONE" config user.email runonfluxbot@gmail.com
+          git -C "$CLONE" config user.name runonfluxbot
+          git -C "$CLONE" add -A services
+          if git -C "$CLONE" diff --cached --quiet; then
+            echo 'docs unchanged, nothing to push'
+            exit 0
+          fi
+          git -C "$CLONE" commit --quiet -m "Update from https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}"
+          git -C "$CLONE" push --quiet \
+            "https://x-access-token:${GH_TOKEN}@github.com/RunOnFlux/fluxjsdocs.git" master

.github/workflows/release-gate.yml

@@ -0,0 +1,161 @@
+name: release-gate
+
+# Required status check on release PRs (development -> master). Verifies, before a release can
+# merge, that what master will become is already approved: the version moves forward, the tree's
+# hash is in the signed document central serves, and the old-fluxbench fallback list carries it.
+#
+# This check is the gate the human sees; release-tag.yml re-verifies the merged tree before
+# cutting the tag and GitHub Release. Enforcement comes from branch protection listing this check
+# as required -- the workflow itself has no power to block a merge.
+#
+# Deliberately strict: central not serving a validly signed document is a red run, not a skip. A
+# gate that passes when its source is missing is not a gate.
+#
+# Read-only: no secrets, no pushes.
+#
+# Tests: tests/ci/release-workflows.sh -- runs offline, no secrets. It extracts steps from this
+# file verbatim, so renaming one breaks it loudly rather than silently skipping it.
+
+on:
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release-gate-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  gate:
+    runs-on: ubuntu-22.04
+    steps:
+      # The default pull_request checkout is the merge preview -- the tree master will actually
+      # hold after the merge -- which is the thing worth judging, not the PR head alone.
+      - uses: actions/checkout@v7
+
+      - name: Version moves forward
+        run: |
+          git fetch --quiet --depth 1 origin master
+          BASE=$(git show origin/master:package.json | jq -r .version)
+          CANDIDATE=$(jq -r .version package.json)
+          echo "master: ${BASE}  candidate: ${CANDIDATE}"
+          # Strictly newer, compared numerically per part. String inequality would call 8.17.10
+          # older than 8.17.9.
+          #
+          # The shape is checked before anything is compared. Splitting on dots and comparing
+          # whatever falls out lets a version the parser cannot read be judged by whichever part
+          # happens to differ: "8.18" has no third part and passes, "8.18.0-rc1" passes on its
+          # minor while "8.17.2-rc1" is refused on a NaN. Refuse what cannot be read instead.
+          node -e '
+            const parse = (label, value) => {
+              const parts = /^(\d+)\.(\d+)\.(\d+)$/.exec(value);
+              if (!parts) {
+                throw new Error(`the ${label} version is not three numeric parts: ${value}`);
+              }
+              return parts.slice(1).map(Number);
+            };
+            const base = parse("master", process.argv[1]);
+            const head = parse("candidate", process.argv[2]);
+            const delta = head[0] - base[0] || head[1] - base[1] || head[2] - base[2];
+            if (!(delta > 0)) {
+              throw new Error("the candidate version does not move past master -- bump package.json in the release PR");
+            }
+          ' "$BASE" "$CANDIDATE"
+
+      # The tree judged below is the merge preview, which is a ref nowhere. Its hash is in the
+      # signed document only because some branch tip already holds the same ZelBack subtree --
+      # normally this PR's head, because master carries no code of its own: every commit landing
+      # directly on master has been version metadata or operational data under helpers/, outside
+      # the hashed tree.
+      #
+      # When master DOES carry ZelBack changes the head lacks, the merged tree exists on no branch,
+      # no signing run can ever cover it, and the check below fails telling the reader to wait for
+      # a signing run that will never come. Refuse here instead, and name the thing that fixes it.
+      #
+      # Deliberately compares the ZelBack subtree, not the whole tree: master diverging under
+      # helpers/ is routine and cannot change the hash.
+      - name: The merged ZelBack is a tree the signer can have seen
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          set -eo pipefail
+          git fetch --quiet --depth 1 origin "refs/pull/${PR_NUMBER}/head"
+          PREVIEW=$(git rev-parse 'HEAD:ZelBack')
+          HEAD_TREE=$(git rev-parse 'FETCH_HEAD:ZelBack')
+          if [ "$PREVIEW" != "$HEAD_TREE" ]; then
+            echo "the merge preview's ZelBack is ${PREVIEW}, this PR head's is ${HEAD_TREE}."
+            echo "master carries ZelBack changes that are not on ${GITHUB_HEAD_REF}, so the merged"
+            echo "tree exists on no branch and no signing run can ever cover it."
+            echo "fix: merge master into ${GITHUB_HEAD_REF} and re-run -- its tip will then carry the"
+            echo "merged tree and the signer will pick it up. Waiting will not help."
+            exit 1
+          fi
+          echo "the merged ZelBack is this PR head's ZelBack (${PREVIEW})"
+
+      - name: Tree hash is in the signed document central serves
+        run: |
+          set -eo pipefail
+          NEW_HASH=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}')
+          # d41d8... is the md5 of an empty stream, produced whenever nothing was hashed: ZelBack
+          # absent (pipefail catches that) or present holding no regular files (find exits 0 and
+          # emits nothing, which pipefail cannot see). The signer refuses to list that value, so
+          # the membership check below would fail anyway -- but it would fail as "not in the signed
+          # document", sending the reader to the signing chain instead of the missing directory.
+          if [ "$NEW_HASH" = d41d8cd98f00b204e9800998ecf8427e ]; then
+            echo 'nothing was hashed -- this checkout has no ZelBack files'
+            exit 1
+          fi
+          echo "tree hash: ${NEW_HASH}"
+          # exported for the node script below; GITHUB_ENV only reaches subsequent steps
+          export NEW_HASH
+          echo "NEW_HASH=${NEW_HASH}" >> "$GITHUB_ENV"
+
+          curl -sS -m 30 -o hashlist-signed.json https://hashes.runonflux.io/hashlist
+
+          # Verify the signature the way a consumer does, against the published public keys
+          # (RunOnFlux/fluxhashes SIGNING.md), so a stale deploy answering every path with the
+          # unsigned array -- or anything else that is not the signed document -- is a red run.
+          #
+          # Single-quoted, with the hash read from the environment: inside a double-quoted shell
+          # string the ${...} below would be the shell's, not node's.
+          node -e '
+            const crypto = require("crypto");
+            const fs = require("fs");
+            const PINNED_PUBLIC_KEYS = [
+              "14837066068b258bfbd0749702056f7065361af44aed48761834744391cbbaaa",
+              "fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8",
+            ];
+            const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex");
+            const document = JSON.parse(fs.readFileSync("hashlist-signed.json", "utf8"));
+            const payload = Buffer.from(document.payload_b64, "base64");
+            const signature = Buffer.from(document.sig_b64, "base64");
+            const verified = PINNED_PUBLIC_KEYS.some((hex) => crypto.verify(
+              null,
+              payload,
+              crypto.createPublicKey({
+                key: Buffer.concat([SPKI_ED25519_PREFIX, Buffer.from(hex, "hex")]),
+                format: "der",
+                type: "spki",
+              }),
+              signature,
+            ));
+            if (!verified) {
+              throw new Error("central is not serving a validly signed hash list");
+            }
+            const { seq, hashes } = JSON.parse(payload.toString("utf8"));
+            if (!hashes.includes(process.env.NEW_HASH)) {
+              throw new Error(`the tree hash is not in the signed document (seq ${seq}) -- wait for the signing run to cover the release candidate`);
+            }
+            console.log(`signed document seq ${seq} carries the tree hash`);
+          '
+
+      # Old fluxbench falls back to helpers/hashes.json via GitHub raw when central is unreachable.
+      # The entry rides the release PR instead of a commit straight to master. This step retires
+      # with old fluxbench.
+      - name: Fallback list carries the hash
+        run: |
+          jq -e --arg h "$NEW_HASH" 'index($h) != null' helpers/hashes.json > /dev/null \
+            || { echo "helpers/hashes.json does not contain ${NEW_HASH} -- add it in the release PR"; exit 1; }
+          echo "helpers/hashes.json carries the tree hash"