diff --git a/.eslintignore b/.eslintignore index bae2ce36d0..56fd31982b 100644 --- a/.eslintignore +++ b/.eslintignore @@ -5,3 +5,8 @@ /ZelApps/ /docs/ /dev/ +# ESM, and these rules are written for the CommonJS backend: an ESM import must +# carry its .js extension, which import/extensions is configured to forbid. It +# needs a config of its own rather than this one, so it is out of scope here +# instead of silently wrong. +/test-infra/ diff --git a/.eslintrc.js b/.eslintrc.js index dc1e82aceb..4057088517 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -15,7 +15,7 @@ module.exports = { extends: [ 'eslint:recommended', ], - plugins: [], + plugins: ['import'], rules: { 'max-len': [ 'error', @@ -25,6 +25,9 @@ module.exports = { ignoreTrailingComments: true, }, ], + // Two spaces, and enforced: the repo carried no indent rule at all, so a + // mis-indented block read as intentional to every reader and to CI alike. + indent: ['error', 2, { SwitchCase: 1 }], 'no-console': 'off', 'linebreak-style': [ 'error', diff --git a/.github/workflows/integration-harness.yml b/.github/workflows/integration-harness.yml index 8a024838b9..6b876d208f 100644 --- a/.github/workflows/integration-harness.yml +++ b/.github/workflows/integration-harness.yml @@ -64,10 +64,10 @@ jobs: echo "leftover harness containers: $(docker ps -aq --filter label=flux-e2e-run | wc -l)" - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: 20 @@ -77,6 +77,12 @@ jobs: - name: Build harness images run: | + # The fixtures' app binary. It is gitignored and checkout runs + # git clean, so it is NEVER present here - eight suites push it, and + # suite 63 needs it built from THIS branch's test-app.c (a cached one + # has no BURN_CPU, so the container idles and 63's throttle wait dies + # at 150s looking like a broken throttler rather than a missing file). + bash test-infra/test-app/build.sh docker compose -f test-infra/docker-compose.yml -p flux-e2e build \ fluxos-01 daemon-stub syncthing-stub fdm-stub peer-stub external-http-stub docker pull mongo:8 @@ -113,7 +119,7 @@ jobs: - name: Upload logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: e2e-logs path: ${{ runner.temp }}/e2e-logs diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 5fbd5b6305..1cfc50ae36 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -2,7 +2,71 @@ name: Node CI on: [push] +# Cross-repo pushes and dispatches use per-run GitHub App tokens minted below; the run's own token +# needs read only. +permissions: + contents: read + jobs: + # Publication is a request, not a write: the fluxhashes signer fetches this commit itself, + # derives the tree hash from the bytes it fetched, and publishes list, signed document and + # provenance in one commit. The credential can start, cancel and re-run workflows on fluxhashes + # and delete their logs; it cannot write to the repository, and it sends pointers rather than hash + # values. That bounds what the credential does, but it does not make the list unreachable through + # it: a fork network shares one object store, so a dispatched commit may be any commit ever pushed + # to this repository or to a public fork of it, and the signer derives what it is pointed at. The + # credential is held by whoever can land a workflow change on any branch here, who can already get + # a hash listed by pushing -- the same trust boundary, not a defence against it. + # NEW_HASH rides along as a tripwire the signer checks against its own computation -- a mismatch + # is a red signing run, never a listed value. Forks publish nothing. + # + # Its own job, deliberately: the hash describes a checkout nothing else has touched, and a + # publication failure (fluxhashes unavailable, token expired) stays a red publish job instead of + # blocking the test suite. + publish: + runs-on: ubuntu-22.04 + if: github.repository == 'RunOnFlux/flux' + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + - name: Check Hash + run: | + set -eo pipefail + newhash=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}') + # d41d8... is the md5 of an empty stream, which the pipeline yields whenever nothing was + # hashed: ZelBack absent (pipefail catches that) or present holding no regular files + # (find exits 0 and emits nothing, which pipefail cannot see). It is a well-formed hash + # meaning "a node whose ZelBack holds no files is genuine FluxOS". The signer refuses to + # list it, but a claim it cannot match aborts the signing run rather than this one -- + # so fail here, in the repository that produced it. + if [ "$newhash" = d41d8cd98f00b204e9800998ecf8427e ]; then + echo 'nothing was hashed -- this checkout has no ZelBack files' + exit 1 + fi + echo $newhash + echo NEW_HASH=$newhash >> $GITHUB_ENV + # The credential is a GitHub App scoped to fluxhashes with Actions permission only: the + # workflow mints a short-lived installation token per run, so there is no long-lived token + # anywhere and nothing to renew. + - name: Mint the dispatch token + id: dispatch-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.FLUXHASHES_APP_ID }} + private-key: ${{ secrets.FLUXHASHES_APP_KEY }} + owner: RunOnFlux + repositories: fluxhashes + - name: Request hash publication + env: + GH_TOKEN: ${{ steps.dispatch-token.outputs.token }} + run: | + gh api -X POST repos/RunOnFlux/fluxhashes/actions/workflows/sign-hashlist.yml/dispatches \ + -f ref=master \ + -f "inputs[commit]=${GITHUB_SHA}" \ + -f "inputs[ref]=${GITHUB_REF_NAME}" \ + -f "inputs[ref_type]=${GITHUB_REF_TYPE}" \ + -f "inputs[claimed_hash]=${NEW_HASH}" + build: runs-on: ${{ matrix.os }} @@ -19,44 +83,11 @@ jobs: - 27017:27017 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v7 - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 + uses: actions/setup-node@v7 with: node-version: ${{ matrix.node-version }} - - name: Check Hash - run: | - newhash=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}') - echo $newhash - echo NEW_HASH=$newhash >> $GITHUB_ENV - - name: Get current hashes - run: | - mkdir hashes - wget 'https://raw.githubusercontent.com/RunOnFlux/fluxhashes/master/src/hashes/hashes.js' -P hashes - hashfile=`cat hashes/hashes.js | sed "s/return/_/gi" | sed "s/\n/_/gi"` - echo HASH_FILE=$hashfile >> $GITHUB_ENV - - name: Show hashes - run: | - echo $HASH_FILE - echo $NEW_HASH - - name: Patch hashes - if: ${{ !contains(env.HASH_FILE, env.NEW_HASH) }} - run: | - newhash=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}') - sed -i "s/ ];/ '$newhash',\n ];/gi" hashes/hashes.js - tail -n 200 hashes/hashes.js - - name: Push hashes to fluxhashes - if: ${{ !contains(env.HASH_FILE, env.NEW_HASH) }} - uses: cpina/github-action-push-to-another-repository@main - env: - API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} - with: - source-directory: "hashes" - destination-github-username: "RunOnFlux" - destination-repository-name: "fluxhashes" - user-email: runonfluxbot@gmail.com - target-branch: master - target-directory: src/hashes/ - name: install flux and flux benchmark daemons run: | echo 'deb https://apt.runonflux.io/ '$(lsb_release -cs)' main' | sudo tee /etc/apt/sources.list.d/flux.list @@ -93,14 +124,43 @@ jobs: CI: true - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - - name: Push docs to other repo # Push services directory from RunOnFlux/flux to RunOnFlux/fluxjsdocs repo to build JSDocs separately. - uses: cpina/github-action-push-to-another-repository@main - env: - API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} + # Only development publishes the docs. Without the ref condition every branch build races for + # fluxjsdocs master, so the published JSDocs are whichever branch happened to build last -- + # an unmerged feature branch as often as not. + - name: Mint the docs token + if: github.repository == 'RunOnFlux/flux' && github.ref == 'refs/heads/development' + id: docs-token + uses: actions/create-github-app-token@v3 with: - source-directory: "ZelBack/src/services" - destination-github-username: "RunOnFlux" - destination-repository-name: "fluxjsdocs" - user-email: runonfluxbot@gmail.com - target-branch: master - target-directory: services/ + app-id: ${{ secrets.FLUXJSDOCS_APP_ID }} + private-key: ${{ secrets.FLUXJSDOCS_APP_KEY }} + owner: RunOnFlux + repositories: fluxjsdocs + # Git directly, rather than a third-party action. The action this replaces was referenced by + # a personal repository's default branch, so every build ran whatever happened to be on it at + # that moment -- and was handed a token that can write to fluxjsdocs. Nothing outside Flux + # touches the credential now. fluxjsdocs is public, so the clone is anonymous and the token + # authenticates only the push, which also keeps it out of the clone's stored config. + - name: Push docs to fluxjsdocs # Publishes ZelBack/src/services so JSDocs builds separately. + if: github.repository == 'RunOnFlux/flux' && github.ref == 'refs/heads/development' + env: + GH_TOKEN: ${{ steps.docs-token.outputs.token }} + run: | + set -eo pipefail + CLONE=$(mktemp -d) + git clone --quiet --depth 1 --single-branch --branch master \ + https://github.com/RunOnFlux/fluxjsdocs.git "$CLONE" + # Replace rather than merge, so a file deleted here disappears there too. + rm -rf "$CLONE/services" + mkdir -p "$CLONE/services" + cp -a ZelBack/src/services/. "$CLONE/services/" + git -C "$CLONE" config user.email runonfluxbot@gmail.com + git -C "$CLONE" config user.name runonfluxbot + git -C "$CLONE" add -A services + if git -C "$CLONE" diff --cached --quiet; then + echo 'docs unchanged, nothing to push' + exit 0 + fi + git -C "$CLONE" commit --quiet -m "Update from https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" + git -C "$CLONE" push --quiet \ + "https://x-access-token:${GH_TOKEN}@github.com/RunOnFlux/fluxjsdocs.git" master diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 0000000000..b8bbff58c4 --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,161 @@ +name: release-gate + +# Required status check on release PRs (development -> master). Verifies, before a release can +# merge, that what master will become is already approved: the version moves forward, the tree's +# hash is in the signed document central serves, and the old-fluxbench fallback list carries it. +# +# This check is the gate the human sees; release-tag.yml re-verifies the merged tree before +# cutting the tag and GitHub Release. Enforcement comes from branch protection listing this check +# as required -- the workflow itself has no power to block a merge. +# +# Deliberately strict: central not serving a validly signed document is a red run, not a skip. A +# gate that passes when its source is missing is not a gate. +# +# Read-only: no secrets, no pushes. +# +# Tests: tests/ci/release-workflows.sh -- runs offline, no secrets. It extracts steps from this +# file verbatim, so renaming one breaks it loudly rather than silently skipping it. + +on: + pull_request: + branches: [master] + +permissions: + contents: read + +concurrency: + group: release-gate-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + gate: + runs-on: ubuntu-22.04 + steps: + # The default pull_request checkout is the merge preview -- the tree master will actually + # hold after the merge -- which is the thing worth judging, not the PR head alone. + - uses: actions/checkout@v7 + + - name: Version moves forward + run: | + git fetch --quiet --depth 1 origin master + BASE=$(git show origin/master:package.json | jq -r .version) + CANDIDATE=$(jq -r .version package.json) + echo "master: ${BASE} candidate: ${CANDIDATE}" + # Strictly newer, compared numerically per part. String inequality would call 8.17.10 + # older than 8.17.9. + # + # The shape is checked before anything is compared. Splitting on dots and comparing + # whatever falls out lets a version the parser cannot read be judged by whichever part + # happens to differ: "8.18" has no third part and passes, "8.18.0-rc1" passes on its + # minor while "8.17.2-rc1" is refused on a NaN. Refuse what cannot be read instead. + node -e ' + const parse = (label, value) => { + const parts = /^(\d+)\.(\d+)\.(\d+)$/.exec(value); + if (!parts) { + throw new Error(`the ${label} version is not three numeric parts: ${value}`); + } + return parts.slice(1).map(Number); + }; + const base = parse("master", process.argv[1]); + const head = parse("candidate", process.argv[2]); + const delta = head[0] - base[0] || head[1] - base[1] || head[2] - base[2]; + if (!(delta > 0)) { + throw new Error("the candidate version does not move past master -- bump package.json in the release PR"); + } + ' "$BASE" "$CANDIDATE" + + # The tree judged below is the merge preview, which is a ref nowhere. Its hash is in the + # signed document only because some branch tip already holds the same ZelBack subtree -- + # normally this PR's head, because master carries no code of its own: every commit landing + # directly on master has been version metadata or operational data under helpers/, outside + # the hashed tree. + # + # When master DOES carry ZelBack changes the head lacks, the merged tree exists on no branch, + # no signing run can ever cover it, and the check below fails telling the reader to wait for + # a signing run that will never come. Refuse here instead, and name the thing that fixes it. + # + # Deliberately compares the ZelBack subtree, not the whole tree: master diverging under + # helpers/ is routine and cannot change the hash. + - name: The merged ZelBack is a tree the signer can have seen + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -eo pipefail + git fetch --quiet --depth 1 origin "refs/pull/${PR_NUMBER}/head" + PREVIEW=$(git rev-parse 'HEAD:ZelBack') + HEAD_TREE=$(git rev-parse 'FETCH_HEAD:ZelBack') + if [ "$PREVIEW" != "$HEAD_TREE" ]; then + echo "the merge preview's ZelBack is ${PREVIEW}, this PR head's is ${HEAD_TREE}." + echo "master carries ZelBack changes that are not on ${GITHUB_HEAD_REF}, so the merged" + echo "tree exists on no branch and no signing run can ever cover it." + echo "fix: merge master into ${GITHUB_HEAD_REF} and re-run -- its tip will then carry the" + echo "merged tree and the signer will pick it up. Waiting will not help." + exit 1 + fi + echo "the merged ZelBack is this PR head's ZelBack (${PREVIEW})" + + - name: Tree hash is in the signed document central serves + run: | + set -eo pipefail + NEW_HASH=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}') + # d41d8... is the md5 of an empty stream, produced whenever nothing was hashed: ZelBack + # absent (pipefail catches that) or present holding no regular files (find exits 0 and + # emits nothing, which pipefail cannot see). The signer refuses to list that value, so + # the membership check below would fail anyway -- but it would fail as "not in the signed + # document", sending the reader to the signing chain instead of the missing directory. + if [ "$NEW_HASH" = d41d8cd98f00b204e9800998ecf8427e ]; then + echo 'nothing was hashed -- this checkout has no ZelBack files' + exit 1 + fi + echo "tree hash: ${NEW_HASH}" + # exported for the node script below; GITHUB_ENV only reaches subsequent steps + export NEW_HASH + echo "NEW_HASH=${NEW_HASH}" >> "$GITHUB_ENV" + + curl -sS -m 30 -o hashlist-signed.json https://hashes.runonflux.io/hashlist + + # Verify the signature the way a consumer does, against the published public keys + # (RunOnFlux/fluxhashes SIGNING.md), so a stale deploy answering every path with the + # unsigned array -- or anything else that is not the signed document -- is a red run. + # + # Single-quoted, with the hash read from the environment: inside a double-quoted shell + # string the ${...} below would be the shell's, not node's. + node -e ' + const crypto = require("crypto"); + const fs = require("fs"); + const PINNED_PUBLIC_KEYS = [ + "14837066068b258bfbd0749702056f7065361af44aed48761834744391cbbaaa", + "fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8", + ]; + const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); + const document = JSON.parse(fs.readFileSync("hashlist-signed.json", "utf8")); + const payload = Buffer.from(document.payload_b64, "base64"); + const signature = Buffer.from(document.sig_b64, "base64"); + const verified = PINNED_PUBLIC_KEYS.some((hex) => crypto.verify( + null, + payload, + crypto.createPublicKey({ + key: Buffer.concat([SPKI_ED25519_PREFIX, Buffer.from(hex, "hex")]), + format: "der", + type: "spki", + }), + signature, + )); + if (!verified) { + throw new Error("central is not serving a validly signed hash list"); + } + const { seq, hashes } = JSON.parse(payload.toString("utf8")); + if (!hashes.includes(process.env.NEW_HASH)) { + throw new Error(`the tree hash is not in the signed document (seq ${seq}) -- wait for the signing run to cover the release candidate`); + } + console.log(`signed document seq ${seq} carries the tree hash`); + ' + + # Old fluxbench falls back to helpers/hashes.json via GitHub raw when central is unreachable. + # The entry rides the release PR instead of a commit straight to master. This step retires + # with old fluxbench. + - name: Fallback list carries the hash + run: | + jq -e --arg h "$NEW_HASH" 'index($h) != null' helpers/hashes.json > /dev/null \ + || { echo "helpers/hashes.json does not contain ${NEW_HASH} -- add it in the release PR"; exit 1; } + echo "helpers/hashes.json carries the tree hash" diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml new file mode 100644 index 0000000000..70930931d2 --- /dev/null +++ b/.github/workflows/release-tag.yml @@ -0,0 +1,207 @@ +name: release-tag + +# Runs when a release lands on master: re-verifies the merged tree against the signed document, +# takes an advisory look at live-node distribution, then cuts the tag and the GitHub Release and +# records the tag on the provenance row in fluxhashes. +# +# The pre-merge gate (release-gate.yml) judged the merge preview; this judges what master actually +# holds, which is the authoritative check -- it covers the only divergence branch protection +# cannot. The tag and Release are outputs of a verified release: nothing in the fleet follows +# them, the fleet follows master. +# +# A red run here means the release is stuck loudly with no tag -- the fleet is already on the +# verified merge tree, so nothing is half-made. +# +# Tests: tests/ci/release-workflows.sh -- runs offline, no secrets. It extracts steps from this file +# verbatim, so renaming one breaks it loudly rather than silently skipping it. Run it after any +# change here. + +on: + push: + branches: [master] + +permissions: + contents: write + +concurrency: + group: release-tag + cancel-in-progress: false + +jobs: + tag: + # Only the real repository. A fork pushing to its own master has neither our credentials + # nor a tree on the signed list, so every run there is noise. Matches nodejs.yml. + if: github.repository == 'RunOnFlux/flux' + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 2 + + # A push that does not change the version is not a release. HEAD^1 is master's previous tip + # whether the push was a merge or a plain commit. + - name: Did the version change? + id: version + run: | + CURRENT=$(jq -r .version package.json) + PREVIOUS=$(git show 'HEAD^1:package.json' 2>/dev/null | jq -r .version || echo "") + echo "previous: ${PREVIOUS:-none} current: ${CURRENT}" + if [ "$CURRENT" = "$PREVIOUS" ]; then + echo "version unchanged, not a release" + echo "release=false" >> "$GITHUB_OUTPUT" + else + echo "release=true" >> "$GITHUB_OUTPUT" + echo "version=${CURRENT}" >> "$GITHUB_OUTPUT" + fi + + # Idempotent on a re-run of THIS commit, loud on anything else. A tag carrying the release + # version but pointing somewhere else is not a re-run -- it is a hand-cut tag, or a master + # that was rolled back behind one. Treating that as "already done" would skip the merged-tree + # verification below, which is the only check covering what master actually holds, and report + # the release green with no tag and nothing verified. + - name: Tag already cut? + id: existing + if: steps.version.outputs.release == 'true' + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -eo pipefail + # An annotated tag advertises a peeled ^{} ref holding the commit; a lightweight tag has + # no ^{} line and points at the commit directly. Everything this workflow cuts is + # annotated; every tag cut by hand is lightweight, and at the time of writing all 312 + # tags in this repository are lightweight -- so reading only one form finds neither + # reliably. ls-remote exits 0 with no output when nothing matches, so an empty answer + # means "no such tag" while a genuine transport failure dies here on pipefail. + PEELED=$(git ls-remote --tags origin "refs/tags/v${VERSION}^{}" | cut -f1) + DIRECT=$(git ls-remote --tags origin "refs/tags/v${VERSION}" | cut -f1) + EXISTING=${PEELED:-$DIRECT} + if [ -z "$EXISTING" ]; then + echo "done=false" >> "$GITHUB_OUTPUT" + elif [ "$EXISTING" = "$GITHUB_SHA" ]; then + echo "tag v${VERSION} already points at this commit -- re-run, nothing to do" + echo "done=true" >> "$GITHUB_OUTPUT" + else + echo "tag v${VERSION} already exists at ${EXISTING}, which is not this commit (${GITHUB_SHA})" + echo "resolve the tag before re-running: this release has not been verified against the signed document" + exit 1 + fi + + # The merge tree normally equals the PR head tree the gate already saw signed, so this + # passes on the first poll. The loop covers the divergent case: the on-push publish and the + # signing run pick the new tree up within minutes, and this waits for them rather than + # failing a release over a race. + - name: Merged tree is in the signed document central serves + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + run: | + set -eo pipefail + NEW_HASH=$(find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}') + # d41d8... is the md5 of an empty stream, produced whenever nothing was hashed: ZelBack + # absent (pipefail catches that) or present holding no regular files (find exits 0 and + # emits nothing, which pipefail cannot see). The signer refuses to list that value, so + # the membership check below would fail anyway -- but it would fail as "not in the signed + # document", sending the reader to the signing chain instead of the missing directory. + if [ "$NEW_HASH" = d41d8cd98f00b204e9800998ecf8427e ]; then + echo 'nothing was hashed -- this checkout has no ZelBack files' + exit 1 + fi + echo "merged tree hash: ${NEW_HASH}" + # exported for the node script below; GITHUB_ENV only reaches subsequent steps + export NEW_HASH + echo "NEW_HASH=${NEW_HASH}" >> "$GITHUB_ENV" + + for attempt in $(seq 1 30); do + curl -sS -m 30 -o hashlist-signed.json https://hashes.runonflux.io/hashlist || true + if node -e ' + const crypto = require("crypto"); + const fs = require("fs"); + const PINNED_PUBLIC_KEYS = [ + "14837066068b258bfbd0749702056f7065361af44aed48761834744391cbbaaa", + "fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8", + ]; + const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); + const document = JSON.parse(fs.readFileSync("hashlist-signed.json", "utf8")); + const payload = Buffer.from(document.payload_b64, "base64"); + const signature = Buffer.from(document.sig_b64, "base64"); + const verified = PINNED_PUBLIC_KEYS.some((hex) => crypto.verify( + null, + payload, + crypto.createPublicKey({ + key: Buffer.concat([SPKI_ED25519_PREFIX, Buffer.from(hex, "hex")]), + format: "der", + type: "spki", + }), + signature, + )); + if (!verified) throw new Error("not a validly signed document"); + const { seq, hashes } = JSON.parse(payload.toString("utf8")); + if (!hashes.includes(process.env.NEW_HASH)) throw new Error(`not in seq ${seq}`); + console.log(`signed document seq ${seq} carries the merged tree hash`); + ' 2> poll-error.txt; then + exit 0 + fi + # the message line, not the 15-line trace, times thirty attempts. Node prints the + # file marker first, so the first line is never the message; head -1 is the fallback + # for a failure that is not a Node error at all, so the log cannot go silent. + echo "attempt ${attempt}: $(grep -m1 -E '^[A-Za-z]*Error: ' poll-error.txt || head -1 poll-error.txt) -- retrying in 30s" + sleep 30 + done + + echo "the signed document never covered the merged tree; no tag is cut" + exit 1 + + # Advisory until the fleet serves /flux/hashlist: results are logged, never fatal. Once the + # FluxOS side is deployed, enforcement is turning the failure at the end into exit 1. + - name: Live-node sample (advisory) + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + run: | + curl -sS -m 60 -o nodelist.json https://api.runonflux.io/daemon/viewdeterministiczelnodelist || { echo "ADVISORY: node list unavailable"; exit 0; } + jq -r '.data[].ip | select(. != null and . != "")' nodelist.json | shuf -n 10 > sample.txt || true + SERVING=0 + while read -r ADDR; do + HOST=${ADDR%:*}; PORT=${ADDR##*:}; [ "$HOST" = "$PORT" ] && PORT=16127 + BODY=$(curl -sS -m 5 "http://${HOST}:${PORT}/flux/hashlist" 2>/dev/null || true) + PAYLOAD=$(printf '%s' "$BODY" | jq -r 'try (.payload_b64 // .data.payload_b64) // empty' 2>/dev/null || true) + if [ -n "$PAYLOAD" ] && printf '%s' "$PAYLOAD" | base64 -d 2>/dev/null | grep -q "$NEW_HASH"; then + SERVING=$((SERVING + 1)) + fi + done < sample.txt + echo "ADVISORY: $(wc -l < sample.txt | tr -d ' ') nodes sampled, ${SERVING} serving a document with the merged tree hash" + + - name: Cut the tag and the GitHub Release + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ steps.version.outputs.version }} + run: | + git config user.email runonfluxbot@gmail.com + git config user.name runonfluxbot + git tag -a "v${VERSION}" -m "FluxOS v${VERSION}" + git push --quiet origin "refs/tags/v${VERSION}" + gh release create "v${VERSION}" --title "FluxOS v${VERSION}" --generate-notes --verify-tag + echo "released v${VERSION}" + + # The tag above was pushed with GITHUB_TOKEN, which triggers no workflows -- so the signer + # would only pick it up on its next run's ls-remote delta. This dispatch makes the + # annotation immediate; the signer resolves the tag against its own view of this repository + # and annotates the provenance row itself. Nothing here writes to fluxhashes. + - name: Mint the dispatch token + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + id: dispatch-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.FLUXHASHES_APP_ID }} + private-key: ${{ secrets.FLUXHASHES_APP_KEY }} + owner: RunOnFlux + repositories: fluxhashes + - name: Request the tag annotation + if: steps.version.outputs.release == 'true' && steps.existing.outputs.done == 'false' + env: + GH_TOKEN: ${{ steps.dispatch-token.outputs.token }} + VERSION: ${{ steps.version.outputs.version }} + run: | + gh api -X POST repos/RunOnFlux/fluxhashes/actions/workflows/sign-hashlist.yml/dispatches \ + -f ref=master \ + -f "inputs[commit]=${GITHUB_SHA}" \ + -f "inputs[ref]=v${VERSION}" \ + -f "inputs[ref_type]=tag" \ + -f "inputs[claimed_hash]=${NEW_HASH}" diff --git a/.gitignore b/.gitignore index 404add61dc..06bf0ab1b0 100644 --- a/.gitignore +++ b/.gitignore @@ -381,6 +381,20 @@ CloudUI/ # compiled test-app fixture binary (built via test-infra/test-app/build.sh) test-infra/test-app/test-app +# and the source hash it is keyed on, so a rebuild is skipped only when the +# binary really was built from the test-app.c currently checked out +test-infra/test-app/test-app.sha256 # per-node logs written by the integration harness on test failure test-infra/runner/test-logs/ + +# published image blobs mirrored into the harness registry (registry-helper +# mirrorImage). Content-addressed and re-fetchable, so only the first suite of +# a gate pays for the download. +test-infra/runner/.image-cache/ + +# node-config loads this last, over everything in the pinned directory, so a +# committed one would redirect a production node's endpoints. The harness writes +# it inside the container at boot; it never belongs in the repo. +ZelBack/config/local.js +ZelBack/config/local-*.js diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000000..668bd1d23b --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,112 @@ +# Releasing FluxOS + +A release is a reviewed pull request from `development` into `master`. That has not changed, and +neither has the review or the merge button. + +What has changed is that a release now has to prove, before it merges, that its code is on the +signed list of approved FluxOS trees — and that the tag and the GitHub Release are cut for you +afterwards instead of by hand. + +## Why there is a list at all + +Every node checks its own code against a published list of approved fingerprints. A fingerprint is +an md5 over the `ZelBack` directory and nothing else — not `helpers/`, not `package.json`, not the +workflows. + +A robot in [RunOnFlux/fluxhashes](https://github.com/RunOnFlux/fluxhashes) builds that list. Every +time a branch or tag in this repository moves, it fingerprints the tree at that branch's tip, adds +it to the list, and signs the result. It publishes to `hashes.runonflux.io`. + +That is the whole reason releasing has a gate: the fleet follows `master`, so nothing should reach +`master` whose fingerprint is not already published and signed. + +## Cutting a release + +**1. Bump the version on `development`.** `package.json`, as before. + +**2. Add the fingerprint to `helpers/hashes.json` on `development`.** This is the step that used to +happen *after* the merge, as a commit straight to `master`. It now rides the release PR. + +Compute it from a clean checkout of `development`: + +```sh +find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}' +``` + +Adding it to `helpers/hashes.json` does not change the fingerprint, because the fingerprint only +covers `ZelBack`. So there is no chicken-and-egg — compute it, commit it, and it is still correct. + +(`helpers/hashes.json` is the fallback list the older benchmark reads when it cannot reach central. +This step retires when that version of the benchmark does.) + +**3. Open the PR, `development` → `master`, and get it reviewed.** As before. + +**4. Merge it.** As before. + +## What the checks do + +Two workflows. Neither of them can merge anything or push anything; the gate is read-only. + +**`release-gate`** runs on the pull request and judges the *merge preview* — what `master` will +actually hold once the PR merges, not the PR branch alone. It checks four things: + +| check | what it means | +|---|---| +| the version moves forward | `package.json` is strictly newer than `master`'s, compared numerically | +| the merged `ZelBack` is a tree the signer can have seen | `master` holds no `ZelBack` changes that `development` is missing | +| the fingerprint is in the signed document | central is serving a validly signed list, and the merged tree's fingerprint is in it | +| the fallback list carries it | `helpers/hashes.json` has the entry from step 2 | + +**`release-tag`** runs after the merge, on `master`. It re-verifies the merged tree against the +signed document — this is the authoritative check, because it judges what `master` actually holds — +then cuts the tag and the GitHub Release, and asks the robot to record the tag against the +fingerprint. + +A push to `master` that does not change the version is not a release, and `release-tag` does +nothing. + +## When something goes red + +**"the candidate version does not move past master"** — bump `package.json` in the release PR. + +**"master carries ZelBack changes that are not on development"** — someone committed code straight +to `master`. The merged tree then exists on no branch, so the robot has never fingerprinted it and +never will. Merge `master` into `development` and re-run. Waiting will not help. + +**"the tree hash is not in the signed document"** — the robot has not caught up with +`development`'s tip yet. It reconciles on every push and on a daily sweep, so this normally clears +within minutes. Re-run the check. + +**"central is not serving a validly signed hash list"** — `hashes.runonflux.io` is not answering +with a signed document. That is an infrastructure problem, not a problem with your release. The +gate is deliberately strict here: a gate that passes when its source is missing is not a gate. + +**"helpers/hashes.json does not contain …"** — step 2 was missed. + +**`release-tag` fails after the merge** — the release is on `master` and the fleet has it, but no +tag was cut. Nothing is half-made: `master` holds the tree the gate already verified. Read the run +to see which check failed before deciding whether to re-run it or cut the tag by hand. + +**`release-tag` says a tag already exists at another commit** — a tag with this version number was +cut by hand, or `master` was rolled back behind one. Resolve the tag, then re-run. The workflow +refuses rather than skipping, because skipping would mean the merged tree was never verified. + +## Things that are no longer done by hand + +- Adding the fingerprint to `helpers/hashes.json` after the merge. It rides the PR now. +- Creating the `vX.Y.Z` tag. +- Creating the GitHub Release. + +## Please do not commit to `master` + +Every commit that lands on `master` outside a release PR makes `master` and `development` diverge. +Commits that only touch `helpers/` or `package.json` are harmless to the gate, since the +fingerprint only covers `ZelBack` — but a commit that touches `ZelBack` produces a merged tree that +exists on no branch, which nothing can ever sign, and blocks the next release until `master` is +merged back into `development`. + +## Tests + +The steps in both workflows are covered by `tests/ci/release-workflows.sh`. It runs offline with no +secrets, extracts the steps from the workflow files verbatim rather than copying them, and +mutation-tests its own assertions. Run it after changing either workflow. diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index 41134cc471..d91637e685 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -1,5 +1,6 @@ // eslint-disable-next-line prefer-const let userconfig = require('../../config/userconfig'); +const volumeToolsImage = require('./volumeToolsImage.json'); const isDevelopment = userconfig.initial.development || false; @@ -44,6 +45,9 @@ module.exports = { benchmark: 'benchmark', appTamperingEvents: 'apptamperingevents', nodeStartupTracker: 'nodestartuptracker', + policyDocuments: 'policydocuments', // last-known-good network policy documents, so an unreachable source does not drop enforcement + ipRanges: 'ipranges', // the IP location baseline, one document per allocated range, rebuilt and swapped in whole + nodeLocations: 'nodelocations', // per-node view derived from the baseline, invalidated when a new baseline lands }, }, daemon: { @@ -119,7 +123,15 @@ module.exports = { minimumSyncthingAllowedVersion: '2.0.10', minimumDockerAllowedVersion: '26.1.2', fluxTeamFluxID: '1hjy4bCYBJr4mny4zCE85J94RXa8W6q37', - fluxSupportTeamFluxID: '16iJqiVbHptCx87q6XQwNpKdgEZnFtKcyP', + // A list, so support can be granted to (or revoked from) an identity without + // touching every privilege check. A bare string is still read as a one entry + // list, so a node carrying an older local override keeps working. + fluxSupportTeamFluxID: [ + '16dNCFf7nR3nx5iwn2RQMBw6KcJXkE3JC1', + '15c3aH6y9Koq1Dg1rGXE9Ypn5nL2AbSJCu', + '1NGqYirE4T9wzd1ZcGrw3HjETiuCkt6Sgy', + '13BBPcpHxwCaC61vjQgK6qeDcprFJEGVkP', + ], deterministicNodesStart: 558000, messagesBroadcastRefactorStart: 1751250, // expected block at 13th Octobor 2024 fluxapps: { @@ -128,6 +140,134 @@ module.exports = { // and the run length that counts as stable (resets the ladder) crashBackoffDelaysMs: [0, 30000, 300000, 900000, 1800000], crashBackoffStableRunMs: 600000, + // Backstop for images whose entrypoint discards the payload's exit status. + // A clean exit proves nothing, so for those images restart RATE is the only + // fault evidence left and this is the only thing that ever paces them. + // This many automatic restarts inside the window is treated as a crash and + // enters the ladder above, which reaches anything restarting closer together + // than window/count - 60s apart at these values. Slower is deliberately left + // alone: Palworld's segfault-restart cycle is ~77s at its worst, and coming + // straight back is better for the customer than being paced. + // Keep the window wider than the reconciler's retry interval times this + // count. A container that fails to START never ran, so it is never a fault + // and never walks the ladder directly - it reaches it only by filling this + // window, and a window narrower than that retries forever. + // Counted as restarts ALREADY RECORDED, so at 5 the sixth restart is the one + // that earns a rung and the seventh is the first one held back. + restartBurstCount: 5, + restartBurstWindowMs: 300000, + // How long a finished operation stays readable at /apps/operations/:jobId, + // and how long a client is told to wait between polls while one runs. A + // RUNNING job never expires - only terminal ones are retained on a clock. + operationRetentionMs: 60 * 60 * 1000, + operationRetryAfterSeconds: 2, + // File operations on an app's volume, each run in a throwaway container. + volumeOperations: { + // The tag is the NAME and the id is the PROOF, and they rotate together. + // + // In their own file because the harness reads them too, and reads them + // from here rather than repeating them: a rebuilt image must not leave + // the harness testing something the fleet does not run. It cannot + // `require` this config to get at them - line 2 pulls in userconfig.js, + // which is gitignored and absent from a fresh checkout - so it used to + // match them out of this source with a regular expression, and changing + // the shape of a pin broke the runner rather than the thing under test. + // + // A tag alone decides nothing: it is mutable at the registry, and one + // inside an image a peer hands over is whatever that peer wrote in it. So + // what a node runs is decided by the image id - the digest of the image's + // own config - which is checked on every path, whether the image arrived + // from the registry, from a peer, or was already here. An id is per + // architecture, hence one for each. + // + // ROTATING THIS MEANS CHANGING BOTH, which is why they sit together. An + // id belongs to a specific build: the same commit rebuilt under a new tag + // carries different labels and therefore a different id. Read them from + // the release that published the tag, never from a previous one. A tag + // moved without its ids is refused by every node, loudly, which is the + // right direction to fail in but is not something to discover during a + // rollout. + // + // It also means rotating the image needs a FluxOS release, which is + // deliberate rather than a limitation to work around. What the image does + // is coupled to the code that drives it - the staging names it creates are the + // ones swept here, so a change to one is a change to both - and the + // alternative, + // publishing the pin where the fleet reads policy, would let a merge + // choose the program every node runs as root over an app's volume, with + // no staged rollout. The urgency that would buy is small: the container + // has no network, a read-only rootfs, every capability dropped but three, + // and one volume mounted, so a CVE in what it packages is not reachable + // the way one in a network-facing service is. + // + // What the image DOES is proven in its own repository, not here: the + // ceiling, the link refusal, discarding staging, the atomic exchange the + // publish is made of, and the signal handling all have tests there + // that run in a container configured exactly as this one configures it, + // on both architectures. Nothing in this repository can exercise them, + // and a reviewer looking only here should not conclude they are + // unexercised. + ...volumeToolsImage, + // One per app stops a single owner monopolising a node; the node-wide cap + // stops the disk being saturated by several at once. A reached limit is + // refused rather than queued - a queued request waits silently behind + // someone else's long copy until an intermediate proxy kills it. + // How widely the fleet's registry fetch is spread. Only the registry is + // spread: it is the one place every node reaches at once, where asking + // peers costs the fleet nothing it does not already have. Configurable so + // a test fleet can watch a window it would otherwise sit inside of. + prefetchWindowMs: 6 * 60 * 60 * 1000, + maxConcurrentPerApp: 1, + maxConcurrentPerNode: 4, + // How long an operation may make NO progress before it is stopped. Not a + // limit on how long it may run: moving a hundred gigabytes legitimately + // outruns any wall clock short enough to be useful, and a fixed ceiling + // cannot tell that from a wedged container. The volume's own usage is + // read every tick anyway, so "has this written or deleted anything at + // all recently" is free and is the question actually worth asking. + // Generous, because a slow disk under load is not a stuck one. + stallTimeoutMs: 10 * 60 * 1000, + // The floor an upload has to keep to count as still sending. Bytes from + // the caller are the only evidence a slow upload is alive - it moves no + // whole filesystem block for minutes, so the volume reads as idle - but + // the evidence has to be a RATE. Treating any byte at all as progress + // lets one byte per window hold a slot until the request itself times + // out two hours later, and four of those block every file operation on + // the node for every app on it. + // + // Set where a caller below it could not finish anyway: 64 kbit/s carries + // ~58MB in the two hours server.requestTimeout allows, so this mostly + // writes down a limit that already exists. It clears the worst usable + // mobile link by a wide margin and sits thousands of times above the + // trickle it is here to stop. + minUploadBitsPerSecond: 64 * 1000, + // Bounds a runaway archive. How much can be WRITTEN is already capped by + // the size of the volume itself. + memoryBytes: 512 * 1024 * 1024, + pidsLimit: 256, + // One core per operation. tar and zip are single-threaded, so this mostly + // writes down what they already use - what it bounds is the tool that is + // not: anything in the image that spawns workers has pidsLimit's worth of + // processes to do it with, and without a quota one operation takes every + // core the node has. The worst case across the pool is + // maxConcurrentPerNode cores, and contention inside it is settled by + // CpuShares in the executor's HostConfig: file operations yield to the + // applications, which are the tenants the node is for. + cpuCores: 1, + // How long a cancelled operation is given to stop of its own accord. The + // container is sent SIGTERM, which flux-op traps to stop the command and + // reclaim its staging directory; only after this does docker escalate to + // SIGKILL, which reaches neither - the executor's own deferred reclaim + // then removes what was staged. Long enough to remove a large staging + // tree, short enough that a cancel still feels like one. + cancelGraceSeconds: 15, + // How often a running operation is looked at: one tick reports that it is + // alive, notices a cancellation, and reads how far it has got. Nothing is + // holding a request open to receive any of it - the endpoints answered 202 + // before the work began - so this is the resolution of a poll, not a + // keepalive. + progressIntervalMs: 2000, + }, // in flux main chain per month (blocksLasting) price: [ { // any price fork can be done by adjusting object similarily. @@ -207,9 +347,30 @@ module.exports = { multiplier: 1, // multiplier in case we want to increase prices globaly minUSDPrice: 0.99, // min. usd price that can be paid with stripe/paypal. }, + // Who the support team is, from which height. Only the latest fork at or below + // a message's own block is consulted, so entries are only ever appended: an + // edit to one below the tip changes which signatures the past accepts. teamSupportAddress: [{ height: 1851659, // height from which address is valid address: '16iJqiVbHptCx87q6XQwNpKdgEZnFtKcyP', + }, { + // From here a fork names a list rather than one address. The address above + // is carried forward - a fork replaces its predecessor rather than adding + // to it, so leaving it out would stop it signing. + // + // Dated about a week ahead of the tip it was written at (2931010, 7th + // September 2026, ~30s blocks) rather than at it: a node on an older FluxOS + // reads only the fork at 1851659 and rejects a message signed by any of the + // new addresses, so the fork has to fall after the network has had time to + // update. + height: 2951000, // ~14th September 2026 + addresses: [ + '16iJqiVbHptCx87q6XQwNpKdgEZnFtKcyP', + '16dNCFf7nR3nx5iwn2RQMBw6KcJXkE3JC1', + '15c3aH6y9Koq1Dg1rGXE9Ypn5nL2AbSJCu', + '1NGqYirE4T9wzd1ZcGrw3HjETiuCkt6Sgy', + '13BBPcpHxwCaC61vjQgK6qeDcprFJEGVkP', + ], }], usersToExtend: ['1MCBJn6qsy3YRY2YasdYMYdJcdhy1ev8Rd'], // addresses that can extend applications on behalf of app owners (expire-only updates) addresses cannot be deleted over time, just adding new ones // restartAlwaysOwners removed — all containers use restart policy 'no', FluxOS manages startup @@ -252,8 +413,21 @@ module.exports = { minUpTime: 1800, // 30 mins appSyncPeerThreshold: 12, // peers needed before starting app sync / spawning appSyncDegradedThreshold: 4, // below this, pause spawner — gossip unreliable - appSyncMinPeerUptime: 7500, // seconds a peer must have been running before we sync from it appSyncMinCompletions: 3, // sync responses needed per type before spawner can start + // Applies ONLY to peers whose build cannot refuse a sync request - one that + // can is asked whatever its uptime, because it answers for itself. Retires + // with the last such build. + appSyncMinPeerUptime: 7500, + // How long a node waits for a state sync before deciding that what it has + // is what it gets. Both roads to readiness, and to answering another node's + // sync request, so a node with a 0 here is authoritative from the moment it + // starts - which is how a fleet gets a peer that can answer at all. + // + // 125 minutes is locationTtlS below, in minutes: one full lifetime of a + // running-app location record, so every holder has had to announce itself + // at least once. That is what makes waiting it out equivalent to a view, + // and it is why there is no shorter variant of it for anyone. + appSyncFallbackMinutes: 125, installation: { probability: 100, // 1% delay: 120, // in seconds @@ -278,7 +452,7 @@ module.exports = { daemonPONFork: 2020000, // block height where PON (Proof of Node) fork activates - chain works 4x faster after this block blocksAllowanceInterval: 1000, // ap differences can be in 1000s - more than 1 day removeBlocksAllowanceIntervalBlock: 1625000, // after this block we can start having app updates without extending subscription - block expected in April 19th 2024 - ownerAppAllowance: 1000, // in case of node owner installing some app, the app will run for this amount of blocks + ownerAppAllowance: 1000, // a by-name local install (fluxteam only) runs for this amount of blocks before the expiry sweep removes it temporaryAppAllowance: 200, // in case of any user installing some temporary app message for testing purposes, the app will run for this many blocks expireFluxAppsPeriod: 100, // every 100 blocks we run a check that deletes apps specifications and stops/removes the application from existence if it has been lastly updated more than 22k blocks ago updateFluxAppsPeriod: 9, // every 9 blocks we check for reinstalling of old application versions @@ -293,13 +467,31 @@ module.exports = { bootDelayMultiplier: 1, spawnDelayMs: 0, removalSpacingMs: 60000, + // Per-document expiry for the ephemeral app collections, in seconds, read by + // appConstants.js. Each record carries its own deadline and the collection + // index is expireAt/expireAfterSeconds:0, so changing one of these takes + // effect on records written after it, not on the ones already stored. + // A running-app location record: 125 minutes. It also SETS how often a node + // announces - appConstants derives the interval from it, two announcements + // to a lifetime with slack, so the pair cannot drift. There is no separate + // announce interval to keep in step with this number. locationTtlS: 7500, - installingTtlS: 900, - installErrorTtlS: 3600, - tempMsgTtlS: 3600, + // Grace after a node announces its own shutdown, before peers drop its + // locations. MUST stay below locationTtlS: appStartupManager expires on + // `(cleanShutdown && downtime > sigterm) || downtime > running`, so a value + // above the running expiry makes this window unreachable and a clean + // shutdown gets no grace at all. + sigtermExpiryS: 420, + installingTtlS: 900, // an in-progress install: 15 minutes + // 24 hours. This was 3600 while a collection-level TTL index on `cachedAt` + // drove it; that index was dropped when expiry moved per-document, and the + // key kept the old mechanism's number for three months while nothing read + // it. The 24h the code has actually run since is the value. + installErrorTtlS: 86400, + tempMsgTtlS: 3600, // collection-level index, serviceManager.js hashSyncIntervalMs: 1800000, - peerNotifyIntervalMs: 3600000, cpuCheckIntervalMs: 900000, + statsSampleIntervalMs: 60000, portRestoreIntervalMs: 600000, imageComplianceIntervalMs: 3600000, forceRemovalIntervalMs: 7200000, @@ -308,9 +500,23 @@ module.exports = { portTestPropagationDelayMs: 10000, portTestPeerTimeoutMs: 30000, portTestMaxAttempts: 5, + // Asking the other Flux nodes at our own public address which ports they + // hold. Short: they are one hop away, and a sibling that does not answer + // promptly is left unasked rather than delaying an install - the port test + // that follows is what decides. + siblingPortsTimeoutMs: 5000, + // How long a signed sibling ask stays good for. The exchange itself is + // bounded by siblingPortsTimeoutMs; the rest is allowance for two nodes + // that were never required to agree on the time. + siblingAskValidityMs: 60000, spawnReconfirmDelayMs: 7500000, nonEnterpriseSpawnDelayMs: 120000, globalCmdDelayMs: 500, + // How many times a global command retries a node that answers 503 while it + // is still reconciling its apps after boot. The refusal carries a 15s + // Retry-After, so this is ~2 minutes of coverage - long enough for a + // booting node to settle, bounded so a genuinely wedged one is not hammered. + globalCmdBootRetries: 8, discoveryAutostart: true, discoveryRetryMs: 60000, discoveryFailRetryMs: 120000, @@ -318,9 +524,29 @@ module.exports = { connectionBackoffMs: [120000, 300000, 600000, 900000], nodeMonitorIntervalMs: 1200000, nodeMonitorRemovalDelayMs: 60000, + // Residential-node staging. The placement hold is immediate and is not + // tunable; these pace only the part that moves customer data. + residentialCheckIntervalMs: 6 * 60 * 60 * 1000, // re-evaluate the verdict + residentialSettleMs: 24 * 60 * 60 * 1000, // verdict must hold before any app moves + residentialEvacuationIntervalMs: 6 * 60 * 60 * 1000, // minimum gap between departures + residentialQueueBaseMs: 30 * 60 * 1000, // every node waits at least this + // Per position in the instance order, and it MUST stay longer than the pass + // that reads it. mayEvacuateApp is reached only from the give-up pass at + // explorerService.js:651, which runs every removeFluxAppsPeriod (11) x + // speedMultiplier (4 post-PON) = 44 blocks = 22 minutes at 30s blocks, and + // wholeSince is stamped inside that pass - so maturity is quantised to a + // 22-minute grid and a shorter step cannot separate two points on it. + // Adjacent positions would mature on the same pass, and the pass is keyed on + // block height so every node evaluates in the same instant. Both holders + // then read the app at full strength, because fluxappremoved is broadcast + // after the volume is already deleted. 40 minutes is 1.8x the pass, so the + // chain would have to slow to ~55s blocks before adjacent positions could + // meet. Asserted against production's own config in the unit tests. + residentialQueueStepMs: 40 * 60 * 1000, nodeMonitorDosRecoveryDelayMs: 600000, nodeMonitorConfirmationLossDelayMs: 1200000, nodeMonitorErrorRecoveryDelayMs: 120000, + nodeMonitorCheckIntervalMs: 120000, nodeMonitorCheckTimeoutMs: 10000, spawnDeferrals: { targetedNodesMs: { enterprise: 1800000, standard: 3420000 }, @@ -334,6 +560,13 @@ module.exports = { }, spawnDelayMultiplier: 1, daemonInfoIntervalMs: 30000, + // NOT how often the chain is asked. pollForNewBlocks reads a height cached + // by daemonServiceMiscRpcs and refreshed on the daemonInfoIntervalMs timer + // above, so this is the rate at which the node works THROUGH blocks once it + // knows it is behind. Its share of that refresh window - 5000/30000, 16.7% - + // is what decides whether a block is still the tip when it is processed, + // and everything hung off block processing inherits that. + explorerPollIntervalMs: 5000, explorerSyncRetryMs: 120000, explorerDeepRestoreBlocks: 100, syncTimeoutMs: 120000, @@ -355,6 +588,7 @@ module.exports = { imageUpdateDelayAfterRedeployMs: 120000, imageUpdateDelayBetweenComponentsMs: 1000, masterSlaveIntervalMs: 30000, // masterSlave (g:) FDM election cycle + masterSlaveStaggerMs: 180000, // per-place wait before an instance may take an empty g: primary }, lockedSystemResources: { cpu: 10, // 1 cpu core @@ -398,6 +632,10 @@ module.exports = { stallNudgeMaxIntervalMs: 900000, // nudge backoff cap (15min) stallRemoveMinWindowMs: 1200000, // 20min minimum evidence window before removal stallRemoveMinNudges: 3, // nudges that must have failed before removal + // Where a legacy node installs syncthing from. Arcane nodes ship it in the image and + // never reach either of these. + aptSourceUrl: 'https://apt.syncthing.net/', + releaseKeyUrl: 'https://syncthing.net/release-key.gpg', }, // enterpriseAppOwners moved to helpers/enterprisenodes.json (synced from github every 6h, see enterpriseConfig) enterprisePublicKeys: [ // list of whitelisted nodes indentity public keys. Most trusted node operators that are publicly known, kyc. Eg Flux team members, Titan. @@ -446,9 +684,32 @@ module.exports = { rawBaseUrl: 'https://raw.githubusercontent.com/RunOnFlux/flux/master', apiBaseUrl: 'https://api.github.com', }, + policy: { + // The directory holding the network's enforcement documents, fetched at runtime by + // policyStore. A repo of its own, so a merge to the application cannot change fleet + // policy as a side effect and a policy change is not a commit to the application's + // default branch. Releases predating this still read RunOnFlux/flux helpers/, so both + // copies are kept in step until minimumFluxOSAllowedVersion is above all of them. + baseUrl: 'https://raw.githubusercontent.com/RunOnFlux/fluxos-network-policy/main', + }, geolocation: { ipApiBaseUrl: 'http://ip-api.com', - statsApiBaseUrl: 'https://stats.runonflux.io', + }, + // The network's statistics service. One host, several paths: node location, + // marketplace listings, app USD pricing, and the minimum module versions a node + // checks its syncthing against at boot. + stats: { + baseUrl: 'https://stats.runonflux.io', + }, + pricing: { + fluxRatesBaseUrl: 'https://viprates.runonflux.io', + // Consulted only when the rates service above is unreachable. + coingeckoBaseUrl: 'https://api.coingecko.com', + }, + mongodb: { + // Where a replacement server signing key is fetched from when the installed one + // has expired. The version is appended: /server-.asc + signingKeyBaseUrl: 'https://pgp.mongodb.com', }, analytics: { url: 'https://cloudaudit.runonflux.io', // analytics server URL (e.g. 'https://analytics.runonflux.io'). Empty = disabled. diff --git a/ZelBack/config/volumeToolsImage.json b/ZelBack/config/volumeToolsImage.json new file mode 100644 index 0000000000..3a3530dc84 --- /dev/null +++ b/ZelBack/config/volumeToolsImage.json @@ -0,0 +1,8 @@ +{ + "image": "ghcr.io/runonflux/flux-volume-tools:v1.4.2", + "indexId": "sha256:cae8a45cf19961e23a618e015ab6fd029f9fd5ec654db92ec60b9dce02ee5fcf", + "imageIds": { + "amd64": "sha256:edf11fd3f9debc0236ab2409963e6a44c18d96e8d65ed5eefaaacbd21c1dd14c", + "arm64": "sha256:041f51a0fbc2fba276fac16f7ba584c2697aaa9d8ac448cd8eaad53951b3c6f3" + } +} diff --git a/ZelBack/pinEnvironment.js b/ZelBack/pinEnvironment.js new file mode 100644 index 0000000000..2e93fc2609 --- /dev/null +++ b/ZelBack/pinEnvironment.js @@ -0,0 +1,28 @@ +// What the process settles about itself before any library reads the environment. +// +// Required as the FIRST line of every entry point. node-config, express and apicache each +// read these as they load, so a require placed above this one that reaches any of them +// answers the question first and the pins below arrive too late to matter. +// +// The four belong together. NODE_ENV and NODE_CONFIG_ENV in particular are a pair: pinning +// NODE_ENV alone sends node-config looking for a deployment file that does not exist. + +process.env.NODE_CONFIG_DIR = `${__dirname}/config/`; +// The directory is pinned above so config loads from the one that ships with the node. +// NODE_CONFIG is the same door: the config package merges whatever JSON it holds over every +// file, after the directory is settled, so leaving it open redirects any endpoint without +// editing a single file. Deleted rather than emptied, because an empty value is parsed and +// fails rather than being ignored. +delete process.env.NODE_CONFIG; + +// The same door, one variable over. Express hands a caller the exception stack instead of +// the status text unless this says production, and apicache stamps its version onto every +// cached response. Assigned rather than defaulted, because a value read from the +// environment changes how the node answers without any file saying so. +process.env.NODE_ENV = 'production'; + +// node-config names its deployment from the environment too, preferring this variable to +// NODE_ENV. Pinned to the deployment it already resolves to, so the line above governs +// express and apicache alone: the config directory holds one file, default.js, and +// node-config warns on every start for a deployment name it cannot find a file for. +process.env.NODE_CONFIG_ENV = 'development'; diff --git a/ZelBack/src/lib/fluxServer.js b/ZelBack/src/lib/fluxServer.js index 8dc307e933..c3872f785c 100644 --- a/ZelBack/src/lib/fluxServer.js +++ b/ZelBack/src/lib/fluxServer.js @@ -8,7 +8,7 @@ const compression = require('compression'); const routes = require('../routes'); const { analyticsMiddleware, startFlushTimer } = require('../services/analyticsService'); -const socketHandlers = require('./socketHandlers'); +const { socketHandlers, admitUpgrade } = require('./socketHandlers'); const socketIoHandlers = require('./socketIoHandlers'); const { FluxWebsocketServer } = require('./socketServer'); const { FluxSocketIoServer } = require('./socketIoServer'); @@ -90,6 +90,7 @@ class FluxServer { this.socketServer = new FluxWebsocketServer({ routes: socketHandlers, + admit: admitUpgrade, errorHandler: this.errorHandler, }); diff --git a/ZelBack/src/lib/socketHandlers.js b/ZelBack/src/lib/socketHandlers.js index 340bf6707d..c3d4e76d85 100644 --- a/ZelBack/src/lib/socketHandlers.js +++ b/ZelBack/src/lib/socketHandlers.js @@ -11,4 +11,30 @@ const socketHandlers = { '/ws/payment/:paymentid': paymentService.wsRespondPayment, }; -module.exports = socketHandlers; +const FLUX_PEER_ROUTE = /^\/ws\/flux(\/|$)/; + +/** + * Whether a websocket upgrade may complete, decided before the handshake does. + * + * A node that is not yet accepting peer connections refuses here rather than + * after the 101. Its HTTP server answers well before its application gate + * opens, and its capabilities ride in the upgrade response headers, so a + * completed handshake is enough for the dialling node to construct a peer, + * count it toward its thresholds and write to it - a boot's state-sync requests + * go into that socket and are lost when this side closes it. The dial fails + * cleanly instead - no peer is constructed, so nothing is queued for reconnect + * and nothing counts it as a lost peer; the next discovery pass dials again. + * + * Only the peer routes. Browsers reach /ws/id, /ws/sign and /ws/payment, and + * those have nothing to do with whether this node has peers yet. + * @param {import('node:http').IncomingMessage} request + * @returns {{status: number, message: string, reason: string}|null} null to admit. + */ +function admitUpgrade(request) { + if (!FLUX_PEER_ROUTE.test(request?.url ?? '')) return null; + if (peerManager.acceptingConnections) return null; + + return { status: 503, message: 'Service Unavailable', reason: 'node-not-accepting-connections' }; +} + +module.exports = { socketHandlers, admitUpgrade }; diff --git a/ZelBack/src/lib/socketIoHandlers/appLogsHandler.js b/ZelBack/src/lib/socketIoHandlers/appLogsHandler.js new file mode 100644 index 0000000000..99089eac63 --- /dev/null +++ b/ZelBack/src/lib/socketIoHandlers/appLogsHandler.js @@ -0,0 +1,589 @@ +const verificationHelper = require('../../services/verificationHelper'); +const { Privilege } = require('../../services/utils/privileges'); +const dockerService = require('../../services/dockerService'); +const LogFrameDecoder = require('../../services/utils/logFrameDecoder'); + +const log = require('../log'); + +/** + * How long lines are collected before they are sent as one message. + * + * A line at a time is what makes a stream cost more than the poll it replaces: + * a container writing 1,000 lines a second would be 1,000 socket.io messages a + * second per node, where a three-second poll answered the same 3,000 lines in + * one bounded read. Collecting them puts the message rate under our control and + * leaves it flat however loud the container is. Loki's tail endpoint carries the + * same idea as `delay_for`. + */ +const BATCH_MS = 250; + +/** + * The most lines held for one container between two flushes. + * + * Reached only by a container writing faster than the socket drains, and the + * answer then is to drop and say so rather than to buffer without limit - an + * unbounded queue turns a loud container into the node's memory problem. What + * was dropped is counted and reported, never passed over in silence. + */ +const MAX_QUEUED_LINES = 20000; + +/** + * Lines sent to a subscriber before the live ones, so a viewer opens with + * context instead of an empty pane until the container next writes. + */ +const BACKFILL_LINES = 200; + +/** + * How many containers one connection may follow. + * + * Ten, because an app is capped at ten components (`appValidator.js:679`), so + * this is a viewer following every container of the largest app it can be + * looking at. + * + * It bounds a connection's own bookkeeping, not the node's work. The most + * streams a node can have open is the number of containers it runs, whatever + * any client does: `feeds` is keyed by container and one stream serves every + * viewer of it, so connecting more times opens no more streams. + */ +const MAX_FOLLOWED = 10; + +/** + * One docker stream per container, however many viewers are watching it. + * + * The alternative is a stream per subscriber, which multiplies the daemon's work + * and this process's decoding by the number of people looking. Fan-out is what + * socket.io rooms are for, so the stream is opened by the first subscriber and + * closed by the last one leaving. Module scope because the streams outlive the + * connection that opened them. + * + * @type {Map}>} + */ +const feeds = new Map(); + +const roomFor = (containerId) => `applogs:${containerId}`; + +/** + * Stop a feed and forget it. Safe to call for a container with no feed. + * + * @param {object} io The namespace the room belongs to + * @param {string} containerId + */ +function closeFeed(io, containerId) { + const feed = feeds.get(containerId); + if (!feed) return; + // The room goes with the feed. A viewer left in it after the stream ended is + // still a member when the container is started again and someone else opens a + // fresh feed for the same id - it would be handed that feed's lines without + // having asked for them, into a pane that has already fallen back to the poll + // and is showing every line twice. Emptied here because a feed knows its room + // and cannot reach the connections that hold it. + io.socketsLeave(roomFor(containerId)); + // And the record each viewer keeps of following this container, which is the + // other half of the same subscription. A connection cannot be reached from + // here except through its viewers, so the record carries the way to forget + // it. Left behind, it counts against that connection's limit for the life of + // the socket while naming a container it no longer follows - and the id it + // names may since have been filed to somebody else's feed. + feed.subscribers.forEach((entry) => entry.forget()); + // Marked as well as dropped, because a feed can be closed while its stream is + // still being opened: the open has no way back to this map once the record is + // gone, and the flag is what tells it the stream it is holding has no viewer. + feed.closed = true; + clearInterval(feed.timer); + // destroy() rather than a docker call: this is the response stream, and + // destroying it is what tells the daemon to stop following. + if (feed.stream) feed.stream.destroy(); + feeds.delete(containerId); +} + +/** + * Send what has collected since the last flush, and say what did not fit. + * + * @param {object} io The namespace to emit on + * @param {string} containerId + */ +function flush(io, containerId) { + const feed = feeds.get(containerId); + if (!feed) return; + + // Named on every message, because a connection may follow several containers + // and a batch that does not say which one it belongs to can only be read by a + // client that follows exactly one. Additive: a client that reads `lines` and + // ignores the rest is unaffected. + if (feed.dropped) { + io.to(roomFor(containerId)).emit('skipped', { container: containerId, count: feed.dropped }); + feed.dropped = 0; + } + if (feed.queued.length) { + const lines = feed.queued; + feed.queued = []; + io.to(roomFor(containerId)).emit('logs', { container: containerId, lines }); + } + + // A line too long to hold was handed over cut, and this is what was cut from + // it. Its own event rather than `skipped`, which counts LINES the queue could + // not carry: a truncated line is one the viewer HAS, missing its tail, and + // reporting it as a skipped line would name the wrong thing and the wrong + // unit. Additive, like `skipped`: a client that reads `lines` and ignores the + // rest is unaffected. + // + // AFTER the lines, unlike `skipped` above. That one announces lines that never + // arrived, which belongs ahead of the ones that did; this one is about a line + // the viewer is being shown, so a reader meets the cut line first and then + // what was cut from it. Sent whether or not this batch carries lines, because + // a line that ends on the last character of a chunk settles what was cut from + // it with nothing queued behind it. + if (feed.truncated) { + io.to(roomFor(containerId)).emit('truncated', { container: containerId, characters: feed.truncated }); + feed.truncated = 0; + } +} + +/** + * Take the container's feed record. Filed before anything is awaited. + * + * Nothing may be awaited between the caller's `feeds.get` and this claim: two + * viewers opening one container in the same tick must find one feed between + * them. The map holds one record per container, so a second stream for the same + * container is held by nothing, cannot be destroyed, and follows the container + * for the life of the process - delivering every line to the room twice. + * + * Returned rather than kept private, because a subscription is held by the + * record and not by the id. Ids survive a container's restart; the record is + * what makes one subscription distinguishable from the next. + * + * @param {string} containerId + * @returns {object} the feed record, already filed + */ +function claimFeed(containerId) { + const feed = { + stream: null, queued: [], dropped: 0, truncated: 0, timer: null, subscribers: new Set(), recent: [], closed: false, + }; + feeds.set(containerId, feed); + return feed; +} + +/** + * Open the docker follow stream for a container, once. + * + * @param {object} io The namespace to emit on + * @param {object} container The dockerode container + * @param {string} containerId + * @param {object} feed The record `claimFeed` filed for this container + * @returns {Promise} resolves once the stream is attached + */ +async function openFeed(io, container, containerId, feed) { + // Bounded, unlike the polling read's decoder: that one is handed a single + // payload and is bounded by it, while this lives for as long as a viewer + // watches, and a container that never writes a newline would otherwise decide + // how much of the node's memory that costs - and then send all of it. + const decoder = new LogFrameDecoder({ + maxLineLength: LogFrameDecoder.MAX_LINE_LENGTH, + timestamped: true, + }); + + let stream; + try { + stream = await container.logs({ + follow: true, + stdout: true, + stderr: true, + timestamps: true, + // Bounded, like every other read this codebase makes of a log. `follow` + // with a `tail` opens at the end of the file and costs nothing to + // establish - measured on a live node at 2 CPU ticks against a 1 tick + // idle baseline, where the same read without a `tail` costs 15. + tail: BACKFILL_LINES, + }); + } catch (error) { + // Through closeFeed, the only thing that removes a feed from the map, so + // "not filed" and "marked closed" always agree - the invariant every holder + // of a subscription reads. It empties the room too: a viewer left in a room + // whose feed has gone is handed the lines of whatever feed opens for that id + // next, into a pane that has already fallen back to the poll. + // + // Guarded by identity, because a failing open can be the stale one - the + // feed it claimed already closed and the container reopened by another + // viewer. Emptying that room would leave a live feed's viewers subscribed to + // a stream that can no longer reach them. + if (feeds.get(containerId) === feed) { + io.to(roomFor(containerId)).emit('error', 'Log stream error.', containerId); + closeFeed(io, containerId); + } + throw error; + } + + // The last viewer left while the daemon was answering, so closeFeed has + // already run and found no stream to destroy. This is the only pass that can. + if (feed.closed) { + stream.destroy(); + return; + } + feed.stream = stream; + + // Called by the stream, not by socket.io, so the guard that answers a failing + // socket listener does not reach these. A throw here is a rejection nobody + // handles, which is a process exit. + const guard = (label, fn) => (...args) => { + try { + return fn(...args); + } catch (error) { + log.error(`appLogsHandler: ${label} for ${containerId}: ${error.message}`); + return undefined; + } + }; + + // This stream outlives the feed it was opened for: closeFeed destroys it, and + // the events that follow a destroy still arrive here - by which time a later + // viewer's feed can be filed under the same container id. So the handlers below + // act on the record this stream belongs to and stand down once it is closed, + // which "no longer filed" always agrees with. Reaching for the id instead hands + // a dead subscription's lines to the live one that replaced it, and lets a dead + // stream's error close a subscription it never had. + const enqueue = (lines) => { + if (!lines.length || feed.closed) return; + + // Appended rather than spread, which is the rule dockerContainerLogsPolling + // states and keeps. One chunk finishes as many lines as it has newlines and + // not as many as it has frames - the decoder splits frame bodies - so a + // 64KB read of the shortest non-empty lines measures 32,768 against the + // 125,263 arguments V8 accepts. That margin belongs to the socket's read + // size rather than to anything here, and crossing it is not a crash: the + // RangeError lands in the data handler's guard, and every line of the chunk + // is lost before one of them reaches a viewer. + // + // `recent` is kept so a viewer that joins a stream already running opens + // with the same context the first one got from docker's `tail`, rather than + // an empty pane until the container next writes. + for (let i = 0; i < lines.length; i += 1) feed.recent.push(lines[i]); + if (feed.recent.length > BACKFILL_LINES) feed.recent = feed.recent.slice(-BACKFILL_LINES); + + const space = MAX_QUEUED_LINES - feed.queued.length; + if (lines.length > space) { + feed.dropped += lines.length - space; + // From the offset rather than through a slice: the tail is all that is + // kept, and building it as an array of its own to hand over is an + // allocation of the same width for nothing. + for (let i = lines.length - space; i < lines.length; i += 1) feed.queued.push(lines[i]); + return; + } + for (let i = 0; i < lines.length; i += 1) feed.queued.push(lines[i]); + }; + + stream.on('data', guard('stream data', (chunk) => { + enqueue(decoder.push(chunk)); + feed.truncated += decoder.takeTruncated(); + })); + + stream.on('error', guard('stream error', (error) => { + if (feed.closed) return; + log.error(`appLogsHandler: stream error for ${containerId}: ${error.message}`); + io.to(roomFor(containerId)).emit('error', 'Log stream error.', containerId); + closeFeed(io, containerId); + })); + + // The container stopped, so docker closed the stream. Told before the feed + // goes, so a viewer shows a stopped container rather than a pane that quietly + // stops updating - and the connection is free to follow it again when it is + // started, because the entry holding it names a feed that is now closed. + stream.on('end', guard('stream end', () => { + if (feed.closed) return; + enqueue(decoder.flush()); + feed.truncated += decoder.takeTruncated(); + flush(io, containerId); + io.to(roomFor(containerId)).emit('ended', { container: containerId }); + closeFeed(io, containerId); + })); + + feed.timer = setInterval(() => flush(io, containerId), BATCH_MS); +} + +/** + * Live application logs, pushed. + * + * The polling endpoint stays exactly as it is and remains the only thing a node + * that predates this can offer, so a viewer tries here and falls back to it. The + * network runs several FluxOS versions at once and always will, which makes that + * fallback permanent rather than a migration step. + * + * @param {object} socket + * @returns {Promise} + */ +async function appLogsHandler(socket) { + const io = socket.nsp; + // What this connection follows, by container, each entry holding the feed + // record itself rather than the id it is filed under. Ids are names and names + // are reused - a container keeps its id across a restart - so an entry that + // held only an id cannot tell its own feed from the one another viewer opens + // for that container later. + // + // A map rather than a single slot, because nothing about a log stream is + // exclusive. The terminal takes one container per connection because `cmd` + // and `resize` name no session; everything here is addressed by room and + // keyed by container, and one docker stream serves every viewer of a + // container however they are connected. A connection following ten containers + // costs the node what ten connections following one each cost it, measured, + // and saves nine sockets. + /** @type {Map} */ + const following = new Map(); + // Subscribes that have not settled. A pass reaches `following` only once the + // daemon has named its container; an unsubscribe arriving before that marks + // the pass here, so a connection cannot end up following a container it has + // asked to leave. + const pending = new Set(); + let clientGone = false; + + // By container rather than by the slot, because the two get out of step + // exactly when it matters: a disconnect during the open runs leave() before + // there is a feed, finds nothing to release, and gives the slot up - so the + // pass that finally has a feed would have nothing to name it by, and the + // stream and its interval would run on with no viewer and nobody to stop them. + const release = (containerId, entry) => { + // Above the return, because the room is the half of a subscription that + // outlives having no feed: a subscribe whose open failed has nothing to + // release and used to carry the room out with it. And a connection that has + // given its slot up is free to follow a second container while a room it + // never left still delivers the first one's lines into that pane. + socket.leave(roomFor(containerId)); + const feed = feeds.get(containerId); + if (!feed) return; + if (entry) feed.subscribers.delete(entry); + // The last viewer left, so nothing is reading what the daemon is sending. + if (!feed.subscribers.size) closeFeed(io, containerId); + }; + + const leave = (containerId) => { + const entry = following.get(containerId); + if (!entry) return; + // Marked as well as dropped, because a subscribe still in setup holds this + // record and has no other way to learn the subscription was given up: the + // flag is what tells that pass to stand down rather than to finish and + // leave the connection following a container it has asked to leave. + entry.abandoned = true; + following.delete(containerId); + release(containerId, entry); + }; + + /** + * Give up everything this connection follows, or only what `name` names. + * + * @param {string|null} name a container id, the name a subscribe asked with, + * or null for all of them + */ + const leaveMatching = (name) => { + pending.forEach((pass) => { + if (!name || pass.name === name) pass.abandoned = true; + }); + [...following.values()] + .filter((entry) => !name || entry.containerId === name || entry.name === name) + .forEach((entry) => leave(entry.containerId)); + }; + + // Registered at connection, ahead of any message: a disconnect can land while + // authorisation and the docker lookup are still in flight, and socket.io emits + // 'disconnect' exactly once - a listener added after it was delivered never + // fires, and the feed it should have released would outlive every viewer. + socket.on('disconnect', () => { + clientGone = true; + leaveMatching(null); + }); + + // Named by whatever the client called it - the id it was given back, or the + // name it subscribed with. Matching both is what lets a viewer give up one + // container without a docker lookup to resolve what it already holds. No + // argument leaves everything, which is what a client that follows one + // container sends. + socket.on('unsubscribe', (nameOrId) => { + leaveMatching(typeof nameOrId === 'string' ? nameOrId : null); + }); + + socket.on('subscribe', async (zelidauth, nameOrId) => { + // Ahead of everything, because this namespace takes no middleware: both + // arguments are whatever an unauthenticated client serialised, and nothing + // upstream makes them strings the way node's http parser does for a header. + // + // zelidauth is refused here rather than at verifyPrivilege, which throws a + // TypeError for a non-string on purpose: that TypeError says our own code + // wired the call wrongly, and it cannot go on meaning that while any + // stranger can raise it on demand. + if (typeof nameOrId !== 'string') { + socket.emit('error', 'No container specified.'); + return; + } + if (typeof zelidauth !== 'string') { + socket.emit('error', 'Not authorized.', nameOrId); + return; + } + // Refused before the signature is checked, so a connection at its limit + // cannot spend the node's verification on a subscribe that cannot be + // accepted. The passes still in setup count too: a pass reaches `following` + // only after the verification and the docker lookup, so counting the + // settled ones alone lets any number of subscribes arriving together run + // both of those in parallel and be refused afterwards - the cost this + // refusal exists to avoid, taken as many times as they were sent. + // + // A pass stays in `pending` until its setup ends, which is after it has taken + // a container, so the two sets overlap. What this gate is owed is the + // containers the connection is committed to - the ones it holds and the + // passes that have yet to name one - and adding the whole of both charges a + // pass in mid-open twice, refusing a tenth container to a connection with + // nine open. A pass holds a container exactly when it carries its id. + // + // The count is checked again at the claim below, where nothing is awaited + // and it cannot move underneath the decision. + const settling = [...pending].filter((pass) => !pass.containerId).length; + if (following.size + settling >= MAX_FOLLOWED) { + socket.emit('error', `This connection already follows ${MAX_FOLLOWED} containers.`, nameOrId); + return; + } + + const mine = { + containerId: null, + name: nameOrId, + feed: null, + abandoned: false, + // How the feed reaches back to this connection when it closes. Nothing at + // module scope can see `following`, so the record carries the way to + // forget it - guarded by identity, because a later pass may have taken + // this container over and its record is not this one's to remove. + forget: () => { + if (mine.containerId && following.get(mine.containerId) === mine) { + following.delete(mine.containerId); + } + }, + }; + pending.add(mine); + + const mainAppName = nameOrId.split('_')[1] || nameOrId; + + // Gives up what this pass took, and only while the entry is still this + // pass's. A pass abandoned mid-setup can be overtaken by a later subscribe + // for the same container: releasing then would take the room and the feed + // out from under the pass that now holds them. + const drop = () => { + if (!mine.containerId || following.get(mine.containerId) !== mine) return; + following.delete(mine.containerId); + release(mine.containerId, mine); + }; + + try { + // Authorise BEFORE touching docker: the lookup below is a remote-controlled + // operation on an attacker-supplied name, and must not be reachable by an + // unauthenticated caller. Through verifyPrivilege like every other caller, + // so this stream carries a privilege a sweep can find. + const authorized = await verificationHelper.verifyPrivilege( + Privilege.APP_OWNER_OR_FLUX_TEAM, + zelidauth, + { appName: mainAppName }, + ); + if (authorized !== true) { + socket.emit('error', 'Not authorized.', nameOrId); + return; + } + + const container = await dockerService.getDockerContainerByIdOrName(nameOrId).catch((error) => { + log.error(`appLogsHandler: container lookup failed for ${nameOrId}: ${error.message}`); + return null; + }); + if (!container) { + socket.emit('error', 'Container not found.', nameOrId); + return; + } + + // The client may have gone, or given this subscription up, while the + // awaits above ran. Nothing is held yet, so there is nothing to release. + if (mine.abandoned || clientGone || !socket.connected) return; + + const containerId = container.id; + + // From here to the feed being in hand, nothing is awaited: the entry, the + // count and the feed identity are settled in one tick, so a second + // subscribe cannot pass a check this pass is about to invalidate. + const held = following.get(containerId); + if (held && held.feed && !held.feed.closed) { + // Already receiving it, which is the only thing 'subscribed' says. + // Answering it again is the honest reply to a client that asked twice, + // and costs the node nothing: the feed is shared and the room already + // carries it. + if (held.feed.subscribers.has(held)) { + socket.emit('subscribed', { container: containerId }); + return; + } + // Held by a pass that is still opening the stream. Answering + // 'subscribed' here would say a feed exists before it does, and leave + // the client believing it while an open that then fails is reported to + // the pass that made it. That pass answers for both. + return; + } + if (!held && following.size >= MAX_FOLLOWED) { + socket.emit('error', `This connection already follows ${MAX_FOLLOWED} containers.`, nameOrId); + return; + } + // Whatever the dead entry's pass is still doing, it stands down rather + // than releasing the container this pass is about to take. + if (held) held.abandoned = true; + + mine.containerId = containerId; + following.set(containerId, mine); + socket.join(roomFor(containerId)); + + const existing = feeds.get(containerId); + if (!existing) { + mine.feed = claimFeed(containerId); + await openFeed(io, container, containerId, mine.feed); + } else { + mine.feed = existing; + if (existing.recent.length) { + // Sent to this socket alone, and only the part the room will NOT send + // again. Every line is put in both `recent` and `queued`, so whatever + // is queued right now is also the tail of `recent` and is about to + // arrive here through the room - handing the whole of `recent` over + // delivers that tail twice, which is the one thing a log pane must + // never do. + // + // The two are read in the same tick with nothing awaited between them, + // and only a stream 'data' event appends to either, so this is a + // consistent snapshot rather than a race narrowed. + const alsoComing = Math.min(existing.queued.length, existing.recent.length); + const backfill = existing.recent.slice(0, existing.recent.length - alsoComing); + if (backfill.length) socket.emit('logs', { container: containerId, lines: backfill }); + } + } + + // Re-checked after that await. The feed was opened during it, so this is + // the pass that has to give it up - and it releases whether or not the + // disconnect that preceded it already did: release() is written to find + // nothing and return. + if (mine.abandoned || clientGone || !socket.connected) { + drop(); + return; + } + + // The feed this pass claimed is no longer the container's: the last viewer + // left while the daemon was answering, or the stream failed on open. + // 'subscribed' here would attach a pane to nothing, with no 'ended' or + // 'error' to fall back from. + if (feeds.get(containerId) !== mine.feed) { + drop(); + socket.emit('error', 'Log stream error.', containerId); + return; + } + + mine.feed.subscribers.add(mine); + socket.emit('subscribed', { container: containerId }); + } catch (error) { + log.error(`appLogsHandler: ${nameOrId}: ${error.message}`); + socket.emit('error', 'Error following logs.', nameOrId); + drop(); + } finally { + pending.delete(mine); + } + }); +} + +module.exports = appLogsHandler; +module.exports.feeds = feeds; +module.exports.BATCH_MS = BATCH_MS; +module.exports.MAX_QUEUED_LINES = MAX_QUEUED_LINES; +module.exports.BACKFILL_LINES = BACKFILL_LINES; +module.exports.MAX_FOLLOWED = MAX_FOLLOWED; diff --git a/ZelBack/src/lib/socketIoHandlers/debugHandler.js b/ZelBack/src/lib/socketIoHandlers/debugHandler.js index ad5a544f99..d2ab0eced7 100644 --- a/ZelBack/src/lib/socketIoHandlers/debugHandler.js +++ b/ZelBack/src/lib/socketIoHandlers/debugHandler.js @@ -1,7 +1,6 @@ -const querystring = require('node:querystring'); - const verificationHelper = require('../../services/verificationHelper'); const log = require('../log'); +const { Privilege } = require('../../services/utils/privileges'); async function debugHandler(socket) { const { handshake: { query, address } } = socket; @@ -15,11 +14,15 @@ async function debugHandler(socket) { return; } - const parsed = querystring.decode(authDetails); - - const req = { headers: { zelidauth: parsed } }; - - const ok = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + // authDetails is the query string the client sent, which is the same form a + // zelidauth header takes. There is no request here to take it from, and there + // never was - the one this built existed only to fit a signature. + const ok = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authDetails) + .catch((error) => { + // A throw in a socket.io listener is unhandled and takes the process down. + log.error(error); + return false; + }); if (ok !== true) { socket.emit('error', 'Unauthorized'); diff --git a/ZelBack/src/lib/socketIoHandlers/dockerTerminalHandler.js b/ZelBack/src/lib/socketIoHandlers/dockerTerminalHandler.js index 8dd7c724d4..529ec8a103 100644 --- a/ZelBack/src/lib/socketIoHandlers/dockerTerminalHandler.js +++ b/ZelBack/src/lib/socketIoHandlers/dockerTerminalHandler.js @@ -1,4 +1,5 @@ -const verificationHelperUtils = require('../../services/verificationHelperUtils'); +const verificationHelper = require('../../services/verificationHelper'); +const { Privilege } = require('../../services/utils/privileges'); const dockerService = require('../../services/dockerService'); const serviceHelper = require('../../services/serviceHelper'); const { trackTerminalSession } = require('../../services/analyticsService'); @@ -8,15 +9,115 @@ const log = require('../log'); async function dockerTerminalHandler(socket) { const clientIp = socket.handshake.headers['x-forwarded-for']?.split(',')[0]?.trim() || socket.handshake.address; - // Anything that throws in a socket.io listener is unhandled and takes the whole - // FluxOS process down, so every failure here has to leave through - // socket.emit('error', ...) instead. + // One terminal per connection, owned by the connection rather than by the + // message that opened it. // - // The try/catch below only covers this listener's own body - the dockerode and - // socket callbacks registered inside it run LATER, after the try has exited, so - // they are each wrapped in `guard` rather than relying on it. + // Everything below used to be declared inside the 'exec' listener, so each + // message brought its own copy of it and registered its own 'cmd', 'resize' + // and 'disconnect' listeners on the shared socket. A client emitting 'exec' + // twice got two shells and one keystroke written to both of them. + // + // A connection carries one terminal, so a second 'exec' on it is refused + // rather than stacked. That is this end's rule and does not rest on the + // client's behaviour, though the client agrees: it opens a socket per + // terminal, and socket.io-client forces a new connection for a namespace it + // already holds, so several terminals on one node are several sockets. + const session = { + // Whether a terminal exists on this connection or is being set up right now + // - not whether one was ever attempted. A setup that fails gives it back, so + // the refusal below can only ever be told to a caller that really has one. + claimed: false, + exec: null, + stream: null, + // What a close has to be able to say, recorded at the moment the open was. + opened: null, + }; + let clientGone = false; + + // Safe to run twice, which is what closes the setup race: a disconnect that + // lands while exec.start is in flight runs this with no stream to destroy, + // and the callback runs it again once there is one. Neither pass can record + // a second close, because the first clears what a close is made from. + const closeSession = () => { + if (session.stream) { + session.stream.destroy(); + session.stream = null; + } + if (session.opened) { + const { zelidauth, appName, component } = session.opened; + trackTerminalSession(zelidauth, appName, 'close', clientIp, component); + session.opened = null; + } + }; + + // Registered once, at connection, ahead of any message. Two reasons: a + // disconnect can land while authorisation and the docker lookups are still in + // flight, and socket.io emits 'disconnect' exactly once - a listener added + // after it has been delivered never fires, which leaked the hijacked stream + // and its exec for as long as the container lived. And a listener registered + // per message is a listener that accumulates. + socket.on('disconnect', () => { + clientGone = true; + closeSession(); + }); + + // Both route to whatever session exists, and do nothing before there is one. + // That is the whole price of registering them ahead of the shell they drive. + socket.on('resize', (data) => { + if (!session.exec) return; + const { rows, cols } = data || {}; + session.exec.resize({ h: rows, w: cols }, () => { }); + }); + + // A keystroke racing the container's teardown fails ASYNCHRONOUSLY through the + // stream's 'error' event - a destroyed socket's write() just returns false. + // The type filter is what stops a non-string payload throwing out of write(). + socket.on('cmd', (data) => { + if (!session.stream) return; + if (typeof data !== 'object') session.stream.write(data); + }); + + // Every way the setup below can fail ends here, so releasing what it took is + // part of failing rather than something each path has to remember. It pairs + // the analytics open too: a setup that recorded one and then failed used to + // leave it hanging until the socket closed. + const abandonSetup = (message) => { + closeSession(); + session.exec = null; + session.claimed = false; + if (message) socket.emit('error', message); + }; + socket.on('exec', async (zelidauth, nameOrId, dockerCmd, dockerEnv, dockerUser) => { - // wrap a callback that runs outside this listener's try/catch + // Ahead of everything, because this namespace takes no middleware: the five + // arguments are whatever an unauthenticated client serialised, and nothing + // upstream makes them strings the way node's http parser does for a header. + // + // nameOrId is split to name the app below, and a throw in an async socket.io + // listener is a rejection nobody handles - it reached apiServer's + // uncaughtException handler and exited the node. + // + // zelidauth is refused here rather than at verifyPrivilege, which throws a + // TypeError for a non-string on purpose: that TypeError says our own code + // wired the call wrongly, and it cannot go on meaning that while any + // stranger can raise it on demand. + if (typeof nameOrId !== 'string') { + socket.emit('error', 'No container specified.'); + return; + } + if (typeof zelidauth !== 'string') { + socket.emit('error', 'Not authorized.'); + return; + } + if (session.claimed) { + socket.emit('error', 'This connection already has a terminal.'); + return; + } + session.claimed = true; + + // Wraps a callback that runs outside this listener's try/catch. The dockerode + // and stream callbacks below are called by libraries, not by socket.io, so + // the guard that catches a failing socket listener does not reach them. const guard = (label, fn) => (...args) => { try { return fn(...args); @@ -26,43 +127,31 @@ async function dockerTerminalHandler(socket) { return undefined; } }; - // Registered BEFORE the awaits below: a disconnect can land while auth and - // the docker lookups are still in flight (deterministically, for a client - // that emits 'exec' and closes), and a listener added after socket.io's - // single 'disconnect' emit never fires - leaking the hijacked stream and - // its exec for as long as the container lives. - let execStream = null; - let clientGone = false; - let analyticsOpened = false; - // Derived synchronously from nameOrId so the disconnect listener below can - // own BOTH halves of the analytics pair. A close listener registered later - // (after the awaits) can never fire when the client left during them - - // socket.io emits 'disconnect' exactly once - which is how an 'open' ends up - // recorded with no matching 'close'. + const mainAppName = nameOrId.split('_')[1] || nameOrId; const parts = nameOrId.split('_'); const component = parts.length > 1 ? parts[0].replace(/^(zel|flux)/, '') || null : null; const analyticsAppName = parts.length > 1 ? mainAppName : mainAppName.replace(/^(zel|flux)/, ''); - socket.on('disconnect', () => { - clientGone = true; - if (execStream) execStream.destroy(); - // pairs whatever was opened, regardless of where in the flow we were - if (analyticsOpened) trackTerminalSession(zelidauth, analyticsAppName, 'close', clientIp, component); - }); + try { - const auth = { - zelidauth, - }; // Authorise BEFORE touching Docker: the lookup below is a remote-controlled // operation on an attacker-supplied name, and must not be reachable by an // unauthenticated caller. - const authorized = await verificationHelperUtils.verifyAppOwnerOrHigherSession(auth, mainAppName); + // + // Through verifyPrivilege like every other caller, so this shell carries a + // privilege a sweep can find. Reaching the verifier directly is what once + // hid an interactive root shell from a search for the privilege it needed. + const authorized = await verificationHelper.verifyPrivilege( + Privilege.APP_OWNER_OR_FLUX_TEAM, + zelidauth, + { appName: mainAppName }, + ); if (authorized !== true) { - socket.emit('error', 'Not authorized.'); + abandonSetup('Not authorized.'); return; } - // getDockerContainerByIdOrName reads .Id off an undefined lookup result when - // the container is absent, so this rejects rather than returning null. An + // getDockerContainerByIdOrName throws `Container not found` when the + // container is absent, so this rejects rather than returning null. An // unreachable daemon rejects here too and reaches the client as the same // "not found" - log the cause so the two are distinguishable. const container = await dockerService.getDockerContainerByIdOrName(nameOrId).catch((error) => { @@ -70,19 +159,22 @@ async function dockerTerminalHandler(socket) { return null; }); if (!container) { - socket.emit('error', 'Container not found.'); + abandonSetup('Container not found.'); return; } - // the client may have gone away during the awaits above - do not create an + // The client may have gone away during the awaits above - do not create an // exec nobody is attached to, and do not record a session that never - // happened. This check has to precede the analytics open: a client that - // left during the awaits has already had its 'disconnect' delivered, so - // nothing downstream can close a session opened after it. - if (clientGone || !socket.connected) return; + // happened. This check has to precede the open: a client that left during + // the awaits has already had its 'disconnect' delivered, and the close it + // ran found nothing to pair. + if (clientGone || !socket.connected) { + abandonSetup(); + return; + } trackTerminalSession(zelidauth, analyticsAppName, 'open', clientIp, component); - analyticsOpened = true; + session.opened = { zelidauth, appName: analyticsAppName, component }; const cmd = { AttachStdout: true, @@ -96,15 +188,15 @@ async function dockerTerminalHandler(socket) { container.exec(cmd, guard('exec create', (err, exec) => { // dockerode passes back a null exec when the daemon rejects the exec // create (most commonly the container is not running - state created or - // exited). Without this guard the code below dereferences null - // (exec.start / exec.resize) and the resulting TypeError is thrown from - // inside this callback, which is unhandled and crashes the whole FluxOS - // process. Fail the terminal session cleanly instead. + // exited). Without this the code below dereferences null and the + // TypeError leaves a library callback, where nothing catches it. if (err || !exec) { log.error(`dockerTerminalHandler: exec create failed for ${nameOrId}: ${err ? err.message : 'no exec instance (is the container running?)'}`); - socket.emit('error', 'Error opening a terminal. Is the container running?'); + abandonSetup('Error opening a terminal. Is the container running?'); return; } + session.exec = exec; + const options = { Tty: true, stream: true, @@ -113,19 +205,13 @@ async function dockerTerminalHandler(socket) { stderr: true, hijack: true, }; - socket.on('resize', guard('resize', (data) => { - const { rows, cols } = data; - exec.resize({ h: rows, w: cols }, () => { - }); - })); /* eslint-disable no-shadow */ exec.start(options, guard('exec start', (err, stream) => { - // Same defensive check as above: on failure stream can be null, and the - // stream.on(...) below would throw an unhandled TypeError out of this - // callback (crashing the process). Bail out cleanly instead. + // Same check as above: on failure stream can be null, and the + // stream.on(...) below would throw out of a library callback. if (err || !stream) { log.error(`dockerTerminalHandler: exec start failed for ${nameOrId}: ${err ? err.message : 'no stream'}`); - socket.emit('error', 'Error executing the command.'); + abandonSetup('Error executing the command.'); return; } stream.on('data', guard('stream data', (chunk) => { @@ -141,33 +227,16 @@ async function dockerTerminalHandler(socket) { socket.emit('error', 'Terminal session error.'); })); - // Hand the stream to the disconnect teardown registered before the - // awaits, and close the race the other way: if the client vanished - // while exec.start was in flight, its disconnect has already fired - // and nothing else would ever destroy this stream. - execStream = stream; - if (clientGone || !socket.connected) { - stream.destroy(); - return; - } - - // A keystroke racing the container's teardown fails ASYNCHRONOUSLY via - // the stream's 'error' event (handled above) - a destroyed socket's - // write() just returns false. The type filter is what stops a - // non-string payload throwing synchronously out of write(). - socket.on('cmd', guard('cmd', (data) => { - if (typeof data !== 'object') { - stream.write(data); - } - })); + // Hand the stream to the session, and close the race the other way: if + // the client vanished while exec.start was in flight its disconnect has + // already run, and nothing else would ever destroy this stream. + session.stream = stream; + if (clientGone || !socket.connected) closeSession(); })); - socket.on('end', () => { - log.info('--------end---------'); - }); })); } catch (error) { log.error(`dockerTerminalHandler: ${nameOrId}: ${error.message}`); - socket.emit('error', 'Error opening a terminal.'); + abandonSetup('Error opening a terminal.'); } }); } diff --git a/ZelBack/src/lib/socketIoHandlers/index.js b/ZelBack/src/lib/socketIoHandlers/index.js index 3dfee0d595..743dd453ac 100644 --- a/ZelBack/src/lib/socketIoHandlers/index.js +++ b/ZelBack/src/lib/socketIoHandlers/index.js @@ -1,7 +1,9 @@ const debugHandler = require('./debugHandler'); const dockerTerminalHandler = require('./dockerTerminalHandler'); +const appLogsHandler = require('./appLogsHandler'); module.exports = { debug: debugHandler, terminal: dockerTerminalHandler, + applogs: appLogsHandler, }; diff --git a/ZelBack/src/lib/socketIoServer.js b/ZelBack/src/lib/socketIoServer.js index fa2c03d99d..abf5f27343 100644 --- a/ZelBack/src/lib/socketIoServer.js +++ b/ZelBack/src/lib/socketIoServer.js @@ -1,5 +1,88 @@ const socketio = require('socket.io'); +const log = require('./log'); + +// socket.io calls a listener and discards the promise it returns - dispatch() +// is `super.emitUntyped.apply` inside a process.nextTick, with no catch - so a +// listener that throws produces a rejection nobody handles, which reaches +// apiServer's uncaughtException handler and exits the node. An unauthenticated +// client emitting one malformed message was a restart of FluxOS. +// +// Every listener registered through this class answers its own client instead: +// the same bargain asyncRoute makes for a route, bounded to the one socket that +// failed, with everything else on the node untouched. Deliberately NOT the +// process-level unhandledRejection handler, which was proposed on this branch +// and rejected - that one swallows every failure everywhere and leaves the node +// running in a state nobody chose. +const LISTENER_FAILED = 'Error handling request.'; + +/** + * A listener that leaves its failure through the socket rather than the process. + * @param {object} socket The socket to answer + * @param {string} event The event being listened for, for the log line + * @param {Function} listener The listener as written + * @returns {Function} the listener to register + */ +function answeringItsOwnClient(socket, event, listener) { + const answer = (error) => { + log.error(`socketIoServer: '${event}' listener failed: ${error.message}`); + socket.emit('error', LISTENER_FAILED); + }; + + return function guarded(...args) { + let result; + try { + result = listener(...args); + } catch (error) { + return answer(error); + } + // Returned as it was when it is not a promise: socket.io ignores what a + // listener returns, and a caller inside this process may not. + return typeof result?.then === 'function' ? result.catch(answer) : result; + }; +} + +/** + * The socket a handler is given, with every listener it registers guarded. + * + * The socket itself is mutated rather than replaced by a facade: a handler + * reads handshake, emits, joins rooms, reads `connected` and disconnects, and a + * facade that forgot one of those would fail a long way from here. socket.io + * registers its own listeners in the Socket constructor, which has already run + * by the time 'connection' is emitted, so none of them pass through this. + * + * The wrapper is a different function from the listener, so removal by identity + * has to be translated or it silently removes nothing. A WeakMap, so an entry + * lasts exactly as long as the listener it belongs to. + * + * Safe to apply twice - two listeners on one namespace are handed the same + * socket - because the inner guard answers first and the outer one never sees + * the failure, and a removal translates through both maps in turn. + * @param {object} socket The connected socket + * @returns {object} the same socket + */ +function guardingItsListeners(socket) { + const wrappers = new WeakMap(); + const register = { on: socket.on, once: socket.once }; + const remove = { off: socket.off, removeListener: socket.removeListener }; + + Object.keys(register).forEach((method) => { + socket[method] = function guardedRegister(event, listener) { + const wrapper = answeringItsOwnClient(socket, event, listener); + wrappers.set(listener, wrapper); + return register[method].call(this, event, wrapper); + }; + }); + + Object.keys(remove).forEach((method) => { + socket[method] = function guardedRemove(event, listener) { + return remove[method].call(this, event, wrappers.get(listener) || listener); + }; + }); + + return socket; +} + class FluxSocketIoServer { static defaultErrorHandler = () => { }; @@ -26,7 +109,12 @@ class FluxSocketIoServer { addListener(event, listener, options = {}) { const namespace = `/${options.namespace}` || '/'; - this.io.of(namespace).on(event, listener); + // A namespace emits its socket as the first argument, so the connection + // handler is guarded against the socket it was handed - and that socket + // guards everything the handler goes on to register on it. + this.io.of(namespace).on(event, (socket, ...rest) => ( + answeringItsOwnClient(socket, event, listener)(guardingItsListeners(socket), ...rest) + )); } attachNamespaceListeners() { diff --git a/ZelBack/src/lib/socketServer.js b/ZelBack/src/lib/socketServer.js index 92e8ee8302..9d71ce5642 100644 --- a/ZelBack/src/lib/socketServer.js +++ b/ZelBack/src/lib/socketServer.js @@ -6,12 +6,19 @@ const { FLUX_VERSION, FLUX_CAPABILITIES } = require('../services/utils/FluxPeerS class FluxWebsocketServer { static defautlErrorHandler = () => { }; + static defaultAdmit = () => null; + #socketServer = new WebSocketServer({ noServer: true, perMessageDeflate: { zlibDeflateOptions: { chunkSize: 1024, - memLevel: 9, + // No-context-takeover resets the stream after every message, so the + // window can never carry history between messages and only ever matches + // within one. Gossip messages are a few KB, so an 8KB window and this + // hash table compress them to the same bytes a 32KB window does, on a + // third of the memory - and every peer socket holds a context. + memLevel: 8, level: 9, }, zlibInflateOptions: { @@ -19,8 +26,13 @@ class FluxWebsocketServer { }, clientNoContextTakeover: true, serverNoContextTakeover: true, - clientMaxWindowBits: true, // Allow Firefox to use default settings - serverMaxWindowBits: true, // Let browsers negotiate (Default 15) + // Both stay as the peer's choice. Pinning a number here rejects the + // handshake outright - with a 400, not a fallback to uncompressed - for + // any client that offers a smaller window than ours, and browsers reach + // this server too (/ws/id, /ws/sign, /ws/payment). Peers running this + // build offer 13 themselves, so negotiation still settles there. + clientMaxWindowBits: true, + serverMaxWindowBits: true, concurrencyLimit: 2, threshold: 128, }, @@ -33,6 +45,7 @@ class FluxWebsocketServer { constructor(options = {}) { this.#routes = options.routes || {}; this.errorHandler = options.errorHandler || FluxWebsocketServer.defautlErrorHandler; + this.admit = options.admit || FluxWebsocketServer.defaultAdmit; this.#routeMatchers = Object.entries(this.#routes).map((entry) => { const [route, handler] = entry; @@ -75,6 +88,14 @@ class FluxWebsocketServer { return this.#routeMatchers.slice(); } + /** + * The underlying ws server, so a caller can observe what the handshake did. + * @returns {WebSocketServer} + */ + get wsServer() { + return this.#socketServer; + } + matchRoute(url) { let routeHandler = null; let params = {}; @@ -99,7 +120,49 @@ class FluxWebsocketServer { return null; } + /** + * Complete the websocket handshake, unless admission refuses it first. + * + * A refusal has to be answered here, with an HTTP status, because a handshake + * that completes is already a connection: the peer reads our capabilities out + * of the 101's headers, builds a peer object and can write to it before we + * have run a single line of our own. Closing afterwards does not undo any of + * that - it leaves the other side holding something it believes in, and what + * it wrote into it is gone. Answering the upgrade instead means no socket, no + * peer, and a dial that fails cleanly and is retried. + * @param {import('node:http').IncomingMessage} request + * @param {import('node:net').Socket} socket + * @param {Buffer} head + * @returns {void} + */ handleUpgrade(request, socket, head) { + const refusal = this.admit(request); + + if (refusal) { + const { status, message, reason } = refusal; + // THIS SOCKET IS OURS NOW. http removes its own error listener before + // emitting 'upgrade', and ws attaches one as the first thing it does with + // a socket it is handed. Writing to one with no listener leaves an + // 'error' with nowhere to go, and node throws those - which reaches the + // process handler in apiServer and exits the node. The peer resetting + // between its request and this reply is enough to raise one, and this + // path only runs while connections are refused, which is a boot. + socket.on('error', () => socket.destroy()); + // Ended rather than written-then-destroyed. destroy() does not wait for a + // queued write, so the status the dialler is meant to read can be thrown + // away with the socket that was carrying it - leaving it a bare reset, + // which is the nothing this answer exists to replace. + socket.once('finish', () => socket.destroy()); + socket.end( + `HTTP/1.1 ${status} ${message}\r\n` + + 'Connection: close\r\n' + + `X-Flux-Refusal: ${reason}\r\n` + + 'Content-Length: 0\r\n' + + '\r\n', + ); + return; + } + this.#socketServer.handleUpgrade(request, socket, head, (ws) => { this.#socketServer.emit('connection', ws, request); }); diff --git a/ZelBack/src/middlewares/alwaysRespond.js b/ZelBack/src/middlewares/alwaysRespond.js new file mode 100644 index 0000000000..d4d934fd89 --- /dev/null +++ b/ZelBack/src/middlewares/alwaysRespond.js @@ -0,0 +1,21 @@ +/** + * Express middleware for endpoints that act on every call. + * + * Express fingerprints each response with an ETag. App control endpoints answer with the + * same body every time, so a client replaying its stored ETag receives a bodiless 304 and + * cannot tell whether the action ran — even though the handler executed in full. Removing + * the request validator opts these routes out of conditional-GET handling, and no-store + * keeps the client from caching the answer for next time. + * + * @param {object} req - Request object + * @param {object} res - Response object + * @param {Function} next - Next middleware + * @returns {void} + */ +function alwaysRespond(req, res, next) { + delete req.headers['if-none-match']; + res.set('Cache-Control', 'no-store'); + return next(); +} + +module.exports = alwaysRespond; diff --git a/ZelBack/src/middlewares/index.js b/ZelBack/src/middlewares/index.js new file mode 100644 index 0000000000..d04fc290b6 --- /dev/null +++ b/ZelBack/src/middlewares/index.js @@ -0,0 +1,16 @@ +/** + * FluxOS Express middlewares + * + * Entry point for the middlewares mounted on individual routes. Middlewares applied + * to every request are registered on the server itself, in lib/fluxServer.js. + */ + +const alwaysRespond = require('./alwaysRespond'); +const isLocal = require('./isLocal'); +const requireHttps = require('./requireHttps'); + +module.exports = { + alwaysRespond, + isLocal, + requireHttps, +}; diff --git a/ZelBack/src/middlewares/isLocal.js b/ZelBack/src/middlewares/isLocal.js new file mode 100644 index 0000000000..becb79d4a2 --- /dev/null +++ b/ZelBack/src/middlewares/isLocal.js @@ -0,0 +1,18 @@ +/** + * Express middleware restricting a route to callers on the node itself. + * + * @param {object} req - Request object + * @param {object} res - Response object + * @param {Function} next - Next middleware + * @returns {void} + */ +function isLocal(req, res, next) { + // Only addresses the socket vouches for. X-Forwarded-For is the caller's own + // claim, and a localhost check that ever believes it is deciding "local" on + // an attacker-controlled header. + const remote = req.ip || req.connection.remoteAddress || req.socket.remoteAddress; + if (remote === 'localhost' || remote === '127.0.0.1' || remote === '::ffff:127.0.0.1' || remote === '::1') return next(); + return res.status(401).send('Access denied'); +} + +module.exports = isLocal; diff --git a/ZelBack/src/middlewares/requireHttps.js b/ZelBack/src/middlewares/requireHttps.js new file mode 100644 index 0000000000..8c901afcc1 --- /dev/null +++ b/ZelBack/src/middlewares/requireHttps.js @@ -0,0 +1,23 @@ +const messageHelper = require('../services/messageHelper'); + +/** + * Express middleware rejecting a route when the connection is not TLS. + * + * @param {object} req - Request object + * @param {object} res - Response object + * @param {Function} next - Next middleware + * @returns {void} + */ +function requireHttps(req, res, next) { + if (!req.secure) { + const errMessage = messageHelper.createErrorMessage( + 'HTTPS required for ArcaneOS authentication endpoints', + 'ForbiddenProtocol', + 403, + ); + return res.status(403).json(errMessage); + } + return next(); +} + +module.exports = requireHttps; diff --git a/ZelBack/src/routes.js b/ZelBack/src/routes.js index 096f3b2a7f..68df662762 100644 --- a/ZelBack/src/routes.js +++ b/ZelBack/src/routes.js @@ -1,5 +1,3 @@ -const apicache = require('apicache'); - const daemonServiceAddressRpcs = require('./services/daemonService/daemonServiceAddressRpcs'); const daemonServiceTransactionRpcs = require('./services/daemonService/daemonServiceTransactionRpcs'); const daemonServiceBlockchainRpcs = require('./services/daemonService/daemonServiceBlockchainRpcs'); @@ -17,7 +15,10 @@ const paymentService = require('./services/paymentService'); const fluxService = require('./services/fluxService'); const fluxCommunication = require('./services/fluxCommunication'); const fluxCommunicationMessagesSender = require('./services/fluxCommunicationMessagesSender'); -const messageHelper = require('./services/messageHelper'); +const { + asyncRoute, cache, rejectQueryParameters, requireBootSettled, +} = require('./services/utils/routeGuards'); +const { alwaysRespond, isLocal, requireHttps } = require('./middlewares'); // App modular services const appQueryService = require('./services/appQuery/appQueryService'); @@ -25,9 +26,12 @@ const resourceQueryService = require('./services/appQuery/resourceQueryService') const deploymentInfoService = require('./services/appQuery/deploymentInfoService'); const fileQueryService = require('./services/appQuery/fileQueryService'); const fileSystemManager = require('./services/appSystem/fileSystemManager'); +const volumeExecutor = require('./services/appSystem/volumeExecutor'); +const operationsController = require('./services/appManagement/operationsController'); const cryptographicKeys = require('./services/appMessaging/cryptographicKeys'); const registryManager = require('./services/appDatabase/registryManager'); const appValidator = require('./services/appRequirements/appValidator'); +const placementFeasibility = require('./services/appPlacement/placementFeasibility'); const appSpecHelpers = require('./services/utils/appSpecHelpers'); const appInspector = require('./services/appManagement/appInspector'); const appController = require('./services/appManagement/appController'); @@ -46,428 +50,480 @@ const generalService = require('./services/generalService'); const upnpService = require('./services/upnpService'); const syncthingService = require('./services/syncthingService'); const fluxNetworkHelper = require('./services/fluxNetworkHelper'); +const portManager = require('./services/appNetwork/portManager'); const enterpriseNodesService = require('./services/enterpriseNodesService'); const backupRestoreService = require('./services/backupRestoreService'); -const IOUtils = require('./services/IOUtils'); const arcaneAuthService = require('./services/arcaneAuthService'); const appTamperingDetectionService = require('./services/appTamperingDetectionService'); const fluxEventBus = require('./services/utils/fluxEventBus'); -function isLocal(req, res, next) { - const remote = req.ip || req.connection.remoteAddress || req.socket.remoteAddress || req.headers['x-forwarded-for']; - if (remote === 'localhost' || remote === '127.0.0.1' || remote === '::ffff:127.0.0.1' || remote === '::1') return next(); - return res.status(401).send('Access denied'); -} - -function requireHttps(req, res, next) { - if (!req.secure) { - const errMessage = messageHelper.createErrorMessage( - 'HTTPS required for ArcaneOS authentication endpoints', - 'ForbiddenProtocol', - 403, - ); - return res.status(403).json(errMessage); - } - return next(); -} - -const cache = apicache.middleware; - module.exports = (app) => { // GET PUBLIC methods - app.get('/daemon/help/:command?', cache('1 hour'), (req, res) => { // accept both help/command and ?command=getinfo. If ommited, default help will be displayed. Other calls works in similar way - daemonServiceControlRpcs.help(req, res); - }); - app.get('/daemon/getinfo', cache('30 seconds'), (req, res) => { - daemonServiceControlRpcs.getInfo(req, res); - }); - app.get('/daemon/getfluxnodestatus', cache('60 seconds'), (req, res) => { - daemonServiceNodeRpcs.getFluxNodeStatusApi(req, res); - }); - app.get('/daemon/getzelnodestatus', cache('60 seconds'), (req, res) => { // DEPRECATED - daemonServiceNodeRpcs.getFluxNodeStatusApi(req, res); - }); - app.get('/daemon/listfluxnodes/:filter?', cache('30 seconds'), (req, res) => { - daemonServiceNodeRpcs.listFluxNodes(req, res); - }); - app.get('/daemon/listzelnodes/:filter?', cache('30 seconds'), (req, res) => { // DEPRECATED - daemonServiceNodeRpcs.listFluxNodes(req, res); - }); - app.get('/daemon/viewdeterministicfluxnodelist/:filter?', cache('30 seconds'), (req, res) => { - daemonServiceNodeRpcs.listFluxNodes(req, res); - }); - app.get('/daemon/viewdeterministiczelnodelist/:filter?', cache('30 seconds'), (req, res) => { // DEPRECATED - daemonServiceNodeRpcs.listFluxNodes(req, res); - }); - app.get('/daemon/getfluxnodecount', cache('30 seconds'), (req, res) => { - daemonServiceNodeRpcs.getFluxNodeCount(req, res); - }); - app.get('/daemon/getzelnodecount', cache('30 seconds'), (req, res) => { // DEPRECATED - daemonServiceNodeRpcs.getFluxNodeCount(req, res); - }); - app.get('/daemon/getdoslist', cache('30 seconds'), (req, res) => { - daemonServiceNodeRpcs.getDOSList(req, res); - }); - app.get('/daemon/getstartlist', cache('30 seconds'), (req, res) => { - daemonServiceNodeRpcs.getStartList(req, res); - }); - app.get('/daemon/fluxnodecurrentwinner', cache('30 seconds'), (req, res) => { - daemonServiceNodeRpcs.fluxNodeCurrentWinner(req, res); - }); - app.get('/daemon/getbestblockhash', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getBestBlockHash(req, res); - }); - app.get('/daemon/getblock/:hashheight?/:verbosity?', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getBlock(req, res); - }); - app.get('/daemon/getblockchaininfo', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getBlockchainInfo(req, res); - }); - app.get('/daemon/getblockcount', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getBlockCount(req, res); - }); - app.get('/daemon/getblockdeltas/:hash?', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getBlockDeltas(req, res); - }); - app.get('/daemon/getblockhashes/:high?/:low?/:noorphans?/:logicaltimes?', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getBlockHashes(req, res); - }); - app.get('/daemon/getblockhash/:index?', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getBlockHash(req, res); - }); - app.get('/daemon/getblockheader/:hash?/:verbose?', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getBlockHeader(req, res); - }); - app.get('/daemon/getchaintips', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getChainTips(req, res); - }); - app.get('/daemon/getdifficulty', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getDifficulty(req, res); - }); - app.get('/daemon/getmempoolinfo', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getMempoolInfo(req, res); - }); - app.get('/daemon/getrawmempool/:verbose?', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getRawMemPool(req, res); - }); - app.get('/daemon/gettxout/:txid?/:n?/:includemempool?', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getTxOut(req, res); - }); - app.get('/daemon/gettxoutproof/:txids?/:blockhash?', cache('30 seconds'), (req, res) => { // comma separated list of txids. For example: /gettxoutproof/abc,efg,asd/blockhash - daemonServiceBlockchainRpcs.getTxOutProof(req, res); - }); - app.get('/daemon/gettxoutsetinfo', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getTxOutSetInfo(req, res); - }); - app.get('/daemon/verifytxoutproof/:proof?', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.verifyTxOutProof(req, res); - }); - app.get('/daemon/getspentinfo/:txid?/:index?', cache('30 seconds'), (req, res) => { - daemonServiceBlockchainRpcs.getSpentInfo(req, res); - }); - app.get('/daemon/getblocksubsidy/:height?', cache('30 seconds'), (req, res) => { - daemonServiceMiningRpcs.getBlockSubsidy(req, res); - }); - app.get('/daemon/getblocktemplate/:jsonrequestobject?', cache('30 seconds'), (req, res) => { - daemonServiceMiningRpcs.getBlockTemplate(req, res); - }); - app.get('/daemon/getlocalsolps', cache('30 seconds'), (req, res) => { - daemonServiceMiningRpcs.getLocalSolPs(req, res); - }); - app.get('/daemon/getmininginfo', cache('30 seconds'), (req, res) => { - daemonServiceMiningRpcs.getMiningInfo(req, res); - }); - app.get('/daemon/getnetworkhashps/:blocks?/:height?', cache('30 seconds'), (req, res) => { - daemonServiceMiningRpcs.getNetworkHashPs(req, res); - }); - app.get('/daemon/getnetworksolps/:blocks?/:height?', cache('30 seconds'), (req, res) => { - daemonServiceMiningRpcs.getNetworkSolPs(req, res); - }); - app.get('/daemon/getconnectioncount', cache('30 seconds'), (req, res) => { - daemonServiceNetworkRpcs.getConnectionCount(req, res); - }); - app.get('/daemon/getdeprecationinfo', cache('30 seconds'), (req, res) => { - daemonServiceNetworkRpcs.getDeprecationInfo(req, res); - }); - app.get('/daemon/getnettotals', cache('30 seconds'), (req, res) => { - daemonServiceNetworkRpcs.getNetTotals(req, res); - }); - app.get('/daemon/getnetworkinfo', cache('30 seconds'), (req, res) => { - daemonServiceNetworkRpcs.getNetworkInfo(req, res); - }); - app.get('/daemon/getpeerinfo', cache('30 seconds'), (req, res) => { - daemonServiceNetworkRpcs.getPeerInfo(req, res); - }); - app.get('/daemon/listbanned', cache('30 seconds'), (req, res) => { - daemonServiceNetworkRpcs.listBanned(req, res); - }); - app.get('/daemon/createrawtransaction/:transactions?/:addresses?/:locktime?/:expiryheight?', (req, res) => { - daemonServiceTransactionRpcs.createRawTransaction(req, res); - }); - app.get('/daemon/decoderawtransaction/:hexstring?', cache('30 seconds'), (req, res) => { - daemonServiceTransactionRpcs.decodeRawTransaction(req, res); - }); - app.get('/daemon/decodescript/:hex?', cache('30 seconds'), (req, res) => { - daemonServiceTransactionRpcs.decodeScript(req, res); - }); - app.get('/daemon/fundrawtransaction/:hexstring?', (req, res) => { - daemonServiceTransactionRpcs.fundRawTransaction(req, res); - }); - app.get('/daemon/getrawtransaction/:txid?/:verbose?', (req, res) => { - daemonServiceTransactionRpcs.getRawTransaction(req, res); - }); - app.get('/daemon/sendrawtransaction/:hexstring?/:allowhighfees?', (req, res) => { - daemonServiceTransactionRpcs.sendRawTransaction(req, res); - }); - app.get('/daemon/createmultisig/:n?/:keys?', (req, res) => { - daemonServiceUtilityRpcs.createMultiSig(req, res); - }); - app.get('/daemon/estimatefee/:nblocks?', cache('30 seconds'), (req, res) => { - daemonServiceUtilityRpcs.estimateFee(req, res); - }); - app.get('/daemon/estimatepriority/:nblocks?', cache('30 seconds'), (req, res) => { - daemonServiceUtilityRpcs.estimatePriority(req, res); - }); - app.get('/daemon/validateaddress/:fluxaddress?', cache('30 seconds'), (req, res) => { - daemonServiceUtilityRpcs.validateAddress(req, res); - }); - app.get('/daemon/verifymessage/:fluxaddress?/:signature?/:message?', cache('30 seconds'), (req, res) => { - daemonServiceUtilityRpcs.verifyMessage(req, res); - }); - app.get('/daemon/gettransaction/:txid?/:includewatchonly?', cache('30 seconds'), (req, res) => { - daemonServiceWalletRpcs.getTransaction(req, res); - }); - app.get('/daemon/zvalidateaddress/:zaddr?', cache('30 seconds'), (req, res) => { - daemonServiceUtilityRpcs.zValidateAddress(req, res); - }); - app.get('/daemon/getbenchmarks', cache('30 seconds'), (req, res) => { - daemonServiceBenchmarkRpcs.getBenchmarks(req, res); - }); - app.get('/daemon/getbenchstatus', cache('30 seconds'), (req, res) => { - daemonServiceBenchmarkRpcs.getBenchStatus(req, res); - }); + app.get('/daemon/help/:command?', cache('1 hour'), asyncRoute((req, res) => { // accept both help/command and ?command=getinfo. If ommited, default help will be displayed. Other calls works in similar way + return daemonServiceControlRpcs.help(req, res); + })); + app.get('/daemon/getinfo', asyncRoute((req, res) => { + return daemonServiceControlRpcs.getInfo(req, res); + })); + app.get('/daemon/getfluxnodestatus', cache('60 seconds'), asyncRoute((req, res) => { + return daemonServiceNodeRpcs.getFluxNodeStatusApi(req, res); + })); + app.get('/daemon/getzelnodestatus', cache('60 seconds'), asyncRoute((req, res) => { // DEPRECATED + return daemonServiceNodeRpcs.getFluxNodeStatusApi(req, res); + })); + app.get('/daemon/listfluxnodes/:filter?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNodeRpcs.listFluxNodes(req, res); + })); + app.get('/daemon/listzelnodes/:filter?', cache('30 seconds'), asyncRoute((req, res) => { // DEPRECATED + return daemonServiceNodeRpcs.listFluxNodes(req, res); + })); + app.get('/daemon/viewdeterministicfluxnodelist/:filter?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNodeRpcs.listFluxNodes(req, res); + })); + app.get('/daemon/viewdeterministiczelnodelist/:filter?', cache('30 seconds'), asyncRoute((req, res) => { // DEPRECATED + return daemonServiceNodeRpcs.listFluxNodes(req, res); + })); + app.get('/daemon/getfluxnodecount', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNodeRpcs.getFluxNodeCount(req, res); + })); + app.get('/daemon/getzelnodecount', cache('30 seconds'), asyncRoute((req, res) => { // DEPRECATED + return daemonServiceNodeRpcs.getFluxNodeCount(req, res); + })); + app.get('/daemon/getdoslist', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNodeRpcs.getDOSList(req, res); + })); + app.get('/daemon/getstartlist', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNodeRpcs.getStartList(req, res); + })); + app.get('/daemon/fluxnodecurrentwinner', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNodeRpcs.fluxNodeCurrentWinner(req, res); + })); + app.get('/daemon/getbestblockhash', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getBestBlockHash(req, res); + })); + app.get('/daemon/getblock/:hashheight?/:verbosity?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getBlock(req, res); + })); + app.get('/daemon/getblockchaininfo', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getBlockchainInfo(req, res); + })); + app.get('/daemon/getblockcount', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getBlockCount(req, res); + })); + app.get('/daemon/getblockdeltas/:hash?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getBlockDeltas(req, res); + })); + app.get('/daemon/getblockhashes/:high?/:low?/:noorphans?/:logicaltimes?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getBlockHashes(req, res); + })); + app.get('/daemon/getblockhash/:index?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getBlockHash(req, res); + })); + app.get('/daemon/getblockheader/:hash?/:verbose?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getBlockHeader(req, res); + })); + app.get('/daemon/getchaintips', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getChainTips(req, res); + })); + app.get('/daemon/getdifficulty', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getDifficulty(req, res); + })); + app.get('/daemon/getmempoolinfo', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getMempoolInfo(req, res); + })); + app.get('/daemon/getrawmempool/:verbose?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getRawMemPool(req, res); + })); + app.get('/daemon/gettxout/:txid?/:n?/:includemempool?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getTxOut(req, res); + })); + app.get('/daemon/gettxoutproof/:txids?/:blockhash?', cache('30 seconds'), asyncRoute((req, res) => { // comma separated list of txids. For example: /gettxoutproof/abc,efg,asd/blockhash + return daemonServiceBlockchainRpcs.getTxOutProof(req, res); + })); + app.get('/daemon/gettxoutsetinfo', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getTxOutSetInfo(req, res); + })); + app.get('/daemon/verifytxoutproof/:proof?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.verifyTxOutProof(req, res); + })); + app.get('/daemon/getspentinfo/:txid?/:index?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getSpentInfo(req, res); + })); + app.get('/daemon/getblocksubsidy/:height?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceMiningRpcs.getBlockSubsidy(req, res); + })); + app.get('/daemon/getblocktemplate/:jsonrequestobject?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceMiningRpcs.getBlockTemplate(req, res); + })); + app.get('/daemon/getlocalsolps', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceMiningRpcs.getLocalSolPs(req, res); + })); + app.get('/daemon/getmininginfo', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceMiningRpcs.getMiningInfo(req, res); + })); + app.get('/daemon/getnetworkhashps/:blocks?/:height?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceMiningRpcs.getNetworkHashPs(req, res); + })); + app.get('/daemon/getnetworksolps/:blocks?/:height?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceMiningRpcs.getNetworkSolPs(req, res); + })); + app.get('/daemon/getconnectioncount', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.getConnectionCount(req, res); + })); + app.get('/daemon/getdeprecationinfo', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.getDeprecationInfo(req, res); + })); + app.get('/daemon/getnettotals', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.getNetTotals(req, res); + })); + app.get('/daemon/getnetworkinfo', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.getNetworkInfo(req, res); + })); + app.get('/daemon/getpeerinfo', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.getPeerInfo(req, res); + })); + app.get('/daemon/listbanned', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.listBanned(req, res); + })); + app.get('/daemon/createrawtransaction/:transactions?/:addresses?/:locktime?/:expiryheight?', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.createRawTransaction(req, res); + })); + app.get('/daemon/decoderawtransaction/:hexstring?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.decodeRawTransaction(req, res); + })); + app.get('/daemon/decodescript/:hex?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.decodeScript(req, res); + })); + app.get('/daemon/fundrawtransaction/:hexstring?', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.fundRawTransaction(req, res); + })); + app.get('/daemon/getrawtransaction/:txid?/:verbose?', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.getRawTransaction(req, res); + })); + app.get('/daemon/sendrawtransaction/:hexstring?/:allowhighfees?', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.sendRawTransaction(req, res); + })); + app.get('/daemon/createmultisig/:n?/:keys?', asyncRoute((req, res) => { + return daemonServiceUtilityRpcs.createMultiSig(req, res); + })); + app.get('/daemon/estimatefee/:nblocks?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceUtilityRpcs.estimateFee(req, res); + })); + app.get('/daemon/estimatepriority/:nblocks?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceUtilityRpcs.estimatePriority(req, res); + })); + app.get('/daemon/validateaddress/:fluxaddress?', asyncRoute((req, res) => { + return daemonServiceUtilityRpcs.validateAddress(req, res); + })); + app.get('/daemon/verifymessage/:fluxaddress?/:signature?/:message?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceUtilityRpcs.verifyMessage(req, res); + })); + app.get('/daemon/gettransaction/:txid?/:includewatchonly?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceWalletRpcs.getTransaction(req, res); + })); + app.get('/daemon/zvalidateaddress/:zaddr?', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceUtilityRpcs.zValidateAddress(req, res); + })); + app.get('/daemon/getbenchmarks', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBenchmarkRpcs.getBenchmarks(req, res); + })); + app.get('/daemon/getbenchstatus', cache('30 seconds'), asyncRoute((req, res) => { + return daemonServiceBenchmarkRpcs.getBenchStatus(req, res); + })); - app.get('/id/loginphrase', (req, res) => { - idService.loginPhrase(req, res); - }); - app.get('/id/emergencyphrase', (req, res) => { - idService.emergencyPhrase(req, res); - }); - app.get('/zelid/loginphrase', (req, res) => { // DEPRECATED - idService.loginPhrase(req, res); - }); - app.get('/zelid/emergencyphrase', (req, res) => { // DEPRECATED - idService.emergencyPhrase(req, res); - }); + app.get('/id/loginphrase', asyncRoute((req, res) => { + return idService.loginPhrase(req, res); + })); + app.get('/id/emergencyphrase', asyncRoute((req, res) => { + return idService.emergencyPhrase(req, res); + })); + app.get('/zelid/loginphrase', asyncRoute((req, res) => { // DEPRECATED + return idService.loginPhrase(req, res); + })); + app.get('/zelid/emergencyphrase', asyncRoute((req, res) => { // DEPRECATED + return idService.emergencyPhrase(req, res); + })); - app.get('/flux/nodetier', cache('30 seconds'), (req, res) => { - fluxService.getNodeTier(req, res); - }); - app.get('/flux/info', cache('60 seconds'), (req, res) => { - fluxService.getFluxInfo(req, res); - }); - app.get('/flux/timezone', (req, res) => { - fluxService.getFluxTimezone(req, res); - }); - app.get('/flux/version', cache('30 seconds'), (req, res) => { - fluxService.getFluxVersion(req, res); - }); - app.get('/flux/nodejsversions', cache('30 seconds'), (req, res) => { - fluxService.getNodeJsVersions(req, res); - }); - app.get('/flux/ip', cache('30 seconds'), (req, res) => { - fluxService.getFluxIP(req, res); - }); - app.get('/flux/staticip', cache('30 seconds'), (req, res) => { - fluxService.isStaticIPapi(req, res); - }); - app.get('/flux/geolocation', cache('30 seconds'), (req, res) => { - fluxService.getFluxGeolocation(req, res); - }); - app.get('/flux/zelid', cache('30 seconds'), (req, res) => { // DEPERCATED - fluxService.getFluxZelID(req, res); - }); - app.get('/flux/id', cache('30 seconds'), (req, res) => { - fluxService.getFluxZelID(req, res); - }); - app.get('/flux/fluxids', cache('30 seconds'), (req, res) => { - fluxService.getFluxIds(req, res); - }); - app.get('/flux/pgp', cache('30 seconds'), (req, res) => { - fluxService.getFluxPGPidentity(req, res); - }); - app.get('/flux/kadena', cache('30 seconds'), (req, res) => { - fluxService.getFluxKadena(req, res); - }); - app.get('/flux/routerip', cache('1 day'), (req, res) => { - fluxService.getRouterIP(req, res); - }); - app.get('/flux/blockedports', cache('1 day'), (req, res) => { - fluxService.getBlockedPorts(req, res); - }); - app.get('/flux/apiport', cache('1 day'), (req, res) => { - fluxService.getAPIPort(req, res); - }); - app.get('/flux/blockedrepositories', cache('1 day'), (req, res) => { - fluxService.getBlockedRepositories(req, res); - }); - app.get('/flux/enterpriseappowners', cache('1 hour'), (req, res) => { - fluxService.getEnterpriseAppOwners(req, res); - }); - app.get('/flux/marketplaceurl', cache('1 day'), (req, res) => { - fluxService.getMarketplaceURL(req, res); - }); - app.get('/flux/restart', cache('30 seconds'), (req, res) => { - fluxService.restartFluxOS(req, res); - }); - app.get('/flux/dosstate', cache('30 seconds'), (req, res) => { - fluxNetworkHelper.getDOSState(req, res); - }); - app.post('/flux/dosstate', (req, res) => { - fluxNetworkHelper.setDOSStateApi(req, res); - }); + app.get('/flux/nodetier', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.getNodeTier(req, res); + })); + app.get('/flux/info', cache('60 seconds'), asyncRoute((req, res) => { + return fluxService.getFluxInfo(req, res); + })); + app.get('/flux/timezone', asyncRoute((req, res) => { + return fluxService.getFluxTimezone(req, res); + })); + app.get('/flux/version', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.getFluxVersion(req, res); + })); + app.get('/flux/nodejsversions', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.getNodeJsVersions(req, res); + })); + app.get('/flux/ip', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.getFluxIP(req, res); + })); + app.get('/flux/staticip', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.isStaticIPapi(req, res); + })); + app.get('/flux/geolocation', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.getFluxGeolocation(req, res); + })); + app.get('/flux/zelid', cache('30 seconds'), asyncRoute((req, res) => { // DEPERCATED + return fluxService.getFluxZelID(req, res); + })); + app.get('/flux/id', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.getFluxZelID(req, res); + })); + app.get('/flux/fluxids', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.getFluxIds(req, res); + })); + app.get('/flux/pgp', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.getFluxPGPidentity(req, res); + })); + app.get('/flux/kadena', cache('30 seconds'), asyncRoute((req, res) => { + return fluxService.getFluxKadena(req, res); + })); + app.get('/flux/routerip', cache('1 day'), asyncRoute((req, res) => { + return fluxService.getRouterIP(req, res); + })); + app.get('/flux/blockedports', cache('1 day'), asyncRoute((req, res) => { + return fluxService.getBlockedPorts(req, res); + })); + app.get('/flux/apiport', cache('1 day'), asyncRoute((req, res) => { + return fluxService.getAPIPort(req, res); + })); + app.get('/flux/blockedrepositories', cache('1 day'), asyncRoute((req, res) => { + return fluxService.getBlockedRepositories(req, res); + })); + app.get('/flux/enterpriseappowners', cache('1 hour'), asyncRoute((req, res) => { + return fluxService.getEnterpriseAppOwners(req, res); + })); + app.get('/flux/marketplaceurl', cache('1 day'), asyncRoute((req, res) => { + return fluxService.getMarketplaceURL(req, res); + })); + app.get('/flux/restart', asyncRoute((req, res) => { + return fluxService.restartFluxOS(req, res); + })); + app.get('/flux/dosstate', cache('30 seconds'), asyncRoute((req, res) => { + return fluxNetworkHelper.getDOSState(req, res); + })); + app.post('/flux/dosstate', asyncRoute((req, res) => { + return fluxNetworkHelper.setDOSStateApi(req, res); + })); // New peer endpoints - app.get('/flux/peers/:filter?', cache('30 seconds'), (req, res) => { - fluxCommunication.getPeers(req, res); - }); - app.get('/flux/unstablenodes', cache('30 seconds'), (req, res) => { - fluxCommunication.getUnstableNodes(req, res); - }); - app.get('/flux/peerhistory', cache('5 seconds'), (req, res) => { - fluxCommunication.getPeerHistory(req, res); - }); - app.get('/flux/topology', cache('5 seconds'), (req, res) => { - fluxCommunication.getTopology(req, res); - }); - app.get('/flux/networkhealth', cache('5 seconds'), (req, res) => { - fluxCommunication.getNetworkHealth(req, res); - }); + app.get('/flux/peers/:filter?', cache('30 seconds'), asyncRoute((req, res) => { + return fluxCommunication.getPeers(req, res); + })); + app.get('/flux/unstablenodes', cache('30 seconds'), asyncRoute((req, res) => { + return fluxCommunication.getUnstableNodes(req, res); + })); + app.get('/flux/peerhistory', asyncRoute((req, res) => { + return fluxCommunication.getPeerHistory(req, res); + })); + app.get('/flux/topology', cache('5 seconds'), asyncRoute((req, res) => { + return fluxCommunication.getTopology(req, res); + })); + app.get('/flux/networkhealth', cache('5 seconds'), asyncRoute((req, res) => { + return fluxCommunication.getNetworkHealth(req, res); + })); // Deprecated peer endpoints — kept for backward compatibility - app.get('/flux/connectedpeers', cache('30 seconds'), (req, res) => { - fluxCommunication.connectedPeers(req, res); - }); - app.get('/flux/connectedpeersinfo', cache('30 seconds'), (req, res) => { - fluxCommunication.connectedPeersInfo(req, res); - }); - app.get('/flux/incomingconnections', cache('30 seconds'), (req, res) => { - fluxNetworkHelper.getIncomingConnections(req, res); - }); - app.get('/flux/incomingconnectionsinfo', cache('30 seconds'), (req, res) => { - fluxNetworkHelper.getIncomingConnectionsInfo(req, res); - }); - app.get('/flux/checkfluxavailability/:ip?/:port?', cache('30 seconds'), (req, res) => { - fluxNetworkHelper.checkFluxAvailability(req, res); - }); - app.post('/flux/checkappavailability', (req, res) => { - fluxNetworkHelper.checkAppAvailability(req, res); - }); - app.post('/flux/keepupnpportsopen', (req, res) => { - fluxNetworkHelper.keepUPNPPortsOpen(req, res); - }); + app.get('/flux/connectedpeers', cache('30 seconds'), asyncRoute((req, res) => { + return fluxCommunication.connectedPeers(req, res); + })); + app.get('/flux/connectedpeersinfo', cache('30 seconds'), asyncRoute((req, res) => { + return fluxCommunication.connectedPeersInfo(req, res); + })); + app.get('/flux/incomingconnections', cache('30 seconds'), asyncRoute((req, res) => { + return fluxNetworkHelper.getIncomingConnections(req, res); + })); + app.get('/flux/incomingconnectionsinfo', cache('30 seconds'), asyncRoute((req, res) => { + return fluxNetworkHelper.getIncomingConnectionsInfo(req, res); + })); + app.get('/flux/checkfluxavailability/:ip?/:port?', cache('30 seconds'), asyncRoute((req, res) => { + return fluxNetworkHelper.checkFluxAvailability(req, res); + })); + app.post('/flux/checkappavailability', asyncRoute((req, res) => { + return fluxNetworkHelper.checkAppAvailability(req, res); + })); + app.post('/flux/keepupnpportsopen', asyncRoute((req, res) => { + return fluxNetworkHelper.keepUPNPPortsOpen(req, res); + })); + // POST because the ask carries a signature, which makes it a body and not a + // URL. No cache() follows from that: apicache keys on the URL alone, so it + // would answer every different ask alike and answer it before the handler ran + // - and it caches only GETs regardless. portsInUse caches the value instead. + // No rejectQueryParameters, because with the ask in the body there is no URL + // surface for a query string to reach. + // + // What this endpoint is for, who may call it and why it is signed is on + // portsInUseApi, with the code that enforces it. + app.post('/flux/portsinuse', asyncRoute((req, res) => { + return portManager.portsInUseApi(req, res); + })); // ArcaneOS Authentication Endpoints (HTTPS only) - app.get('/arcane/authchallenge', requireHttps, arcaneAuthService.authChallengeHandler); + app.get('/arcane/authchallenge', requireHttps, asyncRoute(arcaneAuthService.authChallengeHandler)); // Apps routes - now directly calling modular services - app.get('/apps/listrunningapps', cache('15 seconds'), (req, res) => { - appQueryService.listRunningApps(req, res); - }); - app.get('/apps/listallapps', cache('30 seconds'), (req, res) => { - appQueryService.listAllApps(req, res); - }); - app.get('/apps/listappsimages', cache('30 seconds'), (req, res) => { - appInspector.listAppsImages(req, res); - }); - app.get('/apps/installedapps/:appname?', cache('30 seconds'), (req, res) => { - appQueryService.installedApps(req, res); - }); - app.get('/apps/availableapps', cache('30 seconds'), (req, res) => { - registryManager.availableApps(req, res); - }); - app.get('/apps/fluxusage', cache('30 seconds'), (req, res) => { - resourceQueryService.fluxUsage(req, res); - }); - app.get('/apps/appsresources', cache('30 seconds'), (req, res) => { - resourceQueryService.appsResources(req, res); - }); - app.get('/apps/registrationinformation', cache('30 seconds'), (req, res) => { - registryManager.registrationInformation(req, res); - }); - app.get('/apps/temporarymessages/:hash?', cache('5 seconds'), (req, res) => { - messageVerifier.getAppsTemporaryMessages(req, res); - }); - app.get('/apps/permanentmessages/:hash?/:owner?/:appname?', cache('2 minutes'), (req, res) => { - messageVerifier.getAppsPermanentMessages(req, res); - }); - app.get('/apps/globalappsspecifications/:hash?/:owner?/:appname?', cache('30 seconds'), (req, res) => { - registryManager.getGlobalAppsSpecifications(req, res); - }); - app.get('/apps/latestspecificationversion', cache('5 minutes'), (req, res) => { - appQueryService.getlatestApplicationSpecificationAPI(req, res); - }); - app.get('/apps/updatetolatestspecs/:appname', cache('30 seconds'), (req, res) => { - registryManager.updateApplicationSpecificationAPI(req, res); - }); - app.get('/apps/appspecifications/:appname/:decrypt?', (req, res) => { - registryManager.getApplicationSpecificationAPI(req, res); - }); - app.get('/apps/appowner/:appname?', cache('30 seconds'), (req, res) => { - registryManager.getApplicationOwnerAPI(req, res); - }); - app.get('/apps/apporiginalowner/:appname?', cache('30 seconds'), (req, res) => { - appQueryService.getApplicationOriginalOwner(req, res); - }); - app.get('/apps/messagescount/:appowner?', cache('30 seconds'), (req, res) => { - appQueryService.getAppsMessagesCount(req, res); - }); - app.get('/apps/hashes', cache('30 seconds'), (req, res) => { - registryManager.getAppHashes(req, res); - }); - app.get('/apps/location/:appname?', cache('30 seconds'), (req, res) => { - registryManager.getAppsLocation(req, res); - }); - app.get('/apps/locations', cache('30 seconds'), (req, res) => { - registryManager.getAppsLocations(req, res); - }); - app.get('/apps/installinglocation/:appname?', cache('30 seconds'), (req, res) => { - registryManager.getAppInstallingLocation(req, res); - }); - app.get('/apps/installinglocations', cache('30 seconds'), (req, res) => { - appQueryService.getAppsInstallingLocations(req, res); - }); - app.get('/apps/installingerrorslocation/:appname?', cache('30 seconds'), (req, res) => { - registryManager.getAppInstallingErrorsLocation(req, res); - }); - app.get('/apps/installingerrorslocations', cache('30 seconds'), (req, res) => { - registryManager.getAppsInstallingErrorsLocations(req, res); - }); - app.post('/apps/calculateprice', (req, res) => { // returns price in flux for both new registration of app and update of app - appSpecHelpers.getAppPrice(req, res); - }); - app.post('/apps/calculatefiatandfluxprice', (req, res) => { // returns price in usd and flux for both new registration of app and update of app - appSpecHelpers.getAppFiatAndFluxPrice(req, res); - }); - app.get('/apps/whitelistedrepositories', cache('30 seconds'), (req, res) => { - generalService.whitelistedRepositories(req, res); - }); - app.post('/apps/verifyappregistrationspecifications', (req, res) => { // returns formatted app specifications - appValidator.verifyAppRegistrationParameters(req, res); - }); - app.post('/apps/verifyappupdatespecifications', (req, res) => { // returns formatted app specifications - appValidator.verifyAppUpdateApi(req, res); - }); - app.get('/apps/deploymentinformation', cache('30 seconds'), (req, res) => { - deploymentInfoService.deploymentInformation(req, res); - }); - app.get('/apps/enterprisenodes', cache('30 seconds'), (req, res) => { - enterpriseNodesService.getEnterpriseNodesAPI(req, res); - }); - app.get('/apps/getappspecsusdprice', cache('30 minutes'), (req, res) => { - deploymentInfoService.getAppSpecsUSDPrice(req, res); - }); - app.get('/apps/tamperingevents/:appname?', cache('30 seconds'), (req, res) => { - appTamperingDetectionService.getEvents(req, res); - }); + // + // Several names below say "apps" and mean something else. listrunningapps and + // listallapps return CONTAINERS, listappsimages returns IMAGES, and each repeats + // "apps" inside a path that is already under /apps. The names are wrong and are + // left wrong deliberately: renaming a v1 route is a 404 for every caller, which is + // a different and louder break than changing what a response contains, and v2 gets + // the correct names for free in a new URL space. Do not rename them here. + // + // Both container listings are public and answer every caller identically, with + // {Names, State, Status} built from docker's own fields - which is what lets + // them keep a cache that answers before any handler runs. + // + // Not a filter of listallapps, whatever the name says. This answers "which containers + // should FDM route to", which is the running ones PLUS any stopped container whose app + // is mid-backup or mid-restore - a fact that lives in this process's memory and is not + // on the container, so no caller can derive this list from the one below. FDM's + // checkAppRunning reads it for every app and matches on Names[0] alone, so dropping a + // container from here takes that app out of routing. + app.get('/apps/listrunningapps', cache('15 seconds'), asyncRoute((req, res) => { + return appQueryService.listRunningAppsApi(req, res); + })); + // Read by peers mid-election. Both are unauthenticated, and the API has no rate + // limiting, so neither may do unbounded backend work per request. + // + // heldcomponents still lists docker containers: a component's commitment is + // in-memory and a FluxOS restart drops it while the container keeps running, so + // docker is the only thing that answers for a primary that outlived the process + // holding its intent. Cached at one second - long enough to bound an anonymous + // caller to one docker call a second, short enough to be meaningless against the + // tens of seconds this exists to cover. The cache keys on the request URL, so + // that bound holds only while the URL is the endpoint and nothing else: without + // the guard a caller varies a parameter and every request is a fresh miss. + app.get('/apps/heldcomponents', rejectQueryParameters, cache('1 second'), asyncRoute((req, res) => { + return appQueryService.heldComponents(req, res); + })); + // promotedfolders needs no cache: it is served from the set the syncthing monitor + // already refreshes each pass, so the request touches nothing. Guarded on the + // same terms as its neighbour - it takes no parameters either, and the two are + // read by the same callers on the same path. + app.get('/apps/promotedfolders', rejectQueryParameters, asyncRoute((req, res) => { + return appQueryService.promotedFolders(req, res); + })); + app.get('/apps/listallapps', cache('30 seconds'), asyncRoute((req, res) => { + return appQueryService.listAllAppsApi(req, res); + })); + // Answers the flux team, so it takes no cache: apicache keys an entry on the + // request URL alone and serves it before the handler runs, which would hand + // one authorised answer to every caller after it. + app.get('/apps/listappsimages', asyncRoute((req, res) => { + return appInspector.listAppsImagesApi(req, res); + })); + app.get('/apps/installedapps/:appname?', cache('30 seconds'), asyncRoute((req, res) => { + return appQueryService.installedApps(req, res); + })); + app.get('/apps/availableapps', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.availableApps(req, res); + })); + app.get('/apps/fluxusage', cache('30 seconds'), asyncRoute((req, res) => { + return resourceQueryService.fluxUsage(req, res); + })); + app.get('/apps/appsresources', cache('30 seconds'), asyncRoute((req, res) => { + return resourceQueryService.appsResourcesApi(req, res); + })); + app.get('/apps/registrationinformation', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.registrationInformation(req, res); + })); + app.get('/apps/temporarymessages/:hash?', cache('5 seconds'), asyncRoute((req, res) => { + return messageVerifier.getAppsTemporaryMessages(req, res); + })); + app.get('/apps/permanentmessages/:hash?/:owner?/:appname?', cache('2 minutes'), asyncRoute((req, res) => { + return messageVerifier.getAppsPermanentMessages(req, res); + })); + app.get('/apps/globalappsspecifications/:hash?/:owner?/:appname?', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.getGlobalAppsSpecifications(req, res); + })); + app.get('/apps/latestspecificationversion', cache('5 minutes'), asyncRoute((req, res) => { + return appQueryService.getlatestApplicationSpecificationAPI(req, res); + })); + // Not cached. apicache keys an entry on the URL alone and answers from it + // before the handler runs, so a privilege-checked route behind one hands the + // first caller's response to the next without checking them at all. This + // route's response is also built for one caller in particular - the payload is + // encrypted to a session key they supply in a header - so there is nothing in + // it another caller could use even if it were shared. + app.get('/apps/updatetolatestspecs/:appname', asyncRoute((req, res) => { + return registryManager.updateApplicationSpecificationAPI(req, res); + })); + app.get('/apps/appspecifications/:appname/:decrypt?', asyncRoute((req, res) => { + return registryManager.getApplicationSpecificationAPI(req, res); + })); + // Component names and their election mode, for the flux team. Not cached: the + // answer depends on who is asking, and a shared cache in front of a + // privilege-checked route serves one caller's answer to the next. + app.get('/apps/appcomponentnames/:appname?', asyncRoute((req, res) => { + return registryManager.getApplicationComponentNamesAPI(req, res); + })); + app.get('/apps/appowner/:appname?', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.getApplicationOwnerAPI(req, res); + })); + app.get('/apps/apporiginalowner/:appname?', cache('30 seconds'), asyncRoute((req, res) => { + return appQueryService.getApplicationOriginalOwner(req, res); + })); + app.get('/apps/messagescount/:appowner?', cache('30 seconds'), asyncRoute((req, res) => { + return appQueryService.getAppsMessagesCount(req, res); + })); + app.get('/apps/hashes', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.getAppHashes(req, res); + })); + app.get('/apps/location/:appname?', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.getAppsLocation(req, res); + })); + app.get('/apps/locations', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.getAppsLocations(req, res); + })); + app.get('/apps/installinglocation/:appname?', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.getAppInstallingLocation(req, res); + })); + app.get('/apps/installinglocations', cache('30 seconds'), asyncRoute((req, res) => { + return appQueryService.getAppsInstallingLocations(req, res); + })); + app.get('/apps/installingerrorslocation/:appname?', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.getAppInstallingErrorsLocation(req, res); + })); + app.get('/apps/installingerrorslocations', cache('30 seconds'), asyncRoute((req, res) => { + return registryManager.getAppsInstallingErrorsLocations(req, res); + })); + app.post('/apps/calculateprice', asyncRoute((req, res) => { // returns price in flux for both new registration of app and update of app + return appSpecHelpers.getAppPrice(req, res); + })); + app.post('/apps/calculatefiatandfluxprice', asyncRoute((req, res) => { // returns price in usd and flux for both new registration of app and update of app + return appSpecHelpers.getAppFiatAndFluxPrice(req, res); + })); + app.get('/apps/whitelistedrepositories', cache('30 seconds'), asyncRoute((req, res) => { // deprecated: whitelist retired, always returns [] + return generalService.whitelistedRepositories(req, res); + })); + app.post('/apps/verifyappregistrationspecifications', asyncRoute((req, res) => { // returns formatted app specifications + return appValidator.verifyAppRegistrationParameters(req, res); + })); + app.post('/apps/verifyappupdatespecifications', asyncRoute((req, res) => { // returns formatted app specifications + return appValidator.verifyAppUpdateApi(req, res); + })); + app.post('/apps/placementfeasibility', asyncRoute((req, res) => { // fault domains and per-domain instance share for a prospective spec + return placementFeasibility.placementFeasibilityAPI(req, res); + })); + app.get('/apps/placementlocations', rejectQueryParameters, cache('30 seconds'), asyncRoute((req, res) => { // node, fault-domain and tier counts per continent/country + return placementFeasibility.placementLocationsAPI(req, res); + })); + app.get('/apps/deploymentinformation', cache('30 seconds'), asyncRoute((req, res) => { + return deploymentInfoService.deploymentInformation(req, res); + })); + app.get('/apps/enterprisenodes', cache('30 seconds'), asyncRoute((req, res) => { + return enterpriseNodesService.getEnterpriseNodesAPI(req, res); + })); + app.get('/apps/getappspecsusdprice', cache('30 minutes'), asyncRoute((req, res) => { + return deploymentInfoService.getAppSpecsUSDPrice(req, res); + })); + app.get('/apps/tamperingevents/:appname?', cache('30 seconds'), asyncRoute((req, res) => { + return appTamperingDetectionService.getEvents(req, res); + })); // app.get('/explorer/allutxos', (req, res) => { // explorerService.getAllUtxos(req, res); @@ -479,1086 +535,1174 @@ module.exports = (app) => { // explorerService.getAllAddresses(req, res); // }); - app.get('/explorer/utxo/:address?', cache('30 seconds'), (req, res) => { - explorerService.getAddressUtxos(req, res); - }); - app.get('/explorer/transactions/:address?', cache('30 seconds'), (req, res) => { - explorerService.getAddressTransactions(req, res); - }); - app.get('/explorer/balance/:address?', cache('30 seconds'), (req, res) => { - explorerService.getAddressBalance(req, res); - }); - app.get('/explorer/scannedheight', cache('30 seconds'), (req, res) => { - explorerService.getScannedHeight(req, res); - }); + app.get('/explorer/utxo/:address?', cache('30 seconds'), asyncRoute((req, res) => { + return explorerService.getAddressUtxos(req, res); + })); + app.get('/explorer/transactions/:address?', cache('30 seconds'), asyncRoute((req, res) => { + return explorerService.getAddressTransactions(req, res); + })); + app.get('/explorer/balance/:address?', cache('30 seconds'), asyncRoute((req, res) => { + return explorerService.getAddressBalance(req, res); + })); + app.get('/explorer/scannedheight', cache('30 seconds'), asyncRoute((req, res) => { + return explorerService.getScannedHeight(req, res); + })); // app.get('/explorer/fusion/coinbase/all', cache('30 seconds'), (req, res) => { // explorerService.getAllFusionCoinbase(req, res); // }); - app.get('/explorer/fusion/coinbase/:address?', cache('30 seconds'), (req, res) => { // deprecated - explorerService.getAddressFusionCoinbase(req, res); - }); + app.get('/explorer/fusion/coinbase/:address?', cache('30 seconds'), asyncRoute((req, res) => { // deprecated + return explorerService.getAddressFusionCoinbase(req, res); + })); // GET PROTECTED API - User level - app.get('/daemon/prioritisetransaction/:txid?/:prioritydelta?/:feedelta?', cache('30 seconds'), (req, res) => { - daemonServiceMiningRpcs.prioritiseTransaction(req, res); - }); - app.get('/daemon/submitblock/:hexdata?/:jsonparametersobject?', cache('30 seconds'), (req, res) => { - daemonServiceMiningRpcs.submitBlock(req, res); - }); + app.get('/daemon/prioritisetransaction/:txid?/:prioritydelta?/:feedelta?', asyncRoute((req, res) => { + return daemonServiceMiningRpcs.prioritiseTransaction(req, res); + })); + app.get('/daemon/submitblock/:hexdata?/:jsonparametersobject?', asyncRoute((req, res) => { + return daemonServiceMiningRpcs.submitBlock(req, res); + })); - app.get('/id/loggedsessions', cache('30 seconds'), (req, res) => { - idService.loggedSessions(req, res); - }); - app.get('/id/logoutcurrentsession', cache('30 seconds'), (req, res) => { - idService.logoutCurrentSession(req, res); - }); - app.get('/id/logoutallsessions', cache('30 seconds'), (req, res) => { - idService.logoutAllSessions(req, res); - }); - app.get('/zelid/loggedsessions', cache('30 seconds'), (req, res) => { // DEPRECATED - idService.loggedSessions(req, res); - }); - app.get('/zelid/logoutcurrentsession', cache('30 seconds'), (req, res) => { // DEPRECATED - idService.logoutCurrentSession(req, res); - }); - app.get('/zelid/logoutallsessions', cache('30 seconds'), (req, res) => { // DEPRECATED - idService.logoutAllSessions(req, res); - }); + app.get('/id/loggedsessions', asyncRoute((req, res) => { + return idService.loggedSessions(req, res); + })); + app.get('/id/logoutcurrentsession', asyncRoute((req, res) => { + return idService.logoutCurrentSession(req, res); + })); + app.get('/id/logoutallsessions', asyncRoute((req, res) => { + return idService.logoutAllSessions(req, res); + })); + app.get('/zelid/loggedsessions', asyncRoute((req, res) => { // DEPRECATED + return idService.loggedSessions(req, res); + })); + app.get('/zelid/logoutcurrentsession', asyncRoute((req, res) => { // DEPRECATED + return idService.logoutCurrentSession(req, res); + })); + app.get('/zelid/logoutallsessions', asyncRoute((req, res) => { // DEPRECATED + return idService.logoutAllSessions(req, res); + })); - app.get('/benchmark/getstatus', cache('30 seconds'), (req, res) => { - benchmarkService.getStatus(req, res); - }); - app.get('/benchmark/help/:command?', cache('1 hour'), (req, res) => { - benchmarkService.help(req, res); - }); - app.get('/benchmark/getbenchmarks', cache('30 seconds'), (req, res) => { - benchmarkService.getBenchmarks(req, res); - }); - app.get('/benchmark/getstoredbenchmark', cache('1 hour'), (req, res) => { - benchmarkService.getStoredBenchmark(req, res); - }); - app.get('/benchmark/getinfo', cache('30 seconds'), (req, res) => { - benchmarkService.getInfo(req, res); - }); + app.get('/benchmark/getstatus', cache('30 seconds'), asyncRoute((req, res) => { + return benchmarkService.getStatus(req, res); + })); + app.get('/benchmark/help/:command?', cache('1 hour'), asyncRoute((req, res) => { + return benchmarkService.help(req, res); + })); + app.get('/benchmark/getbenchmarks', cache('30 seconds'), asyncRoute((req, res) => { + return benchmarkService.getBenchmarks(req, res); + })); + app.get('/benchmark/getstoredbenchmark', cache('1 hour'), asyncRoute((req, res) => { + return benchmarkService.getStoredBenchmark(req, res); + })); + app.get('/benchmark/getinfo', cache('30 seconds'), asyncRoute((req, res) => { + return benchmarkService.getInfo(req, res); + })); - app.get('/syncthing/meta', cache('30 seconds'), (req, res) => { - syncthingService.getMeta(req, res); - }); - app.get('/syncthing/deviceid', cache('30 seconds'), (req, res) => { - syncthingService.getDeviceIdApi(req, res); - }); - app.get('/syncthing/health', cache('30 seconds'), (req, res) => { - syncthingService.getHealth(req, res); - }); - app.get('/syncthing/system/browse/:current?', cache('30 seconds'), (req, res) => { - syncthingService.systemBrowse(req, res); - }); - app.get('/syncthing/system/connections', cache('30 seconds'), (req, res) => { - syncthingService.systemConnections(req, res); - }); - app.get('/syncthing/system/debug/:enable?/:disable?', cache('30 seconds'), (req, res) => { - syncthingService.systemDebug(req, res); - }); - app.get('/syncthing/system/discovery/:device?/:addr?', cache('30 seconds'), (req, res) => { - syncthingService.systemDiscovery(req, res); - }); - app.get('/syncthing/system/error/clear', cache('30 seconds'), (req, res) => { - syncthingService.systemErrorClear(req, res); - }); - app.get('/syncthing/system/error/:message?', cache('30 seconds'), (req, res) => { - syncthingService.systemError(req, res); - }); - app.get('/syncthing/system/log/:since?', cache('30 seconds'), (req, res) => { - syncthingService.systemLog(req, res); - }); - app.get('/syncthing/system/logtxt/:since?', cache('30 seconds'), (req, res) => { - syncthingService.systemLogTxt(req, res); - }); - app.get('/syncthing/system/paths', cache('30 seconds'), (req, res) => { - syncthingService.systemPaths(req, res); - }); - app.get('/syncthing/system/pause/:device?', cache('30 seconds'), (req, res) => { - syncthingService.systemPause(req, res); - }); - app.get('/syncthing/system/ping', cache('30 seconds'), (req, res) => { - syncthingService.systemPing(req, res); - }); - app.get('/syncthing/system/reset/:folder?', cache('30 seconds'), (req, res) => { - syncthingService.systemReset(req, res); - }); - app.get('/syncthing/system/restart', cache('30 seconds'), (req, res) => { - syncthingService.systemRestart(req, res); - }); - app.get('/syncthing/system/resume/:device?', cache('30 seconds'), (req, res) => { - syncthingService.systemResume(req, res); - }); - app.get('/syncthing/system/shutdown', cache('30 seconds'), (req, res) => { - syncthingService.systemShutdown(req, res); - }); - app.get('/syncthing/system/status', cache('30 seconds'), (req, res) => { - syncthingService.systemStatus(req, res); - }); - app.get('/syncthing/system/upgrade', cache('30 seconds'), (req, res) => { - syncthingService.systemUpgrade(req, res); - }); - app.get('/syncthing/system/version', cache('30 seconds'), (req, res) => { - syncthingService.systemVersion(req, res); - }); - app.get('/syncthing/config', cache('30 seconds'), (req, res) => { - syncthingService.getConfig(req, res); - }); - app.get('/syncthing/config/restart-required', cache('30 seconds'), (req, res) => { - syncthingService.getConfigRestartRequired(req, res); - }); - app.get('/syncthing/config/folders/:id?', cache('30 seconds'), (req, res) => { - syncthingService.getConfigFolders(req, res); - }); - app.get('/syncthing/config/devices/:id?', cache('30 seconds'), (req, res) => { - syncthingService.getConfigDevices(req, res); - }); - app.get('/syncthing/config/defaults/folder', cache('30 seconds'), (req, res) => { - syncthingService.getConfigDefaultsFolder(req, res); - }); - app.get('/syncthing/config/defaults/device', cache('30 seconds'), (req, res) => { - syncthingService.getConfigDefaultsDevice(req, res); - }); - app.get('/syncthing/config/defaults/ignores', cache('30 seconds'), (req, res) => { - syncthingService.getConfigDefaultsIgnores(req, res); - }); - app.get('/syncthing/config/options', cache('30 seconds'), (req, res) => { - syncthingService.getConfigOptions(req, res); - }); - app.get('/syncthing/config/ldap', cache('30 seconds'), (req, res) => { - syncthingService.getConfigLdap(req, res); - }); - app.get('/syncthing/config/gui', cache('30 seconds'), (req, res) => { - syncthingService.getConfigGui(req, res); - }); - app.get('/syncthing/stats/device', cache('30 seconds'), (req, res) => { - syncthingService.statsDevice(req, res); - }); - app.get('/syncthing/stats/folder', cache('30 seconds'), (req, res) => { - syncthingService.statsFolder(req, res); - }); - app.get('/syncthing/cluster/pending/devices', cache('30 seconds'), (req, res) => { - syncthingService.getClusterPendigDevices(req, res); - }); - app.get('/syncthing/cluster/pending/folders', cache('30 seconds'), (req, res) => { - syncthingService.getClusterPendigFolders(req, res); - }); - app.get('/syncthing/folder/errors/:folder?', cache('30 seconds'), (req, res) => { - syncthingService.getFolderErrors(req, res); - }); - app.get('/syncthing/folder/versions/:folder?', cache('30 seconds'), (req, res) => { - syncthingService.getFolderVersions(req, res); - }); - app.get('/syncthing/db/browse/:folder?/:levels?/:prefix?', cache('30 seconds'), (req, res) => { - syncthingService.getDbBrowse(req, res); - }); - app.get('/syncthing/db/completion/:folder?/:device?', cache('30 seconds'), (req, res) => { - syncthingService.getDbCompletion(req, res); - }); - app.get('/syncthing/db/file/:folder?/:file?', cache('30 seconds'), (req, res) => { - syncthingService.getDbFile(req, res); - }); - app.get('/syncthing/db/ignores/:folder?', cache('30 seconds'), (req, res) => { - syncthingService.getDbIgnores(req, res); - }); - app.get('/syncthing/db/localchanged/:folder?', cache('30 seconds'), (req, res) => { - syncthingService.getDbLocalchanged(req, res); - }); - app.get('/syncthing/db/need/:folder?', cache('30 seconds'), (req, res) => { - syncthingService.getDbNeed(req, res); - }); - app.get('/syncthing/db/remoteneed/:folder?/:device?', cache('30 seconds'), (req, res) => { - syncthingService.getDbRemoteNeed(req, res); - }); - app.get('/syncthing/db/status/:folder?', cache('30 seconds'), (req, res) => { - syncthingService.getDbStatus(req, res); - }); - app.get('/syncthing/events/disk', cache('30 seconds'), (req, res) => { - syncthingService.getEventsDisk(req, res); - }); - app.get('/syncthing/events/:events?/:since?/:limit?/:timeout?', cache('30 seconds'), (req, res) => { - syncthingService.getEvents(req, res); - }); - app.get('/syncthing/svc/random/string/:length?', cache('30 seconds'), (req, res) => { - syncthingService.getSvcRandomString(req, res); - }); - app.get('/syncthing/svc/report', cache('30 seconds'), (req, res) => { - syncthingService.getSvcReport(req, res); - }); - app.get('/syncthing/svc/:deviceid?', cache('30 seconds'), (req, res) => { - syncthingService.getSvcDeviceID(req, res); - }); - app.get('/syncthing/debug/peercompletion', cache('30 seconds'), (req, res) => { - syncthingService.debugPeerCompletion(req, res); - }); - app.get('/syncthing/debug/httpmetrics', cache('30 seconds'), (req, res) => { - syncthingService.debugHttpmetrics(req, res); - }); - app.get('/syncthing/debug/cpuprof', cache('30 seconds'), (req, res) => { - syncthingService.debugCpuprof(req, res); - }); - app.get('/syncthing/debug/heapprof', cache('30 seconds'), (req, res) => { - syncthingService.debugHeapprof(req, res); - }); - app.get('/syncthing/debug/support', cache('30 seconds'), (req, res) => { - syncthingService.debugSupport(req, res); - }); - app.get('/syncthing/debug/file', cache('30 seconds'), (req, res) => { - syncthingService.debugFile(req, res); - }); + app.get('/syncthing/meta', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getMetaApi(req, res); + })); + app.get('/syncthing/deviceid', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getDeviceIdApi(req, res); + })); + app.get('/syncthing/health', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getHealthApi(req, res); + })); + app.get('/syncthing/system/browse/:current?', asyncRoute((req, res) => { + return syncthingService.systemBrowse(req, res); + })); + app.get('/syncthing/system/connections', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.systemConnections(req, res); + })); + app.get('/syncthing/system/debug/:enable?/:disable?', asyncRoute((req, res) => { + return syncthingService.systemDebug(req, res); + })); + app.get('/syncthing/system/discovery/:device?/:addr?', asyncRoute((req, res) => { + return syncthingService.systemDiscovery(req, res); + })); + app.get('/syncthing/system/error/clear', asyncRoute((req, res) => { + return syncthingService.systemErrorClear(req, res); + })); + app.get('/syncthing/system/error/:message?', asyncRoute((req, res) => { + return syncthingService.systemError(req, res); + })); + app.get('/syncthing/system/log/:since?', asyncRoute((req, res) => { + return syncthingService.systemLog(req, res); + })); + app.get('/syncthing/system/logtxt/:since?', asyncRoute((req, res) => { + return syncthingService.systemLogTxt(req, res); + })); + app.get('/syncthing/system/paths', asyncRoute((req, res) => { + return syncthingService.systemPaths(req, res); + })); + app.get('/syncthing/system/pause/:device?', asyncRoute((req, res) => { + return syncthingService.systemPauseApi(req, res); + })); + app.get('/syncthing/system/ping', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.systemPingApi(req, res); + })); + app.get('/syncthing/system/reset/:folder?', asyncRoute((req, res) => { + return syncthingService.systemReset(req, res); + })); + app.get('/syncthing/system/restart', asyncRoute((req, res) => { + return syncthingService.systemRestartApi(req, res); + })); + app.get('/syncthing/system/resume/:device?', asyncRoute((req, res) => { + return syncthingService.systemResumeApi(req, res); + })); + app.get('/syncthing/system/shutdown', asyncRoute((req, res) => { + return syncthingService.systemShutdown(req, res); + })); + app.get('/syncthing/system/status', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.systemStatus(req, res); + })); + app.get('/syncthing/system/upgrade', asyncRoute((req, res) => { + return syncthingService.systemUpgrade(req, res); + })); + app.get('/syncthing/system/version', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.systemVersionApi(req, res); + })); + app.get('/syncthing/config', asyncRoute((req, res) => { + return syncthingService.getConfigApi(req, res); + })); + app.get('/syncthing/config/restart-required', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getConfigRestartRequired(req, res); + })); + app.get('/syncthing/config/folders/:id?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getConfigFoldersApi(req, res); + })); + app.get('/syncthing/config/devices/:id?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getConfigDevicesApi(req, res); + })); + app.get('/syncthing/config/defaults/folder', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getConfigDefaultsFolderApi(req, res); + })); + app.get('/syncthing/config/defaults/device', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getConfigDefaultsDevice(req, res); + })); + app.get('/syncthing/config/defaults/ignores', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getConfigDefaultsIgnores(req, res); + })); + app.get('/syncthing/config/options', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getConfigOptionsApi(req, res); + })); + app.get('/syncthing/config/ldap', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getConfigLdap(req, res); + })); + app.get('/syncthing/config/gui', asyncRoute((req, res) => { + return syncthingService.getConfigGuiApi(req, res); + })); + app.get('/syncthing/stats/device', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.statsDevice(req, res); + })); + app.get('/syncthing/stats/folder', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.statsFolder(req, res); + })); + app.get('/syncthing/cluster/pending/devices', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getClusterPendigDevices(req, res); + })); + app.get('/syncthing/cluster/pending/folders', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getClusterPendigFolders(req, res); + })); + app.get('/syncthing/folder/errors/:folder?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getFolderErrors(req, res); + })); + app.get('/syncthing/folder/versions/:folder?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getFolderVersions(req, res); + })); + app.get('/syncthing/db/browse/:folder?/:levels?/:prefix?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getDbBrowse(req, res); + })); + app.get('/syncthing/db/completion/:folder?/:device?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getDbCompletionApi(req, res); + })); + app.get('/syncthing/db/file/:folder?/:file?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getDbFile(req, res); + })); + app.get('/syncthing/db/ignores/:folder?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getDbIgnores(req, res); + })); + app.get('/syncthing/db/localchanged/:folder?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getDbLocalchanged(req, res); + })); + app.get('/syncthing/db/need/:folder?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getDbNeed(req, res); + })); + app.get('/syncthing/db/remoteneed/:folder?/:device?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getDbRemoteNeed(req, res); + })); + app.get('/syncthing/db/status/:folder?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getDbStatusApi(req, res); + })); + app.get('/syncthing/events/disk', asyncRoute((req, res) => { + return syncthingService.getEventsDisk(req, res); + })); + app.get('/syncthing/events/:events?/:since?/:limit?/:timeout?', asyncRoute((req, res) => { + return syncthingService.getEventsApi(req, res); + })); + app.get('/syncthing/svc/random/string/:length?', asyncRoute((req, res) => { + return syncthingService.getSvcRandomString(req, res); + })); + app.get('/syncthing/svc/report', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getSvcReport(req, res); + })); + app.get('/syncthing/svc/:deviceid?', cache('30 seconds'), asyncRoute((req, res) => { + return syncthingService.getSvcDeviceID(req, res); + })); + app.get('/syncthing/debug/peercompletion', asyncRoute((req, res) => { + return syncthingService.debugPeerCompletion(req, res); + })); + app.get('/syncthing/debug/httpmetrics', asyncRoute((req, res) => { + return syncthingService.debugHttpmetrics(req, res); + })); + app.get('/syncthing/debug/cpuprof', asyncRoute((req, res) => { + return syncthingService.debugCpuprof(req, res); + })); + app.get('/syncthing/debug/heapprof', asyncRoute((req, res) => { + return syncthingService.debugHeapprof(req, res); + })); + app.get('/syncthing/debug/support', asyncRoute((req, res) => { + return syncthingService.debugSupport(req, res); + })); + app.get('/syncthing/debug/file', asyncRoute((req, res) => { + return syncthingService.debugFile(req, res); + })); // BACKUP & RESTORE - app.get('/backup/getvolumedataofcomponent/:appname?/:component?/:multiplier?/:decimal?/:fields?', (req, res) => { - backupRestoreService.getVolumeDataOfComponent(req, res); - }); - app.get('/backup/getremotefilesize/:fileurl?/:multiplier?/:decimal?/:number?/:appname?', (req, res) => { - backupRestoreService.getRemoteFileSize(req, res); - }); - app.get('/backup/getlocalbackuplist/:path?/:multiplier?/:decimal?/:number?/:appname?', (req, res) => { - backupRestoreService.getLocalBackupList(req, res); - }); - app.get('/backup/removebackupfile/:filepath?/:appname?', (req, res) => { - backupRestoreService.removeBackupFile(req, res); - }); - app.get('/backup/downloadlocalfile/:filepath?/:appname?', (req, res) => { - backupRestoreService.downloadLocalFile(req, res); - }); - app.post('/apps/appendbackuptask', (req, res) => { - advancedWorkflows.appendBackupTask(req, res); - }); + app.get('/backup/getvolumedataofcomponent/:appname?/:component?/:multiplier?/:decimal?/:fields?', asyncRoute((req, res) => { + return backupRestoreService.getVolumeDataOfComponent(req, res); + })); + app.get('/backup/getremotefilesize/:fileurl?/:multiplier?/:decimal?/:number?/:appname?', asyncRoute((req, res) => { + return backupRestoreService.getRemoteFileSize(req, res); + })); + app.get('/backup/getlocalbackuplist/:path?/:multiplier?/:decimal?/:number?/:appname?', asyncRoute((req, res) => { + return backupRestoreService.getLocalBackupList(req, res); + })); + app.get('/backup/removebackupfile/:filepath?/:appname?', asyncRoute((req, res) => { + return backupRestoreService.removeBackupFile(req, res); + })); + app.get('/backup/downloadlocalfile/:filepath?/:appname?', asyncRoute((req, res) => { + return backupRestoreService.downloadLocalFile(req, res); + })); + app.post('/apps/appendbackuptask', asyncRoute((req, res) => { + return advancedWorkflows.appendBackupTask(req, res); + })); - app.post('/apps/appendrestoretask', (req, res) => { - advancedWorkflows.appendRestoreTask(req, res); - }); + app.post('/apps/appendrestoretask', asyncRoute((req, res) => { + return advancedWorkflows.appendRestoreTask(req, res); + })); - app.post('/ioutils/fileupload/:type?/:appname?/:component?/:folder?/:filename?', (req, res) => { - IOUtils.fileUpload(req, res); - }); + app.post('/ioutils/fileupload/:type?/:appname?/:component?/:folder?/:filename?', requireBootSettled, asyncRoute((req, res) => { + return fileSystemManager.uploadAppsFiles(req, res); + })); // GET PROTECTED API - Fluxnode Owner - app.get('/daemon/stop', (req, res) => { - daemonServiceControlRpcs.stop(req, res); - }); - app.get('/daemon/reindex', (req, res) => { - fluxService.reindexDaemon(req, res); - }); - app.get('/daemon/createfluxnodekey', (req, res) => { - daemonServiceNodeRpcs.createFluxNodeKey(req, res); - }); - app.get('/daemon/createzelnodekey', (req, res) => { // DEPRECATED - daemonServiceNodeRpcs.createFluxNodeKey(req, res); - }); - app.get('/daemon/listfluxnodeconf/:filter?', (req, res) => { - daemonServiceNodeRpcs.listFluxNodeConf(req, res); - }); - app.get('/daemon/listzelnodeconf/:filter?', (req, res) => { // DEPRECATED - daemonServiceNodeRpcs.listFluxNodeConf(req, res); - }); - app.get('/daemon/getfluxnodeoutputs', (req, res) => { - daemonServiceNodeRpcs.getFluxNodeOutputs(req, res); - }); - app.get('/daemon/getzelnodeoutputs', (req, res) => { // DEPRECATED - daemonServiceNodeRpcs.getFluxNodeOutputs(req, res); - }); - app.get('/daemon/startfluxnode/:set?/:lockwallet?/:alias?', (req, res) => { - daemonServiceNodeRpcs.startFluxNode(req, res); - }); - app.get('/daemon/startzelnode/:set?/:lockwallet?/:alias?', (req, res) => { // DEPRECATED - daemonServiceNodeRpcs.startFluxNode(req, res); - }); - app.get('/daemon/startdeterministicfluxnode/:alias?/:lockwallet?', (req, res) => { - daemonServiceNodeRpcs.startDeterministicFluxNode(req, res); - }); - app.get('/daemon/startdeterministiczelnode/:alias?/:lockwallet?', (req, res) => { // DEPRECATED - daemonServiceNodeRpcs.startDeterministicFluxNode(req, res); - }); - app.get('/daemon/verifychain/:checklevel?/:numblocks?', (req, res) => { - daemonServiceBlockchainRpcs.verifyChain(req, res); - }); - app.get('/daemon/addnode/:node?/:command?', (req, res) => { - daemonServiceNetworkRpcs.addNode(req, res); - }); - app.get('/daemon/clearbanned', (req, res) => { - daemonServiceNetworkRpcs.clearBanned(req, res); - }); - app.get('/daemon/disconnectnode/:node?', (req, res) => { - daemonServiceNetworkRpcs.disconnectNode(req, res); - }); - app.get('/daemon/getaddednodeinfo/:dns?/:node?', (req, res) => { - daemonServiceNetworkRpcs.getAddedNodeInfo(req, res); - }); - app.get('/daemon/setban/:ip?/:command?/:bantime?/:absolute?', (req, res) => { - daemonServiceNetworkRpcs.setBan(req, res); - }); - app.get('/daemon/signrawtransaction/:hexstring?/:prevtxs?/:privatekeys?/:sighashtype?/:branchid?', (req, res) => { - daemonServiceTransactionRpcs.signRawTransaction(req, res); - }); - app.get('/daemon/addmultisigaddress/:n?/:keysobject?', (req, res) => { - daemonServiceWalletRpcs.addMultiSigAddress(req, res); - }); - app.get('/daemon/backupwallet/:destination?', (req, res) => { - daemonServiceWalletRpcs.backupWallet(req, res); - }); - app.get('/daemon/dumpprivkey/:taddr?', (req, res) => { - daemonServiceWalletRpcs.dumpPrivKey(req, res); - }); - app.get('/daemon/getbalance/:minconf?/:includewatchonly?', (req, res) => { - daemonServiceWalletRpcs.getBalance(req, res); - }); - app.get('/daemon/getnewaddress', (req, res) => { - daemonServiceWalletRpcs.getNewAddress(req, res); - }); - app.get('/daemon/getrawchangeaddress', (req, res) => { - daemonServiceWalletRpcs.getRawChangeAddress(req, res); - }); - app.get('/daemon/getreceivedbyaddress/:fluxaddress?/:minconf?', (req, res) => { - daemonServiceWalletRpcs.getReceivedByAddress(req, res); - }); - app.get('/daemon/getunconfirmedbalance', (req, res) => { - daemonServiceWalletRpcs.getUnconfirmedBalance(req, res); - }); - app.get('/daemon/getwalletinfo', (req, res) => { - daemonServiceWalletRpcs.getWalletInfo(req, res); - }); - app.get('/daemon/importaddress/:address?/:label?/:rescan?', (req, res) => { - daemonServiceWalletRpcs.importAddress(req, res); - }); - app.get('/daemon/importprivkey/:fluxprivkey?/:label?/:rescan?', (req, res) => { - daemonServiceWalletRpcs.importPrivKey(req, res); - }); - app.get('/daemon/importwallet/:filename?', (req, res) => { - daemonServiceWalletRpcs.importWallet(req, res); - }); - app.get('/daemon/keypoolrefill/:newsize?', (req, res) => { - daemonServiceWalletRpcs.keyPoolRefill(req, res); - }); - app.get('/daemon/listaddressgroupings', (req, res) => { - daemonServiceWalletRpcs.listAddressGroupings(req, res); - }); - app.get('/daemon/listlockunspent', (req, res) => { - daemonServiceWalletRpcs.listLockUnspent(req, res); - }); - app.get('/daemon/listreceivedbyaddress/:minconf?/:includeempty?/:includewatchonly?', (req, res) => { - daemonServiceWalletRpcs.listReceivedByAddress(req, res); - }); - app.get('/daemon/listsinceblock/:blockhash?/:targetconfirmations?/:includewatchonly?', (req, res) => { - daemonServiceWalletRpcs.listSinceBlock(req, res); - }); - app.get('/daemon/listtransactions/:count?/:from?/:includewatchonly?', (req, res) => { - daemonServiceWalletRpcs.listTransactions(req, res); - }); - app.get('/daemon/listunspent/:minconf?/:maxconf?/:addresses?', (req, res) => { - daemonServiceWalletRpcs.listUnspent(req, res); - }); - app.get('/daemon/lockunspent/:unlock?/:transactions?', (req, res) => { - daemonServiceWalletRpcs.lockUnspent(req, res); - }); - app.get('/daemon/rescanblockchain/:startheight?', (req, res) => { - daemonServiceWalletRpcs.rescanBlockchain(req, res); - }); - app.get('/daemon/sendfrom/:tofluxaddress?/:amount?/:minconf?/:comment?/:commentto?', (req, res) => { - daemonServiceWalletRpcs.sendFrom(req, res); - }); - app.get('/daemon/sendmany/:amounts?/:minconf?/:comment?/:substractfeefromamount?', (req, res) => { - daemonServiceWalletRpcs.sendMany(req, res); - }); - app.get('/daemon/sendtoaddress/:fluxaddress?/:amount?/:comment?/:commentto?/:substractfeefromamount?', (req, res) => { - daemonServiceWalletRpcs.sendToAddress(req, res); - }); - app.get('/daemon/settxfee/:amount?', (req, res) => { - daemonServiceWalletRpcs.setTxFee(req, res); - }); - app.get('/daemon/signmessage/:taddr?/:message?', (req, res) => { - daemonServiceWalletRpcs.signMessage(req, res); - }); - app.get('/daemon/zexportkey/:zaddr?', (req, res) => { - daemonServiceZcashRpcs.zExportKey(req, res); - }); - app.get('/daemon/zexportviewingkey/:zaddr?', (req, res) => { - daemonServiceZcashRpcs.zExportViewingKey(req, res); - }); - app.get('/daemon/zgetbalance/:address?/:minconf?', (req, res) => { - daemonServiceZcashRpcs.zGetBalance(req, res); - }); - app.get('/daemon/zgetmigrationstatus', (req, res) => { - daemonServiceZcashRpcs.zGetMigrationStatus(req, res); - }); - app.get('/daemon/zgetnewaddress/:type?', (req, res) => { - daemonServiceZcashRpcs.zGetNewAddress(req, res); - }); - app.get('/daemon/zgetoperationresult/:operationid?', (req, res) => { - daemonServiceZcashRpcs.zGetOperationResult(req, res); - }); - app.get('/daemon/zgetoperationstatus/:operationid?', (req, res) => { - daemonServiceZcashRpcs.zGetOperationStatus(req, res); - }); - app.get('/daemon/zgettotalbalance/:minconf?/:includewatchonly?', (req, res) => { - daemonServiceZcashRpcs.zGetTotalBalance(req, res); - }); - app.get('/daemon/zimportkey/:zkey?/:rescan?/:startheight?', (req, res) => { - daemonServiceZcashRpcs.zImportKey(req, res); - }); - app.get('/daemon/zimportviewingkey/:vkey?/:rescan?/:startheight?', (req, res) => { - daemonServiceZcashRpcs.zImportViewingKey(req, res); - }); - app.get('/daemon/zimportwallet/:filename?', (req, res) => { - daemonServiceZcashRpcs.zImportWallet(req, res); - }); - app.get('/daemon/zlistaddresses/:includewatchonly?', (req, res) => { - daemonServiceZcashRpcs.zListAddresses(req, res); - }); - app.get('/daemon/zlistoperationids', (req, res) => { - daemonServiceZcashRpcs.zListOperationIds(req, res); - }); - app.get('/daemon/zlistreceivedbyaddress/:address?/:minconf?', (req, res) => { - daemonServiceZcashRpcs.zListReceivedByAddress(req, res); - }); - app.get('/daemon/zlistunspent/:minconf?/:maxonf?/:includewatchonly?/:addresses?', (req, res) => { - daemonServiceZcashRpcs.zListUnspent(req, res); - }); - app.get('/daemon/zmergetoaddress/:fromaddresses?/:toaddress?/:fee?/:transparentlimit?/:shieldedlimit?/:memo?', (req, res) => { - daemonServiceZcashRpcs.zMergeToAddress(req, res); - }); - app.get('/daemon/zsendmany/:fromaddress?/:amounts?/:minconf?/:fee?', (req, res) => { - daemonServiceZcashRpcs.zSendMany(req, res); - }); - app.get('/daemon/zsetmigration/:enabled?', (req, res) => { - daemonServiceZcashRpcs.zSetMigration(req, res); - }); - app.get('/daemon/zshieldcoinbase/:fromaddress?/:toaddress?/:fee?/:limit?', (req, res) => { - daemonServiceZcashRpcs.zShieldCoinBase(req, res); - }); - app.get('/daemon/zcrawjoinsplit/:rawtx?/:inputs?/:outputs?/:vpubold?/:vpubnew?', (req, res) => { - daemonServiceZcashRpcs.zcRawJoinSplit(req, res); - }); - app.get('/daemon/zcrawkeygen', (req, res) => { - daemonServiceZcashRpcs.zcRawKeygen(req, res); - }); - app.get('/daemon/zcrawreceive/:zcsecretkey?/:encryptednote?', (req, res) => { - daemonServiceZcashRpcs.zcRawReceive(req, res); - }); - app.get('/daemon/zcsamplejoinsplit', (req, res) => { - daemonServiceZcashRpcs.zcSampleJoinSplit(req, res); - }); - app.get('/daemon/getaddresstxids/:address?/:start?/:end?', (req, res) => { - daemonServiceAddressRpcs.getSingleAddresssTxids(req, res); - }); - app.get('/daemon/getaddressbalance/:address?', (req, res) => { - daemonServiceAddressRpcs.getSingleAddressBalance(req, res); - }); - app.get('/daemon/getaddressdeltas/:address?/:start?/:end?/:chaininfo?', (req, res) => { - daemonServiceAddressRpcs.getSingleAddressDeltas(req, res); - }); - app.get('/daemon/getaddressutxos/:address?/:chaininfo?', (req, res) => { - daemonServiceAddressRpcs.getSingleAddressUtxos(req, res); - }); - app.get('/daemon/getaddressmempool/:address?', (req, res) => { - daemonServiceAddressRpcs.getSingleAddressMempool(req, res); - }); + app.get('/daemon/stop', asyncRoute((req, res) => { + return daemonServiceControlRpcs.stop(req, res); + })); + app.get('/daemon/reindex', asyncRoute((req, res) => { + return fluxService.reindexDaemon(req, res); + })); + app.get('/daemon/createfluxnodekey', asyncRoute((req, res) => { + return daemonServiceNodeRpcs.createFluxNodeKey(req, res); + })); + app.get('/daemon/createzelnodekey', asyncRoute((req, res) => { // DEPRECATED + return daemonServiceNodeRpcs.createFluxNodeKey(req, res); + })); + app.get('/daemon/listfluxnodeconf/:filter?', asyncRoute((req, res) => { + return daemonServiceNodeRpcs.listFluxNodeConf(req, res); + })); + app.get('/daemon/listzelnodeconf/:filter?', asyncRoute((req, res) => { // DEPRECATED + return daemonServiceNodeRpcs.listFluxNodeConf(req, res); + })); + app.get('/daemon/getfluxnodeoutputs', asyncRoute((req, res) => { + return daemonServiceNodeRpcs.getFluxNodeOutputs(req, res); + })); + app.get('/daemon/getzelnodeoutputs', asyncRoute((req, res) => { // DEPRECATED + return daemonServiceNodeRpcs.getFluxNodeOutputs(req, res); + })); + app.get('/daemon/startfluxnode/:set?/:lockwallet?/:alias?', asyncRoute((req, res) => { + return daemonServiceNodeRpcs.startFluxNode(req, res); + })); + app.get('/daemon/startzelnode/:set?/:lockwallet?/:alias?', asyncRoute((req, res) => { // DEPRECATED + return daemonServiceNodeRpcs.startFluxNode(req, res); + })); + app.get('/daemon/startdeterministicfluxnode/:alias?/:lockwallet?', asyncRoute((req, res) => { + return daemonServiceNodeRpcs.startDeterministicFluxNode(req, res); + })); + app.get('/daemon/startdeterministiczelnode/:alias?/:lockwallet?', asyncRoute((req, res) => { // DEPRECATED + return daemonServiceNodeRpcs.startDeterministicFluxNode(req, res); + })); + app.get('/daemon/verifychain/:checklevel?/:numblocks?', asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.verifyChain(req, res); + })); + app.get('/daemon/addnode/:node?/:command?', asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.addNode(req, res); + })); + app.get('/daemon/clearbanned', asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.clearBanned(req, res); + })); + app.get('/daemon/disconnectnode/:node?', asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.disconnectNode(req, res); + })); + app.get('/daemon/getaddednodeinfo/:dns?/:node?', asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.getAddedNodeInfo(req, res); + })); + app.get('/daemon/setban/:ip?/:command?/:bantime?/:absolute?', asyncRoute((req, res) => { + return daemonServiceNetworkRpcs.setBan(req, res); + })); + app.get('/daemon/signrawtransaction/:hexstring?/:prevtxs?/:privatekeys?/:sighashtype?/:branchid?', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.signRawTransaction(req, res); + })); + app.get('/daemon/addmultisigaddress/:n?/:keysobject?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.addMultiSigAddress(req, res); + })); + app.get('/daemon/backupwallet/:destination?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.backupWallet(req, res); + })); + app.get('/daemon/dumpprivkey/:taddr?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.dumpPrivKey(req, res); + })); + app.get('/daemon/getbalance/:minconf?/:includewatchonly?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.getBalance(req, res); + })); + app.get('/daemon/getnewaddress', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.getNewAddress(req, res); + })); + app.get('/daemon/getrawchangeaddress', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.getRawChangeAddress(req, res); + })); + app.get('/daemon/getreceivedbyaddress/:fluxaddress?/:minconf?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.getReceivedByAddress(req, res); + })); + app.get('/daemon/getunconfirmedbalance', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.getUnconfirmedBalance(req, res); + })); + app.get('/daemon/getwalletinfo', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.getWalletInfo(req, res); + })); + app.get('/daemon/importaddress/:address?/:label?/:rescan?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.importAddress(req, res); + })); + app.get('/daemon/importprivkey/:fluxprivkey?/:label?/:rescan?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.importPrivKey(req, res); + })); + app.get('/daemon/importwallet/:filename?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.importWallet(req, res); + })); + app.get('/daemon/keypoolrefill/:newsize?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.keyPoolRefill(req, res); + })); + app.get('/daemon/listaddressgroupings', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.listAddressGroupings(req, res); + })); + app.get('/daemon/listlockunspent', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.listLockUnspent(req, res); + })); + app.get('/daemon/listreceivedbyaddress/:minconf?/:includeempty?/:includewatchonly?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.listReceivedByAddress(req, res); + })); + app.get('/daemon/listsinceblock/:blockhash?/:targetconfirmations?/:includewatchonly?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.listSinceBlock(req, res); + })); + app.get('/daemon/listtransactions/:count?/:from?/:includewatchonly?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.listTransactions(req, res); + })); + app.get('/daemon/listunspent/:minconf?/:maxconf?/:addresses?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.listUnspent(req, res); + })); + app.get('/daemon/lockunspent/:unlock?/:transactions?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.lockUnspent(req, res); + })); + app.get('/daemon/rescanblockchain/:startheight?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.rescanBlockchain(req, res); + })); + app.get('/daemon/sendfrom/:tofluxaddress?/:amount?/:minconf?/:comment?/:commentto?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.sendFrom(req, res); + })); + app.get('/daemon/sendmany/:amounts?/:minconf?/:comment?/:substractfeefromamount?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.sendMany(req, res); + })); + app.get('/daemon/sendtoaddress/:fluxaddress?/:amount?/:comment?/:commentto?/:substractfeefromamount?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.sendToAddress(req, res); + })); + app.get('/daemon/settxfee/:amount?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.setTxFee(req, res); + })); + app.get('/daemon/signmessage/:taddr?/:message?', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.signMessage(req, res); + })); + app.get('/daemon/zexportkey/:zaddr?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zExportKey(req, res); + })); + app.get('/daemon/zexportviewingkey/:zaddr?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zExportViewingKey(req, res); + })); + app.get('/daemon/zgetbalance/:address?/:minconf?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zGetBalance(req, res); + })); + app.get('/daemon/zgetmigrationstatus', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zGetMigrationStatus(req, res); + })); + app.get('/daemon/zgetnewaddress/:type?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zGetNewAddress(req, res); + })); + app.get('/daemon/zgetoperationresult/:operationid?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zGetOperationResult(req, res); + })); + app.get('/daemon/zgetoperationstatus/:operationid?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zGetOperationStatus(req, res); + })); + app.get('/daemon/zgettotalbalance/:minconf?/:includewatchonly?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zGetTotalBalance(req, res); + })); + app.get('/daemon/zimportkey/:zkey?/:rescan?/:startheight?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zImportKey(req, res); + })); + app.get('/daemon/zimportviewingkey/:vkey?/:rescan?/:startheight?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zImportViewingKey(req, res); + })); + app.get('/daemon/zimportwallet/:filename?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zImportWallet(req, res); + })); + app.get('/daemon/zlistaddresses/:includewatchonly?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zListAddresses(req, res); + })); + app.get('/daemon/zlistoperationids', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zListOperationIds(req, res); + })); + app.get('/daemon/zlistreceivedbyaddress/:address?/:minconf?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zListReceivedByAddress(req, res); + })); + app.get('/daemon/zlistunspent/:minconf?/:maxonf?/:includewatchonly?/:addresses?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zListUnspent(req, res); + })); + app.get('/daemon/zmergetoaddress/:fromaddresses?/:toaddress?/:fee?/:transparentlimit?/:shieldedlimit?/:memo?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zMergeToAddress(req, res); + })); + app.get('/daemon/zsendmany/:fromaddress?/:amounts?/:minconf?/:fee?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zSendMany(req, res); + })); + app.get('/daemon/zsetmigration/:enabled?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zSetMigration(req, res); + })); + app.get('/daemon/zshieldcoinbase/:fromaddress?/:toaddress?/:fee?/:limit?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zShieldCoinBase(req, res); + })); + app.get('/daemon/zcrawjoinsplit/:rawtx?/:inputs?/:outputs?/:vpubold?/:vpubnew?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zcRawJoinSplit(req, res); + })); + app.get('/daemon/zcrawkeygen', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zcRawKeygen(req, res); + })); + app.get('/daemon/zcrawreceive/:zcsecretkey?/:encryptednote?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zcRawReceive(req, res); + })); + app.get('/daemon/zcsamplejoinsplit', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zcSampleJoinSplit(req, res); + })); + app.get('/daemon/getaddresstxids/:address?/:start?/:end?', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getSingleAddresssTxids(req, res); + })); + app.get('/daemon/getaddressbalance/:address?', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getSingleAddressBalance(req, res); + })); + app.get('/daemon/getaddressdeltas/:address?/:start?/:end?/:chaininfo?', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getSingleAddressDeltas(req, res); + })); + app.get('/daemon/getaddressutxos/:address?/:chaininfo?', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getSingleAddressUtxos(req, res); + })); + app.get('/daemon/getaddressmempool/:address?', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getSingleAddressMempool(req, res); + })); - app.get('/id/loggedusers', (req, res) => { - idService.loggedUsers(req, res); - }); - app.get('/id/activeloginphrases', (req, res) => { - idService.activeLoginPhrases(req, res); - }); - app.get('/id/logoutallusers', (req, res) => { - idService.logoutAllUsers(req, res); - }); - app.get('/zelid/loggedusers', (req, res) => { // DEPRECATED - idService.loggedUsers(req, res); - }); - app.get('/zelid/activeloginphrases', (req, res) => { // DEPRECATED - idService.activeLoginPhrases(req, res); - }); - app.get('/zelid/logoutallusers', (req, res) => { // DEPRECATED - idService.logoutAllUsers(req, res); - }); + app.get('/id/loggedusers', asyncRoute((req, res) => { + return idService.loggedUsers(req, res); + })); + app.get('/id/activeloginphrases', asyncRoute((req, res) => { + return idService.activeLoginPhrases(req, res); + })); + app.get('/id/logoutallusers', asyncRoute((req, res) => { + return idService.logoutAllUsers(req, res); + })); + app.get('/zelid/loggedusers', asyncRoute((req, res) => { // DEPRECATED + return idService.loggedUsers(req, res); + })); + app.get('/zelid/activeloginphrases', asyncRoute((req, res) => { // DEPRECATED + return idService.activeLoginPhrases(req, res); + })); + app.get('/zelid/logoutallusers', asyncRoute((req, res) => { // DEPRECATED + return idService.logoutAllUsers(req, res); + })); - app.get('/flux/adjustkadena/:account?/:chainid?', (req, res) => { // note this essentially rebuilds flux use with caution! - fluxService.adjustKadenaAccount(req, res); - }); - app.get('/flux/adjustrouterip/:routerip?', (req, res) => { // note this essentially rebuilds flux use with caution! - fluxService.adjustRouterIP(req, res); - }); - app.post('/flux/adjustblockedports', (req, res) => { // note this essentially rebuilds flux use with caution! - fluxService.adjustBlockedPorts(req, res); - }); - app.get('/flux/adjustapiport/:apiport?', (req, res) => { // note this essentially rebuilds flux use with caution! - fluxService.adjustAPIPort(req, res); - }); - app.post('/flux/adjustblockedrepositories', (req, res) => { // note this essentially rebuilds flux use with caution! - fluxService.adjustBlockedRepositories(req, res); - }); - app.get('/flux/reindexdaemon', (req, res) => { - fluxService.reindexDaemon(req, res); - }); + app.get('/flux/adjustkadena/:account?/:chainid?', asyncRoute((req, res) => { // note this essentially rebuilds flux use with caution! + return fluxService.adjustKadenaAccount(req, res); + })); + app.get('/flux/adjustrouterip/:routerip?', asyncRoute((req, res) => { // note this essentially rebuilds flux use with caution! + return fluxService.adjustRouterIP(req, res); + })); + app.post('/flux/adjustblockedports', asyncRoute((req, res) => { // note this essentially rebuilds flux use with caution! + return fluxService.adjustBlockedPorts(req, res); + })); + app.get('/flux/adjustapiport/:apiport?', asyncRoute((req, res) => { // note this essentially rebuilds flux use with caution! + return fluxService.adjustAPIPort(req, res); + })); + app.post('/flux/adjustblockedrepositories', asyncRoute((req, res) => { // note this essentially rebuilds flux use with caution! + return fluxService.adjustBlockedRepositories(req, res); + })); + app.get('/flux/reindexdaemon', asyncRoute((req, res) => { + return fluxService.reindexDaemon(req, res); + })); - app.get('/benchmark/signfluxnodetransaction/:hexstring?', (req, res) => { - benchmarkService.signFluxTransaction(req, res); - }); - app.get('/benchmark/signzelnodetransaction/:hexstring?', (req, res) => { // DEPRECATED - benchmarkService.signFluxTransaction(req, res); - }); - app.get('/benchmark/stop', (req, res) => { - benchmarkService.stop(req, res); - }); + app.get('/benchmark/signfluxnodetransaction/:hexstring?', asyncRoute((req, res) => { + return benchmarkService.signFluxTransaction(req, res); + })); + app.get('/benchmark/signzelnodetransaction/:hexstring?', asyncRoute((req, res) => { // DEPRECATED + return benchmarkService.signFluxTransaction(req, res); + })); + app.get('/benchmark/stop', asyncRoute((req, res) => { + return benchmarkService.stop(req, res); + })); // GET PROTECTED API - FluxTeam - app.get('/daemon/start', (req, res) => { - fluxService.startDaemon(req, res); - }); - app.get('/daemon/restart', (req, res) => { - fluxService.restartDaemon(req, res); - }); - app.get('/daemon/ping', (req, res) => { // we do not want this to be issued by anyone. - daemonServiceNetworkRpcs.ping(req, res); - }); - app.get('/daemon/zcbenchmark/:benchmarktype?/:samplecount?', (req, res) => { - daemonServiceZcashRpcs.zcBenchmark(req, res); - }); - app.get('/daemon/startbenchmark', (req, res) => { - daemonServiceBenchmarkRpcs.startBenchmarkD(req, res); - }); - app.get('/daemon/stopbenchmark', (req, res) => { - daemonServiceBenchmarkRpcs.stopBenchmarkD(req, res); - }); + app.get('/daemon/start', asyncRoute((req, res) => { + return fluxService.startDaemon(req, res); + })); + app.get('/daemon/restart', asyncRoute((req, res) => { + return fluxService.restartDaemon(req, res); + })); + app.get('/daemon/ping', asyncRoute((req, res) => { // we do not want this to be issued by anyone. + return daemonServiceNetworkRpcs.ping(req, res); + })); + app.get('/daemon/zcbenchmark/:benchmarktype?/:samplecount?', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zcBenchmark(req, res); + })); + app.get('/daemon/startbenchmark', asyncRoute((req, res) => { + return daemonServiceBenchmarkRpcs.startBenchmarkD(req, res); + })); + app.get('/daemon/stopbenchmark', asyncRoute((req, res) => { + return daemonServiceBenchmarkRpcs.stopBenchmarkD(req, res); + })); - app.get('/flux/startbenchmark', (req, res) => { - fluxService.startBenchmark(req, res); - }); - app.get('/flux/restartbenchmark', (req, res) => { - fluxService.restartBenchmark(req, res); - }); - app.get('/flux/startdaemon', (req, res) => { - fluxService.startDaemon(req, res); - }); - app.get('/flux/restartdaemon', (req, res) => { - fluxService.restartDaemon(req, res); - }); - app.get('/flux/entermaster', (req, res) => { - fluxService.enterMaster(req, res); - }); - app.get('/flux/enterdevelopment', (req, res) => { - fluxService.enterDevelopment(req, res); - }); - app.get('/flux/updateflux', (req, res) => { // method shall be called only if flux version is obsolete. - fluxService.updateFlux(req, res); - }); - app.get('/flux/softupdateflux', (req, res) => { // method shall be called only if flux version is obsolete. - fluxService.softUpdateFlux(req, res); - }); - app.get('/flux/softupdatefluxinstall', (req, res) => { // method shall be called only if flux version is obsolete. - fluxService.softUpdateFluxInstall(req, res); - }); - app.get('/flux/hardupdateflux', (req, res) => { // method shall be called only if flux version is obsolete and updatezeflux is not working correctly - fluxService.hardUpdateFlux(req, res); - }); - app.get('/flux/rebuildhome', (req, res) => { - fluxService.rebuildHome(req, res); - }); - app.get('/flux/updatedaemon', (req, res) => { // method shall be called only if daemon version is obsolete - fluxService.updateDaemon(req, res); - }); - app.get('/flux/updatebenchmark', (req, res) => { // method shall be called only if benchamrk version is obsolete - fluxService.updateBenchmark(req, res); - }); - app.get('/flux/daemondebug', (req, res) => { - fluxService.daemonDebug(req, res); - }); - app.get('/flux/benchmarkdebug', (req, res) => { - fluxService.benchmarkDebug(req, res); - }); - app.get('/flux/taildaemondebug', (req, res) => { - fluxService.tailDaemonDebug(req, res); - }); - app.get('/flux/tailbenchmarkdebug', (req, res) => { - fluxService.tailBenchmarkDebug(req, res); - }); - app.get('/flux/errorlog', (req, res) => { - fluxService.fluxErrorLog(req, res); - }); - app.get('/flux/warnlog', (req, res) => { - fluxService.fluxWarnLog(req, res); - }); - app.get('/flux/debuglog', (req, res) => { - fluxService.fluxDebugLog(req, res); - }); - app.get('/flux/infolog', (req, res) => { - fluxService.fluxInfoLog(req, res); - }); - app.get('/flux/tailerrorlog', (req, res) => { - fluxService.tailFluxErrorLog(req, res); - }); - app.get('/flux/tailwarnlog', (req, res) => { - fluxService.tailFluxWarnLog(req, res); - }); - app.get('/flux/taildebuglog', (req, res) => { - fluxService.tailFluxDebugLog(req, res); - }); - app.get('/flux/tailinfolog', (req, res) => { - fluxService.tailFluxInfoLog(req, res); - }); + app.get('/flux/startbenchmark', asyncRoute((req, res) => { + return fluxService.startBenchmark(req, res); + })); + app.get('/flux/restartbenchmark', asyncRoute((req, res) => { + return fluxService.restartBenchmark(req, res); + })); + app.get('/flux/startdaemon', asyncRoute((req, res) => { + return fluxService.startDaemon(req, res); + })); + app.get('/flux/restartdaemon', asyncRoute((req, res) => { + return fluxService.restartDaemon(req, res); + })); + // What this node reports it is running, read from the working tree it was deployed + // from. A diagnostic for whoever switches the branch below, and only that: it is the + // node's own account of itself, so it answers what is on disk rather than settling + // whether that is what should be there. + app.get('/flux/currentbranch', asyncRoute((req, res) => { + return fluxService.getCurrentBranchApi(req, res); + })); + app.get('/flux/currentcommitid', asyncRoute((req, res) => { + return fluxService.getCurrentCommitIdApi(req, res); + })); + app.get('/flux/entermaster', asyncRoute((req, res) => { + return fluxService.enterMasterApi(req, res); + })); + app.get('/flux/enterdevelopment', asyncRoute((req, res) => { + return fluxService.enterDevelopmentApi(req, res); + })); + app.get('/flux/updateflux', asyncRoute((req, res) => { // method shall be called only if flux version is obsolete. + return fluxService.updateFlux(req, res); + })); + app.get('/flux/softupdateflux', asyncRoute((req, res) => { // method shall be called only if flux version is obsolete. + return fluxService.softUpdateFluxApi(req, res); + })); + app.get('/flux/softupdatefluxinstall', asyncRoute((req, res) => { // method shall be called only if flux version is obsolete. + return fluxService.softUpdateFluxInstallApi(req, res); + })); + app.get('/flux/hardupdateflux', asyncRoute((req, res) => { // method shall be called only if flux version is obsolete and updatezeflux is not working correctly + return fluxService.hardUpdateFlux(req, res); + })); + app.get('/flux/rebuildui', asyncRoute((req, res) => { + return fluxService.rebuildUi(req, res); + })); + app.get('/flux/updatedaemon', asyncRoute((req, res) => { // method shall be called only if daemon version is obsolete + return fluxService.updateDaemon(req, res); + })); + app.get('/flux/updatebenchmark', asyncRoute((req, res) => { // method shall be called only if benchamrk version is obsolete + return fluxService.updateBenchmark(req, res); + })); + app.get('/flux/daemondebug', asyncRoute((req, res) => { + return fluxService.daemonDebug(req, res); + })); + app.get('/flux/benchmarkdebug', asyncRoute((req, res) => { + return fluxService.benchmarkDebug(req, res); + })); + app.get('/flux/taildaemondebug', asyncRoute((req, res) => { + return fluxService.tailDaemonDebug(req, res); + })); + app.get('/flux/tailbenchmarkdebug', asyncRoute((req, res) => { + return fluxService.tailBenchmarkDebug(req, res); + })); + app.get('/flux/errorlog', asyncRoute((req, res) => { + return fluxService.fluxErrorLog(req, res); + })); + app.get('/flux/warnlog', asyncRoute((req, res) => { + return fluxService.fluxWarnLog(req, res); + })); + app.get('/flux/debuglog', asyncRoute((req, res) => { + return fluxService.fluxDebugLog(req, res); + })); + app.get('/flux/infolog', asyncRoute((req, res) => { + return fluxService.fluxInfoLog(req, res); + })); + app.get('/flux/tailerrorlog', asyncRoute((req, res) => { + return fluxService.tailFluxErrorLog(req, res); + })); + app.get('/flux/tailwarnlog', asyncRoute((req, res) => { + return fluxService.tailFluxWarnLog(req, res); + })); + app.get('/flux/taildebuglog', asyncRoute((req, res) => { + return fluxService.tailFluxDebugLog(req, res); + })); + app.get('/flux/tailinfolog', asyncRoute((req, res) => { + return fluxService.tailFluxInfoLog(req, res); + })); - app.get('/flux/broadcastmessage/:data?', (req, res) => { - fluxCommunicationMessagesSender.broadcastMessageFromUser(req, res); - }); - app.get('/flux/broadcastmessagetooutgoing/:data?', (req, res) => { - fluxCommunicationMessagesSender.broadcastMessageToOutgoingFromUser(req, res); - }); - app.get('/flux/broadcastmessagetoincoming/:data?', (req, res) => { - fluxCommunicationMessagesSender.broadcastMessageToIncomingFromUser(req, res); - }); - app.get('/flux/addpeer/:ip?', (req, res) => { - fluxCommunication.addPeer(req, res); - }); - app.get('/flux/removepeer/:ip?', (req, res) => { - fluxCommunication.removePeer(req, res); - }); - app.get('/flux/addoutgoingpeer/:ip?', (req, res) => { - fluxCommunication.addOutgoingPeer(req, res); - }); - app.get('/flux/removeincomingpeer/:ip?', (req, res) => { - fluxCommunication.removeIncomingPeer(req, res); - }); - app.get('/flux/startdiscovery', (req, res) => { - fluxCommunication.startDiscoveryApi(req, res); - }); - app.get('/flux/allowport/:port?', (req, res) => { - fluxNetworkHelper.allowPortApi(req, res); - }); - app.get('/flux/checkcommunication', (req, res) => { - fluxNetworkHelper.isCommunicationEstablished(req, res); - }); - app.get('/flux/uptime', cache('30 seconds'), (req, res) => { - fluxNetworkHelper.fluxUptime(req, res); - }); - app.get('/flux/systemuptime', cache('30 seconds'), (req, res) => { - fluxNetworkHelper.fluxSystemUptime(req, res); - }); - app.get('/flux/clockdrift', cache('30 seconds'), (req, res) => { - fluxNetworkHelper.clockDrift(req, res); - }); - app.get('/flux/backendfolder', isLocal, (req, res) => { - fluxService.fluxBackendFolder(req, res); - }); - app.get('/flux/mapport/:port?', (req, res) => { - upnpService.mapPortApi(req, res); - }); - app.get('/flux/unmapport/:port?', (req, res) => { - upnpService.removeMapPortApi(req, res); - }); - app.get('/flux/getmap', (req, res) => { - upnpService.getMapApi(req, res); - }); - app.get('/flux/getip', (req, res) => { - upnpService.getIpApi(req, res); - }); - app.get('/flux/getgateway', (req, res) => { - upnpService.getGatewayApi(req, res); - }); - app.get('/flux/isarcaneos', cache('1 day'), (req, res) => { - fluxService.isArcaneOs(req, res); - }); + app.get('/flux/broadcastmessage/:data?', asyncRoute((req, res) => { + return fluxCommunicationMessagesSender.broadcastMessageFromUser(req, res); + })); + app.get('/flux/broadcastmessagetooutgoing/:data?', asyncRoute((req, res) => { + return fluxCommunicationMessagesSender.broadcastMessageToOutgoingFromUser(req, res); + })); + app.get('/flux/broadcastmessagetoincoming/:data?', asyncRoute((req, res) => { + return fluxCommunicationMessagesSender.broadcastMessageToIncomingFromUser(req, res); + })); + app.get('/flux/addpeer/:ip?', asyncRoute((req, res) => { + return fluxCommunication.addPeer(req, res); + })); + app.get('/flux/removepeer/:ip?', asyncRoute((req, res) => { + return fluxCommunication.removePeer(req, res); + })); + app.get('/flux/addoutgoingpeer/:ip?', asyncRoute((req, res) => { + return fluxCommunication.addOutgoingPeer(req, res); + })); + app.get('/flux/removeincomingpeer/:ip?', asyncRoute((req, res) => { + return fluxCommunication.removeIncomingPeer(req, res); + })); + app.get('/flux/startdiscovery', asyncRoute((req, res) => { + return fluxCommunication.startDiscoveryApi(req, res); + })); + app.get('/flux/allowport/:port?', asyncRoute((req, res) => { + return fluxNetworkHelper.allowPortApi(req, res); + })); + app.get('/flux/checkcommunication', asyncRoute((req, res) => { + return fluxNetworkHelper.isCommunicationEstablished(req, res); + })); + app.get('/flux/uptime', cache('30 seconds'), asyncRoute((req, res) => { + return fluxNetworkHelper.fluxUptime(req, res); + })); + app.get('/flux/systemuptime', cache('30 seconds'), asyncRoute((req, res) => { + return fluxNetworkHelper.fluxSystemUptime(req, res); + })); + app.get('/flux/clockdrift', cache('30 seconds'), asyncRoute((req, res) => { + return fluxNetworkHelper.clockDrift(req, res); + })); + app.get('/flux/backendfolder', isLocal, asyncRoute((req, res) => { + return fluxService.fluxBackendFolder(req, res); + })); + app.get('/flux/mapport/:port?', asyncRoute((req, res) => { + return upnpService.mapPortApi(req, res); + })); + app.get('/flux/unmapport/:port?', asyncRoute((req, res) => { + return upnpService.removeMapPortApi(req, res); + })); + app.get('/flux/getmap', asyncRoute((req, res) => { + return upnpService.getMapApi(req, res); + })); + app.get('/flux/getip', asyncRoute((req, res) => { + return upnpService.getIpApi(req, res); + })); + app.get('/flux/getgateway', asyncRoute((req, res) => { + return upnpService.getGatewayApi(req, res); + })); + app.get('/flux/isarcaneos', cache('1 day'), asyncRoute((req, res) => { + return fluxService.isArcaneOs(req, res); + })); - app.get('/benchmark/start', (req, res) => { - fluxService.startBenchmark(req, res); - }); - app.get('/benchmark/restart', (req, res) => { - fluxService.restartBenchmark(req, res); - }); - app.get('/benchmark/restartnodebenchmarks', (req, res) => { - benchmarkService.restartNodeBenchmarks(req, res); - }); + app.get('/benchmark/start', asyncRoute((req, res) => { + return fluxService.startBenchmark(req, res); + })); + app.get('/benchmark/restart', asyncRoute((req, res) => { + return fluxService.restartBenchmark(req, res); + })); + app.get('/benchmark/restartnodebenchmarks', asyncRoute((req, res) => { + return benchmarkService.restartNodeBenchmarks(req, res); + })); - app.get('/explorer/reindex/:reindexapps?', (req, res) => { - explorerService.reindexExplorer(req, res); - }); - app.get('/explorer/restart', (req, res) => { - explorerService.restartBlockProcessing(req, res); - }); - app.get('/explorer/stop', (req, res) => { - explorerService.stopBlockProcessing(req, res); - }); - app.get('/explorer/rescan/:blockheight?/:rescanapps?', (req, res) => { - explorerService.rescanExplorer(req, res); - }); + app.get('/explorer/reindex/:reindexapps?', asyncRoute((req, res) => { + return explorerService.reindexExplorer(req, res); + })); + app.get('/explorer/restart', asyncRoute((req, res) => { + return explorerService.restartBlockProcessing(req, res); + })); + app.get('/explorer/stop', asyncRoute((req, res) => { + return explorerService.stopBlockProcessing(req, res); + })); + app.get('/explorer/rescan/:blockheight?/:rescanapps?', asyncRoute((req, res) => { + return explorerService.rescanExplorer(req, res); + })); - app.get('/apps/checkhashes', (req, res) => { - appHashSyncService.triggerAppHashesCheckAPI(req, res); - }); - app.get('/apps/requestmessage/:hash', (req, res) => { - messageVerifier.requestAppMessageAPI(req, res); - }); - app.get('/apps/appstart/:appname?/:global?', (req, res) => { - appController.appStart(req, res); - }); - app.get('/apps/appstop/:appname?/:global?', (req, res) => { - appController.appStop(req, res); - }); - app.get('/apps/apprestart/:appname?/:global?', (req, res) => { - appController.appRestart(req, res); - }); - app.get('/apps/apppause/:appname?/:global?', (req, res) => { - appController.appPause(req, res); - }); - app.get('/apps/appunpause/:appname?/:global?', (req, res) => { - appController.appUnpause(req, res); - }); - app.get('/apps/apptop/:appname?', (req, res) => { - appInspector.appTop(req, res); - }); - app.get('/apps/applog/:appname?/:lines?', (req, res) => { - appInspector.appLog(req, res); - }); - app.get('/apps/applogpolling/:appname?/:lines?/:since?', (req, res) => { - appInspector.appLogPolling(req, res); - }); - app.get('/apps/appinspect/:appname?', (req, res) => { - appInspector.appInspect(req, res); - }); - app.get('/apps/appstats/:appname?', (req, res) => { - appInspector.appStats(req, res); - }); - app.get('/apps/appmonitor/:appname?/:range?', (req, res) => { - monitoringOrchestrator.appMonitor(req, res); - }); - app.get('/apps/appmonitorstream/:appname?', (req, res) => { - appInspector.appMonitorStream(req, res); - }); - app.get('/apps/appchanges/:appname?', (req, res) => { - appInspector.appChanges(req, res); - }); - app.post('/apps/appexec', (req, res) => { - appInspector.appExec(req, res); - }); - app.get('/apps/appremove/:appname?/:force?/:global?', (req, res) => { - appUninstaller.removeAppLocallyApi(req, res); - }); - app.get('/apps/installapplocally/:appname?', (req, res) => { - appInstaller.installAppLocally(req, res); - }); - app.get('/apps/testappinstall/:appname?', (req, res) => { - appInstaller.testAppInstall(req, res); - }); - app.get('/apps/createfluxnetwork', (req, res) => { - systemIntegration.createFluxNetworkAPI(req, res); - }); - app.get('/apps/rescanglobalappsinformation/:blockheight?/:removelastinformation?', (req, res) => { - registryManager.rescanGlobalAppsInformationAPI(req, res); - }); - app.get('/apps/reindexglobalappsinformation', (req, res) => { - registryManager.reindexGlobalAppsInformationAPI(req, res); - }); - app.get('/apps/reindexglobalappslocation', (req, res) => { - registryManager.reindexGlobalAppsLocationAPI(req, res); - }); - app.get('/apps/redeploy/:appname?/:force?/:global?', (req, res) => { - advancedWorkflows.redeployAPI(req, res); - }); - app.get('/apps/redeploycomponent/:appname?/:component?/:force?', (req, res) => { - advancedWorkflows.redeployComponentAPI(req, res); - }); - app.get('/apps/reconstructhashes', (req, res) => { - registryManager.reconstructAppMessagesHashCollectionAPI(req, res); - }); - app.get('/apps/startmonitoring/:appname?', (req, res) => { - monitoringOrchestrator.startAppMonitoringAPI(req, res); - }); - app.get('/apps/stopmonitoring/:appname?/:deletedata?', (req, res) => { - monitoringOrchestrator.stopAppMonitoringAPI(req, res); - }); + app.get('/apps/checkhashes', asyncRoute((req, res) => { + return appHashSyncService.triggerAppHashesCheckAPI(req, res); + })); + app.get('/apps/requestmessage/:hash', asyncRoute((req, res) => { + return messageVerifier.requestAppMessageAPI(req, res); + })); + // alwaysRespond BEFORE requireBootSettled, on all eight app-control routes. + // + // Not for the reason it looks like. A 503 from the boot gate cannot collapse + // into a bodiless 304 whichever way round these go: express's req.fresh + // returns false unless the status is 2xx or 304, so a conditional request can + // never turn a 503 into one, whatever ETag it carries. Verified across + // bootSettled x order x six request shapes. + // + // The real reason is smaller: reversed, the 503 goes out with no Cache-Control + // header at all instead of no-store, because alwaysRespond never runs. Worth + // keeping, and worth writing down - two middlewares in an order with no stated + // reason is an invitation to swap them. + app.get('/apps/appstart/:appname?/:global?', alwaysRespond, requireBootSettled, asyncRoute((req, res) => { + return appController.appStart(req, res); + })); + app.get('/apps/appstop/:appname?/:global?', alwaysRespond, requireBootSettled, asyncRoute((req, res) => { + return appController.appStop(req, res); + })); + app.get('/apps/apprestart/:appname?/:global?', alwaysRespond, requireBootSettled, asyncRoute((req, res) => { + return appController.appRestart(req, res); + })); + // No :global - a kill is deliberately per-node. Its privilege is the same as + // its siblings' above: every run-state verb asks for appownerorfluxteam, so + // the node operator can order none of them on an app they only host. + app.get('/apps/appkill/:appname?', alwaysRespond, requireBootSettled, asyncRoute((req, res) => { + return appController.appKill(req, res); + })); + app.get('/apps/apppause/:appname?/:global?', alwaysRespond, requireBootSettled, asyncRoute((req, res) => { + return appController.appPause(req, res); + })); + app.get('/apps/appunpause/:appname?/:global?', alwaysRespond, requireBootSettled, asyncRoute((req, res) => { + return appController.appUnpause(req, res); + })); + app.get('/apps/apptop/:appname?', asyncRoute((req, res) => { + return appInspector.appTop(req, res); + })); + app.get('/apps/applog/:appname?/:lines?', asyncRoute((req, res) => { + return appInspector.appLog(req, res); + })); + app.get('/apps/applogpolling/:appname?/:lines?/:since?', asyncRoute((req, res) => { + return appInspector.appLogPolling(req, res); + })); + app.get('/apps/appinspect/:appname?', asyncRoute((req, res) => { + return appInspector.appInspect(req, res); + })); + app.get('/apps/appstats/:appname?', asyncRoute((req, res) => { + return appInspector.appStats(req, res); + })); + app.get('/apps/appmonitor/:appname?/:range?', asyncRoute((req, res) => { + return appInspector.appMonitorAPI(req, res); + })); + app.get('/apps/appchanges/:appname?', asyncRoute((req, res) => { + return appInspector.appChanges(req, res); + })); + app.post('/apps/appexec', asyncRoute((req, res) => { + return appInspector.appExec(req, res); + })); + app.get('/apps/appremove/:appname?/:force?/:global?', alwaysRespond, requireBootSettled, asyncRoute((req, res) => { + return appUninstaller.removeAppLocallyApi(req, res); + })); + app.get('/apps/installapplocally/:appname?', requireBootSettled, asyncRoute((req, res) => { + return appInstaller.installAppLocally(req, res); + })); + app.get('/apps/testappinstall/:appname?', requireBootSettled, asyncRoute((req, res) => { + return appInstaller.testAppInstall(req, res); + })); + app.get('/apps/createfluxnetwork', asyncRoute((req, res) => { + return systemIntegration.createFluxNetworkAPI(req, res); + })); + app.get('/apps/rescanglobalappsinformation/:blockheight?/:removelastinformation?', asyncRoute((req, res) => { + return registryManager.rescanGlobalAppsInformationAPI(req, res); + })); + app.get('/apps/reindexglobalappsinformation', asyncRoute((req, res) => { + return registryManager.reindexGlobalAppsInformationAPI(req, res); + })); + app.get('/apps/reindexglobalappslocation', asyncRoute((req, res) => { + return registryManager.reindexGlobalAppsLocationAPI(req, res); + })); + app.get('/apps/redeploy/:appname?/:force?/:global?', alwaysRespond, requireBootSettled, asyncRoute((req, res) => { + return advancedWorkflows.redeployAPI(req, res); + })); + app.get('/apps/redeploycomponent/:appname?/:component?/:force?', alwaysRespond, requireBootSettled, asyncRoute((req, res) => { + return advancedWorkflows.redeployComponentAPI(req, res); + })); + app.get('/apps/reconstructhashes', asyncRoute((req, res) => { + return registryManager.reconstructAppMessagesHashCollectionAPI(req, res); + })); + // alwaysRespond, like the other retired routes: the body is byte-identical on + // every call, so express fingerprints it with a strong ETag and any caller that + // revalidates gets an empty 304 from the second call on. Explaining why the + // endpoint stopped working is the only job these routes have left, and a repeat + // caller could never read the explanation. + app.get('/apps/startmonitoring/:appname?', alwaysRespond, asyncRoute((req, res) => { + return monitoringOrchestrator.startAppMonitoringAPI(req, res); + })); + app.get('/apps/stopmonitoring/:appname?/:deletedata?', alwaysRespond, asyncRoute((req, res) => { + return monitoringOrchestrator.stopAppMonitoringAPI(req, res); + })); + app.get('/apps/appmonitorstream/:appname?', alwaysRespond, asyncRoute((req, res) => { + return monitoringOrchestrator.appMonitorStreamAPI(req, res); + })); - app.get('/syncthing/metrics', cache('10 seconds'), (req, res) => { - syncthingService.getSyncthingMetrics(req, res); - }); - app.get('/syncthing/metrics/health', cache('10 seconds'), (req, res) => { - syncthingService.getSyncthingHealthSummary(req, res); - }); - app.get('/syncthing/metrics/history/:limit?', cache('10 seconds'), (req, res) => { - syncthingService.getSyncthingMetricsHistory(req, res); - }); - app.get('/syncthing/peer/diagnostics', cache('10 seconds'), (req, res) => { - syncthingService.getPeerSyncDiagnosticsApi(req, res); - }); + app.get('/syncthing/metrics', asyncRoute((req, res) => { + return syncthingService.getSyncthingMetrics(req, res); + })); + app.get('/syncthing/metrics/health', asyncRoute((req, res) => { + return syncthingService.getSyncthingHealthSummary(req, res); + })); + app.get('/syncthing/metrics/history/:limit?', asyncRoute((req, res) => { + return syncthingService.getSyncthingMetricsHistory(req, res); + })); + app.get('/syncthing/peer/diagnostics', asyncRoute((req, res) => { + return syncthingService.getPeerSyncDiagnosticsApi(req, res); + })); // POST PUBLIC methods route // ArcaneOS Authentication Endpoints (HTTPS only) - app.post('/arcane/configsync', requireHttps, arcaneAuthService.configSyncHandler); + app.post('/arcane/configsync', requireHttps, asyncRoute(arcaneAuthService.configSyncHandler)); - app.post('/id/verifylogin', (req, res) => { - idService.verifyLogin(req, res); - }); - app.post('/id/providesign', (req, res) => { - idService.provideSign(req, res); - }); - app.post('/id/checkprivilege', (req, res) => { - idService.checkLoggedUser(req, res); - }); - app.post('/zelid/verifylogin', (req, res) => { // DEPRECATED - idService.verifyLogin(req, res); - }); - app.post('/zelid/providesign', (req, res) => { // DEPRECATED - idService.provideSign(req, res); - }); - app.post('/zelid/checkprivilege', (req, res) => { // DEPRECATED - idService.checkLoggedUser(req, res); - }); + app.post('/id/verifylogin', asyncRoute((req, res) => { + return idService.verifyLogin(req, res); + })); + app.post('/id/providesign', asyncRoute((req, res) => { + return idService.provideSign(req, res); + })); + app.post('/id/checkprivilege', asyncRoute((req, res) => { + return idService.checkLoggedUser(req, res); + })); + app.post('/zelid/verifylogin', asyncRoute((req, res) => { // DEPRECATED + return idService.verifyLogin(req, res); + })); + app.post('/zelid/providesign', asyncRoute((req, res) => { // DEPRECATED + return idService.provideSign(req, res); + })); + app.post('/zelid/checkprivilege', asyncRoute((req, res) => { // DEPRECATED + return idService.checkLoggedUser(req, res); + })); // Payment request routes - app.get('/payment/paymentrequest', (req, res) => { - paymentService.paymentRequest(req, res); - }); - app.post('/payment/verifypayment', (req, res) => { - paymentService.verifyPayment(req, res); - }); + app.get('/payment/paymentrequest', asyncRoute((req, res) => { + return paymentService.paymentRequest(req, res); + })); + app.post('/payment/verifypayment', asyncRoute((req, res) => { + return paymentService.verifyPayment(req, res); + })); - app.post('/daemon/createrawtransaction', (req, res) => { - daemonServiceTransactionRpcs.createRawTransactionPost(req, res); - }); - app.post('/daemon/decoderawtransaction', (req, res) => { - daemonServiceTransactionRpcs.decodeRawTransactionPost(req, res); - }); - app.post('/daemon/decodescript', (req, res) => { - daemonServiceTransactionRpcs.decodeScriptPost(req, res); - }); - app.post('/daemon/fundrawtransaction', (req, res) => { - daemonServiceTransactionRpcs.fundRawTransactionPost(req, res); - }); - app.post('/daemon/sendrawtransaction', (req, res) => { - daemonServiceTransactionRpcs.sendRawTransactionPost(req, res); - }); - app.post('/daemon/createmultisig', (req, res) => { - daemonServiceUtilityRpcs.createMultiSigPost(req, res); - }); - app.post('/daemon/verifymessage', (req, res) => { - daemonServiceUtilityRpcs.verifyMessagePost(req, res); - }); - app.post('/daemon/getblockhashes', (req, res) => { - daemonServiceBlockchainRpcs.getBlockHashesPost(req, res); - }); - app.post('/daemon/getspentinfo', (req, res) => { - daemonServiceBlockchainRpcs.getSpentInfoPost(req, res); - }); - app.post('/daemon/getaddresstxids', (req, res) => { - daemonServiceAddressRpcs.getAddressTxids(req, res); - }); - app.post('/daemon/getaddressbalance', (req, res) => { - daemonServiceAddressRpcs.getAddressBalance(req, res); - }); - app.post('/daemon/getaddressdeltas', (req, res) => { - daemonServiceAddressRpcs.getAddressDeltas(req, res); - }); - app.post('/daemon/getaddressutxos', (req, res) => { - daemonServiceAddressRpcs.getAddressUtxos(req, res); - }); - app.post('/daemon/getaddressmempool', (req, res) => { - daemonServiceAddressRpcs.getAddressMempool(req, res); - }); - app.get('/flux/streamchainpreparation', (req, res) => { - fluxService.streamChainPreparation(req, res); - }); - app.post('/flux/streamchain', (req, res) => { - fluxService.streamChain(req, res); - }); + app.post('/daemon/createrawtransaction', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.createRawTransactionPost(req, res); + })); + app.post('/daemon/decoderawtransaction', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.decodeRawTransactionPost(req, res); + })); + app.post('/daemon/decodescript', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.decodeScriptPost(req, res); + })); + app.post('/daemon/fundrawtransaction', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.fundRawTransactionPost(req, res); + })); + app.post('/daemon/sendrawtransaction', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.sendRawTransactionPost(req, res); + })); + app.post('/daemon/createmultisig', asyncRoute((req, res) => { + return daemonServiceUtilityRpcs.createMultiSigPost(req, res); + })); + app.post('/daemon/verifymessage', asyncRoute((req, res) => { + return daemonServiceUtilityRpcs.verifyMessagePost(req, res); + })); + app.post('/daemon/getblockhashes', asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getBlockHashesPost(req, res); + })); + app.post('/daemon/getspentinfo', asyncRoute((req, res) => { + return daemonServiceBlockchainRpcs.getSpentInfoPost(req, res); + })); + app.post('/daemon/getaddresstxids', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getAddressTxids(req, res); + })); + app.post('/daemon/getaddressbalance', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getAddressBalance(req, res); + })); + app.post('/daemon/getaddressdeltas', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getAddressDeltas(req, res); + })); + app.post('/daemon/getaddressutxos', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getAddressUtxos(req, res); + })); + app.post('/daemon/getaddressmempool', asyncRoute((req, res) => { + return daemonServiceAddressRpcs.getAddressMempool(req, res); + })); + app.get('/flux/streamchainpreparation', asyncRoute((req, res) => { + return fluxService.streamChainPreparation(req, res); + })); + app.post('/flux/streamchain', asyncRoute((req, res) => { + return fluxService.streamChain(req, res); + })); // POST PROTECTED API - USER LEVEL - app.post('/id/logoutspecificsession', (req, res) => { // requires the knowledge of a session loginPhrase so users level is sufficient and user cannot logout another user as he does not know the loginPhrase. - idService.logoutSpecificSession(req, res); - }); - app.post('/zelid/logoutspecificsession', (req, res) => { // DEPRECATED - idService.logoutSpecificSession(req, res); - }); + app.post('/id/logoutspecificsession', asyncRoute((req, res) => { // requires the knowledge of a session loginPhrase so users level is sufficient and user cannot logout another user as he does not know the loginPhrase. + return idService.logoutSpecificSession(req, res); + })); + app.post('/zelid/logoutspecificsession', asyncRoute((req, res) => { // DEPRECATED + return idService.logoutSpecificSession(req, res); + })); - app.post('/daemon/submitblock', (req, res) => { - daemonServiceMiningRpcs.submitBlockPost(req, res); - }); + app.post('/daemon/submitblock', asyncRoute((req, res) => { + return daemonServiceMiningRpcs.submitBlockPost(req, res); + })); - app.post('/apps/checkdockerexistance', (req, res) => { - imageManager.checkDockerAccessibility(req, res); - }); - app.post('/apps/appregister', (req, res) => { - registryManager.registerAppGlobalyApi(req, res); - }); - app.post('/apps/appupdate', (req, res) => { - advancedWorkflows.updateAppGlobalyApi(req, res); - }); - app.post('/apps/getpublickey', (req, res) => { - cryptographicKeys.getPublicKey(req, res); - }); + app.post('/apps/checkdockerexistance', asyncRoute((req, res) => { + return imageManager.checkDockerAccessibility(req, res); + })); + app.post('/apps/appregister', asyncRoute((req, res) => { + return registryManager.registerAppGlobalyApi(req, res); + })); + app.post('/apps/appupdate', asyncRoute((req, res) => { + return advancedWorkflows.updateAppGlobalyApi(req, res); + })); + app.post('/apps/getpublickey', asyncRoute((req, res) => { + return cryptographicKeys.getPublicKey(req, res); + })); // POST PROTECTED API - FluxNode owner level - app.post('/daemon/signrawtransaction', (req, res) => { - daemonServiceTransactionRpcs.signRawTransactionPost(req, res); - }); - app.post('/daemon/addmultisigaddress', (req, res) => { - daemonServiceWalletRpcs.addMultiSigAddressPost(req, res); - }); - app.post('/daemon/sendfrom', (req, res) => { - daemonServiceWalletRpcs.sendFromPost(req, res); - }); - app.post('/daemon/sendmany', (req, res) => { - daemonServiceWalletRpcs.sendManyPost(req, res); - }); - app.post('/daemon/sendtoaddress', (req, res) => { - daemonServiceWalletRpcs.sendToAddressPost(req, res); - }); - app.post('/daemon/signmessage', (req, res) => { - daemonServiceWalletRpcs.signMessagePost(req, res); - }); - app.post('/daemon/zsendmany', (req, res) => { - daemonServiceZcashRpcs.zSendManyPost(req, res); - }); - app.post('/daemon/zcrawjoinsplit', (req, res) => { - daemonServiceZcashRpcs.zcRawJoinSplitPost(req, res); - }); - app.post('/daemon/zcrawreceive', (req, res) => { - daemonServiceZcashRpcs.zcRawReceivePost(req, res); - }); + app.post('/daemon/signrawtransaction', asyncRoute((req, res) => { + return daemonServiceTransactionRpcs.signRawTransactionPost(req, res); + })); + app.post('/daemon/addmultisigaddress', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.addMultiSigAddressPost(req, res); + })); + app.post('/daemon/sendfrom', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.sendFromPost(req, res); + })); + app.post('/daemon/sendmany', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.sendManyPost(req, res); + })); + app.post('/daemon/sendtoaddress', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.sendToAddressPost(req, res); + })); + app.post('/daemon/signmessage', asyncRoute((req, res) => { + return daemonServiceWalletRpcs.signMessagePost(req, res); + })); + app.post('/daemon/zsendmany', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zSendManyPost(req, res); + })); + app.post('/daemon/zcrawjoinsplit', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zcRawJoinSplitPost(req, res); + })); + app.post('/daemon/zcrawreceive', asyncRoute((req, res) => { + return daemonServiceZcashRpcs.zcRawReceivePost(req, res); + })); - app.post('/benchmark/signfluxnodetransaction', (req, res) => { - benchmarkService.signFluxTransactionPost(req, res); - }); - app.post('/benchmark/signzelnodetransaction', (req, res) => { // DEPRECATED - benchmarkService.signFluxTransactionPost(req, res); - }); + app.post('/benchmark/signfluxnodetransaction', asyncRoute((req, res) => { + return benchmarkService.signFluxTransactionPost(req, res); + })); + app.post('/benchmark/signzelnodetransaction', asyncRoute((req, res) => { // DEPRECATED + return benchmarkService.signFluxTransactionPost(req, res); + })); // POST PROTECTED API - FluxTeam - app.post('/flux/broadcastmessage', (req, res) => { - fluxCommunicationMessagesSender.broadcastMessageFromUserPost(req, res); - }); - app.post('/flux/broadcastmessagetooutgoing', (req, res) => { - fluxCommunicationMessagesSender.broadcastMessageToOutgoingFromUserPost(req, res); - }); - app.post('/flux/broadcastmessagetoincoming', (req, res) => { - fluxCommunicationMessagesSender.broadcastMessageToIncomingFromUserPost(req, res); - }); + app.post('/flux/broadcastmessage', asyncRoute((req, res) => { + return fluxCommunicationMessagesSender.broadcastMessageFromUserPost(req, res); + })); + app.post('/flux/broadcastmessagetooutgoing', asyncRoute((req, res) => { + return fluxCommunicationMessagesSender.broadcastMessageToOutgoingFromUserPost(req, res); + })); + app.post('/flux/broadcastmessagetoincoming', asyncRoute((req, res) => { + return fluxCommunicationMessagesSender.broadcastMessageToIncomingFromUserPost(req, res); + })); - app.post('/syncthing/system/error', (req, res) => { - syncthingService.postSystemError(req, res); - }); - app.post('/syncthing/system/upgrade', (req, res) => { - syncthingService.postSystemUpgrade(req, res); - }); - app.post('/syncthing/config', (req, res) => { - syncthingService.postConfig(req, res); - }); - app.post('/syncthing/config/folders', (req, res) => { - syncthingService.postConfigFolders(req, res); - }); - app.post('/syncthing/config/devices', (req, res) => { - syncthingService.postConfigDevices(req, res); - }); - app.post('/syncthing/config/defaults/folder', (req, res) => { - syncthingService.postConfigDefaultsFolder(req, res); - }); - app.post('/syncthing/config/defaults/device', (req, res) => { - syncthingService.postConfigDefaultsDevice(req, res); - }); - app.post('/syncthing/config/defaults/ignores', (req, res) => { - syncthingService.postConfigDefaultsIgnores(req, res); - }); - app.post('/syncthing/config/options', (req, res) => { - syncthingService.postConfigOptions(req, res); - }); - app.post('/syncthing/config/gui', (req, res) => { - syncthingService.postConfigGui(req, res); - }); - app.post('/syncthing/config/ldap', (req, res) => { - syncthingService.postConfigLdap(req, res); - }); - app.post('/syncthing/cluster/pending/devices', (req, res) => { - syncthingService.postClusterPendigDevices(req, res); - }); - app.post('/syncthing/cluster/pending/folders', (req, res) => { - syncthingService.postClusterPendigFolders(req, res); - }); - app.post('/syncthing/folder/versions', (req, res) => { - syncthingService.postFolderVersions(req, res); - }); - app.post('/syncthing/db/ignores', (req, res) => { - syncthingService.postDbIgnores(req, res); - }); - app.post('/syncthing/db/override', (req, res) => { - syncthingService.postDbOverride(req, res); - }); - app.post('/syncthing/db/prio', (req, res) => { - syncthingService.postDbPrio(req, res); - }); - app.post('/syncthing/db/revert', (req, res) => { - syncthingService.postDbRevert(req, res); - }); - app.post('/syncthing/db/scan', (req, res) => { - syncthingService.postDbScan(req, res); - }); + app.post('/syncthing/system/error', asyncRoute((req, res) => { + return syncthingService.postSystemError(req, res); + })); + app.post('/syncthing/system/upgrade', asyncRoute((req, res) => { + return syncthingService.postSystemUpgrade(req, res); + })); + app.post('/syncthing/config', asyncRoute((req, res) => { + return syncthingService.postConfig(req, res); + })); + app.post('/syncthing/config/folders', asyncRoute((req, res) => { + return syncthingService.postConfigFolders(req, res); + })); + app.post('/syncthing/config/devices', asyncRoute((req, res) => { + return syncthingService.postConfigDevices(req, res); + })); + app.post('/syncthing/config/defaults/folder', asyncRoute((req, res) => { + return syncthingService.postConfigDefaultsFolder(req, res); + })); + app.post('/syncthing/config/defaults/device', asyncRoute((req, res) => { + return syncthingService.postConfigDefaultsDevice(req, res); + })); + app.post('/syncthing/config/defaults/ignores', asyncRoute((req, res) => { + return syncthingService.postConfigDefaultsIgnores(req, res); + })); + app.post('/syncthing/config/options', asyncRoute((req, res) => { + return syncthingService.postConfigOptions(req, res); + })); + app.post('/syncthing/config/gui', asyncRoute((req, res) => { + return syncthingService.postConfigGui(req, res); + })); + app.post('/syncthing/config/ldap', asyncRoute((req, res) => { + return syncthingService.postConfigLdap(req, res); + })); + app.post('/syncthing/cluster/pending/devices', asyncRoute((req, res) => { + return syncthingService.postClusterPendigDevices(req, res); + })); + app.post('/syncthing/cluster/pending/folders', asyncRoute((req, res) => { + return syncthingService.postClusterPendigFolders(req, res); + })); + app.post('/syncthing/folder/versions', asyncRoute((req, res) => { + return syncthingService.postFolderVersions(req, res); + })); + app.post('/syncthing/db/ignores', asyncRoute((req, res) => { + return syncthingService.postDbIgnores(req, res); + })); + app.post('/syncthing/db/override', asyncRoute((req, res) => { + return syncthingService.postDbOverride(req, res); + })); + app.post('/syncthing/db/prio', asyncRoute((req, res) => { + return syncthingService.postDbPrio(req, res); + })); + app.post('/syncthing/db/revert', asyncRoute((req, res) => { + return syncthingService.postDbRevert(req, res); + })); + app.post('/syncthing/db/scan', asyncRoute((req, res) => { + return syncthingService.postDbScan(req, res); + })); // FluxShare - app.get('/apps/fluxshare/getfile/:file?/:token?', (req, res) => { - fluxshareService.fluxShareDownloadFile(req, res); - }); - app.get('/apps/fluxshare/getfolder/:folder?', (req, res) => { - fluxshareService.fluxShareGetFolder(req, res); - }); - app.get('/apps/fluxshare/createfolder/:folder?', (req, res) => { - fluxshareService.fluxShareCreateFolder(req, res); - }); - app.post('/apps/fluxshare/uploadfile/:folder?', (req, res) => { - fluxshareService.fluxShareUpload(req, res); - }); - app.get('/apps/fluxshare/removefile/:file?', (req, res) => { - fluxshareService.fluxShareRemoveFile(req, res); - }); - app.get('/apps/fluxshare/removefolder/:folder?', (req, res) => { - fluxshareService.fluxShareRemoveFolder(req, res); - }); - app.get('/apps/fluxshare/fileexists/:file?', (req, res) => { - fluxshareService.fluxShareFileExists(req, res); - }); - app.get('/apps/fluxshare/stats', (req, res) => { - fluxshareService.fluxShareStorageStats(req, res); - }); - app.get('/apps/fluxshare/sharefile/:file?', (req, res) => { - fluxshareService.fluxShareShareFile(req, res); - }); - app.get('/apps/fluxshare/unsharefile/:file?', (req, res) => { - fluxshareService.fluxShareUnshareFile(req, res); - }); - app.get('/apps/fluxshare/sharedfiles', (req, res) => { - fluxshareService.fluxShareGetSharedFiles(req, res); - }); - app.get('/apps/fluxshare/rename/:oldpath?/:newname?', (req, res) => { - fluxshareService.fluxShareRename(req, res); - }); - app.get('/apps/fluxshare/downloadfolder/:folder?', (req, res) => { - fluxshareService.fluxShareDownloadFolder(req, res); - }); + app.get('/apps/fluxshare/getfile/:file?/:token?', asyncRoute((req, res) => { + return fluxshareService.fluxShareDownloadFile(req, res); + })); + app.get('/apps/fluxshare/getfolder/:folder?', asyncRoute((req, res) => { + return fluxshareService.fluxShareGetFolder(req, res); + })); + app.get('/apps/fluxshare/createfolder/:folder?', asyncRoute((req, res) => { + return fluxshareService.fluxShareCreateFolder(req, res); + })); + app.post('/apps/fluxshare/uploadfile/:folder?', asyncRoute((req, res) => { + return fluxshareService.fluxShareUpload(req, res); + })); + app.get('/apps/fluxshare/removefile/:file?', asyncRoute((req, res) => { + return fluxshareService.fluxShareRemoveFile(req, res); + })); + app.get('/apps/fluxshare/removefolder/:folder?', asyncRoute((req, res) => { + return fluxshareService.fluxShareRemoveFolder(req, res); + })); + app.get('/apps/fluxshare/fileexists/:file?', asyncRoute((req, res) => { + return fluxshareService.fluxShareFileExists(req, res); + })); + app.get('/apps/fluxshare/stats', asyncRoute((req, res) => { + return fluxshareService.fluxShareStorageStats(req, res); + })); + app.get('/apps/fluxshare/sharefile/:file?', asyncRoute((req, res) => { + return fluxshareService.fluxShareShareFile(req, res); + })); + app.get('/apps/fluxshare/unsharefile/:file?', asyncRoute((req, res) => { + return fluxshareService.fluxShareUnshareFile(req, res); + })); + app.get('/apps/fluxshare/sharedfiles', asyncRoute((req, res) => { + return fluxshareService.fluxShareGetSharedFiles(req, res); + })); + app.get('/apps/fluxshare/rename/:oldpath?/:newname?', asyncRoute((req, res) => { + return fluxshareService.fluxShareRename(req, res); + })); + app.get('/apps/fluxshare/downloadfolder/:folder?', asyncRoute((req, res) => { + return fluxshareService.fluxShareDownloadFolder(req, res); + })); + // Handing the file operation image to a node that cannot reach the registry. + // Open to other Flux nodes rather than to an owner: it carries no app data, + // and a node needing it has nobody to authenticate as. + app.get('/apps/fileoperationimage/:imageid', asyncRoute((req, res) => { + return volumeExecutor.serveImageToPeer(req, res); + })); // Volume Browser - app.get('/apps/getfolderinfo/:appname?/:component?/:folder?', (req, res) => { - fileQueryService.getAppsFolder(req, res); - }); - app.get('/apps/createfolder/:appname?/:component?/:folder?', (req, res) => { - fileSystemManager.createAppsFolder(req, res); - }); - app.get('/apps/renameobject/:appname?/:component?/:oldpath?/:newname?', (req, res) => { - fileSystemManager.renameAppsObject(req, res); - }); - app.get('/apps/removeobject/:appname?/:component?/:object?', (req, res) => { - fileSystemManager.removeAppsObject(req, res); - }); - app.get('/apps/downloadfile/:appname?/:component?/:file?', (req, res) => { - fileSystemManager.downloadAppsFile(req, res); - }); - app.get('/apps/downloadfolder/:appname?/:component?/:folder?', (req, res) => { - fileSystemManager.downloadAppsFolder(req, res); - }); - app.get('/explorer/issynced', cache('30 seconds'), (req, res) => { - explorerService.isExplorerSynced(req, res); - }); + app.get('/apps/getfolderinfo/:appname?/:component?/:folder?', asyncRoute((req, res) => { + return fileQueryService.getAppsFolder(req, res); + })); + app.get('/apps/createfolder/:appname?/:component?/:folder?', requireBootSettled, asyncRoute((req, res) => { + return fileSystemManager.createAppsFolder(req, res); + })); + app.get('/apps/renameobject/:appname?/:component?/:oldpath?/:newname?', requireBootSettled, asyncRoute((req, res) => { + return fileSystemManager.renameAppsObject(req, res); + })); + app.get('/apps/removeobject/:appname?/:component?/:object?', requireBootSettled, asyncRoute((req, res) => { + return fileSystemManager.removeAppsObject(req, res); + })); + // Every endpoint that answers 202 points here: one status resource, one + // status enum, one error shape, so a client polls the same way whatever it + // started. + app.get('/apps/operations/:jobId', asyncRoute((req, res) => { + return operationsController.getOperation(req, res); + })); + app.delete('/apps/operations/:jobId', asyncRoute((req, res) => { + return operationsController.cancelOperation(req, res); + })); + // POST, with the operands in a JSON body: + // { appname, component, source, destination, overwrite? } + // + // Not GET. These create, overwrite and destroy, and a GET may be replayed by + // a proxy, a retry or a refresh - `overwrite: true` sitting in a URL is a + // destructive operation waiting to be repeated. Thirty of the /apps/ GET + // routes in this file are served through apicache, which makes a cached + // destructive GET one config line away rather than impossible. A path also + // puts every filename into the access log and the URL length limit. + // + // It matches the endpoints that already answer 202 here - imagepreflight, + // playground and imagecache are all POST - rather than the older file API + // whose GET shape these otherwise sit beside. + // + // `destination` is the full target path INCLUDING the new name, not the + // parent directory, so copy and move share -T semantics and there is no + // paste-into versus paste-as ambiguity. + // + // All four answer 202 with a jobId to poll at /apps/operations/:jobId. Move + // included, even though its visible part is a rename: paste is one gesture in + // a file browser, and cut-paste returning a result while copy-paste returns a + // job would put two response shapes inside one user action. + app.post('/apps/moveobject', requireBootSettled, asyncRoute((req, res) => { + return fileSystemManager.moveAppsObject(req, res); + })); + app.post('/apps/copyobject', requireBootSettled, asyncRoute((req, res) => { + return fileSystemManager.copyAppsObject(req, res); + })); + app.post('/apps/compressobject', requireBootSettled, asyncRoute((req, res) => { + return fileSystemManager.compressAppsObject(req, res); + })); + app.post('/apps/extractobject', requireBootSettled, asyncRoute((req, res) => { + return fileSystemManager.extractAppsObject(req, res); + })); + app.get('/apps/downloadfile/:appname?/:component?/:file?', asyncRoute((req, res) => { + return fileSystemManager.downloadAppsFile(req, res); + })); + app.get('/apps/downloadfolder/:appname?/:component?/:folder?', asyncRoute((req, res) => { + return fileSystemManager.downloadAppsFolder(req, res); + })); + app.get('/explorer/issynced', cache('30 seconds'), asyncRoute((req, res) => { + return explorerService.isExplorerSynced(req, res); + })); + + app.get('/flux/eventstream', asyncRoute((req, res) => { + return fluxEventBus.sseHandler(req, res); + })); - app.get('/flux/eventstream', (req, res) => { - fluxEventBus.sseHandler(req, res); - }); + // Cadence, read rather than streamed - see the rule at the top of + // fluxEventBus.js. 404s in production, like the stream above. + app.get('/flux/testcounters', asyncRoute((req, res) => { + return fluxEventBus.countersHandler(req, res); + })); }; diff --git a/ZelBack/src/services/IOUtils.js b/ZelBack/src/services/IOUtils.js index 891b0880d6..8b4c5c9b6d 100644 --- a/ZelBack/src/services/IOUtils.js +++ b/ZelBack/src/services/IOUtils.js @@ -1,17 +1,12 @@ -const df = require('node-df'); const fs = require('fs').promises; const fs2 = require('fs'); -const util = require('util'); const log = require('../lib/log'); const axios = require('axios'); const path = require('path'); -const { formidable } = require('formidable'); +const deviceHelper = require('./deviceHelper'); const serviceHelper = require('./serviceHelper'); -const messageHelper = require('./messageHelper'); -const verificationHelper = require('./verificationHelper'); -const exec = util.promisify(require('child_process').exec); const { URL } = require('url'); -const { sanitizePath, validateFilename, verifyRealPathOfExistingPath } = require('./utils/pathSecurity'); +const { measureTree } = require('./utils/treeSize'); const { validateUrlWithDns } = require('./utils/urlSecurity'); /** @@ -60,7 +55,7 @@ async function requestWithValidatedRedirects(url, method = 'GET', axiosOptions = } // Handle redirect - const location = response.headers.location; + const { location } = response.headers; if (!location) { throw new Error('Redirect response missing Location header'); } @@ -141,32 +136,70 @@ function convertFileSize(sizes, targetUnit = 'auto', decimal = 2, returnNumber = /** * Get the total size of a folder, including its subdirectories and files. + * + * Follows no symlink. It used to `stat` its way down with unbounded recursion + * and a Promise.all fan-out, which an app owner could turn on the node hosting + * them: this measures volumes the apps themselves write to, so a `loop -> ..` + * planted in one measured itself until the process died, and an `escape -> /` + * measured the host. Both callers run as the FluxOS process, before any + * container exists - a copy's capacity check, and the file browser, which + * measures every directory it lists. + * + * Approximate upwards of nothing: an entry that cannot be stat'ed is skipped + * and a directory that cannot be opened is walked no further, so a tree only + * partly readable reports less than it holds. That is what a size in a listing + * means, and what `du` does with the same problem. Anything deciding whether + * something FITS needs a bound applied to what actually lands, not this. + * * @param {string} folderPath - The path to the folder. - * @returns {string|boolean} - The total size of the folder formatted with the specified multiplier and decimal places, or false if an error occurs. + * @returns {Promise} - Total bytes. */ async function getFolderSize(folderPath) { - try { - let totalSize = 0; - const calculateSize = async (filePath) => { - const stats = await fs.stat(filePath); - - if (stats.isFile()) { - return stats.size; - // eslint-disable-next-line no-else-return - } else if (stats.isDirectory()) { - const files = await fs.readdir(filePath); - const sizes = await Promise.all(files.map((file) => calculateSize(path.join(filePath, file)))); - return sizes.reduce((acc, size) => acc + size, 0); - } - - return 0; // Unknown file type - }; + // What the files say, which is what a listing means by the size of a folder + // and what every file browser shows. Anything comparing a figure against + // free space wants measureTree's `occupied` instead, because that is a count + // of blocks and this is not. + return measureTree(folderPath, fs); +} - totalSize = await calculateSize(folderPath); - return totalSize; - } catch (err) { - console.error(`Error getting folder size: ${err}`); - return false; +/** + * The size of a whole app volume's directory tree, in bytes. `du` walks it in + * one process with bounded memory; getFolderSize recurses in-process and fans + * out a Promise per entry at every level, which is fine for the handful of + * entries a folder listing shows and unbounded on an app's real data. + * + * Null rather than false when the size cannot be established: zero is a real + * answer for an empty directory, and a falsy sentinel makes the two + * indistinguishable at every call site that tests the result for truth. + * + * @param {string} dirPath - The path of the directory to measure. + * @returns {Promise} - Size in bytes, or null if it could not be measured. + */ +async function getDirectorySizeBytes(dirPath) { + try { + // Without -s, du reports each directory as it walks it and its own total + // last, so the answer is unchanged while the walk becomes observable - which + // is what an idle limit needs to mean anything. argv, no shell, for the same + // reason as the tar calls. + let total = null; + const result = await serviceHelper.runStreamingCommand('du', { + runAsRoot: true, + params: ['-b', dirPath], + idleTimeout: 5 * 60 * 1000, + onLine: (line) => { + const value = Number.parseInt(line.split(/\s+/)[0], 10); + if (Number.isFinite(value)) total = value; + }, + }); + if (result.error) { + const message = (result.stderr || result.error.message || '').replace(/\n/g, ' ').trim(); + log.error(`Error measuring directory ${dirPath}: ${message}`); + return null; + } + return total; + } catch (error) { + log.error(`Error measuring directory ${dirPath}: ${error.message}`); + return null; } } @@ -219,56 +252,60 @@ async function getRemoteFileSize(fileurl, multiplier, decimal, number = false) { * @param {string} multiplier - Unit multiplier for displaying sizes (B, KB, MB, GB). * @param {number} decimal - Number of decimal places for precision. * @param {string} fields - Optional comma-separated list of fields to include in the response. Possible fields: 'mount', 'size', 'used', 'available', 'capacity', 'filesystem'. - * @returns {Array|boolean} - Array of objects containing volume information for the specified component, or false if no matching mount is found. + * @returns {Promise<{error: Error|null, mounts: object[]}>} - `mounts` is the + * matching mount info (empty when the volume is not mounted - df only + * reports mounted filesystems), and `error` is set only when the mount + * table itself could not be read. An empty `mounts` with no `error` is + * an answer ("not mounted"); an `error` is a failure to answer, which a + * destructive caller must refuse on rather than read as "not mounted". */ async function getVolumeInfo(appname, component, multiplier, decimal, fields) { try { - const options = { - prefixMultiplier: multiplier, - isDisplayPrefixMultiplier: false, - precision: +decimal, + const mounts = await deviceHelper.listMountedFilesystems(); + + // The identifier is `flux_`, and neither name may contain an + // underscore (components are alphanumeric, app names alphanumeric plus + // internal hyphens), so the pair cannot be ambiguous. Both are validated + // against those charsets before reaching here, which is also what keeps them + // safe to interpolate into a pattern. + const identifier = component === 'null' ? `flux${appname}` : `flux${component}_${appname}`; + + // A path the KERNEL reports as a mountpoint, selected by the request - never + // a path built from it. The worst a hostile appname can do is match nothing. + const matched = mounts.filter((mount) => path.basename(mount.target) === identifier); + if (!matched.length) return { error: null, mounts: [] }; + + const divisor = { + b: 1, kb: 1024, mb: 1024 ** 2, gb: 1024 ** 3, + }[String(multiplier || 'B').toLowerCase()] ?? 1; + // Two argument orders for this function exist in the codebase, so `decimal` + // sometimes arrives as a field name. Anything non-numeric means "no rounding" + // rather than NaN, which is what the previous implementation produced for + // every size it returned to the file API. + const precision = Number.isFinite(+decimal) ? +decimal : null; + const toUnit = (bytes) => { + const value = bytes / divisor; + return precision === null ? value : Number(value.toFixed(precision)); }; - const dfAsync = util.promisify(df); - const dfData = await dfAsync(options); - let regex; - if (component === 'null') { - regex = new RegExp(`flux${appname}$`); - } else { - regex = new RegExp(`flux${component}_${appname}$`); - } - const allowedFields = fields ? fields.split(',') : null; - const adjustValue = (value) => (multiplier.toLowerCase() === 'b' ? value * 1024 : value); - const dfSorted = dfData - .filter((entry) => { - const testResult = regex.test(entry.mount); - return testResult; - }) - .map((entry) => { - const filteredEntry = allowedFields - ? Object.fromEntries(Object.entries(entry).filter(([key]) => allowedFields.includes(key))) - : entry; - - if (allowedFields && allowedFields.some((field) => ['size', 'available', 'used'].includes(field))) { - ['size', 'available', 'used'].forEach((property) => { - if (filteredEntry[property] !== undefined) { - filteredEntry[property] = adjustValue(filteredEntry[property]); - } - }); - } - return filteredEntry; - }) - .filter((entry) => { - if (allowedFields) { - return Object.keys(entry).length > 0; - // eslint-disable-next-line no-else-return - } else { - return true; - } - }); - return dfSorted.length > 0 ? dfSorted : false; + + const allowedFields = fields ? String(fields).split(',') : null; + const mountsInfo = matched.map((mount) => { + const full = { + filesystem: mount.source, + size: toUnit(mount.sizeBytes), + used: toUnit(mount.usedBytes), + available: toUnit(mount.availableBytes), + capacity: mount.usePercent / 100, + mount: mount.target, + }; + return allowedFields + ? Object.fromEntries(Object.entries(full).filter(([key]) => allowedFields.includes(key))) + : full; + }).filter((entry) => Object.keys(entry).length > 0); + return { error: null, mounts: mountsInfo }; } catch (error) { log.error(error); - return false; + return { error, mounts: [] }; } } @@ -286,8 +323,12 @@ async function getPathFileList(targetpath, multiplier, decimal, filterKeywords = // eslint-disable-next-line no-restricted-syntax for (const file of files) { const filePath = `${targetpath}/${file}`; + // lstat, so every entry describes ITSELF. This lists a directory on an + // application's own volume, where the application can put a link pointing + // anywhere on the node - and following one would answer with the size and + // creation time of whatever it names. // eslint-disable-next-line no-await-in-loop - const stats = await fs.stat(filePath); + const stats = await fs.lstat(filePath); // eslint-disable-next-line no-await-in-loop const passesFilter = filterKeywords.length === 0 || filterKeywords.some((keyword) => { const includes = file.includes(keyword); @@ -406,14 +447,95 @@ async function downloadFileFromUrl(url, localpath, component, rename = false, re async function untarFile(extractPath, tarFilePath) { try { await fs.mkdir(extractPath, { recursive: true }); - const unpackCmd = `sudo tar -xvzf ${tarFilePath} -C ${extractPath}`; - await exec(unpackCmd, { maxBuffer: 1024 * 1024 * 10 }); + // argv, not a command string: a path reaching this from anywhere a user can + // name a file would otherwise turn a filename containing $( ) into + // arbitrary root execution on the node. + // + // -v is also gone. It printed every extracted filename into a 10MB buffer, + // and a file-count-heavy tree overflowed it - which threw partway through + // an extraction, leaving a half-extracted tree and no way back. + const result = await serviceHelper.runCommand('tar', { + runAsRoot: true, + params: ['-xzf', tarFilePath, '-C', extractPath], + }); + if (result.error) { + const message = (result.stderr || result.stdout || result.error.message || '').replace(/\n/g, ' '); + log.error(`Error during extraction: ${message}`); + return { status: false, error: message }; + } return { status: true }; } catch (error) { - const stringstderr = error.stderr.replace(/\n/g, ' '); - const stringstdout = error.stdout.replace(/\n/g, ' '); - log.error('Error during extraction:', error.stderr || error.stdout); - return { status: false, error: stringstderr || stringstdout }; + log.error('Error during extraction:', error); + return { status: false, error: error.message }; + } +} + +/** + * Read a gzipped tar without extracting it. One decompression pass, nothing + * written to disk and no space consumed, establishing that the archive is + * complete and readable BEFORE anything is deleted to make room for its + * contents, and yielding the numbers needed to decide whether those contents + * will fit. + * + * The whole stream has to be inflated: gzip's CRC is in the trailing bytes, so + * a truncated or corrupt archive cannot be recognised any other way, and the + * ISIZE field beside it wraps at 4 GiB - useless on exactly the archives where + * the size answer matters. The listing is counted as it arrives and never held, + * so an archive of any member count costs the same to read. + * + * @param {string} tarFilePath - The path of the tarball (tar.gz) file to read. + * @returns {Promise<{status: boolean, entries?: number, bytes?: number, error?: string}>} + * entries is the member count; bytes is their total uncompressed size. + */ +async function inspectTarGz(tarFilePath) { + try { + let entries = 0; + let bytes = 0; + let sized = 0; + + // argv, and no shell: root is the only reason this is a child process, and + // a path reaches tar as an argument rather than as anything parsed. + // + // `sized` counts the members whose size column actually parsed as a number, + // which is what separates a differently-shaped listing from an archive whose + // members are all genuinely zero length. + const result = await serviceHelper.runStreamingCommand('tar', { + runAsRoot: true, + params: ['-tzvf', tarFilePath], + // Bounded by work rather than by size. These archives are the largest + // thing the node handles, and a total limit can only kill the ones that + // are merely big - which this then reports to an operator as their backup + // being unreadable. Every line of the listing is proof of progress, so + // silence this long is a read that has stalled. + idleTimeout: 5 * 60 * 1000, + onLine: (line) => { + entries += 1; + const size = line.split(/\s+/)[2]; + if (/^[0-9]+$/.test(size)) { + sized += 1; + bytes += Number(size); + } + }, + }); + + if (result.error) { + const message = (result.stderr || result.error.message || '').replace(/\n/g, ' ').trim(); + log.error(`Error reading archive: ${message}`); + return { status: false, error: message }; + } + // The size column is the third field of GNU tar's verbose listing. If no + // member's third field parsed as a number the listing is a different tar's + // column layout, and reporting its total as zero would walk an unmeasured + // archive through the free-space check. Testing the total rather than the + // parse would also condemn an archive whose members are all genuinely empty, + // which is a real thing to restore. + if (entries > 0 && sized === 0) { + return { status: false, error: 'archive listing not in the expected format' }; + } + return { status: true, entries, bytes }; + } catch (error) { + log.error('Error reading archive:', error); + return { status: false, error: error.message }; } } @@ -428,14 +550,20 @@ async function createTarGz(sourceDirectory, outputFileName) { try { const outputDirectory = outputFileName.substring(0, outputFileName.lastIndexOf('/')); await fs.mkdir(outputDirectory, { recursive: true }); - const packCmd = `sudo tar -czvf ${outputFileName} -C ${sourceDirectory} .`; - await exec(packCmd, { maxBuffer: 1024 * 1024 * 10 }); + // argv, and without -v, for the same two reasons as untarFile above. + const result = await serviceHelper.runCommand('tar', { + runAsRoot: true, + params: ['-czf', outputFileName, '-C', sourceDirectory, '.'], + }); + if (result.error) { + const message = (result.stderr || result.stdout || result.error.message || '').replace(/\n/g, ' '); + log.error(`Error creating tarball: ${message}`); + return { status: false, error: message }; + } return { status: true }; } catch (error) { - const stringstderr = error.stderr.replace(/\n/g, ' '); - const stringstdout = error.stdout.replace(/\n/g, ' '); - log.error('Error creating tarball:', error.stderr || error.stdout); - return { status: false, error: stringstderr || stringstdout }; + log.error('Error creating tarball:', error); + return { status: false, error: error.message }; } } @@ -448,13 +576,21 @@ async function createTarGz(sourceDirectory, outputFileName) { */ async function removeDirectory(rpath, directory = false) { try { - let execFinal; - if (directory === false) { - execFinal = `sudo rm -rf "${rpath}"`; - } else { - execFinal = `sudo find "${rpath}" -mindepth 1 -exec rm -rf {} +`; + // argv, not a command string. fluxshareService passes a path built from a + // caller-supplied folder name straight into this, so a name containing + // $( ) or a backtick was arbitrary root execution the moment the character + // rule that happened to exclude them was relaxed. + const result = directory + ? await serviceHelper.runCommand('find', { + runAsRoot: true, + params: [rpath, '-mindepth', '1', '-exec', 'rm', '-rf', '{}', '+'], + }) + : await serviceHelper.runCommand('rm', { runAsRoot: true, params: ['-rf', rpath] }); + + if (result.error) { + log.error(result.error); + return false; } - await exec(execFinal, { maxBuffer: 1024 * 1024 * 10 }); return true; } catch (error) { log.error(error); @@ -462,139 +598,6 @@ async function removeDirectory(rpath, directory = false) { } } -/** - * To upload a specified folder to FluxShare. Checks that there is enough space available. Only accessible by admins. - * @param {object} req Request. - * @param {object} res Response. - */ -async function fileUpload(req, res) { - try { - let { appname } = req.params; - appname = appname || req.query.appname || ''; - if (!appname) { - throw new Error('appname parameter is mandatory.'); - } - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); - if (!authorized) { - throw new Error('Unauthorized. Access denied.'); - } - let { component } = req.params; - component = component || req.query.component || ''; - let { filename } = req.params; - filename = filename || req.query.filename || ''; - let { folder } = req.params; - folder = folder || req.query.folder || ''; - let { type } = req.params; - type = type || req.query.type || ''; - if (!type || !component) { - throw new Error('component and type parameters are mandatory'); - } - let filepath; - const appVolumePath = await getVolumeInfo(appname, component, 'B', 'mount', 0); - if (appVolumePath.length > 0) { - if (type === 'backup') { - filepath = `${appVolumePath[0].mount}/backup/upload/`; - } else { - // Use appid level to access appdata and all other mount points - // Sanitize folder path to prevent directory traversal attacks - filepath = sanitizePath(folder, appVolumePath[0].mount); - } - } else { - throw new Error('Application volume not found'); - } - // Verify resolved path stays within the allowed base directory - await verifyRealPathOfExistingPath(filepath, appVolumePath[0].mount); - const options = { - multiples: true, - uploadDir: `${filepath}`, - maxFileSize: 10 * 1024 * 1024 * 1024, // 10gb - hashAlgorithm: false, - keepExtensions: true, - // eslint-disable-next-line no-unused-vars - filename: (name, ext, part, form) => { - const { originalFilename } = part; - return originalFilename; - }, - }; - await fs.mkdir(filepath, { recursive: true }); - const permission = `sudo chmod 777 "${filepath}"`; - await exec(permission, { maxBuffer: 1024 * 1024 * 10 }); - const form = formidable(options); - - form - // eslint-disable-next-line no-unused-vars - .on('fileBegin', (name, file) => { - // Validate filename to prevent path traversal via filename parameter - let safeFilename; - if (!filename) { - // Use form field name - validate it doesn't contain path separators - safeFilename = validateFilename(name); - } else { - // Use provided filename - validate it doesn't contain path separators - safeFilename = validateFilename(filename); - } - // eslint-disable-next-line no-param-reassign - file.filepath = `${filepath}/${safeFilename}`; - }) - .on('progress', (bytesReceived, bytesExpected) => { - try { - res.write(serviceHelper.ensureString([bytesReceived, bytesExpected])); - if (res.flush) res.flush(); - } catch (error) { - log.error(error); - } - }) - // eslint-disable-next-line no-unused-vars - .on('field', (name, field) => { - - }) - // eslint-disable-next-line no-unused-vars - .on('file', (name, file) => { - try { - res.write(serviceHelper.ensureString(name)); - if (res.flush) res.flush(); - } catch (error) { - log.error(error); - } - }) - .on('aborted', () => { - console.error('Request aborted by the user'); - }) - .on('error', (error) => { - log.error(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - try { - res.write(serviceHelper.ensureString(errorResponse)); - if (res.flush) res.flush(); - } catch (e) { - log.error(e); - } - }) - .on('end', () => { - try { - res.end(); - } catch (error) { - log.error(error); - } - }); - - form.parse(req); - } catch (error) { - log.error(error); - if (res) { - try { - res.connection.destroy(); - } catch (e) { - log.error(e); - } - } - } -} - module.exports = { getVolumeInfo, getPathFileList, @@ -605,8 +608,9 @@ module.exports = { convertFileSize, downloadFileFromUrl, untarFile, + inspectTarGz, createTarGz, removeDirectory, getFolderSize, - fileUpload, + getDirectorySizeBytes, }; diff --git a/ZelBack/src/services/analyticsService.js b/ZelBack/src/services/analyticsService.js index 918359b498..a49c1a2994 100644 --- a/ZelBack/src/services/analyticsService.js +++ b/ZelBack/src/services/analyticsService.js @@ -176,7 +176,7 @@ function analyticsMiddleware(req, res, next) { return; } - const zelidauth = req.headers.zelidauth; + const { zelidauth } = req.headers; if (!zelidauth) { next(); return; diff --git a/ZelBack/src/services/appDatabase/policyArtifactRepository.js b/ZelBack/src/services/appDatabase/policyArtifactRepository.js new file mode 100644 index 0000000000..3a95110d08 --- /dev/null +++ b/ZelBack/src/services/appDatabase/policyArtifactRepository.js @@ -0,0 +1,144 @@ +const config = require('config'); +const { GridFSBucket } = require('mongodb'); +const dbHelper = require('../dbHelper'); +const log = require('../../lib/log'); + +// Last-known-good storage for policy artifacts: documents too large to sit inline in a +// mongo document, which the 16 MiB BSON cap forbids. GridFS chunks them, so size is not +// a constraint, and they stay in the same local database — no untracked files in the repo +// tree, and identical on Arcane and legacy. +// +// Bytes go to the bucket; the etag and the id of the stored file go to the ordinary +// policyDocuments row, so there is one place to look for "what do we have and how fresh". +const BUCKET_NAME = 'policyartifacts'; +const policyDocumentsCollection = config.database.local.collections.policyDocuments; + +function db() { + const connection = dbHelper.databaseConnection(); + return connection ? connection.db(config.database.local.database) : null; +} + +function bucket(database) { + return new GridFSBucket(database, { bucketName: BUCKET_NAME }); +} + +/** + * What we hold for an artifact: the id of its stored bytes and the etag they were served + * with, or null when there is nothing. + * @param {string} name Registry key. + * @returns {Promise<{fileId: object, etag: string|null, fetchedAt: number|null}|null>} + */ +async function getArtifactRecord(name) { + const database = db(); + if (!database) return null; + const doc = await dbHelper.findOneInDatabase( + database, + policyDocumentsCollection, + { _id: name }, + ); + if (!doc || !doc.fileId) return null; + return { fileId: doc.fileId, etag: doc.etag ?? null, fetchedAt: doc.fetchedAt ?? null }; +} + +/** + * The stored bytes for an artifact, or null when the file is missing. + * + * A record whose file has gone (an interrupted write, a dropped bucket) reads as absent + * rather than throwing: the caller's next fetch replaces it. + * @param {object} fileId GridFS file id from the artifact record. + * @returns {Promise} + */ +async function readArtifactBytes(fileId) { + const database = db(); + if (!database) return null; + try { + const chunks = []; + const stream = bucket(database).openDownloadStream(fileId); + for await (const chunk of stream) chunks.push(chunk); + return Buffer.concat(chunks); + } catch (error) { + log.warn(`policyArtifact - could not read stored bytes for ${fileId}: ${error.message}`); + return null; + } +} + +/** + * Store bytes for an artifact and point its record at them. + * + * GridFS does not overwrite — every upload is a new file with its own chunks — so the + * previous file is deleted once the new one is committed and the record moved. Skipping + * that would grow the local database by the artifact's size on every refresh. + * @param {string} name Registry key. + * @param {Buffer} bytes The artifact. + * @param {string|null} [etag] Response ETag, for the next conditional request. + * @returns {Promise} true when stored and recorded. + */ +async function writeArtifactBytes(name, bytes, etag = null) { + const database = db(); + if (!database) return false; + + const previous = await getArtifactRecord(name); + + const fileId = await new Promise((resolve, reject) => { + const upload = bucket(database).openUploadStream(name); + upload.on('error', reject); + upload.on('finish', () => resolve(upload.id)); + upload.end(bytes); + }); + + await dbHelper.findOneAndUpdateInDatabase( + database, + policyDocumentsCollection, + { _id: name }, + { $set: { fileId, etag, fetchedAt: Date.now() } }, + { upsert: true }, + ); + + // Only now is the old file unreferenced. Failing to delete it is untidy, not incorrect, + // so it must not fail the write that already succeeded. + if (previous) { + await bucket(database).delete(previous.fileId) + .catch((error) => log.warn(`policyArtifact - could not delete superseded ${name} file: ${error.message}`)); + } + + return true; +} + +/** + * Delete stored files for an artifact that its record does not point at. + * + * A process killed between the upload finishing and the record moving leaves a file + * nothing references, and nothing else would ever reclaim it. Run at startup. + * @param {string} name Registry key. + * @returns {Promise} How many files were removed. + */ +async function sweepOrphanedArtifacts(name) { + const database = db(); + if (!database) return 0; + try { + const current = await getArtifactRecord(name); + const currentId = current ? String(current.fileId) : null; + const files = await bucket(database).find({ filename: name }).toArray(); + const orphans = files.filter((file) => String(file._id) !== currentId); + + // eslint-disable-next-line no-restricted-syntax + for (const orphan of orphans) { + // eslint-disable-next-line no-await-in-loop + await bucket(database).delete(orphan._id) + .catch((error) => log.warn(`policyArtifact - could not sweep ${orphan._id}: ${error.message}`)); + } + if (orphans.length) log.info(`policyArtifact - swept ${orphans.length} orphaned ${name} file(s)`); + return orphans.length; + } catch (error) { + log.warn(`policyArtifact - sweep of ${name} failed: ${error.message}`); + return 0; + } +} + +module.exports = { + getArtifactRecord, + readArtifactBytes, + writeArtifactBytes, + sweepOrphanedArtifacts, + BUCKET_NAME, +}; diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index 7c66162e54..54272b7357 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -9,6 +9,8 @@ const fluxEventBus = require('../utils/fluxEventBus'); // Removed appsService to avoid circular dependency - will use dynamic require where needed const { checkAndDecryptAppSpecs, encryptEnterpriseFromSession } = require('../utils/enterpriseHelper'); const { specificationFormatter, updateToLatestAppSpecifications } = require('../utils/appUtilities'); +const placementFeasibility = require('../appPlacement/placementFeasibility'); +const mountParser = require('../utils/mountParser'); const { SIGTERM_EXPIRY_MS, globalAppsInformation, @@ -22,6 +24,7 @@ const { appsHashesCollection, scannedHeightCollection, } = require('../utils/appConstants'); +const { Privilege, authOf } = require('../utils/privileges'); let reindexRunning = false; @@ -406,6 +409,28 @@ async function appInstallingLocation(appname) { return results; } +/** + * How many nodes are claiming each app, counted in one grouped pass. + * + * The spawner needs this for every candidate at once, to decide which apps + * still need a node before it picks one. Asking per app would be a read per + * candidate; this is a single scan of a collection that holds only live claims, + * since they expire on a TTL index. + * + * Names are lowercased because an app is addressed case-insensitively + * everywhere else here, so a caller must not have to know which case the + * claiming node happened to send. + * @returns {Promise>} Lowercased app name to claim count. + */ +async function installingCountsByApp() { + const dbopen = dbHelper.databaseConnection(); + const database = dbopen.db(config.database.appsglobal.database); + const rows = await dbHelper.aggregateInDatabase(database, globalAppsInstallingLocations, [ + { $group: { _id: { $toLower: '$name' }, count: { $sum: 1 } } }, + ]); + return new Map(rows.map((row) => [row._id, row.count])); +} + /** * Get app installing errors locations for a specific app or all apps * @param {string} appname - Application name (optional) @@ -758,6 +783,99 @@ async function getApplicationSpecifications(appName) { return appInfo; } +/** + * What the flux team may know about an app whose specification they cannot + * read: the names of its components, whether each is election managed, and what + * the app costs in total. + * + * Deliberately NOT specification-shaped. A redacted specification is the + * dangerous thing here: it is indistinguishable from a complete one, so an + * update composed from it writes its blanks back over the customer's app. + * Nothing can mistake this shape for a spec or submit it as one. + * + * Two different claims, which is why they are two fields: + * + * `resources` is public information that happens to be sealed today. v9 keeps + * the totals OUTSIDE the encrypted envelope on purpose - a node has to judge + * whether it can host an app without being able to read it - and binds them + * into the AAD so a relayer cannot understate them. Returning them here is that + * same decision, made for a spec version that has not shipped it yet. + * + * `components` is not. v9 seals the component list and publishes only a count, + * so handing over the names is a deliberate exception rather than a claim they + * are harmless: the container tools address a component as `_`, + * so logs, terminal, monitoring and file changes cannot function without them. + * The exception is granted to an authenticated flux team caller, on a node that + * already holds the plaintext, and to nobody else. + * + * Withheld either way: environment parameters, repository credentials, secrets, + * commands, image tags, ports and domains. + * + * @param {object} req - Request object + * @param {object} res - Response object + */ +async function getApplicationComponentNamesAPI(req, res) { + try { + let { appname } = req.params; + appname = appname || req.query.appname; + + if (!appname) { + throw new Error('No Application Name specified'); + } + + const mainAppName = appname.split('_')[1] || appname; + + // fluxteam, not appownerorfluxteam: an owner reads the specification itself + // and has no use for this, and the node operator is not a party to a + // customer's app at all. + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); + if (!authorized) { + const errMessage = messageHelper.errUnauthorizedMessage(); + return res.json(errMessage); + } + + const specifications = await getApplicationSpecifications(mainAppName); + if (!specifications) { + throw new Error(`Application: ${mainAppName} not found`); + } + + const components = specifications.version >= 4 && Array.isArray(specifications.compose) + ? specifications.compose + : [specifications]; + + // Totals, not per-component sizing: it is what an app costs, it is what the + // app list renders, and it is the granularity v9 publishes. Named for the + // fields a v8 spec carries - v9 calls the same three cpu, memoryMb and + // storageGb. + const resources = components.reduce((total, component) => ({ + cpu: total.cpu + (Number(component.cpu) || 0), + ram: total.ram + (Number(component.ram) || 0), + hdd: total.hdd + (Number(component.hdd) || 0), + }), { cpu: 0, ram: 0, hdd: 0 }); + + const response = messageHelper.createDataMessage({ + components: components.map((component) => ({ + name: component.name, + // The classifier the election itself uses. A sync flag counts only on the + // primary mount, so this must agree with what decides the component's + // fate rather than with anywhere the letters happen to appear. + masterSlave: mountParser.isGComponent(component.containerData || ''), + })), + resources, + }); + + return res.json(response); + } catch (error) { + log.error(error); + const errorResponse = messageHelper.createErrorMessage( + error.message || error, + error.name, + error.code, + ); + return res.json(errorResponse); + } +} + /** * Get application specification via API * @param {object} req - Request object @@ -813,34 +931,26 @@ async function getApplicationSpecificationAPI(req, res) { throw new Error('Header with enterpriseKey is mandatory for enterprise Apps.'); } - const ownerAuthorized = await verificationHelper.verifyPrivilege( - 'appowner', - req, - mainAppName, + // Decrypting a spec is the owner's alone. A partly-redacted one would be + // neither usable nor safe: an update composed from a spec whose + // environmentParameters and repoauth have been blanked writes those blanks + // back over the customer's app, and everything left in it is still theirs. + // + // The flux team decrypts out of band instead, which keeps a decryption a + // deliberate act by a named person rather than a side effect of opening a + // page. + const authorized = await verificationHelper.verifyPrivilege( + Privilege.APP_OWNER, + authOf(req), + { appName: mainAppName }, ); - const fluxTeamAuthorized = ownerAuthorized === true - ? false - : await verificationHelper.verifyPrivilege( - 'appownerabove', - req, - mainAppName, - ); - - if (ownerAuthorized !== true && fluxTeamAuthorized !== true) { + if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); return null; } - if (fluxTeamAuthorized) { - specifications.compose.forEach((component) => { - const comp = component; - comp.environmentParameters = []; - comp.repoauth = ''; - }); - } - // this seems a bit weird, but the client can ask for the specs encrypted or decrypted. // If decrypted, they pass us another session key and we use that to encrypt. specifications.enterprise = await encryptEnterpriseFromSession( @@ -937,10 +1047,14 @@ async function updateApplicationSpecificationAPI(req, res) { } } + // The owner alone, as for the decrypt path above. This upgrades the stored + // spec and hands it back whole, encrypted to a session key the caller + // supplies - environmentParameters and repoauth included - which is the + // spec its owner is about to re-sign, and nobody else's business. const authorized = await verificationHelper.verifyPrivilege( - 'appownerabove', - req, - mainAppName, + Privilege.APP_OWNER, + authOf(req), + { appName: mainAppName }, ); if (!authorized) { @@ -1685,7 +1799,7 @@ async function reconstructAppMessagesHashCollection() { */ async function reconstructAppMessagesHashCollectionAPI(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized) { const result = await reconstructAppMessagesHashCollection(); const message = messageHelper.createSuccessMessage(result); @@ -1736,7 +1850,7 @@ async function registerAppGlobalyApi(req, res) { }); req.on('end', async () => { try { - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -1797,6 +1911,11 @@ async function registerAppGlobalyApi(req, res) { // parameters are now proper format and assigned. Check for their validity, if they are within limits, have propper ports, repotag exists, string lengths, specs are ok await appValidator.verifyAppSpecifications(appSpecFormatted, daemonHeight, true); + // placement feasibility at the front door, while the spec is still + // decrypted: an impossible spec is rejected before it is paid for, a + // diversity-constrained one is accepted with a warning + await placementFeasibility.checkPlacementFeasibility(appSpecFormatted, 'registerAppGlobalyApi'); + if (appSpecFormatted.version === 7 && appSpecFormatted.nodes.length > 0) { // eslint-disable-next-line no-restricted-syntax for (const appComponent of appSpecFormatted.compose) { @@ -1919,7 +2038,7 @@ async function reindexGlobalAppsLocation() { */ async function reindexGlobalAppsLocationAPI(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { await reindexGlobalAppsLocation(); const message = messageHelper.createSuccessMessage('Reindex successfull'); @@ -1946,7 +2065,7 @@ async function reindexGlobalAppsLocationAPI(req, res) { */ async function reindexGlobalAppsInformationAPI(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { await reindexGlobalAppsInformation(); const message = messageHelper.createSuccessMessage('Reindex successfull'); @@ -2013,7 +2132,7 @@ async function rescanGlobalAppsInformation(height = 0, removeLastInformation = f */ async function rescanGlobalAppsInformationAPI(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { let { blockheight } = req.params; // we accept both help/command and help?command=getinfo blockheight = blockheight || req.query.blockheight; @@ -2132,6 +2251,7 @@ module.exports = { appLocation, appLocationFromEvents, appInstallingLocation, + installingCountsByApp, appInstallingErrorsLocation, countAppInstallingErrors, storeAppInstallingMessage, @@ -2143,6 +2263,7 @@ module.exports = { getApplicationGlobalSpecifications, getApplicationLocalSpecifications, getApplicationSpecifications, + getApplicationComponentNamesAPI, getApplicationSpecificationAPI, updateApplicationSpecificationAPI, getApplicationOwner, diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 6104a665d0..2679e0e739 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -1,8 +1,10 @@ const config = require('config'); const util = require('util'); -const df = require('node-df'); const fs = require('node:fs'); const path = require('node:path'); +const { + SYNCTHING_FOLDER_MARKER, SYNCTHING_IGNORE_FILE, SYNCTHING_IGNORE_LINES, +} = require('../appSystem/volumeReservedNames'); const nodecmd = require('node-cmd'); const axios = require('axios'); const dbHelper = require('../dbHelper'); @@ -16,7 +18,10 @@ const fluxNetworkHelper = require('../fluxNetworkHelper'); const { DEFAULT_API_PORT, extractIp, extractPort, socketAddressesMatch, ipsMatch, } = require('../utils/socketAddressUtils'); +const fluxEventBus = require('../utils/fluxEventBus'); +const { InstallOutcome } = require('../utils/installOutcome'); const generalService = require('../generalService'); +const placementFeasibility = require('../appPlacement/placementFeasibility'); // eslint-disable-next-line no-unused-vars const upnpService = require('../upnpService'); const { @@ -27,22 +32,83 @@ const { appsFolder, appVolumesPath, legacyAppVolumesPath, + APP_VOLUME_MOUNT_OPTIONS, } = require('../utils/appConstants'); const { specificationFormatter } = require('../utils/appSpecHelpers'); +const { compareInstanceSeniority } = require('../utils/instanceOrdering'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const volumeService = require('../utils/volumeService'); const mountParser = require('../utils/mountParser'); const appReconciler = require('../appMonitoring/appReconciler'); +const { createPeerFolderLiveness, silenceVerdict, SilenceVerdict } = require('../appMonitoring/peerFolderLiveness'); +const syncthingFolderStateMachine = require('../appMonitoring/syncthingFolderStateMachine'); +const syncthingServiceModule = require('../syncthingService'); +const { getContainerDataFlags, requiresSyncing } = require('../appMonitoring/syncthingMonitorHelpers'); const appsRuntimeState = require('../appManagement/appsRuntimeState'); const { stopAppMonitoring } = require('../appManagement/appInspector'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const globalState = require('../utils/globalState'); const appNetworkLinker = require('./appNetworkLinker'); +const { Privilege, authOf } = require('../utils/privileges'); const isArcane = Boolean(process.env.FLUXOS_PATH); // Master/slave app tracking const mastersRunningGSyncthingApps = new Map(); +// When the election last reached a CONCLUSIVE verdict for an identifier: FDM +// answered, and either named a primary or named none. An absent or stale entry +// means the election is not running - it returns early before it reaches any +// app while syncthing's first-run mount-safety is outstanding, while syncthing +// is unhealthy, and per app whenever FDM cannot be reached - and "the election +// is not running" must not be readable as "there is no primary here". +// +// Without this the presence of an entry in mastersRunningGSyncthingApps is the +// only evidence there is, and its absence carries two opposite meanings at +// once. +const primaryElectionCheckedAt = new Map(); +// Consecutive passes on which the safety gate has refused to give up an app, +// keyed by name. A first refusal is the gate working and says nothing; the +// twentieth is a node that cannot establish something it needs and has not been +// able to for hours. Those two are indistinguishable at info level, and the +// giveUp:safety event does not close the gap - fluxEventBus is disabled on a +// real node (config.testEventStream is false), so the event exists for the +// harness and nothing reads it in production. +// +// Counted rather than timed because the pass is the unit: it is what re-asks +// the question, and how long it has been stuck is only meaningful in passes. +const giveUpRefusals = new Map(); +// About four hours of block passes at the production cadence - long enough that +// a folder mid-resync, a peer rebooting or a load balancer blipping has been +// and gone, short enough to still be the same day. +const REFUSALS_BEFORE_ESCALATING = 12; + +// Components this node has stopped in order to hand their app back, mapped to +// the number of give-up passes since. An evacuating node that is the elected +// primary cannot leave while it is the one writing to the volume, so it stops +// writing and asks again next pass, by which time the ordinary election has +// given the role to a peer. +// +// THIS IS ALSO WHAT KEEPS THE COMPONENT STOPPED. masterSlaveApps runs every +// 30s and would otherwise see the component not running here, clear this node's +// own stale primary record, find no peer running it yet, and start it straight +// back up - undoing the stand-down within one election cycle, every cycle. It +// is read there as a "not a candidate right now" filter, which is the whole +// mechanism: no new wire state, no negotiation, just this node declining to +// stand for an office it is trying to leave. +const standingDown = new Map(); +// Give-up passes a stood-down component may wait before this node gives up on +// leaving and becomes electable again. The alternative to a cap is an app that +// is stopped here AND not running anywhere else, which is worse than the +// stuck-but-serving state the stand-down exists to fix. On expiry the entry is +// simply dropped: masterSlaveApps then clears this node's stale primary record +// and the normal paths restart the component wherever it belongs. +const STAND_DOWN_PASSES_BEFORE_GIVING_UP = 6; +// Ten election cycles. Derived from the election's own cadence rather than +// written as its own number, so it stays in the same proportion to the pass +// that refreshes it at every scale - the harness compresses masterSlaveApps by +// 10x and this follows exactly, instead of being a constant that silently +// becomes hundreds of cycles under compression. +const PRIMARY_ELECTION_STALE_MS = (config.fluxapps.masterSlaveIntervalMs ?? 30 * 1000) * 10; const timeTostartNewMasterApp = new Map(); // Components already reported as operator-stopped, so the exclusion is announced // on entry (and again after a restart) instead of every 30s cycle. An operator @@ -52,6 +118,13 @@ const timeTostartNewMasterApp = new Map(); // misread. Cleared when the lock lifts so a later stop announces again. const operatorStoppedNoted = new Set(); +// The directories a restore may read an archive from, and so the only values +// its `type` may take. It names a directory inside the app's volume and reaches +// a shell through tar, so an unrecognised one is refused rather than +// interpolated. Matches the three prefixes backupRestoreService's path +// validation accepts. +const RESTORE_TYPES = ['local', 'remote', 'upload']; + // Promisified functions const cmdAsync = util.promisify(nodecmd.run); @@ -94,7 +167,7 @@ async function getInstalledAppsFromDb(options = {}) { }; let apps = await dbHelper.findInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); if (decryptApps) { - apps = await decryptEnterpriseApps(apps, { formatSpecs: false }); + ({ inPlace: apps } = await decryptEnterpriseApps(apps, { formatSpecs: false })); } return messageHelper.createDataMessage(apps); } catch (error) { @@ -204,7 +277,9 @@ function getFdmIndex(appName) { * @param {string} appName - Application name * @param {Object} axiosOptions - Axios request options * @returns {Promise<{ip: string|null, fdmOk: boolean}>} The master IP (FDM returns a bare IP; - * compare it with ipsMatch, which ignores the port) and success status + * compare it with ipsMatch, which ignores the port), and whether any region answered. + * A null ip with fdmOk true is FDM saying this app has no primary yet; fdmOk false is + * no region having said anything. The election acts on those two facts differently. */ async function getMasterIpFromFdm(appName, axiosOptions) { const fdmIndex = getFdmIndex(appName); @@ -214,6 +289,12 @@ async function getMasterIpFromFdm(appName, axiosOptions) { { name: 'ASIA', baseUrl: `http://fdm-sg-1-${fdmIndex}.runonflux.io:16130` }, ]; + // A region has answered when it gave a verdict about this app: a success body, + // or a 404 saying FDM holds no record of it. A 503 is FDM reporting itself as + // not ready to answer, and a body that is not success is not a verdict either - + // neither one tells us whether a primary exists, so neither may pass for one. + let answered = false; + for (const region of fdmRegions) { try { const url = `${region.baseUrl}/appips/${appName}`; @@ -221,6 +302,7 @@ async function getMasterIpFromFdm(appName, axiosOptions) { const response = await serviceHelper.axiosGet(url, axiosOptions); if (response.data && response.data.status === 'success' && response.data.data) { + answered = true; const { ips } = response.data.data; if (ips && ips.length > 0) { const ip = extractIp(ips[0]); @@ -232,8 +314,13 @@ async function getMasterIpFromFdm(appName, axiosOptions) { log.debug(`getMasterIpFromFdm: No IPs returned from ${region.name} FDM for app ${appName}`); } catch (error) { if (error.response && error.response.status === 404) { + // FDM holds no record of the app. That is an answer, and it is the answer a + // g: app gets before its first primary is ever elected - standing down on it + // would leave a newly deployed app waiting for a primary FDM cannot name. + answered = true; log.debug(`getMasterIpFromFdm: App ${appName} not found in ${region.name} FDM`); } else if (error.response && error.response.status === 503) { + // Starting up - it is not answering yet, so it has told us nothing. log.debug(`getMasterIpFromFdm: ${region.name} FDM service starting up for app ${appName}`); } else { log.error(`getMasterIpFromFdm: Failed to reach ${region.name} FDM for app ${appName}: ${error.message}`); @@ -242,8 +329,9 @@ async function getMasterIpFromFdm(appName, axiosOptions) { } } - // All regions failed or returned no IPs - return { ip: null, fdmOk: true }; + // No region named a primary. Whether that is because they said there is none + // or because none of them answered is the distinction fdmOk carries. + return { ip: null, fdmOk: answered }; } /** @@ -411,7 +499,6 @@ let dosMountMessage = ''; * @returns {Promise} */ async function createAppVolume(appSpecifications, appName, isComponent, res) { - const dfAsync = util.promisify(df); const identifier = isComponent ? `${appSpecifications.name}_${appName}` : appName; const appId = dockerService.getAppIdentifier(identifier); @@ -424,22 +511,7 @@ async function createAppVolume(appSpecifications, appName, isComponent, res) { if (res.flush) res.flush(); } - // we want whole numbers in GB - const options = { - prefixMultiplier: 'GB', - isDisplayPrefixMultiplier: false, - precision: 0, - }; - - const dfres = await dfAsync(options); - const okVolumes = []; - dfres.forEach((volume) => { - if (volume.filesystem.includes('/dev/') && !volume.filesystem.includes('loop') && !volume.mount.includes('boot')) { - okVolumes.push(volume); - } else if (volume.filesystem.includes('loop') && volume.mount === '/') { - okVolumes.push(volume); - } - }); + const okVolumes = await volumeService.capacityVolumesInGib(); // Dynamic require to avoid circular dependency // eslint-disable-next-line global-require @@ -598,7 +670,7 @@ async function createAppVolume(appSpecifications, appName, isComponent, res) { res.write(serviceHelper.ensureString(mountingStatus)); if (res.flush) res.flush(); } - await execAsRoot('mount', ['-o', 'loop', volumeFile, appDir]); + await execAsRoot('mount', ['-o', APP_VOLUME_MOUNT_OPTIONS, volumeFile, appDir]); const mountingStatus2 = { status: 'Volume mounted', }; @@ -770,7 +842,7 @@ async function createAppVolume(appSpecifications, appName, isComponent, res) { // mountpoint - it is syncthing's own guard against syncing an unmounted // dir, and the immutable bare mountpoint guarantees it can never be // recreated there. - await execAsRoot('mkdir', ['-p', path.join(appDir, '.stfolder')]); + await execAsRoot('mkdir', ['-p', path.join(appDir, SYNCTHING_FOLDER_MARKER)]); const stFolderCreation2 = { status: '.stfolder created', }; @@ -780,9 +852,10 @@ async function createAppVolume(appSpecifications, appName, isComponent, res) { if (res.flush) res.flush(); } - // Create .stignore file to exclude backup directory (in parent - // directory; the app dir is 777 by now so no elevation is needed) - await fs.promises.writeFile(path.join(appDir, '.stignore'), '/backup\n'); + // Create .stignore with the FluxOS policy lines - what keeps backup and + // an operation's staging off the network (in parent directory; the app + // dir is 777 by now so no elevation is needed) + await fs.promises.writeFile(path.join(appDir, SYNCTHING_IGNORE_FILE), `${SYNCTHING_IGNORE_LINES.join('\n')}\n`); const stiFileCreation = { status: '.stignore created', }; @@ -848,35 +921,40 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { // check if hash is in blockchain // register and launch according to specifications in message // throw without catching + // Whether THIS call raised the install hold. The guards below refuse because + // someone else is holding the node, and a refusal must not release their hold + // on its way out. + let acquired = false; try { if (globalState.removalInProgress) { const rStatus = messageHelper.createErrorMessage('Another application is undergoing removal'); log.error(rStatus); if (res) { res.write(serviceHelper.ensureString(rStatus)); - res.end(); + if (res.flush) res.flush(); } - return; + return InstallOutcome.REFUSED; } if (globalState.installationInProgress) { const rStatus = messageHelper.createErrorMessage('Another application is undergoing installation'); log.error(rStatus); if (res) { res.write(serviceHelper.ensureString(rStatus)); - res.end(); + if (res.flush) res.flush(); } - return; + return InstallOutcome.REFUSED; } globalState.installationInProgress = true; + acquired = true; const tier = await generalService.nodeTier().catch((error) => log.error(error)); if (!tier) { const rStatus = messageHelper.createErrorMessage('Failed to get Node Tier'); log.error(rStatus); if (res) { res.write(serviceHelper.ensureString(rStatus)); - res.end(); + if (res.flush) res.flush(); } - return; + return InstallOutcome.REFUSED; } const appSpecifications = appSpecs; const appComponent = componentSpecs; @@ -921,14 +999,13 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { } const appResult = await dbHelper.findOneInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); if (appResult && !isComponent) { - globalState.installationInProgress = false; const rStatus = messageHelper.createErrorMessage(`Flux App ${appName} already installed`); log.error(rStatus); if (res) { res.write(serviceHelper.ensureString(rStatus)); - res.end(); + if (res.flush) res.flush(); } - return; + return InstallOutcome.REFUSED; } // Verify the apps this app must be networked with (networkWith token in the @@ -1052,12 +1129,14 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { const successStatus = messageHelper.createSuccessMessage(`Flux App ${appName} successfully installed and launched`); log.info(successStatus); if (res) { + // Written, not closed. Every caller here is mid-stream on a response its + // own endpoint opened, and closing it from inside the installer is what + // left that endpoint writing into a finished response. res.write(serviceHelper.ensureString(successStatus)); - res.end(); + if (res.flush) res.flush(); } - globalState.installationInProgress = false; + return InstallOutcome.INSTALLED; } catch (error) { - globalState.installationInProgress = false; const errorResponse = messageHelper.createErrorMessage( error.message || error, error.name, @@ -1077,7 +1156,24 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { } // eslint-disable-next-line global-require const appUninstaller = require('./appUninstaller'); - appUninstaller.removeAppLocally(appSpecs.name, res, true); + // The teardown reports its progress into this response, and the endpoint + // closes it the moment this function returns - so it finishes first. A write + // arriving after the close is ERR_STREAM_WRITE_AFTER_END on a response still + // draining to a browser, which reaches apiServer's uncaughtException handler + // and exits the node. + await appUninstaller.removeAppLocally(appSpecs.name, res, true, false); + // The app is gone from this node, and removed without telling anyone - + // `sendMessage` is false above, so peers keep their location record until it + // expires. A caller that reads this outcome the same as a refusal either + // announces an installation that is not there, or destroys a running app + // over a scheduling collision. + return InstallOutcome.FAILED; + } finally { + // The one place the hold is released, so every way out of this function + // releases it. A tier lookup that failed used to return without releasing, + // and the node then refused every install, redeploy, spawn and reinstall + // pass it was offered until FluxOS restarted. + if (acquired) globalState.installationInProgress = false; } } @@ -1243,37 +1339,23 @@ async function softRemoveAppLocally(app, res) { * @param {object} res - Response object */ async function softRedeploy(appSpecs, res) { + // Whether softRemoveAppLocally ran to completion. False means the removal did + // not FINISH - which is not the same as "it never started": softRemoveAppLocally + // is a sequence (guards, spec lookup, per-component uninstall, database + // cleanup) and a failure part way through can leave one component's container + // already gone. What the flag is good for is the only decision made on it: the + // forced, network-broadcast uninstall below is justified once the app is + // demonstrably down, and never before. + let softRemoved = false; try { - if (globalState.removalInProgress) { - log.warn('Another application is undergoing removal'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing removal'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.installationInProgress) { - log.warn('Another application is undergoing installation'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing installation'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.softRedeployInProgress) { - log.warn('Another application is undergoing soft redeploy'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing soft redeploy'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.hardRedeployInProgress) { - log.warn('Another application is undergoing hard redeploy'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing hard redeploy'); + // Every operation, including the periodic reinstall pass - which sets its + // flag before it tears anything down and holds it across the wait, so the + // node it comes back to is still its own. + const holder = globalState.operationHolding(); + if (holder) { + const message = `Another application is undergoing ${holder}`; + log.warn(message); + const appRedeployResponse = messageHelper.createWarningMessage(message); if (res) { res.write(serviceHelper.ensureString(appRedeployResponse)); if (res.flush) res.flush(); @@ -1315,6 +1397,7 @@ async function softRedeploy(appSpecs, res) { log.info('Starting softRedeploy'); try { await softRemoveAppLocally(appSpecs.name, res); + softRemoved = true; } catch (error) { log.error(error); globalState.softRedeployInProgress = false; @@ -1332,17 +1415,68 @@ async function softRedeploy(appSpecs, res) { const appInstaller = require('./appInstaller'); await appInstaller.checkAppRequirements(appSpecs); // register - await softRegisterAppLocally(appSpecs, undefined, res); + const outcome = await softRegisterAppLocally(appSpecs, undefined, res); + if (outcome !== InstallOutcome.INSTALLED) { + // Neither outcome is undone here, and neither leaves the app as it was: + // the removal above has already taken its containers AND its local row, + // and this is the only pass that would have put them back. + // + // REFUSED is the node being held by another operation for the length of + // the delay above. FAILED is the installer's own teardown, which removes + // locally without telling anyone (`sendMessage` is false at its call). So + // in both cases this node ends with no containers, no row, and peers + // holding a location record until it expires on its own - nothing here + // announces the loss, and with no row there is nothing for the reconciler + // to converge either. + // + // Left as it is deliberately: v9 replaces this path with the operation + // registry, which is where the recovery belongs. Uninstalling and + // broadcasting from here would answer it at the cost of the app's data, + // and a retry belongs to something that owns the whole redeploy rather + // than to its last step. + const notReinstalled = messageHelper.createErrorMessage( + `Application ${appSpecs.name} was not reinstalled (${outcome})`, + ); + log.warn(notReinstalled); + if (res) { + res.write(serviceHelper.ensureString(notReinstalled)); + if (res.flush) res.flush(); + } + globalState.softRedeployInProgress = false; + return; + } log.info('Application softly redeployed'); globalState.softRedeployInProgress = false; } catch (error) { log.info('Error on softRedeploy'); log.error(error); - log.warn(`REMOVAL REASON: Soft redeploy failure - ${appSpecs.name} failed during soft redeploy: ${error.message} (softRedeploy)`); globalState.softRedeployInProgress = false; + if (!softRemoved) { + // The removal never completed, so the app was not taken down as a unit. + // Uninstalling it here - forced, and broadcast to the network - turned a + // transient failure (a concurrent reconcile racing the removal, a docker + // call finding no container) into the loss of a running application. + // Whatever state the removal did reach, the reconciler converges it: + // a container it left behind is recreated, one it left running is kept. + const failedDuringRemoval = messageHelper.createErrorMessage( + `Soft redeploy of ${appSpecs.name} failed during removal: ${error.message}. ` + + 'No forced uninstall - the app is not known to be down, and convergence is left to the reconciler.', + ); + log.warn(failedDuringRemoval); + // Told to the caller, not only to the log. Returning quietly closes the + // stream on whatever the teardown last wrote - a progress line - so a + // redeploy that did not happen answers 200 and reads as one that did. + if (res) { + res.write(serviceHelper.ensureString(failedDuringRemoval)); + if (res.flush) res.flush(); + } + return; + } + log.warn(`REMOVAL REASON: Soft redeploy failure - ${appSpecs.name} failed during soft redeploy: ${error.message} (softRedeploy)`); // eslint-disable-next-line global-require const appUninstaller = require('./appUninstaller'); - await appUninstaller.removeAppLocally(appSpecs.name, res, true, true, true); + // endResponse false: redeployAPI opened this response and closes it. + await appUninstaller.removeAppLocally(appSpecs.name, res, true, false, true); log.info(`Cleanup completed for ${appSpecs.name} after soft redeploy failure`); } } @@ -1356,36 +1490,14 @@ async function hardRedeploy(appSpecs, res) { // eslint-disable-next-line global-require const appUninstaller = require('./appUninstaller'); try { - if (globalState.removalInProgress) { - log.warn('Another application is undergoing removal'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing removal'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.installationInProgress) { - log.warn('Another application is undergoing installation'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing installation'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.softRedeployInProgress) { - log.warn('Another application is undergoing soft redeploy'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing soft redeploy'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.hardRedeployInProgress) { - log.warn('Another application is undergoing hard redeploy'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing hard redeploy'); + // Every operation, including the periodic reinstall pass - which sets its + // flag before it tears anything down and holds it across the wait, so the + // node it comes back to is still its own. + const holder = globalState.operationHolding(); + if (holder) { + const message = `Another application is undergoing ${holder}`; + log.warn(message); + const appRedeployResponse = messageHelper.createWarningMessage(message); if (res) { res.write(serviceHelper.ensureString(appRedeployResponse)); if (res.flush) res.flush(); @@ -1407,14 +1519,27 @@ async function hardRedeploy(appSpecs, res) { const appInstaller = require('./appInstaller'); await appInstaller.checkAppRequirements(appSpecs); // register - await appInstaller.registerAppLocally(appSpecs, undefined, res, false, true); // can throw + const outcome = await appInstaller.registerAppLocally(appSpecs, undefined, res, false, true); // can throw + if (outcome !== InstallOutcome.INSTALLED) { + const notReinstalled = messageHelper.createErrorMessage( + `Application ${appSpecs.name} was not reinstalled (${outcome})`, + ); + log.warn(notReinstalled); + if (res) { + res.write(serviceHelper.ensureString(notReinstalled)); + if (res.flush) res.flush(); + } + globalState.hardRedeployInProgress = false; + return; + } log.info('Application redeployed'); globalState.hardRedeployInProgress = false; } catch (error) { log.error(error); log.warn(`REMOVAL REASON: Hard redeploy failure - ${appSpecs.name} failed during hard redeploy: ${error.message} (hardRedeploy)`); globalState.hardRedeployInProgress = false; - await appUninstaller.removeAppLocally(appSpecs.name, res, true, true, true); + // endResponse false: redeployAPI opened this response and closes it. + await appUninstaller.removeAppLocally(appSpecs.name, res, true, false, true); log.info(`Cleanup completed for ${appSpecs.name} after hard redeploy failure`); } } @@ -1432,36 +1557,14 @@ async function softRedeployComponent(appName, componentName, res) { const appInstaller = require('./appInstaller'); try { - if (globalState.removalInProgress) { - log.warn('Another application is undergoing removal'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing removal'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.installationInProgress) { - log.warn('Another application is undergoing installation'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing installation'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.softRedeployInProgress) { - log.warn('Another application is undergoing soft redeploy'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing soft redeploy'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.hardRedeployInProgress) { - log.warn('Another application is undergoing hard redeploy'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing hard redeploy'); + // Every operation, including the periodic reinstall pass - which sets its + // flag before it tears anything down and holds it across the wait, so the + // node it comes back to is still its own. + const holder = globalState.operationHolding(); + if (holder) { + const message = `Another application is undergoing ${holder}`; + log.warn(message); + const appRedeployResponse = messageHelper.createWarningMessage(message); if (res) { res.write(serviceHelper.ensureString(appRedeployResponse)); if (res.flush) res.flush(); @@ -1494,10 +1597,33 @@ async function softRedeployComponent(appName, componentName, res) { } const fullComponentName = `${componentName}_${appName}`; + const componentAppId = dockerService.getAppIdentifier(fullComponentName); + + // Whether softUninstallComponent ran to completion. False means the removal + // did not FINISH, which is not the same as "it never started" - and it is the + // only thing the forced, network-broadcast uninstall below is decided on. + let componentRemoved = false; try { log.warn(`Beginning Soft Redeployment of component ${fullComponentName}...`); - await appUninstaller.softUninstallComponent(fullComponentName, null, componentSpec, res, stopAppMonitoring); + // Both arguments used to be wrong, and softUninstallComposedApp is the + // reference for what they should be: the BARE app name, and the docker id + // of the component. + // + // The id was passed as `null`. softUninstallComponent hands it straight to + // appDockerStop (whose `.catch` swallowed it) and then appDockerRemove + // (which does not), where it reached getAppIdentifier and threw on + // `null.startsWith` before any container was looked up. So EVERY soft + // component redeploy failed, and the catch below answered by uninstalling + // the whole app - forced, and broadcast to the network. + // + // The name was passed already joined. The callee builds + // `${component}_${appName}` from it for the monitoring key and hands it to + // cleanupPorts, so `frontend_myapp` became `frontend_frontend_myapp`: the + // stop targeted a monitor that does not exist and the real one kept + // sampling a container that was gone. + await appUninstaller.softUninstallComponent(appName, componentAppId, componentSpec, res, stopAppMonitoring); + componentRemoved = true; const appRedeployResponse = messageHelper.createSuccessMessage(`Component ${fullComponentName} softly removed. Awaiting installation...`); log.info(appRedeployResponse); @@ -1513,15 +1639,62 @@ async function softRedeployComponent(appName, componentName, res) { // Register component log.warn(`Continuing Soft Redeployment of component ${fullComponentName}...`); - await softRegisterAppLocally(appSpecifications, componentSpec, res); + const outcome = await softRegisterAppLocally(appSpecifications, componentSpec, res); + if (outcome !== InstallOutcome.INSTALLED) { + // Neither outcome reaches the catch below, which would uninstall the + // WHOLE app over one component: a scheduling collision is not grounds + // for that, and on FAILED the installer's teardown is already running. + // + // What neither outcome does is put the component back. The soft removal + // above has taken it down, so this node is left short a component with + // nothing announcing it - the same gap the whole-app path has, and left + // to v9's operation registry for the same reason. + const notReinstalled = messageHelper.createErrorMessage( + `Component ${fullComponentName} was not reinstalled (${outcome})`, + ); + log.warn(notReinstalled); + // Reported, even though there is nothing to undo. Returning quietly here + // leaves the caller's stream closing on whatever the installer wrote last + // - and on FAILED that is its teardown's "was successfuly removed", so a + // destroyed app reads as a completed redeploy. + if (res) { + res.write(serviceHelper.ensureString(notReinstalled)); + if (res.flush) res.flush(); + } + globalState.softRedeployInProgress = false; + return; + } log.info(`Component ${fullComponentName} softly redeployed`); + // The only report that a SINGLE component was replaced. app:installed and + // app:removed both speak for a whole app, so neither fires here and neither + // could: this path leaves the app installed throughout, which is the point + // of it. Nothing observing the node could tell a component redeploy from + // never having been asked - which is how this endpoint failing on every + // call went unnoticed. + fluxEventBus.publish('app:componentRedeployed', { + name: appName, component: componentName, identifier: fullComponentName, hard: false, + }); globalState.softRedeployInProgress = false; } catch (error) { log.error(error); - log.warn(`REMOVAL REASON: Soft redeploy failure - ${appName} being removed after component ${fullComponentName} failed during soft redeploy: ${error.message} (softRedeployComponent)`); globalState.softRedeployInProgress = false; - await appUninstaller.removeAppLocally(appName, res, true, true, true); + if (!componentRemoved) { + // One component's removal did not complete, and the answer to that was + // to uninstall the WHOLE app - forced, and broadcast to the network. A + // transient failure (the reconciler racing the removal, appDockerRemove + // finding no container) took every other component of the app with it. + // Whatever state the removal did reach, the reconciler converges it: a + // container it left behind is recreated, one it left running is kept. + // The throw is what tells the caller, and redeployComponentAPI answers + // on it. + log.warn(`Soft redeploy of ${fullComponentName} failed during removal: ${error.message}. ` + + 'No forced uninstall - the app is not known to be down, and convergence is left to the reconciler.'); + throw error; + } + log.warn(`REMOVAL REASON: Soft redeploy failure - ${appName} being removed after component ${fullComponentName} failed during soft redeploy: ${error.message} (softRedeployComponent)`); + // endResponse false: the endpoint opened this response and closes it. + await appUninstaller.removeAppLocally(appName, res, true, false, true); log.info(`Cleanup completed for ${appName} after component ${fullComponentName} soft redeploy failure`); throw error; } @@ -1546,36 +1719,14 @@ async function hardRedeployComponent(appName, componentName, res) { const appInstaller = require('./appInstaller'); try { - if (globalState.removalInProgress) { - log.warn('Another application is undergoing removal'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing removal'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.installationInProgress) { - log.warn('Another application is undergoing installation'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing installation'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.softRedeployInProgress) { - log.warn('Another application is undergoing soft redeploy'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing soft redeploy'); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - return; - } - if (globalState.hardRedeployInProgress) { - log.warn('Another application is undergoing hard redeploy'); - const appRedeployResponse = messageHelper.createWarningMessage('Another application is undergoing hard redeploy'); + // Every operation, including the periodic reinstall pass - which sets its + // flag before it tears anything down and holds it across the wait, so the + // node it comes back to is still its own. + const holder = globalState.operationHolding(); + if (holder) { + const message = `Another application is undergoing ${holder}`; + log.warn(message); + const appRedeployResponse = messageHelper.createWarningMessage(message); if (res) { res.write(serviceHelper.ensureString(appRedeployResponse)); if (res.flush) res.flush(); @@ -1608,11 +1759,17 @@ async function hardRedeployComponent(appName, componentName, res) { const fullComponentName = `${componentName}_${appName}`; + // Same decision as the soft path: whether hardUninstallComponent finished is + // the only thing the forced, network-broadcast uninstall below is decided on. + let componentRemoved = false; + try { log.warn(`Beginning Hard Redeployment of component ${fullComponentName}...`); log.warn(`REMOVAL REASON: Hard redeploy initiated - ${fullComponentName} being removed as part of hard redeploy process (hardRedeployComponent)`); - await appUninstaller.hardUninstallComponent(fullComponentName, null, componentSpec, res, stopAppMonitoring, false); + // same contract as the soft path above: bare app name, real docker id + await appUninstaller.hardUninstallComponent(appName, dockerService.getAppIdentifier(fullComponentName), componentSpec, res, stopAppMonitoring, false); + componentRemoved = true; const appRedeployResponse = messageHelper.createSuccessMessage(`Component ${fullComponentName} removed. Awaiting installation...`); log.info(appRedeployResponse); @@ -1628,15 +1785,53 @@ async function hardRedeployComponent(appName, componentName, res) { // Register component log.warn(`Continuing Hard Redeployment of component ${fullComponentName}...`); - await appInstaller.registerAppLocally(appSpecifications, componentSpec, res); + const outcome = await appInstaller.registerAppLocally(appSpecifications, componentSpec, res); + if (outcome !== InstallOutcome.INSTALLED) { + // Neither outcome reaches the catch below, which would uninstall the + // WHOLE app over one component: a scheduling collision is not grounds + // for that, and on FAILED the installer's teardown is already running. + // + // What neither outcome does is put the component back. The soft removal + // above has taken it down, so this node is left short a component with + // nothing announcing it - the same gap the whole-app path has, and left + // to v9's operation registry for the same reason. + const notReinstalled = messageHelper.createErrorMessage( + `Component ${fullComponentName} was not reinstalled (${outcome})`, + ); + log.warn(notReinstalled); + // Reported, even though there is nothing to undo. Returning quietly here + // leaves the caller's stream closing on whatever the installer wrote last + // - and on FAILED that is its teardown's "was successfuly removed", so a + // destroyed app reads as a completed redeploy. + if (res) { + res.write(serviceHelper.ensureString(notReinstalled)); + if (res.flush) res.flush(); + } + globalState.hardRedeployInProgress = false; + return; + } log.info(`Component ${fullComponentName} hard redeployed`); + // Same fact as the soft path, and `hard` is the consequence that differs: + // the component's volume was rebuilt, so its data on this node is gone. + fluxEventBus.publish('app:componentRedeployed', { + name: appName, component: componentName, identifier: fullComponentName, hard: true, + }); globalState.hardRedeployInProgress = false; } catch (error) { log.error(error); - log.warn(`REMOVAL REASON: Hard redeploy failure - ${appName} being removed after component ${fullComponentName} failed during hard redeploy: ${error.message} (hardRedeployComponent)`); globalState.hardRedeployInProgress = false; - await appUninstaller.removeAppLocally(appName, res, true, true, true); + if (!componentRemoved) { + // See the soft path. A hard redeploy asks for one component's volume to + // be rebuilt, and a removal that did not finish is not grounds for + // destroying the components beside it and telling the network. + log.warn(`Hard redeploy of ${fullComponentName} failed during removal: ${error.message}. ` + + 'No forced uninstall - the app is not known to be down, and convergence is left to the reconciler.'); + throw error; + } + log.warn(`REMOVAL REASON: Hard redeploy failure - ${appName} being removed after component ${fullComponentName} failed during hard redeploy: ${error.message} (hardRedeployComponent)`); + // endResponse false: the endpoint opened this response and closes it. + await appUninstaller.removeAppLocally(appName, res, true, false, true); log.info(`Cleanup completed for ${appName} after component ${fullComponentName} hard redeploy failure`); throw error; } @@ -1685,8 +1880,12 @@ async function redeployComponentAPI(req, res) { force = force || req.query.force || false; force = serviceHelper.ensureBoolean(force); - // Authorization check - must be app owner or above - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); + // This refuses the node operator, and a redeploy is an + // uninstall followed by a reinstall. With force it is the hard one, which + // unmounts the component's volume and rm -rf's it - the app's data on this + // node is gone. The same gate appremove asks for, which this would otherwise + // be the way around. + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -1695,14 +1894,15 @@ async function redeployComponentAPI(req, res) { res.setHeader('Content-Type', 'application/json'); + // This response has one owner and it is here. The redeploy paths below write + // their progress into it and none of them close it, so every exit - a + // completed redeploy, a failure, and the four guards that refuse the request + // outright - reaches the same close. if (force) { await hardRedeployComponent(appname, component, res); } else { await softRedeployComponent(appname, component, res); } - - const successMessage = messageHelper.createSuccessMessage(`Component ${component} of ${appname} redeployed successfully`); - res.json(successMessage); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage( @@ -1710,7 +1910,20 @@ async function redeployComponentAPI(req, res) { error.name, error.code, ); - res.json(errorResponse); + // Before anything has been written the status line is still ours, so a + // refusal can be answered as one. Once the body has started it cannot, and + // the envelope goes into the stream where a client parses it out. + if (res.headersSent) { + res.write(serviceHelper.ensureString(errorResponse)); + } else { + res.json(errorResponse); + } + } finally { + // Writing to a closed response is not a caught error: node reports it a tick + // later as an `error` event on res, nothing listens for one, and an unheard + // `error` reaches apiServer's uncaughtException handler and exits the + // process. Closing from one place is what keeps every write above it live. + if (!res.writableEnded) res.end(); } } @@ -1738,6 +1951,9 @@ async function redeployAPI(req, res) { const redeploySkip = globalState.restoreInProgress.some((backupItem) => appname === backupItem); if (redeploySkip) { log.info(`Restore is running for ${appname}, redeploy skipped...`); + // Answered, not just closed: the caller asked for a redeploy and did not + // get one, and an empty body would read as success. + res.json(messageHelper.createWarningMessage(`Restore is running for ${appname}, redeploy skipped`)); return; } @@ -1745,7 +1961,12 @@ async function redeployAPI(req, res) { force = force || req.query.force || false; force = serviceHelper.ensureBoolean(force); - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); + // This refuses the node operator, and a redeploy is an + // uninstall followed by a reinstall. With force it is the hard one, which + // unmounts the component's volume and rm -rf's it - the app's data on this + // node is gone. The same gate appremove asks for, which this would otherwise + // be the way around. + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -1755,7 +1976,7 @@ async function redeployAPI(req, res) { // Dynamic require to avoid circular dependency // eslint-disable-next-line global-require const appController = require('../appManagement/appController'); - appController.executeAppGlobalCommand(appname, 'redeploy', req.headers.zelidauth, force); // do not wait + appController.executeAppGlobalCommand(appname, 'redeploy', authOf(req), force); // do not wait const hardOrSoft = force ? 'hard' : 'soft'; const appResponse = messageHelper.createSuccessMessage(`${appname} queried for global ${hardOrSoft} redeploy`); res.json(appResponse); @@ -1784,24 +2005,33 @@ async function redeployAPI(req, res) { error.name, error.code, ); - res.json(errorResponse); + if (res.headersSent) { + res.write(serviceHelper.ensureString(errorResponse)); + } else { + res.json(errorResponse); + } + } finally { + // Same owner rule as redeployComponentAPI: the redeploy writes progress, this + // closes. The guards inside softRedeploy return without closing, and a + // failure after the stream has started leaves nothing else that can. + if (!res.writableEnded) res.end(); } } /** - * Helper function to send chunk of data to response stream with delay + * Write one progress line to the response stream. + * + * The flush is what makes the line arrive: Express's compression middleware + * buffers small res.write calls to build compressible chunks, so a progress + * stream without it sits in that buffer instead of reaching the caller. + * * @param {object} res - Response object * @param {string} chunk - Data chunk to send * @returns {Promise} */ async function sendChunk(res, chunk) { - return new Promise((resolve) => { - setTimeout(() => { - res.write(`${chunk}\n`); - if (res.flush) res.flush(); - resolve(); - }, 3000); // Adjust the delay as needed - }); + res.write(`${chunk}\n`); + if (res.flush) res.flush(); } /** @@ -1818,12 +2048,9 @@ async function stopSyncthingApp(appComponentName, res) { // eslint-disable-next-line global-require const syncthingService = require('../syncthingService'); const allSyncthingFolders = await syncthingService.getConfigFolders(); - if (allSyncthingFolders.status === 'error') { - return; - } let folderId = null; // eslint-disable-next-line no-restricted-syntax - for (const syncthingFolder of allSyncthingFolders.data) { + for (const syncthingFolder of allSyncthingFolders) { if (syncthingFolder.path === folder || syncthingFolder.path.includes(`${folder}/`)) { folderId = syncthingFolder.id; } @@ -1834,14 +2061,6 @@ async function stopSyncthingApp(appComponentName, res) { // remove folder from syncthing // eslint-disable-next-line no-await-in-loop await syncthingService.adjustConfigFolders('delete', undefined, folderId); - // check if restart is needed - // eslint-disable-next-line no-await-in-loop - const restartRequired = await syncthingService.getConfigRestartRequired(); - if (restartRequired.status === 'success' && restartRequired.data.requiresRestart === true) { - log.info('Syncthing restart required, restarting...'); - // eslint-disable-next-line no-await-in-loop - await syncthingService.systemRestart(); - } const adjustSyncthingB = { status: 'Syncthing adjusted', }; @@ -1876,16 +2095,12 @@ async function changeSyncthingFolderType(folderId, folderType) { log.info(`Changing syncthing folder ${folderId} to ${folderType} mode`); // Get current folder configuration - const foldersResponse = await syncthingService.getConfigFolders(); - if (foldersResponse.status !== 'success') { - log.error(`Failed to get syncthing folders: ${JSON.stringify(foldersResponse)}`); - return false; - } + const folders = await syncthingService.getConfigFolders(); // Find the folder by path // Syncthing syncs the entire appId folder (includes all subdirectories) const folderPath = `${appsFolder}${folderId}`; - const folder = foldersResponse.data.find((f) => f.path === folderPath); + const folder = folders.find((f) => f.path === folderPath); if (!folder) { log.error(`Syncthing folder not found for path: ${folderPath}`); @@ -1914,6 +2129,114 @@ async function changeSyncthingFolderType(folderId, folderType) { } } +/** + * The syncthing folder id backing a component's volume. Folder ids ARE the + * docker app identifiers, so a composed app has one folder PER COMPONENT and + * the app name alone never names a folder. Legacy (version <= 3) apps pass the + * literal 'null' component, mirroring IOUtils.getVolumeInfo. + * @param {string} appname - Application name + * @param {string} componentName - Component name, or 'null' for a legacy app + * @returns {string} Syncthing folder id + */ +function syncthingFolderIdForComponent(appname, componentName) { + return componentName === 'null' + ? dockerService.getAppIdentifier(appname) + : dockerService.getAppIdentifier(`${componentName}_${appname}`); +} + +/** + * Pause or resume one syncthing folder. Pausing stops that folder's runner - + * and therefore all writes to its directory - while leaving the folder config + * and its index in place, so it is the right way to hold data still for the + * duration of an operation. Deleting the folder instead loses the config, and + * only the syncthing monitor's per-app pass ever recreates it. + * + * Scoped to a single folder: the daemon and every other folder keep running. + * A paused/resumed folder never sets syncthing's restart-required flag, so no + * process restart is involved (verified against syncthing v2 - the folder + * runner is stopped and, when unpausing, started again in place). + * @param {string} folderId - Syncthing folder ID + * @param {boolean} paused - Desired paused state + * @returns {Promise} - true if applied, false otherwise + */ +async function setSyncthingFolderPaused(folderId, paused) { + try { + const response = await syncthingServiceModule.adjustConfigFolders('patch', { paused }, folderId); + if (response.status === 'success') { + log.info(`setSyncthingFolderPaused - ${folderId} paused=${paused}`); + return 'held'; + } + // Only a bare 404 proves syncthing replied and holds no such folder, so + // nothing is replicating it and the caller may proceed. The HTTP status + // itself decides - the axios code cannot, ERR_BAD_REQUEST spans every 4xx, + // so a 403 from a stale api key would read the same as absence. Any other + // answer leaves the folder possibly live and unheld, which the caller + // must refuse on. + if (response.data?.httpStatus === 404) { + log.info(`setSyncthingFolderPaused - ${folderId} is unknown to syncthing; nothing to hold still`); + return 'absent'; + } + log.error(`setSyncthingFolderPaused - ${folderId} paused=${paused} failed: ${JSON.stringify(response)}`); + return 'failed'; + } catch (error) { + log.error(`setSyncthingFolderPaused - ${folderId} paused=${paused} failed: ${error.message}`); + return 'failed'; + } +} + +/** + * An app's components in one shape whatever its version. A version <= 3 app has + * no compose array and one implicit component, addressed as the literal 'null' - + * the identifier IOUtils.getVolumeInfo and the syncthing folder ids both use for + * it. + * @param {object} appDetails - Global app specification + * @returns {Array<{name: string, containerData: string}>} Components + */ +function componentsOfApp(appDetails) { + return appDetails.version <= 3 + ? [{ name: 'null', containerData: appDetails.containerData }] + : (appDetails.compose || []); +} + +/** + * How a component's data is replicated, which is what decides whether an + * operation on it has to reach beyond this node: + * + * - `none` the data is this instance's own and reaches nobody. + * - `elected` (g:) one instance runs at a time, so every other copy is + * quiescent and syncthing carries a change to them. + * - `shared` (r:/s:) every instance runs and writes, so their containers are + * holding data a change here has just replaced underneath them. + * + * Read from the PRIMARY mount's flags, which is what syncthing itself is + * configured from - a flag on a later mount configures nothing, so a substring + * search over the whole containerData reports sync on apps that have none. + * @param {string} containerData - Component containerData + * @returns {string} 'none' | 'elected' | 'shared' + */ +function syncModeOfComponent(containerData) { + const flags = getContainerDataFlags((containerData || '').split('|')[0]); + if (!requiresSyncing(flags)) return 'none'; + return flags.includes('g') ? 'elected' : 'shared'; +} + +/** + * The components of an app whose data is synced, paired with their syncthing + * folder ids. Uses the same predicate that decides a folder is created at all, + * so this can never disagree with what syncthing is actually configured with. + * @param {object} appDetails - Global app specification + * @param {string} appname - Application name + * @returns {Array<{componentName: string, folderId: string}>} Synced components + */ +function syncedComponentsOfApp(appDetails, appname) { + return componentsOfApp(appDetails) + .filter((comp) => syncModeOfComponent(comp.containerData) !== 'none') + .map((comp) => ({ + componentName: comp.name, + folderId: syncthingFolderIdForComponent(appname, comp.name), + })); +} + /** * Helper function to apply permissions fix on persistent container data * Fixes permissions on appdata and all additional mount points @@ -1979,41 +2302,130 @@ async function appDockerStart(appname) { } } +// A dockerd restart takes seconds, and the reconciler already retries an +// unreachable daemon on this cadence rather than acting on what it could not +// read. Waiting here is not a stand-in for a fact - it is how the fact becomes +// obtainable, and the attempt ceiling is what stops it waiting forever. +const DOCKER_SETTLE_POLL_MS = 5000; +const DOCKER_SETTLE_ATTEMPTS = 12; + /** - * Helper function to stop app docker containers - * @param {string} appname - App name - * @returns {Promise} + * This container's run state, once docker is in a position to answer for it. + * + * appReconciler.dockerActual tells three failures apart that an inspect error + * cannot: the daemon being unreachable, the daemon being up but that one + * inspect failing, and the container genuinely being gone. The first two mean + * we did not learn anything, so they are waited out rather than read as an + * answer - and the wait holds the caller's backup/restore lease, which is the + * point. An operation that gives up here releases that lease and hands the app + * back to the reconciler in the middle of its own work. + * + * @param {string} identifier - Component identifier + * @returns {Promise} dockerActual's verdict, or null if the daemon + * never became able to answer */ -async function appDockerStop(appname) { - try { - // eslint-disable-next-line global-require - const registryManager = require('../appDatabase/registryManager'); +async function settledDockerState(identifier, onWait) { + // eslint-disable-next-line no-plusplus + for (let attempt = 1; attempt <= DOCKER_SETTLE_ATTEMPTS; attempt++) { + // eslint-disable-next-line no-await-in-loop + const actual = await appReconciler.dockerActual(identifier); + if (actual.reachable && !actual.indeterminate) return actual; + if (attempt === DOCKER_SETTLE_ATTEMPTS) break; + const waiting = `Docker is not answering for ${identifier} yet, waiting (${attempt}/${DOCKER_SETTLE_ATTEMPTS - 1})...`; + log.warn(`appDockerStop - ${waiting}`); + // The response is a stream that has already returned 200, so a minute of + // silence is a minute in which anything between here and the browser may + // decide the connection is idle. Saying what we are waiting for keeps it + // alive and tells the operator something true. + // eslint-disable-next-line no-await-in-loop + if (onWait) await onWait(waiting); + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(DOCKER_SETTLE_POLL_MS); + } + return null; +} + +/** + * Stop every container the given app name covers, and answer whether they are + * all actually down. + * + * Every component is attempted even when one fails. The catch used to sit + * outside the loop, so the first component that would not stop skipped every + * component after it, and the caller - which then went on to replace the app's + * data - saw nothing at all. + * + * The verdict is read back from docker rather than taken from the stop call. A + * stop that returned is not the same as a container that is down, and appdata + * lives on a volume a running container is still writing to: clearing it under + * one leaves the app writing into a half-emptied tree and able to save its own + * state back over whatever the restore puts there. + * + * @param {string} appname - App name, or a single component identifier + * @param {Function} [onWait] - Called with a progress line while waiting for docker + * @returns {Promise<{stopped: boolean, running: string[], unavailable: boolean, errors: string[]}>} + * `running` names the components docker reports as still up - what a caller + * about to destroy data must refuse on. `unavailable` says docker never became + * able to answer, which is a different refusal with a different remedy. + */ +async function appDockerStop(appname, onWait) { + // eslint-disable-next-line global-require + const registryManager = require('../appDatabase/registryManager'); + let identifiers = []; + try { const mainAppName = appname.split('_')[1] || appname; - const isComponent = appname.includes('_'); - if (isComponent) { - await dockerService.appDockerStop(appname); - stopAppMonitoring(appname, false); + if (appname.includes('_')) { + identifiers = [appname]; } else { const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - if (appSpecs.version <= 3) { - await dockerService.appDockerStop(appname); - stopAppMonitoring(appname, false); - } else { - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose) { - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerStop(`${appComponent.name}_${appSpecs.name}`); - stopAppMonitoring(`${appComponent.name}_${appSpecs.name}`, false); - } - } + if (!appSpecs) throw new Error('Application not found'); + identifiers = appSpecs.version <= 3 + ? [appname] + : appSpecs.compose.map((component) => `${component.name}_${appSpecs.name}`); } } catch (error) { log.error(error); + // The component list itself is unknown, so nothing can be asserted about + // what is running - which is a refusal, not an empty success. + return { + stopped: false, running: [], unavailable: false, errors: [error.message], + }; + } + + const running = []; + const errors = []; + let unavailable = false; + // eslint-disable-next-line no-restricted-syntax + for (const identifier of identifiers) { + try { + // eslint-disable-next-line no-await-in-loop + await dockerService.appDockerStop(identifier); + stopAppMonitoring(identifier, false); + } catch (error) { + // The stop failing is not itself the verdict - docker may simply have been + // mid-restart. What counts is what it says afterwards. + log.error(`appDockerStop - ${identifier}: ${error.message}`); + errors.push(`${identifier}: ${error.message}`); + } + // eslint-disable-next-line no-await-in-loop + const actual = await settledDockerState(identifier, onWait); + if (!actual) { + // The daemon is a property of the node, not of this component: having + // waited it out once, waiting again for each remaining component only + // multiplies the refusal's latency by the compose count. + unavailable = true; + errors.push(`${identifier}: docker never became able to answer`); + break; + } + if (actual.running) { + running.push(identifier); + } + // reachable and not running - stopped, or gone, which is also not running } + + return { + stopped: running.length === 0 && !unavailable, running, unavailable, errors, + }; } /** @@ -2089,6 +2501,17 @@ async function appDockerRestart(appname) { * @returns {Promise} */ async function requestMasterStartWithPermissionsFix(appname, appId) { + // Claimed before the ownership fix, not after it: the fix takes long enough + // that a peer probing "is anyone running this?" would otherwise get a truthful + // no from a node that has already committed, and start alongside it. Released + // in the finally - from a successful start the controllerDesired below carries + // the claim, and a failed one must stop claiming. + appReconciler.claimStarting(appname); + // A fact - this node has decided to become primary and is committing to it. + // The cadence around this decision is a counter, not an event: see the rule at + // the top of fluxEventBus.js. + fluxEventBus.publish('masterSlave:started', { identifier: appname }); + fluxEventBus.count('masterSlave:decision', appname, 'started'); try { log.info(`Preparing masterSlave primary ${appname}: fixing permissions before start`); @@ -2117,6 +2540,8 @@ async function requestMasterStartWithPermissionsFix(appname, appId) { } catch (error) { log.error(`Error preparing masterSlave primary ${appname}: ${error.message}`); // leave it stopped if the permissions-fix workflow failed + } finally { + appReconciler.releaseStarting(appname); } } @@ -2128,6 +2553,9 @@ async function requestMasterStartWithPermissionsFix(appname, appId) { async function appendBackupTask(req, res) { let appname; let backup; + let force; + // folders this task paused, so a failure anywhere below can resume them + const pausedFolderIds = []; try { const processedBody = serviceHelper.ensureObject(req.body); log.info(processedBody); @@ -2135,17 +2563,24 @@ async function appendBackupTask(req, res) { appname = processedBody.appname; // eslint-disable-next-line prefer-destructuring backup = processedBody.backup; + force = processedBody.force === true || processedBody.force === 'true'; if (!appname || !backup) { throw new Error('appname and backup parameters are mandatory'); } - const indexBackup = globalState.backupInProgress.indexOf(appname); - if (indexBackup !== -1) { - throw new Error('Backup in progress...'); + if (!Array.isArray(backup)) { + throw new Error('backup must be a list of components'); } const hasTrueBackup = backup.some((backupitem) => backupitem.backup); if (hasTrueBackup === false) { throw new Error('No backup jobs...'); } + // The claim, before any awaited work: a second request for the same app + // finds it taken here rather than passing an emptied check and racing to + // the archive alongside the first. Last in this synchronous block, so a + // validation throw above never leaves it claimed. + if (!globalState.tryStartBackup(appname)) { + throw new Error('Backup in progress...'); + } } catch (error) { log.error(error); await sendChunk(res, `${error?.message}\n`); @@ -2153,24 +2588,91 @@ async function appendBackupTask(req, res) { return false; } try { - const authorized = res ? await verificationHelper.verifyPrivilege('appownerabove', req, appname) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (authorized === true) { - globalState.backupInProgress.push(appname); - // Check if app using syncthing, stop syncthing for all component that using it // eslint-disable-next-line global-require const registryManager = require('../appDatabase/registryManager'); const appDetails = await registryManager.getApplicationGlobalSpecifications(appname); + if (!appDetails) { + throw new Error(`Refused: no specifications found for ${appname}`); + } + const requested = new Set(backup.filter((item) => item.backup).map((item) => item.component)); + const allSyncedComponents = syncedComponentsOfApp(appDetails, appname); + const syncedComponents = allSyncedComponents.filter((comp) => requested.has(comp.componentName)); + + // An archive is only worth keeping if this instance holds a complete copy. + // A synced app's data lives on every instance, and a backup is deliberately + // taken from a standby - the quiescent one - so the question is never "is + // this the primary" but "is this copy whole". An index that is behind, or + // absent entirely (a folder syncthing was never configured with), yields an + // archive of whatever happens to be on disk, which can be nothing at all. + // Checked BEFORE anything is stopped: a refusal must not cost a healthy app + // an outage. + const incomplete = []; // eslint-disable-next-line no-restricted-syntax - const syncthing = appDetails.compose.find((comp) => comp.containerData.includes('g:') || comp.containerData.includes('r:') || comp.containerData.includes('s:')); - if (syncthing) { - // eslint-disable-next-line no-await-in-loop - await sendChunk(res, `Stopping syncthing for ${appname}\n`); + for (const { componentName, folderId } of syncedComponents) { // eslint-disable-next-line no-await-in-loop - await stopSyncthingApp(appname, res); + const { status: syncStatus, reason } = await syncthingFolderStateMachine + .probeFolderSyncCompletion(folderId); + if (reason === 'absent') { + incomplete.push(`${componentName}: no syncthing folder - this instance has never synced`); + } else if (reason === 'unknown') { + // Syncthing not answering says nothing about the data. Refusing is + // still right - an archive of an unverified copy is the thing that + // looks fine now and loses data when it is restored months later - + // but the reason given has to be the one that actually happened. + incomplete.push(`${componentName}: syncthing did not answer - sync state could not be determined`); + } else if (syncStatus.globalBytes === 0) { + // With nothing in the global index there is nothing to be a fraction + // of, and the percentage defaults to 100 - which would tell an operator + // the copy is complete in the same breath as refusing it. isSynced is + // already false here for the same reason; only the wording was wrong. + incomplete.push(`${componentName}: nothing in the sync index yet - cannot confirm this copy holds the data`); + } else if (!syncStatus.isSynced) { + incomplete.push(`${componentName}: ${syncStatus.syncPercentage.toFixed(2)}% synced (${syncStatus.inSyncBytes}/${syncStatus.globalBytes} bytes)`); + } + } + if (incomplete.length > 0) { + const summary = incomplete.join('; '); + if (!force) { + throw new Error(`Refusing to back up an incomplete copy - ${summary}. Back up from a fully synced instance, or repeat with force to archive what is on disk anyway.`); + } + log.warn(`appendBackupTask - ${appname} forced over an incomplete copy - ${summary}`); + await sendChunk(res, `WARNING: backing up an incomplete copy - ${summary}\n`); } - await sendChunk(res, 'Stopping application...\n'); - await appDockerStop(appname); + // Hold the data still by pausing the folders being archived - the folder + // runner stops, so nothing writes underneath the archive. Deleting them + // instead (as this once did) loses the folder config, and only the + // syncthing monitor's per-app pass ever puts it back; on a node where that + // pass cannot complete, the app silently stops being redundant forever. + // eslint-disable-next-line no-restricted-syntax + for (const { folderId } of syncedComponents) { + // eslint-disable-next-line no-await-in-loop + await sendChunk(res, `Pausing syncthing folder ${folderId}\n`); + // eslint-disable-next-line no-await-in-loop + const held = await setSyncthingFolderPaused(folderId, true); + if (held === 'held') pausedFolderIds.push(folderId); + // An unheld folder is still pulling, so the archive would be taken over + // data that moves underneath it. A torn archive is the thing this whole + // path exists to stop being created. + if (held === 'failed') { + throw new Error(`Refused: ${folderId} could not be held still, so an archive taken now could be inconsistent`); + } + } + const syncthing = allSyncedComponents.length > 0; + + await sendChunk(res, 'Stopping application...\n'); + const stopVerdict = await appDockerStop(appname, (line) => sendChunk(res, `${line}\n`)); + // Same reason the folders are held: an archive taken while a container is + // still writing is torn, and a torn archive is what this path exists to + // stop being created. + if (stopVerdict.unavailable) { + throw new Error(`Refused: docker is not answering on this node, so ${appname} cannot be confirmed stopped - try again shortly`); + } + if (!stopVerdict.stopped) { + throw new Error(`Refused: ${stopVerdict.running.join(', ') || appname} could not be stopped, so an archive taken now could be inconsistent`); + } await serviceHelper.delay(5 * 1000); // eslint-disable-next-line global-require const IOUtils = require('../IOUtils'); @@ -2178,16 +2680,19 @@ async function appendBackupTask(req, res) { for (const component of backup) { if (component.backup) { // eslint-disable-next-line no-await-in-loop - const componentPath = await IOUtils.getVolumeInfo(appname, component.component, 'B', 0, 'mount'); - const targetPath = `${componentPath[0].mount}/appdata`; - const tarGzPath = `${componentPath[0].mount}/backup/local/backup_${component.component.toLowerCase()}.tar.gz`; + const { error: mountError, mounts } = await IOUtils.getVolumeInfo(appname, component.component, 'B', 0, 'mount'); + if (mountError || !mounts.length) { + throw new Error(`Refused: ${component.component} volume is not mounted, so it cannot be archived`); + } + const targetPath = `${mounts[0].mount}/appdata`; + const tarGzPath = `${mounts[0].mount}/backup/local/backup_${component.component.toLowerCase()}.tar.gz`; // eslint-disable-next-line no-await-in-loop - const existStatus = await IOUtils.checkFileExists(`${componentPath[0].mount}/backup/local/backup_${component.component.toLowerCase()}.tar.gz`); + const existStatus = await IOUtils.checkFileExists(`${mounts[0].mount}/backup/local/backup_${component.component.toLowerCase()}.tar.gz`); if (existStatus === true) { // eslint-disable-next-line no-await-in-loop await sendChunk(res, `Removing exists backup archive for ${component.component.toLowerCase()}...\n`); // eslint-disable-next-line no-await-in-loop - await IOUtils.removeFile(`${componentPath[0].mount}/backup/local/backup_${component.component.toLowerCase()}.tar.gz`); + await IOUtils.removeFile(`${mounts[0].mount}/backup/local/backup_${component.component.toLowerCase()}.tar.gz`); } // eslint-disable-next-line no-await-in-loop await sendChunk(res, `Creating backup archive for ${component.component.toLowerCase()}...\n`); @@ -2195,27 +2700,37 @@ async function appendBackupTask(req, res) { const tarStatus = await IOUtils.createTarGz(targetPath, tarGzPath); if (tarStatus.status === false) { // eslint-disable-next-line no-await-in-loop - await IOUtils.removeFile(`${componentPath[0].mount}/backup/local/backup_${component.component.toLowerCase()}.tar.gz`); + await IOUtils.removeFile(`${mounts[0].mount}/backup/local/backup_${component.component.toLowerCase()}.tar.gz`); throw new Error(`Error: Failed to create backup archive for ${component.component.toLowerCase()}, ${tarStatus.error}`); } } } await serviceHelper.delay(5 * 1000); + // the archive is written - let the folders sync again before the app is + // brought back, so redundancy is restored at the earliest safe moment + // eslint-disable-next-line no-restricted-syntax + for (const folderId of pausedFolderIds) { + // eslint-disable-next-line no-await-in-loop + await setSyncthingFolderPaused(folderId, false); + } + pausedFolderIds.length = 0; await sendChunk(res, 'Starting application...\n'); if (!syncthing) { await appDockerStart(appname); } else { - const componentsWithoutGSyncthing = appDetails.compose.filter((comp) => !comp.containerData.includes('g:')); + // A g: component's run state belongs to the election, not to this task: + // starting it here would put a second writer on the shared volume. + // Every other component is this task's to bring back. + const componentsToStart = componentsOfApp(appDetails) + .filter((comp) => syncModeOfComponent(comp.containerData) !== 'elected'); // eslint-disable-next-line no-restricted-syntax - for (const component of componentsWithoutGSyncthing) { + for (const component of componentsToStart) { // eslint-disable-next-line no-await-in-loop - await appDockerStart(`${component.name}_${appname}`); + await appDockerStart(component.name === 'null' ? appname : `${component.name}_${appname}`); } } await sendChunk(res, 'Finalizing...\n'); await serviceHelper.delay(5 * 1000); - const indexToRemove = globalState.backupInProgress.indexOf(appname); - globalState.backupInProgress.splice(indexToRemove, 1); res.end(); return true; // eslint-disable-next-line no-else-return @@ -2225,18 +2740,35 @@ async function appendBackupTask(req, res) { } } catch (error) { log.error(error); - const indexToRemove = globalState.backupInProgress.indexOf(appname); - if (indexToRemove >= 0) { - globalState.backupInProgress.splice(indexToRemove, 1); + // eslint-disable-next-line no-restricted-syntax + for (const folderId of pausedFolderIds) { + // eslint-disable-next-line no-await-in-loop + await setSyncthingFolderPaused(folderId, false); } await sendChunk(res, `${error?.message}\n`); res.end(); return false; + } finally { + // The one release, reached by every exit the claim can survive to: success, + // unauthorized, and error alike. The claim is made in the block above this + // try, so a request that never claimed never reaches here. + globalState.finishBackup(appname); } } /** * Append a restore task based on the provided parameters. + * + * Nothing is deleted until a complete, readable replacement is known to exist: + * the archive is fetched and read end to end first, and only then does appdata + * make way for it. The order is the whole point - this ran the other way round, + * and a restore of an archive that turned out to hold one config file was what + * destroyed the app it was meant to protect. + * + * The other instances are never told to redeploy. They hold the only other + * copies, a forced redeploy deletes their volumes, and syncthing already + * carries a restored folder to them. Where their containers are running they + * are restarted, which recreates nothing. * @async * @param {object} req - Request object. * @param {object} res - Response object. @@ -2247,6 +2779,14 @@ async function appendRestoreTask(req, res) { let appname; let restore; let type; + let force; + // folders this task paused, so a failure anywhere below can resume them + const pausedFolderIds = []; + // the component whose appdata is mid-replacement, if any. Set before its data + // makes way and cleared once the archive is fully unpacked, so it names a + // directory that is neither the old copy nor the new one - and nothing else. + // A component that finished is whole, however the components after it fare. + let swapInFlight = null; try { const processedBody = serviceHelper.ensureObject(req.body); log.info(processedBody); @@ -2256,17 +2796,27 @@ async function appendRestoreTask(req, res) { restore = processedBody.restore; // eslint-disable-next-line prefer-destructuring type = processedBody.type; + force = processedBody.force === true || processedBody.force === 'true'; if (!appname || !restore || !type) { throw new Error('appname, restore and type parameters are mandatory'); } - const indexRestore = globalState.restoreInProgress.indexOf(appname); - if (indexRestore !== -1) { - throw new Error(`Restore for app ${appname} is running...`); + if (!Array.isArray(restore)) { + throw new Error('restore must be a list of components'); + } + if (!RESTORE_TYPES.includes(type)) { + throw new Error(`Refused: type must be one of ${RESTORE_TYPES.join(', ')}`); } const hasTrueRestore = restore.some((restoreitem) => restoreitem.restore); if (hasTrueRestore === false) { throw new Error('No restore jobs...'); } + // The claim, before any awaited work: a second request for the same app + // finds it taken here rather than passing an emptied check and racing to + // the clear alongside the first. Last in this synchronous block, so a + // validation throw above never leaves it claimed. + if (!globalState.tryStartRestore(appname)) { + throw new Error(`Restore for app ${appname} is running...`); + } } catch (error) { log.error(error); await sendChunk(res, `${error?.message}\n`); @@ -2274,125 +2824,333 @@ async function appendRestoreTask(req, res) { return false; } try { - const authorized = res ? await verificationHelper.verifyPrivilege('appownerabove', req, appname) : true; - if (authorized === true) { - const componentItem = restore.map((restoreItem) => restoreItem); - globalState.restoreInProgress.push(appname); - // eslint-disable-next-line global-require - const registryManager = require('../appDatabase/registryManager'); - const appDetails = await registryManager.getApplicationGlobalSpecifications(appname); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); + if (authorized !== true) { + const errMessage = messageHelper.errUnauthorizedMessage(); + return res.json(errMessage); + } + // eslint-disable-next-line global-require + const registryManager = require('../appDatabase/registryManager'); + const appDetails = await registryManager.getApplicationGlobalSpecifications(appname); + if (!appDetails) { + throw new Error(`Refused: no specifications found for ${appname}`); + } + + // Only the components the caller asked for. The UI sends every component of + // the app on every request, the unselected ones flagged false and, in remote + // mode, carrying an empty url - reading the list rather than the flags would + // turn every restore into a whole-app restore. + const components = componentsOfApp(appDetails); + // Named once each: a component listed twice would be paused, downloaded and + // unpacked twice over, the second pass clearing what the first had just put + // in place. + const requested = [...new Map( + restore.filter((item) => item.restore).map((item) => [item.component, item]), + ).values()]; + const targets = requested.map((item) => { + const component = components.find((comp) => comp.name === item.component); + if (!component) { + throw new Error(`Refused: ${item.component} is not a component of ${appname}`); + } + return { + name: component.name, + // the folder id IS the docker app identifier, so it addresses the + // syncthing folder, the receiveonly cache and the reconciler alike + folderId: syncthingFolderIdForComponent(appname, component.name), + syncMode: syncModeOfComponent(component.containerData), + url: item.url, + }; + }); + + // A g: component has exactly one writer and it is the instance FDM points + // at. Restoring onto any other copy puts the data where the primary is + // still overwriting it: it reports success and is quietly undone. Only a + // positive answer disqualifies this node - an unreachable FDM must not + // block a restore, the same way it does not block an election. + // + // The answer decides two things: whether to refuse here, and whether this + // task may start the elected components at the end. Silence is not consent: + // "FDM named no primary" and "FDM gave no answer" both arrive as a null ip, and + // starting a writer on the second is what turns "we do not know who the primary + // is" into two of them on one volume. fdmOk is what tells them apart. + let primaryConfirmedLocal = false; + if (!force && targets.some((target) => target.syncMode === 'elected')) { + const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); + const { ip: primaryIp, fdmOk } = await getMasterIpFromFdm(appname, { timeout: 10000 }); + if (fdmOk && primaryIp && !ipsMatch(primaryIp, localSocketAddr)) { + throw new Error(`Refused: restore on ${primaryIp}, it holds the live copy`); + } + primaryConfirmedLocal = Boolean(fdmOk && primaryIp && ipsMatch(primaryIp, localSocketAddr)); + } + + // Hold the data still by pausing the folders being replaced. Deleting them + // instead (as this once did, by an app-level identifier that matched no + // composed app's folder and so did nothing at all) loses the folder config, + // and only the syncthing monitor's per-app pass ever puts it back. + // eslint-disable-next-line no-restricted-syntax + for (const target of targets.filter((item) => item.syncMode !== 'none')) { + // eslint-disable-next-line no-await-in-loop + await sendChunk(res, `Pausing syncthing folder ${target.folderId}\n`); + // eslint-disable-next-line no-await-in-loop + const held = await setSyncthingFolderPaused(target.folderId, true); + if (held === 'held') pausedFolderIds.push(target.folderId); + // Clearing appdata happens INSIDE the replicated folder - the folder path + // is the mount root, not appdata - so an unheld sendreceive folder turns + // the clear into deletions this node broadcasts to every healthy peer. + // Refuse while the data is still there rather than find out afterwards. + if (held === 'failed') { + throw new Error(`Refused: ${target.folderId} could not be held still, so clearing its data would propagate the deletions to the other instances`); + } + } + + await sendChunk(res, 'Stopping application...\n'); + const stopVerdict = await appDockerStop(appname, (line) => sendChunk(res, `${line}\n`)); + // A container still up is still writing to the volume whose appdata is + // about to be emptied - it would write into a half-cleared tree and can + // save its own state back over what the archive puts there. + if (stopVerdict.unavailable) { + throw new Error(`Refused: docker is not answering on this node, so ${appname} cannot be confirmed stopped - try again shortly`); + } + if (!stopVerdict.stopped) { + throw new Error(`Refused: ${stopVerdict.running.join(', ') || appname} could not be stopped, so its data cannot be replaced safely`); + } + await serviceHelper.delay(5 * 1000); + + // eslint-disable-next-line global-require + const IOUtils = require('../IOUtils'); + // eslint-disable-next-line no-restricted-syntax + for (const target of targets) { + // eslint-disable-next-line no-await-in-loop + const { error: mountError, mounts } = await IOUtils.getVolumeInfo(appname, target.name, 'B', 0, 'mount'); + if (mountError) { + throw new Error(`Refused: ${target.name} mount could not be read, so its data cannot be replaced safely`); + } + if (!mounts.length) { + throw new Error(`Refused: ${target.name} volume is not mounted`); + } + target.mount = mounts[0].mount; + target.appDataPath = `${mounts[0].mount}/appdata`; + target.archivePath = `${mounts[0].mount}/backup/${type}/backup_${target.name.toLowerCase()}.tar.gz`; + } + + if (type === 'remote') { // eslint-disable-next-line no-restricted-syntax - const syncthing = appDetails.compose.find((comp) => comp.containerData.includes('g:') || comp.containerData.includes('r:') || comp.containerData.includes('s:')); - if (syncthing) { + for (const target of targets) { + if (!target.url) { + throw new Error(`Refused: no url given for ${target.name}`); + } // eslint-disable-next-line no-await-in-loop - await sendChunk(res, `Stopping syncthing for ${appname}\n`); + await IOUtils.removeDirectory(`${target.mount}/backup/remote`, true); // eslint-disable-next-line no-await-in-loop - await stopSyncthingApp(appname, res); - } - await sendChunk(res, 'Stopping application...\n'); - await appDockerStop(appname); - await serviceHelper.delay(5 * 1000); - // eslint-disable-next-line global-require - const IOUtils = require('../IOUtils'); - // eslint-disable-next-line no-restricted-syntax - for (const component of restore) { - if (component.restore) { - // eslint-disable-next-line no-await-in-loop - const componentVolumeInfo = await IOUtils.getVolumeInfo(appname, component.component, 'B', 0, 'mount'); - const appDataPath = `${componentVolumeInfo[0].mount}/appdata`; - // eslint-disable-next-line no-await-in-loop - await sendChunk(res, `Removing ${component.component} component data...\n`); - // eslint-disable-next-line no-await-in-loop - await serviceHelper.delay(2 * 1000); - // eslint-disable-next-line no-await-in-loop - await IOUtils.removeDirectory(appDataPath, true); + await sendChunk(res, `Downloading ${target.url}...\n`); + // eslint-disable-next-line no-await-in-loop + const downloadStatus = await IOUtils.downloadFileFromUrl(target.url, `${target.mount}/backup/remote`, target.name, true); + if (downloadStatus !== true) { + throw new Error(`Error: Failed to download ${target.url}...`); + } + // This copy is ours, so this task is the one that removes it + target.downloaded = true; + // A connection that dropped, or an error page served as 200, lands here + // as a short file. The archive read below would catch it too, but only + // after inflating what did arrive, and it cannot say what was expected. + // eslint-disable-next-line no-await-in-loop + const expectedBytes = await IOUtils.getRemoteFileSize(target.url, 'B', 0, true); + // eslint-disable-next-line no-await-in-loop + const receivedBytes = await IOUtils.getFileSize(target.archivePath); + if (Number.isFinite(expectedBytes) && expectedBytes > 0 && receivedBytes !== expectedBytes) { + throw new Error(`Error: download incomplete, got ${receivedBytes} of ${expectedBytes} bytes`); } } + } - if (type === 'remote') { - // eslint-disable-next-line no-restricted-syntax - for (const restoreItem of componentItem) { - if (restoreItem?.url !== '') { - // eslint-disable-next-line no-await-in-loop - const componentPath = await IOUtils.getVolumeInfo(appname, restoreItem.component, 'B', 0, 'mount'); - // eslint-disable-next-line no-await-in-loop - await IOUtils.removeDirectory(`${componentPath[0].mount}/backup/remote`, true); - // eslint-disable-next-line no-await-in-loop - await sendChunk(res, `Downloading ${restoreItem.url}...\n`); - // eslint-disable-next-line no-await-in-loop - const downloadStatus = await IOUtils.downloadFileFromUrl(restoreItem.url, `${componentPath[0].mount}/backup/remote`, restoreItem.component, true); - if (downloadStatus !== true) { - throw new Error(`Error: Failed to download ${restoreItem.url}...`); - } - } - } + // Read every archive before any of them is acted on: one decompression + // pass that writes nothing, proving the archive is whole and readable while + // the data it would replace is still there. Its true size is what the space + // check below needs, and the compressed size cannot supply it. + // eslint-disable-next-line no-restricted-syntax + for (const target of targets) { + // eslint-disable-next-line no-await-in-loop + await sendChunk(res, `Checking archive for ${target.name.toLowerCase()}...\n`); + // eslint-disable-next-line no-await-in-loop + const archive = await IOUtils.inspectTarGz(target.archivePath); + if (!archive.status) { + throw new Error(`Error: archive for ${target.name.toLowerCase()} is unreadable, ${archive.error}`); + } + if (archive.entries === 0) { + throw new Error(`Error: archive for ${target.name.toLowerCase()} is empty`); + } + target.archive = archive; + + // Deleting appdata is what frees the room the archive needs, so that is + // what the archive is measured against. A volume that cannot be measured + // is judged on its free space alone - under-stating the room refuses a + // restore that would have fit, which is recoverable; over-stating it runs + // out of space halfway through, which is not. + // Free space is read HERE, not when the mount was resolved: a remote + // restore has just written the archive into this same volume, so a figure + // taken before the download over-states the room by the size of the + // archive itself - and every FluxDrive restore is a remote one. + // eslint-disable-next-line no-await-in-loop + const { error: mountError, mounts } = await IOUtils.getVolumeInfo(appname, target.name, 'B', 0, 'available'); + if (mountError) { + throw new Error(`Refused: ${target.name} mount could not be read, so its data cannot be replaced safely`); + } + if (!mounts.length) { + throw new Error(`Refused: ${target.name} volume is not mounted`); + } + // eslint-disable-next-line no-await-in-loop + const appDataBytes = await IOUtils.getDirectorySizeBytes(target.appDataPath); + const room = mounts[0].available + (appDataBytes ?? 0); + if (archive.bytes > room) { + const needed = IOUtils.convertFileSize(archive.bytes, 'GB', 2); + const have = IOUtils.convertFileSize(room, 'GB', 2); + throw new Error(`Refused: ${target.name} needs ${needed}, has ${have}`); } + } - // eslint-disable-next-line no-restricted-syntax - for (const component of restore) { - if (component.restore) { - // eslint-disable-next-line no-await-in-loop - const componentPath = await IOUtils.getVolumeInfo(appname, component.component, 'B', 0, 'mount'); - const targetPath = `${componentPath[0].mount}/appdata`; - const tarGzPath = `${componentPath[0].mount}/backup/${type}/backup_${component.component.toLowerCase()}.tar.gz`; - // eslint-disable-next-line no-await-in-loop - await sendChunk(res, `Unpacking backup archive for ${component.component.toLowerCase()}...\n`); - // eslint-disable-next-line no-await-in-loop - const tarStatus = await IOUtils.untarFile(targetPath, tarGzPath); - if (tarStatus.status === false) { - throw new Error(`Error: Failed to unpack archive file for ${component.component.toLowerCase()}, ${tarStatus.error}`); - } else { - // eslint-disable-next-line no-await-in-loop - await sendChunk(res, `Removing backup file for ${component.component.toLowerCase()}...\n`); - // eslint-disable-next-line no-await-in-loop - await IOUtils.removeFile(tarGzPath); - } - const syncthingAux = appDetails.compose.find((comp) => comp.name === component.component && (comp.containerData.includes('g:') || comp.containerData.includes('r:'))); - if (syncthingAux) { - // eslint-disable-next-line global-require - const identifier = `${component.component}_${appname}`; - const appId = dockerService.getAppIdentifier(identifier); - // eslint-disable-next-line global-require - const { receiveOnlySyncthingAppsCache } = require('../utils/appCaches'); - const cache = { - restarted: true, - numberOfExecutionsRequired: 4, - numberOfExecutions: 10, - }; - receiveOnlySyncthingAppsCache.set(appId, cache); - } - } + // eslint-disable-next-line no-restricted-syntax + for (const target of targets) { + // eslint-disable-next-line no-await-in-loop + await sendChunk(res, `Restoring ${target.name.toLowerCase()}...\n`); + // from here the directory is neither copy. A clearing that reports + // failure may still have removed most of it, so the mark goes on first. + swapInFlight = target; + // eslint-disable-next-line no-await-in-loop + const cleared = await IOUtils.removeDirectory(target.appDataPath, true); + if (cleared !== true) { + throw new Error(`Error: could not clear ${target.name.toLowerCase()} appdata before unpacking`); } - await serviceHelper.delay(1 * 5 * 1000); - await sendChunk(res, 'Starting application...\n'); - await appDockerStart(appname); - if (syncthing) { - await sendChunk(res, 'Redeploying other instances...\n'); - // eslint-disable-next-line global-require - const appController = require('../appManagement/appController'); - appController.executeAppGlobalCommand(appname, 'redeploy', req.headers.zelidauth, true); - await serviceHelper.delay(1 * 60 * 1000); + // eslint-disable-next-line no-await-in-loop + const tarStatus = await IOUtils.untarFile(target.appDataPath, target.archivePath); + if (tarStatus.status === false) { + throw new Error(`Error: Failed to unpack archive file for ${target.name.toLowerCase()}, ${tarStatus.error}`); } - await sendChunk(res, 'Finalizing...\n'); - await serviceHelper.delay(5 * 1000); - const indexToRemove = globalState.restoreInProgress.indexOf(appname); - globalState.restoreInProgress.splice(indexToRemove, 1); - res.end(); - return true; - // eslint-disable-next-line no-else-return - } else { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res.json(errMessage); + swapInFlight = null; + log.info(`appendRestoreTask - ${appname} ${target.name} restored ${target.archive.entries} entries, ${target.archive.bytes} bytes`); + + // Mark the folder settled so the receiveonly machinery leaves this copy + // alone: it is the one the other instances are meant to take. + if (target.syncMode !== 'none') { + globalState.receiveOnlySyncthingAppsCache.set(target.folderId, { + restarted: true, + numberOfExecutionsRequired: 4, + numberOfExecutions: 10, + }); + } + } + + // The folders carry the restored data out to the other instances the moment + // they resume, so this is the propagation - there is nothing to ask the + // peers to do about their data. + // eslint-disable-next-line no-restricted-syntax + for (const folderId of pausedFolderIds) { + // eslint-disable-next-line no-await-in-loop + await setSyncthingFolderPaused(folderId, false); + } + pausedFolderIds.length = 0; + + await serviceHelper.delay(1 * 5 * 1000); + await sendChunk(res, 'Starting application...\n'); + // A bare app name fans out to every component, and the stop above fans out + // the same way - so this covers a g: component that is no target of this + // restore at all. An elected component is started only where FDM confirmed + // this node holds the primary; unconfirmed - FDM unreachable, or force + // skipping the check - it belongs to the election, which is what starts it + // on every other node anyway. The folders are back in sendreceive by here, + // so a container started now writes into live replicated storage at once. + const componentsToStart = componentsOfApp(appDetails).filter( + (comp) => primaryConfirmedLocal || syncModeOfComponent(comp.containerData) !== 'elected', + ); + // eslint-disable-next-line no-restricted-syntax + for (const component of componentsToStart) { + // eslint-disable-next-line no-await-in-loop + await appDockerStart(component.name === 'null' ? appname : `${component.name}_${appname}`); + } + + // Only the copy this task downloaded is ours to remove. An uploaded or + // local archive is the owner's restore point, and restoring from it must + // not consume it. Held until here so that a failure at any point above can + // be retried from the archive rather than re-fetched. + // eslint-disable-next-line no-restricted-syntax + for (const target of targets.filter((item) => item.downloaded)) { + // eslint-disable-next-line no-await-in-loop + await IOUtils.removeFile(target.archivePath); + } + + // An r:/s: component runs on every instance at once, so the peers' running + // containers are holding the data this restore has just replaced; a restart + // is what makes them read it again, and it recreates no volume. A g: + // component has no peer container to disturb - the other instances are + // stopped and adopt the restored data when the role next moves - and an + // unsynced component's data never left this node. + if (targets.some((target) => target.syncMode === 'shared')) { + await sendChunk(res, 'Restarting other instances...\n'); + // eslint-disable-next-line global-require + const appController = require('../appManagement/appController'); + appController.executeAppGlobalCommand(appname, 'apprestart', authOf(req), undefined, true); // do not wait } + + await sendChunk(res, 'Finalizing...\n'); + await serviceHelper.delay(5 * 1000); + res.end(); + return true; } catch (error) { log.error(error); - const indexToRemove = globalState.restoreInProgress.indexOf(appname); - if (indexToRemove >= 0) { - globalState.restoreInProgress.splice(indexToRemove, 1); + // A component whose appdata was replaced and then failed holds a partial + // directory, and that is not a copy the other instances should be given. + // Demoting the folder stops it being sent and disqualifies this node from + // election until syncthing has healed it from a peer; the cache entry has + // to say NOT settled, or the folder state machine skips the healing path + // and starts the container on the partial data. + let undemotedFolderId = null; + if (swapInFlight) { + // The demotion only means anything where there is a folder: it stops this + // copy being sent, and disqualifies the node from election until syncthing + // has healed it from a peer. The cache entry has to say NOT settled, or + // the folder state machine skips the healing path and starts the container + // on the partial data. + if (swapInFlight.syncMode !== 'none') { + // Patched straight at the folder id, the way the monitor's mount-safety + // block does: a safety action must not be conditioned on a fallible + // read whose failure silently reads as "nothing to protect". A folder + // syncthing does not know answers 404 - nothing is replicating the + // partial data, so there is nothing to demote. + const demote = await syncthingServiceModule.adjustConfigFolders('patch', { type: 'receiveonly' }, swapInFlight.folderId); + if (demote.status !== 'success' && demote.data?.httpStatus !== 404) { + // Still sendreceive over partial data. Paused it transmits nothing; + // resumed it would hand the deletions and the wreckage to every + // healthy peer, so the resume below skips it. The monitor resumes a + // paused folder as drift eventually - this is damage limitation with + // a loud log, not a seal. + undemotedFolderId = swapInFlight.folderId; + log.error(`appendRestoreTask - SAFETY: ${swapInFlight.folderId} holds partial data and could not be demoted to receiveonly (${demote.data?.message || JSON.stringify(demote.data)}); leaving it paused`); + } + globalState.receiveOnlySyncthingAppsCache.set(swapInFlight.folderId, { + restarted: false, + numberOfExecutions: 0, + }); + } + // The hold means something for every component, and used to be applied + // only to the synced ones. A component that syncs has peers to be put + // right by, so holding it costs it minutes; a component that does not has + // no repair path at all - it is the one where running on a half-replaced + // directory is least recoverable, because the app writes fresh state over + // the wreckage and the next restore lands on top of that. + appReconciler.setControllerDesired(swapInFlight.folderId, 'stopped', 'restore did not complete'); + } + // eslint-disable-next-line no-restricted-syntax + for (const folderId of pausedFolderIds.filter((id) => id !== undemotedFolderId)) { + // eslint-disable-next-line no-await-in-loop + await setSyncthingFolderPaused(folderId, false); } await sendChunk(res, `${error?.message}\n`); res.end(); return false; + } finally { + // The one release, reached by every exit the claim can survive to: success, + // unauthorized, and error alike. The claim is made in the block above this + // try, so a request that never claimed never reaches here. + globalState.finishRestore(appname); } } @@ -2444,28 +3202,12 @@ async function testAppMount() { await removeTestAppMount(); const appSize = 1; const overHeadRequired = 2; - const dfAsync = util.promisify(df); const appId = 'flux_fluxTestVol'; log.info('Mount Test: started'); log.info('Mount Test: Searching available space...'); - // we want whole numbers in GB - const options = { - prefixMultiplier: 'GB', - isDisplayPrefixMultiplier: false, - precision: 0, - }; - - const dfres = await dfAsync(options); - const okVolumes = []; - dfres.forEach((volume) => { - if (volume.filesystem.includes('/dev/') && !volume.filesystem.includes('loop') && !volume.mount.includes('boot')) { - okVolumes.push(volume); - } else if (volume.filesystem.includes('loop') && volume.mount === '/') { - okVolumes.push(volume); - } - }); + const okVolumes = await volumeService.capacityVolumesInGib(); // check if space is not sharded in some bad way. Always count the fluxSystemReserve let useThisVolume = null; @@ -2508,7 +3250,7 @@ async function testAppMount() { log.info('Mount Test: Directory made'); log.info('Mount Test: Mounting volume...'); - await execAsRoot('mount', ['-o', 'loop', volumePath, path.join(appsFolder, appId)]); + await execAsRoot('mount', ['-o', APP_VOLUME_MOUNT_OPTIONS, volumePath, path.join(appsFolder, appId)]); log.info('Mount Test: Volume mounted. Test completed.'); dosMountMessage = ''; // run removal @@ -2638,27 +3380,6 @@ function getRemovalInProgress() { return globalState.removalInProgress; } -/** - * Add app to restore progress - * @param {string} appname - App name - */ -function addToRestoreProgress(appname) { - if (!globalState.restoreInProgress.includes(appname)) { - globalState.restoreInProgress.push(appname); - } -} - -/** - * Remove app from restore progress - * @param {string} appname - App name - */ -function removeFromRestoreProgress(appname) { - const index = globalState.restoreInProgress.indexOf(appname); - if (index > -1) { - globalState.restoreInProgress.splice(index, 1); - } -} - /** * Reset removal progress state */ @@ -2801,6 +3522,14 @@ async function updateAppGlobaly(params) { // Validate structural compatibility await validateApplicationUpdateCompatibility(appSpecFormatted, appInfo); + // placement feasibility applies to updates too: a narrowed geolocation, + // raised instance count or grown sizing must not buy a spec the network + // provably cannot satisfy - the redeploy would strip the out-of-geo + // instances and leave the app below its count, or at zero. Placed after the + // previous spec is resolved so an update that changes nothing + // placement-relevant - a renewal, a cancellation - is never refused. + await placementFeasibility.checkPlacementFeasibility(appSpecFormatted, 'updateAppGlobaly', previousAppSpec); + if (isEnterprise) { appSpecFormatted.contacts = []; appSpecFormatted.compose = []; @@ -2854,7 +3583,7 @@ async function updateAppGlobalyApi(req, res) { }); req.on('end', async () => { try { - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -2893,7 +3622,224 @@ async function updateAppGlobalyApi(req, res) { } /** - * To find and remove apps that are spawned more than maximum number of instances allowed locally. + * The app/component identifier a docker container name carries, with the + * runtime prefix removed. Containers are named `/flux`, and + * `/zel` for anything installed before the rename. + * @param {string} containerName Docker's name, leading slash included. + * @returns {string} The identifier the election and the specs both use. + */ +function identifierFromContainerName(containerName) { + return containerName.startsWith('/zel') ? containerName.slice(4) : containerName.slice(5); +} + +/** + * Whether this node is the primary currently elected to run an app's `g:` + * component. + * + * A `g:` component runs on one node at a time, and that node is the one writing + * to the volume. Handing the app back from under it drops whatever it has + * written since the peer last reported the folder complete, so the primary + * stands down and lets masterSlaveApps elect a successor before it may leave. + * + * The election keys one identifier per app - the app name below v4, and + * `_` above it - so both forms are matched. + * + * Three states, because two cannot express what is known here. An empty + * election table means either "no node is primary" or "the election has not + * run", and the caller destroys a volume on the difference. Every other + * unavailable input in canSafelyRemoveApp refuses; so does this one. + * + * Only a fresh verdict that NAMES a primary answers false. "FDM named nobody" + * is null rather than false, because FDM registration lags a node actually + * starting the component by ~110s (see the note at the `all` peer check below), + * and throughout that window it reports no primary while an instance is live - + * so on a node running the component it is the likeliest single reading of "no + * primary" that this node is the one just promoted. + * + * Null is also returned when no verdict has been recorded for the app, or when + * the one on record has gone stale: the election refreshes every + * masterSlaveIntervalMs, so an entry older than PRIMARY_ELECTION_STALE_MS means + * it has stopped running rather than that nothing has changed. + * @param {string} appName Global app name. + * @param {string} localSocketAddr This node's socket address. + * @param {number} [now] Epoch ms, injectable for tests. + * @returns {boolean|null} True if this node is the elected primary, false if + * another node provably is, null if the election cannot say. + */ +function isElectedPrimaryHere(appName, localSocketAddr, now = Date.now()) { + let namesAPrimary = false; + // eslint-disable-next-line no-restricted-syntax + for (const [identifier, checkedAt] of primaryElectionCheckedAt) { + const namesThisApp = identifier === appName || identifier.endsWith(`_${appName}`); + if (!namesThisApp) continue; + if (now - checkedAt > PRIMARY_ELECTION_STALE_MS) continue; + const masterIp = mastersRunningGSyncthingApps.get(identifier); + if (!masterIp) continue; + namesAPrimary = true; + if (ipsMatch(masterIp, localSocketAddr)) return true; + } + return namesAPrimary ? false : null; +} + +/** + * The identifier of an app's `g:` component, or null when it has none. + * + * Derived the same way masterSlaveApps derives it - the app name below v4, and + * `_` above it - because it names the same thing: the one + * component that runs on a single node at a time and writes to the volume. + * @param {object} installedApp Locally installed app record. + * @returns {string|null} + */ +function gComponentIdentifier(installedApp) { + if (installedApp.version <= 3) { + return mountParser.isGComponent(installedApp.containerData) ? installedApp.name : null; + } + const component = (installedApp.compose || []).find((c) => mountParser.isGComponent(c.containerData)); + return component ? `${component.name}_${installedApp.name}` : null; +} + +/** + * Whether this node should give up an app, and why. + * + * Two reasons, one answer. SURPLUS: the app runs on more nodes than it needs and + * this node holds the junior instance. EVACUATION: the node is shedding what it + * holds because it is no longer fit to serve, and this app's turn has come. + * + * Only the reason is decided here. Whether it is SAFE to act on it is + * appEvacuationSafety's question, and both must agree - a count has never been + * able to tell a redundant copy from the last one that holds the data. + * @param {object} installedApp Locally installed app record. + * @param {object[]} runningAppList Instance locations for the app. + * @param {string} localSocketAddr This node's socket address. + * @param {object} [deps] Injected collaborators for the surplus probe. + * @param {Function} [deps.isComponentRunningLocally] Whether a component + * identifier is running on this node right now. + * @param {object} [deps.liveness] Peer folder liveness, for judging a silent peer. + * @returns {Promise<{giveUp: boolean, reason: string, detail: string}>} + */ +async function reasonToGiveUpApp(installedApp, runningAppList, localSocketAddr, deps = {}) { + // lazy load to avoid circular dependency + // eslint-disable-next-line global-require + const residentialNodeDosService = require('../residentialNodeDosService'); + const minInstances = installedApp.instances || config.fluxapps.minimumInstances; + + // A surplus this node declined to act on, carried out of the block so the + // decision can be reported without returning here - a node that is also + // evacuating must still reach the evacuation gate below. + let surplusDeclined = null; + + if (runningAppList.length > minInstances) { + // junior end first: the newest instance stands aside, ties broken + // by the shared ordering so every node names the same surplus + const ordered = [...runningAppList].sort((a, b) => compareInstanceSeniority(b, a)); + const index = ordered.findIndex((x) => socketAddressesMatch(x.ip, localSocketAddr)); + const writer = gComponentIdentifier(installedApp); + const detail = `running on ${runningAppList.length} instances (max: ${minInstances}) and this node is the newest`; + + // "THE NEWEST STANDS ASIDE" IS A STAND-IN FOR "THE LEAST VALUABLE COPY + // STANDS ASIDE", and when the newest copy is the one WRITING the stand-in + // is backwards - that is the most valuable copy on the network, not the + // least. The election is allowed to seat the writer anywhere in the order: + // it skips instances whose data has not finished syncing and starts + // whichever one is ready, and the designated-leader branch leaves the order + // outright. So the two rules can land on the same node. + // + // The node stays, and the next copy trims instead. What it does NOT do is + // stop the writer to make the ordering come true: an app is over-served, + // not down, and interrupting the one node serving it to tidy up the count + // is a worse outcome than the count being wrong for another pass. + if (index === 0) { + // eslint-disable-next-line no-await-in-loop + const runsWriter = Boolean(writer) && Boolean(deps.isComponentRunningLocally) + && await deps.isComponentRunningLocally(writer); + if (!runsWriter) return { giveUp: true, reason: 'SURPLUS', detail }; + // Carried out to the evacuation gate rather than returned past it, for the + // reason the second-newest branch below is: a node that is also draining + // must still be asked whether it should hand this app back. Returned here, + // an evacuating node that is the newest copy AND runs the writer answers + // "staying" forever - when the gate would have stood it down, let a peer + // take the writer, and released it on the next pass. + surplusDeclined = { + code: 'NEWEST_HOLDS_WRITER', + detail: `this node is the newest but holds ${writer}; the next copy trims instead`, + }; + } + + // The next copy, and it steps in ONLY on a positive confirmation that the + // newest is running the writer. Every node ranks the same shared order, but + // "who is writing" is each node's own reading and FDM's registration lags + // it by ~110s - so a second node acting on a guess is how two copies leave + // at once, which is the failure the shared order exists to prevent. + // + // Silence, a timeout, a refusal, "not running": all mean this node does + // nothing, and nothing is exactly today's behaviour. The rule can only ever + // fail towards no trim, never towards two. + if (index === 1 && writer && deps.liveness) { + const appId = dockerService.getAppIdentifier(writer); + // eslint-disable-next-line no-await-in-loop + const newestState = await peerComponentState(ordered[0].ip, { + appId, + identifier: writer, + appName: installedApp.name, + liveness: deps.liveness, + label: 'the newest copy', + logPrefix: 'giveUpApp', + }); + if (newestState === PeerComponent.RUNNING) { + return { + giveUp: true, + reason: 'SURPLUS', + detail: `${detail.replace('this node is the newest', `the newest copy holds ${writer}, so this node trims`)}`, + }; + } + // DECLINING IS A DECISION, and it is reported as one. The newest copy's + // own refusal already reports SURPLUS with giveUp false; this one fell + // through to NONE - which is what the pass reports when the app has no + // surplus at all. So "there is a surplus and I will not act on a guess" + // and "there is nothing here to trim" reached the event stream + // identically, and the single observation that would catch this rule + // failing open was not available to anything watching it. + surplusDeclined = { + code: 'WRITER_UNCONFIRMED', + detail: `${detail} but the newest copy could not be confirmed to hold ${writer} (${newestState}); nothing is trimmed`, + }; + } + } + + if (residentialNodeDosService.isEvacuating()) { + const verdict = residentialNodeDosService.mayEvacuateApp(installedApp.name, runningAppList, localSocketAddr, minInstances); + if (verdict.ok) { + return { + giveUp: true, + reason: 'EVACUATION', + detail: 'node is not fit to serve and is handing its apps back', + }; + } + return { + giveUp: false, reason: 'EVACUATION', code: verdict.code, detail: verdict.reason, + }; + } + + if (surplusDeclined) { + return { + giveUp: false, reason: 'SURPLUS', code: surplusDeclined.code, detail: surplusDeclined.detail, + }; + } + return { giveUp: false, reason: 'NONE', detail: '' }; +} + +/** + * The single pass that decides whether this node should stop holding an app. + * + * At most one app goes per pass, and the PASS is the spacing: this returns as + * soon as one app has gone, and explorerService runs it again every + * removeFluxAppsPeriod * speedMultiplier blocks. config.fluxapps.removal.delay + * is not read here, or anywhere else - it paced a serviceHelper.delay() inside + * a loop that removed several apps in one pass, and that sleep held the whole + * pass open: everything behind it waited, and every decision after it was made + * against an installed-app list read minutes earlier. The pass is fired + * unawaited from the block handler and guards nothing itself, so a pass long + * enough to outlive its own interval could also overlap the next one. * @returns {void} Return statement is only used here to interrupt the function and nothing is returned. */ async function checkAndRemoveApplicationInstance() { @@ -2917,54 +3863,263 @@ async function checkAndRemoveApplicationInstance() { const appUninstaller = require('./appUninstaller'); // eslint-disable-next-line global-require const registryManager = require('../appDatabase/registryManager'); + // eslint-disable-next-line global-require + const evacuationSafety = require('./appEvacuationSafety'); + // eslint-disable-next-line global-require + const residentialNodeDosService = require('../residentialNodeDosService'); + // eslint-disable-next-line global-require + const { findSyncedPeer } = require('../appMonitoring/syncthingFolderStateMachine'); + + const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); + if (!localSocketAddr) { + log.info('Give-up-an-app pass skipped: local socket address unknown'); + return; + } + + // removeAppLocally refuses when another removal or install holds the lock, + // and it refuses by returning - no throw, no status, nothing the caller can + // read. So a pass that ran into one logged "locally removed", called + // noteEvacuated, burned the whole departure interval and discarded the + // app's queue wait, for a removal that never happened. + // + // They do collide: explorerService invokes this pass WITHOUT awaiting it, + // so it outlives the block that started it, and two blocks later the same + // scanner awaits expireGlobalApplications, which holds removalInProgress + // through a real uninstall. + // + // Checked here rather than by making removeAppLocally report back: the file + // it lives in is untouched by this branch, every force=false caller shares + // the same refusal, and a pass that knows it would be refused has no reason + // to start. The same guard reinstallOldApplications and softRemoveAppLocally + // already use. + if (globalState.removalInProgress) { + log.info('Give-up-an-app pass skipped: another removal is in progress'); + return; + } + if (globalState.installationInProgress) { + log.info('Give-up-an-app pass skipped: an installation is in progress'); + return; + } + + // Which components this node is actually running, read once for the pass + // rather than once per app. Only the g: ones matter downstream: a node not + // running the writer component cannot be the writer, which is what lets the + // safety gate answer without FDM on every node that is merely holding a + // synced copy. + // eslint-disable-next-line global-require + const appQueryService = require('../appQuery/appQueryService'); + const runningRes = await appQueryService.listRunningApps(); + const runningIdentifiers = runningRes && runningRes.status === 'success' && Array.isArray(runningRes.data) + ? new Set(runningRes.data.map((app) => identifierFromContainerName(app.Names[0]))) + : null; + if (!runningIdentifiers) { + log.warn('Give-up-an-app pass: running container list unreadable; every g: component is treated as running here'); + } + // Unreadable is not "not running". A container list this node cannot read + // says nothing about what is on its disk, and answering "not running" would + // route every app straight past the primary check. + const isComponentRunningLocally = async (identifier) => ( + runningIdentifiers ? runningIdentifiers.has(identifier) : true + ); + + // One per pass, so a peer asked about twice is asked once. Lazy - it does no + // work at all unless a probe below actually goes silent. + const liveness = createPeerFolderLiveness(); + + // An app can leave by routes this pass never sees - an operator removal, a + // redeploy - and a counter for one this node no longer holds is a leak. + const heldNames = new Set(appsInstalled.map((app) => app.name)); + // eslint-disable-next-line no-restricted-syntax + for (const name of giveUpRefusals.keys()) { + if (!heldNames.has(name)) giveUpRefusals.delete(name); + } + // Stood-down components age on the pass that would have removed them, so the + // cap is counted in the thing that re-asks the question. Two ways out: the + // app left by any route, or this node waited out the cap without being able + // to leave. The second matters more than it looks - a component stopped here + // and running nowhere is a worse state than the one the stand-down exists to + // fix, so the node gives up leaving and stands for election again rather + // than holding a stopped app indefinitely. + // eslint-disable-next-line no-restricted-syntax + for (const [identifier, passes] of standingDown) { + const owner = identifier.includes('_') ? identifier.slice(identifier.indexOf('_') + 1) : identifier; + if (!heldNames.has(owner)) { + standingDown.delete(identifier); + // eslint-disable-next-line no-continue + continue; + } + if (passes >= STAND_DOWN_PASSES_BEFORE_GIVING_UP) { + standingDown.delete(identifier); + log.warn(`${identifier} stood down ${passes} passes without being able to leave; standing for election again`); + // eslint-disable-next-line no-continue + continue; + } + standingDown.set(identifier, passes + 1); + } + // eslint-disable-next-line no-restricted-syntax for (const installedApp of appsInstalled) { // eslint-disable-next-line no-await-in-loop const runningAppList = await registryManager.appLocation(installedApp.name); - const minInstances = installedApp.instances || config.fluxapps.minimumInstances; // introduced in v3 of apps specs - if (runningAppList.length > minInstances) { - // eslint-disable-next-line no-await-in-loop - const appDetails = await registryManager.getApplicationGlobalSpecifications(installedApp.name); - if (appDetails) { - log.info(`Application ${installedApp.name} is already spawned on ${runningAppList.length} instances. Checking if should be unninstalled from the FluxNode..`); - runningAppList.sort((a, b) => { - if (!a.runningSince && b.runningSince) { - return 1; - } - if (a.runningSince && !b.runningSince) { - return -1; - } - if (a.runningSince < b.runningSince) { - return 1; - } - if (a.runningSince > b.runningSince) { - return -1; - } - if (a.ip < b.ip) { - return 1; - } - if (a.ip > b.ip) { - return -1; + // eslint-disable-next-line no-await-in-loop + const decision = await reasonToGiveUpApp(installedApp, runningAppList, localSocketAddr, { + isComponentRunningLocally, + liveness, + }); + // Every pass reports what it decided about every app it holds. Without + // this the pass is invisible: it logs nothing at all when it has nothing + // to give up, so "never ran" and "ran and declined" read identically, and + // a suite can only tell them apart by scraping logs. + fluxEventBus.publish('giveUp:considered', { + appName: installedApp.name, + giveUp: decision.giveUp, + reason: decision.reason, + code: decision.code, + detail: decision.detail, + }); + if (!decision.giveUp) { + if (decision.reason === 'EVACUATION') { + // A node held below the instance count WANTS to leave and cannot, and + // it is counted and escalated exactly as a safety refusal is - because + // until the strength test moved into the pacing gate, that is where + // this was answered and what it did. Left uncounted, a node stuck on + // an app the fleet can never bring back to strength says nothing + // louder than an info line, forever. + // + // Only this code. Waiting a turn, and pausing between departures, are + // this working: counting those would escalate every evacuating node on + // its twelfth pass and teach everyone to ignore the warning. + if (decision.code === 'BELOW_INSTANCE_COUNT') { + const shortRefusals = (giveUpRefusals.get(installedApp.name) ?? 0) + 1; + giveUpRefusals.set(installedApp.name, shortRefusals); + if (shortRefusals % REFUSALS_BEFORE_ESCALATING === 0) { + log.warn(`${installedApp.name} has been refused ${shortRefusals} passes running (EVACUATION, ${decision.code}): ${decision.detail}`); + } else { + log.info(`${installedApp.name} not handed back yet: ${decision.detail}`); } - return 0; - }); - // eslint-disable-next-line no-await-in-loop - const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); - if (localSocketAddr) { - const index = runningAppList.findIndex((x) => socketAddressesMatch(x.ip, localSocketAddr)); - if (index === 0) { - log.info(`Application ${installedApp.name} going to be removed from node as it was the latest one running it to install it..`); - log.warn(`REMOVAL REASON: Too many instances - ${installedApp.name} running on ${runningAppList.length} instances (max: ${minInstances}) - This node is the newest instance`); - log.warn(`Removing application ${installedApp.name} locally`); - // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(installedApp.name, null, false, true, true); - log.warn(`Application ${installedApp.name} locally removed`); - // eslint-disable-next-line no-await-in-loop - await serviceHelper.delay(config.fluxapps.removal.delay * 1000); // wait for 6 mins so we don't have more removals at the same time + } else { + log.info(`${installedApp.name} not handed back yet: ${decision.detail}`); + } + } + // eslint-disable-next-line no-continue + continue; + } + + // eslint-disable-next-line no-await-in-loop + const safety = await evacuationSafety.canSafelyRemoveApp(installedApp.name, { + appLocation: registryManager.appLocation, + getApplicationGlobalSpecifications: registryManager.getApplicationGlobalSpecifications, + findSyncedPeer, + isElectedPrimary: (name) => isElectedPrimaryHere(name, localSocketAddr), + isComponentRunningLocally, + }); + fluxEventBus.publish('giveUp:safety', { + appName: installedApp.name, + reason: decision.reason, + safe: safety.safe, + code: safety.code, + detail: safety.reason, + }); + if (safety.code === 'STAND_DOWN_REQUIRED' && decision.reason === 'EVACUATION') { + // Every other condition has already passed - a connected peer holds each + // synced folder in full, and the app is at strength - so the only thing + // between this node and leaving is that it is the one writing. Stop + // writing. The election hands the role to a peer within a cycle or two, + // and the next pass finds the component not running here, re-proves the + // peer is complete with nothing left to write, and removes. + // + // EVACUATION only. SURPLUS picks the JUNIOR instance and the primary is + // the senior one, so a surplus giver-up is never the primary; wiring + // this there would stop a container for a case that cannot arise. + // eslint-disable-next-line no-restricted-syntax + for (const identifier of safety.standDown) { + try { + // The controller's opinion FIRST, and it is not optional. For a g: + // component appReconciler reads its desired state from + // controllerDesired, so a container stopped while that still says + // 'running' is one the reconciler starts again on its next sweep: + // the stand-down reports success, the component keeps running, the + // election entry goes stale because this node has excluded itself, + // and every later pass refuses with ELECTION_UNKNOWN while the node + // never leaves. This is the same lever masterSlaveApps pulls to put + // a node into standby, which is what standing down makes this one. + appReconciler.setControllerDesired(identifier, 'stopped', 'standing down to hand the app back'); + // eslint-disable-next-line no-await-in-loop + const stop = await appDockerStop(identifier); + // THE VERDICT, not the fact that the call returned. appDockerStop + // REPORTS a refusal rather than throwing one - it catches internally + // and answers { stopped, running, unavailable, errors }, where + // `stopped` is read back from docker rather than taken from the stop + // call. So the catch below can only fire on an unexpected throw, and + // marking here on the strength of having CALLED the stop marked a + // component that may still be up: docker refusing, or never becoming + // able to answer, both come back as stopped:false with no throw. + if (!stop || !stop.stopped) { + // Same reasoning as the catch, for the case that actually happens + // on a node. Unmarked, so the next pass tries again rather than + // this node excluding itself from the election for a component it + // is still running. + log.error(`${installedApp.name}: could not stand down ${identifier}: ` + + `running=[${(stop?.running ?? []).join(', ')}] ` + + `unavailable=${stop?.unavailable ?? 'unknown'} ` + + `errors=[${(stop?.errors ?? []).join('; ')}]`); + } else { + standingDown.set(identifier, 0); + log.warn(`${installedApp.name}: standing down as ${identifier}'s primary so the app can be handed back`); } + } catch (error) { + // Left unmarked deliberately: a component this node failed to stop + // is one it is still writing to, and marking it would make the node + // unelectable for a component it is running. The next pass retries. + log.error(`${installedApp.name}: could not stand down ${identifier}: ${error.message}`); } } + fluxEventBus.publish('giveUp:standDown', { + appName: installedApp.name, + components: safety.standDown, + }); + // One action per pass, exactly as a removal is. + return; + } + if (!safety.safe) { + if (decision.reason === 'EVACUATION') { + // The observation window restarts, so the queue wait is served against + // an uninterrupted period of the app being whole rather than + // accumulated across a gap. Only the evacuation path has a window; + // calling this on a surplus refusal cleared a mark nothing had set. + residentialNodeDosService.forgetAppObservation(installedApp.name); + } + const refusals = (giveUpRefusals.get(installedApp.name) ?? 0) + 1; + giveUpRefusals.set(installedApp.name, refusals); + // No removal follows from this, however long it lasts, and that is + // deliberate. Every reason the gate refuses is a reason removing would + // be wrong: the peers really are incomplete, so this copy is one of the + // few that is not; or this node cannot see them, which is not evidence + // about them. Deleting after a timeout does not fix a wedged folder, it + // just loses the data more slowly. What a node stuck here needs is to + // be VISIBLE - the app is over-served, not down, so nothing about it is + // urgent, and the node being unable to establish anything is the part + // worth acting on. + if (refusals % REFUSALS_BEFORE_ESCALATING !== 0) { + log.info(`${installedApp.name} would be given up (${decision.reason}) but it is not safe: ${safety.reason}`); + } else { + log.warn(`${installedApp.name} has been refused ${refusals} passes running (${decision.reason}, ${safety.code}): ${safety.reason}`); + } + // eslint-disable-next-line no-continue + continue; + } + giveUpRefusals.delete(installedApp.name); + + log.warn(`REMOVAL REASON: ${decision.reason} - ${installedApp.name} ${decision.detail}. Safe: ${safety.reason}`); + // eslint-disable-next-line no-await-in-loop + await appUninstaller.removeAppLocally(installedApp.name, null, false, true, true); + log.warn(`Application ${installedApp.name} locally removed`); + if (decision.reason === 'EVACUATION') { + residentialNodeDosService.noteEvacuated(installedApp.name); } + // One per pass. Removing a second here would take two instances off the + // network before the spawner has replaced either. + return; } } catch (error) { log.error(error); @@ -3082,23 +4237,12 @@ async function reinstallOldApplications() { // eslint-disable-next-line no-await-in-loop const tier = await generalService.nodeTier(); if (appSpecifications.version >= 4 && installedApp.version <= 3) { - if (globalState.removalInProgress) { - log.warn(`Another application is undergoing removal. Skipping ${installedApp.name} for this cycle.`); - // eslint-disable-next-line no-continue - continue; - } - if (globalState.installationInProgress) { - log.warn(`Another application is undergoing installation. Skipping ${installedApp.name} for this cycle.`); - // eslint-disable-next-line no-continue - continue; - } - if (globalState.softRedeployInProgress) { - log.warn(`Another application is undergoing soft redeploy. Skipping ${installedApp.name} for this cycle.`); - // eslint-disable-next-line no-continue - continue; - } - if (globalState.hardRedeployInProgress) { - log.warn(`Another application is undergoing hard redeploy. Skipping ${installedApp.name} for this cycle.`); + // Its own flag excluded and no other: this pass set + // reinstallationOfOldAppsInProgress before the loop, and asking + // without the exclusion would skip every app on its own account. + const heldBy = globalState.operationHolding('reinstallation'); + if (heldBy) { + log.warn(`Another application is undergoing ${heldBy}. Skipping ${installedApp.name} for this cycle.`); // eslint-disable-next-line no-continue continue; } @@ -3149,18 +4293,29 @@ async function reinstallOldApplications() { // Now install components - containers will be created but app is already in DB // eslint-disable-next-line no-restricted-syntax + let allComponentsBack = true; for (const appComponent of appSpecifications.compose) { log.warn(`Continuing Hard Redeployment of component ${appComponent.name}_${appSpecifications.name}...`); // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(config.fluxapps.redeploy.composedDelay * 1000); // install the app // eslint-disable-next-line no-await-in-loop - await appInstaller.registerAppLocally(appSpecifications, appComponent); // component + const outcome = await appInstaller.registerAppLocally(appSpecifications, appComponent); // component + if (outcome !== InstallOutcome.INSTALLED) { + log.error(`Component ${appComponent.name}_${appSpecifications.name} was not reinstalled (${outcome}). ${appSpecifications.name} is part-built; the app row describes every component, so the reconciler recreates what is missing.`); + allComponentsBack = false; + break; + } + } + // Announced and restarted only if it is actually back. Saying so + // regardless is what turned a refused install into a node reporting + // an app it had taken apart. + if (allComponentsBack) { + log.warn(`Composed application ${appSpecifications.name} updated.`); + log.warn(`Restarting application ${appSpecifications.name}`); + // eslint-disable-next-line no-await-in-loop, no-use-before-define + await appDockerRestart(appSpecifications.name); } - log.warn(`Composed application ${appSpecifications.name} updated.`); - log.warn(`Restarting application ${appSpecifications.name}`); - // eslint-disable-next-line no-await-in-loop, no-use-before-define - await appDockerRestart(appSpecifications.name); } else if (appSpecifications.version <= 3) { if (appSpecifications.tiered) { const hddTier = `hdd${tier}`; @@ -3171,23 +4326,12 @@ async function reinstallOldApplications() { appSpecifications.hdd = appSpecifications[hddTier] || appSpecifications.hdd; } - if (globalState.removalInProgress) { - log.warn(`Another application is undergoing removal. Skipping ${installedApp.name} for this cycle.`); - // eslint-disable-next-line no-continue - continue; - } - if (globalState.installationInProgress) { - log.warn(`Another application is undergoing installation. Skipping ${installedApp.name} for this cycle.`); - // eslint-disable-next-line no-continue - continue; - } - if (globalState.softRedeployInProgress) { - log.warn(`Another application is undergoing soft redeploy. Skipping ${installedApp.name} for this cycle.`); - // eslint-disable-next-line no-continue - continue; - } - if (globalState.hardRedeployInProgress) { - log.warn(`Another application is undergoing hard redeploy. Skipping ${installedApp.name} for this cycle.`); + // Its own flag excluded and no other: this pass set + // reinstallationOfOldAppsInProgress before the loop, and asking + // without the exclusion would skip every app on its own account. + const heldBy = globalState.operationHolding('reinstallation'); + if (heldBy) { + log.warn(`Another application is undergoing ${heldBy}. Skipping ${installedApp.name} for this cycle.`); // eslint-disable-next-line no-continue continue; } @@ -3217,23 +4361,12 @@ async function reinstallOldApplications() { } else { // composed application log.warn(`Beginning Redeployment of ${appSpecifications.name}...`); - if (globalState.removalInProgress) { - log.warn(`Another application is undergoing removal. Skipping ${installedApp.name} for this cycle.`); - // eslint-disable-next-line no-continue - continue; - } - if (globalState.installationInProgress) { - log.warn(`Another application is undergoing installation. Skipping ${installedApp.name} for this cycle.`); - // eslint-disable-next-line no-continue - continue; - } - if (globalState.softRedeployInProgress) { - log.warn(`Another application is undergoing soft redeploy. Skipping ${installedApp.name} for this cycle.`); - // eslint-disable-next-line no-continue - continue; - } - if (globalState.hardRedeployInProgress) { - log.warn(`Another application is undergoing hard redeploy. Skipping ${installedApp.name} for this cycle.`); + // Its own flag excluded and no other: this pass set + // reinstallationOfOldAppsInProgress before the loop, and asking + // without the exclusion would skip every app on its own account. + const heldBy = globalState.operationHolding('reinstallation'); + if (heldBy) { + log.warn(`Another application is undergoing ${heldBy}. Skipping ${installedApp.name} for this cycle.`); // eslint-disable-next-line no-continue continue; } @@ -3275,8 +4408,16 @@ async function reinstallOldApplications() { // register // eslint-disable-next-line no-await-in-loop - await appInstaller.registerAppLocally(appSpecifications, undefined, null, false, true); - log.info(`Application ${appSpecifications.name} redeployed with new component structure`); + const outcome = await appInstaller.registerAppLocally(appSpecifications, undefined, null, false, true); + if (outcome === InstallOutcome.INSTALLED) { + log.info(`Application ${appSpecifications.name} redeployed with new component structure`); + } else { + // The removal above deleted the local row, and only a successful + // install writes it back. Nothing reconciles an app with no row, + // so this node has simply lost the instance until it is placed + // here again. + log.error(`Application ${appSpecifications.name} was removed for a component structure change and NOT reinstalled (${outcome}). This node no longer holds it and cannot recover it on its own.`); + } // eslint-disable-next-line no-continue continue; @@ -3303,8 +4444,10 @@ async function reinstallOldApplications() { log.warn(`Beginning Soft Redeployment of component ${appComponent.name}_${appSpecifications.name}...`); // soft redeployment const appId = dockerService.getAppIdentifier(`${appComponent.name}_${appSpecifications.name}`); + // Bare app name: the callee joins it with the component's own + // name for the monitoring key and for cleanupPorts. // eslint-disable-next-line no-await-in-loop - await appUninstaller.softUninstallComponent(`${appComponent.name}_${appSpecifications.name}`, appId, appComponent, null, stopAppMonitoring); + await appUninstaller.softUninstallComponent(appSpecifications.name, appId, appComponent, null, stopAppMonitoring); log.warn(`Application component ${appComponent.name}_${appSpecifications.name} softly removed. Awaiting installation...`); // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(config.fluxapps.redeploy.composedDelay * 1000); @@ -3313,8 +4456,9 @@ async function reinstallOldApplications() { log.warn(`REMOVAL REASON: Hard redeployment (component) - ${appComponent.name}_${appSpecifications.name} HDD changed from ${installedComponent.hdd} to ${appComponent.hdd}`); // hard redeployment const appId = dockerService.getAppIdentifier(`${appComponent.name}_${appSpecifications.name}`); + // same contract as the soft branch above // eslint-disable-next-line no-await-in-loop - await appUninstaller.hardUninstallComponent(`${appComponent.name}_${appSpecifications.name}`, appId, appComponent, null, stopAppMonitoring); + await appUninstaller.hardUninstallComponent(appSpecifications.name, appId, appComponent, null, stopAppMonitoring); log.warn(`Application component ${appComponent.name}_${appSpecifications.name} removed. Awaiting installation...`); // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(config.fluxapps.redeploy.composedDelay * 1000); @@ -3357,6 +4501,7 @@ async function reinstallOldApplications() { } log.info(`Database entry created for ${appSpecifications.name} BEFORE component Docker container creation (composed redeployment path)`); + let allComponentsBack = true; // Now install components - containers will be created but app is already in DB // eslint-disable-next-line no-restricted-syntax for (const appComponent of appSpecifications.compose) { @@ -3379,20 +4524,33 @@ async function reinstallOldApplications() { await serviceHelper.delay(config.fluxapps.redeploy.composedDelay * 1000); // install the app // eslint-disable-next-line no-await-in-loop - await softRegisterAppLocally(appSpecifications, appComponent); // component + const outcome = await softRegisterAppLocally(appSpecifications, appComponent); // component + if (outcome !== InstallOutcome.INSTALLED) { + log.error(`Component ${appComponent.name}_${appSpecifications.name} was not reinstalled (${outcome}). ${appSpecifications.name} is part-built; the app row describes every component, so the reconciler recreates what is missing.`); + allComponentsBack = false; + break; + } } else { log.warn(`Continuing Hard Redeployment of component ${appComponent.name}_${appSpecifications.name}...`); // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(config.fluxapps.redeploy.composedDelay * 1000); // install the app // eslint-disable-next-line no-await-in-loop - await appInstaller.registerAppLocally(appSpecifications, appComponent); // component + const outcome = await appInstaller.registerAppLocally(appSpecifications, appComponent); // component + if (outcome !== InstallOutcome.INSTALLED) { + log.error(`Component ${appComponent.name}_${appSpecifications.name} was not reinstalled (${outcome}). ${appSpecifications.name} is part-built; the app row describes every component, so the reconciler recreates what is missing.`); + allComponentsBack = false; + break; + } } } - log.warn(`Composed application ${appSpecifications.name} updated.`); - log.warn(`Restarting application ${appSpecifications.name}`); - // eslint-disable-next-line no-await-in-loop, no-use-before-define - await appDockerRestart(appSpecifications.name); + // Announced and restarted only if it is actually back. + if (allComponentsBack) { + log.warn(`Composed application ${appSpecifications.name} updated.`); + log.warn(`Restarting application ${appSpecifications.name}`); + // eslint-disable-next-line no-await-in-loop, no-use-before-define + await appDockerRestart(appSpecifications.name); + } } catch (error) { log.error(error); log.warn(`REMOVAL REASON: Redeployment error - ${appSpecifications.name} failed during redeployment: ${error.message}`); @@ -3530,17 +4688,178 @@ async function forceAppRemovals() { } /** - * Manages syncthing master/slave application coordination using FDM services - * @param {object} globalState - Global state object containing masterSlaveAppsRunning, etc. + * What this node can show about a peer's copy of a g: component. UNKNOWN is not + * a soft NOT_RUNNING: only NOT_RUNNING releases the component for a start here, + * because starting is what puts a second writer on a shared volume. + */ +const PeerComponent = Object.freeze({ + RUNNING: 'running', + NOT_RUNNING: 'notRunning', + UNKNOWN: 'unknown', +}); + +// Bounded so a slow peer cannot hold a promotion open, and deliberately not +// shortened: a peer cut short answers UNKNOWN and holds the start, so a tighter +// budget buys nothing and costs availability. +const PEER_PROBE_TIMEOUT_MS = 10 * 1000; + +/** + * How long an instance waits per place in the queue before it may take a primary + * FDM is reporting as empty. + * + * The stagger serialises the candidates so they do not all start against the same + * volume at once, and its length is set by how long FDM takes to register a node + * that HAS started - measured at ~110s in production. A place is worth more than + * that or the wait does not cover what it exists to cover. + * + * Read from config on every call, and per place rather than as a total, because + * the only consumer that overrides it is a test: at three minutes a place, every + * staggered-start path costs minutes of wall clock to reach, which is why none of + * them has rig coverage. A suite exercising one compresses it in its own + * configOverrides - not in the shared harness config, which would re-time every + * existing g: election suite for the benefit of the one that needs it. + * + * @param {number} places how far down the election order, 0 for no wait + * @returns {number} milliseconds + */ +function staggerMs(places) { + return places * (config.fluxapps.masterSlaveStaggerMs ?? 3 * 60 * 1000); +} + +// Why a silent peer was left alone, in the words an operator reading the log +// needs: each one is a different thing to go and look at. +const SILENCE_REASONS = Object.freeze({ + [SilenceVerdict.CONNECTION_ALIVE]: "this node's syncthing still holds a live connection to it", + [SilenceVerdict.NO_EVIDENCE]: 'this node cannot ask its own syncthing about it', + [SilenceVerdict.LOCALLY_ISOLATED]: 'this node cannot see the fleet either', +}); + +/** + * What this node can show one peer to be doing with a component. + * + * Lifted out of masterSlaveApps so the election and the surplus rule ask this + * question through the same code. Two implementations of "is that peer running + * it" drift, and they drift towards whatever answer each caller finds + * convenient - which for one of them is a removal. + * + * `label` names the peer the way its caller knows it, so a log line reads the + * same whether the peer came from the election order, from the remembered + * primary, or from the instance order the surplus rule ranks. + * @param {string} peerSocketAddr The peer's socket address. + * @param {object} ctx Everything the probe needs that is not the peer. + * @param {string} ctx.appId Container name for the component. + * @param {string} ctx.identifier `_` the election keys on. + * @param {string} ctx.appName Global app name, for the log lines. + * @param {object} ctx.liveness Peer folder liveness, for judging silence. + * @param {string} ctx.label How the caller knows this peer. + * @param {string} ctx.logPrefix Which caller is asking. + * @returns {Promise} A PeerComponent state. + */ +async function peerComponentState(peerSocketAddr, { + appId, identifier, appName, liveness, label, logPrefix, +}) { + // Docker reports names with a leading slash, and getAppIdentifier yields + // exactly the container name for this component. Compare whole names: a + // substring test also matches a longer app whose name merely begins the same + // way - myapp against myapp2, or simplexsmp against simplexsmp1 - and a false + // positive here means the component is never started at all. + const peerRunsThisComponent = (appsRunning) => appsRunning.some( + (app) => (app.Names || []).some((name) => name.replace(/^\//, '') === appId), + ); + const ipToCheck = extractIp(peerSocketAddr); + const portToCheck = extractPort(peerSocketAddr); + const { CancelToken } = axios; + const source = CancelToken.source(); + // Cleared once the request settles: every probe otherwise leaves a + // live 10s timer behind, and this runs for each peer on every pass + // until the component is running locally. + const cancelTimer = setTimeout(() => source.cancel('Operation canceled by timeout.'), PEER_PROBE_TIMEOUT_MS); + + try { + // heldcomponents, not listrunningapps: a peer part-way through + // its own pre-start ownership fix has committed but has no + // container, and answering from containers alone reports the + // component free. A peer too old to serve it falls back below. + const heldResponse = await axios.get(`http://${ipToCheck}:${portToCheck}/apps/heldcomponents`, { timeout: PEER_PROBE_TIMEOUT_MS, cancelToken: source.token }) + .catch((error) => { + // A status is an answer: the peer is alive and merely too old + // for this endpoint, so fall through to the container list. No + // reply at all is the case this function exists to judge, and + // it belongs to the handler below. + if (!error.response) throw error; + return null; + }); + const held = heldResponse?.data?.data; + if (Array.isArray(held)) { + if (held.includes(appId)) { + fluxEventBus.count('masterSlave:decision', identifier, 'heldOnPeer'); + log.info(`${logPrefix}: component:${identifier} is held on peer node (${label}) at ${ipToCheck}, will not start`); + return PeerComponent.RUNNING; + } + return PeerComponent.NOT_RUNNING; + } + + // The peer HAS the endpoint and it failed. FluxOS answers + // errors in band, so this arrives as a 200 carrying an error + // object rather than a list - indistinguishable from a peer + // too old for the route by shape alone, which is why it is + // separated here. + // Falling through would answer from the container list a + // question the peer has just said it cannot answer, and that + // list cannot see the durable stop lock at all: a primary its + // owner stopped to work on reads as free, and this node + // elects itself over them. Alive and unreadable is UNKNOWN. + if (heldResponse?.data?.status === 'error') { + log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} could not answer what it holds for app:${appName} - alive, and cannot be ruled out, will not start`); + return PeerComponent.UNKNOWN; + } + + const response = await axios.get(`http://${ipToCheck}:${portToCheck}/apps/listrunningapps`, { timeout: PEER_PROBE_TIMEOUT_MS, cancelToken: source.token }); + const appsRunning = response.data?.data; + // A reply this node cannot read is not a clearance. The peer + // answered, so it is alive; what it is running is simply unknown. + if (!Array.isArray(appsRunning)) { + log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is alive but did not list what it runs for app:${appName}, will not start`); + return PeerComponent.UNKNOWN; + } + // Match on the g: component identifier, not the app name: non-g siblings + // (e.g. a DB cluster component) run on every node and must not be mistaken + // for the master/slave component being active there. + if (peerRunsThisComponent(appsRunning)) { + log.info(`${logPrefix}: component:${identifier} is running on peer node (${label}) at ${ipToCheck}, will not start`); + return PeerComponent.RUNNING; + } + return PeerComponent.NOT_RUNNING; + } catch (error) { + if (error.response) { + log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} answered ${error.response.status} for app:${appName} - alive, and cannot be ruled out, will not start`); + return PeerComponent.UNKNOWN; + } + const verdict = await silenceVerdict(appId, peerSocketAddr, liveness); + if (verdict === SilenceVerdict.GONE) { + log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent and this node's syncthing shows its connection for ${appId} gone - the component is free there`); + return PeerComponent.NOT_RUNNING; + } + log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent for app:${appName} and ${SILENCE_REASONS[verdict]}, will not start`); + return PeerComponent.UNKNOWN; + } finally { + clearTimeout(cancelTimer); + } +} +/** + * Manages syncthing master/slave application coordination using FDM services. + * State (the busy lists, the receive-only cache) is read off globalStateParam at + * the point of each decision, never taken as separate parameters: the getters + * hand out snapshots, so any list captured at call time is a photograph of that + * moment - and this function is invoked once at boot and re-invokes itself for + * the life of the process. + * @param {object} globalStateParam - Global state module (busy lists, receive-only cache, run flags) * @param {Function} installedApps - Function to get installed apps * @param {Function} listRunningApps - Function to get running apps - * @param {Map} receiveOnlySyncthingAppsCache - Cache for receive-only syncthing apps - * @param {Array} backupInProgress - Array of apps with backup in progress - * @param {Array} restoreInProgress - Array of apps with restore in progress * @param {object} https - HTTPS module * @returns {Promise} */ -async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, receiveOnlySyncthingAppsCache, backupInProgressParam, restoreInProgressParam, https) { +async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, https) { try { // eslint-disable-next-line no-param-reassign globalStateParam.masterSlaveAppsRunning = true; @@ -3565,7 +4884,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line global-require const syncthingService = require('../syncthingService'); const syncthingHealth = await syncthingService.getHealth(); - if (syncthingHealth.status !== 'success' || !syncthingHealth.data || syncthingHealth.data.status !== 'OK') { + if (syncthingHealth?.status !== 'OK') { log.warn('masterSlaveApps: Syncthing is not available or not healthy, skipping this cycle'); return; } @@ -3586,13 +4905,8 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, } // Decrypt enterprise apps (version 8 with encrypted content) - appsInstalled.data = await decryptEnterpriseApps(appsInstalled.data); - const runningAppsNames = runningApps.map((app) => { - if (app.Names[0].startsWith('/zel')) { - return app.Names[0].slice(4); - } - return app.Names[0].slice(5); - }); + ({ inPlace: appsInstalled.data } = await decryptEnterpriseApps(appsInstalled.data)); + const runningAppsNames = runningApps.map((app) => identifierFromContainerName(app.Names[0])); const agent = new https.Agent({ rejectUnauthorized: false, }); @@ -3601,6 +4915,13 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, httpsAgent: agent, }; + // This pass's view of the fleet, shared by every g: app it elects. Only its + // localConnectivity() is read here - the peers themselves are probed for what + // they are running, below - and that answer is decided once for the pass: two + // apps must not reach opposite conclusions about whether a silence is a peer's + // or this node's own. + const liveness = createPeerFolderLiveness(); + // Cleanup stale entries from maps to prevent memory leaks const validIdentifiers = new Set(); // eslint-disable-next-line no-restricted-syntax @@ -3628,6 +4949,15 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, } } + // And the verdict record beside it. An identifier the node no longer holds + // must not keep answering for the app it names. + // eslint-disable-next-line no-restricted-syntax + for (const identifier of primaryElectionCheckedAt.keys()) { + if (!validIdentifiers.has(identifier)) { + primaryElectionCheckedAt.delete(identifier); + } + } + // Remove stale entries from timeTostartNewMasterApp // eslint-disable-next-line no-restricted-syntax for (const identifier of timeTostartNewMasterApp.keys()) { @@ -3653,10 +4983,11 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, let identifier; let needsToBeChecked = false; let appId; - const backupSkip = backupInProgressParam.some((backupItem) => installedApp.name === backupItem); - const restoreSkip = restoreInProgressParam.some((backupItem) => installedApp.name === backupItem); + const backupSkip = globalStateParam.backupInProgress.some((backupItem) => installedApp.name === backupItem); + const restoreSkip = globalStateParam.restoreInProgress.some((backupItem) => installedApp.name === backupItem); if (backupSkip || restoreSkip) { log.info(`masterSlaveApps: Backup/Restore is running for ${installedApp.name}, syncthing masterSlave check is disabled for that app`); + fluxEventBus.count('masterSlave:decision', installedApp.name, 'skippedBusy'); // eslint-disable-next-line no-continue continue; } @@ -3676,9 +5007,25 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, } } if (needsToBeChecked) { + // This node stopped the component in order to hand the app back, so it + // is not a candidate. Without this the election restores it within one + // cycle: the component is not running here, this node's own stale + // primary record is cleared below, no peer has picked it up yet, and the + // index-0 branch starts it again - every 30s, forever. Checked in the + // same place and for the same reason as operator-stopped, which is the + // other way a component this node holds is deliberately not running. + if (standingDown.has(identifier)) { + log.info(`masterSlaveApps: ${identifier} is standing down to be handed back - excluded from primary election`); + // eslint-disable-next-line no-continue + continue; + } // operator explicitly stopped this g: component; don't elect or act on it // eslint-disable-next-line no-await-in-loop if (await appsRuntimeState.isOperatorStopped(identifier)) { + // Outside the once-guard below on purpose: the log line reports the + // state change, the counter reports every pass that honoured it, which + // is what a test asserting "the election kept skipping it" needs. + fluxEventBus.count('masterSlave:decision', identifier, 'operatorStopped'); if (!operatorStoppedNoted.has(identifier)) { operatorStoppedNoted.add(identifier); log.info(`masterSlaveApps: ${identifier} is operator-stopped - excluded from primary election until it is started`); @@ -3718,11 +5065,17 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-continue continue; } + // FDM answered and `ip` is either a primary's address or null for + // none. Both are verdicts, and it is the verdict rather than the + // table entry that isElectedPrimaryHere reads - a null ip writes + // nothing to mastersRunningGSyncthingApps, so without this line the + // two ways of having no entry stay indistinguishable. + primaryElectionCheckedAt.set(identifier, Date.now()); if ((!ip)) { log.info(`masterSlaveApps: app:${installedApp.name} has currently no primary set`); if (!runningAppsNames.includes(identifier)) { // Check if app is ready (syncthing data is synced) before allowing it to become primary - let isReady = receiveOnlySyncthingAppsCache.has(appId) && receiveOnlySyncthingAppsCache.get(appId).restarted; + let isReady = globalStateParam.receiveOnlySyncthingAppsCache.has(appId) && globalStateParam.receiveOnlySyncthingAppsCache.get(appId).restarted; // Fallback: If not in cache or not ready, check if syncthing folder is already in sendreceive mode // This handles the case where folder is synced but cache was cleared/lost @@ -3732,11 +5085,11 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, const syncthingService = require('../syncthingService'); // eslint-disable-next-line no-await-in-loop const allSyncthingFolders = await syncthingService.getConfigFolders(); - if (allSyncthingFolders.status === 'success') { + if (Array.isArray(allSyncthingFolders)) { // Syncthing syncs the entire appId folder (includes all subdirectories) const folder = `${appsFolder}${appId}`; // eslint-disable-next-line no-restricted-syntax - for (const syncthingFolder of allSyncthingFolders.data) { + for (const syncthingFolder of allSyncthingFolders) { if (syncthingFolder.path === folder && syncthingFolder.type === 'sendreceive') { log.info(`masterSlaveApps: app:${installedApp.name} folder is already in sendreceive mode, treating as ready`); isReady = true; @@ -3760,27 +5113,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, const registryManager = require('../appDatabase/registryManager'); // eslint-disable-next-line no-await-in-loop const runningAppList = await registryManager.appLocation(installedApp.name); - runningAppList.sort((a, b) => { - if (!a.runningSince && b.runningSince) { - return -1; - } - if (a.runningSince && !b.runningSince) { - return 1; - } - if (a.runningSince < b.runningSince) { - return -1; - } - if (a.runningSince > b.runningSince) { - return 1; - } - if (a.ip < b.ip) { - return -1; - } - if (a.ip > b.ip) { - return 1; - } - return 0; - }); + runningAppList.sort(compareInstanceSeniority); const index = runningAppList.findIndex((x) => ipsMatch(x.ip, localSocketAddr)); // The remembered primary is this node, but the component is not @@ -3811,10 +5144,20 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // FDM reports no primary while an instance is live - so // without this an index-0 node starts a second writer // on a shared volume. - // Best-effort by design, matching the existing probe: an unreachable - // peer is treated as not-running so a network fault cannot strand an - // app forever. It narrows the window rather than closing it; real - // mutual exclusion needs a lease, which is out of scope here. + // A peer that does not answer is UNKNOWN, not free. FluxOS and the + // container fail independently: a node whose API is down for a + // restart still holds the volume and still writes to it, so its + // silence is the strongest reason to suspect it is running the + // component - not a clearance to start beside it. Silence is acted + // on only with evidence: this node's own syncthing showing the + // peer's connection to the folder gone, read on a node that can + // still see the fleet. That is the same proof the election demands + // before it drops a holder, asked here one step later. + // Holding strands nothing indefinitely. A peer that has genuinely + // died loses its sync connection on its own, which releases the + // start; and if it stays dead it stops broadcasting, so it drops + // out of the location list this probes and stops being asked about + // at all. // Probed CONCURRENTLY, not in sequence. Each probe is bounded at // 10s, and an unreachable peer burns the whole budget, so a // sequential walk costs 10s x peers - paid on the promotion path, @@ -3822,64 +5165,50 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // duration of the permissions fix, so every 30s pass re-enters // here), and ahead of every later g: app in the same pass. Running // them together bounds the wait at one timeout regardless of peer - // count. The per-probe timeout is deliberately NOT shortened: this - // check fails open, so a peer that answers slowly must still be - // given its full budget or a live primary reads as absent and we - // start a second writer - the exact outcome the probe exists to - // prevent. - // Docker reports names with a leading slash, and getAppIdentifier - // yields exactly the container name for this component. Compare whole - // names: a substring test also matches a longer app whose name merely - // begins the same way - myapp against myapp2, or simplexsmp against - // simplexsmp1 - and a false positive here means the component is never - // started at all. - const peerRunsThisComponent = (appsRunning) => appsRunning.some( - (app) => (app.Names || []).some((name) => name.replace(/^\//, '') === appId), - ); - const checkPeersRunning = async (scope) => { - const limit = scope === 'all' ? runningAppList.length : index; - if (limit <= 0) return false; // not found, or no lower nodes to check + // count. + const probeCtx = { + appId, identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', + }; - const { CancelToken } = axios; - const timeout = 10 * 1000; + const checkPeersRunning = async (scope) => { + // A lower-only scope with nobody in it is not an answer. At index 0 + // there is no node ahead to ask, and index -1 - this node absent + // from the location list - has none either, so the walk below asks + // NOBODY and the caller reads that as clear. + // + // That is the same blind start the index-0 branch takes scope 'all' + // to avoid, reached from the staggered paths instead. It is not + // unreachable there: the stagger is booked at index >= 2, index is + // re-derived from the location list every pass, and the instances + // ahead can age out of that list before the booked turn arrives - + // leaving a node at index 0 holding a schedule. FDM's registration + // lags a node actually starting (~110s in production), so through + // that whole window it reports no primary while an instance is live, + // and starting on "nobody is ahead of me" puts a second writer on + // the shared volume. + // + // Escalate rather than answer: a start is never issued without some + // peer having been asked. An empty 'all' is a real answer - there is + // genuinely no one to ask - and falls through below. + const effectiveScope = scope === 'lower' && index <= 0 ? 'all' : scope; + const limit = effectiveScope === 'all' ? runningAppList.length : index; + if (limit <= 0) return PeerComponent.NOT_RUNNING; // nobody to ask at all const peers = []; for (let i = 0; i < limit; i += 1) { if (i === index) continue; // never probe ourselves if (runningAppList[i]) peers.push({ i, node: runningAppList[i] }); } - if (!peers.length) return false; - - const probes = peers.map(async ({ i, node }) => { - const ipToCheck = extractIp(node.ip); - const portToCheck = extractPort(node.ip); - const source = CancelToken.source(); - // Cleared once the request settles: every probe otherwise leaves a - // live 10s timer behind, and this runs for each peer on every pass - // until the component is running locally. - const cancelTimer = setTimeout(() => source.cancel('Operation canceled by timeout.'), timeout); - - try { - const response = await axios.get(`http://${ipToCheck}:${portToCheck}/apps/listrunningapps`, { timeout, cancelToken: source.token }); - const appsRunning = response.data.data; - // Match on the g: component identifier, not the app name: non-g siblings - // (e.g. a DB cluster component) run on every node and must not be mistaken - // for the master/slave component being active there. - if (peerRunsThisComponent(appsRunning)) { - log.info(`masterSlaveApps: component:${identifier} is running on peer node (index ${i}) at ${ipToCheck}, will not start`); - return true; - } - } catch (error) { - log.info(`masterSlaveApps: Failed to check peer node ${i} at ${ipToCheck} for app:${installedApp.name}, error: ${error.message}`); - // an unreachable peer is treated as not-running - } finally { - clearTimeout(cancelTimer); - } - return false; - }); - - const found = await Promise.all(probes); - return found.some(Boolean); + if (!peers.length) return PeerComponent.NOT_RUNNING; + + const states = await Promise.all( + peers.map(({ i, node }) => peerComponentState(node.ip, { ...probeCtx, label: `index ${i}` })), + ); + // One peer that cannot be ruled out holds the start on its own: + // every other peer answering "not me" says nothing about that one. + if (states.includes(PeerComponent.RUNNING)) return PeerComponent.RUNNING; + if (states.includes(PeerComponent.UNKNOWN)) return PeerComponent.UNKNOWN; + return PeerComponent.NOT_RUNNING; }; const checkLowerIndexNodesRunning = () => checkPeersRunning('lower'); @@ -3889,44 +5218,28 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // blind, and FDM's registration lag makes "FDM says no primary" // an unreliable proxy for "nobody is running it". // eslint-disable-next-line no-await-in-loop - const peerRunning = await checkPeersRunning('all'); - if (peerRunning) { - log.info(`masterSlaveApps: not starting app:${installedApp.name} index: ${index} - a peer is already running it`); + const peerState = await checkPeersRunning('all'); + if (peerState !== PeerComponent.NOT_RUNNING) { + log.info(`masterSlaveApps: not starting app:${installedApp.name} index: ${index} - a peer ${peerState === PeerComponent.RUNNING ? 'is already running it' : 'could not be ruled out'}`); } else { requestMasterStartWithPermissionsFix(identifier, appId); log.info(`masterSlaveApps: starting docker component:${identifier} index: ${index}`); } } else if (!timeTostartNewMasterApp.has(identifier) && mastersRunningGSyncthingApps.has(identifier) && !ipsMatch(mastersRunningGSyncthingApps.get(identifier), localSocketAddr)) { // There was a previous master (not me), and it's no longer on FDM - const { CancelToken } = axios; - const source = CancelToken.source(); - const timeout = 10 * 1000; // 10 seconds - // Cleared once the request settles, so a completed check leaves no - // live timer behind. - const cancelTimer = setTimeout(() => source.cancel('Operation canceled by the user.'), timeout * 2); const previousMasterIp = mastersRunningGSyncthingApps.get(identifier); // Look up the correct port from runningAppList since FDM API returns IP without port const previousMasterNode = runningAppList.find((x) => ipsMatch(x.ip, previousMasterIp)); - const ipToCheckAppRunning = extractIp(previousMasterIp); - const portToCheckAppRunning = previousMasterNode ? extractPort(previousMasterNode.ip) : DEFAULT_API_PORT; - let previousMasterStillRunning = false; - try { - // eslint-disable-next-line no-await-in-loop - const response = await axios.get(`http://${ipToCheckAppRunning}:${portToCheckAppRunning}/apps/listrunningapps`, { timeout, cancelToken: source.token }); - const appsRunning = response.data.data; - // Match on the g: component identifier, not the app name: non-g siblings - // running on the previous master must not be mistaken for the master/slave - // component still being active there. - if (peerRunsThisComponent(appsRunning)) { - log.info(`masterSlaveApps: component:${identifier} is not on fdm but previous master is running it at: ${ipToCheckAppRunning}:${portToCheckAppRunning}`); - previousMasterStillRunning = true; - } - } catch (error) { - log.info(`masterSlaveApps: Failed to reach previous master at ${ipToCheckAppRunning}:${portToCheckAppRunning} for app:${installedApp.name}, will proceed with primary selection. Error: ${error.message}`); - } finally { - clearTimeout(cancelTimer); - } - if (previousMasterStillRunning) { + const previousMasterAddr = `${extractIp(previousMasterIp)}:${previousMasterNode ? extractPort(previousMasterNode.ip) : DEFAULT_API_PORT}`; + // Asked the same way, and released on the same terms, as any other + // peer. FDM dropping a primary is not evidence that it stopped - + // its registration lags reality in both directions - so a previous + // primary this node cannot read keeps the component. It is the + // instance most likely to still hold the volume, and electing over + // it is exactly the split-brain this branch is reached to avoid. + // eslint-disable-next-line no-await-in-loop + const previousMasterState = await peerComponentState(previousMasterAddr, { ...probeCtx, label: 'previous primary' }); + if (previousMasterState !== PeerComponent.NOT_RUNNING) { // Only THIS app is settled - the previous master still holds it, // so there is nothing to elect. Returning here would abandon the // whole pass and silently skip every remaining g: app, for as @@ -3946,20 +5259,22 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, if (previousMasterIndex >= 0) { log.info(`masterSlaveApps: app:${installedApp.name} had primary running at index: ${previousMasterIndex}`); if (index > previousMasterIndex) { - timetoStartApp += (index - 1) * 3 * 60 * 1000; + timetoStartApp += staggerMs(index - 1); } else { - timetoStartApp += index * 3 * 60 * 1000; + timetoStartApp += staggerMs(index); } } else { - timetoStartApp += index * 3 * 60 * 1000; + timetoStartApp += staggerMs(index); } if (timetoStartApp <= Date.now()) { // Time to start, but check if lower-index nodes are running // eslint-disable-next-line no-await-in-loop - const lowerNodeRunning = await checkLowerIndexNodesRunning(); - if (!lowerNodeRunning) { + const lowerNodeState = await checkLowerIndexNodesRunning(); + if (lowerNodeState === PeerComponent.NOT_RUNNING) { requestMasterStartWithPermissionsFix(identifier, appId); log.info(`masterSlaveApps: starting docker component:${identifier} index: ${index}`); + } else { + log.info(`masterSlaveApps: not starting app:${installedApp.name} index: ${index} - a lower-index node ${lowerNodeState === PeerComponent.RUNNING ? 'is already running it' : 'could not be ruled out'}`); } } else { log.info(`masterSlaveApps: will start docker app:${installedApp.name} at ${timetoStartApp.toString()}`); @@ -3969,18 +5284,68 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, } else if (timeTostartNewMasterApp.has(identifier) && timeTostartNewMasterApp.get(identifier) <= Date.now()) { // Scheduled start time has arrived, check if lower-index nodes are running // eslint-disable-next-line no-await-in-loop - const lowerNodeRunning = await checkLowerIndexNodesRunning(); - if (!lowerNodeRunning) { + const lowerNodeState = await checkLowerIndexNodesRunning(); + if (lowerNodeState === PeerComponent.NOT_RUNNING) { requestMasterStartWithPermissionsFix(identifier, appId); log.info(`masterSlaveApps: starting docker component:${identifier} index: ${index} that was scheduled to start at ${timeTostartNewMasterApp.get(identifier).toString()}`); timeTostartNewMasterApp.delete(identifier); - } else { + } else if (lowerNodeState === PeerComponent.RUNNING) { log.info(`masterSlaveApps: not starting app:${installedApp.name} index: ${index} - lower-index node is already running`); timeTostartNewMasterApp.delete(identifier); + } else { + // The schedule is KEPT. Its due time has passed, so the next pass + // re-probes and starts the moment the peer can be ruled out - + // whereas dropping it sends this node back through a fresh + // index * 3min wait for a peer it may be able to read in seconds. + log.info(`masterSlaveApps: holding the scheduled start of app:${installedApp.name} index: ${index} - a lower-index node could not be ruled out`); + } + } else if (index > 0 && !mastersRunningGSyncthingApps.has(identifier) + && globalStateParam.receiveOnlySyncthingAppsCache.get(appId)?.designatedLeader) { + // The state machine's confirmed designated leader is the only + // instance that can seed a newborn app: at genesis every other + // instance is receiveonly with nothing to sync from, so serving + // the index stagger would wait on nodes that provably cannot + // become ready. + // + // Every peer is probed, not just the lower-index ones. A + // lower-only check belongs to the staggered starts, where index + // order is what serialises the candidates; this branch exists + // precisely to leave that order, so it starts as blind as an + // index-0 start does and needs the same 'all' scope. + // eslint-disable-next-line no-await-in-loop + const peerState = await checkPeersRunning('all'); + if (peerState === PeerComponent.UNKNOWN) { + // The claim is NOT spent. It is what this decision is made from, + // and no decision was reached - a peer this node could not read + // is not a peer that took the seed. Spending it here would drop + // the node to the index stagger on no evidence, which is the + // shape of failure this branch was added to remove. + log.info(`masterSlaveApps: holding the seed of app:${installedApp.name} index: ${index} - a peer could not be ruled out`); + } else { + if (peerState === PeerComponent.RUNNING) { + log.info(`masterSlaveApps: not starting app:${installedApp.name} index: ${index} - a peer is already running it`); + } else { + requestMasterStartWithPermissionsFix(identifier, appId); + log.info(`masterSlaveApps: starting docker component:${identifier} index: ${index} - designated leader seeds without the index stagger`); + } + // Any stagger already scheduled for this node is moot: the seed has + // just been handled here, and leaving the entry lets the scheduled + // branch start it a second time when that time arrives. Whether the + // schedule was set before the election confirmed the leader - which + // is a race this branch has to win, not defer to - or after, the + // answer is the same. + timeTostartNewMasterApp.delete(identifier); + // The claim covers genesis only, and nothing else retracts it: + // once the folder is sendreceive the state machine returns on its + // already-syncing branch and never reaches the election again. + // Spent here, so it cannot take this node out of the stagger on + // later primary losses. + const seedCache = globalStateParam.receiveOnlySyncthingAppsCache.get(appId); + if (seedCache) seedCache.designatedLeader = false; } } else if (index > 0 && !mastersRunningGSyncthingApps.has(identifier) && !timeTostartNewMasterApp.has(identifier)) { // Non-primary node with no history - schedule start based on index - const timetoStartApp = Date.now() + (index * 3 * 60 * 1000); + const timetoStartApp = Date.now() + staggerMs(index); log.info(`masterSlaveApps: scheduling app:${installedApp.name} index: ${index} to start at ${timetoStartApp.toString()}`); timeTostartNewMasterApp.set(identifier, timetoStartApp); } else { @@ -3989,6 +5354,10 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, } } } else { + // This pass read a primary off FDM. Counted rather than published: + // it is the loop's cadence, not an event - see the rule at the top + // of fluxEventBus.js. + fluxEventBus.count('masterSlave:decision', identifier, 'primaryObserved'); mastersRunningGSyncthingApps.set(identifier, ip); if (timeTostartNewMasterApp.has(identifier)) { log.info(`masterSlaveApps: app:${installedApp.name} removed from timeTostartNewMasterApp cache, already started on another standby node`); @@ -4001,7 +5370,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, log.info(`masterSlaveApps: requesting stop of component:${identifier} - primary runs on ip:${ip}, localSocketAddr is: ${localSocketAddr}`); } else if (ipsMatch(localSocketAddr, ip) && !runningAppsNames.includes(identifier)) { // Check if app is ready (syncthing data is synced) before starting - let isReady = receiveOnlySyncthingAppsCache.has(appId) && receiveOnlySyncthingAppsCache.get(appId).restarted; + let isReady = globalStateParam.receiveOnlySyncthingAppsCache.has(appId) && globalStateParam.receiveOnlySyncthingAppsCache.get(appId).restarted; // Fallback: If not in cache or not ready, check if syncthing folder is already in sendreceive mode if (!isReady) { @@ -4010,11 +5379,11 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, const syncthingService = require('../syncthingService'); // eslint-disable-next-line no-await-in-loop const allSyncthingFolders = await syncthingService.getConfigFolders(); - if (allSyncthingFolders.status === 'success') { + if (Array.isArray(allSyncthingFolders)) { // Syncthing syncs the entire appId folder (includes all subdirectories) const folder = `${appsFolder}${appId}`; // eslint-disable-next-line no-restricted-syntax - for (const syncthingFolder of allSyncthingFolders.data) { + for (const syncthingFolder of allSyncthingFolders) { if (syncthingFolder.path === folder && syncthingFolder.type === 'sendreceive') { log.info(`masterSlaveApps: app:${installedApp.name} folder is already in sendreceive mode, treating as ready`); isReady = true; @@ -4044,8 +5413,9 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, } finally { // eslint-disable-next-line no-param-reassign globalStateParam.masterSlaveAppsRunning = false; + fluxEventBus.count('masterSlave:cycles'); await serviceHelper.delay(config.fluxapps.masterSlaveIntervalMs ?? 30 * 1000); - masterSlaveApps(globalStateParam, installedApps, listRunningApps, receiveOnlySyncthingAppsCache, backupInProgressParam, restoreInProgressParam, https); + masterSlaveApps(globalStateParam, installedApps, listRunningApps, https); } } @@ -4071,13 +5441,13 @@ module.exports = { setRemovalInProgress, getInstallationInProgress, getRemovalInProgress, - addToRestoreProgress, - removeFromRestoreProgress, removalInProgressReset, setRemovalInProgressToTrue, installationInProgressReset, setInstallationInProgressTrue, checkAndRemoveApplicationInstance, + reasonToGiveUpApp, + isElectedPrimaryHere, reinstallOldApplications, checkAndRemoveEnterpriseAppsOnNonArcane, forceAppRemovals, diff --git a/ZelBack/src/services/appLifecycle/appEvacuationSafety.js b/ZelBack/src/services/appLifecycle/appEvacuationSafety.js new file mode 100644 index 0000000000..b2d7ada375 --- /dev/null +++ b/ZelBack/src/services/appLifecycle/appEvacuationSafety.js @@ -0,0 +1,256 @@ +// Whether this node may give up an app right now. +// +// "Evacuation" is a node shedding the apps it holds. Deliberately not called +// draining: that word already means a socket or buffer emptying in this codebase, +// and in v9 it means an app shedding traffic. +// +// Every existing removal path decides on an instance count alone, and a count +// cannot tell a redundant copy from the last one that holds the data. Two of the +// paths that do it - surplus removal and the geolocation-change redeploy - have +// already destroyed customer volumes. This is the predicate they should all ask. +// +// The answer is deliberately conservative: anything it cannot establish is a +// refusal, because the cost of waiting is a delay and the cost of being wrong is +// an unrecoverable `rm -rf`. + +const config = require('config'); +const log = require('../../lib/log'); +const dockerService = require('../dockerService'); +const globalState = require('../utils/globalState'); +const mountParser = require('../utils/mountParser'); +const fluxNetworkHelper = require('../fluxNetworkHelper'); +const { socketAddressesMatch, extractIp } = require('../utils/socketAddressUtils'); + +/** + * How many running instances the network insists this app keeps. Mirrors the + * spawner's deficit test (`$lt: [actual, {$ifNull: [instances, 3]}]`) exactly: + * the drain works by creating a deficit the spawner then fills, so a different + * idea of "enough" here would either stall the drain or leave the app short. + * @param {object} spec Global app specification. + * @returns {number} + */ +function requiredInstances(spec) { + return spec.instances ?? config.fluxapps.minimumInstances; +} + +/** + * The components of an app that keep synced state, with the syncthing folder id + * each one owns. Folder ids ARE app identifiers. + * @param {object} spec Global app specification. + * @returns {Array<{name: string, syncMode: string, folderId: string}>} + */ +function syncedComponents(spec) { + const components = spec.version >= 4 && Array.isArray(spec.compose) ? spec.compose : [spec]; + return components.reduce((acc, component) => { + const syncMode = mountParser.getComponentSyncMode(component.containerData || ''); + if (!syncMode) return acc; + const identifier = spec.version >= 4 && Array.isArray(spec.compose) + ? `${component.name}_${spec.name}` + : spec.name; + acc.push({ + name: component.name || spec.name, + syncMode, + // The key masterSlaveApps elects on, and the name the container carries + // once the prefix is stripped - so a caller can ask whether THIS node is + // running the component without reconstructing either. + identifier, + folderId: dockerService.getAppIdentifier(identifier), + }); + return acc; + }, []); +} + +/** + * Instances that are not this node, counted one per physical host. + * + * Several FluxNode registrations routinely share one machine and one connection, + * so two locations at the same address were never two copies: they fail together, + * and on a residential line they are being drained together. Counting them + * separately is how an app with "two instances" loses both. + * @param {Array<{ip: string}>} locations Instance locations for the app. + * @param {string} localSocketAddr This node's socket address. + * @returns {number} Distinct other hosts holding the app. + */ +function otherHostCount(locations, localSocketAddr) { + const localIp = extractIp(localSocketAddr); + const hosts = new Set(); + locations.forEach((location) => { + if (socketAddressesMatch(location.ip, localSocketAddr)) return; + const ip = extractIp(location.ip); + if (ip && ip !== localIp) hosts.add(ip); + }); + return hosts.size; +} + +/** + * May this node remove this app right now? + * + * @param {string} appName Global app name. + * @param {object} deps Injected collaborators, so this is testable without a + * database, a docker daemon or a syncthing process. + * @param {Function} deps.appLocation Instance locations for an app name. + * @param {Function} deps.getApplicationGlobalSpecifications Global spec for an app name. + * @param {Function} deps.findSyncedPeer Connected peer that holds a folder, or null. + * @param {Function} deps.isElectedPrimary Three-state: true if this node is the + * elected `g:` primary, false if another node provably is, null if the + * election cannot say. Required - null and false are not interchangeable here. + * @param {Function} deps.isComponentRunningLocally Whether a component + * identifier is running on this node right now. Required. + * @returns {Promise<{safe: boolean, code: string, reason: string}>} `code` is + * the machine-readable verdict - a refusal because the election cannot answer + * reads identically to an idle pass without it, and a node stuck that way + * should be visible rather than silent. + */ +async function canSafelyRemoveApp(appName, deps) { + const { + appLocation, + getApplicationGlobalSpecifications, + findSyncedPeer, + isElectedPrimary, + isComponentRunningLocally, + } = deps; + + try { + // No default. These used to default to `async () => false`, which answered + // "no, you are not the primary" to a caller that had not asked anyone - the + // one unavailable input in this function that proceeded rather than + // refused. A caller that cannot answer must not be told the removal is + // safe, so the omission throws and the catch below turns it into a refusal + // that says so. + if (typeof isElectedPrimary !== 'function' || typeof isComponentRunningLocally !== 'function') { + throw new Error('isElectedPrimary and isComponentRunningLocally are required'); + } + if (globalState.backupInProgress.includes(appName)) { + return { safe: false, code: 'BACKUP_IN_PROGRESS', reason: 'backup in progress' }; + } + if (globalState.restoreInProgress.includes(appName)) { + return { safe: false, code: 'RESTORE_IN_PROGRESS', reason: 'restore in progress' }; + } + + const spec = await getApplicationGlobalSpecifications(appName); + if (!spec) { + // No spec means we cannot tell how many instances the app needs, nor + // whether it keeps state. Both are required to answer safely. + return { safe: false, code: 'NO_SPEC', reason: 'no global specification available' }; + } + + const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); + if (!localSocketAddr) { + return { safe: false, code: 'NO_LOCAL_ADDRESS', reason: 'local socket address unknown' }; + } + + const locations = await appLocation(appName); + if (!Array.isArray(locations) || !locations.length) { + // An empty list is far more likely to mean the location view has not + // populated than that the app runs nowhere while installed here. + return { safe: false, code: 'NO_LOCATIONS', reason: 'no instance locations known' }; + } + + // The serialisation gate. Removing takes the app to N-1, so every other + // draining holder sees it short and waits; the spawner fills the gap and + // releases the next one. Acting while it is ALREADY short would take a + // second copy off an app that is mid-replacement. + const required = requiredInstances(spec); + if (locations.length < required) { + return { + safe: false, + code: 'BELOW_INSTANCE_COUNT', + reason: `app is below its instance count (${locations.length}/${required}), another move is in flight`, + }; + } + + const synced = syncedComponents(spec); + const otherHosts = otherHostCount(locations, localSocketAddr); + + if (!synced.length) { + // Nothing to lose but the container, which the spawner rebuilds from the + // specification. Removing the only instance is a gap, not a loss. + return { safe: true, code: 'STATELESS', reason: `stateless app, ${otherHosts} other host(s) hold it` }; + } + + // Past here the volume IS the product. + if (otherHosts < 1) { + return { safe: false, code: 'ONLY_HOST', reason: 'stateful app and this is the only host holding it' }; + } + + // ASKED BEFORE THE ELECTION, and that order is the whole of what makes a + // stand-down safe. A node that holds the only good copy refuses here and + // never reaches the question below, so standing down can never be the thing + // that strands an app. + // + // On the pass that removes, this is also the proof that everything this node + // ever wrote has landed elsewhere: by then the component is stopped, so a + // peer at 100% is a peer holding the final state rather than the state as of + // a moment before the next write. + // eslint-disable-next-line no-restricted-syntax + for (const component of synced) { + // eslint-disable-next-line no-await-in-loop + const peer = await findSyncedPeer(component.folderId); + if (!peer) { + return { + safe: false, + code: 'NO_SYNCED_PEER', + reason: `no connected peer holds ${component.folderId} in full`, + }; + } + } + + // A g: component runs on one node at a time and that node is the one + // writing to the volume; the rest hold synced copies with the component + // stopped. So a node not running it cannot be the writer - a local fact, + // needing neither FDM nor the election, and the reason a load-balancer + // outage stalls the trim only on the nodes actually holding a writer + // instead of on all of them. + const gComponents = synced.filter((component) => component.syncMode === 'g'); + const runningHere = await Promise.all( + gComponents.map((component) => isComponentRunningLocally(component.identifier)), + ); + const runningIdentifiers = gComponents + .filter((_component, index) => runningHere[index]) + .map((component) => component.identifier); + if (runningIdentifiers.length) { + const primary = await isElectedPrimary(appName); + if (primary === null) { + // Not "there is no primary" - "nobody can tell me who it is". This node + // is running the writer, so the likeliest reading is that it is the one + // just promoted and FDM has not caught up. Handing the app back from + // under it drops whatever it has written since the peer last reported + // the folder complete. + return { + safe: false, + code: 'ELECTION_UNKNOWN', + reason: 'this node runs the g: component and the election cannot say who is primary', + }; + } + if (primary === true) { + // Not a removal, and not a refusal to leave - an instruction to stop + // writing first. The caller stops these components and asks again next + // pass, by which time the election has given the role to a peer and the + // check above means something stronger. `standDown` names what to stop, + // so the caller does not re-derive it from the spec. + return { + safe: false, + code: 'STAND_DOWN_REQUIRED', + reason: 'this node is the elected primary; stop the component before handing the app back', + standDown: runningIdentifiers, + }; + } + } + + return { + safe: true, + code: 'SYNCED_ELSEWHERE', + reason: `${synced.length} synced component(s) held in full by a connected peer, ${otherHosts} other host(s)`, + }; + } catch (error) { + log.warn(`appEvacuationSafety - ${appName}: ${error.message}`); + return { safe: false, code: 'CHECK_FAILED', reason: `safety check failed: ${error.message}` }; + } +} + +module.exports = { + canSafelyRemoveApp, + requiredInstances, + syncedComponents, + otherHostCount, +}; diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 69d70f0250..0e4e49e331 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -6,6 +6,7 @@ const verificationHelper = require('../verificationHelper'); const dockerService = require('../dockerService'); const dbHelper = require('../dbHelper'); const messageHelper = require('../messageHelper'); +const { InstallOutcome } = require('../utils/installOutcome'); const generalService = require('../generalService'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const fluxNetworkHelper = require('../fluxNetworkHelper'); @@ -23,9 +24,6 @@ const { systemArchitecture } = require('../appSystem/systemIntegration'); const { checkApplicationImagesCompliance, verifyRepository } = require('../appSecurity/imageManager'); const { startAppMonitoring } = require('../appManagement/appInspector'); const imageVerifier = require('../utils/imageVerifier'); -// pgpService is used in commented out code -// eslint-disable-next-line no-unused-vars -const pgpService = require('../pgpService'); const registryCredentialHelper = require('../utils/registryCredentialHelper'); const upnpService = require('../upnpService'); const globalState = require('../utils/globalState'); @@ -40,6 +38,7 @@ const hwRequirements = require('../appRequirements/hwRequirements'); const config = require('config'); const fluxEventBus = require('../utils/fluxEventBus'); const volumeService = require('../utils/volumeService'); +const { Privilege, authOf } = require('../utils/privileges'); // Legacy apps that use old gateway IP assignment method const appsThatMightBeUsingOldGatewayIpAssignment = ['HNSDoH', 'dane', 'fdm', 'Jetpack2', 'fdmdedicated', 'isokosse', 'ChainBraryDApp', 'health', 'ethercalc']; @@ -50,69 +49,26 @@ const appsThatMightBeUsingOldGatewayIpAssignment = ['HNSDoH', 'dane', 'fdm', 'Je const legacyPinnedOctets = appsThatMightBeUsingOldGatewayIpAssignment.map((name) => name.charCodeAt(name.length - 1)); // Helper functions and constants for installApplicationHard -const util = require('util'); -const dockerPullStreamPromise = util.promisify(dockerService.dockerPullStream); const supportedArchitectures = ['amd64', 'arm64']; /** - * Perform Docker cleanup (prune containers, networks, volumes, images) + * Reclaim disk before an install by removing unreferenced docker images. + * + * Only images. Containers, networks and volumes were pruned here too, keyed on + * docker's notion of "unused" - nothing attached right now - which is equally + * true of a healthy app whose container is momentarily down, of a container + * FluxOS is running for its own purposes, and of anything the node operator + * left stopped. The guard in front of this only ever knew about installed app + * components, so those other three were never covered. An image, by contrast, + * is unreferenced or it is not, and re-pulling one is a download rather than a + * loss. + * * @param {object} res - Response object for streaming * @returns {Promise} */ async function performDockerCleanup(res) { - const dockerContainers = { - status: 'Clearing up unused docker containers...', - }; - log.info(dockerContainers); - if (res) { - res.write(serviceHelper.ensureString(dockerContainers)); - if (res.flush) res.flush(); - } - await dockerService.pruneContainers(); - const dockerContainers2 = { - status: 'Docker containers cleaned.', - }; - if (res) { - res.write(serviceHelper.ensureString(dockerContainers2)); - if (res.flush) res.flush(); - } - - const dockerNetworks = { - status: 'Clearing up unused docker networks...', - }; - log.info(dockerNetworks); - if (res) { - res.write(serviceHelper.ensureString(dockerNetworks)); - if (res.flush) res.flush(); - } - await dockerService.pruneNetworks(); - const dockerNetworks2 = { - status: 'Docker networks cleaned.', - }; - if (res) { - res.write(serviceHelper.ensureString(dockerNetworks2)); - if (res.flush) res.flush(); - } - - const dockerVolumes = { - status: 'Clearing up unused docker volumes...', - }; - log.info(dockerVolumes); - if (res) { - res.write(serviceHelper.ensureString(dockerVolumes)); - if (res.flush) res.flush(); - } - await dockerService.pruneVolumes(); - const dockerVolumes2 = { - status: 'Docker volumes cleaned.', - }; - if (res) { - res.write(serviceHelper.ensureString(dockerVolumes2)); - if (res.flush) res.flush(); - } - const dockerImages = { status: 'Clearing up unused docker images...', }; @@ -297,7 +253,7 @@ async function verifyAndPullImage(appSpecifications, appName, isComponent, res, pullConfig.provider = imgVerifier.provider; // eslint-disable-next-line no-unused-vars - await dockerPullStreamPromise(pullConfig, res); + await dockerService.pullImage(pullConfig, res); const pullStatus = { status: isComponent ? `Pulling component ${appSpecifications.name} of Flux App ${appName}` : `Pulling global Flux App ${appName} was successful`, @@ -417,35 +373,40 @@ async function registerAppLocally(appSpecs, componentSpecs, res, test = false, s // get applications specifics from app messages database // check if hash is in blockchain // register and launch according to specifications in message + // Whether THIS call raised the install hold. The guards below refuse because + // someone else is holding the node, and a refusal must not release their hold + // on its way out. + let acquired = false; try { if (globalState.removalInProgress) { const rStatus = messageHelper.createWarningMessage('Another application is undergoing removal. Installation not possible.'); log.error(rStatus); if (res) { res.write(serviceHelper.ensureString(rStatus)); - res.end(); + if (res.flush) res.flush(); } - return false; + return InstallOutcome.REFUSED; } if (globalState.installationInProgress) { const rStatus = messageHelper.createWarningMessage('Another application is undergoing installation. Installation not possible'); log.error(rStatus); if (res) { res.write(serviceHelper.ensureString(rStatus)); - res.end(); + if (res.flush) res.flush(); } - return false; + return InstallOutcome.REFUSED; } globalState.installationInProgress = true; + acquired = true; const tier = await generalService.nodeTier().catch((error) => log.error(error)); if (!tier) { const rStatus = messageHelper.createErrorMessage('Failed to get Node Tier'); log.error(rStatus); if (res) { res.write(serviceHelper.ensureString(rStatus)); - res.end(); + if (res.flush) res.flush(); } - return false; + return InstallOutcome.REFUSED; } const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); @@ -496,14 +457,13 @@ async function registerAppLocally(appSpecs, componentSpecs, res, test = false, s } const appResult = await dbHelper.findOneInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); if (appResult && !isComponent) { - globalState.installationInProgress = false; const rStatus = messageHelper.createErrorMessage(`Flux App ${appName} already installed`); log.error(rStatus); if (res) { res.write(rStatus); - res.end(); + if (res.flush) res.flush(); } - return false; + return InstallOutcome.REFUSED; } // Lazy-load appQueryService to avoid circular dependency issues @@ -518,7 +478,10 @@ async function registerAppLocally(appSpecs, componentSpecs, res, test = false, s throw new Error('Unable to check running Apps'); } const appsInstalled = installedAppsRes.data; - const decryptedAppsInstalled = await appQueryService.decryptEnterpriseApps(appsInstalled, { formatSpecs: false }); + const { readable: decryptedAppsInstalled, unreadable } = await appQueryService.decryptEnterpriseApps(appsInstalled, { formatSpecs: false }); + if (unreadable.length) { + log.warn(`Component names unavailable for undecryptable apps: ${unreadable.map((app) => app.name).join(', ')}`); + } const runningApps = runningAppsRes.data; const installedAppComponentNames = []; decryptedAppsInstalled.forEach((app) => { @@ -678,26 +641,13 @@ async function registerAppLocally(appSpecs, componentSpecs, res, test = false, s const successStatus = messageHelper.createSuccessMessage(`Flux App ${appName} successfully installed and launched`); log.info(successStatus); if (res) { + // Written, not closed. Every caller here is mid-stream on a response its + // own endpoint opened; closing it from in here is what let a failed + // reinstall answer with the teardown's success line. res.write(serviceHelper.ensureString(successStatus)); - res.end(); - } - globalState.installationInProgress = false; - - // Broadcast this node's running apps AFTER releasing the install lock. - // onInstallComplete() -> checkAndNotifyPeersOfRunningApps() relies on - // containerHealthMonitor.monitorAndRecoverApps() to force-include syncthing - // apps whose components are not all simultaneously "running" at this instant - // (e.g. a component mid receive-only resync). That recovery path bails out - // while globalState.isOperationInProgress() is true, so broadcasting before - // installationInProgress is cleared would exclude the just-installed app from - // its own announcement. checkAndNotifyPeersOfRunningApps never throws (it - // catches internally), so running it after res.end() is safe. - if (!test && onInstallComplete) { - await onInstallComplete(); - fluxEventBus.publish('app:installed', { name: appSpecifications.name, hash: appSpecifications.hash }); + if (res.flush) res.flush(); } } catch (error) { - globalState.installationInProgress = false; const errorResponse = messageHelper.createErrorMessage( error.message || error, error.name, @@ -716,12 +666,25 @@ async function registerAppLocally(appSpecs, componentSpecs, res, test = false, s res.write(serviceHelper.ensureString(removeStatus)); if (res.flush) res.flush(); } - await appUninstaller.removeAppLocally(appSpecs.name, res, true, true, sendRemovalMessage); + // endResponse false: the endpoint opened this response and closes it. With + // true, this teardown's own "was successfuly removed" landed as the last + // thing the caller saw, and the reinstall failure that caused it was + // written into a response that had already closed. + await appUninstaller.removeAppLocally(appSpecs.name, res, true, false, sendRemovalMessage); log.info(`Cleanup completed for ${appSpecs.name} after installation failure`); } - return false; + // The app is gone from this node - the teardown above removed it. A caller + // that reads this as "nothing happened" leaves a half-removed app behind; + // one that reads REFUSED as this destroys a running app for a scheduling + // collision. They are not the same answer. + return InstallOutcome.FAILED; } finally { + // The one place the hold is released, so every way out of this function + // releases it. A tier lookup that failed used to return without releasing, + // and the node then refused every install, redeploy, spawn and reinstall + // pass it was offered until FluxOS restarted. + if (acquired) globalState.installationInProgress = false; if (test) { try { await appUninstaller.removeAppLocally(appSpecs.name, null, true, false, false); @@ -731,7 +694,21 @@ async function registerAppLocally(appSpecs, componentSpecs, res, test = false, s } } } - return true; + + // Announced with the node already released, which is what the finally above + // has just done. checkAndNotifyPeersOfRunningApps leans on + // containerHealthMonitor.monitorAndRecoverApps() to force-include syncthing + // apps whose components are not all simultaneously "running" at this instant + // (e.g. a component mid receive-only resync), and that recovery path bails out + // while globalState.isOperationInProgress() is true - so announcing from + // inside the hold left the app just installed out of its own announcement. + // Below the block rather than ordered by hand inside it, so the release + // cannot drift back after it. checkAndNotifyPeersOfRunningApps never throws. + if (!test && onInstallComplete) { + await onInstallComplete(); + fluxEventBus.publish('app:installed', { name: appSpecs.name, hash: appSpecs.hash }); + } + return InstallOutcome.INSTALLED; } /** @@ -999,8 +976,17 @@ async function installAppLocally(req, res) { } let blockAllowance = config.fluxapps.ownerAppAllowance; // needs to be logged in - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (authorized) { + // registerAppLocally refuses a concurrent removal or installation but not a + // redeploy or the periodic reinstall pass, and those hold the node across a + // teardown they intend to rebuild from. Asked here, before anything is + // written, so the refusal is still a status line rather than an envelope in + // a half-streamed body. + const heldBy = globalState.operationHolding(); + if (heldBy) { + throw new Error(`Another application is undergoing ${heldBy}. Installation not possible.`); + } let appSpecifications; // anyone can deploy temporary app // favor temporary to launch test temporary apps @@ -1013,8 +999,19 @@ async function installAppLocally(req, res) { blockAllowance = config.fluxapps.temporaryAppAllowance; } if (!appSpecifications) { - // only owner can deploy permanent message or existing app - const ownerAuthorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + // Placing a registered app on a node is not the node operator's call, for + // the same reason removing one is not: hosting an app is not owning it. + // This branch resolves an app BY NAME from the marketplace, the global + // registry or a permanent message, so an operator reaching it is choosing + // which customer's app runs on hardware they control - and for a g:/r: app + // the new instance syncs that customer's data down to it. The spawner + // decides placement; the owner decides everything else. + // + // The temporary-message branch above is untouched and stays open to any + // logged-in user: that is how an app is tested before it is registered, + // it is addressed by hash rather than by name, and it expires on its own + // (temporaryAppAllowance). + const ownerAuthorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); if (!ownerAuthorized) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -1108,7 +1105,17 @@ async function installAppLocally(req, res) { error.name, error.code, ); - res.json(errorResponse); + if (res.headersSent) { + res.write(serviceHelper.ensureString(errorResponse)); + } else { + res.json(errorResponse); + } + } finally { + // This response has one owner and it is here. registerAppLocally writes its + // progress into it and does not close it, so a failure that arrives after + // the stream has started still reaches the caller instead of landing in a + // response the installer had already ended. + if (!res.writableEnded) res.end(); } } @@ -1161,8 +1168,14 @@ async function testAppInstall(req, res) { let blockAllowance = config.fluxapps.ownerAppAllowance; // needs to be logged in - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (authorized) { + // A test install creates and starts real containers, so it takes the node + // the same way a real one does. + const heldBy = globalState.operationHolding(); + if (heldBy) { + throw new Error(`Another application is undergoing ${heldBy}. Test installation not possible.`); + } let appSpecifications; // anyone can deploy temporary app @@ -1177,8 +1190,19 @@ async function testAppInstall(req, res) { } if (!appSpecifications) { - // only owner can deploy permanent message or existing app - const ownerAuthorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + // Placing a registered app on a node is not the node operator's call, for + // the same reason removing one is not: hosting an app is not owning it. + // This branch resolves an app BY NAME from the marketplace, the global + // registry or a permanent message, so an operator reaching it is choosing + // which customer's app runs on hardware they control - and for a g:/r: app + // the new instance syncs that customer's data down to it. The spawner + // decides placement; the owner decides everything else. + // + // The temporary-message branch above is untouched and stays open to any + // logged-in user: that is how an app is tested before it is registered, + // it is addressed by hash rather than by name, and it expires on its own + // (temporaryAppAllowance). + const ownerAuthorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); if (!ownerAuthorized) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -1270,7 +1294,6 @@ async function testAppInstall(req, res) { status: `Test installation validation passed. Installation skipped due to architecture incompatibility: this node is ${localArch} but app requires [${commonArchitectures.join(', ')}]`, }; res.write(serviceHelper.ensureString(successMessage)); - res.end(); return; } @@ -1287,7 +1310,17 @@ async function testAppInstall(req, res) { error.name, error.code, ); - res.json(errorResponse); + if (res.headersSent) { + res.write(serviceHelper.ensureString(errorResponse)); + } else { + res.json(errorResponse); + } + } finally { + // This response has one owner and it is here. registerAppLocally writes its + // progress into it and does not close it, so a failure that arrives after + // the stream has started still reaches the caller instead of landing in a + // response the installer had already ended. + if (!res.writableEnded) res.end(); } } diff --git a/ZelBack/src/services/appLifecycle/appSpawner.js b/ZelBack/src/services/appLifecycle/appSpawner.js index 6fa795bb93..6d0488b4ad 100644 --- a/ZelBack/src/services/appLifecycle/appSpawner.js +++ b/ZelBack/src/services/appLifecycle/appSpawner.js @@ -9,20 +9,64 @@ const geolocationService = require('../geolocationService'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const log = require('../../lib/log'); const { normalizeSocketAddress, extractIp, extractPort, socketAddressesMatch } = require('../utils/socketAddressUtils'); +const { compareInstallingClaims, compareInstanceSeniority, describeRanking } = require('../utils/instanceOrdering'); // Import modular services const appQueryService = require('../appQuery/appQueryService'); + +// What this node last concluded about each app: the stage that removed it from +// the candidate list, or 'candidate' when it survived to the draw. +// +// Kept so the verdict can be published on CHANGE ONLY. "This pass passed over +// that app" is true every pass for every app the fleet already covers - it is a +// fact about the clock, and fluxEventBus says plainly that a cadence is a +// counter, not an event. What actually happens, rarely, is a verdict FLIPPING: +// an app this node was excluded from becoming one it may take again, which is +// the thing a caller wants to know and the thing no log line makes assertable. +const lastCandidacy = new Map(); + +/** + * Which stage removed each app, from the survivor snapshots taken through the + * filter chain, and publish only the ones whose answer changed since last pass. + * + * @param {Array<[string, Set]>} stages Ordered [stageName, names still in]. + */ +function publishCandidacyChanges(stages) { + if (!stages.length) return; + const [, initial] = stages[0]; + const verdicts = new Map(); + for (const name of initial) { + let stage = 'candidate'; + for (let i = 1; i < stages.length; i += 1) { + if (!stages[i][1].has(name)) { stage = stages[i][0]; break; } + } + verdicts.set(name, stage); + } + for (const [name, stage] of verdicts) { + if (lastCandidacy.get(name) === stage) continue; + lastCandidacy.set(name, stage); + fluxEventBus.publish('spawner:candidacy', { name, stage, candidate: stage === 'candidate' }); + } + for (const name of [...lastCandidacy.keys()]) { + if (!verdicts.has(name)) lastCandidacy.delete(name); + } +} +const resourceQueryService = require('../appQuery/resourceQueryService'); +const messageStore = require('../appMessaging/messageStore'); const registryManager = require('../appDatabase/registryManager'); const imageManager = require('../appSecurity/imageManager'); const hwRequirements = require('../appRequirements/hwRequirements'); const portManager = require('../appNetwork/portManager'); const appUtilities = require('../utils/appUtilities'); const mountParser = require('../utils/mountParser'); +const ipLocationStore = require('../appPlacement/ipLocationStore'); +const placementFeasibility = require('../appPlacement/placementFeasibility'); const systemIntegration = require('../appSystem/systemIntegration'); const globalState = require('../utils/globalState'); const enterpriseNetwork = require('../utils/enterpriseNetwork'); const { FluxCacheManager } = require('../utils/cacheManager'); const appInstaller = require('./appInstaller'); +const { InstallOutcome } = require('../utils/installOutcome'); const appUninstaller = require('./appUninstaller'); const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../utils/appSyncEvents'); const fluxEventBus = require('../utils/fluxEventBus'); @@ -30,7 +74,7 @@ const fluxEventBus = require('../utils/fluxEventBus'); let appsCountAvailableToInstallOnMyNode = 0; const collisionWaitMs = config.fluxapps.installCollisionWaitMs; -const spawnReconfirmDelayMs = config.fluxapps.spawnReconfirmDelayMs; +const { spawnReconfirmDelayMs } = config.fluxapps; const nonEnterpriseSpawnDelayMs = config.fluxapps.nonEnterpriseSpawnDelayMs ?? 2 * 60 * 1000; let spawnLoopRunning = false; @@ -104,6 +148,12 @@ async function trySpawningGlobalApplication() { return installDelay; } + if (fluxNetworkHelper.isPlacementHeld()) { + log.info(`Node held back from new placements (${fluxNetworkHelper.getPlacementHold()}). Global applications will not be installed`); + fluxEventBus.publish('spawner:blocked', { reason: 'placement_hold' }); + return installDelay; + } + let isNodeConfirmed = false; isNodeConfirmed = await generalService.isNodeStatusConfirmed().catch(() => null); if (!isNodeConfirmed) { @@ -140,6 +190,11 @@ async function trySpawningGlobalApplication() { throw new Error('Unable to detect Flux IP address'); } + // Our address without the port, derived once. It was being recomputed in + // four places under three different names, so nothing told a reader they + // were the same value. + const localIp = extractIp(localSocketAddr); + const runningApps = await appQueryService.listRunningApps(); if (runningApps.status !== 'success') { throw new Error('trySpawningGlobalApplication - Unable to check running apps on this Flux'); @@ -155,7 +210,7 @@ async function trySpawningGlobalApplication() { const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); const currentHeight = syncStatus.data.height; const ponFork = config.fluxapps.daemonPONFork; - const blocksLasting = config.fluxapps.blocksLasting; + const { blocksLasting } = config.fluxapps; const minBlocksAllowance = config.fluxapps.newMinBlocksAllowance; const pipeline = [ // Filter out apps that are expired or expiring within minBlocksAllowance (100) blocks @@ -279,11 +334,24 @@ async function trySpawningGlobalApplication() { } else { const myNodeLocation = await systemIntegration.nodeFullGeolocation(); + // Where the candidates went. Every filter below removes apps for a + // different and entirely reasonable reason, and none of them says so - the + // pass ends with "No app currently to be processed" whether one filter + // dropped everything or five each took a share. From outside the process + // that is indistinguishable from an app nobody wanted, which is how a port + // collision looked like a placement failure for a whole day. + const survivors = { found: globalAppNamesLocation.length }; + const nameSet = () => new Set(globalAppNamesLocation.map((app) => app.name)); + const stages = [['found', nameSet()]]; + // filter apps that failed to install before globalAppNamesLocation = globalAppNamesLocation.filter((app) => !runningApps.data.find((appsRunning) => appsRunning.Names[0].slice(5) === app.name) && !globalState.spawnErrorsLongerAppCache.has(app.hash) && !globalState.trySpawningGlobalAppCache.has(app.hash) && !appsToBeCheckedLater.some((appAux) => appAux.appName === app.name)); + survivors.afterAlreadyHeldOrTried = globalAppNamesLocation.length; + stages.push(['afterAlreadyHeldOrTried', nameSet()]); + // filter apps that are non enterprise or are marked to install on my node. // Enterprise-owned apps that target specific node IPs are strict: only a node // whose IP is listed may install them, regardless of version (the version>=8 @@ -294,26 +362,93 @@ async function trySpawningGlobalApplication() { } return app.nodes.length === 0 || app.nodes.find((ip) => socketAddressesMatch(ip, localSocketAddr)) || app.version >= 8; }); - // filter apps that dont have geolocation or that are forbidden to spawn on my node geolocation - globalAppNamesLocation = globalAppNamesLocation.filter((app) => (app.geolocation.length === 0 || app.geolocation.filter((loc) => loc.startsWith('a!c')).length === 0 || !app.geolocation.find((loc) => loc.startsWith('a!c') && `a!c${myNodeLocation}`.startsWith(loc.replace('_NONE', ''))))); - // filter apps that dont have geolocation or have and match my node geolocation - globalAppNamesLocation = globalAppNamesLocation.filter((app) => (app.geolocation.length === 0 || app.geolocation.filter((loc) => loc.startsWith('ac')).length === 0 || app.geolocation.find((loc) => loc.startsWith('ac') && `ac${myNodeLocation}`.startsWith(loc)))); + // Selection uses the SAME eligibility implementation as candidate counting + // and the install gate, over the SAME source for where this node is - the + // published table, which is the only thing the count can read for the + // thousands of nodes it cannot ask. Taking continent and country from the + // node's ip-api self-report instead put this one reader on a different + // source from the other two: measured across the fleet, the two disagree on + // country for about one node in thirteen, and where they disagree the table + // is right roughly eighteen times out of nineteen. A node the count credits + // to the table's country would then never volunteer for an app pinned there, + // and the app sits below its instance count with candidates that look + // available. + // + // The self-report is the fallback, for a node the table cannot place at all + // - the same fallback, in the same direction, as the install gate. The old + // string-prefix filters here hid every table-vocabulary region pin from + // spawning and stripped _NONE, which turned a no-op deny into a whole-country + // selection ban. + const [selfContinentCode, selfCountryCode] = (myNodeLocation ?? '').split('_'); + let myContinentCode = selfContinentCode ?? null; + let myCountryCode = selfCountryCode ?? null; + let myTableRegion = null; + try { + const localHit = await ipLocationStore.lookup(localIp); + // Both or neither: a hit carrying one without the other cannot place the + // node any better than its own report can. + if (localHit?.continentCode && localHit?.countryCode) { + myContinentCode = localHit.continentCode; + myCountryCode = localHit.countryCode; + } + myTableRegion = localHit?.region ?? null; + } catch (error) { + // store unreadable = the table cannot place this node, so its self-report + // stands and the region is unknown; selection over-includes and the + // installer arbitrates + } + const myLocation = { continentCode: myContinentCode, countryCode: myCountryCode, region: myTableRegion }; + survivors.afterNodePin = globalAppNamesLocation.length; + stages.push(['afterNodePin', nameSet()]); + globalAppNamesLocation = globalAppNamesLocation.filter( + (app) => placementFeasibility.nodeLocationMatchesGeolocation(myLocation, app.geolocation), + ); + survivors.afterGeolocation = globalAppNamesLocation.length; + stages.push(['afterGeolocation', nameSet()]); globalAppNamesLocation = enterpriseNetwork.filterAppsByOwnership(globalAppNamesLocation, isEnterprise); + survivors.afterOwnership = globalAppNamesLocation.length; + stages.push(['afterOwnership', nameSet()]); + + // Drop candidates whose remaining slots are already claimed, before one is + // picked at random. The pool counts running instances only, so an app that + // other nodes are already installing still reads as short - and selection + // is a lottery, so such a candidate does not merely waste its own cycle: + // it can win the draw ahead of one this node could have installed, and the + // node then spawns nothing for a whole pass. Counting every candidate's + // claims costs one grouped read of a collection that holds only live + // claims. The re-read before claiming still runs and is the authority; + // this only spares the draw candidates it would have turned away. + const claimsByApp = await registryManager.installingCountsByApp(); + globalAppNamesLocation = globalAppNamesLocation.filter( + (app) => app.actual + (claimsByApp.get(app.name.toLowerCase()) ?? 0) < app.required, + ); appsCountAvailableToInstallOnMyNode = globalAppNamesLocation.length + appsSyncthingToBeCheckedLater.length + appsToBeCheckedLater.length; ({ shortDelayTime, delayTime } = enterpriseNetwork.getSpawnDelays(isEnterprise, appsCountAvailableToInstallOnMyNode)); + survivors.afterClaims = globalAppNamesLocation.length; + stages.push(['afterClaims', nameSet()]); + + publishCandidacyChanges(stages); + if (globalAppNamesLocation.length === 0) { - log.info('trySpawningGlobalApplication - No app currently to be processed'); + log.info(`trySpawningGlobalApplication - No app currently to be processed (${JSON.stringify(survivors)})`); + // A TALLY, not a stream. This is true on every pass of a fleet whose apps + // are all at their instance count - roughly every 240ms per node under + // the harness multiplier - and as an event it spent a ring every other + // consumer shares, which is why nothing could afford to subscribe to it. + // The breakdown stays in the log line above, and what CHANGED went out as + // spawner:candidacy. + fluxEventBus.count('spawner:noCandidates'); return delayTime; } log.info(`trySpawningGlobalApplication - Found ${globalAppNamesLocation.length} apps that are missing instances on the network and can be selected to try to spawn on my node.`); let random = Math.floor(Math.random() * globalAppNamesLocation.length); appToRunAux = globalAppNamesLocation[random]; - const filterAppsWithNyNodeIP = globalAppNamesLocation.filter((app) => app.nodes.find((ip) => socketAddressesMatch(ip, localSocketAddr))); - if (filterAppsWithNyNodeIP.length > 0) { - random = Math.floor(Math.random() * filterAppsWithNyNodeIP.length); - appToRunAux = filterAppsWithNyNodeIP[random]; + const appsNamingThisNode = globalAppNamesLocation.filter((app) => app.nodes.find((ip) => socketAddressesMatch(ip, localSocketAddr))); + if (appsNamingThisNode.length > 0) { + random = Math.floor(Math.random() * appsNamingThisNode.length); + appToRunAux = appsNamingThisNode[random]; } appToRun = appToRunAux.name; @@ -323,7 +458,7 @@ async function trySpawningGlobalApplication() { log.info(`trySpawningGlobalApplication - Application ${appToRun} selected to try to spawn. Reported as been running in ${appToRunAux.actual} instances and ${appToRunAux.required} are required.`); runningAppList = await registryManager.appLocation(appToRun); installingAppList = await registryManager.appInstallingLocation(appToRun); - if (runningAppList.length + installingAppList.length > minInstances) { + if (runningAppList.length + installingAppList.length >= minInstances) { log.info(`trySpawningGlobalApplication - Application ${appToRun} is already spawned or being installed on ${runningAppList.length + installingAppList.length} instances.`); return shortDelayTime; } @@ -348,13 +483,12 @@ async function trySpawningGlobalApplication() { runningAppList = await registryManager.appLocation(appToRun); - const adjustedIP = extractIp(localSocketAddr); // just IP address // check if app not running on this device - if (runningAppList.find((document) => document.ip.includes(adjustedIP))) { + if (runningAppList.find((document) => document.ip.includes(localIp))) { log.info(`trySpawningGlobalApplication - Application ${appToRun} is reported as already running on this Flux IP`); return delayTime; } - if (installingAppList.find((document) => document.ip.includes(adjustedIP))) { + if (installingAppList.find((document) => document.ip.includes(localIp))) { log.info(`trySpawningGlobalApplication - Application ${appToRun} is reported as already being installed on this Flux IP`); return delayTime; } @@ -417,6 +551,20 @@ async function trySpawningGlobalApplication() { throw error; }); + // Refused before taking on new work, and only here. An application this node + // cannot read contributes nothing to the totals the check below subtracts + // from its capacity, so the space it believes is free includes space already + // spoken for and this node would over-commit. The same refusal on the + // maintenance paths would be wrong: a redeploy of an app already counted adds + // nothing, and one unreadable application would freeze every other one on the + // node. Named, because a node that quietly stops accepting work is a long + // afternoon for whoever has to find out why. + const unaccounted = resourceQueryService.unaccountedApps(await resourceQueryService.appsResources()); + if (unaccounted.length) { + log.error(`trySpawningGlobalApplication - cannot account for what this node has committed: ${unaccounted.join(', ')} could not be read. Not taking on more.`); + return shortDelayTime; + } + // verify requirements await hwRequirements.checkAppRequirements(appSpecifications); // enterprise network nodes: reserve >4 vCores of burst headroom (automatic CPU burst) @@ -426,24 +574,88 @@ async function trySpawningGlobalApplication() { // ensure ports unused // Get apps running specifically on this IP - const localSocketAddrAddress = extractIp(localSocketAddr); // just IP address without port - const runningAppsOnThisIP = await registryManager.getRunningAppIpList(localSocketAddrAddress); - const runningAppsNames = runningAppsOnThisIP.map((app) => app.name); + const appsRunningAtOurIp = await registryManager.getRunningAppIpList(localIp); + const runningAppsNames = appsRunningAtOurIp.map((app) => app.name); await portManager.ensureApplicationPortsNotUsed(appSpecifications, runningAppsNames); + // The check above reads a sibling's ports from the specifications the + // network broadcasts, so it sees only what has been reported as RUNNING. A + // sibling that has installed an application and not started it, or is still + // installing it, appears nowhere in that list and holds the router's forward + // regardless. Ask the other Flux nodes at this address directly, here rather + // than during the port test, so a refusal costs no firewall rule and no port + // mapping to unwind. + // + // Not because an enterprise application hides its ports. It seals them, and + // a node running ArcaneOS opens them - assignedPortsGlobalApps decrypts. On + // a node that is not running ArcaneOS it cannot, and there this ask is the + // only thing that sees a sealed neighbour's ports at all. + // + // Answered rather than raised, and handled exactly as an unreachable port is + // below: this node cannot host this app, which is an ordinary answer and not + // a fault. What separates the two is how it is told, not what it costs the + // app: raised, this is an error with a stack trace and no event; answered, + // it is one line and a deferral the fleet can observe. The app holds the + // entry every selection takes in the spawn cache either way - it was filed + // at selection, and the catch below adds nothing for a hash already there - + // so this node stops considering it until that expires, which is right, + // because nothing changes here until the sibling gives the port up. + const sibling = await portManager.siblingHoldingPort(appPorts, localSocketAddr); + if (sibling) { + log.error(`trySpawningGlobalApplication - ${appSpecifications.name} port ${sibling.port} is held by the Flux node at ${sibling.address}, which shares this public address. Installation aborted.`); + // A deferral, published as one: this stands the node down and returns + // shortDelayTime exactly as the seven reasons below it do, so it belongs + // in that vocabulary rather than in an event of its own. + fluxEventBus.publish('spawner:deferred', { + appName: appSpecifications.name, + reason: 'sibling_holds_port', + delayMs: shortDelayTime, + port: sibling.port, + address: sibling.address, + }); + return shortDelayTime; + } + // Note: User-blocked port check happens earlier (line ~353) before Docker Hub calls // Check if ports are publicly available - critical for proper Flux network operation - const portsPubliclyAvailable = await portManager.checkInstallingAppPortAvailable(appPorts); - if (portsPubliclyAvailable === false) { + const portVerdict = await portManager.checkInstallingAppPortAvailable(appPorts); + if (portVerdict.ok === false) { log.error(`trySpawningGlobalApplication - Some of application ports of ${appSpecifications.name} are not available publicly. Installation aborted.`); + // The cause lives in portManager, which says which port and which peers; + // this says the spawner deferred, and on which of its verdicts. + fluxEventBus.publish('spawner:deferred', { + appName: appSpecifications.name, + reason: 'ports_not_available', + portVerdict: portVerdict.reason, + delayMs: shortDelayTime, + }); return shortDelayTime; } // double check if app is installed on the number of instances requested runningAppList = await registryManager.appLocation(appToRun); installingAppList = await registryManager.appInstallingLocation(appToRun); - if (runningAppList.length + installingAppList.length > minInstances) { + if (runningAppList.length + installingAppList.length >= minInstances) { + // KEPT when the running copies alone meet the count, CLEARED when the + // claims were needed to reach it. + // + // A running copy is a durable fact and caching it is the point of the + // cache - the app is covered, and re-deciding that every pass is waste. + // A claim is not: it is withdrawn as soon as its node finds the share + // already filled, seconds later and by design, because the share is + // checked after the claim goes out. Cached on a count that needed those + // claims, this node remembers "covered" for the cache's twelve hours and + // never reconsiders, so an app that falls back below its instance count + // waits out the day on every node that glanced inside that window. + // + // Clearing unconditionally is the other way to be wrong: an app whose + // count is genuinely met would re-enter the candidate pool on every pass + // and be declined again forever, never cached because it was never + // installed. + if (runningAppList.length < minInstances) { + globalState.trySpawningGlobalAppCache.delete(appHash); + } log.info(`trySpawningGlobalApplication - Application ${appToRun} is already spawned or being installed on ${runningAppList.length + installingAppList.length} instances.`); return shortDelayTime; } @@ -458,22 +670,58 @@ async function trySpawningGlobalApplication() { syncthingApp = appSpecifications.compose.some((comp) => mountParser.isSyncedComponent(comp.containerData)); } - const localIp = extractIp(localSocketAddr); - const lastIndex = localIp.lastIndexOf('.'); - const secondLastIndex = localIp.substring(0, lastIndex).lastIndexOf('.'); - const ipPrefix = localIp.substring(0, secondLastIndex + 1); // includes the '.' e.g. "192.168." - + // An owner who names exactly as many nodes as instances has assigned the + // placement, and the diversity share does not second-guess it. A longer + // list is a candidate pool - `nodes` may carry up to 120 entries against + // an instance count as low as one - so the share still governs, computed + // over that pool (placementFeasibility restricts its candidate set to it). + // The bypass applies only when THIS node is named: a v8+ app spawning on + // an off-list node is subject to the share either way. + let ownerNamedThisNode = false; if (syncthingApp) { - let sameIpRangeNode = runningAppList.find((location) => location.ip.startsWith(ipPrefix)); - if (sameIpRangeNode) { - log.info(`trySpawningGlobalApplication - Application ${appToRun} uses syncthing and it is already spawned on Fluxnode with same ip range`); + const pinList = appSpecifications.nodes ?? []; + ownerNamedThisNode = pinList.length > 0 && pinList.length <= minInstances + && await placementFeasibility.specNamesThisNode(appSpecifications, localSocketAddr); + } + + // A synced app may only be refused when a better-placed candidate provably + // exists: this domain is refused once it holds its share of the instances, + // computed over the app's eligible candidate set - never refused outright. + let placementShare = null; + let placementDomainOf = null; + let myDomain = null; + if (syncthingApp && !ownerNamedThisNode) { + // placementComputation refuses a geo-restricted question while the location + // table is still loading, because answering it over the whole network would + // advise on numbers that mean nothing. That refusal is addressed to the HTTP + // caller; reaching the catch below instead would read as a pre-install error + // and park this app for six hours over a table that is seconds from ready. + let computation; + try { + computation = await placementFeasibility.placementComputation(appSpecifications, minInstances); + } catch (error) { + if (error.statusCode !== 503) throw error; + log.info(`trySpawningGlobalApplication - ${appSpecifications.name} deferred: ${error.message}`); return shortDelayTime; } - sameIpRangeNode = installingAppList.find((location) => location.ip.startsWith(ipPrefix)); - if (sameIpRangeNode) { - log.info(`trySpawningGlobalApplication - Application ${appToRun} uses syncthing and it is already being installed on Fluxnode with same ip range`); + placementShare = computation.feasibility; + placementDomainOf = computation.domainOf; + myDomain = placementDomainOf(localIp); + // No `placeable` gate here, deliberately. This node reached the placement + // check having passed its own geolocation filter, so it is itself an + // eligible candidate - a table that resolves zero candidates network-wide + // is contradicting the node's own location rather than proving the app + // unplaceable, and refusing on that would strand the app everywhere. + // Install-time geolocation checks remain authoritative. + const heldInMine = await placementFeasibility.countHeldInDomain(runningAppList, myDomain, placementDomainOf) + + await placementFeasibility.countHeldInDomain(installingAppList, myDomain, placementDomainOf); + if (heldInMine >= placementShare.maxPerDomain) { + log.info(`trySpawningGlobalApplication - Application ${appToRun} uses syncthing and fault domain ${myDomain} already holds ${heldInMine} of its ${placementShare.maxPerDomain}-instance share (${placementShare.domainCount} eligible domains)`); return shortDelayTime; } + } + + if (syncthingApp) { if (!appFromAppsToBeCheckedLater && !appFromAppsSyncthingToBeCheckedLater && runningAppList.length < 6) { // check if there are connectivity to all nodes // eslint-disable-next-line no-restricted-syntax @@ -637,7 +885,7 @@ async function trySpawningGlobalApplication() { // eslint-disable-next-line no-restricted-syntax for (const componentToInstall of compositedSpecification) { - // check image is whitelisted and repotag is available for download + // check repotag is available for download // eslint-disable-next-line no-await-in-loop await imageManager.verifyRepository(componentToInstall.repotag, { repoauth: componentToInstall.repoauth, @@ -657,11 +905,72 @@ async function trySpawningGlobalApplication() { // triple check if app is installed on the number of instances requested runningAppList = await registryManager.appLocation(appToRun); installingAppList = await registryManager.appInstallingLocation(appToRun); - if (runningAppList.length + installingAppList.length > minInstances) { + if (runningAppList.length + installingAppList.length >= minInstances) { + // KEPT when the running copies alone meet the count, CLEARED when the + // claims were needed to reach it. + // + // A running copy is a durable fact and caching it is the point of the + // cache - the app is covered, and re-deciding that every pass is waste. + // A claim is not: it is withdrawn as soon as its node finds the share + // already filled, seconds later and by design, because the share is + // checked after the claim goes out. Cached on a count that needed those + // claims, this node remembers "covered" for the cache's twelve hours and + // never reconsiders, so an app that falls back below its instance count + // waits out the day on every node that glanced inside that window. + // + // Clearing unconditionally is the other way to be wrong: an app whose + // count is genuinely met would re-enter the candidate pool on every pass + // and be declined again forever, never cached because it was never + // installed. + if (runningAppList.length < minInstances) { + globalState.trySpawningGlobalAppCache.delete(appHash); + } log.info(`trySpawningGlobalApplication - Application ${appToRun} is already spawned or being installed on ${runningAppList.length + installingAppList.length} instances.`); return shortDelayTime; } + // Retract this node's installing claim, network-wide. A silent back-out + // leaves the fluxappinstalling broadcast alive for its full TTL, and that + // ghost keeps counting against instance totals and domain shares - and can + // even win the cold-start seed election - for up to 15 minutes. On a small + // eligible pool (a pinned org or region) one collision round of ghosts + // stalls the whole domain for that window, so every withdrawal must say so. + // + // The retraction is a version 2 fluxappinstalling: the claim's own message, + // withdrawing the claim. NOT an installing error - that means an install was + // attempted and failed, it is counted and acted on as such, and a node + // standing aside has attempted nothing. Counting these would make the apps + // most in demand, whose races have the most losers, look the most broken. + // + // A node that does not know version 2 rejects the message whole, so it + // neither acts on it nor refreshes the claim's clock: the claim expires on + // its own, exactly as it did before any of this existed. + // Standing aside costs no eligibility. A node that reconsiders this app while + // the winner is still installing is turned away by the guards above - they + // count claims as well as running instances - so it never re-claims and + // nothing loops. And when the app IS short again because a holder died, a + // node that once lost the race is exactly the one that should take it. + const withdrawInstallingClaim = async (reason) => { + log.info(`trySpawningGlobalApplication - withdrawing installing claim for ${appToRun}: ${reason}`); + try { + const withdrawal = { + type: 'fluxappinstalling', + version: 2, + name: appSpecifications.name, + ip: localSocketAddr, + broadcastedAt: Date.now(), + withdrawn: true, + }; + await messageStore.storeAppInstallingMessage(withdrawal); + // eslint-disable-next-line global-require + const fluxCommMessagesSenderLib = require('../fluxCommunicationMessagesSender'); + await fluxCommMessagesSenderLib.broadcastMessageToAll(withdrawal); + } catch (error) { + // best effort - the installing TTL remains the backstop + log.warn(`trySpawningGlobalApplication - could not retract installing claim for ${appToRun}: ${error.message}`); + } + }; + // an application was selected and checked that it can run on this node. try to install and run it locally // lets broadcast to the network the app is going to be installed on this node, so we don't get lot's of intances installed when it's not needed let broadcastedAt = Date.now(); @@ -686,59 +995,85 @@ async function trySpawningGlobalApplication() { runningAppList = await registryManager.appLocation(appToRun); installingAppList = await registryManager.appInstallingLocation(appToRun); if (runningAppList.length + installingAppList.length > minInstances) { - installingAppList.sort((a, b) => { - if (a.broadcastedAt < b.broadcastedAt) { - return -1; - } - if (a.broadcastedAt > b.broadcastedAt) { - return 1; - } - return 0; - }); + installingAppList.sort(compareInstallingClaims); + log.info(`trySpawningGlobalApplication - Application ${appToRun} contended: ${runningAppList.length} running, claims after wait: ${describeRanking(installingAppList, 'broadcastedAt')}`); broadcastedAt = Date.now(); const index = installingAppList.findIndex((x) => socketAddressesMatch(x.ip, localSocketAddr)); if (runningAppList.length + index + 1 > minInstances) { log.info(`trySpawningGlobalApplication - Application ${appToRun} is already spawned or being installed on ${runningAppList.length + installingAppList.length} instances, my instance is number ${runningAppList.length + index + 1}`); + await withdrawInstallingClaim('instance count filled by earlier claimants'); + globalState.trySpawningGlobalAppCache.delete(appHash); return shortDelayTime; } } - if (syncthingApp) { - const sameIpRangeNode = runningAppList.find((location) => location.ip.startsWith(ipPrefix)); - if (sameIpRangeNode) { - log.info(`trySpawningGlobalApplication - Application ${appToRun} uses syncthing and it is already spawned on Fluxnode with same ip range`); + if (syncthingApp && !ownerNamedThisNode && placementShare) { + // Re-check the domain share against the propagated lists, keyed by the + // same computation that produced the share - a fresher view of the + // network would move nodes between domains the share was never computed + // for. Running instances consume the share outright; among simultaneous + // installing claimants the earliest broadcasts win the remainder - the + // generalisation of the old oldest-wins resolver to shares above one. + const runningInMine = await placementFeasibility.countHeldInDomain(runningAppList, myDomain, placementDomainOf); + const remainingShare = placementShare.maxPerDomain - runningInMine; + if (remainingShare <= 0) { + log.info(`trySpawningGlobalApplication - Application ${appToRun} uses syncthing and fault domain ${myDomain} already runs ${runningInMine} of its ${placementShare.maxPerDomain}-instance share`); + await withdrawInstallingClaim('domain share held by running instances'); + globalState.trySpawningGlobalAppCache.delete(appHash); return shortDelayTime; } - const sameIpRangeInstallingNodes = installingAppList.filter((location) => location.ip.startsWith(ipPrefix)); - if (sameIpRangeInstallingNodes.length > 0) { - // Find the node with the oldest broadcastedAt (first to start installing) - const oldestNode = sameIpRangeInstallingNodes.reduce((oldest, current) => { - if (!oldest.broadcastedAt) return current; - if (!current.broadcastedAt) return oldest; - return current.broadcastedAt < oldest.broadcastedAt ? current : oldest; - }); - // If our node is not the oldest one, skip - let the first node continue - if (!socketAddressesMatch(oldestNode.ip, localSocketAddr)) { - log.info(`trySpawningGlobalApplication - Application ${appToRun} uses syncthing and it is already being installed on Fluxnode with same ip range`); - return shortDelayTime; - } - // Our node is the oldest - we were first, continue with installation - log.info(`trySpawningGlobalApplication - Application ${appToRun} uses syncthing, we are the first node in ip range to start installing, continuing`); + const claimantsInMine = installingAppList + .filter((location) => placementDomainOf(location.ip) === myDomain) + .sort(compareInstallingClaims); + const myIndex = claimantsInMine.findIndex((location) => socketAddressesMatch(location.ip, localSocketAddr)); + const claimantsAhead = myIndex === -1 ? claimantsInMine.length : myIndex; + if (claimantsAhead >= remainingShare) { + log.info(`trySpawningGlobalApplication - Application ${appToRun} uses syncthing and ${claimantsAhead} earlier claimants in fault domain ${myDomain} fill its remaining share of ${remainingShare} (claims: ${describeRanking(claimantsInMine, 'broadcastedAt')})`); + await withdrawInstallingClaim('domain share filled by earlier claimants'); + globalState.trySpawningGlobalAppCache.delete(appHash); + return shortDelayTime; + } + if (claimantsInMine.length > 1) { + log.info(`trySpawningGlobalApplication - Application ${appToRun} uses syncthing, this node is claim ${claimantsAhead + 1} of ${remainingShare} remaining in fault domain ${myDomain}, continuing (claims: ${describeRanking(claimantsInMine, 'broadcastedAt')})`); } } + // The node is already doing something to an app, and the spawner is the one + // that gives way: the periodic reinstall pass holds this flag across its own + // teardown-and-rebuild, and an install started inside that window is refused + // when the pass comes back for its node - leaving the app it tore down with + // nothing to rebuild it. The claim is withdrawn rather than held, so another + // node can take the placement now instead of waiting this one out. + const heldBy = globalState.operationHolding(); + if (heldBy) { + log.info(`trySpawningGlobalApplication - Application ${appToRun} not installed, this node is undergoing ${heldBy}`); + await withdrawInstallingClaim(`node is undergoing ${heldBy}`); + globalState.trySpawningGlobalAppCache.delete(appHash); + return shortDelayTime; + } + // install the app let registerOk = false; + // The installer still signals some failures by throwing, and only the reason + // it throws with says WHICH check refused - a port already held by another + // app is raised that way, and reporting the failure without it leaves a suite + // unable to tell a refusal from an app that was simply never selected. What + // it no longer does is collapse "I touched nothing" and "I tore the app down" + // into one false: the outcome says which. + let installError = null; try { - registerOk = await appInstaller.registerAppLocally(appSpecifications, null, null, false); // can throw + const outcome = await appInstaller.registerAppLocally(appSpecifications, null, null, false); // can throw + registerOk = outcome === InstallOutcome.INSTALLED; + if (!registerOk) installError = `installer ${outcome}`; } catch (error) { log.error(error); + installError = error.message ?? String(error); registerOk = false; } if (!registerOk) { log.info(`trySpawningGlobalApplication - Install failed for ${appToRun}, adding to local error cache`); globalState.spawnErrorsLongerAppCache.set(appHash, ''); - fluxEventBus.publish('spawner:installFailed', { appName: appToRun, hash: appHash }); + fluxEventBus.publish('spawner:installFailed', { appName: appToRun, hash: appHash, error: installError }); return shortDelayTime; } @@ -746,23 +1081,9 @@ async function trySpawningGlobalApplication() { // double check if app is installed in more of the instances requested runningAppList = await registryManager.appLocation(appToRun); if (runningAppList.length > minInstances) { - runningAppList.sort((a, b) => { - if (!a.runningSince && b.runningSince) { - return -1; - } - if (a.runningSince && !b.runningSince) { - return 1; - } - if (a.runningSince < b.runningSince) { - return -1; - } - if (a.runningSince > b.runningSince) { - return 1; - } - return 0; - }); + runningAppList.sort(compareInstanceSeniority); const index = runningAppList.findIndex((x) => socketAddressesMatch(x.ip, localSocketAddr)); - log.info(`trySpawningGlobalApplication - Application ${appToRun} is already spawned on ${runningAppList.length} instances, my instance is number ${index + 1}`); + log.info(`trySpawningGlobalApplication - Application ${appToRun} is already spawned on ${runningAppList.length} instances, my instance is number ${index + 1} (instances: ${describeRanking(runningAppList, 'runningSince')})`); if (index + 1 > minInstances) { log.info(`trySpawningGlobalApplication - Application ${appToRun} is going to be removed as already passed the instances required.`); log.warn(`REMOVAL REASON: Exceeded required instances - ${appSpecifications.name} already has sufficient instances, removing local installation (appSpawner)`); diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 4134da617c..fcf432f1a8 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -24,6 +24,28 @@ const { getNonGComponentIdentifiers, parseContainerName, appHasValidLocationOnNo const SYNC_TIMEOUT_MS = config.system.bootSyncTimeoutMs ?? 300000; +/** + * Await a promise, giving up after a deadline. The timer is cleared however the + * race ends, so work that finishes early leaves nothing pending behind it. + * @param {Promise} work What to wait for. + * @param {number} timeoutMs How long to wait. + * @param {string} reason Message of the error thrown when the deadline passes. + * @returns {Promise} Whatever `work` resolved to. + */ +async function awaitWithin(work, timeoutMs, reason) { + let timer; + try { + return await Promise.race([ + work, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(reason)), timeoutMs); + }), + ]); + } finally { + clearTimeout(timer); + } +} + /** * Get all installed apps from local database * @returns {Promise} Array of installed app specifications @@ -276,10 +298,7 @@ async function manageAppsOnBoot(bootContext) { // Locations still valid — wait for daemon + sync then reconcile. const DAEMON_TIMEOUT_MS = config.system.bootDaemonTimeoutMs ?? 300000; try { - await Promise.race([ - globalState.waitForDaemonReady(), - new Promise((_, reject) => { setTimeout(() => reject(new Error('daemon_timeout')), DAEMON_TIMEOUT_MS); }), - ]); + await awaitWithin(globalState.waitForDaemonReady(), DAEMON_TIMEOUT_MS, 'daemon_timeout'); } catch (error) { if (error.message === 'daemon_timeout') { log.error(`appStartupManager - Daemon not ready after ${DAEMON_TIMEOUT_MS / 1000}s, removing all apps`); @@ -303,10 +322,7 @@ async function manageAppsOnBoot(bootContext) { } try { - await Promise.race([ - globalState.waitForDbReady(), - new Promise((_, reject) => { setTimeout(() => reject(new Error('sync_timeout')), SYNC_TIMEOUT_MS); }), - ]); + await awaitWithin(globalState.waitForDbReady(), SYNC_TIMEOUT_MS, 'sync_timeout'); } catch (error) { if (error.message === 'sync_timeout') { log.error(`appStartupManager - DB not ready after ${SYNC_TIMEOUT_MS / 1000}s, removing all apps`); diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 38bc746fd3..6e10cae7e4 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -21,8 +21,8 @@ const { specificationFormatter } = require('../utils/appSpecHelpers'); const { stopAppMonitoring } = require('../appManagement/appInspector'); const appsRuntimeState = require('../appManagement/appsRuntimeState'); const volumeService = require('../utils/volumeService'); -const imageManager = require('../appSecurity/imageManager'); const fluxEventBus = require('../utils/fluxEventBus'); +const { Privilege, authOf } = require('../utils/privileges'); const fluxDirPath = process.env.FLUXOS_PATH || path.join(process.env.HOME, 'zelflux'); const appsFolderPath = process.env.FLUX_APPS_FOLDER || path.join(fluxDirPath, 'ZelApps'); @@ -32,8 +32,8 @@ const crontabLoad = util.promisify(systemcrontab.load); // Fired once per component identifier after a successful local removal, beside // the durable runtime-state clear (mirrors appInstaller.setOnInstallComplete). -// serviceManager wires it to appReconciler.clearControllerDesired so the -// reconciler's in-memory controller verdict dies with the component - a +// serviceManager wires it to appReconciler.forgetDesiredState so every +// in-memory verdict about the component dies with it - a // back-require of appReconciler here would capture a stale partial export // (appReconciler already requires this module and both replace module.exports). let onComponentRemoved = null; @@ -1021,6 +1021,13 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen if (res.flush) res.flush(); } await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); + // The app is gone for good: nothing reconciles an app with no row, so its + // removal records have no reader left. Only this full-uninstall path clears + // them - softRemoveAppLocally deletes the row too, but as one step of a + // redeploy whose containers are coming straight back, and its records are + // exactly what stops a teardown that fails part way being read as + // tampering. + dockerService.clearFluxRemovedContainers(appName); const databaseStatus2 = { status: 'Database cleaned', }; @@ -1202,43 +1209,26 @@ async function removeAppLocallyApi(req, res) { throw new Error('No Flux App specified'); } - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); + // The node operator is deliberately NOT here. Hosting an app is not owning it: + // an operator who can remove one can script the removal against every install + // and keep a customer's app off their node indefinitely, which the customer + // experiences as an app that will not stay deployed and cannot diagnose. + // Ending an app is the owner's call, or the team's on their behalf. + // + // One gate, and vetted status does not narrow it further: appownerorfluxteam + // admits exactly {owner, fluxTeam, fluxSupport}, which is who may uninstall a + // vetted app too. A second check against the same set can only ever agree, and + // asking it costs two database reads and a vetted lookup per uninstall. + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); } - // For vetted apps, only app owner or Flux Team can uninstall - // First, get app specifications to check if vetted - const dbopen = dbHelper.databaseConnection(); - const appsDatabase = dbopen.db(config.database.appslocal.database); - const database = dbopen.db(config.database.appsglobal.database); - const appsQuery = { name: appname }; - const appsProjection = {}; - - let appSpecsForVettedCheck = await dbHelper.findOneInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); - if (!appSpecsForVettedCheck) { - appSpecsForVettedCheck = await dbHelper.findOneInDatabase(database, globalAppsInformation, appsQuery, appsProjection); - } - - if (appSpecsForVettedCheck) { - const appIsVetted = await imageManager.isAppVetted(appSpecsForVettedCheck); - if (appIsVetted) { - // Check if user is specifically the app owner or Flux Team - const isAppOwner = await verificationHelper.verifyPrivilege('appowner', req, appname); - const isFluxTeam = await verificationHelper.verifyPrivilege('fluxteam', req); - - if (!isAppOwner && !isFluxTeam) { - const errMessage = messageHelper.createErrorMessage('This is a vetted application. Only the app owner or InFlux Support Team are allowed to uninstall it.'); - return res.json(errMessage); - } - } - } - if (global) { // eslint-disable-next-line global-require const appController = require('../appManagement/appController'); - appController.executeAppGlobalCommand(appname, 'appremove', req.headers.zelidauth); // do not wait + appController.executeAppGlobalCommand(appname, 'appremove', authOf(req)); // do not wait const appResponse = messageHelper.createSuccessMessage(`${appname} queried for global reinstallation`); return res.json(appResponse); } diff --git a/ZelBack/src/services/appManagement/appController.js b/ZelBack/src/services/appManagement/appController.js index e81781a341..46885ec4fd 100644 --- a/ZelBack/src/services/appManagement/appController.js +++ b/ZelBack/src/services/appManagement/appController.js @@ -4,14 +4,70 @@ const serviceHelper = require('../serviceHelper'); // Removed verificationHelper to avoid circular dependency - will use dynamic require where needed const messageHelper = require('../messageHelper'); const dockerService = require('../dockerService'); -const registryManager = require('../appDatabase/registryManager'); -const appInspector = require('./appInspector'); const appsRuntimeState = require('./appsRuntimeState'); +const appReconciler = require('../appMonitoring/appReconciler'); const fluxNetworkHelper = require('../fluxNetworkHelper'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); +const fluxEventBus = require('../utils/fluxEventBus'); const log = require('../../lib/log'); +const { Privilege, authOf } = require('../utils/privileges'); -const globalCmdDelayMs = config.fluxapps.globalCmdDelayMs; +const { globalCmdDelayMs } = config.fluxapps; +// Guaranteed a finite non-negative integer, so a missing or malformed config +// value can never spin the retry loop below forever. +const globalCmdBootRetries = (Number.isInteger(config.fluxapps.globalCmdBootRetries) + && config.fluxapps.globalCmdBootRetries >= 0) + ? config.fluxapps.globalCmdBootRetries + : 8; + +// A node still reconciling its apps after boot refuses these routes with 15s. +const BOOT_RETRY_AFTER_FALLBACK_S = 15; +// Caps a node's Retry-After so a hostile or absurd value cannot stall delivery. +const BOOT_RETRY_MAX_WAIT_MS = 60 * 1000; + +/** + * Send one global command to one instance, retrying only a boot-gate refusal. + * + * A node that has not finished reconciling its apps after boot answers these + * routes with 503 + Retry-After (see requireBootSettled), which is + * self-resolving - it settles within its boot window. Retrying that a bounded + * number of times keeps a global command from being dropped on the first + * refusal: without it a global appremove aimed at a node mid-restart never + * lands, the app stays installed and running, and the owner was already told + * the removal was queried. ONLY a 503 is retried; any other status is the + * node's real answer and is final, and a node still refusing after the bound is + * warned about rather than hammered forever. + * + * Errors are handled internally, so this never rejects - callers fire it and + * move on. + * + * @param {string} url + * @param {object} axiosConfig + */ +async function deliverGlobalCommand(url, axiosConfig) { + for (let attempt = 0; ; attempt += 1) { + try { + // eslint-disable-next-line no-await-in-loop + const response = await axios.get(url, axiosConfig); + log.info(`Successfully sent command to ${url}: ${response.status}`); + return; + } catch (error) { + const status = error.response && error.response.status; + if (status !== 503) { + log.error(`Axios request failed for ${url}`, error); + return; + } + if (attempt >= globalCmdBootRetries) { + log.warn(`Node at ${url} still reconciling apps after boot; command not delivered after ${globalCmdBootRetries} retries`); + return; + } + const headerRetryAfter = Number(error.response.headers && error.response.headers['retry-after']); + const retryAfterS = headerRetryAfter > 0 ? headerRetryAfter : BOOT_RETRY_AFTER_FALLBACK_S; + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(Math.min(retryAfterS * 1000, BOOT_RETRY_MAX_WAIT_MS)); + } + } +} /** * Get application locations from the global database @@ -82,13 +138,9 @@ async function executeAppGlobalCommand(appname, command, zelidauth, paramA, bypa if (paramA) { url += `/${paramA}`; } - axios.get(url, axiosConfig) - .then((response) => { - log.info(`Successfully sent command to ${url}: ${response.status}`); - }) - .catch((error) => { - log.error(`Axios request failed for ${url}`, error); - }); + // Fire-and-forget: each node's delivery, with its own bounded retry of a + // boot-gate 503, runs on its own while the loop paces the sends. + deliverGlobalCommand(url, axiosConfig); // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(globalCmdDelayMs); } @@ -110,17 +162,176 @@ async function executeAppGlobalCommand(appname, command, zelidauth, paramA, bypa * * @param {string} appname app or component identifier * @param {object|null} appSpecs full app spec (null for a component command) + * The components are resolved from what the app is actually made of, never from a + * stored spec's `compose`: an enterprise app keeps its component names inside an + * encrypted blob, so reading compose yields an empty list and a whole-app command + * addresses nothing while reporting success. + * * @param {boolean} stopped + * @param {object} [options] + * @param {boolean} [options.awaitPass] hold until the reconcile pass has run + * @param {boolean} [options.force] a stop is a hard kill, not a graceful stop + * @param {boolean} [options.alsoRestart] raise the restart generation with the lock */ -async function setAppOperatorStopped(appname, appSpecs, stopped) { - const ids = (!appname.includes('_') && appSpecs && appSpecs.version > 3) - ? appSpecs.compose.map((c) => `${c.name}_${appSpecs.name}`) - : [appname]; +async function operatorTargetIds(appname) { + const mainAppName = appname.split('_')[1] || appname; + // eslint-disable-next-line global-require + const appQueryService = require('../appQuery/appQueryService'); + const installedRes = await appQueryService.installedApps(mainAppName); + if (!installedRes || installedRes.status !== 'success' || !installedRes.data.length) { + throw new Error(`Application ${mainAppName} is not installed on this node`); + } + const appName = installedRes.data[0].name; + const ids = await appReconciler.componentIdsOf(installedRes.data); + // An installed app with nothing to address is not an app that needs nothing + // done to it - it is one whose parts this node cannot work out: a spec that + // will not decrypt, falling back to a docker listing that is empty or that + // failed outright. Acting on the empty list wrote no intent, settled + // vacuously against nothing, and answered that the command had succeeded. + // Refused rather than reported, in the operator's terms: what they need to + // know is that nothing happened. + if (!ids.length) { + throw new Error(`Application ${appName} was not changed: this node cannot determine its components`); + } + if (!appname.includes('_')) return { ids, appName }; + // A component is addressed by name, and a name that is not one of this app's + // components addresses nothing. Taking it verbatim wrote a durable operator + // lock under a component that does not exist - nothing clears one, and it holds + // the real component down if one is ever created with that name. + if (!ids.includes(appname)) { + throw new Error(`Component ${appname} is not installed on this node`); + } + return { ids: [appname], appName: appname }; +} + +async function setAppOperatorStopped(appname, stopped, { awaitPass = false, force = false, alsoRestart = false } = {}) { + const { ids, appName } = await operatorTargetIds(appname); + // Components come up in compose order and go down in the reverse of it, so a + // dependency outlives what writes to it: the database stops after the server it + // serves, not before it. awaitPass holds each component's pass open before the + // next id is touched, so this order is the order the containers move in. + // Reversed on the mapped ids, which is a fresh array - never on the spec, whose + // compose array is shared with whatever the caller fetched it from. + if (stopped) ids.reverse(); + let allActuated = true; // eslint-disable-next-line no-restricted-syntax for (const id of ids) { + // Written through the reconciler's per-key slot rather than straight to the + // store. A pass reads the lock and acts on that answer once docker has + // replied, so a write landing in between is not seen: the pass starts a + // container the operator has just stopped and the next pass stops it again. + // applyIntent waits out any pass deciding for this id, holds the key while + // the write lands, and enqueues on release - so the two cannot interleave, + // and the next pass reads what was just written. // eslint-disable-next-line no-await-in-loop - await appsRuntimeState.setOperatorStopped(id, stopped); + const actuated = await appReconciler.applyIntent(id, async () => { + await appsRuntimeState.setOperatorStopped(id, stopped, { force }); + // Raised inside the same slot as the lock, so a pass cannot read one + // without the other and bounce a container the operator meant to keep down. + if (alsoRestart) await appsRuntimeState.requestRestart(id); + // The operator's intent is the one desired-state write in this flow that + // announced nothing, so nothing could be ordered against it - and the + // failure it hides is an actuation on the PREVIOUS intent arriving after + // this one landed. Published from inside the slot: after the write, so it + // can never claim an intent that did not persist, and before the pass, + // which is what makes it the ordering point. + fluxEventBus.publish('app:operatorIntent', { + // The bare component id the reconciler publishes its own actuations + // under. An event carrying a different spelling of the same component + // cannot be ordered against them, which is the only thing it is for. + identifier: dockerService.getBaseAppName(id), stopped, force, restartRequested: alsoRestart, + }); + }, { awaitPass }); + if (!actuated) allActuated = false; + // A stop retracts the controller's desire as well as taking the lock. The + // lock only suppresses the reconciler while it is held; a desire left + // standing is reconciled against the stopped container the moment the lock + // lifts, restarting a g:/r: component with no election pass and putting it + // beside whichever peer took over. Retracted, the component sits at "no + // controller opinion" - take no action - until its decider re-derives + // intent. Plain apps do not consult the controller, so their + // resume-on-start is unchanged. + // + // The RUN opinion only. A pending appdata clear is the sync layer's finding + // that the local data must not be trusted, and an operator stopping the app + // says nothing about that - dropping it here would lose it for good, since + // the sync layer marks a component processed before it asks. + if (stopped) appReconciler.clearControllerDesired(id); } + return { ids, actuated: allActuated, appName }; +} + +/** + * What the containers are actually doing, once the reconciler has had its pass. + * + * `actuated` says a pass ran, not that it achieved anything: a pass that finds + * docker unreachable completes by deferring. So the answer to "is it stopped" + * comes from probing, and dockerActual is the probe that can tell a container + * being gone from docker being unreachable - which is the difference between + * reporting done and reporting pending. + * @param {string[]} ids Component identifiers. + * @returns {Promise<{settled: boolean, reason: string|null}>} + */ +async function containersReachedStopped(ids) { + // eslint-disable-next-line no-restricted-syntax + for (const id of ids) { + // eslint-disable-next-line no-await-in-loop + const actual = await appReconciler.dockerActual(id); + if (!actual.reachable) return { settled: false, reason: 'docker is not reachable' }; + // Nothing there is not the same as stopped. dockerActual distinguishes the two + // and this read the pair as one, so a command against a container that does not + // exist settled - and answered "stopped" for something that was never running. + if (!actual.exists) return { settled: false, reason: 'it is not installed on this node' }; + if (actual.running) return { settled: false, reason: 'the reconciler has not stopped it yet' }; + } + return { settled: true, reason: null }; +} + +// Why the reconciler is not running a component, in the operator's terms. The +// election cases are not failures: a synced component runs on the node the +// election made the writer, so "not started" is the correct outcome elsewhere +// and saying so is more use than a generic wait. +const NOT_RUNNING_REASONS = { + awaitingController: 'waiting for the election', + controllerDesired: 'the election has not made this node the writer', + policy: 'its restart policy does not allow it to run', + invalidSpec: 'its specification cannot be actuated', + notInstalled: 'it is not installed on this node', +}; + +/** + * What the containers are actually doing, once the reconciler has had its pass. + * + * The mirror of containersReachedStopped, with one asymmetry: a container that + * is not running may be one the reconciler is right to leave alone, so the + * reason comes from the reconciler's own verdict rather than from the absence. + * + * @param {string[]} ids Component identifiers. + * @returns {Promise<{settled: boolean, reason: string|null}>} + */ +async function containersReachedRunning(ids) { + // eslint-disable-next-line no-restricted-syntax + for (const id of ids) { + // eslint-disable-next-line no-await-in-loop + const actual = await appReconciler.dockerActual(id); + if (!actual.reachable) return { settled: false, reason: 'docker is not reachable' }; + if (actual.running) { + // eslint-disable-next-line no-continue + continue; + } + let verdict; + try { + // eslint-disable-next-line no-await-in-loop + verdict = await appReconciler.desiredRunState(id); + } catch (err) { + return { settled: false, reason: `its state could not be read: ${err.message}` }; + } + return { + settled: false, + reason: NOT_RUNNING_REASONS[verdict.reason] || 'the reconciler has not started it yet', + }; + } + return { settled: true, reason: null }; } async function appStart(req, res) { @@ -137,113 +348,60 @@ async function appStart(req, res) { const mainAppName = appname.split('_')[1] || appname; - // eslint-disable-next-line global-require // Use dynamic require to avoid circular dependency // eslint-disable-next-line global-require const verificationHelper = require('../verificationHelper'); - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + // This refuses the node operator, and whether someone + // else's app runs is not theirs to decide. The same gate appkill and + // appremove ask for; the argument is on verifyAppOwnerOrFluxTeamSession. + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; + return res.json(errMessage); } if (global) { - executeAppGlobalCommand(appname, 'appstart', req.headers.zelidauth); // do not wait + executeAppGlobalCommand(appname, 'appstart', authOf(req)); // do not wait const appResponse = messageHelper.createSuccessMessage(`${appname} queried for global start`); - return res ? res.json(appResponse) : appResponse; + return res.json(appResponse); } - const isComponent = appname.includes('_'); // it is a component start - let appRes; - - if (isComponent) { - // user-initiated start clears the operator stop lock so the reconciler keeps it running - await setAppOperatorStopped(appname, null, false); - // For component start, check if it uses g:syncthing mode - const componentMainApp = appname.split('_')[1]; - const appSpecs = await registryManager.getApplicationSpecifications(componentMainApp); - if (appSpecs && appSpecs.version > 3) { - const componentSpec = appSpecs.compose.find((comp) => `${comp.name}_${appSpecs.name}` === appname); - if (componentSpec && componentSpec.containerData && componentSpec.containerData.includes('g:')) { - // Check if component is running - try { - const containers = await dockerService.dockerListContainers(false); // Get only running containers - const isRunning = containers.some((container) => container.Names[0] === dockerService.getAppDockerNameIdentifier(appname) || container.Id === appname); - if (!isRunning) { - log.info(`Skipping start for g:syncthing component ${appname} - not currently running`); - appRes = `Component ${appname} uses g:syncthing mode and is not running - skipped start`; - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; - } - } catch (error) { - log.warn(`Could not check running status for ${appname}: ${error.message}`); - } - } - } - appRes = await dockerService.appDockerStart(appname); - appInspector.startAppMonitoring(appname); - } else { - // Check if app exists before starting - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - // user-initiated start clears the operator stop lock so the reconciler keeps it running - await setAppOperatorStopped(appname, appSpecs, false); - - if (appSpecs.version <= 3) { - // For non-composed apps, check if it uses g:syncthing mode - if (appSpecs.containerData && appSpecs.containerData.includes('g:')) { - try { - const containers = await dockerService.dockerListContainers(false); - const isRunning = containers.some((container) => container.Names[0] === dockerService.getAppDockerNameIdentifier(appname) || container.Id === appname); - if (!isRunning) { - log.info(`Skipping start for g:syncthing app ${appname} - not currently running`); - appRes = `Application ${appname} uses g:syncthing mode and is not running - skipped start`; - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; - } - } catch (error) { - log.warn(`Could not check running status for ${appname}: ${error.message}`); - } - } - appRes = await dockerService.appDockerStart(appname); - appInspector.startAppMonitoring(appname); - } else { - // For composed applications (version > 3), start all components - log.info(`Starting composed app ${appSpecs.name} with ${appSpecs.compose.length} components`); - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose) { - const componentName = `${appComponent.name}_${appSpecs.name}`; - // Check if component uses g:syncthing mode - if (appComponent.containerData && appComponent.containerData.includes('g:')) { - try { - // eslint-disable-next-line no-await-in-loop - const containers = await dockerService.dockerListContainers(false); - const isRunning = containers.some((container) => container.Names[0] === dockerService.getAppDockerNameIdentifier(componentName) || container.Id === componentName); - if (!isRunning) { - log.info(`Skipping start for g:syncthing component ${componentName} - not currently running`); - // eslint-disable-next-line no-continue - continue; - } - } catch (error) { - log.warn(`Could not check running status for ${componentName}: ${error.message}`); - } - } - log.info(`Starting component: ${componentName}`); - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerStart(componentName); - log.info(`Component ${componentName} started, starting monitoring`); - appInspector.startAppMonitoring(componentName); - log.info(`Monitoring started for ${componentName}`); - } - log.info(`All components started for ${appSpecs.name}`); - appRes = `Application ${appSpecs.name} started`; - } + // THE RECONCILER STARTS IT, NOT THIS HANDLER. + // + // Clearing the lock is the whole of an operator start: whether the container + // may run is a decision the election already owns for a g:/r: component, and + // the reconciler consults it on every pass. A handler that also probed docker + // was asking a different question - "is this container running now" as a proxy + // for "should this node be running it" - and those diverge both ways: a + // primary whose container is stopped was refused a start, a standby whose + // container happened to be up was started. + // + // awaitPass holds this handler until the pass has run, so a success still + // means the container is running in the same wall-clock the direct call took. + // Which components this addresses - and whether the app or component even + // exists here - is resolved from what the app is made of, in one place. + const { ids, actuated, appName: startedName } = await setAppOperatorStopped(appname, false, { awaitPass: true }); + + // A pass that completed is not a container that started - docker being + // unreachable completes by deferring, and a synced component the election + // holds elsewhere is a pass that correctly did nothing. Probe rather than + // infer, and name which of the two it was. + const outcome = actuated + ? await containersReachedRunning(ids) + : { settled: false, reason: 'no reconcile has run yet' }; + + if (!outcome.settled) { + // Accepted, not applied. The intent is durable and the reconciler converges + // on it; where the reason is the election, "not started here" is the correct + // outcome rather than a failure, and the operator is told which it is. + const pending = messageHelper.createDataMessage( + `Application ${startedName} will be started: ${outcome.reason}`, + ); + return res.json(pending); } - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; + const appResponse = messageHelper.createDataMessage(`Application ${startedName} started`); + return res.json(appResponse); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage( @@ -251,7 +409,7 @@ async function appStart(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -279,56 +437,61 @@ async function appStop(req, res) { // Use dynamic require to avoid circular dependency // eslint-disable-next-line global-require const verificationHelper = require('../verificationHelper'); - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + // This refuses the node operator, and whether someone + // else's app runs is not theirs to decide. The same gate appkill and + // appremove ask for; the argument is on verifyAppOwnerOrFluxTeamSession. + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; + return res.json(errMessage); } if (global) { - executeAppGlobalCommand(appname, 'appstop', req.headers.zelidauth); // do not wait + executeAppGlobalCommand(appname, 'appstop', authOf(req)); // do not wait const appResponse = messageHelper.createSuccessMessage(`${appname} queried for global stop`); - return res ? res.json(appResponse) : appResponse; + return res.json(appResponse); } - const isComponent = appname.includes('_'); // it is a component stop - let appRes; - - if (isComponent) { - // lock BEFORE the docker op (matching the whole-app path): a crash between - // the stop and the lock write would leave a stopped container the - // reconciler restarts against the operator's intent - await setAppOperatorStopped(appname, null, true); - appInspector.stopAppMonitoring(appname, false); - appRes = await dockerService.appDockerStop(appname); - } else { - // Check if app exists before stopping - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - // operator stop persists so the reconciler does not restart it - await setAppOperatorStopped(appname, appSpecs, true); - - // eslint-disable-next-line no-restricted-syntax - if (appSpecs.version <= 3) { - // eslint-disable-next-line no-await-in-loop - appInspector.stopAppMonitoring(appname, false); - appRes = await dockerService.appDockerStop(appname); - } else { - // For composed applications (version > 3), stop all components in reverse order - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose.reverse()) { - appInspector.stopAppMonitoring(`${appComponent.name}_${appSpecs.name}`, false); - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerStop(`${appComponent.name}_${appSpecs.name}`); - } - appRes = `Application ${appSpecs.name} stopped`; - } + // THE RECONCILER STOPS IT, NOT THIS HANDLER. + // + // Two things drove the container before this: the handler called + // appDockerStop directly while the reconciler actuated off its own per-key + // queue. Two writers to one container is what made an operator stop + // interleave with a pass - the pass read the lock, the stop landed, the pass + // started the container it had already decided to start. Writing the intent + // and letting the single actuator converge removes the second writer rather + // than narrowing the window between them. + // + // The contract is unchanged: awaitPass holds this handler until the pass has + // run, so a success still means the container is stopped, in the same + // wall-clock the direct call took. + // + // Monitoring goes with the container, so the reconciler turns it off when it + // stops one. Doing it here stopped the sampler for a container the stop had + // not reached - an unreachable docker left it running and unwatched. + // Which components this addresses - and whether the app or component even + // exists here - is resolved from what the app is made of, in one place. + const { ids, actuated, appName: stoppedName } = await setAppOperatorStopped(appname, true, { awaitPass: true }); + + // A pass that completed is not a container that stopped - docker being + // unreachable completes by deferring. Probe rather than infer, so a stop + // that has not happened yet is never reported as one that has. + const outcome = actuated + ? await containersReachedStopped(ids) + : { settled: false, reason: 'no reconcile has run yet' }; + + if (!outcome.settled) { + // Accepted, not applied. The intent is durable and the reconciler will + // converge, so an error here would be false - the old direct call threw + // in exactly this case, after the lock had already been written. + const pending = messageHelper.createDataMessage( + `Application ${stoppedName} will be stopped: ${outcome.reason}`, + ); + return res.json(pending); } - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; + const appResponse = messageHelper.createDataMessage(`Application ${stoppedName} stopped`); + return res.json(appResponse); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage( @@ -336,7 +499,7 @@ async function appStop(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -354,7 +517,6 @@ async function appRestart(req, res) { global = global || req.query.global || false; global = serviceHelper.ensureBoolean(global); - // eslint-disable-next-line global-require if (!appname) { throw new Error('No Flux App specified'); } @@ -364,104 +526,46 @@ async function appRestart(req, res) { // Use dynamic require to avoid circular dependency // eslint-disable-next-line global-require const verificationHelper = require('../verificationHelper'); - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + // This refuses the node operator, and whether someone + // else's app runs is not theirs to decide. The same gate appkill and + // appremove ask for; the argument is on verifyAppOwnerOrFluxTeamSession. + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; + return res.json(errMessage); } if (global) { - executeAppGlobalCommand(appname, 'apprestart', req.headers.zelidauth); // do not wait + executeAppGlobalCommand(appname, 'apprestart', authOf(req)); // do not wait const appResponse = messageHelper.createSuccessMessage(`${appname} queried for global restart`); - return res ? res.json(appResponse) : appResponse; + return res.json(appResponse); } - const isComponent = appname.includes('_'); // it is a component restart - let appRes; - - if (isComponent) { - // user-initiated restart means "make it run": clear the operator stop lock - // (before the docker op) so the reconciler keeps it running afterwards - await setAppOperatorStopped(appname, null, false); - // For component restart, check if it uses g:syncthing mode - const componentMainApp = appname.split('_')[1]; - const appSpecs = await registryManager.getApplicationSpecifications(componentMainApp); - if (appSpecs && appSpecs.version > 3) { - const componentSpec = appSpecs.compose.find((comp) => `${comp.name}_${appSpecs.name}` === appname); - if (componentSpec && componentSpec.containerData && componentSpec.containerData.includes('g:')) { - // Check if component is running - try { - const containers = await dockerService.dockerListContainers(false); // Get only running containers - const isRunning = containers.some((container) => container.Names[0] === dockerService.getAppDockerNameIdentifier(appname) || container.Id === appname); - if (!isRunning) { - log.info(`Skipping restart for g:syncthing component ${appname} - not currently running`); - appRes = `Component ${appname} uses g:syncthing mode and is not running - skipped restart`; - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; - } - } catch (error) { - log.warn(`Could not check running status for ${appname}: ${error.message}`); - } - } - } - appRes = await dockerService.appDockerRestart(appname); - } else { - // Check if app exists before restarting - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - // eslint-disable-next-line no-restricted-syntax - if (!appSpecs) { - throw new Error('Application not found'); - } - // user-initiated restart means "make it run": clear the operator stop lock - // for every component (before the docker ops), matching appStart - await setAppOperatorStopped(appname, appSpecs, false); - - if (appSpecs.version <= 3) { - // For non-composed apps, check if it uses g:syncthing mode - if (appSpecs.containerData && appSpecs.containerData.includes('g:')) { - try { - const containers = await dockerService.dockerListContainers(false); - const isRunning = containers.some((container) => container.Names[0] === dockerService.getAppDockerNameIdentifier(appname) || container.Id === appname); - if (!isRunning) { - log.info(`Skipping restart for g:syncthing app ${appname} - not currently running`); - appRes = `Application ${appname} uses g:syncthing mode and is not running - skipped restart`; - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; - } - } catch (error) { - log.warn(`Could not check running status for ${appname}: ${error.message}`); - } - } - appRes = await dockerService.appDockerRestart(appname); - } else { - // For composed applications (version > 3), restart all components - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose) { - const componentName = `${appComponent.name}_${appSpecs.name}`; - // Check if component uses g:syncthing mode - if (appComponent.containerData && appComponent.containerData.includes('g:')) { - try { - // eslint-disable-next-line no-await-in-loop - const containers = await dockerService.dockerListContainers(false); - const isRunning = containers.some((container) => container.Names[0] === dockerService.getAppDockerNameIdentifier(componentName) || container.Id === componentName); - if (!isRunning) { - log.info(`Skipping restart for g:syncthing component ${componentName} - not currently running`); - // eslint-disable-next-line no-continue - continue; - } - } catch (error) { - log.warn(`Could not check running status for ${componentName}: ${error.message}`); - } - } - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerRestart(`${appComponent.name}_${appSpecs.name}`); - } - appRes = `Application ${appSpecs.name} restarted`; - } + // A RESTART IS DESIRED STATE, NOT A DOCKER CALL. + // + // "Make it run now" is the lock cleared and the restart generation raised. + // The reconciler bounces a running container once the generation passes the + // one it last actuated, and a stopped container is simply started - which is + // the same request satisfied. Expressing it as a level rather than an action + // is what removes the race: there is no window between this handler deciding + // and the reconciler deciding, because only one of them decides. + // Which components this addresses - and whether the app or component even + // exists here - is resolved from what the app is made of, in one place. + const { ids, actuated, appName: restartedName } = await setAppOperatorStopped(appname, false, { awaitPass: true, alsoRestart: true }); + + const outcome = actuated + ? await containersReachedRunning(ids) + : { settled: false, reason: 'no reconcile has run yet' }; + + if (!outcome.settled) { + const pending = messageHelper.createDataMessage( + `Application ${restartedName} will be restarted: ${outcome.reason}`, + ); + return res.json(pending); } - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; + const appResponse = messageHelper.createDataMessage(`Application ${restartedName} restarted`); + return res.json(appResponse); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage( @@ -469,7 +573,7 @@ async function appRestart(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -482,7 +586,6 @@ async function appRestart(req, res) { async function appKill(req, res) { try { let { appname } = req.params; - // eslint-disable-next-line global-require appname = appname || req.query.appname; if (!appname) { @@ -494,44 +597,36 @@ async function appKill(req, res) { // Use dynamic require to avoid circular dependency // eslint-disable-next-line global-require const verificationHelper = require('../verificationHelper'); - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + // This refuses the node operator, and a hard kill of + // someone else's app is not theirs to order. The owner and the flux team + // only, as for every other verb that decides whether the app runs. + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; + return res.json(errMessage); } - const isComponent = appname.includes('_'); // it is a component kill. Proceed with killing just component - let appRes; - - if (isComponent) { - // lock BEFORE the docker op (matching the whole-app path) - crash-safe direction - await setAppOperatorStopped(appname, null, true); - appRes = await dockerService.appDockerKill(appname); - } else { - // eslint-disable-next-line no-restricted-syntax - // Check if app exists before killing - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - // operator kill persists so the reconciler does not restart it - await setAppOperatorStopped(appname, appSpecs, true); - - if (appSpecs.version <= 3) { - appRes = await dockerService.appDockerKill(appname); - } else { - // For composed applications (version > 3), kill all components in reverse order - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose.reverse()) { - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerKill(`${appComponent.name}_${appSpecs.name}`); - } - appRes = `Application ${appSpecs.name} killed`; - } + // A kill is a stop that carries a signal, so it is the same desired state + // with a mode: the lock, plus force. The reconciler reads the mode where it + // stops the container, which keeps the choice of signal beside the decision + // to stop rather than in a handler racing it. + // Which components this addresses - and whether the app or component even + // exists here - is resolved from what the app is made of, in one place. + const { ids, actuated, appName: killedName } = await setAppOperatorStopped(appname, true, { awaitPass: true, force: true }); + + const outcome = actuated + ? await containersReachedStopped(ids) + : { settled: false, reason: 'no reconcile has run yet' }; + + if (!outcome.settled) { + const pending = messageHelper.createDataMessage( + `Application ${killedName} will be killed: ${outcome.reason}`, + ); + return res.json(pending); } - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; + const appResponse = messageHelper.createDataMessage(`Application ${killedName} killed`); + return res.json(appResponse); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage( @@ -539,74 +634,54 @@ async function appKill(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } /** - * Pause an application + * Pause and unpause were removed: docker reports a paused container as running, so the + * reconciler and the load balancer both treat it as healthy and keep routing to it, + * while nothing in FluxOS can see that it is frozen. The routes answer with an error + * rather than a success so a caller is not told the container stopped when it has not. * @param {object} req - Request object * @param {object} res - Response object * @returns {object} Response message */ -async function appPause(req, res) { +async function deprecatedPauseResponse(req, res) { try { let { appname } = req.params; appname = appname || req.query.appname; - // eslint-disable-next-line global-require - let { global } = req.params; - global = global || req.query.global || false; - global = serviceHelper.ensureBoolean(global); - - if (!appname) { - throw new Error('No Flux App specified'); - } - - const mainAppName = appname.split('_')[1] || appname; - - // Use dynamic require to avoid circular dependency - // eslint-disable-next-line global-require - const verificationHelper = require('../verificationHelper'); - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); - if (!authorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - - if (global) { - executeAppGlobalCommand(appname, 'apppause', req.headers.zelidauth); // do not wait - const appResponse = messageHelper.createSuccessMessage(`${appname} queried for global pause`); - return res ? res.json(appResponse) : appResponse; - } - const isComponent = appname.includes('_'); // it is a component pause - let appRes; - - if (isComponent) { - // eslint-disable-next-line no-restricted-syntax - appRes = await dockerService.appDockerPause(appname); - } else { - // Check if app exists before pausing - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); + if (appname) { + // Validated before anything is done with it. Express's default extended + // query parser turns ?appname=a&appname=b into an ARRAY and ?appname[x]=1 + // into an object, neither of which has .split - and this runs ahead of + // verifyPrivilege because the app name is what the privilege is scoped to, + // so it is reachable unauthenticated from the open internet. + // + // Unguarded, the rejection was dropped and the response never written: the + // socket stayed open with nothing left to answer it, since fluxServer sets + // a two-hour requestTimeout and node stops applying it once the request has + // been received. + if (typeof appname !== 'string') { + throw new Error('Invalid Flux App name specified'); } - - if (appSpecs.version <= 3) { - appRes = await dockerService.appDockerPause(appname); - } else { - // For composed applications (version > 3), pause all components - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose.reverse()) { - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerPause(`${appComponent.name}_${appSpecs.name}`); - } - appRes = `Application ${appSpecs.name} paused`; + const mainAppName = appname.split('_')[1] || appname; + // eslint-disable-next-line global-require + const verificationHelper = require('../verificationHelper'); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); + if (!authorized) { + const errMessage = messageHelper.errUnauthorizedMessage(); + return res.json(errMessage); } } - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; + const errorResponse = messageHelper.createErrorMessage( + 'Pausing applications is no longer supported. Use appstop to stop an application.', + 'Deprecated', + 410, + ); + return res.json(errorResponse); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage( @@ -614,120 +689,47 @@ async function appPause(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } /** - * Unpause an application + * Pause an application * @param {object} req - Request object * @param {object} res - Response object * @returns {object} Response message */ -async function appUnpause(req, res) { - try { - // eslint-disable-next-line global-require - let { appname } = req.params; - appname = appname || req.query.appname; - let { global } = req.params; - global = global || req.query.global || false; - global = serviceHelper.ensureBoolean(global); - - if (!appname) { - throw new Error('No Flux App specified'); - } - - const mainAppName = appname.split('_')[1] || appname; - - // Use dynamic require to avoid circular dependency - // eslint-disable-next-line global-require - const verificationHelper = require('../verificationHelper'); - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); - if (!authorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - - if (global) { - executeAppGlobalCommand(appname, 'appunpause', req.headers.zelidauth); // do not wait - const appResponse = messageHelper.createSuccessMessage(`${appname} queried for global unpase`); - return res ? res.json(appResponse) : appResponse; - } - - const isComponent = appname.includes('_'); // it is a component unpause - let appRes; - // eslint-disable-next-line no-restricted-syntax - - if (isComponent) { - appRes = await dockerService.appDockerUnpause(appname); - } else { - // Check if app exists before unpausing - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - - if (appSpecs.version <= 3) { - appRes = await dockerService.appDockerUnpause(appname); - } else { - // For composed applications (version > 3), unpause all components - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose) { - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerUnpause(`${appComponent.name}_${appSpecs.name}`); - } - appRes = `Application ${appSpecs.name} unpaused`; - } - } - - const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; - } catch (error) { - log.error(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - return res ? res.json(errorResponse) : errorResponse; - } +async function appPause(req, res) { + return deprecatedPauseResponse(req, res); } /** - * Docker restart app (internal function) - * @param {string} appname - Application name - * @returns {Promise} + * Unpause an application + * @param {object} req - Request object + * @param {object} res - Response object + * @returns {object} Response message */ -async function appDockerRestart(appname) { - try { - // mainAppName extracted for potential future use - // eslint-disable-next-line no-unused-vars - const mainAppName = appname.split('_')[1] || appname; - const isComponent = appname.includes('_'); // it is a component restart. Proceed with restarting just component - if (isComponent) { - await dockerService.appDockerRestart(appname); - // Note: startAppMonitoring would need to be injected or called separately - log.info(`Component ${appname} restarted successfully`); - } else { - // ask for restarting entire composed application - // This would need getApplicationSpecifications from registryManager - log.info(`Restarting entire application ${appname}`); - await dockerService.appDockerRestart(appname); - } - } catch (error) { - log.error(`Docker restart failed for ${appname}: ${error.message}`); - throw error; - } +async function appUnpause(req, res) { + return deprecatedPauseResponse(req, res); } +// Nothing in this module drives a container. Every route here records the +// operator's desired state and lets the reconciler actuate it. /** * To stop all non Flux running apps. Executes continuously at regular intervals. + * + * What is kept is everything FluxOS owns, not everything that is an app: the + * node runs short-lived containers of its own - a file operation is one - and + * those are unnamed, so docker gives them a random name that no prefix test can + * tell from a tenant's. Selecting on the ownership label instead means a long + * copy is not stopped out from under its caller by a sweep that runs every two + * hours. */ async function stopAllNonFluxRunningApps() { try { log.info('Running non Flux apps check...'); let apps = await dockerService.dockerListContainers(false); - apps = apps.filter((app) => (app.Names[0].slice(1, 4) !== 'zel' && app.Names[0].slice(1, 5) !== 'flux')); + apps = apps.filter((app) => !dockerService.isFluxOwnedContainer(app)); if (apps.length > 0) { log.info(`Found ${apps.length} apps to be stopped...`); // eslint-disable-next-line no-restricted-syntax @@ -757,12 +759,12 @@ async function stopAllNonFluxRunningApps() { module.exports = { executeAppGlobalCommand, + deliverGlobalCommand, appStart, appStop, appRestart, appKill, appPause, appUnpause, - appDockerRestart, stopAllNonFluxRunningApps, }; diff --git a/ZelBack/src/services/appManagement/appInspector.js b/ZelBack/src/services/appManagement/appInspector.js index d4f74f3bf0..161c83b32f 100644 --- a/ZelBack/src/services/appManagement/appInspector.js +++ b/ZelBack/src/services/appManagement/appInspector.js @@ -1,30 +1,19 @@ -const path = require('path'); +const config = require('config'); const serviceHelper = require('../serviceHelper'); const verificationHelper = require('../verificationHelper'); const messageHelper = require('../messageHelper'); const dockerService = require('../dockerService'); +const logCursor = require('../utils/logCursor'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); +const globalState = require('../utils/globalState'); const cpuBurstHelper = require('../utils/cpuBurstHelper'); const log = require('../../lib/log'); -// eslint-disable-next-line no-unused-vars -const { appConstants } = require('../utils/appConstants'); const { getContainerStorage } = require('../utils/appUtilities'); - -// eslint-disable-next-line import/no-extraneous-dependencies -const util = require('util'); -// eslint-disable-next-line import/no-extraneous-dependencies -const nodecmd = require('node-cmd'); - -// eslint-disable-next-line no-unused-vars -const fluxDirPath = process.env.FLUXOS_PATH || path.join(process.env.HOME, 'zelflux'); -// eslint-disable-next-line no-unused-vars -const appsFolderPath = process.env.FLUX_APPS_FOLDER || path.join(fluxDirPath, 'ZelApps'); +const { Privilege, authOf } = require('../utils/privileges'); const dosState = 0; const dosMessage = null; -const cmdAsync = util.promisify(nodecmd.run); -const dockerStatsStreamPromise = util.promisify(dockerService.dockerContainerStatsStream); /** * Get top processes running in an application container @@ -43,15 +32,22 @@ async function appTop(req, res) { const mainAppName = appname.split('_')[1] || appname; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + // Every endpoint in this module asks for appownerorfluxteam, which refuses + // the node operator. Hosting a container is a reason to know what it costs + // you, and /apps/appsresources answers that without authentication; it is + // not a reason to read what is inside it. What these return is the + // customer's: a process list carries argv, appInspect returns dockerode's + // object whole - Config.Env and all - and the log endpoints are whatever + // the application prints. + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; + return res.json(errMessage); } const appRes = await dockerService.appDockerTop(appname); const appResponse = messageHelper.createDataMessage(appRes); - return res ? res.json(appResponse) : appResponse; + return res.json(appResponse); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage( @@ -59,7 +55,7 @@ async function appTop(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -83,7 +79,7 @@ async function appLog(req, res) { const mainAppName = appname.split('_')[1] || appname; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (authorized === true) { let logs = await dockerService.dockerContainerLogs(appname, lines); logs = serviceHelper.dockerBufferToString(logs); @@ -104,55 +100,6 @@ async function appLog(req, res) { } } -/** - * Stream application logs - * @param {object} req - Request object - * @param {object} res - Response object - * @returns {Promise} - */ -async function appLogStream(req, res) { - try { - let { appname } = req.params; - appname = appname || req.query.appname; - - if (!appname) { - throw new Error('No Flux App specified'); - } - - const mainAppName = appname.split('_')[1] || appname; - - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); - if (authorized === true) { - res.setHeader('Content-Type', 'application/json'); - dockerService.dockerContainerLogsStream(appname, res, (error) => { - if (error) { - log.error(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - res.write(errorResponse); - res.end(); - } else { - res.end(); - } - }); - } else { - const errMessage = messageHelper.errUnauthorizedMessage(); - res.json(errMessage); - } - } catch (error) { - log.error(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - res.json(errorResponse); - } -} - /** * Poll application logs with filtering * @param {object} req - Request object @@ -167,6 +114,10 @@ async function appLogPolling(req, res) { lines = lines || req.query.lineCount || 'all'; let { since } = req.params; since = since || req.query.since || ''; + // A query parameter, not a path segment: the route has three optional + // segments and a fourth would not match, so a reader sending its position to + // a node that predates this would be answered with a 404 rather than logs. + const { cursor } = req.query; if (!appname) { throw new Error('No Flux App specified'); @@ -174,7 +125,7 @@ async function appLogPolling(req, res) { const mainAppName = appname.split('_')[1] || appname; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (authorized === true) { let parsedLineCount; if (lines === 'all') { @@ -183,25 +134,46 @@ async function appLogPolling(req, res) { parsedLineCount = parseInt(lines, 10) || 100; } - const logs = []; - await new Promise((resolve, reject) => { - dockerService.dockerContainerLogsPolling(appname, parsedLineCount, since, (err, logLine) => { - if (err) { - reject(err); - } else if (logLine === 'Stream ended') { - resolve(); - } else if (logLine) { - logs.push(logLine); - } - }); + // A reader that sends a position gets everything after it. One that does + // not gets the most recent lines, which is what every reader written + // before positions existed asks for and still receives. + const position = logCursor.decode(cursor); + // The `since` box in a log viewer is a filter a person typed, not a claim + // to hold lines. It keeps its line count and can never be answered with + // rolledOver, which is about a position that no longer exists. + const sinceMs = position ? null : Date.parse(since); + + const result = await dockerService.dockerContainerLogsPolling(appname, { + position, + since: Number.isFinite(sinceMs) ? sinceMs : null, + lineCount: parsedLineCount, }); res.json({ - logs, + logs: result.lines, lineCount: parsedLineCount, - logCount: logs.length, + logCount: result.lines.length, sinceTimestamp: since, - truncated: parsedLineCount === 'all' ? false : logs.length >= parsedLineCount, + // The position reached. A reader hands it back and is answered with what + // it has not seen; it never has to read it. + cursor: result.position ? logCursor.encode(result.position) : cursor || null, + // The line this reader asked from no longer exists - docker discarded the + // file holding it. What sat between it and the oldest line below is gone + // and cannot be fetched by anyone. + rolledOver: result.rolledOver, + // The line this reader asked from is further back than one read reaches, + // so it has been moved to the end of the log and what sat between was + // not delivered. Those lines still exist, unlike rolledOver's - reaching + // them costs a read of the whole retained log, which is 349ms of blocked + // event loop per poll against 1ms bounded, and the position that asks + // for it is a value the caller writes. A reader that polls often enough + // never sees this. + skipped: result.skipped, + // The log holds more than the line limit asked for. Only a caller that + // sent one is told, and it is the same answer this field has always + // given. A positioned reader is never told it: nothing walks backwards, + // so the only thing it could do about it is a poll returning nothing. + truncated: result.truncated, status: 'success', }); } else { @@ -236,7 +208,7 @@ async function appInspect(req, res) { const mainAppName = appname.split('_')[1] || appname; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (authorized === true) { const response = await dockerService.dockerContainerInspect(appname); const appResponse = messageHelper.createDataMessage(response); @@ -273,14 +245,9 @@ async function appStats(req, res) { const mainAppName = appname.split('_')[1] || appname; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (authorized === true) { - const response = await dockerService.dockerContainerStats(appname); - const containerStorageInfo = await getContainerStorage(appname); - response.disk_stats = containerStorageInfo; - const inspect = await dockerService.dockerContainerInspect(appname); - response.nanoCpus = inspect.HostConfig.NanoCpus; - const appResponse = messageHelper.createDataMessage(response); + const appResponse = messageHelper.createDataMessage(await latestStats(appname)); res.json(appResponse); } else { const errMessage = messageHelper.errUnauthorizedMessage(); @@ -298,74 +265,250 @@ async function appStats(req, res) { } /** - * Get application monitoring data - * @param {object} req - Request object - * @param {object} res - Response object - * @param {object} appsMonitored - Apps monitoring data - * @returns {Promise} + * Milliseconds on the monotonic clock. Every elapsed-time decision here — how long + * a sample is kept, which samples a CPU decision may count — reads this rather than + * the wall clock, so an NTP step cannot expire the store early or hand a decision + * samples it already counted. Samples carry the wall-clock time too: that is what + * the charts plot and what a requested range means. + * @returns {number} Milliseconds since an arbitrary fixed origin */ -async function appMonitor(req, res, appsMonitored) { - try { - let { appname, range } = req.params; - appname = appname || req.query.appname; - range = range || req.query.range || null; +function monotonicMs() { + return Number(process.hrtime.bigint() / 1000000n); +} - if (!appname) { - throw new Error('No Flux App specified'); +/** + * Reduce a docker stats reading to the values the monitoring consumers read. + * + * Docker returns several kilobytes a sample — per-core usage arrays repeated for + * the previous reading, the whole kernel memory counter set, one blkio entry per + * device in the stack — and between them the charts and the CPU throttler read + * the fourteen values below. Keeping the extract is what makes a week of samples + * affordable to hold: measured over ten components at a week of samples each, + * 469MB of full readings against 36MB of these. + * + * A value belongs here if a consumer cannot do its job without it, not if it is + * cheap to carry. memoryCache is the case that proves it — one number that costs + * 0.9% of what dropping the other twenty-seven saved, and without which nobody + * can report container memory the way docker does. + * @param {object} stats - A docker stats reading, with disk_stats and nanoCpus attached + * @returns {object} The values worth keeping + */ +function extractSample(stats) { + const blkio = stats.blkio_stats?.io_service_bytes_recursive; + // a container's traffic passes through its loop device, device-mapper and the + // physical disk, so every entry is a partial view; docker stats sums them + const sumBlkio = (op) => (blkio + ? blkio + .filter((entry) => entry.op?.toLowerCase() === op) + .reduce((total, entry) => total + (entry.value || 0), 0) + : null); + + return { + cpuTotal: stats.cpu_stats?.cpu_usage?.total_usage ?? 0, + cpuTotalBefore: stats.precpu_stats?.cpu_usage?.total_usage ?? 0, + cpuSystem: stats.cpu_stats?.system_cpu_usage ?? 0, + cpuSystemBefore: stats.precpu_stats?.system_cpu_usage ?? 0, + onlineCpus: stats.cpu_stats?.online_cpus ?? 0, + nanoCpus: stats.nanoCpus ?? null, + memoryUsage: stats.memory_stats?.usage ?? null, + memoryLimit: stats.memory_stats?.limit ?? null, + // Docker's `usage` counts page cache - file data the kernel is holding only + // because something read it, and drops the moment anything else wants the + // space. `docker stats` subtracts it, so every consumer that reports a + // container's memory subtracts it too, and without this figure none of them + // can: they subtract zero and report a number that climbs with disk reads. + // cgroup v2 names it inactive_file and v1 names it cache; they are not the + // same counter (v1's is the whole page cache, v2's the reclaimable part) but + // docker's own CLI reads them from the same slot, so this does too. Null + // where neither is present, which leaves the consumer exactly where it was. + memoryCache: stats.memory_stats?.stats?.inactive_file + ?? stats.memory_stats?.stats?.cache ?? null, + ioRead: sumBlkio('read'), + ioWrite: sumBlkio('write'), + // EVERY interface, not just eth0. appNetworkLinker connects a container to + // additional networks when a spec declares networkWith, and reading eth0 + // alone made that traffic vanish from a reading the base returned in full. + // Still a narrowing - the per-interface packet, error and dropped counters + // are dropped, as everywhere else here - and `networks.eth0` remains a key, + // so a consumer reading it is untouched. + networks: Object.fromEntries( + Object.entries(stats.networks ?? {}).map(([iface, counters]) => [iface, { + rx_bytes: counters?.rx_bytes ?? null, + tx_bytes: counters?.tx_bytes ?? null, + }]), + ), + disk: stats.disk_stats ?? null, + }; +} + +/** + * Put an extracted sample back into the shape callers parse. + * @param {object} sample - An extracted sample + * @returns {object} The docker stats shape the monitoring endpoints return + */ +function expandSample(sample) { + const io = sample.ioRead === null && sample.ioWrite === null + ? null + : [{ op: 'read', value: sample.ioRead }, { op: 'write', value: sample.ioWrite }]; + + return { + cpu_stats: { + cpu_usage: { total_usage: sample.cpuTotal }, + system_cpu_usage: sample.cpuSystem, + online_cpus: sample.onlineCpus, + }, + precpu_stats: { + cpu_usage: { total_usage: sample.cpuTotalBefore }, + system_cpu_usage: sample.cpuSystemBefore, + }, + // Reported under the cgroup v2 name whichever key it was read from: a + // consumer's `inactive_file ?? cache` chain then settles on the first, and + // the value is the same either way. + memory_stats: { + usage: sample.memoryUsage, + limit: sample.memoryLimit, + stats: { inactive_file: sample.memoryCache }, + }, + blkio_stats: { io_service_bytes_recursive: io }, + networks: sample.networks ?? {}, + nanoCpus: sample.nanoCpus, + disk_stats: sample.disk, + }; +} + +/** + * How long a reading stands in for the present. The chart polls every five seconds + * per viewer, and every viewer of the same app wants the same reading, so this + * bounds collection by time rather than by request count: one viewer costs what it + * costs today, ten cost the same. + */ +const statsFreshnessMs = 5 * 1000; + +// Readings in flight, keyed by app name. The freshness window above only helps a +// caller who arrives AFTER one has landed; callers who arrive together all find +// nothing fresh and all go and take their own. That is not a spare API call - a +// reading walks the app's whole volume with du to size it, so ten viewers opening +// the same server at once meant ten simultaneous walks of one disk, in precisely +// the situation where a server is popular enough for several people to be looking. +// +// Same shape as the enterprise-spec decryption single-flight in appQueryService: +// the first caller records its promise, everyone arriving mid-flight awaits that +// one, and the entry is dropped once it settles - including on failure, so a +// rejected reading is never handed to the next caller. +const statsInFlight = new Map(); + +/** + * The most recent reading for an app, taking one only if what is held has aged out. + * + * The node already samples every running container a minute; without this the live + * view collected the same three docker readings again on every poll and threw them + * away, so the copy that was kept was the one nothing read. + * @param {string} appname - Application name, optionally component-qualified + * @returns {Promise} The reading, in the docker stats shape callers parse + */ +async function latestStats(appname) { + const monitored = globalState.appsMonitored[appname]; + const stored = monitored && monitored.statsStore + ? monitored.statsStore[monitored.statsStore.length - 1] + : null; + const held = [monitored && monitored.latest, stored] + .filter(Boolean) + .sort((a, b) => b.elapsed - a.elapsed)[0]; + + if (held && monotonicMs() - held.elapsed < statsFreshnessMs) { + return expandSample(held); + } + + const inFlight = statsInFlight.get(appname); + if (inFlight) return expandSample(await inFlight); + + const reading = (async () => { + const stats = await dockerService.dockerContainerStats(appname); + stats.disk_stats = await getContainerStorage(appname); + const inspect = await dockerService.dockerContainerInspect(appname); + stats.nanoCpus = inspect.HostConfig.NanoCpus; + + const sample = { timestamp: Date.now(), elapsed: monotonicMs(), ...extractSample(stats) }; + if (monitored) { + monitored.latest = sample; } + return sample; + })(); - if (range !== null) { - range = parseInt(range, 10); - if (!Number.isInteger(range) || range <= 0) { - throw new Error('Invalid range value. It must be a positive integer or null.'); - } + statsInFlight.set(appname, reading); + try { + return expandSample(await reading); + } finally { + statsInFlight.delete(appname); + } +} + +/** + * Get the collected monitoring statistics for an application + * @param {string} appname - Application name, optionally component-qualified + * @param {number|string} [range] - Window in milliseconds to report on, or null for everything + * @returns {Array} Collected statistics + */ +function appMonitor(appname, range = null) { + if (!appname) { + throw new Error('No Flux App specified'); + } + + let window = range; + if (window !== null) { + window = parseInt(window, 10); + if (!Number.isInteger(window) || window <= 0) { + throw new Error('Invalid range value. It must be a positive integer or null.'); } + } - const mainAppName = appname.split('_')[1] || appname; + const monitored = globalState.appsMonitored[appname]; + if (!monitored) { + throw new Error('No data available'); + } - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); - if (authorized === true) { - if (appsMonitored[appname]) { - let appStatsMonitoring = appsMonitored[appname].statsStore; - if (range) { - const now = Date.now(); - const cutoffTimestamp = now - range; - const hoursInMs = 24 * 60 * 60 * 1000; - appStatsMonitoring = appStatsMonitoring.filter((stats) => stats.timestamp >= cutoffTimestamp); - if (range > hoursInMs) { - appStatsMonitoring = appStatsMonitoring.filter((_, index, array) => index % 20 === 0 || index === array.length - 1); - } + let appStatsMonitoring = monitored.statsStore; + if (window) { + const cutoffTimestamp = Date.now() - window; + const dayInMs = 24 * 60 * 60 * 1000; + appStatsMonitoring = appStatsMonitoring.filter((stats) => stats.timestamp >= cutoffTimestamp); + if (window > dayInMs) { + // Past a day the series is thinned to one sample an hour: a week of + // minute-resolution samples is neither sendable nor plottable. Thinning by + // timestamp rather than by position keeps the spacing an hour whatever the + // sampler's cadence is. + const hourInMs = 60 * 60 * 1000; + const thinned = []; + let lastKept = null; + appStatsMonitoring.forEach((stats) => { + if (lastKept === null || stats.timestamp - lastKept >= hourInMs) { + thinned.push(stats); + lastKept = stats.timestamp; } - const appResponse = messageHelper.createDataMessage(appStatsMonitoring); - res.json(appResponse); - } else { - throw new Error('No data available'); + }); + const newest = appStatsMonitoring[appStatsMonitoring.length - 1]; + if (newest && thinned[thinned.length - 1] !== newest) { + thinned.push(newest); } - } else { - const errMessage = messageHelper.errUnauthorizedMessage(); - res.json(errMessage); + appStatsMonitoring = thinned; } - } catch (error) { - log.error(error); - const errMessage = messageHelper.createErrorMessage( - error.message, - error.name, - error.code, - ); - res.json(errMessage); } + return appStatsMonitoring.map((stats) => ({ + timestamp: stats.timestamp, + data: expandSample(stats), + })); } /** - * Stream application monitoring data + * Get application monitoring data * @param {object} req - Request object * @param {object} res - Response object * @returns {Promise} */ -async function appMonitorStream(req, res) { +async function appMonitorAPI(req, res) { try { - let { appname } = req.params; - appname = appname || req.query.appname; + const appname = req.params.appname || req.query.appname; + const range = req.params.range || req.query.range || null; if (!appname) { throw new Error('No Flux App specified'); @@ -373,10 +516,10 @@ async function appMonitorStream(req, res) { const mainAppName = appname.split('_')[1] || appname; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (authorized === true) { - await dockerStatsStreamPromise(appname, req, res); - res.end(); + const appResponse = messageHelper.createDataMessage(appMonitor(appname, range)); + res.json(appResponse); } else { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -392,148 +535,161 @@ async function appMonitorStream(req, res) { } } -/** - * Get application folder size - * @param {string} appName - Application name - * @returns {Promise} Folder size in bytes - */ -async function getAppFolderSize(appName) { - try { - const appsDirPath = process.env.FLUX_APPS_FOLDER || path.join(fluxDirPath, 'ZelApps'); - const directoryPath = path.join(appsDirPath, appName); - const exec = `sudo du -s --block-size=1 ${directoryPath}`; - const cmdres = await cmdAsync(exec); - const size = serviceHelper.ensureString(cmdres).split('\t')[0] || 0; - return size; - } catch (error) { - log.error(error); - return 0; - } -} - /** * Start monitoring an application * @param {string} appName - Application name - * @param {object} [appsMonitored] - Apps monitoring data reference (optional, will get from appsService if not provided) * @returns {void} */ -function startAppMonitoring(appName, appsMonitored) { +function startAppMonitoring(appName) { if (!appName) { throw new Error('No App specified'); } - // eslint-disable-next-line global-require - // Get appsMonitored from globalState if not provided (to avoid circular dependency) - if (!appsMonitored) { - // eslint-disable-next-line global-require - const globalState = require('../utils/globalState'); - // eslint-disable-next-line prefer-destructuring, no-param-reassign - appsMonitored = globalState.appsMonitored; - } - - // Safety check: if appsMonitored is still undefined, throw a more descriptive error - if (!appsMonitored) { - // eslint-disable-next-line no-param-reassign - throw new Error('Failed to initialize app monitoring: appsMonitored object is undefined'); - // eslint-disable-next-line no-param-reassign - } + const { appsMonitored } = globalState; - // eslint-disable-next-line no-param-reassign log.info('Initialize Monitoring...'); // Clear previous interval for this app to prevent multiple intervals if (appsMonitored[appName] && appsMonitored[appName].oneMinuteInterval) { clearInterval(appsMonitored[appName].oneMinuteInterval); } - // eslint-disable-next-line no-param-reassign - appsMonitored[appName] = {}; // Initialize the app's monitoring object - if (!appsMonitored[appName].statsStore) { - // eslint-disable-next-line no-param-reassign - appsMonitored[appName].statsStore = []; - } - if (!appsMonitored[appName].lastHourstatsStore) { - // eslint-disable-next-line no-param-reassign - appsMonitored[appName].lastHourstatsStore = []; + appsMonitored[appName] = { + statsStore: [], + // the throttler reads everything past this, then moves it forward; a + // watermark rather than a wipe, so the series stays whole for the charts + lastCpuDecisionAt: 0, + nanoCpus: null, + run: 0, + }; + // Validated, because setInterval does not. A missing key coerces to NaN and + // anything below 1 is treated as ONE MILLISECOND - so the sampler would ask + // docker for stats on every monitored component about a thousand times a + // second instead of once a minute. That has happened to this exact key + // before; the test config's own comment still names the incident. ?? alone + // covers only the missing key, so a value that is present but wrong - zero, + // negative, a string - is refused here too, loudly. + const configuredInterval = config.fluxapps.statsSampleIntervalMs; + const sampleIntervalMs = Number.isFinite(configuredInterval) && configuredInterval >= 1000 + ? configuredInterval + : 60 * 1000; + if (sampleIntervalMs !== configuredInterval) { + log.warn(`statsSampleIntervalMs is ${JSON.stringify(configuredInterval)}, not a number of at least 1000; sampling every 60s instead`); } - // eslint-disable-next-line no-param-reassign - appsMonitored[appName].run = 0; - // eslint-disable-next-line no-param-reassign appsMonitored[appName].oneMinuteInterval = setInterval(async () => { try { if (!appsMonitored[appName]) { log.error(`Monitoring of ${appName} already stopped`); return; - // eslint-disable-next-line no-param-reassign } const dockerContainer = await dockerService.getDockerContainerOnly(appName); if (!dockerContainer) { log.error(`Monitoring of ${appName} not possible. App does not exist. Forcing stopping of monitoring`); // eslint-disable-next-line no-use-before-define - stopAppMonitoring(appName, true, appsMonitored); + stopAppMonitoring(appName, true); + return; + } + // a container that is created, exited or dead reports no usage; sampling it + // fills the store with empty readings and gives the throttler nothing to + // read. The listing above already carries the state, so this costs nothing. + if (dockerContainer.State !== 'running') { return; } - // eslint-disable-next-line no-param-reassign appsMonitored[appName].run += 1; const statsNow = await dockerService.dockerContainerStats(appName); - const containerStorageInfo = await getContainerStorage(appName); - // eslint-disable-next-line no-param-reassign - statsNow.disk_stats = containerStorageInfo; - const now = Date.now(); - if (appsMonitored[appName].run % 3 === 0) { + statsNow.disk_stats = await getContainerStorage(appName); + // the allocation only moves when the throttler moves it, so it is read on + // the same cadence as before and carried onto the samples in between + if (appsMonitored[appName].run % 3 === 1) { const inspect = await dockerService.dockerContainerInspect(appName); - // eslint-disable-next-line no-param-reassign - statsNow.nanoCpus = inspect.HostConfig.NanoCpus; - appsMonitored[appName].statsStore.push({ timestamp: now, data: statsNow }); - const statsStoreSizeInBytes = new TextEncoder().encode(JSON.stringify(appsMonitored[appName].statsStore)).length; - const estimatedSizeInMB = statsStoreSizeInBytes / (1024 * 1024); - log.info(`Size of stats for ${appName}: ${estimatedSizeInMB.toFixed(2)} MB`); - // eslint-disable-next-line no-param-reassign - appsMonitored[appName].statsStore = appsMonitored[appName].statsStore.filter( - (stat) => now - stat.timestamp <= 7 * 24 * 60 * 60 * 1000, - ); + appsMonitored[appName].nanoCpus = inspect?.HostConfig?.NanoCpus ?? null; + } + statsNow.nanoCpus = appsMonitored[appName].nanoCpus; + + // A PARTIAL disk reading is a floor, not a total: some mount could not be + // sized, so the figure is lower than the truth by an unknown amount. + // getContainerStorage already refuses to CACHE one for even sixty seconds - + // "a short one is recomputed next tick, which is where it gets the chance to + // come good" - and this store keeps a sample for SEVEN DAYS and serves it to + // every chart request. The same reading cannot be too untrustworthy to hold + // for a minute and trustworthy enough to hold for a week. + // + // Carried forward rather than dropped. Storing null would be worse than the + // dip it fixes: the dashboards read `disk_stats.used || 0`, so an absent + // figure charts as ZERO - a drop to the floor instead of a partial one. The + // last measured value is at most one sample interval stale, which is inside + // the noise of a disk chart. + // + // The live path is unaffected and stays honest: /apps/appstats takes a fresh + // reading rather than serving this store, so a caller asking what the usage + // is RIGHT NOW still gets status 'partial' and can act on it. + const lastKnownDisk = appsMonitored[appName].statsStore.length + ? appsMonitored[appName].statsStore[appsMonitored[appName].statsStore.length - 1].disk + : null; + // Any reading the sizing step did not stand behind, not just a partial one. + // Its catch returns { used: 0, status: 'error' } - a dockerd blip mid-tick + // is enough - and storing that charts a drop to the FLOOR, which is worse + // than the partial dip this guard was written for. + if (statsNow.disk_stats?.status && statsNow.disk_stats.status !== 'success' && lastKnownDisk) { + // Relabelled, because the carried copy arrives saying 'success'. Under a + // PERMANENT failure - a volume unmounted, du refused for good - the + // chart would otherwise show a flat line labelled success indefinitely, + // with nothing in the sample saying it was carried. The number is kept + // (the dashboards only blank their figures on 'error'), the label stops + // claiming the reading is fresh. + statsNow.disk_stats = { ...lastKnownDisk, status: 'stale' }; } - appsMonitored[appName].lastHourstatsStore.push({ timestamp: now, data: statsNow }); - // eslint-disable-next-line no-param-reassign - appsMonitored[appName].lastHourstatsStore = appsMonitored[appName].lastHourstatsStore.filter( - (stat) => now - stat.timestamp <= 60 * 60 * 1000, + + const elapsed = monotonicMs(); + appsMonitored[appName].statsStore.push({ + timestamp: Date.now(), + elapsed, + ...extractSample(statsNow), + }); + appsMonitored[appName].statsStore = appsMonitored[appName].statsStore.filter( + (stat) => elapsed - stat.elapsed <= 7 * 24 * 60 * 60 * 1000, ); } catch (error) { log.error(error); } - }, 1 * 60 * 1000); + }, sampleIntervalMs); } -// eslint-disable-next-line global-require /** * Stop monitoring an application * @param {string} appName - Application name * @param {boolean} deleteData - Whether to delete monitoring data - * @param {object} [appsMonitored] - Apps monitoring data reference (optional, will get from appsService if not provided) * @returns {void} */ -function stopAppMonitoring(appName, deleteData, appsMonitored) { - // Get appsMonitored from globalState if not provided (to avoid circular dependency) - if (!appsMonitored) { - // eslint-disable-next-line global-require - const globalState = require('../utils/globalState'); - // eslint-disable-next-line prefer-destructuring, no-param-reassign - appsMonitored = globalState.appsMonitored; - } - - // Safety check: if appsMonitored is still undefined, log warning and return early - if (!appsMonitored) { - log.warn(`Cannot stop monitoring for ${appName}: appsMonitored object is undefined`); - return; - } +function stopAppMonitoring(appName, deleteData) { + const { appsMonitored } = globalState; if (appsMonitored[appName]) { clearInterval(appsMonitored[appName].oneMinuteInterval); + // Dropped, not just cleared: a stale handle is indistinguishable from a live + // one, and ensureAppMonitoring has to be able to tell them apart. + appsMonitored[appName].oneMinuteInterval = null; if (deleteData) { - // eslint-disable-next-line no-param-reassign delete appsMonitored[appName]; } } } +/** + * Starts monitoring only if it is not already running. + * + * startAppMonitoring resets statsStore, so calling it for a container that is + * already monitored discards the series the charts read. The reconciler reaches + * a running container on every pass, so it needs the question asked rather than + * the reset repeated. + * + * @param {string} appName - Application name + * @returns {void} + */ +function ensureAppMonitoring(appName) { + const { appsMonitored } = globalState; + if (appsMonitored[appName] && appsMonitored[appName].oneMinuteInterval) return; + startAppMonitoring(appName); +} + /** * Execute command in application container * @param {object} req - Request object @@ -559,7 +715,10 @@ async function appExec(req, res) { const mainAppName = processedBody.appname.split('_')[1] || processedBody.appname; - const authorized = await verificationHelper.verifyPrivilege('appowner', req, mainAppName); + // The container terminal's privilege: it reaches this component with more - + // an interactive session on a caller-named user - so a narrower gate here + // refuses nothing. + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (authorized === true) { let cmd = processedBody.cmd || []; let env = processedBody.env || []; @@ -581,7 +740,11 @@ async function appExec(req, res) { error.name, error.code, ); - res.write(errorResponse); + // createErrorMessage returns an OBJECT and res.write takes only a + // string or a buffer, so the path that exists to report the failure + // threw while reporting it - from inside the same callback, which is + // an exit rather than a 500. + res.write(serviceHelper.ensureString(errorResponse)); res.end(); } else { res.end(); @@ -620,7 +783,7 @@ async function appChanges(req, res) { const mainAppName = appname.split('_')[1] || appname; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: mainAppName }); if (authorized === true) { const response = await dockerService.dockerContainerChanges(appname); const appResponse = messageHelper.createDataMessage(response); @@ -641,24 +804,54 @@ async function appChanges(req, res) { } /** - * List Docker images used by apps - * @param {object} req - Request object - * @param {object} res - Response object - * @returns {Promise} List of Docker images + * Every docker image on this node. + * @returns {Promise} Message carrying the image list */ -async function listAppsImages(req, res) { +async function listAppsImages() { try { const apps = await dockerService.dockerListImages(); - const appsResponse = messageHelper.createDataMessage(apps); - return res ? res.json(appsResponse) : appsResponse; + return messageHelper.createDataMessage(apps); } catch (error) { log.error(error); - const errorResponse = messageHelper.createErrorMessage( + return messageHelper.createErrorMessage( error.message || error, error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + } +} + +/** + * GET /apps/listappsimages - the image list, for the flux team. + * + * An image entry cannot be attributed to the application that uses it: nothing + * in a row names one, and the field that would - Containers - reads -1, because + * docker does not compute it for a plain list. So the row is the smallest thing + * this can answer with, and the whole list is either given or it is not. + * + * The route carries no cache. apicache answers from its store before the + * handler runs and keys on the request URL alone, so a privilege checked here + * would be checked for the first caller and no one after them. + * + * @param {object} req - Request object + * @param {object} res - Response object + * @returns {Promise} + */ +async function listAppsImagesApi(req, res) { + try { + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); + if (!authorized) { + res.json(messageHelper.errUnauthorizedMessage()); + return; + } + res.json(await listAppsImages()); + } catch (error) { + log.error(error); + res.json(messageHelper.createErrorMessage( + error.message || error, + error.name, + error.code, + )); } } @@ -677,6 +870,36 @@ function getAppsDOSState(req, res) { return res ? res.json(response) : response; } +/** + * The samples a CPU decision is entitled to: everything recorded since the last + * decision, so no sample is counted twice. + * + * The hour bound matters. A decision is skipped whenever docker cannot inspect the + * container or too few samples have arrived, and the watermark does not move on a + * skip — so without it, a long run of skips would eventually hand a decision days + * of samples and the 80% rule would be measured against a different population + * than it was designed for. + * @param {object} monitored - The monitoring entry for one app or component + * @returns {Array} Samples this decision may count + */ +function cpuDecisionWindow(monitored) { + if (!monitored || !monitored.statsStore) return []; + const floor = Math.max(monitored.lastCpuDecisionAt || 0, monotonicMs() - 60 * 60 * 1000); + return monitored.statsStore.filter((stat) => stat.elapsed > floor); +} + +/** + * One sample's CPU use as a percentage of what the spec asked for. + * @param {object} sample - An extracted sample + * @param {number} specifiedCpu - The cpu the app or component specified + * @returns {number} Percentage of the specified cpu in use + */ +function sampleCpuLoad(sample, specifiedCpu) { + const cpuUsage = sample.cpuTotal - sample.cpuTotalBefore; + const systemCpuUsage = sample.cpuSystem - sample.cpuSystemBefore; + return ((cpuUsage / systemCpuUsage) * sample.onlineCpus * 100) / specifiedCpu || 0; +} + /** * Check if applications are throttling CPU and adjust CPU limits * @param {object} appsMonitored - Applications monitoring data @@ -691,13 +914,19 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { throw new Error('Failed to get installed Apps'); } // Decrypt enterprise apps (version 8 with encrypted content) - installedAppsRes.data = await decryptEnterpriseApps(installedAppsRes.data); + ({ inPlace: installedAppsRes.data } = await decryptEnterpriseApps(installedAppsRes.data)); const appsInstalled = installedAppsRes.data; let stats; // eslint-disable-next-line no-restricted-syntax for (const app of appsInstalled) { if (app.version <= 3) { - stats = appsMonitored[app.name]?.lastHourstatsStore; + // Stamped from BEFORE the window was taken, not from after the awaits + // below. The watermark marks where this decision stopped looking; a + // sample arriving during the inspect is not in the snapshot, so dating + // the watermark later would put it behind the next window too and no + // decision would ever count it. + const decisionAt = monotonicMs(); + stats = cpuDecisionWindow(appsMonitored[app.name]); // eslint-disable-next-line no-await-in-loop const inspect = await dockerService.dockerContainerInspect(app.name); // Skip CPU throttling for containers with CFS burst actively applied. @@ -709,7 +938,7 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { log.info(`checkApplicationsCpuUSage ${app.name} burst-active, skipping CPU throttling`); if (appsMonitored[app.name]) { // eslint-disable-next-line no-param-reassign - appsMonitored[app.name].lastHourstatsStore = []; + appsMonitored[app.name].lastCpuDecisionAt = decisionAt; } // eslint-disable-next-line no-continue continue; @@ -719,22 +948,27 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { let cpuThrottlingRuns = 0; let cpuThrottling = false; const cpuPercentage = nanoCpus / app.cpu / 1e9; - // eslint-disable-next-line no-restricted-syntax - for (const stat of stats) { - const cpuUsage = stat.data.cpu_stats.cpu_usage.total_usage - stat.data.precpu_stats.cpu_usage.total_usage; - const systemCpuUsage = stat.data.cpu_stats.system_cpu_usage - stat.data.precpu_stats.system_cpu_usage; - const cpu = ((cpuUsage / systemCpuUsage) * stat.data.cpu_stats.online_cpus * 100) / app.cpu || 0; - const realCpu = cpu / cpuPercentage; + stats.forEach((stat) => { + const realCpu = sampleCpuLoad(stat, app.cpu) / cpuPercentage; if (realCpu >= 92) { cpuThrottlingRuns += 1; } - } + }); if (cpuThrottlingRuns >= stats.length * 0.8) { // cpu was high on 80% of the checks cpuThrottling = true; } - // eslint-disable-next-line no-param-reassign - appsMonitored[app.name].lastHourstatsStore = []; + // Guarded like the burst-skip write above it. The entry can be gone by + // now: the window was snapshotted before two awaits, and an uninstall, + // a reconciler recreate, or the sampler noticing the container vanish + // all call stopAppMonitoring in between. Writing to undefined throws to + // the function-level catch, which abandons the pass - so every app + // ordered after this one goes un-inspected until the next attempt, + // fifteen minutes later. + if (appsMonitored[app.name]) { + // eslint-disable-next-line no-param-reassign + appsMonitored[app.name].lastCpuDecisionAt = decisionAt; + } log.info(`checkApplicationsCpuUSage ${app.name} cpu high load: ${cpuThrottling}`); log.info(`checkApplicationsCpuUSage ${cpuPercentage}`); if (cpuThrottling && app.cpu > 1) { @@ -770,7 +1004,10 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { // eslint-disable-next-line no-restricted-syntax for (const appComponent of app.compose) { const compName = `${appComponent.name}_${app.name}`; - stats = appsMonitored[compName]?.lastHourstatsStore; + // As above: the watermark dates the snapshot, not the decision, so a + // sample landing during the awaits is not lost between the two. + const decisionAt = monotonicMs(); + stats = cpuDecisionWindow(appsMonitored[compName]); // eslint-disable-next-line no-await-in-loop const inspect = await dockerService.dockerContainerInspect(compName); // Skip CPU throttling for components with CFS burst actively applied. @@ -781,7 +1018,7 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { log.info(`checkApplicationsCpuUSage ${compName} burst-active, skipping CPU throttling`); if (appsMonitored[compName]) { // eslint-disable-next-line no-param-reassign - appsMonitored[compName].lastHourstatsStore = []; + appsMonitored[compName].lastCpuDecisionAt = decisionAt; } // eslint-disable-next-line no-continue continue; @@ -791,22 +1028,21 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { let cpuThrottlingRuns = 0; let cpuThrottling = false; const cpuPercentage = nanoCpus / appComponent.cpu / 1e9; - // eslint-disable-next-line no-restricted-syntax - for (const stat of stats) { - const cpuUsage = stat.data.cpu_stats.cpu_usage.total_usage - stat.data.precpu_stats.cpu_usage.total_usage; - const systemCpuUsage = stat.data.cpu_stats.system_cpu_usage - stat.data.precpu_stats.system_cpu_usage; - const cpu = ((cpuUsage / systemCpuUsage) * 100 * stat.data.cpu_stats.online_cpus) / appComponent.cpu || 0; - const realCpu = cpu / cpuPercentage; + stats.forEach((stat) => { + const realCpu = sampleCpuLoad(stat, appComponent.cpu) / cpuPercentage; if (realCpu >= 92) { cpuThrottlingRuns += 1; } - } + }); if (cpuThrottlingRuns >= stats.length * 0.8) { // cpu was high on 80% of the checks cpuThrottling = true; } - // eslint-disable-next-line no-param-reassign - appsMonitored[`${appComponent.name}_${app.name}`].lastHourstatsStore = []; + // Guarded like the burst-skip write above it - see the v3 path. + if (appsMonitored[compName]) { + // eslint-disable-next-line no-param-reassign + appsMonitored[compName].lastCpuDecisionAt = decisionAt; + } log.info(`checkApplicationsCpuUSage ${appComponent.name}_${app.name} cpu high load: ${cpuThrottling}`); log.info(`checkApplicationsCpuUSage ${cpuPercentage}`); if (cpuThrottling && appComponent.cpu > 1) { @@ -862,7 +1098,7 @@ async function monitorSharedDBApps(installedApps, removeAppLocally, globalState) // get list of all installed apps const appsInstalled = await installedApps(); // Decrypt enterprise apps (version 8 with encrypted content) - appsInstalled.data = await decryptEnterpriseApps(appsInstalled.data); + ({ inPlace: appsInstalled.data } = await decryptEnterpriseApps(appsInstalled.data)); // eslint-disable-next-line no-restricted-syntax for (const installedApp of appsInstalled.data.filter((app) => app.version > 3)) { @@ -896,7 +1132,6 @@ async function monitorSharedDBApps(installedApps, removeAppLocally, globalState) } finally { await serviceHelper.delay(5 * 60 * 1000); monitorSharedDBApps(installedApps, removeAppLocally, globalState); - // eslint-disable-next-line global-require } } @@ -910,15 +1145,13 @@ async function monitorSharedDBApps(installedApps, removeAppLocally, globalState) */ async function checkStorageSpaceForApps(installedApps, removeAppLocally, softRedeploy, appsStorageViolations) { try { - // eslint-disable-next-line global-require - const config = require('config'); // get list of locally installed apps. const installedAppsRes = await installedApps(); if (installedAppsRes.status !== 'success') { throw new Error('Failed to get installed Apps'); } // Decrypt enterprise apps (version 8 with encrypted content) - installedAppsRes.data = await decryptEnterpriseApps(installedAppsRes.data); + ({ inPlace: installedAppsRes.data } = await decryptEnterpriseApps(installedAppsRes.data)); const appsInstalled = installedAppsRes.data; const dockerSystemDF = await dockerService.dockerGetUsage(); const allowedMaximum = (config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap) * 1000 * 1024 * 1024; @@ -1009,18 +1242,18 @@ async function checkStorageSpaceForApps(installedApps, removeAppLocally, softRed module.exports = { appTop, appLog, - appLogStream, appLogPolling, appInspect, appStats, appMonitor, - appMonitorStream, + appMonitorAPI, appExec, appChanges, - getAppFolderSize, startAppMonitoring, + ensureAppMonitoring, stopAppMonitoring, listAppsImages, + listAppsImagesApi, getAppsDOSState, checkApplicationsCpuUSage, monitorSharedDBApps, diff --git a/ZelBack/src/services/appManagement/appsRuntimeState.js b/ZelBack/src/services/appManagement/appsRuntimeState.js index 31fdd2c6a0..d347f14f31 100644 --- a/ZelBack/src/services/appManagement/appsRuntimeState.js +++ b/ZelBack/src/services/appManagement/appsRuntimeState.js @@ -20,6 +20,19 @@ const STABLE_RUN_MS = config.fluxapps.crashBackoffStableRunMs ?? 10 * 60 * 1000; // only the count (capped by the ladder) and the last timestamp are ever read, // so the persisted history never needs to grow beyond the ladder length const MAX_HISTORY = BACKOFF_DELAYS_MS.length; +// The exit code cannot prove a fault: an image whose entrypoint is a wrapper +// script ending in `exit 0` reports a clean stop for a segfault, and no init we +// wrap around it can recover a status the image already discarded. So pacing on +// the code alone would leave such a container restarting without limit. This is +// the cause-blind backstop - this many automatic restarts ALREADY RECORDED +// inside the window is evidence of a fault whatever Docker reported, and it +// disposes into the same ladder rather than into a state a human has to clear. +// +// The count is of restarts already behind it, so at 5 the SIXTH restart is the +// one that earns a rung and the seventh is the first one held back. Six free +// restarts from a knob that reads as five is worth knowing before tuning it. +const RESTART_BURST_COUNT = config.fluxapps.restartBurstCount ?? 5; +const RESTART_BURST_WINDOW_MS = config.fluxapps.restartBurstWindowMs ?? 5 * 60 * 1000; function collection() { const db = dbHelper.databaseConnection(); @@ -57,14 +70,13 @@ function isDuplicateKeyError(err) { return err && (err.code === 11000 || /E11000/.test(err.message || '')); } -async function setFields(rawIdentifier, fields) { - const identifier = canonical(rawIdentifier); +async function upsertState(identifier, update) { const database = collection(); const write = () => dbHelper.updateOneInDatabase( database, appsRuntimeState, { identifier }, - { $set: { identifier, ...fields, updatedAt: Date.now() } }, + update, { upsert: true }, ); try { @@ -73,30 +85,105 @@ async function setFields(rawIdentifier, fields) { // Under the unique index, the loser of a concurrent first upsert THROWS a // duplicate-key error instead of converting to an update. The document // exists at that point, so one retry takes the update path - without it the - // loser's write (possibly the operator stop lock) would be silently dropped. + // loser's write (possibly the operator stop lock, or a restart request) + // would be silently dropped. if (!isDuplicateKeyError(err)) throw err; await write(); } } +async function setFields(rawIdentifier, fields) { + const identifier = canonical(rawIdentifier); + await upsertState(identifier, { $set: { identifier, ...fields, updatedAt: Date.now() } }); +} + /** * Sets the operator stop lock. This is the highest-priority desired-state * input — when true the reconciler must never auto-start the component. A * deliberate (re)start (operatorStopped=false, also install/redeploy) clears * the crash-recovery backoff so the component gets a fresh start. * + * "Operator" here is the human working the app, not the node operator. The + * distinction the name draws is a person deciding this component stays down + * against the reconciler, the master/slave election or crash recovery bringing + * it back up — so it still reads correctly now that the node operator is not + * one of the people who can ask for it. + * + * The field is persisted and queried by name (`{ operatorStopped: true }`), so + * renaming it is a migration: without one, that query stops matching on upgrade + * and every live stop lock releases at once. + * * @param {string} identifier * @param {boolean} stopped */ -async function setOperatorStopped(identifier, stopped) { +async function setOperatorStopped(identifier, stopped, opts = {}) { // No catch: the lock is the contract that the reconciler will not restart the // app. Swallowing a write failure would let the API report success while the // lock never persisted - the caller must surface the failure instead. const fields = { operatorStopped: stopped }; - if (!stopped) fields.restartHistory = []; + if (stopped) { + // A hard kill skips the graceful shutdown window. Durable, so a crash between + // the write and the stop cannot quietly downgrade "kill now" to a drain that + // waits for the app to finish what it is doing. + fields.operatorStopForce = opts.force === true; + } else { + fields.operatorStopForce = false; + fields.restartHistory = []; + fields.autoRestartWindow = []; + } await setFields(identifier, fields); } +/** + * Raises the component's restart generation, which is how an operator restart is + * expressed as desired state rather than as a docker call from the handler. + * + * The reconciler bounces the container when the generation exceeds the one it + * last actuated, so the request is level-based - replaying it changes nothing - + * and durable, so it survives a restart of FluxOS itself. + * + * No catch, for setOperatorStopped's reason: a request that did not persist must + * not be reported to the operator as one that did. + * + * @param {string} identifier + */ +async function requestRestart(rawIdentifier) { + // Incremented by the database, never read and rewritten here. Reading first + // asked getState for a number and got null for two different answers - "no + // record yet" and "the read failed" - and treating the second as zero wrote a + // generation BELOW the one already actuated, which reads as nothing pending: the + // restart is reported and never happens. $inc has no such answer to + // misread, and creates the field at 1 when there is no record, which is what + // the read was for. It also counts two concurrent requests as two, where + // read-then-write had both read the same number and one silently overwrite the + // other. + const identifier = canonical(rawIdentifier); + await upsertState(identifier, { + $inc: { restartGeneration: 1 }, + $set: { identifier, updatedAt: Date.now() }, + }); +} + +/** + * Marks a restart generation as actuated, so the pass that follows the bounce + * does not bounce it again. + * + * Throws, unlike the recorders either side of it, because this write is not + * history - it is the only thing that stops the next pass bouncing the container + * again. Swallowed, a failure here read as "recorded" and the pass that followed + * found the request still outstanding: on a node whose reads work and whose + * writes do not, that restarted the app every POST_START_VERIFY_MS forever, on + * the one path deliberately exempt from the backoff ladder. Losing an exit code + * (recordExit) costs a log line; losing this one costs the app. + * + * @param {string} identifier + * @param {number} generation + * @throws when the write fails + */ +async function recordRestartGeneration(identifier, generation) { + await setFields(identifier, { actuatedRestartGeneration: generation }); +} + /** * Whether the operator has deliberately stopped this component, so the * reconciler (and the masterSlave/syncthing deciders) must leave it stopped. @@ -110,20 +197,96 @@ async function isOperatorStopped(identifier) { } /** - * Appends a restart attempt (wall-clock) and trims the history to the ladder - * length so a perpetually crashing container never grows the array unbounded. + * The operator's stop lock and the mode they asked for, from ONE read. + * + * They are two fields of one document, and a caller needing both must not ask + * twice: getState returns null for a read failure exactly as it does for "no + * record", so a second read that failed reported no force flag and turned the + * operator's "kill now" into a drain they did not ask for. Answering both from a + * single read is a window that cannot open - and one round-trip fewer on every + * stop the reconciler performs. + * + * @param {string} identifier + * @returns {Promise<{stopped: boolean, force: boolean}>} + */ +async function operatorStopState(identifier) { + const state = await getState(identifier); + return { + stopped: state?.operatorStopped === true, + force: state?.operatorStopForce === true, + }; +} + +/** + * Every component identifier on this node the operator has deliberately stopped. + * + * One query rather than isOperatorStopped per component. The caller is the + * held-components answer, served on an unauthenticated route a peer reads + * mid-election, so a findOne apiece would scale its cost with the number of + * components installed here. + * + * Throws where getState swallows, and the difference is the point. getState's + * callers ask about one component and act on this node: a read failure there + * reads as "no lock", the reconciler leaves the container alone anyway, and the + * next pass asks again. This answer LEAVES the node - an empty list tells a peer + * nothing is held here, and the peer acts on that by starting a second writer on + * the shared volume. There is no safe way to report a lock set we could not + * read, so the caller has to fail rather than answer. + * + * @returns {Promise} bare component identifiers, unprefixed + */ +async function operatorStoppedIdentifiers() { + const database = collection(); + const docs = await dbHelper.findInDatabase( + database, + appsRuntimeState, + { operatorStopped: true }, + { projection: { _id: 0, identifier: 1 } }, + ); + return docs.map((doc) => doc.identifier).filter(Boolean); +} + +/** + * Whether the restarts already recorded fill the burst window, read before the + * current attempt is appended - so it answers "have there already been enough", + * and the restart asking is the one that carries the count over. + * + * That restart is counted, not held: restartWaitMs runs ahead of recordRestart + * and finds an empty ladder, so it goes back immediately and earns the first + * rung on its way past. The one after it is the first to be made to wait. + * + * @param {object|null} state + * @returns {boolean} + */ +function burstExceeded(state) { + const recent = (state && state.autoRestartWindow) || []; + if (recent.length < RESTART_BURST_COUNT) return false; + return Date.now() - recent[0] <= RESTART_BURST_WINDOW_MS; +} + +/** + * Appends a restart attempt (wall-clock). Every automatic restart lands in the + * burst window; only one with crash evidence - or one that fills the burst + * window, which is the same conclusion reached without the exit code - also + * walks the ladder. Both arrays are trimmed to their own bound so a + * perpetually restarting container never grows the document unbounded. * * @param {string} identifier + * @param {boolean} crashed - Docker reported a fault (non-zero exit or OOM kill) */ -async function recordRestart(identifier) { +async function recordRestart(identifier, crashed = true) { try { const state = await getState(identifier); - const history = (state && state.restartHistory) || []; - history.push(Date.now()); - if (history.length > MAX_HISTORY) { - history.splice(0, history.length - MAX_HISTORY); + const fields = { + autoRestartWindow: [...((state && state.autoRestartWindow) || []), Date.now()].slice(-RESTART_BURST_COUNT), + }; + // A rung is earned by evidence of a fault - a non-zero exit, or restarts + // arriving fast enough to be one whatever the code said - and by a component + // that already holds rungs, which is that finding still standing. + if (crashed || burstExceeded(state) || ((state && state.restartHistory) || []).length > 0) { + fields.restartHistory = [...((state && state.restartHistory) || []), Date.now()].slice(-MAX_HISTORY); } - await setFields(identifier, { restartHistory: history }); + await setFields(identifier, fields); } catch (err) { log.error(`appsRuntimeState - failed to record restart for ${identifier}: ${err.message}`); } @@ -154,9 +317,20 @@ async function recordRestart(identifier) { */ async function restartWaitMs(identifier, lastFinishedAtMs = null) { const state = await getState(identifier); + // An empty ladder is a component nothing has found fault with: a clean exit is + // the operator's own restart far more often than it is a fault, and pacing that + // makes a deliberate restart look like an outage. recordRestart decides what + // earns a rung; once a component holds one it is paced until it clears them, + // whatever its exit code reads. The burst window empties while a component + // sits out a wait, so the ladder is what has to carry the finding across one. const history = (state && state.restartHistory) || []; if (history.length === 0) return 0; + // The ladder's last rung IS the component's last start, because a component + // holding rungs earns one for every restart. That is what makes this a stable + // run and not a wait it just served: if a restart could go unrecorded while + // rungs stood, the gap measured here would be the wait itself, and any rung + // longer than STABLE_RUN_MS would clear the ladder every time it fired. const lastRestart = history[history.length - 1]; const lastDeath = Math.max(state.lastDiedAt || 0, lastFinishedAtMs || 0); if (lastDeath > lastRestart && lastDeath - lastRestart > STABLE_RUN_MS) { @@ -326,11 +500,19 @@ async function prepareCollection() { identifier, // a lock anywhere is a lock: never auto-start a deliberately stopped app operatorStopped: twins.some((t) => t.operatorStopped === true), + // a force anywhere is a force: a kill must never be merged down into a + // graceful stop the operator did not ask for + operatorStopForce: twins.some((t) => t.operatorStopForce === true), + // the highest request wins and the lowest actuation does, so a restart + // asked for on either doc still bounces the container once + restartGeneration: Math.max(0, ...twins.map((t) => t.restartGeneration || 0)), + actuatedRestartGeneration: Math.min(...twins.map((t) => t.actuatedRestartGeneration || 0)), // likewise, a heal-removal anywhere means a heal may be mid-flight: keep // the reconciler on the recreate path rather than the uninstall path networkHealRemoval: twins.some((t) => t.networkHealRemoval === true), networkHealHistory: [...new Set(twins.flatMap((t) => t.networkHealHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), restartHistory: [...new Set(twins.flatMap((t) => t.restartHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), + autoRestartWindow: [...new Set(twins.flatMap((t) => t.autoRestartWindow || []))].sort((a, b) => a - b).slice(-RESTART_BURST_COUNT), updatedAt: Math.max(...twins.map((t) => t.updatedAt || 0)), }; const newestExit = twins.filter((t) => t.lastDiedAt !== undefined).sort((a, b) => b.lastDiedAt - a.lastDiedAt)[0]; @@ -356,7 +538,11 @@ module.exports = { getState, setOperatorStopped, isOperatorStopped, + operatorStopState, + operatorStoppedIdentifiers, recordRestart, + requestRestart, + recordRestartGeneration, restartWaitMs, setNetworkHealRemoval, isNetworkHealRemoval, @@ -368,4 +554,6 @@ module.exports = { BACKOFF_DELAYS_MS, STABLE_RUN_MS, MAX_HISTORY, + RESTART_BURST_COUNT, + RESTART_BURST_WINDOW_MS, }; diff --git a/ZelBack/src/services/appManagement/dockerOperations.js b/ZelBack/src/services/appManagement/dockerOperations.js deleted file mode 100644 index 7c9672eb81..0000000000 --- a/ZelBack/src/services/appManagement/dockerOperations.js +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Docker Operations Module - * - * This module contains Docker-related helper functions for managing app containers. - * These are internal operations that work directly with Docker containers and monitoring. - */ - -const util = require('util'); -const dockerService = require('../dockerService'); -const log = require('../../lib/log'); - -const cmdAsync = util.promisify(require('child_process').exec); - -// Import app constants -const { appsFolder } = require('../utils/appConstants'); - -/** - * Stop a Docker container for a specific app (with monitoring integration) - * @param {string} appname - Application name or component name - * @param {Function} stopMonitoringCallback - Callback function to stop app monitoring - * @param {Map} appsMonitored - Map of currently monitored apps - * @param {Function} getApplicationSpecifications - Function to get app specifications - * @returns {Promise} - */ -async function appDockerStop(appname, stopMonitoringCallback, appsMonitored, getApplicationSpecifications) { - try { - const mainAppName = appname.split('_')[1] || appname; - const isComponent = appname.includes('_'); // it is a component stop. Proceed with stopping just component - if (isComponent) { - await dockerService.appDockerStop(appname); - if (stopMonitoringCallback) { - stopMonitoringCallback(appname, false, appsMonitored); - } - } else { - // ask for stopping entire composed application - const appSpecs = await getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - if (appSpecs.version <= 3) { - await dockerService.appDockerStop(appname); - if (stopMonitoringCallback) { - stopMonitoringCallback(appname, false, appsMonitored); - } - } else { - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose) { - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerStop(`${appComponent.name}_${appSpecs.name}`); - if (stopMonitoringCallback) { - stopMonitoringCallback(`${appComponent.name}_${appSpecs.name}`, false, appsMonitored); - } - } - } - } - } catch (error) { - log.error(error); - } -} - -/** - * Restart a Docker container for a specific app (with monitoring integration) - * @param {string} appname - Application name or component name - * @param {Function} startMonitoringCallback - Callback function to start app monitoring - * @param {Map} appsMonitored - Map of currently monitored apps - * @param {Function} getApplicationSpecifications - Function to get app specifications - * @returns {Promise} - */ -async function appDockerRestart(appname, startMonitoringCallback, appsMonitored, getApplicationSpecifications) { - try { - const mainAppName = appname.split('_')[1] || appname; - const isComponent = appname.includes('_'); // it is a component restart. Proceed with restarting just component - if (isComponent) { - await dockerService.appDockerRestart(appname); - if (startMonitoringCallback) { - startMonitoringCallback(appname, appsMonitored); - } - } else { - // ask for restarting entire composed application - const appSpecs = await getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - if (appSpecs.version <= 3) { - await dockerService.appDockerRestart(appname); - if (startMonitoringCallback) { - startMonitoringCallback(appname, appsMonitored); - } - } else { - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose) { - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerRestart(`${appComponent.name}_${appSpecs.name}`); - if (startMonitoringCallback) { - startMonitoringCallback(`${appComponent.name}_${appSpecs.name}`, appsMonitored); - } - } - } - } - } catch (error) { - log.error(error); - } -} - -/** - * Delete all data in the mount point for a specific app - * @param {string} appId - Application ID - * @returns {Promise} - */ -async function appDeleteDataInMountPoint(appId) { - // Implementation for deleting app data in mount point - try { - const execDelete = `sudo rm -rf ${appsFolder}${appId}/appdata/*`; - await cmdAsync(execDelete); - log.info(`Deleted data for app ${appId}`); - } catch (error) { - log.error(`Error deleting data for app ${appId}: ${error.message}`); - } -} - -module.exports = { - appDockerStop, - appDockerRestart, - appDeleteDataInMountPoint, -}; diff --git a/ZelBack/src/services/appManagement/operationsController.js b/ZelBack/src/services/appManagement/operationsController.js new file mode 100644 index 0000000000..996f7bef77 --- /dev/null +++ b/ZelBack/src/services/appManagement/operationsController.js @@ -0,0 +1,180 @@ +const messageHelper = require('../messageHelper'); +const serviceHelper = require('../serviceHelper'); +const verificationHelper = require('../verificationHelper'); +const jobRegistry = require('../utils/jobRegistry'); +const log = require('../../lib/log'); +const { Privilege, authOf } = require('../utils/privileges'); + +// The one status resource for every long-running operation this node accepts. +// Endpoints that start work answer 202 with a job handle and point here; this +// owns the polling contract so no endpoint has to invent one. + +/** + * The FluxID a caller is authenticated as, or null. Jobs registered with an + * owner are only readable by that identity; jobs registered without one treat + * the jobId itself as the capability. + */ +async function callerFluxId(req) { + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); + if (!authorized) return null; + const auth = serviceHelper.ensureObject(authOf(req)); + return auth ? auth.zelid : null; +} + +/** + * Shape a 202 for an endpoint that has just started work. Location and + * Operation-Id are the RFC 9110 / Azure long-running-operation spelling; the + * body repeats them so a client that cannot read headers is not stuck. + * + * @param {import('express').Response} res + * @param {{jobId: string, statusUrl: string}} handle + * @param {object} [extra] additional body fields the endpoint wants echoed + */ +function accepted(res, handle, extra = {}) { + res.setHeader('Location', handle.statusUrl); + res.setHeader('Operation-Id', handle.jobId); + res.setHeader('Retry-After', String(jobRegistry.retryAfterSeconds())); + return res.status(202).json(messageHelper.createDataMessage({ + jobId: handle.jobId, + statusUrl: handle.statusUrl, + status: jobRegistry.JobStatus.RUNNING, + ...extra, + })); +} + +/** + * How far through an operation's line-numbered output the caller has already + * read. Anything that is not a non-negative whole number is treated as no + * cursor at all - a client sending nonsense gets the whole retained view rather + * than a silently truncated one. + * + * @param {import('express').Request} req + * @returns {number} 0 when absent or unusable + */ +/** + * Answer an operation that finished while the caller was still waiting. + * + * 200 rather than 202, because there is nothing to come back for: the job is in + * its terminal state and the body carries it. A client that predates jobs sees + * what it has always seen - a completed request - and one that knows about them + * reads the same status it would have polled for. + * + * @param {object} res + * @param {{jobId: string, statusUrl: string}} handle + * @returns {object} + */ +function completed(res, handle, owner = null) { + res.setHeader('Operation-Id', handle.jobId); + + // The owner has to be given, because a job carries one and the registry + // refuses a read that does not match it. Read without it, every owned job + // came back null and the answer fell through to a hardcoded Succeeded - so an + // operation that FAILED was reported to its caller as having worked. + const job = jobRegistry.get(handle.jobId, owner); + + // Gone or unreadable is not the same as succeeded. The caller is told the + // work is over, and is not told that it worked. + if (!job) { + return res.status(200).json(messageHelper.createErrorMessage( + 'The operation finished but its result could not be read', + )); + } + + if (job.status !== jobRegistry.JobStatus.SUCCEEDED) { + // message, name and code, which is the shape a failed file operation has + // always answered with and what the dashboards read. The problem document + // is built from an Error, so its title IS that error's name. + const problem = job.error || {}; + return res.status(200).json(messageHelper.createErrorMessage( + problem.detail || problem.title || `Operation ${job.status}`, + problem.title, + problem.code, + )); + } + + return res.status(200).json(messageHelper.createDataMessage({ + jobId: handle.jobId, + statusUrl: handle.statusUrl, + status: job.status, + })); +} + +function readCursor(req) { + const raw = req.query && req.query.sinceSeq; + if (raw === undefined || raw === null || raw === '') return 0; + const seq = Number(raw); + if (!Number.isSafeInteger(seq) || seq < 0) return 0; + return seq; +} + +/** + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +async function getOperation(req, res) { + try { + const jobId = req.params.jobId || (req.query && req.query.jobId); + if (!jobId) { + return res.status(400).json(messageHelper.createErrorMessage('Missing jobId')); + } + + const view = jobRegistry.get(jobId, await callerFluxId(req), { sinceSeq: readCursor(req) }); + // Unknown, expired and not-yours are one answer: a jobId must not tell a + // caller whether someone else has an operation running. + if (!view) { + return res.status(404).json(messageHelper.createErrorMessage('Operation not found')); + } + + // A running operation is a 200 with a non-terminal body. Completion is read + // from the status field, never inferred from the HTTP code - a failed + // operation is still a successful poll. + if (jobRegistry.isTerminal(view.status)) { + res.setHeader('Expires', new Date(view.lastUpdatedAt + 60 * 60 * 1000).toUTCString()); + } else { + res.setHeader('Retry-After', String(jobRegistry.retryAfterSeconds())); + } + + return res.json(messageHelper.createDataMessage(view)); + } catch (error) { + log.error(`operationsController getOperation: ${error.message}`); + return res.status(500).json(messageHelper.createErrorMessage(error.message)); + } +} + +/** + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +async function cancelOperation(req, res) { + try { + const { jobId } = req.params; + if (!jobId) { + return res.status(400).json(messageHelper.createErrorMessage('Missing jobId')); + } + + const owner = await callerFluxId(req); + const view = jobRegistry.get(jobId, owner); + if (!view) { + return res.status(404).json(messageHelper.createErrorMessage('Operation not found')); + } + + // Best effort, and said so plainly: the flag is raised here and the worker + // stops at its next checkpoint, so the status stays Running until it does. + const requested = jobRegistry.requestCancel(jobId); + return res.json(messageHelper.createDataMessage({ + jobId, + cancelRequested: requested, + status: jobRegistry.get(jobId, owner).status, + })); + } catch (error) { + log.error(`operationsController cancelOperation: ${error.message}`); + return res.status(500).json(messageHelper.createErrorMessage(error.message)); + } +} + +module.exports = { + completed, + accepted, + getOperation, + cancelOperation, +}; diff --git a/ZelBack/src/services/appMessaging/appHashSyncService.js b/ZelBack/src/services/appMessaging/appHashSyncService.js index 9c2172a622..6765c8530a 100644 --- a/ZelBack/src/services/appMessaging/appHashSyncService.js +++ b/ZelBack/src/services/appMessaging/appHashSyncService.js @@ -21,6 +21,7 @@ const { appSyncEvents, EVENTS } = require('../utils/appSyncEvents'); const { HASH_EXPIRY_BLOCKS, HASH_RETRY_BACKOFF } = require('../utils/appConstants'); const log = require('../../lib/log'); const { invalidMessages } = require('../invalidMessages'); +const { Privilege, authOf } = require('../utils/privileges'); const appsHashesCollection = config.database.daemon.collections.appsHashes; const globalAppsMessages = config.database.appsglobal.collections.appsMessages; @@ -487,6 +488,7 @@ async function broadcastHashRequest(hashes, peers) { hashes, }; const signed = await serialiseAndSignFluxBroadcast(message); + if (!signed) return; for (const peer of peers) { peer.send(signed); } @@ -720,7 +722,7 @@ async function syncMissingHashes(options = {}) { async function triggerAppHashesCheckAPI(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); diff --git a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js index 04e92fff06..8b4112a796 100644 --- a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js +++ b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js @@ -7,10 +7,9 @@ const peerNotification = require('./peerNotification'); const registryManager = require('../appDatabase/registryManager'); const globalState = require('../utils/globalState'); const peerCodec = require('../utils/peerCodec'); -const fluxNetworkHelper = require('../fluxNetworkHelper'); -const verificationHelper = require('../verificationHelper'); const { appSyncEvents, EVENTS } = require('../utils/appSyncEvents'); const fluxEventBus = require('../utils/fluxEventBus'); +const { nodeSigner } = require('../utils/nodeSigner'); const startupCollection = config.database.local.collections.nodeStartupTracker; @@ -24,10 +23,52 @@ const STATES = Object.freeze({ const MIN_SYNC_COMPLETIONS = config.fluxapps.appSyncMinCompletions ?? 3; const SYNC_TIMEOUT_MS = config.fluxapps.syncTimeoutMs ?? 120000; -const MIN_UPTIME_SECONDS = config.fluxapps.appSyncMinPeerUptime ?? 7500; const HASH_SYNC_MAX_RETRIES = config.fluxapps.hashSyncMaxRetries ?? 3; const HASH_SYNC_RETRY_MS = config.fluxapps.hashSyncRetryMs ?? 300000; const FALLBACK_RECHECK_BLOCKS = config.fluxapps.hashSyncFallbackRecheckBlocks ?? 100; +const FALLBACK_MINUTES = config.fluxapps.appSyncFallbackMinutes ?? 125; +// A chain fact, not a policy: blocks are 30 seconds since the PON fork +// (config.fluxapps.daemonPONFork), so two a minute. Not a knob - a node that +// disagrees with the chain about this converts appSyncFallbackMinutes into the +// wrong number of blocks and waits the wrong length of time in silence. +const BLOCKS_PER_MINUTE = 2; +// THE TWO WAYS A PEER CAN BE QUIET, and they mean different things. +// +// A slot must be able to fail and its replacement still finish inside the +// budget, so with S as a healthy peer's completion time - about a minute on +// our fleet - both of these have to satisfy `deadline + S <= SYNC_TIMEOUT_MS`. +// At 120s that leaves 50s and 30s of slack respectively. +// +// FIRST_RESPONSE is "never spoke". The only work between our send and the +// peer's first batch is a signature check, one indexed query and serialising +// 2000 documents, so a tenth of the budget is about an order of magnitude more +// than it needs - and a peer that has sent NOTHING is unambiguous, because a +// peer with nothing to report still sends an empty final batch. +// +// STALL is "spoke, then stopped", which needs more room because a peer may +// legitimately be working between batches. A quarter caps what a stalled peer +// can spend. In production it also lands inside the transport's own liveness +// window (wsPingIntervalMs * wsMaxMissedPongs = 45s), so the sync replaces a +// stalled peer before the socket layer has decided it is dead. +const FIRST_RESPONSE_MS = Math.max(1, Math.floor(SYNC_TIMEOUT_MS / 12)); +const STALL_MS = Math.max(1, Math.floor(SYNC_TIMEOUT_MS / 4)); + +/** + * A fresh record of which peers have answered which stream. + * + * One place, because the tally is built twice - once at construction and again + * whenever a sync starts over - and a stream present in one copy and not the + * other is a requirement that quietly stops being checked. + * @returns {{[syncType: string]: Set}} + */ +function freshSyncCompletions() { + return { + apprunning: new Set(), + appinstalling: new Set(), + apperrors: new Set(), + apptemp: new Set(), + }; +} class AppSyncOrchestrator { #state = STATES.INITIALIZING; @@ -35,9 +76,6 @@ class AppSyncOrchestrator { #getEligibleSyncPeers = null; #onPeerEvent = null; #offPeerEvent = null; - #markSyncRequested = null; - #clearSyncRequested = null; - #isEnterprise = null; #waitForNetworkState = null; #networkReady = false; #peersReady = false; @@ -45,20 +83,61 @@ class AppSyncOrchestrator { #hashSyncComplete = false; #dbRebuilt = false; #blocksSinceSyncStarted = 0; - #blockThreshold = 0; #blockReceivedHandler = null; #peerThresholdHandler = null; #peersBelowHandler = null; + #peerConnectedHandler = null; + #peerDisconnectedHandler = null; #ephemeralSyncHandler = null; + #ephemeralRefusedHandler = null; + #ephemeralUnverifiedHandler = null; + #ephemeralProgressHandler = null; #hashUnresolvedHandler = null; #hashesChangedHandler = null; #broadcastStarted = null; #started = false; #syncInProgress = false; - #askedPeers = new Set(); - #syncCompletions = { apprunning: 0, appinstalling: 0, apperrors: 0 }; + // WHICH peers answered, not how many answers arrived. Three responses from + // one peer are one peer's view of the network, and counting them as three + // satisfied the requirement without ever asking anyone else. + // + // EVERY stream this node asked for, pending registrations included. A peer + // answers all four or refuses all four, so a tally that stopped at three + // credited a peer while it was still delivering - and the record closing on + // that credit is what stopped this node listening to the rest of it. + #syncCompletions = freshSyncCompletions(); #stateSyncComplete = false; #syncTimeout = null; + /** + * peerKey -> the request outstanding to that peer, and how it ended. + * + * ONE record. The deadline hangs off it, the pool counts it, the candidate + * filter reads it, and the response path asks it whether an arriving answer + * is still wanted. Those were separate records with separate owners and + * separate clearing rules, and every way they could disagree was a defect: + * a round that ended cleared one and left the others running, and a decline + * written onto a socket had no lifetime at all. + * + * A record is OPEN until it has an outcome, then it stands - a peer that has + * answered or been set aside is not a candidate - until #sweepRequests + * decides it no longer describes anything. + * @type {Map|null, outcome: string|null, closedAt: number}>} + */ + #requests = new Map(); + /** + * Whether this node has spent its state-sync budget. + * + * The budget bounds THE attempt, not one round inside an open-ended series of + * them: when it runs out the attempt is over, the block fallback is what + * carries the node to readiness, and nothing asks anyone anything until the + * sync genuinely starts again - which is a drop below the peer threshold and + * a recovery, or a restart. Without it "how long before this node gives up" + * has no answer, because a peer joining an hour later would open another one. + */ + #syncBudgetSpent = false; + #reconciling = false; + #reconcileAgain = false; #hashSyncAttempts = 0; #hashSyncRetryTimer = null; #nextHashRetryHeight = 0; @@ -74,9 +153,6 @@ class AppSyncOrchestrator { this.#getEligibleSyncPeers = options.getEligibleSyncPeers; this.#onPeerEvent = options.onPeerEvent; this.#offPeerEvent = options.offPeerEvent; - this.#markSyncRequested = options.markSyncRequested ?? (() => {}); - this.#clearSyncRequested = options.clearSyncRequested ?? (() => {}); - this.#isEnterprise = options.isEnterprise ?? (() => false); this.#peerCountIfAboveThreshold = options.peerCountIfAboveThreshold ?? (() => 0); this.#waitForNetworkState = options.networkStateReady ?? null; this.#fluxVersion = options.fluxVersion ?? null; @@ -116,8 +192,22 @@ class AppSyncOrchestrator { log.info(`AppSyncOrchestrator - Peers below threshold (${count} peers)`); this.#onPeersDegraded(); }; + // Every join, because a peer arriving while the pool is short is what can + // fill it. `peerThresholdReached` is a latched edge that fires once, so it + // says nothing about a pool that has since lost a member; without this the + // only remaining road was the block timer, 125 minutes of SYNCING with the + // spawner paused. + // + // Note it announces a change rather than deciding anything: the threshold + // crossing emits both events from the same call, so this and the handler + // above run in the same tick, and it is the reconciler that makes that + // safe rather than either of them knowing about the other. + this.#peerConnectedHandler = () => this.#reconcile(); this.#onPeerEvent('peerThresholdReached', this.#peerThresholdHandler); this.#onPeerEvent('peersBelowThreshold', this.#peersBelowHandler); + this.#onPeerEvent('peerConnected', this.#peerConnectedHandler); + this.#peerDisconnectedHandler = (peerKey, connectionId) => this.#onPeerDisconnected(peerKey, connectionId); + this.#onPeerEvent('peerDisconnected', this.#peerDisconnectedHandler); // peerThresholdReached is edge-triggered and latched in FluxPeerManager: // if peers connected fast enough that the threshold was crossed BEFORE the @@ -130,9 +220,18 @@ class AppSyncOrchestrator { this.#peerThresholdHandler(peersAlready); } - this.#ephemeralSyncHandler = (syncType) => this.#onEphemeralSyncComplete(syncType); + this.#ephemeralSyncHandler = (syncType, peerKey) => this.#onEphemeralSyncComplete(syncType, peerKey); appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_COMPLETE, this.#ephemeralSyncHandler); + this.#ephemeralRefusedHandler = (syncType, peerKey) => this.#onEphemeralSyncRefused(syncType, peerKey); + appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_REFUSED, this.#ephemeralRefusedHandler); + + this.#ephemeralUnverifiedHandler = (peerKey) => this.#onEphemeralSyncUnverified(peerKey); + appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_UNVERIFIED, this.#ephemeralUnverifiedHandler); + + this.#ephemeralProgressHandler = (peerKey) => this.#onEphemeralSyncProgress(peerKey); + appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_PROGRESS, this.#ephemeralProgressHandler); + this.#hashUnresolvedHandler = () => this.#onHashUnresolved(); appSyncEvents.on(EVENTS.HASH_UNRESOLVED, this.#hashUnresolvedHandler); @@ -153,6 +252,10 @@ class AppSyncOrchestrator { } else { this.#networkReady = true; } + // Before any block arrives, because a node whose fallback is 0 blocks is + // authoritative from the moment it starts and a peer may ask it first. + this.#publishStateSyncAuthority(); + // #peersReady may already be true here (live edge during the network-state // wait, or the latched-level check above), so always attempt the start. this.#tryStartSync(); @@ -163,35 +266,323 @@ class AppSyncOrchestrator { this.#onPeersReady(); } - #onEphemeralSyncComplete(syncType) { + /** + * Record that one peer finished one sync type. + * @param {string} syncType apprunning | appinstalling | apperrors + * @param {string} peerKey ip:port of the peer that answered. + * @returns {void} + */ + #onEphemeralSyncComplete(syncType, peerKey) { if (this.#stateSyncComplete) return; - if (this.#syncCompletions[syncType] === undefined) return; - this.#syncCompletions[syncType] += 1; - log.info(`AppSyncOrchestrator - ${syncType} sync complete (${this.#syncCompletions[syncType]}/${MIN_SYNC_COMPLETIONS})`); + const answered = this.#syncCompletions[syncType]; + if (answered === undefined) return; + // An answer nobody can attribute cannot be counted. Counting it is the + // defect this records peers to avoid, and a completion whose peer is + // missing means the response path stopped saying who it came from - which + // is a fault to report, not to absorb. + if (!peerKey) { + log.error(`AppSyncOrchestrator - ${syncType} sync complete with no peer, not counted`); + return; + } + answered.add(peerKey); + // Its answer is in, so it is no longer something being waited on. The + // record stands as answered rather than being dropped: a peer that has + // given its whole view has nothing left to add and must not be re-asked. + if (this.#peerAnswered(peerKey)) this.#closeRequest(peerKey, 'answered'); + log.info(`AppSyncOrchestrator - ${syncType} sync complete from ${peerKey} (${answered.size}/${MIN_SYNC_COMPLETIONS} peers)`); fluxEventBus.publish('ephemeralSync:peerComplete', { syncType, - completions: this.#syncCompletions[syncType], + peer: peerKey, + completions: answered.size, required: MIN_SYNC_COMPLETIONS, }); - if (this.#syncCompletions.apprunning >= MIN_SYNC_COMPLETIONS - && this.#syncCompletions.appinstalling >= MIN_SYNC_COMPLETIONS - && this.#syncCompletions.apperrors >= MIN_SYNC_COMPLETIONS) { + if (Object.values(this.#syncCompletions).every((peers) => peers.size >= MIN_SYNC_COMPLETIONS)) { this.#stateSyncComplete = true; + this.#publishStateSyncAuthority(); if (this.#syncTimeout) { clearTimeout(this.#syncTimeout); this.#syncTimeout = null; } - this.#clearSyncRequested(); + this.#closeRound('the sync completed'); log.info('AppSyncOrchestrator - All state syncs complete'); - fluxEventBus.publish('ephemeralSync:allComplete', { - apprunning: this.#syncCompletions.apprunning, - appinstalling: this.#syncCompletions.appinstalling, - apperrors: this.#syncCompletions.apperrors, - }); + fluxEventBus.publish('ephemeralSync:allComplete', this.#completionCounts()); this.#checkReadiness(); } } + /** + * Whether one peer has delivered every stream it was asked for. + * + * Read off the tally rather than named stream by stream, so a stream added + * to the tally is one a peer has to be seen answering. + * @param {string} peerKey ip:port + * @returns {boolean} + */ + #peerAnswered(peerKey) { + for (const answered of Object.values(this.#syncCompletions)) { + if (!answered.has(peerKey)) return false; + } + return true; + } + + /** + * How many peers have answered every stream. + * + * One set is enough to walk: a peer that answered everything is in all of + * them, so any of them holds every candidate. + * @returns {number} + */ + #completedPeerCount() { + let complete = 0; + for (const peerKey of this.#syncCompletions.apprunning) { + if (this.#peerAnswered(peerKey)) complete += 1; + } + return complete; + } + + /** + * How many peers have answered each stream, for a log line or an event. + * @returns {{[syncType: string]: number}} + */ + #completionCounts() { + return Object.fromEntries( + Object.entries(this.#syncCompletions).map(([type, peers]) => [type, peers.size]), + ); + } + + /** + * How many requests are still waiting on an answer. + * @returns {number} + */ + #openRequestCount() { + let open = 0; + for (const request of this.#requests.values()) if (!request.outcome) open += 1; + return open; + } + + /** + * Start waiting on a peer, with a deadline for it saying anything at all. + * @param {{key: string, connectionId?: number}} peer + * @returns {void} + */ + #openRequest(peer) { + this.#discardRequest(peer.key); + const request = { + peerKey: peer.key, + connectionId: peer.connectionId ?? null, + spoken: false, + timer: null, + outcome: null, + closedAt: 0, + }; + this.#requests.set(peer.key, request); + request.timer = setTimeout(() => this.#onRequestDeadline(peer.key, 'said nothing'), FIRST_RESPONSE_MS); + if (request.timer.unref) request.timer.unref(); + } + + /** + * Stop waiting on a peer, recording why. + * + * The record STANDS after this. It is what keeps a peer that has answered, + * declined or run out of time from being asked again in the next breath, and + * #sweepRequests is the only thing that decides it has stopped meaning + * anything. + * @param {string} peerKey ip:port + * @param {string} outcome answered | declined | timedOut + * @returns {boolean} true if the request was still open. + */ + #closeRequest(peerKey, outcome) { + const request = this.#requests.get(peerKey); + if (!request || request.outcome) return false; + if (request.timer) { + clearTimeout(request.timer); + request.timer = null; + } + request.outcome = outcome; + request.closedAt = Date.now(); + return true; + } + + /** + * Forget a request entirely, so the peer is a candidate again. + * @param {string} peerKey ip:port + * @returns {void} + */ + #discardRequest(peerKey) { + const request = this.#requests.get(peerKey); + if (!request) return; + if (request.timer) clearTimeout(request.timer); + this.#requests.delete(peerKey); + } + + /** + * End every request still outstanding, because the round they belong to has. + * + * Closing them is what stops their answers being accepted, so the response + * gate and the deadlines read one fact and cannot disagree about whether a + * peer is still being waited on. A peer that was mid-answer when the budget + * ran out is recorded as having run out of time, which is what happened - + * not as having stalled, which is what it would look like to a deadline left + * armed over a gate that had already stopped listening. + * @param {string} why For the log. + * @returns {number} how many were still outstanding. + */ + #closeRound(why) { + let outstanding = 0; + for (const [peerKey, request] of this.#requests) { + if (request.outcome) continue; + outstanding += 1; + this.#closeRequest(peerKey, 'timedOut'); + } + if (outstanding) { + log.info(`AppSyncOrchestrator - ${outstanding} state-sync ${outstanding === 1 ? 'request was' : 'requests were'} still outstanding when ${why}`); + } + return outstanding; + } + + /** + * Anything arriving from a peer proves it is working, so its clock restarts. + * + * The first arrival moves it off the short "never spoke" deadline and onto + * the longer stall one, because a peer part-way through a large answer is + * doing exactly what was asked and may legitimately pause between batches. + * + * Which stream it arrived on does not matter - the question this answers is + * whether the peer is still there, and any of its four responses says so. + * @param {string} peerKey ip:port + * @returns {void} + */ + #onEphemeralSyncProgress(peerKey) { + const request = this.#requests.get(peerKey); + if (!request || request.outcome) return; + clearTimeout(request.timer); + request.spoken = true; + request.timer = setTimeout(() => this.#onRequestDeadline(peerKey, 'stopped mid-answer'), STALL_MS); + if (request.timer.unref) request.timer.unref(); + } + + /** + * A peer that is still connected and is not talking. + * + * The only case a deadline is for: a closed socket ends its request the + * moment it closes, a refusal ends it on the answer, and the round ending + * ends every one of them. So reaching here means the peer is still there and + * has nothing to show for the time. + * @param {string} peerKey ip:port + * @param {string} why What the peer did, for the log. + * @returns {void} + */ + #onRequestDeadline(peerKey, why) { + if (!this.#closeRequest(peerKey, 'timedOut')) return; + if (this.#stateSyncComplete) return; + log.warn(`AppSyncOrchestrator - ${peerKey} ${why} within its deadline, asking another peer`); + fluxEventBus.publish('ephemeralSync:peerTimedOut', { peer: peerKey, reason: why }); + this.#reconcile(); + } + + /** + * A peer whose connection ended can never answer it, so its request ends too. + * + * A peer gets ONE attempt per sync. Its connection dying is the peer's + * answer to this attempt - it dropped in the middle of it - and a node that + * re-asked it would spend another of very few slots on a peer that has just + * shown it cannot hold a socket long enough to answer. The record therefore + * closes rather than being discarded, and stands: a peer that dials back in + * is one already tried. + * + * Told rather than inferred, and told about the CONNECTION. The two ways one + * ends - the peer leaving, and a dead socket being replaced by the peer's + * own reconnect - used to announce themselves differently, and the second not + * at all; what noticed it was a sweep looking for a connection id that had + * changed underneath a record. An announcement naming the connection is the + * whole of that, and an announcement about a connection this request was not + * written into says nothing about it. + * @param {string} peerKey ip:port + * @param {number|null} connectionId The connection that ended. + * @returns {void} + */ + #onPeerDisconnected(peerKey, connectionId) { + const request = this.#requests.get(peerKey); + if (!request) return; + if (request.connectionId !== (connectionId ?? null)) return; + if (!this.#closeRequest(peerKey, 'disconnected')) return; + if (this.#stateSyncComplete) return; + log.info(`AppSyncOrchestrator - ${peerKey} went away with a sync outstanding, asking another peer`); + fluxEventBus.publish('ephemeralSync:peerDisconnected', { peer: peerKey, connectionId: connectionId ?? null }); + this.#reconcile(); + } + + /** + * Whether a sync response arriving on this connection is still wanted. + * + * The request record answers it, so the gate that admits a response and the + * deadline that gives up on one read the same fact and cannot drift apart. + * Asked with the socket because a reconnected peer is a different connection: + * the request went into the old one, and nothing on the new one answers it. + * @param {{key: string, connectionId?: number}} peerSocket + * @returns {boolean} + */ + isSyncResponseWanted(peerSocket) { + if (!peerSocket) return false; + const request = this.#requests.get(peerSocket.key); + if (!request || request.outcome) return false; + return request.connectionId === (peerSocket.connectionId ?? null); + } + + /** + * A peer answered by declining, which is not a completion. + * + * Its request ends as declined, so it stops being a candidate and the pool + * shows a deficit that the next pass fills from a peer that may actually + * know something. A peer refuses all three types when it refuses any, and + * only the first of those closes the request - so the log says once what + * happened once. + * @param {string} syncType apprunning | appinstalling | apperrors + * @param {string} peerKey ip:port of the peer that declined. + * @returns {void} + */ + #onEphemeralSyncRefused(syncType, peerKey) { + if (this.#stateSyncComplete) return; + if (!peerKey) { + log.error(`AppSyncOrchestrator - ${syncType} sync declined with no peer, cannot replace it`); + return; + } + if (this.#closeRequest(peerKey, 'declined')) { + log.info(`AppSyncOrchestrator - ${peerKey} declined the ${syncType} sync, asking another peer`); + } + // Unconditional, because the deficit decides. A refusal from a peer this + // node never had in its pool leaves the pool whole, and a whole pool asks + // nobody - so an early return here would only be a second way of saying + // the same thing, and one that no test could tell from its absence. + this.#reconcile(); + } + + /** + * A peer sent something this node cannot attribute to it. + * + * Its request ends here rather than on a deadline, because the answer is + * already known: a stream with a hole in it is not a survey, and waiting the + * peer out would spend one of very few slots on an answer that cannot be + * counted whatever else arrives. The record stands, so the peer is not asked + * again on this connection. + * @param {string} peerKey ip:port of the peer whose response failed. + * @returns {void} + */ + #onEphemeralSyncUnverified(peerKey) { + if (this.#stateSyncComplete) return; + if (!peerKey) { + log.error('AppSyncOrchestrator - An unverifiable sync response named no peer, cannot replace it'); + return; + } + if (this.#closeRequest(peerKey, 'unverified')) { + log.warn(`AppSyncOrchestrator - ${peerKey} sent a response this node could not verify, asking another peer`); + fluxEventBus.publish('ephemeralSync:peerUnverified', { peer: peerKey }); + } + // Unconditional for the same reason a refusal is: the deficit decides, and + // a pool that is already whole asks nobody. + this.#reconcile(); + } + async #onPeersReady() { if (this.#state === STATES.DEGRADED) { this.#setState(STATES.RESYNCING); @@ -199,7 +590,7 @@ class AppSyncOrchestrator { } this.#startAppRunningBroadcast(); - this.#requestSyncs(); + this.#reconcile(); if (this.#state === STATES.RESYNCING) { if (this.#syncInProgress) return; @@ -208,64 +599,171 @@ class AppSyncOrchestrator { } } - async #requestSyncs() { - const eligible = this.#getEligibleSyncPeers(MIN_UPTIME_SECONDS); - const fresh = eligible.filter((p) => !this.#askedPeers.has(p.key)); + /** + * How many more peers have to be asked for the sync to be able to complete. + * + * Completion needs MIN_SYNC_COMPLETIONS peers to have answered in full, so + * that many requests are outstanding at once - no more, which is what stops + * a boot becoming a second round for every peer that arrives, and no fewer, + * which is what left one waiting on a peer that was never going to reply. + * @returns {number} + */ + #syncDeficit() { + return MIN_SYNC_COMPLETIONS - this.#completedPeerCount() - this.#openRequestCount(); + } - if (fresh.length < MIN_SYNC_COMPLETIONS && this.#askedPeers.size === 0) { - log.info(`AppSyncOrchestrator - Only ${fresh.length} eligible sync peers (need ${MIN_SYNC_COMPLETIONS}), falling back to block timer`); + /** + * Bring the pool of outstanding sync requests back to what it should be. + * + * The only reader-and-writer of the request table. Everything that changes + * what the pool ought to look like - the peer threshold, a peer joining or + * leaving, a refusal, a deadline - says so by calling this, and none of them + * decides anything itself. That is the difference between a level and a + * poke: a trigger that decided would have to know what the other four had + * just done. + * + * A call arriving while a pass is running marks the table dirty and returns, + * and the pass runs again to pick up whatever changed. Serialising them is + * what holds the pool cap: a pass counts the deficit, then fetches a signing + * key before it can reserve anything, so a second one admitted in that window + * would count the same deficit and fill it a second time. The threshold + * crossing emits two triggers from one call, so that window is every boot + * rather than a corner. It also means a burst of joins fetches the key once. + * + * The re-run is not a formality. What lands during a pass is a peer leaving + * or a deadline firing, both of which close a request and widen the deficit + * the pass already counted - so the shortfall it left behind is asked for + * immediately rather than waiting on the next unrelated event. + * + * It terminates: the only thing a pass can do to dirty the table itself is + * lose a peer while writing to it, and that peer is then not a candidate, so + * the loop is bounded by the number of candidates. + * @returns {Promise} + */ + async #reconcile() { + if (this.#reconciling) { + this.#reconcileAgain = true; return; } - - if (fresh.length === 0) { - log.info('AppSyncOrchestrator - No new eligible sync peers to ask'); - return; + this.#reconciling = true; + try { + do { + this.#reconcileAgain = false; + // eslint-disable-next-line no-await-in-loop + await this.#reconcilePass(); + } while (this.#reconcileAgain); + } catch (error) { + // NOBODY IS HOLDING THIS PROMISE. Every one of the five triggers calls + // and returns - a peer joining, a peer leaving, a refusal, a deadline, + // the threshold - and one of them is a timer, so a throw here is a + // rejection with no owner, which node raises to the process handler in + // apiServer and answers by exiting. A failed pass is not evidence that + // the node is broken, and it already has a name: it leaves the pool + // short exactly as the two passes that give up and return do, and the + // next trigger asks again. Caught here rather than at the call sites + // because they all come through here, so a sixth cannot forget. + log.error(`AppSyncOrchestrator - Reconcile pass failed: ${error.message}`); + } finally { + this.#reconciling = false; } + } - const peersToAsk = fresh.slice(0, MIN_SYNC_COMPLETIONS); - - let pubkey; - let requestTs; - let signMsg; + async #reconcilePass() { + if (this.#stateSyncComplete) return; + if (this.#syncBudgetSpent) return; + // Has the sync ever been allowed to start. A latch, and never cleared: + // before the threshold is first crossed there is nobody worth asking. + if (!this.#networkReady || !this.#peersReady) return; + // Are there enough peers to trust an answer RIGHT NOW. A level, and the + // reason the latch is not enough on its own: DEGRADED is this node's own + // verdict that it has too few peers for gossip to be reliable, so a survey + // gathered from them is not one to complete a sync on. + // + // Only the gathering. Authority already earned is not revoked here: a node + // past its block fallback stays authoritative through a degrade, because + // losing peers does not erase what it has already learned and it holds a + // full location lifetime's worth of view. What this stops is a node that + // has NOT earned it taking a short cut to it through the few peers it has + // left. + // + // Recovery needs nothing here: crossing the threshold again moves the + // state to RESYNCING before #onPeersReady reconciles. + if (this.#state === STATES.DEGRADED) return; + + // Counted once, before the key fetch, and it can only be too LOW by the + // time that returns: nothing opens a request but this pass, and the guard + // means no other pass is running. Anything that CLOSES one in the meantime + // - a deadline, a peer leaving - marks the table dirty on its way past, so + // the re-run below asks for whatever this pass left behind. + const open = this.#syncDeficit(); + if (open <= 0) return; + + let signer; try { - pubkey = await fluxNetworkHelper.getFluxNodePublicKey(); - const privkey = await fluxNetworkHelper.getFluxNodePrivateKey(); - requestTs = Date.now(); - signMsg = (type, sinceTs) => { - const msg = peerCodec.buildSyncSignatureMessage(type, sinceTs, requestTs); - return verificationHelper.signMessage(msg, privkey); - }; + signer = await nodeSigner(); + if (!signer) throw new Error('this node cannot sign as itself'); } catch (error) { log.error(`AppSyncOrchestrator - Failed to sign sync requests: ${error.message}`); return; } - for (const peer of peersToAsk) { - this.#askedPeers.add(peer.key); - this.#markSyncRequested(peer.key); + // A peer with any record has had its turn in this attempt, whether it + // answered, declined, ran out of time or dropped. A record is only ever + // dropped when the whole attempt restarts. + const peersToAsk = this.#getEligibleSyncPeers() + .filter((peer) => !this.#requests.has(peer.key)) + .slice(0, open); + + if (!peersToAsk.length) { + log.info(`AppSyncOrchestrator - No peer left to ask, ${open} state-sync ${open === 1 ? 'answer is' : 'answers are'} still needed`); + return; } + const requestTs = Date.now(); + const pubkey = signer.pubKey; + const signMsg = (type, sinceTs) => signer.sign(peerCodec.buildSyncSignatureMessage(type, sinceTs, requestTs)); + + // Every signature is in hand before the first record opens. Signing can + // still fail once the key is known - it answers null rather than throwing - + // and a record opened ahead of one is a peer marked asked with a deadline + // armed and nothing sent. const tempSig = signMsg(peerCodec.MSG_TYPE.REQUEST_TEMP_MESSAGES, 0); const runningSig = signMsg(peerCodec.MSG_TYPE.REQUEST_APP_RUNNING, 0); const installingSig = signMsg(peerCodec.MSG_TYPE.REQUEST_APP_INSTALLING, 0); const errorsSig = signMsg(peerCodec.MSG_TYPE.REQUEST_APP_INSTALLING_ERRORS, 0); + if (!tempSig || !runningSig || !installingSig || !errorsSig) { + log.error('AppSyncOrchestrator - Failed to sign sync requests: this node could not sign as itself'); + return; + } + + for (const peer of peersToAsk) this.#openRequest(peer); + this.#sendRequests(peersToAsk, 'temp messages', peerCodec.encodeRequestTempMessages(0, requestTs, pubkey, tempSig)); this.#sendRequests(peersToAsk, 'apprunning', peerCodec.encodeRequestAppRunning(0, requestTs, pubkey, runningSig)); this.#sendRequests(peersToAsk, 'appinstalling', peerCodec.encodeRequestAppInstalling(0, requestTs, pubkey, installingSig)); this.#sendRequests(peersToAsk, 'apperrors', peerCodec.encodeRequestAppInstallingErrors(0, requestTs, pubkey, errorsSig)); + // OUTSTANDING IS THE POOL CAP ITSELF, and it is published because nothing + // outside can work it out. A round's own size is not the cap - two rounds + // opened in one pass are two events, and a decline is answered here without + // reaching the event stream at all, so the peers named across events cannot + // be added up into the number of requests actually open at any moment. fluxEventBus.publish('ephemeralSync:requested', { peerCount: peersToAsk.length, peers: peersToAsk.map((p) => p.key), + outstanding: this.#openRequestCount(), }); if (!this.#syncTimeout && !this.#stateSyncComplete) { this.#syncTimeout = setTimeout(() => { this.#syncTimeout = null; - this.#clearSyncRequested(); - if (!this.#stateSyncComplete) { - log.warn(`AppSyncOrchestrator - Sync timeout, completions: apprunning=${this.#syncCompletions.apprunning} appinstalling=${this.#syncCompletions.appinstalling} apperrors=${this.#syncCompletions.apperrors}`); - } + if (this.#stateSyncComplete) return; + this.#closeRound('the budget ran out'); + this.#syncBudgetSpent = true; + for (const peerKey of [...this.#requests.keys()]) this.#discardRequest(peerKey); + const answered = Object.entries(this.#completionCounts()) + .map(([type, count]) => `${type}=${count}`).join(' '); + log.warn(`AppSyncOrchestrator - Sync timeout, peers answered: ${answered}`); }, SYNC_TIMEOUT_MS); } } @@ -294,10 +792,14 @@ class AppSyncOrchestrator { } #resetSyncState() { - this.#askedPeers.clear(); - this.#clearSyncRequested(); - this.#syncCompletions = { apprunning: 0, appinstalling: 0, apperrors: 0 }; + // Everything asked in the round that is ending is forgotten outright, not + // set aside: the sync starts over, so a peer already tried is a peer to + // try again rather than one to skip. + for (const peerKey of [...this.#requests.keys()]) this.#discardRequest(peerKey); + this.#syncBudgetSpent = false; + this.#syncCompletions = freshSyncCompletions(); this.#stateSyncComplete = false; + this.#publishStateSyncAuthority(); this.#hashSyncAttempts = 0; if (this.#syncTimeout) { clearTimeout(this.#syncTimeout); @@ -317,12 +819,12 @@ class AppSyncOrchestrator { log.info(`AppSyncOrchestrator - Explorer synced at block ${blockHeight}`); if (this.#state === STATES.INITIALIZING) { this.#setState(STATES.SYNCING); - this.#ensureBlockThreshold(); this.#runInitialSync(); } } if (this.#state === STATES.SYNCING || this.#state === STATES.READY || this.#state === STATES.RESYNCING) { this.#blocksSinceSyncStarted += count; + this.#publishStateSyncAuthority(); this.#checkReadiness(); this.#checkHashRetry(blockHeight); } @@ -451,19 +953,20 @@ class AppSyncOrchestrator { } } - #ensureBlockThreshold() { - if (this.#blockThreshold === 0) { - const enterprise = this.#isEnterprise(); - const blocksPerMinute = 2; - this.#blockThreshold = enterprise - ? 62 * blocksPerMinute - : 125 * blocksPerMinute; - } - } - + // ONE NUMBER, and it is not a preference. FALLBACK_MINUTES is the lifetime of + // a running-app location record, so it is the point at which every holder has + // had to announce itself at least once: wait it out and what this node holds + // is a full view, whether or not a sync ever completed. + // + // There used to be a second, shorter value for enterprise nodes, halved in + // the manner of the spawner's enterprise deferrals. Those are a priority - + // how long before a node may compete for an app - and halving one grants an + // advantage. This is not that: it is how long before a node assumes it knows + // what the network looks like, and there is no advantage in assuming it + // sooner. A node can be given priority; it cannot be given information it has + // not received. #isBlockTimerExpired() { - this.#ensureBlockThreshold(); - return this.#blocksSinceSyncStarted >= this.#blockThreshold; + return this.#blocksSinceSyncStarted >= FALLBACK_MINUTES * BLOCKS_PER_MINUTE; } #isStateSyncReady() { @@ -471,6 +974,20 @@ class AppSyncOrchestrator { return this.#isBlockTimerExpired(); } + /** + * Mirror the state-sync verdict where the sync responder can read it. + * + * Called wherever an input to #isStateSyncReady moves, so the value never + * disagrees with the rule. A peer asking us for app state gets a refusal + * while this is false, because an empty answer from a node that does not yet + * know is indistinguishable from an empty answer from a node that does - and + * the asker counts both as a completed survey. + * @returns {void} + */ + #publishStateSyncAuthority() { + globalState.appStateAuthoritative = this.#isStateSyncReady(); + } + async #checkReadiness() { if (this.#state !== STATES.SYNCING && this.#state !== STATES.RESYNCING) return; if (!this.#explorerSynced) return; @@ -518,7 +1035,7 @@ class AppSyncOrchestrator { this.#broadcastStarted = true; log.info('AppSyncOrchestrator - App running broadcast started'); await globalState.waitForBootContainerStateSettled(); - peerNotification.checkAndNotifyPeersOfRunningApps(); + peerNotification.startBroadcasting(); } get bootContext() { @@ -612,7 +1129,16 @@ class AppSyncOrchestrator { } } - stop() { + /** + * Tear the orchestrator down, and do not return until it is torn down. + * + * Async because the announcement loop's stop waits for the cycle in flight: + * a teardown that returns while that cycle is still running leaves it to + * finish against services this method has already taken apart. + * + * @returns {Promise} + */ + async stop() { this.#started = false; if (this.#heartbeatInterval) { clearInterval(this.#heartbeatInterval); @@ -621,6 +1147,15 @@ class AppSyncOrchestrator { if (this.#ephemeralSyncHandler) { appSyncEvents.removeListener(EVENTS.EPHEMERAL_SYNC_COMPLETE, this.#ephemeralSyncHandler); } + if (this.#ephemeralProgressHandler) { + appSyncEvents.removeListener(EVENTS.EPHEMERAL_SYNC_PROGRESS, this.#ephemeralProgressHandler); + } + if (this.#ephemeralRefusedHandler) { + appSyncEvents.removeListener(EVENTS.EPHEMERAL_SYNC_REFUSED, this.#ephemeralRefusedHandler); + } + if (this.#ephemeralUnverifiedHandler) { + appSyncEvents.removeListener(EVENTS.EPHEMERAL_SYNC_UNVERIFIED, this.#ephemeralUnverifiedHandler); + } if (this.#hashUnresolvedHandler) { appSyncEvents.removeListener(EVENTS.HASH_UNRESOLVED, this.#hashUnresolvedHandler); } @@ -636,7 +1171,14 @@ class AppSyncOrchestrator { if (this.#peersBelowHandler) { this.#offPeerEvent('peersBelowThreshold', this.#peersBelowHandler); } - peerNotification.stopBroadcastInterval(); + if (this.#peerConnectedHandler) { + this.#offPeerEvent('peerConnected', this.#peerConnectedHandler); + } + if (this.#peerDisconnectedHandler) { + this.#offPeerEvent('peerDisconnected', this.#peerDisconnectedHandler); + } + for (const peerKey of [...this.#requests.keys()]) this.#discardRequest(peerKey); + await peerNotification.stopBroadcasting(); this.#broadcastStarted = null; if (this.#syncTimeout) { clearTimeout(this.#syncTimeout); @@ -646,6 +1188,11 @@ class AppSyncOrchestrator { clearTimeout(this.#hashSyncRetryTimer); this.#hashSyncRetryTimer = null; } + // Authority belongs to a running orchestrator. It is this node's claim to + // know what the network runs, and the two guards answering a peer's sync + // request read it - so leaving it set serves a survey drawn from state + // nothing is maintaining any more. + globalState.appStateAuthoritative = false; } } diff --git a/ZelBack/src/services/appMessaging/cryptographicKeys.js b/ZelBack/src/services/appMessaging/cryptographicKeys.js index d282b8e493..a35775536a 100644 --- a/ZelBack/src/services/appMessaging/cryptographicKeys.js +++ b/ZelBack/src/services/appMessaging/cryptographicKeys.js @@ -5,6 +5,7 @@ const serviceHelper = require('../serviceHelper'); const benchmarkService = require('../benchmarkService'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const log = require('../../lib/log'); +const { Privilege, authOf } = require('../utils/privileges'); // Check if running on Arcane OS const isArcane = Boolean(process.env.FLUXOS_PATH); @@ -54,7 +55,7 @@ async function getPublicKey(req, res) { }); req.on('end', async () => { try { - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); diff --git a/ZelBack/src/services/appMessaging/messageStore.js b/ZelBack/src/services/appMessaging/messageStore.js index 1f62528f67..72fbb053cb 100644 --- a/ZelBack/src/services/appMessaging/messageStore.js +++ b/ZelBack/src/services/appMessaging/messageStore.js @@ -19,7 +19,7 @@ const { globalAppStateEvents, appsHashesCollection, } = require('../utils/appConstants'); -const appsInstallingBroadcasts = config.database.appsglobal.collections.appsInstallingBroadcasts; +const { appsInstallingBroadcasts } = config.database.appsglobal.collections; const { specificationFormatter } = require('../utils/appSpecHelpers'); const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../utils/appSyncEvents'); @@ -31,6 +31,11 @@ const { EVICTED_EXPIRY_MS, } = require('../utils/appConstants'); +// Cap on how many location operations are handed to one bulk write. The driver +// encodes the whole batch before sending it, and that encoding is what a large +// sync response leaves behind in memory. +const LOCATION_OPS_PER_WRITE = 500; + const APP_STATE_EVENT_TYPES = Object.freeze({ APPRUNNING: 'apprunning', SIGTERM: 'sigterm', @@ -383,7 +388,22 @@ async function storeAppRunningMessage(message) { } /** - * Store app installing message + * Store app installing message, or apply a version 2 withdrawal of one. + * + * A node claims an app before it knows whether it is needed - the claim is what + * lets every contender see the contention - so losing that race is ordinary and + * has to be retractable. Version 2 is that retraction: it carries + * `withdrawn: true` and removes the sender's claim instead of recording one. + * + * A claim stays at version 1 on purpose. Moving claims to 2 would have nodes + * that do not know the version reject them, and they would stop seeing + * contention at all - so the retraction is what carries the new version, and a + * node that rejects it simply lets the claim expire as it does today. + * + * Never an installing ERROR. That message means an install was attempted and + * failed, it is counted as such, and a node standing aside has attempted + * nothing - counting it would turn the apps most in demand, whose races have + * the most losers, into the apps that look most broken. * @param {object} message - Message to store * @returns {Promise} Whether message should be rebroadcast or Error if invalid */ @@ -394,16 +414,23 @@ async function storeAppInstallingMessage(message) { * @param broadcastedAt number * @param name string * @param ip string + * @param withdrawn boolean - version 2 only, always true */ if (!message || typeof message !== 'object' || typeof message.type !== 'string' || typeof message.version !== 'number' || typeof message.broadcastedAt !== 'number' || typeof message.ip !== 'string' || typeof message.name !== 'string') { return new Error('Invalid Flux App Installing message for storing'); } - if (message.version !== 1) { + if (message.version !== 1 && message.version !== 2) { return new Error(`Invalid Flux App Installing message for storing version ${message.version} not supported`); } + // version 2 exists only to withdraw; anything else at that version is not + // something this protocol emits + if (message.version === 2 && message.withdrawn !== true) { + return new Error('Invalid Flux App Installing message for storing version 2 must be a withdrawal'); + } + if (message.broadcastedAt + GOSSIP_VALIDITY_MS < Date.now()) { log.warn(`Rejecting old/not valid fluxappinstalling message, message:${JSON.stringify(message)}`); return false; @@ -430,6 +457,30 @@ async function storeAppInstallingMessage(message) { return false; } + if (message.version === 2) { + // The comparison above is what makes a withdrawal safe to apply late: a node + // may claim, stand aside, and claim again on a later pass, and a withdrawal + // that arrives after the newer claim must not erase it. Reaching here means + // the stored claim is older than this withdrawal, so it is the one being + // retracted. Nothing is recorded in its place - the sender holds no claim. + // Carried on both deletes rather than rested on the read above. The read + // proves the stored claim was older a moment ago; the guard proves it at the + // moment of deletion, which is what the batch path does and what closes the + // window where a newer claim lands in between. The broadcast row needs it in + // its own right too - it is a separate collection written by a separate path, + // so a claim newer than this withdrawal can exist there while the location + // the read consulted is still the old one. + const olderThanWithdrawal = { broadcastedAt: { $lt: newAppInstallingMessage.broadcastedAt } }; + await dbHelper.removeDocumentsFromCollection( + database, globalAppsInstallingLocations, { ...queryFind, ...olderThanWithdrawal }, + ); + await dbHelper.removeDocumentsFromCollection( + database, appsInstallingBroadcasts, + { 'data.name': message.name, 'data.ip': message.ip, ...olderThanWithdrawal }, + ); + return true; + } + const queryUpdate = { name: newAppInstallingMessage.name, ip: newAppInstallingMessage.ip }; const update = { $set: newAppInstallingMessage }; const options = { @@ -600,14 +651,28 @@ async function storeIPChangedMessage(message) { } async function storeBatchAppRunningMessages(verifiedBroadcasts) { - if (verifiedBroadcasts.length === 0) return { stored: 0 }; + if (verifiedBroadcasts.length === 0) return { stored: 0, writeFailed: false }; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); const { stored } = await storeBatchAppRunningEvents(verifiedBroadcasts); + // One operation is built per app per broadcast and each carries a merge + // pipeline, so a sync response of a few thousand broadcasts expands into tens + // of thousands of them. Writing in bounded batches keeps both this array and + // the driver's encoding of it small; the operations are independent of one + // another, which is what makes the unordered write safe to split. const locationOps = []; - const v2AppsByIp = new Map(); + let writeFailed = false; + const flushLocationOps = async () => { + if (!locationOps.length) return; + const batch = locationOps.splice(0, locationOps.length); + await database.collection(globalAppsLocations).bulkWrite(batch, { ordered: false }) + .catch((err) => { + writeFailed = true; + log.error(`storeBatchAppRunningMessages locations: ${err.message}`); + }); + }; for (const broadcast of verifiedBroadcasts) { const { data } = broadcast; @@ -615,12 +680,6 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { if (validTill < Date.now()) continue; const apps = data.version === 2 ? (data.apps || []) : [{ name: data.name, hash: data.hash }]; - if (data.version === 2 && apps.length > 0) { - const existing = v2AppsByIp.get(data.ip); - if (!existing || data.broadcastedAt > existing.broadcastedAt) { - v2AppsByIp.set(data.ip, { names: apps.map((a) => a.name), broadcastedAt: data.broadcastedAt }); - } - } const incomingDate = new Date(data.broadcastedAt); const incomingExpiry = new Date(validTill); const isNewer = { $gt: [incomingDate, { $ifNull: ['$broadcastedAt', new Date(0)] }] }; @@ -645,24 +704,61 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { upsert: true, }, }); + + if (locationOps.length >= LOCATION_OPS_PER_WRITE) { + // eslint-disable-next-line no-await-in-loop + await flushLocationOps(); + } } } - for (const [ip, { names, broadcastedAt }] of v2AppsByIp) { - const cutoff = new Date(broadcastedAt); - locationOps.push({ + await flushLocationOps(); + + // Upserts only. Removing the rows a node no longer reports belongs to + // pruneAppRunningLocations, which is driven from the newest broadcast per node + // rather than from whatever happened to arrive in this batch. + return { stored, writeFailed }; +} + +/** + * Drop location rows a node no longer reports. + * + * Only the newest broadcast from a node says which apps it still runs, so this + * cannot be done from part of a sync response - a slice holding an older + * broadcast would prune against a stale app list, and one holding a newer + * broadcast would leave rows an earlier slice had already written. The caller + * passes the newest broadcast seen per node across the whole response. + * + * @param {Map, broadcastedAt: number}>} newestByIp Newest broadcast per node. + * @returns {Promise} + */ +async function pruneAppRunningLocations(newestByIp) { + if (!newestByIp || newestByIp.size === 0) return; + + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + const ops = []; + + const flush = async () => { + if (!ops.length) return; + const batch = ops.splice(0, ops.length); + await database.collection(globalAppsLocations).bulkWrite(batch, { ordered: false }) + .catch((err) => log.error(`pruneAppRunningLocations: ${err.message}`)); + }; + + for (const [ip, { names, broadcastedAt }] of newestByIp) { + ops.push({ deleteMany: { - filter: { ip, name: { $nin: names }, broadcastedAt: { $lte: cutoff } }, + filter: { ip, name: { $nin: names }, broadcastedAt: { $lte: new Date(broadcastedAt) } }, }, }); + if (ops.length >= LOCATION_OPS_PER_WRITE) { + // eslint-disable-next-line no-await-in-loop + await flush(); + } } - if (locationOps.length > 0) { - await database.collection(globalAppsLocations).bulkWrite(locationOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppRunningMessages locations: ${err.message}`)); - } - - return { stored }; + await flush(); } // --- Event Log Functions --- @@ -858,11 +954,30 @@ async function storeBatchAppInstallingMessages(verifiedBroadcasts) { const signedOps = []; const locationOps = []; + const withdrawalOps = []; + for (const broadcast of verifiedBroadcasts) { const { data } = broadcast; const validTill = data.broadcastedAt + INSTALLING_EXPIRY_MS; if (validTill < Date.now()) continue; + // A version 2 message withdraws its sender's claim. Reaching the claim path + // with one would store the very claim it retracts, so it is deleted here + // instead - and only where the stored claim is older, so a withdrawal that + // arrives after the sender has claimed again cannot erase the newer claim. + if (data.version === 2) { + // version 2 exists only to withdraw; anything else at that version is + // not something this protocol emits. The single-message path refuses + // it, and this path must not read it as a withdrawal. + if (data.withdrawn !== true) continue; + withdrawalOps.push({ + deleteOne: { + filter: { name: data.name, ip: data.ip, broadcastedAt: { $lt: new Date(data.broadcastedAt) } }, + }, + }); + continue; + } + signedOps.push({ updateOne: { filter: { 'data.name': data.name, 'data.ip': data.ip }, @@ -907,7 +1022,29 @@ async function storeBatchAppInstallingMessages(verifiedBroadcasts) { await database.collection(globalAppsInstallingLocations).bulkWrite(locationOps, { ordered: false }) .catch((err) => log.error(`storeBatchAppInstallingMessages locations: ${err.message}`)); } - return { stored: signedOps.length }; + // after the claims, so a claim and its withdrawal arriving in one batch settle + // in the order they were sent rather than the order they were read + if (withdrawalOps.length > 0) { + await database.collection(globalAppsInstallingLocations).bulkWrite(withdrawalOps, { ordered: false }) + .catch((err) => log.error(`storeBatchAppInstallingMessages withdrawals: ${err.message}`)); + await database.collection(appsInstallingBroadcasts).bulkWrite( + // The location's own guard, carried over rather than restated: the + // broadcast is what proves the location, so deleting one without the + // other leaves a claim nothing can serve during sync. Same object, so + // the two cannot come apart. + withdrawalOps.map((op) => ({ + deleteOne: { + filter: { + 'data.name': op.deleteOne.filter.name, + 'data.ip': op.deleteOne.filter.ip, + broadcastedAt: op.deleteOne.filter.broadcastedAt, + }, + }, + })), + { ordered: false }, + ).catch((err) => log.error(`storeBatchAppInstallingMessages withdrawal broadcasts: ${err.message}`)); + } + return { stored: signedOps.length + withdrawalOps.length }; } function storeSignedAppInstallingErrorBroadcast(signedBroadcast) { @@ -1002,6 +1139,7 @@ module.exports = { storeAppPermanentMessage, storeAppRunningMessage, storeBatchAppRunningMessages, + pruneAppRunningLocations, storeAppStateEvent, storeBatchAppRunningEvents, APP_STATE_EVENT_TYPES, diff --git a/ZelBack/src/services/appMessaging/messageVerifier.js b/ZelBack/src/services/appMessaging/messageVerifier.js index 337844b07b..b44dc3f739 100644 --- a/ZelBack/src/services/appMessaging/messageVerifier.js +++ b/ZelBack/src/services/appMessaging/messageVerifier.js @@ -26,6 +26,41 @@ const { } = require('../utils/appConstants'); const fluxNetworkHelper = require('../fluxNetworkHelper'); const globalState = require('../utils/globalState'); +const { Privilege, authOf } = require('../utils/privileges'); + +/** + * The support team addresses a teamSupportAddress fork names. + * + * A fork used to name one address and now names a list, and both shapes have to + * be read: the forks already in force were written under the old one, and a + * message is verified against the fork in force at its own block. Rewriting those + * entries to the new shape would change which signatures the past accepts, so + * they stay as they are and this reads either. + * + * @param {{address?: string, addresses?: string[]}} fork + * @returns {string[]} + */ +function supportAddressesOf(fork) { + if (Array.isArray(fork.addresses)) return fork.addresses.filter(Boolean); + return fork.address ? [fork.address] : []; +} + +/** + * Whether any of these addresses signed the message. + * + * Stops at the first that did, so a message signed by the first address costs + * exactly what it cost when a fork could only name one. + * + * @param {string} message + * @param {string[]} addresses + * @param {string} signature + * @returns {boolean} + */ +function verifySignatureFromAny(message, addresses, signature) { + return addresses.some( + (address) => signatureVerifier.verifySignature(message, address, signature) === true, + ); +} /** * Verify app hash against message content @@ -255,7 +290,7 @@ async function verifyAppMessageUpdateSignature(type, version, appSpec, timestamp // signature is already validated as string in the if check above, no need to ensureString let marketplaceApp = false; - let fluxSupportTeamFluxID = null; + let supportTeamAddresses = []; const messageToVerify = type + version + JSON.stringify(appSpec) + timestamp; let isValidSignature = signatureVerifier.verifySignature(messageToVerify, appOwner, signature); // btc, eth if (isValidSignature !== true) { @@ -265,7 +300,7 @@ async function verifyAppMessageUpdateSignature(type, version, appSpec, timestamp if (intervals && intervals.length) { const addressInfo = intervals[intervals.length - 1]; // always defined if (addressInfo && addressInfo.height && daemonHeight >= addressInfo.height) { // unneeded check for safety - fluxSupportTeamFluxID = addressInfo.address; + supportTeamAddresses = supportAddressesOf(addressInfo); const numbersOnAppName = appSpec.name.match(/\d+/g); if (numbersOnAppName && numbersOnAppName.length > 0) { const dateBeforeReleaseMarketplace = Date.parse('2020-01-01'); @@ -277,7 +312,7 @@ async function verifyAppMessageUpdateSignature(type, version, appSpec, timestamp } } if (marketplaceApp) { - isValidSignature = signatureVerifier.verifySignature(messageToVerify, fluxSupportTeamFluxID, signature); // btc, eth + isValidSignature = verifySignatureFromAny(messageToVerify, supportTeamAddresses, signature); // btc, eth } } } @@ -306,7 +341,7 @@ async function verifyAppMessageUpdateSignature(type, version, appSpec, timestamp const messageToVerifyB = type + version + JSON.stringify(appSpecOld) + timestamp; isValidSignature = signatureVerifier.verifySignature(messageToVerifyB, appOwner, signature); // btc, eth if (isValidSignature !== true && marketplaceApp) { - isValidSignature = signatureVerifier.verifySignature(messageToVerifyB, fluxSupportTeamFluxID, signature); // btc, eth + isValidSignature = verifySignatureFromAny(messageToVerifyB, supportTeamAddresses, signature); // btc, eth } // fix for repoauth / secrets order change for apps created after 1750273721000 } else if (isValidSignature !== true && appSpec.version === 7) { @@ -325,7 +360,7 @@ async function verifyAppMessageUpdateSignature(type, version, appSpec, timestamp // we can just use the btc / eth verifier as v7 specs came out at 1688749251 isValidSignature = signatureVerifier.verifySignature(messageToVerifyC, appOwner, signature); if (isValidSignature !== true && marketplaceApp) { - isValidSignature = signatureVerifier.verifySignature(messageToVerifyC, fluxSupportTeamFluxID, signature); + isValidSignature = verifySignatureFromAny(messageToVerifyC, supportTeamAddresses, signature); } } @@ -426,7 +461,7 @@ async function requestAppsMessage(apps, incoming) { async function requestAppMessageAPI(req, res) { try { // only flux team and node owner can do this - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); diff --git a/ZelBack/src/services/appMessaging/peerNotification.js b/ZelBack/src/services/appMessaging/peerNotification.js index dd07a396e8..6056d38fc5 100644 --- a/ZelBack/src/services/appMessaging/peerNotification.js +++ b/ZelBack/src/services/appMessaging/peerNotification.js @@ -13,30 +13,113 @@ const appQueryService = require('../appQuery/appQueryService'); const appReconciler = require('../appMonitoring/appReconciler'); const fluxEventBus = require('../utils/fluxEventBus'); +const { nodeSigner } = require('../utils/nodeSigner'); +const { ANNOUNCE_INTERVAL_MS } = require('../utils/appConstants'); +const { AsyncLock } = require('../utils/asyncLock'); const globalAppsLocations = config.database.appsglobal.collections.appsLocations; -let broadcastInterval = null; +let broadcastTimer = null; let broadcastInProgress = false; let rebroadcastNeeded = false; +let overrunning = false; -function resetBroadcastInterval() { - if (broadcastInterval) clearInterval(broadcastInterval); - broadcastInterval = setInterval(() => { +// Whether this node announces at all, and the only thing that decides whether a +// cycle arms the next one. A cycle ends by scheduling its successor, so a stop +// that clears the pending timer alone is undone a moment later by the work it +// was trying to end - the announcement loop outlives every stop taken while it +// is running. +// +// Ours rather than an abort signal's: FluxController's `aborted` is reset the +// moment its lock frees, so which of the two resumes first decides whether the +// loop survives its own abort. Nothing resets this. +let broadcasting = false; + +// Held for the whole of a cycle, so a stop can wait for the cycle in flight +// rather than returning while it is still running. A teardown that returns +// before the thing is torn down is the same lie as a stop that does not stop. +const cycleLock = new AsyncLock(); + +/** + * Schedule the next announcement so that the PERIOD is fixed, rather than the + * gap between one cycle ending and the next beginning. + * + * The cycle's own duration is subtracted, so the work does not sit inside the + * thing it is timed against: a timer recreated after the cycle makes the real + * period `interval + however long the cycle took`, and a node announcing every + * 70s while its configuration says 30 keeps a row alive that expires at 63. + * + * Measured monotonically. A wall clock can step backwards over an NTP + * correction, and a negative elapsed would push the next announcement away by + * the size of the step. + * + * @param {bigint} startedAt process.hrtime.bigint() taken when the cycle began + */ +function scheduleNextBroadcast(startedAt) { + if (broadcastTimer) clearTimeout(broadcastTimer); + // Asked here rather than at the caller, because every path that ends a cycle + // arrives here and a stop must be honoured by all of them. + if (!broadcasting) { + broadcastTimer = null; + return; + } + + const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1e6; + + // Said once on the transition and once when it clears, never per cycle. A + // node whose cycle no longer fits its interval announces itself less often + // than the row it keeps alive, and its presence on the network decays with + // nothing anywhere saying so. + if (elapsedMs > ANNOUNCE_INTERVAL_MS) { + if (!overrunning) { + overrunning = true; + log.warn(`peerNotification - a broadcast cycle took ${Math.round(elapsedMs)}ms against a ${ANNOUNCE_INTERVAL_MS}ms interval; this node is announcing itself less often than the location row it refreshes`); + } + } else if (overrunning) { + overrunning = false; + log.info('peerNotification - broadcast cycles fit inside their interval again'); + } + + // Clamped at zero rather than scheduled into the past. A cycle that outran + // its interval runs again immediately, which is the most the node can do. + broadcastTimer = setTimeout(() => { checkAndNotifyPeersOfRunningApps(); - }, config.fluxapps.peerNotifyIntervalMs ?? 3600000); + }, Math.max(0, ANNOUNCE_INTERVAL_MS - elapsedMs)); +} + +/** + * Start announcing, and keep announcing. + * + * Idempotent: a second call while the loop is running is not a second loop. + * + * @returns {void} + */ +function startBroadcasting() { + if (broadcasting) return; + broadcasting = true; + checkAndNotifyPeersOfRunningApps(); } -function stopBroadcastInterval() { - if (broadcastInterval) { - clearInterval(broadcastInterval); - broadcastInterval = null; +/** + * Stop announcing, and return once the cycle in flight has finished. + * + * @returns {Promise} + */ +async function stopBroadcasting() { + broadcasting = false; + if (broadcastTimer) { + clearTimeout(broadcastTimer); + broadcastTimer = null; } + // The queued repeat goes with the stop. Left set, it is a cycle the next + // start would run before the one it schedules for itself. + rebroadcastNeeded = false; + await cycleLock.waitReady(); } function initialize() { nodeConfirmationService.onMessageCapabilityChange((capable) => { - if (capable && broadcastInterval) { + if (capable && broadcasting) { log.info('peerNotification - Message capability regained, triggering immediate broadcast'); checkAndNotifyPeersOfRunningApps(); } @@ -50,6 +133,10 @@ async function checkAndNotifyPeersOfRunningApps() { return; } broadcastInProgress = true; + // Taken before any work, so the schedule below subtracts the WHOLE cycle - + // including the paths that give up early, which cost time too. + const startedAt = process.hrtime.bigint(); + await cycleLock.enable(); try { if (!nodeConfirmationService.canSendMessages()) { log.info('checkAndNotifyPeersOfRunningApps - Node cannot send messages, skipping broadcast'); @@ -73,7 +160,7 @@ async function checkAndNotifyPeersOfRunningApps() { throw new Error('Failed to get installed Apps'); } let appsInstalled = installedAppsRes.data; - appsInstalled = await decryptEnterpriseApps(appsInstalled, { formatSpecs: false }); + ({ inPlace: appsInstalled } = await decryptEnterpriseApps(appsInstalled, { formatSpecs: false })); const runningAppsRes = await appQueryService.listRunningApps(); if (runningAppsRes.status !== 'success') { throw new Error('Unable to check running Apps'); @@ -154,6 +241,16 @@ async function checkAndNotifyPeersOfRunningApps() { osUptime: os.uptime(), staticIp: geolocationService.isStaticIP(), }; + // The announcement is one fact, recorded twice - the location table, and + // the event log that peers sync from - and sent once. A node that cannot + // sign as itself sends nothing a peer would accept, so it records nothing + // either: its own view of where it runs stays the network's view. Asked + // before the first write, for that reason. + const signer = await nodeSigner(); + if (!signer) { + log.warn('checkAndNotifyPeersOfRunningApps - this node cannot sign as itself; its running apps are not announced'); + return; + } await messageStore.storeAppRunningMessage(appRunningMessage); const signed = await fluxCommunicationMessagesSender.broadcastMessageToAll(appRunningMessage); await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed }); @@ -162,7 +259,7 @@ async function checkAndNotifyPeersOfRunningApps() { } catch (err) { log.error(err); } - const runningAppsCache = globalState.runningAppsCache; + const { runningAppsCache } = globalState; runningAppsCache.clear(); apps.forEach((app) => { runningAppsCache.add(app.name); @@ -173,11 +270,12 @@ async function checkAndNotifyPeersOfRunningApps() { log.error(error); } finally { broadcastInProgress = false; + cycleLock.disable(); if (rebroadcastNeeded) { rebroadcastNeeded = false; setImmediate(() => checkAndNotifyPeersOfRunningApps()); } else { - resetBroadcastInterval(); + scheduleNextBroadcast(startedAt); } } } @@ -185,5 +283,6 @@ async function checkAndNotifyPeersOfRunningApps() { module.exports = { initialize, checkAndNotifyPeersOfRunningApps, - stopBroadcastInterval, + startBroadcasting, + stopBroadcasting, }; diff --git a/ZelBack/src/services/appMonitoring/appReconciler.js b/ZelBack/src/services/appMonitoring/appReconciler.js index ff74745869..00eb4959b7 100644 --- a/ZelBack/src/services/appMonitoring/appReconciler.js +++ b/ZelBack/src/services/appMonitoring/appReconciler.js @@ -4,7 +4,6 @@ const fluxEventBus = require('../utils/fluxEventBus'); const dbHelper = require('../dbHelper'); const serviceHelper = require('../serviceHelper'); const dockerService = require('../dockerService'); -const dockerOperations = require('../appManagement/dockerOperations'); const volumeService = require('../utils/volumeService'); const mountParser = require('../utils/mountParser'); const globalState = require('../utils/globalState'); @@ -44,14 +43,25 @@ const controllerDesired = new Map(); // a start can never race it. const dataDesired = new Map(); +// Components a decider has committed to running but has not started yet, because +// its pre-start data-safety work is still in flight. Read by the peer probe: a +// node that answers only with running containers withholds an intent it already +// holds, and the asking node starts a second writer. In-memory for the same +// reason as controllerDesired - a claim must not survive the process that made it. +const startingClaims = new Set(); + // brief settle between the stop and the rm -rf so the container has fully released // its appdata mount before the wipe (mirrors the sync layer's prior 500ms delay). const DATA_CLEAR_SETTLE_MS = 500; -const inFlight = new Set(); // ids currently reconciling (per-key single-flight) +// id -> promise of the pass (or intent write) currently holding this key. A Map +// rather than a Set so a caller can WAIT for the holder: applyIntent below needs +// to know when a pass has finished, not merely that one is running. +const inFlight = new Map(); // per-key single-flight const dirty = new Set(); // ids re-requested while in flight -> reconcile again const bootPending = new Set(); // ids enqueued before the boot gate opened const backoffTimers = new Map(); // id -> scheduled retry timeout +const unhandledFailures = new Map(); // id -> consecutive passes that threw // The boot-drain gate: opens once every boot-held component has completed ONE // reconcile pass (started, backoff-deferred, awaiting-controller, or failed @@ -100,6 +110,18 @@ function notifyContainerStarted(identifier) { // completion, so this is just a backstop) const MANAGED_RETRY_MS = 5000; +// How many consecutive passes may throw before the component is left to the +// hourly sweep. A pass that throws is by definition one whose failure nobody +// anticipated: every failure this file expects returns after deciding either to +// pace a retry (a transient fault) or deliberately not to (an invalid spec, which +// no retry can fix). An unhandled throw made neither decision, so it reached the +// sweep - and with the operator's stop now actuated here rather than inline, a +// container could keep running for an hour after a stop reported success. +// Retrying is safe because a pass is level-based: it re-derives desired against +// actual rather than resuming half-finished work. The bound is what keeps a +// permanent fault from becoming a five-second log loop forever. +const UNHANDLED_FAILURE_RETRIES = 3; + // an unmountable volume usually means its host filesystem is still coming up // (e.g. the encrypted data partition after a reboot) - retry on a pace that // won't spam, and keep deferring until it mounts @@ -222,17 +244,18 @@ async function getLocalComponentSpec(identifier) { throw error; } if (!appSpec) return null; - try { - [appSpec] = await appQueryService.decryptEnterpriseApps([appSpec], { formatSpecs: false, throwOnError: true }); - } catch (err) { + const { readable: [decryptedSpec] } = await appQueryService + .decryptEnterpriseApps([appSpec], { formatSpecs: false }); + if (!decryptedSpec) { // Decryption failed (e.g. the enterprise key isn't loaded yet at boot). Never // proceed on still-encrypted data: containerData would be unreadable, so we'd // misclassify g:/r: or start a container on garbage. Treat as transient like a // DB read failure so reconcile defers and retries once the key is available. - const error = new Error(`failed to decrypt enterprise spec for ${identifier}: ${err.message}`); + const error = new Error(`failed to decrypt enterprise spec for ${identifier}`); error.transient = true; throw error; } + appSpec = decryptedSpec; let comp; if (appSpec.version >= 4 && Array.isArray(appSpec.compose)) { @@ -296,8 +319,23 @@ async function dockerActual(identifier) { return { reachable: true, exists: true, - running: !!(info.State && info.State.Running), + // A PAUSED container reports Running: true - docker freezes the processes + // and leaves the record saying they are up. Reading that as running is how + // a frozen container became invisible to everything: this function said + // "healthy, nothing to do", the sampler skipped it so its charts flatlined + // with no explanation, and FDM went on routing traffic to something that + // would never answer. + // + // It also put this function at odds with appStartupManager, which + // enumerates boot candidates from the LISTING's State (where paused is not + // 'running'). Boot handed the container over saying it needed starting and + // this said it was already up - every boot, forever. The two now agree. + running: !!(info.State && info.State.Running) && !info.State.Paused, + paused: !!(info.State && info.State.Paused), exitCode: everRan ? (info.State.ExitCode ?? null) : null, + // the kernel's verdict, not the entrypoint's: an image that swallows its + // payload's status still cannot hide this one + oomKilled: !!(info.State && info.State.OOMKilled), finishedAt, // classified from THIS inspect so the running-branch network check needs no // second docker call (and no TOCTOU between two inspects). @@ -340,8 +378,21 @@ function isManagedElsewhere(identifier) { return false; } +/** + * What this component should be doing, why, and - when the answer is an operator + * stop - whether that operator asked for a hard kill. + * + * The lock and the force flag are fields of one document, so they come from one + * read. The stop branch used to re-read the same document for the flag alone, + * and getState returns null for a read failure exactly as it does for "no + * record" - so a second read that failed turned "kill now" into a drain. One + * read is a window that cannot open, and one round-trip fewer on every stop. + * + * @returns {Promise<{desired: boolean|null, reason: string, force: boolean}>} + */ async function effectiveDesiredRunning(identifier, spec, exitCode) { - if (await appsRuntimeState.isOperatorStopped(identifier)) return { desired: false, reason: 'operatorStopped' }; + const operatorStop = await appsRuntimeState.operatorStopState(identifier); + if (operatorStop.stopped) return { desired: false, reason: 'operatorStopped', force: operatorStop.force }; if (spec.isG || spec.isR) { const cd = controllerDesired.get(identifier) ?? null; // No controller opinion yet. controllerDesired is in-memory, so a FluxOS @@ -349,11 +400,36 @@ async function effectiveDesiredRunning(identifier, spec, exitCode) { // the FluxOS process). Take no action - leave the container as-is until the // masterSlave/syncthing decider re-derives intent. Treating "unset" as "stop" // here would bounce every running syncthing app on every FluxOS restart. - if (cd === null) return { desired: null, reason: 'awaitingController' }; - if (cd !== 'running') return { desired: false, reason: 'controllerDesired' }; + if (cd === null) return { desired: null, reason: 'awaitingController', force: false }; + if (cd !== 'running') return { desired: false, reason: 'controllerDesired', force: false }; } const desired = policyAllowsRun(getRestartPolicy(spec), exitCode); - return { desired, reason: desired ? 'running' : 'policy' }; + // Only an operator asks for a hard kill; every other stop reason is a drain. + return { desired, reason: desired ? 'running' : 'policy', force: false }; +} + +/** + * The run state the reconciler would converge this component to right now, and + * why — the same spec read, docker probe and policy a reconcile pass uses, so a + * caller reporting on an operator command never re-derives the decision. + * + * `desired` is null for a g:/r: component whose decider has not spoken: the + * reconciler takes no action, which is neither running nor stopped. + * + * Throws what getLocalComponentSpec throws — a transient spec read is not an + * answer, and reporting one as "not running" would tell an operator their app + * is held when nothing has decided anything. + * + * @param {string} rawIdentifier + * @returns {Promise<{desired: boolean|null, reason: string, force: boolean}>} + */ +async function desiredRunState(rawIdentifier) { + const identifier = canonical(rawIdentifier); + const spec = await getLocalComponentSpec(identifier); + if (!spec) return { desired: false, reason: 'notInstalled', force: false }; + if (spec.invalidSpec) return { desired: false, reason: 'invalidSpec', force: false }; + const actual = await dockerActual(identifier); + return effectiveDesiredRunning(identifier, spec, actual.exitCode); } /** @@ -364,10 +440,26 @@ async function effectiveDesiredRunning(identifier, spec, exitCode) { async function recreateMissing(identifier) { const mainAppName = identifier.split('_')[1] || identifier; - await appTamperingDetectionService.recordEvent(mainAppName, 'container_vanished', `Container ${identifier} missing, not found in Docker`); + // `container_vanished` is the heaviest tampering signal this node emits and it + // asserts one thing: a container went away and FluxOS did not take it. FluxOS's + // own removals are in globalState.fluxRemovedContainers, and an absence FluxOS + // caused is evidence of nothing. Either way the container is recreated below - + // only the accusation is withheld. + // + // The entry is dropped by appDockerCreate succeeding, not by being read here. + // A recreate that fails therefore keeps it, which is what the next pass needs: + // the container is still absent and that absence is still FluxOS's, so reading + // the entry away would have this node accuse itself on the following pass - + // an image pull spans several of them. + const removedByFluxOs = globalState.fluxRemovedContainers.has(dockerService.getAppIdentifier(identifier)); + if (removedByFluxOs) { + log.info(`appReconciler - ${identifier} is missing because FluxOS removed it; recreating without recording a tampering event`); + } else { + await appTamperingDetectionService.recordEvent(mainAppName, 'container_vanished', `Container ${identifier} missing, not found in Docker`); + } try { await containerHealthMonitor.recreateMissingContainers(identifier); - appInspector.startAppMonitoring(identifier, globalState.appsMonitored); + appInspector.startAppMonitoring(identifier); log.info(`appReconciler - recreated missing container ${identifier}`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'recreated' }); notifyContainerStarted(identifier); @@ -414,7 +506,7 @@ async function recreateForNetworkHeal(identifier) { // fallocates + mke2fs). We removed a live container whose data was intact, so a // recreate that cannot verify the volume must fail and be retried - never wipe it. await containerHealthMonitor.recreateMissingContainers(identifier, { softOnly: true }); - appInspector.startAppMonitoring(identifier, globalState.appsMonitored); + appInspector.startAppMonitoring(identifier); log.info(`appReconciler - recreated ${identifier} to clear a detached network endpoint`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'recreated', reason: 'networkDetached' }); notifyContainerStarted(identifier); @@ -618,7 +710,7 @@ async function healDetachedNetwork(identifier, mainAppName, spec) { // Stop the per-minute stats monitor before removing the container (mirrors the // uninstaller). Otherwise its interval runs against a gone container, leaking and // error-spamming. The recreate re-establishes it via startAppMonitoring. - appInspector.stopAppMonitoring(identifier, true, globalState.appsMonitored); + appInspector.stopAppMonitoring(identifier, true); try { // v=false: Flux data lives on bind mounts; the recreate reuses them via a soft // install (enforced: recreateForNetworkHeal passes softOnly). @@ -628,7 +720,7 @@ async function healDetachedNetwork(identifier, mainAppName, spec) { // container we did NOT manage to remove is not left unmonitored. The heal flag // stays set on purpose - the remove may have partially succeeded, and a stale // flag only keeps us on the recreate path (never the uninstall one). - appInspector.startAppMonitoring(identifier, globalState.appsMonitored); + appInspector.startAppMonitoring(identifier); log.error(`appReconciler - failed to remove detached ${identifier}: ${err.message}; will retry`); scheduleRetry(identifier, MANAGED_RETRY_MS); return; @@ -690,13 +782,38 @@ async function reconcile(rawIdentifier) { // container running over a missing volume with the mount-safety hold // unenforceable - the incident's app kept running through the gutted // window exactly this way. Honor a pending stop; defer everything else. - if (controllerDesired.get(identifier) === 'stopped') { + // + // Paused counts: dockerActual reports a paused container as not running, + // and this branch returns before the paused normalisation below is ever + // reached - skipping the stop here would leave a frozen container over the + // missing volume with nothing left to release it. docker stop works on a + // paused container. + // The operator's stop counts here for the same reason the controller's does, + // and is the more urgent of the two: a volume that will not mount is the state + // support reaches for a stop IN, so a stop that waits for the mount to come + // back is a stop that never arrives when it is wanted. It outranks the + // controller everywhere else in this pass (see effectiveDesiredRunning) and + // reading only the controller here was the one place it did not. + const operatorStop = await appsRuntimeState.operatorStopState(identifier); + if (operatorStop.stopped || controllerDesired.get(identifier) === 'stopped') { try { const actualNow = await dockerActual(identifier); - if (actualNow.reachable && !actualNow.indeterminate && actualNow.running) { - log.info(`appReconciler - ${identifier} data volume unavailable but a stop is desired; stopping the container`); - await dockerService.appDockerStop(identifier); - fluxEventBus.publish('reconciler:actuated', { identifier, action: 'stopped', reason: 'controllerDesired' }); + if (actualNow.reachable && !actualNow.indeterminate && (actualNow.running || actualNow.paused)) { + // Only an operator asks for a hard kill; every other stop reason is a + // drain. Carried through here too, or an appkill against an unmounted + // volume quietly becomes a graceful stop. + const forceKill = operatorStop.stopped && operatorStop.force === true; + const reason = operatorStop.stopped ? 'operatorStopped' : 'controllerDesired'; + log.info(`appReconciler - ${identifier} data volume unavailable but a stop is desired; ${forceKill ? 'killing' : 'stopping'} the container`); + if (forceKill) { + await dockerService.appDockerKill(identifier); + } else { + await dockerService.appDockerStop(identifier); + } + appInspector.stopAppMonitoring(identifier, false); + fluxEventBus.publish('reconciler:actuated', { + identifier, action: 'stopped', reason, forced: forceKill, + }); } } catch (err) { log.error(`appReconciler - ${identifier} stop under unavailable volume failed: ${err.message}`); @@ -737,6 +854,44 @@ async function reconcile(rawIdentifier) { return; } + // NORMALISE A PAUSED CONTAINER BEFORE DECIDING ANYTHING ELSE. + // + // Paused is a state nothing downstream can act on. It is not startable - + // docker refuses with "cannot start a paused container, try unpause instead" - + // and appDockerUnpause was retired with the rest of pause, so no primitive + // remains that releases one directly. Left alone it is invisible and permanent. + // + // Stopping it converts an unrecognised state into a known one: docker stop + // works on a paused container and leaves it exited. From there this function + // needs no special case at all - the branches below start it on the normal path + // (with the backoff pacing, the mount-path recreation, the controller re-read + // and the CFS burst reapplication that appDockerStart owns), or leave it + // stopped if that is what is wanted. Handled ahead of the desired-state branch + // deliberately, so it is correct in both directions rather than only when the + // component is meant to be running. + // + // Nothing can create a paused container from here on - pause is retired - so + // this exists for the ones that already are, and for those a node that has not + // upgraded yet can still make during the rollout. A daemon or host restart + // clears them too (they come back exited), but a FluxOS restart does not, and + // that is the one an upgrade performs. + if (actual.paused) { + log.warn(`appReconciler - ${identifier} is paused, which nothing can act on; stopping it so it can be reconciled normally`); + try { + await dockerService.appDockerStop(identifier); + appInspector.stopAppMonitoring(identifier, false); + fluxEventBus.publish('reconciler:actuated', { identifier, action: 'unpaused' }); + } catch (err) { + log.error(`appReconciler - failed to stop the paused ${identifier}: ${err.message}; retrying. No FluxOS primitive releases a paused container - manual remedy on the node: docker unpause ${dockerService.getAppIdentifier(identifier)}`); + scheduleRetry(identifier, MANAGED_RETRY_MS); + return; + } + // The container is exited now, so what was sampled at entry is stale in the + // one field the branches below read. Re-enqueue rather than reason from it. + scheduleRetry(identifier, MANAGED_RETRY_MS); + return; + } + // The heal state says "this container is absent because I removed it". The moment // the container exists and is not detached, that is stale - whatever its run state, // and whatever the desired state below turns out to be. Clearing here (rather than @@ -759,10 +914,11 @@ async function reconcile(rawIdentifier) { if (actual.running) { log.info(`appReconciler - ${identifier} stopping before local appdata clear`); await dockerService.appDockerStop(identifier); + appInspector.stopAppMonitoring(identifier, false); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'stopped', reason: 'dataClear' }); } await serviceHelper.delay(DATA_CLEAR_SETTLE_MS); - await dockerOperations.appDeleteDataInMountPoint(dockerService.getAppIdentifier(identifier)); + await volumeService.clearAppVolumeData(identifier); } catch (err) { // A failed stop/wipe is the only actuation path here that would otherwise drop // to the hourly sweep (~1h down). Leave dataDesired 'clear' - so the retried @@ -783,7 +939,7 @@ async function reconcile(rawIdentifier) { return; } - const { desired, reason } = await effectiveDesiredRunning(identifier, spec, actual.exitCode); + const { desired, reason, force } = await effectiveDesiredRunning(identifier, spec, actual.exitCode); // null = no controller opinion yet for a g:/r: component: neither start nor stop, // leave the container in its current state until the decider speaks. @@ -791,9 +947,24 @@ async function reconcile(rawIdentifier) { if (!desired) { if (actual.running) { - log.info(`appReconciler - ${identifier} desired stopped, stopping`); - await dockerService.appDockerStop(identifier); - fluxEventBus.publish('reconciler:actuated', { identifier, action: 'stopped', reason }); + // A hard kill skips the graceful shutdown window, and only an operator asks + // for one - every other stop reason is a drain. The flag arrives with the + // decision that read it, from the same document and the same read as the + // lock itself, so there is no second read here to disagree with the first. + const forceKill = force === true; + log.info(`appReconciler - ${identifier} desired stopped, ${forceKill ? 'killing' : 'stopping'}`); + if (forceKill) { + await dockerService.appDockerKill(identifier); + } else { + await dockerService.appDockerStop(identifier); + } + // Monitoring follows the container. The per-minute sampler otherwise runs + // against a stopped container, logging an error a minute until something + // else happens to stop it. + appInspector.stopAppMonitoring(identifier, false); + fluxEventBus.publish('reconciler:actuated', { + identifier, action: 'stopped', reason, forced: forceKill, + }); } return; } @@ -809,6 +980,47 @@ async function reconcile(rawIdentifier) { // Verify the attachment (from the inspect dockerActual already did) before // trusting "running"; heal by recreating, confirmed in-pass and paced. if (!dockerService.isContainerDetachedFromNetwork(actual.attachment)) { + // An operator restart is a level, not an action: it raises a generation and + // this bounces the container once the generation passes the one already + // actuated. Not paced by the backoff ladder - a deliberate bounce is not + // crash recovery, and pacing it is what made six restarts look like an app + // that could not stay up. + const restartState = await appsRuntimeState.getState(identifier); + const desiredGeneration = (restartState && restartState.restartGeneration) || 0; + const actuatedGeneration = (restartState && restartState.actuatedRestartGeneration) || 0; + if (desiredGeneration > actuatedGeneration) { + log.info(`appReconciler - ${identifier} restart requested (generation ${desiredGeneration}); restarting`); + try { + await dockerService.appDockerRestart(identifier); + } catch (err) { + log.error(`appReconciler - failed to restart ${identifier} on request: ${err.message}; retrying`); + fluxEventBus.publish('reconciler:actuated', { identifier, action: 'restartRequestFailed', reason: err.message }); + scheduleRetry(identifier, MANAGED_RETRY_MS); + return; + } + fluxEventBus.publish('reconciler:actuated', { identifier, action: 'restarted', reason: 'operatorRequested' }); + notifyContainerStarted(identifier); + // A restart is a start, so it can come up on a stale endpoint the same way. + scheduleRetry(identifier, POST_START_VERIFY_MS); + // Last, because it throws. The bounce above already happened, so a write + // failure must not also cost the event, the peer notification and the + // attachment check a successful restart is owed - it is the record that + // failed, not the restart. + // + // The throw reaches the pass-level retry, which PACES it - a rate, not a + // bound, and the difference matters. UNHANDLED_FAILURE_RETRIES clears only + // on a pass that succeeds, so a database that reads but cannot write never + // records the generation: four bounces over fifteen seconds, then one per + // hourly sweep for as long as the condition holds. Bounding it needs that + // condition to be something the node observes centrally rather than each + // write site discovering it alone, which is its own change. + await appsRuntimeState.recordRestartGeneration(identifier, desiredGeneration); + return; + } + // The container is where it should be; monitoring may not be. A stop turns + // it off, and a stop docker never carried out leaves a running container + // unmonitored with no later pass to notice. + appInspector.ensureAppMonitoring(identifier); return; // running and properly attached (heal state was cleared above) } await healDetachedNetwork(identifier, mainAppName, spec); @@ -854,12 +1066,33 @@ async function reconcile(rawIdentifier) { return; } - // exists but stopped, should run -> backoff-paced restart (no sleeping; the - // worker re-enqueues when the backoff window elapses) + // exists but stopped, should run -> restart, paced by the ladder only when the + // stop carries evidence of a fault (no sleeping; the worker re-enqueues when + // the backoff window elapses). A clean exit goes back immediately: it is an + // operator restarting their own app far more often than it is a crash, and + // pacing that turns a deliberate restart into what looks like an outage. + // exitCode null is a container that has never run - an initial start, not a death. + const crashed = !!actual.oomKilled || (actual.exitCode !== null && actual.exitCode !== 0); const wait = await appsRuntimeState.restartWaitMs(identifier, actual.finishedAt); if (wait > 0) { - log.warn(`appReconciler - ${identifier} stopped, backing off ${Math.round(wait / 1000)}s before restart`); - fluxEventBus.publish('reconciler:actuated', { identifier, action: 'backoff', waitMs: wait }); + // name which of the two put it here: a reported fault, or restarts arriving + // fast enough to be one whatever the exit code said. Support cannot tell + // these apart from the outside, and the difference decides what they do next. + const cause = crashed + ? `exit ${actual.exitCode}${actual.oomKilled ? ' (OOM-killed)' : ''}` + : 'restarting too fast to be healthy'; + // How far up the ladder this is. waitMs alone cannot say: it is what REMAINS + // of the rung, and the worker re-enqueues during a wait, so one rung reports + // several times, each smaller than the last. Two backoffs cannot be compared + // without it - which is how far a component has escalated, and whether it + // ever went backwards. Read on the backoff path only, which is a paced + // restart and therefore cold. + const backoffState = await appsRuntimeState.getState(identifier); + const rung = ((backoffState && backoffState.restartHistory) || []).length; + log.warn(`appReconciler - ${identifier} stopped, ${cause}; backing off ${Math.round(wait / 1000)}s before restart (rung ${rung})`); + fluxEventBus.publish('reconciler:actuated', { + identifier, action: 'backoff', waitMs: wait, rung, crashed, + }); scheduleRetry(identifier, wait); return; } @@ -880,20 +1113,30 @@ async function reconcile(rawIdentifier) { return; } - await appsRuntimeState.recordRestart(identifier); + await appsRuntimeState.recordRestart(identifier, crashed); try { await dockerService.appDockerStart(identifier); } catch (err) { // No die event fires for a failed start (the container never ran), so a // dropped throw here leaves the component down until the hourly sweep. - // Schedule our own retry; pacing is free - the attempt was recorded above, - // so a persistent failure walks the backoff ladder instead of hammering. + // Schedule our own retry. A start that never ran carries no exit code, so it + // is not a fault and does not walk the ladder directly - it reaches the + // ladder by filling the burst window, which these retries do comfortably + // (restartBurstCount x MANAGED_RETRY_MS against restartBurstWindowMs). That + // relationship is what bounds a permanently failing start, and the config + // comment on the window is where it is stated. log.error(`appReconciler - failed to start ${identifier}: ${err.message}; retrying`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'startFailed', reason: err.message }); scheduleRetry(identifier, MANAGED_RETRY_MS); return; } - appInspector.startAppMonitoring(identifier, globalState.appsMonitored); + appInspector.startAppMonitoring(identifier); + // A restart of a container that was already stopped IS this start, so the + // request is satisfied here. Left pending, the pass that next finds it running + // would bounce a container the operator has just watched come up. + const startedState = await appsRuntimeState.getState(identifier); + const pendingGeneration = (startedState && startedState.restartGeneration) || 0; + const satisfiesRestart = pendingGeneration > ((startedState && startedState.actuatedRestartGeneration) || 0); log.info(`appReconciler - ${identifier} restarted`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'started', exitCode: actual.exitCode }); notifyContainerStarted(identifier); @@ -902,6 +1145,11 @@ async function reconcile(rawIdentifier) { // BEFORE this start, so verify the new one shortly - otherwise a detached-at-boot // container waits for the hourly sweep. scheduleRetry(identifier, POST_START_VERIFY_MS); + // Last, because it throws - the start above already happened, and the record + // failing must not cost the bookkeeping that start is owed. + if (satisfiesRestart) { + await appsRuntimeState.recordRestartGeneration(identifier, pendingGeneration); + } } // --- workqueue (per-key single-flight, boot-gated) ----------------------- @@ -917,8 +1165,29 @@ function scheduleRetry(identifier, delayMs) { } function runReconcile(identifier) { - reconcile(identifier) - .catch((err) => log.error(`appReconciler - reconcile ${identifier} failed: ${err.message}`)) + const pass = reconcile(identifier) + .then(() => { + // A pass that got through is the only evidence the fault has cleared. + unhandledFailures.delete(identifier); + }) + .catch((err) => { + const attempt = (unhandledFailures.get(identifier) || 0) + 1; + unhandledFailures.set(identifier, attempt); + const retrying = attempt <= UNHANDLED_FAILURE_RETRIES; + log.error( + `appReconciler - reconcile ${identifier} failed: ${err.message}` + + (retrying + ? `; retrying (${attempt}/${UNHANDLED_FAILURE_RETRIES})` + : `; ${attempt} consecutive failures, leaving it to the hourly sweep`), + ); + // Published for every unhandled failure rather than at each throw site: the + // sites that can throw are the ones nobody thought to guard, so an event + // added per site would miss exactly the same ones the retry did. + fluxEventBus.publish('reconciler:actuated', { + identifier, action: 'reconcileFailed', reason: err.message, attempt, retrying, + }); + if (retrying) scheduleRetry(identifier, MANAGED_RETRY_MS); + }) .finally(() => { inFlight.delete(identifier); // one completed pass (actuated or deferred) is all the boot drain needs @@ -930,6 +1199,10 @@ function runReconcile(identifier) { setImmediate(() => enqueue(identifier)); } }); + // Registered synchronously: promise callbacks are microtasks, so the finally + // above cannot run before this line and clear an entry that is not there yet. + inFlight.set(identifier, pass); + return pass; } /** @@ -941,62 +1214,193 @@ function enqueue(rawIdentifier) { const identifier = canonical(rawIdentifier); if (!globalState.bootContainerStateSettled) { bootPending.add(identifier); - return; + return null; } if (inFlight.has(identifier)) { dirty.add(identifier); - return; + // The pass already running was started against state older than whatever + // just changed, so it is NOT the pass a caller wanting actuation should + // wait on. The re-run this marks dirty is, and it has no promise yet. + return null; } - inFlight.add(identifier); - runReconcile(identifier); + return runReconcile(identifier); +} + +/** + * Change what a component is supposed to be doing, without racing a pass that is + * deciding what to do about it. + * + * A reconcile reads the desired state, then acts on that answer some + * milliseconds later once docker has answered. An intent written in that gap is + * not seen: the pass starts a container an operator has just stopped, and the + * next pass stops it again. The lock is written correctly and early - the + * problem is that the check and the action are not atomic against a concurrent + * writer, so narrowing the gap with a second check before acting would leave the + * same defect with a smaller window. + * + * Instead the write takes the same per-key slot a pass takes. It waits out a + * pass already deciding, holds the key while it writes so `enqueue` marks the + * key dirty rather than starting one, and enqueues on release so the next pass + * reads the intent it just wrote. The two can no longer interleave because they + * are mutually exclusive by construction. + * + * The wait is one pass of ONE component - a docker probe and at most one action - + * so an operator's command is never behind unrelated work. That is a BOUND only + * while passes terminate, and the docker calls a pass makes carry no timeout of + * their own: a daemon that HANGS rather than fails leaves the pass unfinished and + * this wait with nothing to wake it. + * + * What that costs is durability, not correctness: nothing wrong is recorded, and + * the caller's request hangs on a wedged daemon regardless. What is lost is a + * FluxOS restart during the hang - the write has not landed, so the intent does + * not survive one. + * + * Bounding THIS wait is not the repair - giving up on it and writing anyway + * restores the interleave described above, which is the defect this exists to + * fix. Bounding the docker calls is, and that is fleet-wide work rather than + * something this function can do alone. + * @param {string} rawIdentifier Component identifier. + * @param {Function} mutate Writes the new intent. Awaited while the key is held. + * @param {object} [opts] + * @param {boolean} [opts.awaitPass] Wait for the reconcile that follows, so a + * caller can report what was DONE rather than what was asked for. The pass is + * awaited to completion, actuated or deferred - it never throws here, since + * runReconcile absorbs its own failures. + * @returns {Promise} True when a pass ran to completion. False when the + * intent is durable but nothing has acted on it yet - the boot gate is shut, + * or another pass is mid-flight and the re-run has not started. Callers that + * report to a user must not present false as success. + */ +async function applyIntent(rawIdentifier, mutate, { awaitPass = false } = {}) { + const identifier = canonical(rawIdentifier); + + // A loop, not a single await: releasing the key lets a queued pass start + // before this continues, and that pass would be reading the state we are + // about to replace. + // eslint-disable-next-line no-await-in-loop + while (inFlight.has(identifier)) await inFlight.get(identifier).catch(() => {}); + + let release; + const held = new Promise((resolve) => { release = resolve; }); + inFlight.set(identifier, held); + try { + await mutate(); + } finally { + inFlight.delete(identifier); + release(); + } + + const pass = enqueue(identifier); + if (!awaitPass) return Boolean(pass); + if (!pass) return false; + await pass; + return true; +} + +/** + * The component identifiers of a set of installed apps. + * + * Enterprise specs are stored encrypted (compose: []) and the component names + * live INSIDE the blob, so the set is decrypted first - leniently: one app + * failing to decrypt must not cost the rest their components. An app that stays + * encrypted is enumerated from the containers docker is already holding for it, + * matched on the `_` suffix, so it is never silently skipped. That + * source can only see components that EXIST, which is the most that can be known + * about such an app anyway: a vanished component of one cannot be recreated + * either, because recreating it needs the spec. + * + * The listing is taken once for the whole set, and only if something failed to + * decrypt. + * + * @param {Array} installed Records from appQueryService.installedApps(). + * @returns {Promise} Bare component identifiers (`_`, + * or `` for v1-3) across every app given - one spelling whichever source + * they came from. + */ +async function componentIdsOf(installed) { + const { readable, unreadable } = await appQueryService.decryptEnterpriseApps(installed, { formatSpecs: false }); + const ids = []; + + readable.forEach((app) => { + if (app.version >= 4 && Array.isArray(app.compose)) { + app.compose.forEach((c) => ids.push(`${c.name}_${app.name}`)); + } else { + ids.push(app.name); + } + }); + + if (!unreadable.length) return ids; + + let dockerNames; + try { + const containers = await dockerService.dockerListContainers(true); + dockerNames = containers.map((c) => (c.Names && c.Names[0] ? c.Names[0].slice(1) : '')); + } catch (err) { + // The list is returned short rather than refused, so one app's failure + // cannot cost the readable apps their sweep. Named at error level because a + // short list is indistinguishable from a complete one at every call site: + // the app is simply absent from what the caller acts on. + log.error(`appReconciler - cannot list containers, dropping undecryptable apps [${unreadable.map((app) => app.name).join(', ')}]: ${err.message}`); + return ids; + } + unreadable.forEach((app) => { + const suffix = `_${app.name}`; + // Docker holds the namespaced name (`flux_`); the readable + // branch above produces the bare one. One list carries one spelling, so a + // consumer that compares it against a component name an operator typed + // matches it, rather than refusing every component of an app whose spec + // will not decrypt. The consumers that canonicalise on ingest cannot tell + // the two apart, which is why they coexisted unnoticed. + dockerNames.filter((name) => name.endsWith(suffix)).forEach((name) => ids.push(canonical(name))); + }); + return ids; } /** * Enqueue every installed component (hourly tick / reconnect / boot drift). - * Enterprise specs are stored encrypted (compose: []), so the sweep decrypts - * to enumerate components — leniently: one app failing to decrypt must not - * abort the sweep for the rest. An app that stays encrypted is still covered - * via its existing docker containers (see below); never silently skipped. */ async function enqueueAll(reason = 'resync') { const res = await appQueryService.installedApps(); if (!res || res.status !== 'success') return; - const apps = await appQueryService.decryptEnterpriseApps(res.data, { formatSpecs: false }); - let count = 0; - let dockerNames = null; // fetched once, only if some app failed to decrypt - for (const app of apps) { - const stillEncrypted = app.version >= 8 && app.enterprise - && (!Array.isArray(app.compose) || app.compose.length === 0); - if (stillEncrypted) { - // Decryption failed (already logged by decryptEnterpriseApps). The component - // names live inside the blob, so enumerate the app's EXISTING docker - // containers instead: their reconciles defer on the same decrypt failure - // and converge the moment fluxbenchd answers again. A vanished container of - // an undecryptable app cannot be recovered anyway (recreation needs the - // spec); the next sweep retries, so coverage resumes with decryption. - if (dockerNames === null) { - try { - // eslint-disable-next-line no-await-in-loop - const containers = await dockerService.dockerListContainers(true); - dockerNames = containers.map((c) => (c.Names && c.Names[0] ? c.Names[0].slice(1) : '')); - } catch (err) { - log.warn(`appReconciler - enqueueAll cannot list containers for undecryptable apps: ${err.message}`); - dockerNames = []; - } - } - const suffix = `_${app.name}`; - dockerNames.filter((name) => name.endsWith(suffix)).forEach((name) => { - enqueue(name); // canonicalised to the bare component identifier by enqueue - count += 1; - }); - } else if (app.version >= 4 && Array.isArray(app.compose)) { - app.compose.forEach((c) => { enqueue(`${c.name}_${app.name}`); count += 1; }); - } else { - enqueue(app.name); - count += 1; - } + const ids = await componentIdsOf(res.data); + // canonicalised to the bare component identifier by enqueue + ids.forEach((id) => enqueue(id)); + fluxEventBus.publish('reconciler:swept', { reason, count: ids.length }); +} + +/** + * Ask every component of these apps to restart, and let the normal machinery + * carry it out. + * + * For a caller that knows the node has changed underneath its apps - the address + * moved, so the containers have to come up on the new one - and knows nothing + * else about them. Everything an app is made of is worked out here rather than by + * the caller: which components it has, whether its specs can be read, and what a + * restart request even is. + * + * Durable and paced, deliberately. A generation survives a FluxOS restart part + * way through, where a docker call issued from the caller is simply lost, and it + * queues behind the same slot as every other intent instead of racing it. Each + * request is independent: one component that cannot be recorded must not cost the + * others theirs. + * + * @param {Array} installed Records from appQueryService.installedApps(). + * @param {string} reason Why, for the log. + * @returns {Promise} How many components were asked. + */ +async function requestRestartOf(installed, reason) { + const ids = await componentIdsOf(installed); + let asked = 0; + // eslint-disable-next-line no-restricted-syntax + for (const id of ids) { + // eslint-disable-next-line no-await-in-loop + await applyIntent(id, async () => { + await appsRuntimeState.requestRestart(id); + }).then(() => { asked += 1; }) + .catch((err) => log.error(`appReconciler - could not request restart of ${id} (${reason}): ${err.message}`)); } - fluxEventBus.publish('reconciler:swept', { reason, count }); + log.info(`appReconciler - restart requested for ${asked}/${ids.length} components (${reason})`); + return asked; } // --- controllerDesired seam (written by masterSlave/syncthing deciders) --- @@ -1033,10 +1437,70 @@ function requestStopAndClearData(rawIdentifier, reason) { enqueue(identifier); } +/** + * Retract the controller's opinion about whether this component should run, + * leaving every other desired input standing. + * + * A pending data clear is NOT an opinion about running - it is the sync layer's + * finding that the local appdata must not be trusted - so it survives. It has + * to: the sync layer marks a component processed BEFORE asking, so a request + * dropped here is never made again and the component eventually starts on the + * data the clear existed to remove. The reconciler resolves a pending clear + * ahead of any run decision, so one left standing on a stopped component simply + * waits. + */ function clearControllerDesired(rawIdentifier) { const identifier = canonical(rawIdentifier); controllerDesired.delete(identifier); +} + +/** + * Forget every desired input for a component - it is gone, and nothing about it + * is worth acting on. Removal only: for anything short of that, retract the + * specific opinion. + */ +function forgetDesiredState(rawIdentifier) { + const identifier = canonical(rawIdentifier); + controllerDesired.delete(identifier); dataDesired.delete(identifier); + // A removed component keeps no failure history: the map is keyed by identifier + // and a reinstall under the same name would otherwise start part-way up the + // count and reach the sweep sooner than a first failure should. + unhandledFailures.delete(identifier); +} + +/** + * A decider has committed to running this component but cannot start it yet - + * the masterSlave primary path fixes ownership on the persistent data first, + * which takes long enough that a peer asking "is anyone running this?" gets a + * truthful no and starts a second writer. Held from the decision, released when + * the attempt ends: a start that succeeds is covered by controllerDesired from + * then on, and one that fails is correctly no longer a claim. + * + * Deliberately not time-bounded. The claimant knows when it has finished, so + * there is nothing to guess at, and the state is process-local - a crash or a + * FluxOS restart drops it with no way for a stale claim to outlive its owner. + */ +function claimStarting(rawIdentifier) { + startingClaims.add(canonical(rawIdentifier)); +} + +function releaseStarting(rawIdentifier) { + startingClaims.delete(canonical(rawIdentifier)); +} + +/** + * Component identifiers this node runs or is committed to running, from its own + * state alone. The running containers are the caller's to add - this is the part + * Docker cannot answer. + * @returns {string[]} + */ +function committedIdentifiers() { + const ids = new Set(startingClaims); + controllerDesired.forEach((state, identifier) => { + if (state === 'running') ids.add(identifier); + }); + return [...ids]; } // --- lifecycle ----------------------------------------------------------- @@ -1067,6 +1531,7 @@ function stop() { started = false; backoffTimers.forEach((t) => clearTimeout(t)); backoffTimers.clear(); + unhandledFailures.clear(); if (bootDrainCapTimer) { clearTimeout(bootDrainCapTimer); bootDrainCapTimer = null; @@ -1079,14 +1544,33 @@ function stop() { module.exports = { enqueue, + applyIntent, enqueueAll, + requestRestartOf, setControllerDesired, clearControllerDesired, + forgetDesiredState, + claimStarting, + releaseStarting, + committedIdentifiers, requestStopAndClearData, setOnContainerStarted, waitForBootDrainSettled: () => bootDrainGate.wait(), start, stop, + // The one answer to "what is this container actually doing" - it probes the + // daemon rather than pattern-matching an inspect error, so it can tell docker + // being unreachable from the container being gone. Anything that acts on a + // container's run state needs that distinction, not just the reconciler. + dockerActual, + // What the reconciler would do with this component now, for a caller that has + // to report an operator command's outcome truthfully. + desiredRunState, + // What an app is actually made of. Enterprise specs keep their component names + // inside an encrypted blob, so reading `compose` off a stored spec yields an + // empty list and silently addresses nothing - which is why every caller that + // needs an app's components asks here rather than working it out again. + componentIdsOf, // exposed for tests reconcile, policyAllowsRun, diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index 1a00b5d750..d5774cd09e 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -4,7 +4,6 @@ const config = require('config'); const serviceHelper = require('../serviceHelper'); const generalService = require('../generalService'); const fluxNetworkHelper = require('../fluxNetworkHelper'); -const verificationHelper = require('../verificationHelper'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const upnpService = require('../upnpService'); const networkStateService = require('../networkStateService'); @@ -12,13 +11,7 @@ const fluxHttpTestServer = require('../utils/fluxHttpTestServer'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); - -// Helper function to sign check app data -async function signCheckAppData(message) { - const privKey = await fluxNetworkHelper.getFluxNodePrivateKey(); - const signature = await verificationHelper.signMessage(message, privKey); - return signature; -} +const { nodeSigner } = require('../utils/nodeSigner'); // Helper function to handle test shutdown async function handleTestShutdown(testingPort, testHttpServer, isArcane, options = {}) { @@ -146,7 +139,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin } // Decrypt enterprise apps (version 8 with encrypted content) - installedAppsRes.data = await decryptEnterpriseApps(installedAppsRes.data); + ({ inPlace: installedAppsRes.data } = await decryptEnterpriseApps(installedAppsRes.data)); const apps = installedAppsRes.data; const appPorts = []; @@ -215,7 +208,10 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin return; } - const remoteSocketAddress = await networkStateService.getRandomSocketAddress(localSocketAddress); + // An external observer: this asks a peer whether it can reach US, and a Flux + // node sharing our public address cannot answer that. Null when there is no + // such node, which the retry below already handles. + const remoteSocketAddress = await networkStateService.getRandomExternalObserver(localSocketAddress); if (!remoteSocketAddress) { await serviceHelper.delay(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); @@ -295,7 +291,9 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin headers: { 'content-type': '' }, }; - const pubKey = await fluxNetworkHelper.getFluxNodePublicKey(); + const signer = await nodeSigner(); + if (!signer) throw new Error('checkMyAppsAvailability - this node cannot sign the port test'); + const localIp = extractIp(localSocketAddress); const localPort = extractPort(localSocketAddress); const remoteIp = extractIp(remoteSocketAddress); @@ -306,10 +304,11 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin port: String(localPort), appname: 'appPortsTest', ports: [dosState.testingPort], - pubKey, + pubKey: signer.pubKey, }; - const signature = await signCheckAppData(JSON.stringify(data)); + const signature = signer.sign(JSON.stringify(data)); + if (!signature) throw new Error('checkMyAppsAvailability - the port test could not be signed'); data.signature = signature; const resMyAppAvailability = await axios diff --git a/ZelBack/src/services/appMonitoring/containerHealthMonitor.js b/ZelBack/src/services/appMonitoring/containerHealthMonitor.js index 474cd32f6b..2bdc9b825d 100644 --- a/ZelBack/src/services/appMonitoring/containerHealthMonitor.js +++ b/ZelBack/src/services/appMonitoring/containerHealthMonitor.js @@ -37,9 +37,12 @@ async function recreateMissingContainers(componentIdentifier, options = {}) { throw new Error(`App ${mainAppName} not found in local database`); } - appSpec = await decryptEnterpriseApps([appSpec], { formatSpecs: false }); - // eslint-disable-next-line prefer-destructuring - appSpec = appSpec[0]; + const { readable: [decryptedSpec] } = await decryptEnterpriseApps([appSpec], { formatSpecs: false }); + if (!decryptedSpec) { + // its components are inside the blob, so there is nothing to recreate from + throw new Error(`App ${mainAppName} could not be decrypted`); + } + appSpec = decryptedSpec; if (!appSpec.compose || appSpec.compose.length === 0) { throw new Error(`App ${mainAppName} has no components to install`); diff --git a/ZelBack/src/services/appMonitoring/monitoringOrchestrator.js b/ZelBack/src/services/appMonitoring/monitoringOrchestrator.js index 830eb5d9af..8ac4db736f 100644 --- a/ZelBack/src/services/appMonitoring/monitoringOrchestrator.js +++ b/ZelBack/src/services/appMonitoring/monitoringOrchestrator.js @@ -1,79 +1,119 @@ // Monitoring Orchestrator - Functions to start/stop monitoring and handle API endpoints const messageHelper = require('../messageHelper'); -const serviceHelper = require('../serviceHelper'); -const verificationHelper = require('../verificationHelper'); const appInspector = require('../appManagement/appInspector'); +const appQueryService = require('../appQuery/appQueryService'); const log = require('../../lib/log'); +// Monitoring is started by the node whenever a container comes up and feeds the CPU +// throttling loop, so it is not a setting an operator turns on or off. The routes stay +// so callers are told that rather than silently succeeding against a control that is +// gone; they go at the next major version. +const DEPRECATION_MESSAGE = 'Application monitoring is managed by the node and runs for every app. This endpoint no longer has any effect and will be removed.'; + /** - * Start monitoring multiple applications - * @param {Array} appSpecsToMonitor - Array of app specifications to monitor - * @param {object} appsMonitored - Apps monitored structure from appsService - * @param {Function} installedAppsFn - Function to get installed apps - * @returns {Promise} Result of monitoring start + * Resolve the app specifications monitoring should act on + * @param {Array} appSpecsToMonitor - Explicit specifications, or null for every installed app + * @returns {Promise} App specifications */ -async function startMonitoringOfApps(appSpecsToMonitor, appsMonitored, installedAppsFn) { - try { - let apps = appSpecsToMonitor; - if (!apps) { - const installedAppsRes = await installedAppsFn(); - if (installedAppsRes.status !== 'success') { - throw new Error('Failed to get installed Apps'); - } - apps = installedAppsRes.data; +async function resolveAppSpecs(appSpecsToMonitor) { + if (appSpecsToMonitor) { + // Shape-checked for the same reason the compose list is below: for-of accepts + // anything iterable, so a string here would be walked character by character + // and every character treated as an app. Nothing passes a non-null value + // today - serviceManager is the only caller and always passes null - so this + // is the guard arriving before the caller that would need it. + if (!Array.isArray(appSpecsToMonitor)) { + throw new Error('appSpecsToMonitor must be an array of app specifications'); } - - // eslint-disable-next-line no-restricted-syntax - for (const app of apps) { - if (app.version <= 3) { - appInspector.startAppMonitoring(app.name, appsMonitored); - } else { - // eslint-disable-next-line no-restricted-syntax - for (const component of app.compose) { - const monitoredName = `${component.name}_${app.name}`; - appInspector.startAppMonitoring(monitoredName, appsMonitored); - } - } - } - } catch (error) { - log.error(error); + return appSpecsToMonitor; + } + const installedAppsRes = await appQueryService.installedApps(); + if (installedAppsRes.status !== 'success') { + throw new Error('Failed to get installed Apps'); } + return installedAppsRes.data; } /** - * Stop monitoring multiple applications - * @param {Array} appSpecsToMonitor - Array of app specifications to stop monitoring - * @param {boolean} deleteData - Whether to delete monitoring data - * @param {object} appsMonitored - Apps monitored structure from appsService - * @param {Function} installedAppsFn - Function to get installed apps - * @returns {Promise} Result of monitoring stop + * Start monitoring multiple applications + * @param {Array} appSpecsToMonitor - Array of app specifications to monitor, or null for every installed app + * @returns {Promise} */ -// eslint-disable-next-line default-param-last -async function stopMonitoringOfApps(appSpecsToMonitor, deleteData = false, appsMonitored, installedAppsFn) { - try { - let apps = appSpecsToMonitor; - if (!apps) { - const installedAppsRes = await installedAppsFn(); - if (installedAppsRes.status !== 'success') { - throw new Error('Failed to get installed Apps'); +async function startMonitoringOfApps(appSpecsToMonitor) { + const apps = await resolveAppSpecs(appSpecsToMonitor); + + // Monitoring drives CPU throttling, so one app that cannot be monitored must not + // leave the rest of them unthrottled. The same holds one level down: a composed + // app's components are monitored independently, so one that cannot be started + // must not take the components after it in the same compose with it. Catching per + // app alone did exactly that, and named the app in the log rather than the + // component that actually failed. + // + // The name is built INSIDE the try. A component that is null throws on + // `component.name` before startAppMonitoring is ever reached, so a guard placed + // any later would not see the case it exists for. + // A monitored name is `_`, and startAppMonitoring only refuses a + // FALSY one - so gluing two strings together always produces something it + // accepts. A component with no name became `undefined_App`, a monitor armed + // against a container that cannot exist: a timer, a store, and a sampler asking + // docker about it once a minute, forever, with nothing to say it went wrong. + // Both halves have to be real before there is anything worth monitoring. + // A label that cannot itself throw, whatever the entry turns out to be. Reading + // a property off the value that caused the failure is how an error handler + // becomes the failure, and a throw raised inside a catch is not caught by it. + const labelOf = (value) => { + try { + const name = value?.name; + return typeof name === 'string' && name ? name : ''; + } catch (error) { + return ''; + } + }; + + const startComponent = (app, component) => { + const appLabel = labelOf(app); + try { + const componentName = component?.name; + if (typeof componentName !== 'string' || !componentName || !app.name) { + log.error(`startMonitoringOfApps - skipping a component of ${appLabel}: no usable name to monitor it under`); + return; } - apps = installedAppsRes.data; + appInspector.startAppMonitoring(`${componentName}_${app.name}`); + } catch (error) { + // Labels captured BEFORE the try, never re-read here. `component.name` is a + // property access on a value this catch exists because of - a getter that + // threw once throws again, and a throw inside a catch is not caught by it. + log.error(`startMonitoringOfApps - could not start monitoring a component of ${appLabel}: ${error.message}`); } + }; - // eslint-disable-next-line no-restricted-syntax - for (const app of apps) { + // eslint-disable-next-line no-restricted-syntax + for (const app of apps) { + try { if (app.version <= 3) { - appInspector.stopAppMonitoring(app.name, deleteData, appsMonitored); + appInspector.startAppMonitoring(app.name); + } else if (!Array.isArray(app.compose)) { + // for-of accepts anything iterable, and a STRING is iterable: a compose of + // 'nope' walked its four characters and armed four monitors, none of which + // named a container. A missing compose threw and was at least logged; a + // malformed one was silent, which is the worse of the two. + log.error(`startMonitoringOfApps - ${labelOf(app)} has no component list to monitor`); } else { // eslint-disable-next-line no-restricted-syntax for (const component of app.compose) { - const monitoredName = `${component.name}_${app.name}`; - appInspector.stopAppMonitoring(monitoredName, deleteData, appsMonitored); + startComponent(app, component); } } + } catch (error) { + // Still needed for what is not one component's failure: a compose that is not + // iterable at all, which no per-component catch can be reached to see. + // + // Labelled defensively: this catch is reached when `app` itself is what is + // wrong - a null entry throws on `app.version` before anything else runs - + // and `app.name` here would then throw a second time, uncaught, abandoning + // the loop and leaving every remaining app unmonitored with nothing logged. + log.error(`startMonitoringOfApps - could not start monitoring ${labelOf(app)}: ${error.message}`); } - } catch (error) { - log.error(error); } } @@ -81,150 +121,45 @@ async function stopMonitoringOfApps(appSpecsToMonitor, deleteData = false, appsM * Start monitoring API endpoint * @param {object} req Request. * @param {object} res Response. - * @param {object} appsMonitored - Apps monitored structure from appsService - * @param {Function} installedAppsFn - Function to get installed apps * @returns {object} Message. */ -async function startAppMonitoringAPI(req, res, appsMonitored, installedAppsFn) { - try { - let { appname } = req.params; - appname = appname || req.query.appname; - - if (!appname) { - // Only flux team and node owner can monitor all apps - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); - if (!authorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - await stopMonitoringOfApps(null, false, appsMonitored, installedAppsFn); - await startMonitoringOfApps(null, appsMonitored, installedAppsFn); - const monitoringResponse = messageHelper.createSuccessMessage('Application monitoring started for all apps'); - return res ? res.json(monitoringResponse) : monitoringResponse; - } - const mainAppName = appname.split('_')[1] || appname; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); - if (!authorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - const installedAppsRes = await installedAppsFn(mainAppName); - if (installedAppsRes.status !== 'success') { - throw new Error('Failed to get installed Apps'); - } - const apps = installedAppsRes.data; - const appSpecs = apps[0]; - if (!appSpecs) { - throw new Error(`Application ${mainAppName} is not installed`); - } - if (mainAppName === appname) { - await stopMonitoringOfApps(null, false, appsMonitored, installedAppsFn); - await startMonitoringOfApps([appSpecs], appsMonitored, installedAppsFn); - } else { // component based or <= 3 - appInspector.stopAppMonitoring(appname, false, appsMonitored); - appInspector.startAppMonitoring(appname, appsMonitored); - } - const monitoringResponse = messageHelper.createSuccessMessage(`Application monitoring started for ${appSpecs.name}`); - return res ? res.json(monitoringResponse) : monitoringResponse; - } catch (error) { - log.error(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - return res ? res.json(errorResponse) : errorResponse; - } +async function startAppMonitoringAPI(req, res) { + const errMessage = messageHelper.createErrorMessage(DEPRECATION_MESSAGE, 'Deprecated', 410); + return res ? res.json(errMessage) : errMessage; } /** * Stop monitoring API endpoint * @param {object} req Request. * @param {object} res Response. - * @param {object} appsMonitored - Apps monitored structure from appsService - * @param {Function} installedAppsFn - Function to get installed apps * @returns {object} Message. */ -async function stopAppMonitoringAPI(req, res, appsMonitored, installedAppsFn) { - try { - let { appname } = req.params; - appname = appname || req.query.appname; - let { deletedata } = req.params; - deletedata = deletedata || req.query.deletedata || false; - deletedata = serviceHelper.ensureBoolean(deletedata); - - if (!appname) { - // Only flux team and node owner can stop monitoring for all apps - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); - if (!authorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - await stopMonitoringOfApps(null, deletedata, appsMonitored, installedAppsFn); - let successMessage = ''; - if (!deletedata) { - successMessage = 'Application monitoring stopped for all apps. Existing monitoring data maintained.'; - } else { - successMessage = 'Application monitoring stopped for all apps. Monitoring data deleted for all apps.'; - } - const monitoringResponse = messageHelper.createSuccessMessage(successMessage); - return res ? res.json(monitoringResponse) : monitoringResponse; - } - const mainAppName = appname.split('_')[1] || appname; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, mainAppName); - if (!authorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - let successMessage = ''; - if (mainAppName === appname) { - // get appSpecs - const installedAppsRes = await installedAppsFn(mainAppName); - if (installedAppsRes.status !== 'success') { - throw new Error('Failed to get installed Apps'); - } - const apps = installedAppsRes.data; - const appSpecs = apps[0]; - if (!appSpecs) { - throw new Error(`Application ${mainAppName} is not installed`); - } - await stopMonitoringOfApps([appSpecs], deletedata, appsMonitored, installedAppsFn); - } else { // component based or <= 3 - appInspector.stopAppMonitoring(appname, deletedata, appsMonitored); - } - if (deletedata) { - successMessage = `Application monitoring stopped and monitoring data deleted for ${appname}.`; - } else { - successMessage = `Application monitoring stopped for ${appname}. Existing monitoring data maintained.`; - } - const monitoringResponse = messageHelper.createSuccessMessage(successMessage); - return res ? res.json(monitoringResponse) : monitoringResponse; - } catch (error) { - log.error(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - return res ? res.json(errorResponse) : errorResponse; - } +async function stopAppMonitoringAPI(req, res) { + const errMessage = messageHelper.createErrorMessage(DEPRECATION_MESSAGE, 'Deprecated', 410); + return res ? res.json(errMessage) : errMessage; } +// The stream's job is served by polling: /apps/appstats answers with a reading +// at most five seconds old, /apps/appmonitor with the collected series. The +// route stays so a caller is told that rather than getting an anonymous 404 - +// the same contract as the two controls above; it goes at the next major +// version. +const STREAM_DEPRECATION_MESSAGE = 'The stats stream has been removed. Poll /apps/appstats for a live reading or /apps/appmonitor for the collected series.'; + /** - * Enhanced appMonitor function that uses the inspector module but adds the monitored data + * Stats stream API endpoint * @param {object} req Request. * @param {object} res Response. - * @param {object} appsMonitored - Apps monitored structure from appsService - * @returns {object} Monitoring data. + * @returns {object} Message. */ -async function appMonitor(req, res, appsMonitored) { - return appInspector.appMonitor(req, res, appsMonitored); +async function appMonitorStreamAPI(req, res) { + const errMessage = messageHelper.createErrorMessage(STREAM_DEPRECATION_MESSAGE, 'Deprecated', 410); + return res ? res.json(errMessage) : errMessage; } module.exports = { + appMonitorStreamAPI, startMonitoringOfApps, - stopMonitoringOfApps, startAppMonitoringAPI, stopAppMonitoringAPI, - appMonitor, }; diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 1dd5044dea..e07d5f20a7 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -7,6 +7,7 @@ const fluxNetworkHelper = require('../fluxNetworkHelper'); const fluxCommunicationUtils = require('../fluxCommunicationUtils'); const messageStore = require('../appMessaging/messageStore'); const nodeConfirmationService = require('../nodeConfirmationService'); +const networkStateService = require('../networkStateService'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); @@ -77,6 +78,17 @@ async function monitorNodeStatus(installedAppsFn, removeAppLocallyFn) { return monitorNodeStatus(installedAppsFn, removeAppLocallyFn); } if (nodeConfirmationService.isConfirmed()) { log.info('monitorNodeStatus - Node is Confirmed'); + // Everything above this point is app removal and needs no node list. What + // follows compares every app location against it, so an unknown list + // makes every location read as departed and sends the whole set down the + // HTTP probe path. The accessors wait for the list, and this loop only + // re-arms once it has finished, so it checks and comes back rather than + // awaiting in here - the same shape as the branches above. + if (!networkStateService.isReady()) { + log.info('monitorNodeStatus - node list not known yet, deferring the location sweep'); + await serviceHelper.delay(config.fluxapps.nodeMonitorCheckIntervalMs ?? 120000); + return monitorNodeStatus(installedAppsFn, removeAppLocallyFn); + } // lets remove from locations when nodes are no longer confirmed const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); diff --git a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js new file mode 100644 index 0000000000..e5319b4dab --- /dev/null +++ b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js @@ -0,0 +1,250 @@ +// One answer per peer per monitor pass to "are you there, and which folders do +// you hold?" +// +// Both promotion decisions ask that of the same peers on the same endpoint, and +// the reply carries the peer's whole folder list - so the question is per PEER, +// and only its interpretation is per folder. Asked inside the folder loop it +// became per folder as well: the loop is sequential, an unreachable peer costs +// the full timeout, and a node recovering nine synced folders paid that timeout +// nine times. Past three the pass outruns its own 30s interval and the monitor +// drops whole cycles, so promotions, stall detection and error draining stop for +// every folder on the node, not just the slow one. +// +// Answers live exactly as long as the pass that created them. A peer's liveness +// is the one thing here that must not be remembered: carried into the next pass +// it would report a recovered holder as dead, or a dead one as serving, which is +// the judgement this whole path exists to make. +const axios = require('axios'); +const log = require('../../lib/log'); +const fluxCommunication = require('../fluxCommunication'); +const syncthingService = require('../syncthingService'); +const globalState = require('../utils/globalState'); +const { extractIp, extractPort } = require('../utils/socketAddressUtils'); + +// Bounded because this runs on the pass a node is about to promote, and a slow +// peer must not hold the promotion open. +const PROBE_TIMEOUT_MS = 10 * 1000; + +// What proportion of this node's peers must still be answering before it will +// conclude that an unreachable holder is dead rather than that it is itself cut +// off. A proportion, not a count: an absolute floor is a fleet size in disguise, +// and a node holding two peers could never clear one written for a node holding +// twelve - trading a two-hour stall for a permanent one. +// +// This detects total isolation, which is what it claims. It does NOT establish +// that this node is on the majority side of a partial split; no local count can, +// and pretending otherwise is how the second writer gets made. +const MIN_RESPONDING_PEER_FRACTION = 0.5; + +/** + * Ask one peer what it is holding. + * + * Three outcomes, because the callers need to tell them apart: + * + * REACHABLE AND ANSWERABLE - the peer replied with an answer. `ready` and + * `folders` carry it. + * + * REACHABLE BUT NOT ANSWERABLE - the peer replied with an error status. It is + * alive, and that is the half that matters most: a peer that answered anything + * is not a peer that has died, and treating it as dead drops a live holder out + * of the election. It cannot answer THIS question - the endpoint is new, so + * every node that has not been upgraded yet replies 404 - and that is not the + * same as "has not looked yet": an older peer will never grow the endpoint, so + * it never resolves the way an unready peer does. + * + * NOT REACHABLE - no reply at all. Whether the peer is dead or this node is cut + * off is the question its callers then have to answer. + * + * @param {string} socketAddr Peer socket address + * @returns {Promise<{reachable: boolean, answerable: boolean, ready: boolean, folders: string[]}>} + */ +async function probePeer(socketAddr) { + const ip = extractIp(socketAddr); + const port = extractPort(socketAddr); + try { + const response = await axios.get(`http://${ip}:${port}/apps/promotedfolders`, { timeout: PROBE_TIMEOUT_MS }); + const answer = response.data?.data; + // A peer that has not completed its first monitor pass cannot tell "I hold + // nothing" from "I have not looked", so its empty list is not a clearance. + const ready = answer?.ready === true; + const folders = Array.isArray(answer?.folders) ? answer.folders : []; + return { reachable: true, answerable: true, ready, folders }; + } catch (error) { + // error.response exists only when the peer sent one, so this separates a + // reply we cannot use from no reply at all. + if (error.response) { + log.info(`peerFolderLiveness - ${ip} answered ${error.response.status} and cannot say which folders it holds`); + return { reachable: true, answerable: false, ready: false, folders: [] }; + } + log.info(`peerFolderLiveness - could not read ${ip}: ${error.message}`); + return { reachable: false, answerable: false, ready: false, folders: [] }; + } +} + +/** + * A pass's view of its peers. Every peer is asked at most once; `prewarm` asks a + * whole set at once so the pass pays one timeout rather than one per folder, and + * `read` answers from that set or asks on demand for a peer it did not cover. + * Both share one map of in-flight requests, so a peer is never asked twice even + * when the two paths race. + * @returns {{read: Function, prewarm: Function, localConnectivity: Function}} + */ +function createPeerFolderLiveness() { + const answers = new Map(); + let connectivity = null; + + const read = (socketAddr) => { + if (!answers.has(socketAddr)) answers.set(socketAddr, probePeer(socketAddr)); + return answers.get(socketAddr); + }; + + return { + read, + + /** + * Ask every given peer at once. Duplicates collapse, and a peer already read + * is not asked again. + * @param {Iterable} socketAddrs Peer socket addresses + * @returns {Promise} + */ + async prewarm(socketAddrs) { + await Promise.all([...new Set(socketAddrs)].map((addr) => read(addr))); + }, + + /** + * Whether this node can still see the fleet, decided once for the pass. Two + * folders in one pass must not reach opposite conclusions about whether the + * silence is a peer's or this node's own. + * @returns {{connected: boolean, responding: number, total: number}} + */ + localConnectivity() { + if (connectivity === null) { + const { responding, total } = fluxCommunication.peerResponsiveness(); + // No peers at all is not evidence of health: this node holds an app whose + // other holders exist, so having nobody to talk to is itself the isolation + // case. + const connected = total > 0 && responding >= Math.ceil(total * MIN_RESPONDING_PEER_FRACTION); + connectivity = { connected, responding, total }; + } + return connectivity; + }, + }; +} + +const PeerConnection = Object.freeze({ + CONNECTED: 'connected', + DISCONNECTED: 'disconnected', + UNKNOWN: 'unknown', +}); + +/** + * Why this node cannot hear a peer. `GONE` is the only answer that authorises + * acting on the silence; the other three are reasons to leave the peer alone. + */ +const SilenceVerdict = Object.freeze({ + GONE: 'gone', + CONNECTION_ALIVE: 'connectionAlive', + NO_EVIDENCE: 'noEvidence', + LOCALLY_ISOLATED: 'locallyIsolated', +}); + +/** + * The syncthing device id this node knows the peer by, or null. + * + * @param {string} peerIp + * @returns {Promise} + */ +async function peerDeviceId(peerIp) { + const name = `${extractIp(peerIp)}:${extractPort(peerIp)}`; + const cached = globalState.syncthingDevicesIDCache.get(name); + if (cached) return cached; + + // That cache is in-memory and is filled by asking the PEER, so it is empty for + // exactly the peer this matters for: one that died while this node's own process + // was restarting, and can no longer be asked anything. This node's syncthing + // holds the answer on disk - the monitor configured the device under this same + // name while the peer was still up, and that config outlives both processes. + // Left unread, the node with a perfectly good local record would report itself + // ignorant and hold a start for as long as the dead peer's location record lives. + const devices = await syncthingService.getConfigDevices().catch(() => null); + if (!Array.isArray(devices)) return null; + return devices.find((device) => device.name === name)?.deviceID ?? null; +} + +/** + * This node's syncthing's view of its own connection to a peer, for one + * folder: PeerConnection.CONNECTED, DISCONNECTED, or UNKNOWN. + * + * FluxOS and syncthing are separate processes on that peer and fail + * independently, so silence from its API is not evidence that it has stopped + * writing - a FluxOS restart takes the API away for tens of seconds while + * syncthing and the container carry on. A live sync connection IS evidence: + * syncthing reports remoteState 'valid' only for a device whose connection is + * open, and clears it the moment the connection closes. + * + * The three answers are kept apart because a silence may be acted on only with + * evidence. 'disconnected' is an answer - this node's syncthing was asked + * about the device and does not consider it connected. 'unknown' is the + * absence of one - the peer's device is in neither this node's cache nor its + * syncthing's own device config, or this node's syncthing did not answer. + * Collapsing 'unknown' into 'disconnected' would let the one node with the + * least knowledge authorise a second writer. + * + * The peer's own syncthing API is not reachable - it binds to localhost - and + * does not need to be. This is local state, maintained by the connection + * itself. + * + * @param {string} folderId Folder id, always passed explicitly: the completion + * endpoint's aggregate form never sets remoteState and reports 'unknown'. + * @param {string} peerIp + * @returns {Promise} One of PeerConnection + */ +async function peerSyncthingConnection(folderId, peerIp) { + const deviceId = await peerDeviceId(peerIp); + if (!deviceId) return PeerConnection.UNKNOWN; + + const completion = await syncthingService.getDbCompletion({ + folder: folderId, + device: deviceId, + }).catch(() => null); + + if (!completion) return PeerConnection.UNKNOWN; + return completion.remoteState === 'valid' ? PeerConnection.CONNECTED : PeerConnection.DISCONNECTED; +} + +/** + * What a peer's silence is worth as evidence that it has stopped: one of + * SilenceVerdict. The caller has already established the silence - this + * answers only whether it may be acted on. + * + * Two decisions rest on this and they ask it identically: dropping a dead + * holder out of the election, and starting a container a peer may still be + * running. Both turn a silence into an action over a shared volume, so both + * owe the same proof. + * + * Silence alone is never enough. The verdict needs this node's own syncthing + * to have been asked about the peer's device and to have answered that it is + * not connected, AND this node to still be able to see the fleet - a node + * whose peers have all gone quiet is the one that fell over, and the peer is + * very likely still serving on the other side of the split. + * + * @param {string} folderId + * @param {string} peerIp + * @param {Object} liveness This pass's peer view + * @returns {Promise} One of SilenceVerdict + */ +async function silenceVerdict(folderId, peerIp, liveness) { + const connection = await peerSyncthingConnection(folderId, peerIp); + if (connection === PeerConnection.CONNECTED) return SilenceVerdict.CONNECTION_ALIVE; + if (connection === PeerConnection.UNKNOWN) return SilenceVerdict.NO_EVIDENCE; + if (!liveness.localConnectivity().connected) return SilenceVerdict.LOCALLY_ISOLATED; + return SilenceVerdict.GONE; +} + +module.exports = { + createPeerFolderLiveness, + silenceVerdict, + SilenceVerdict, + PROBE_TIMEOUT_MS, + MIN_RESPONDING_PEER_FRACTION, +}; diff --git a/ZelBack/src/services/appMonitoring/syncthingEventsConsumer.js b/ZelBack/src/services/appMonitoring/syncthingEventsConsumer.js index 85e3e2ed16..7ad6f10de3 100644 --- a/ZelBack/src/services/appMonitoring/syncthingEventsConsumer.js +++ b/ZelBack/src/services/appMonitoring/syncthingEventsConsumer.js @@ -50,23 +50,27 @@ const controller = new FluxController(); // folderId -> { time, errors } from the last FolderErrors event const folderErrorsByFolder = new Map(); -// folder ids seen in FolderErrors since the last drain - the monitor pass -// drains this to mount-verify exactly the flagged folders (targeted reaction, -// never a steady-state sweep) -const erroredFolderIdsSinceLastDrain = new Set(); +// Folder ids whose FolderErrors signal has not yet been ACTED on. This is a +// durable level, not a drained edge: reading it never clears it - an entry +// resolves only when the monitor's safety action completes (demotion landed, +// folder verified safe, or no installed app carries the folder any more). A +// pass that fails mid-action therefore changes nothing and the next pass +// retries by construction; the guard never depends on the signal re-firing. +const mountVerifyPending = new Set(); /** * One long-poll iteration: fetch events after `since`, detect lost-events * conditions, surface folder activity to the monitor. */ async function pollOnce() { - const response = await syncthingService.getEvents({ - params: {}, - query: { since, events: SUBSCRIBED_EVENTS, timeout: EVENTS_LONGPOLL_TIMEOUT_S }, + const events = await syncthingService.getEvents({ + since, + events: SUBSCRIBED_EVENTS, + timeout: EVENTS_LONGPOLL_TIMEOUT_S, signal: controller.signal, - }, null); + }); - if (!response || response.status !== 'success' || !Array.isArray(response.data)) { + if (!Array.isArray(events)) { throw new Error('syncthing events endpoint unavailable'); } @@ -77,7 +81,6 @@ async function pollOnce() { if (callbacks.onResync) callbacks.onResync(); } - const events = response.data; if (events.length === 0) return; // long-poll timeout with nothing new const firstId = events[0].id; @@ -105,7 +108,7 @@ async function pollOnce() { } if (event.type === 'FolderErrors') { folderErrorsByFolder.set(folder, { time: event.time, errors: event.data.errors || [] }); - erroredFolderIdsSinceLastDrain.add(folder); + mountVerifyPending.add(folder); fluxEventBus.publish('syncthing:folderErrors', { folder, time: event.time, errors: event.data.errors || [] }); } if (callbacks.onFolderActivity) callbacks.onFolderActivity(folder, event.type); @@ -116,8 +119,13 @@ async function pollOnce() { // controller's lock for each iteration so stop() (controller.abort()) returns only // after the in-flight iteration finishes; the long-poll is aborted via the signal // and every inter-poll wait is a cancellable controller.sleep, so stop() is prompt. +// +// Conditioned on `active` rather than on the signal: the signal is reissued when +// the abort finishes, so an iteration that yields and re-reads it afterwards +// sees a controller that was never stopped and polls on for the life of the +// process. async function runLoop() { - while (!controller.aborted) { + while (controller.active) { const startedAt = Date.now(); // eslint-disable-next-line no-await-in-loop await controller.lock.enable(); @@ -132,7 +140,7 @@ async function runLoop() { } catch (error) { // a deliberate stop() aborts the in-flight long-poll (or the sleep above), // which surfaces here - exit rather than re-anchoring and backing off - if (controller.aborted) break; + if (!controller.active) break; log.warn(`syncthingEventsConsumer - poll failed (${error.message}); retrying in ${EVENTS_RETRY_DELAY_MS / 1000}s (the periodic poll keeps covering meanwhile)`); // a syncthing restart resets the event ids, and the API never returns // events below a stale `since` - after any failure the position is @@ -184,14 +192,26 @@ function getFolderErrors(folderId) { } /** - * Folder ids flagged by FolderErrors since the last drain; draining clears - * the accumulator. Consumed by the monitor pass for targeted mount verifies. + * Folder ids flagged by FolderErrors and not yet acted on. Non-destructive: + * the monitor pass reads this to mount-verify exactly the flagged folders + * (targeted reaction, never a steady-state sweep) and resolves each id only + * once its safety action actually completed. * @returns {string[]} Folder ids */ -function drainErroredFolderIds() { - const ids = [...erroredFolderIdsSinceLastDrain]; - erroredFolderIdsSinceLastDrain.clear(); - return ids; +function mountVerifyPendingIds() { + return [...mountVerifyPending]; +} + +/** + * The flagged folder's safety handling completed: demotion landed, the mount + * verified safe, or no installed app carries the folder. Only these outcomes + * clear the flag - never the act of reading it. The diagnostic record in + * folderErrorsByFolder is deliberately untouched: pull errors must survive + * for diagnosis past the mount question. + * @param {string} folderId + */ +function resolveMountVerify(folderId) { + mountVerifyPending.delete(folderId); } module.exports = { @@ -199,5 +219,6 @@ module.exports = { stop, isRunning, getFolderErrors, - drainErroredFolderIds, + mountVerifyPendingIds, + resolveMountVerify, }; diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index e9578746fb..605a33e09d 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -5,12 +5,14 @@ const log = require('../../lib/log'); const dockerService = require('../dockerService'); const appReconciler = require('./appReconciler'); const appUninstaller = require('../appLifecycle/appUninstaller'); +const messageHelper = require('../messageHelper'); const syncthingService = require('../syncthingService'); const serviceHelper = require('../serviceHelper'); -const volumeService = require('../utils/volumeService'); const { appsFolder } = require('../utils/appConstants'); const appTamperingDetectionService = require('../appTamperingDetectionService'); -const { socketAddressesMatch } = require('../utils/socketAddressUtils'); +const { socketAddressesMatch, extractIp } = require('../utils/socketAddressUtils'); +const fluxEventBus = require('../utils/fluxEventBus'); +const { silenceVerdict, SilenceVerdict } = require('./peerFolderLiveness'); const { LEADER_CONFIRM_COUNT, SYNC_COMPLETE_PERCENTAGE, @@ -22,7 +24,7 @@ const { ACTIVE_FOLDER_STATES, } = require('./syncthingMonitorConstants'); -const { isPathMounted } = volumeService; +const { isPathMounted } = require('../utils/volumeService'); const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); @@ -283,51 +285,81 @@ async function fixAppdataPermissions(appId) { } /** - * Helper function to get Syncthing folder sync completion status + * Reads a folder's sync completion, and says which of the two ways it failed. + * + * "Syncthing says there is no such folder" is a finding about the data. "Syncthing + * did not answer" is a finding about syncthing, and the caller that refuses a + * backup must not report the second as the first - telling an operator their + * instance has never synced, when what happened is that a daemon was restarting, + * is a false statement about their data at the moment they are trying to protect + * it. + * + * Only an HTTP status proves syncthing replied at all, and performRequest keeps + * it in the error message, so that is what separates the two. Anything that is + * not a plain 404 - a transport failure, a 500, an unreadable api key - is + * unknown rather than absent, because none of them are the folder telling us + * anything. + * * @param {string} folderId - The Syncthing folder ID - * @returns {Promise} Sync status object or null if unavailable + * @returns {Promise<{status: Object|null, reason: 'ok'|'absent'|'unknown'}>} */ -async function getFolderSyncCompletion(folderId) { +async function probeFolderSyncCompletion(folderId) { try { - const statusResponse = await syncthingService.getDbStatus({ - query: { folder: folderId }, - }, null); - - if (statusResponse && statusResponse.status === 'success') { - const { - globalBytes = 0, inSyncBytes = 0, state, receiveOnlyChangedFiles = 0, - } = statusResponse.data; - - const syncPercentage = globalBytes > 0 ? (inSyncBytes / globalBytes) * 100 : 100; - - return { - syncPercentage, - globalBytes, - inSyncBytes, - state, - // local additions/modifications in a receiveonly folder; invisible to the - // completion metrics above (they only count cluster data) - receiveOnlyChangedFiles, - // An EMPTY global index (globalBytes 0) means "unknown / not yet synced", - // never "done": a node holding the only copy before its peers reconnect - // reads globalBytes 0, and syncPercentage defaults to 100 there (vacuous). - // Gating on globalBytes > 0 stops the promotion gate from reverting (which - // would delete the only copy) or promoting unverified data against an empty - // global; such a folder falls through to the wait branch instead. The - // leader/cold-start path (the legitimate empty-folder seed) is exempt and - // handled separately above. - isSynced: globalBytes > 0 && syncPercentage === SYNC_COMPLETE_PERCENTAGE, - }; - } - - log.warn(`Failed to get sync status for folder ${folderId}`); - return null; + const { + globalBytes = 0, inSyncBytes = 0, state, receiveOnlyChangedFiles = 0, + } = await syncthingService.getDbStatus(folderId); + + const syncPercentage = globalBytes > 0 ? (inSyncBytes / globalBytes) * 100 : 100; + + const status = { + syncPercentage, + globalBytes, + inSyncBytes, + state, + // local additions/modifications in a receiveonly folder; invisible to the + // completion metrics above (they only count cluster data) + receiveOnlyChangedFiles, + // An EMPTY global index (globalBytes 0) means "unknown / not yet synced", + // never "done": a node holding the only copy before its peers reconnect + // reads globalBytes 0, and syncPercentage defaults to 100 there (vacuous). + // Gating on globalBytes > 0 stops the promotion gate from reverting (which + // would delete the only copy) or promoting unverified data against an empty + // global; such a folder falls through to the wait branch instead. The + // leader/cold-start path (the legitimate empty-folder seed) is exempt and + // handled separately above. + isSynced: globalBytes > 0 && syncPercentage === SYNC_COMPLETE_PERCENTAGE, + }; + + return { status, reason: 'ok' }; } catch (error) { - log.error(`Error checking sync completion for ${folderId}: ${error.message}`); - return null; + // A 404 is syncthing answering that it holds no such folder. Anything else - + // transport, a refused key, a malformed reply - leaves the folder's state + // unknown, which is a different claim and must never read as absence. + if (error.httpStatus === 404) { + log.warn(`No syncthing folder ${folderId}`); + return { status: null, reason: 'absent' }; + } + log.warn(`Could not read sync status for folder ${folderId}: ${error.message}`); + return { status: null, reason: 'unknown' }; } } +/** + * The folder's sync status, or null when it cannot be read for any reason. + * + * Callers that only need "do I have a usable reading" keep this contract: both + * failures are equally unusable to them, and both must be treated conservatively. + * Callers that report the failure to a person want probeFolderSyncCompletion. + * + * @param {string} folderId - The Syncthing folder ID + * @returns {Promise} Sync status object or null if unavailable + */ +async function getFolderSyncCompletion(folderId) { + const { status } = await probeFolderSyncCompletion(folderId); + return status; +} + + /** * Determines if this node should be the designated leader for starting an app first. * Uses deterministic leader election to prevent race conditions. @@ -336,6 +368,17 @@ async function getFolderSyncCompletion(folderId) { * @param {string} localSocketAddr - The current node's IP address * @returns {boolean} True if this node is the designated leader */ +// Lowest IP among the holders - the deterministic pick every node computes +// identically. Identity only: it says nothing about whether that node still exists. +function lowestIpHolder(allPeersList) { + const sorted = [...(allPeersList || [])].sort((a, b) => { + if (a.ip < b.ip) return -1; + if (a.ip > b.ip) return 1; + return 0; + }); + return sorted[0]?.ip ?? null; +} + function isDesignatedLeader(allPeersList, localSocketAddr, deferToRunningPeers = true) { if (!allPeersList || allPeersList.length === 0) { return false; // Be conservative - wait for peers to broadcast @@ -364,18 +407,170 @@ function isDesignatedLeader(allPeersList, localSocketAddr, deferToRunningPeers = // re-broadcast time and propagates with per-node delay, so on a fresh cluster each // node can momentarily order the timestamps differently and every node elects itself // (split-brain). The lowest IP is the single, agreed cold-start seed. - const sortedPeers = [...allPeersList].sort((a, b) => { - if (a.ip < b.ip) return -1; - if (a.ip > b.ip) return 1; - return 0; - }); - - const leader = sortedPeers[0]; - const isLeader = socketAddressesMatch(leader?.ip, localSocketAddr); + const leader = lowestIpHolder(allPeersList); + const isLeader = socketAddressesMatch(leader, localSocketAddr); return isLeader && allPeersList.some((peer) => socketAddressesMatch(peer.ip, localSocketAddr)); } +/** + * Whether this node can show that a holder is gone, rather than merely silent to + * it. Same question the promotion check asks, one step earlier: the election picks + * by identity and has no liveness in it, so a holder that dies keeps being elected + * by everyone else and they defer to it until its location broadcast expires - + * 125 minutes, with the app down throughout. + * + * Answered from this node's own connectivity, which is the only half it can know: + * a node still trading pings with the fleet is watching one holder fall over, a + * node whose peers have all gone quiet is the one that fell over and must keep + * deferring - the holder is very likely still serving on the other side of the + * split. + * + * "Gone" means silent, and only silent. A holder that answers - even to say it + * cannot answer this question - is alive, and dropping a live holder out of the + * election is the one thing this must never do: every survivor would then pick the + * next IP and promote alongside a holder that is still writing. + * + * And silence alone is still not enough: gone requires evidence. The verdict + * needs this node's own syncthing to have been asked about the holder's device + * and answered that it is not connected. A device this node cannot ask about + * proves nothing, and a proof of nothing keeps the holder. + * + * @param {string} appId + * @param {string} holderIp + * @returns {Promise} + */ +async function holderIsGone(appId, holderIp, liveness) { + const answer = await liveness.read(holderIp); + if (answer.reachable) return false; + + const verdict = await silenceVerdict(appId, holderIp, liveness); + if (verdict === SilenceVerdict.CONNECTION_ALIVE) { + log.info(`holderIsGone - ${extractIp(holderIp)}'s API is silent, but this node's syncthing still holds a live connection to it for ${appId}; it is restarting, not gone`); + fluxEventBus.publish('syncthing:holderRetained', { folder: appId, holder: holderIp, reason: 'connectionAlive' }); + return false; + } + if (verdict === SilenceVerdict.NO_EVIDENCE) { + log.info(`holderIsGone - ${extractIp(holderIp)} is unreachable, but this node cannot ask its own syncthing about it for ${appId}; gone requires evidence, keeping it`); + fluxEventBus.publish('syncthing:holderRetained', { folder: appId, holder: holderIp, reason: 'noEvidence' }); + return false; + } + if (verdict === SilenceVerdict.LOCALLY_ISOLATED) { + const { responding, total } = liveness.localConnectivity(); + log.info(`holderIsGone - ${extractIp(holderIp)} is unreachable, but only ${responding} of this node's ${total} peers are answering; treating this node as the isolated one`); + return false; + } + return true; +} + +/** + * The holder list with the elected leader removed when this node can show it is + * gone. One holder per pass: if the next-lowest is also gone, the following pass + * drops that one too, so a run of failures converges without a loop here. Every + * survivor drops the same holder and then picks the same lowest IP of what is + * left, so they agree without coordinating, and the promotion check still catches + * any second node that acts on it. + * + * @param {string} appId + * @param {Array} allPeersList + * @param {string} localSocketAddr + * @param {Object} liveness This pass's peer view + * @returns {Promise>} + */ +async function holderListExcludingDead(appId, allPeersList, localSocketAddr, liveness) { + const leader = lowestIpHolder(allPeersList); + if (!leader || socketAddressesMatch(leader, localSocketAddr)) return allPeersList; + if (!await holderIsGone(appId, leader, liveness)) return allPeersList; + log.warn(`holderListExcludingDead - elected holder ${leader} is gone and this node's own connectivity is healthy; re-electing without it`); + fluxEventBus.publish('syncthing:holderExcluded', { folder: appId, holder: leader }); + return allPeersList.filter((peer) => !socketAddressesMatch(peer.ip, leader)); +} + +/** + * The first peer found already holding a writable copy of this folder, or null. + * + * Two answers block, for different reasons, and the difference is the whole point + * of asking: + * + * UNREACHABLE blocks only while this node cannot show that the silence is the + * peer's and not its own. "That peer is dead" and "I have been cut off" look + * identical from the failed request, and they need opposite answers: the first + * means promote, the second means do not. The node cannot prove a peer is alive + * without agreement, but it can answer whether IT is - a node still exchanging + * pings with the rest of the fleet is watching one node fall over, while a node + * whose peers have all gone quiet is the one that fell over. Once that is + * established the peer is treated as gone, because a dead node must never strand + * an app with no writable copy anywhere. + * + * UNREADY does block, with no bound and none needed. The peer is alive and + * saying it cannot answer yet, which is a live node that may already be holding. + * It resolves itself: the peer finishes its first monitor pass and answers, or + * it stops responding and becomes the unreachable case above. A peer that stays + * alive and permanently unreadable is the one case that waits indefinitely, and + * waiting is right there - promoting because we gave up is exactly the second + * writer this exists to prevent, and a stalled app is visible where diverged + * data is not. + * + * UNANSWERABLE does NOT block, and must not. A peer that predates this endpoint + * is alive and cannot be asked, ever - it will not finish a pass and start + * answering, so the unbounded wait UNREADY earns by resolving itself is not + * earned here. Blocking on it would hold promotion open until somebody upgrades + * that node: on a cold start every other holder defers to the same lowest IP, so + * one un-upgraded peer would stop the app starting anywhere. This check simply + * cannot cover a peer that cannot answer, and the honest reading of that is that + * the folder gets the behaviour it had before this check existed - decided by the + * election alone - rather than a guess dressed as a guarantee. Peers that CAN + * answer are still checked, so the cover grows as the fleet upgrades and is + * complete once it has. + * + * It narrows the window rather than closing it: two nodes that both ask before + * either promotes still both promote. Closing that needs the consensus-grounded + * election the residual-limitation note above describes. + * + * @param {string} appId Folder id + * @param {Array} peers App location entries + * @param {string} localSocketAddr This node's socket address + * @param {Object} liveness This pass's peer view + * @returns {Promise<{ip: string, reason: string}|null>} The blocking peer, or null + */ +async function findPeerBlockingPromotion(appId, peers, localSocketAddr, liveness) { + const others = (peers || []).filter((peer) => peer?.ip && !socketAddressesMatch(peer.ip, localSocketAddr)); + if (!others.length) return null; + + const answers = await Promise.all(others.map( + async (peer) => ({ ip: peer.ip, ...await liveness.read(peer.ip) }), + )); + + const holder = answers.find((answer) => answer.reachable && answer.ready && answer.folders.includes(appId)); + if (holder) return { ip: holder.ip, reason: 'already holds the writable copy' }; + const unready = answers.find((answer) => answer.reachable && answer.answerable && !answer.ready); + if (unready) return { ip: unready.ip, reason: 'has not determined its folder state yet' }; + + // Recorded, not blocked - see UNANSWERABLE above. Worth a line of its own so a + // promotion made without full cover is visible as that, and so the remedy reads + // as "upgrade that node" rather than "debug its monitor". + const unanswerable = answers.find((answer) => answer.reachable && !answer.answerable); + if (unanswerable) { + log.info(`findPeerBlockingPromotion - ${appId}: ${extractIp(unanswerable.ip)} is alive but cannot be asked which folders it holds; promoting on the election alone, as this node would have before this check existed`); + } + + const unreachable = answers.find((answer) => !answer.reachable); + if (unreachable) { + // Whose silence is it? A node still trading pings with the fleet is watching a + // peer die; a node whose own peers have gone quiet is the one that is cut off, + // and must not promote over a holder that is very likely still running on the + // other side of the split. + const { connected, responding, total } = liveness.localConnectivity(); + if (!connected) { + return { + ip: unreachable.ip, + reason: `is unreachable, and only ${responding} of this node's ${total} peers are answering - it cannot tell that peer apart from its own isolation`, + }; + } + } + return null; +} + /** * Handle first run scenario for an app/component * @param {Object} params - Parameters @@ -471,12 +666,9 @@ async function handleSkippedAppSecondEncounter(params) { async function checkIfPeersAreSynced(folderId) { try { // Get all Syncthing folders - const configResponse = await syncthingService.getConfig({}, null); - if (!configResponse || configResponse.status !== 'success') { - return false; - } + const config = await syncthingService.getConfig(); - const folder = configResponse.data.folders?.find((f) => f.id === folderId); + const folder = config.folders?.find((f) => f.id === folderId); if (!folder) { return false; } @@ -487,53 +679,105 @@ async function checkIfPeersAreSynced(folderId) { return true; } - // Check remote devices for this folder + return Boolean(await findSyncedPeer(folderId)); + } catch (error) { + log.error(`checkIfPeersAreSynced - Error checking peers for ${folderId}: ${error.message}`); + return false; + } +} + +/** + * The first connected peer that demonstrably holds everything in this folder. + * + * Unlike checkIfPeersAreSynced this never answers from our OWN folder mode: a + * caller about to delete its local copy needs a peer that holds the data, and + * "we are sendreceive" says nothing about where else the data lives. + * @param {string} folderId - Syncthing folder ID + * @returns {Promise<{deviceID: string, globalBytes: number}|null>} The peer, or + * null when no peer can be shown to hold this folder. + */ +async function findSyncedPeer(folderId) { + try { + // getConfig takes no request and answers with the config itself, not a + // {status, data} envelope - the same call its sibling checkIfPeersAreSynced + // makes twenty lines up. Called the old way, this returned early on every + // invocation and findSyncedPeer answered null without ever asking a device, + // which reads as "no peer holds this data" and is the answer that keeps an + // app rather than removing it. Six unit tests caught it; nothing in the + // stacking rebase did, because both spellings parse. + const config = await syncthingService.getConfig(); + + const folder = config?.folders?.find((f) => f.id === folderId); + if (!folder) { + return null; + } + const { devices = [] } = folder; if (devices.length === 0) { - return false; + return null; } + // Every folder's device list BEGINS with this node's own device - see + // syncthingMonitorHelpers, `const devices = [{ deviceID: myDeviceId }]` - + // and this walk had no self-exclusion, so it asked /rest/db/completion + // about the local device first. Our own copy trivially reports completion + // 100 with globalBytes > 0. + // + // What separates "a peer holds it" from "I hold it" today is only that + // syncthing does not report remoteState 'valid' for the local device: an + // incidental property of a field read defensively below, with a default, + // rather than an intention. Every other device walk in this codebase + // excludes local explicitly. If that assumption were ever wrong, + // canSafelyRemoveApp would return safe for a single-copy stateful app on + // the strength of the copy it is about to delete. + const localDeviceId = await syncthingService.getDeviceId().catch(() => null); + // Get device completion status for each remote device // eslint-disable-next-line no-restricted-syntax for (const device of devices) { + // A device id this node could not establish excludes nothing, which is + // the safe direction: the completion checks below still have to pass. + if (localDeviceId && device.deviceID === localDeviceId) { + // eslint-disable-next-line no-continue + continue; + } try { // eslint-disable-next-line no-await-in-loop - const completionResponse = await syncthingService.getDbCompletion({ - query: { folder: folderId, device: device.deviceID }, - }, null); - - if (completionResponse?.status === 'success' && completionResponse.data) { - const { completion = 0, globalBytes = 0, remoteState = 'unknown' } = completionResponse.data; - // A peer is a safe source only if it is CONNECTED (remoteState 'valid'), - // reports 100%, AND actually holds data: - // - db/completion is computed from the peer's last-known index, so a dead or - // offline peer still reports completion 100. Trusting that stale figure - // turns a source-node reboot into followers deleting their partial copies. - // remoteState is the connectivity discriminator ('valid' iff connected); - // when absent, there is no evidence and the peer must not be trusted. - // - Syncthing reports completion 100 for an empty folder (globalBytes 0) too, - // so without the globalBytes check a peer that synced empty/wrong data from - // a bad seed would falsely satisfy "peers are synced" and we would remove - // the good local copy in favour of an empty one (data loss). - if (remoteState === 'valid' && completion === 100 && globalBytes > 0) { - log.info(`checkIfPeersAreSynced - Found synced peer for ${folderId}: device ${device.deviceID.substring(0, 7)}... at ${completion}% (${globalBytes} bytes, connected)`); - return true; - } - if (completion === 100 && remoteState !== 'valid') { - log.warn(`checkIfPeersAreSynced - ${folderId}: device ${device.deviceID.substring(0, 7)}... reports 100% but is not connected (remoteState ${remoteState}); stale index, not a synced source`); - } else if (completion === 100) { - log.warn(`checkIfPeersAreSynced - ${folderId}: device ${device.deviceID.substring(0, 7)}... reports 100% but 0 bytes (empty); not treating it as a synced source`); - } + const { completion = 0, globalBytes = 0, remoteState = 'unknown' } = await syncthingService.getDbCompletion({ + folder: folderId, + device: device.deviceID, + }); + // A peer is a safe source only if it is CONNECTED (remoteState 'valid'), + // reports 100%, AND actually holds data: + // - db/completion is computed from the peer's last-known index, so a dead or + // offline peer still reports completion 100. Trusting that stale figure + // turns a source-node reboot into followers deleting their partial copies. + // remoteState is the connectivity discriminator ('valid' iff connected); + // when absent, there is no evidence and the peer must not be trusted. + // - Syncthing reports completion 100 for an empty folder (globalBytes 0) too, + // so without the globalBytes check a peer that synced empty/wrong data from + // a bad seed would falsely satisfy "peers are synced" and we would remove + // the good local copy in favour of an empty one (data loss). + if (remoteState === 'valid' && completion === 100 && globalBytes > 0) { + log.info(`findSyncedPeer - Found synced peer for ${folderId}: device ${device.deviceID.substring(0, 7)}... at ${completion}% (${globalBytes} bytes, connected)`); + return { deviceID: device.deviceID, globalBytes }; + } + if (completion === 100 && remoteState !== 'valid') { + log.warn(`findSyncedPeer - ${folderId}: device ${device.deviceID.substring(0, 7)}... reports 100% but is not connected (remoteState ${remoteState}); stale index, not a synced source`); + } else if (completion === 100) { + log.warn(`findSyncedPeer - ${folderId}: device ${device.deviceID.substring(0, 7)}... reports 100% but 0 bytes (empty); not treating it as a synced source`); } } catch (deviceError) { - log.warn(`checkIfPeersAreSynced - Error checking device ${device.deviceID}: ${deviceError.message}`); + // a failed completion read silently skipping the device would read as + // "peer not synced" with zero diagnostics - fail-safe, but loud + log.warn(`findSyncedPeer - ${folderId}: completion read for device ${device.deviceID.substring(0, 7)}... failed: ${deviceError.message}`); } } - return false; + return null; } catch (error) { - log.error(`checkIfPeersAreSynced - Error checking peers for ${folderId}: ${error.message}`); - return false; + log.error(`findSyncedPeer - Error checking peers for ${folderId}: ${error.message}`); + return null; } } @@ -548,16 +792,15 @@ async function checkIfPeersAreSynced(folderId) { */ async function nudgeFolderDevices(folderId) { try { - const configResponse = await syncthingService.getConfig({}, null); - if (!configResponse || configResponse.status !== 'success') return; - const folder = configResponse.data.folders?.find((f) => f.id === folderId); + const config = await syncthingService.getConfig(); + const folder = config.folders?.find((f) => f.id === folderId); if (!folder) return; // eslint-disable-next-line no-restricted-syntax for (const device of folder.devices || []) { let paused = false; try { // eslint-disable-next-line no-await-in-loop - await syncthingService.systemPause({ params: { device: device.deviceID }, query: {} }, null); + await syncthingService.systemPause(device.deviceID); paused = true; // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(OPERATION_DELAY_MS); @@ -572,7 +815,7 @@ async function nudgeFolderDevices(folderId) { if (paused) { try { // eslint-disable-next-line no-await-in-loop - await syncthingService.systemResume({ params: { device: device.deviceID }, query: {} }, null); + await syncthingService.systemResume(device.deviceID); } catch (error) { log.error(`nudgeFolderDevices - ${folderId}: RESUME of device ${device.deviceID.substring(0, 7)} FAILED - device left paused (its connection stays suspended): ${error.message}`); } @@ -597,10 +840,9 @@ async function handleReceiveOnlyTransition(params) { localSocketAddr, containerDataFlags, syncthingFolder, + liveness, } = params; - log.info(`handleReceiveOnlyTransition - ${appId} in cache and not restarted, processing receive-only logic`); - const folderPath = syncthingFolder.path || `${appsFolder}${appId}/appdata`; // Whether any CONNECTED peer genuinely holds the data. Gates the election (a true @@ -622,9 +864,30 @@ async function handleReceiveOnlyTransition(params) { // LEADER_CONFIRM_COUNT consecutive cycles, so a single transient peer-visibility blip // doesn't flip a follower to leader. Defer to a running peer UNLESS this is a true, // safe cold start (no peer serving AND this node holds no data) - then elect one seed. - const electedLeader = isDesignatedLeader(runningAppList, localSocketAddr, aPeerHasData || !folderIsEmpty); - cache.leaderStreak = electedLeader ? (cache.leaderStreak || 0) + 1 : 0; + // The election picks by identity and carries no liveness, so a holder that dies + // keeps winning and every survivor defers to it until its location broadcast + // expires - 125 minutes with the app down. Dropped from the list here, before the + // pick, when this node can show the holder is gone rather than merely silent to it. + const electionList = await holderListExcludingDead(appId, runningAppList, localSocketAddr, liveness); + const electedLeader = isDesignatedLeader(electionList, localSocketAddr, aPeerHasData || !folderIsEmpty); + // The floor holderIsGone asks of a silent holder, asked of this node before + // its own win can count: a node whose peers have gone quiet is the one that + // fell over, and a win it confirms in that state seeds the app on a + // partition's minority side while the majority defers to its IP. Isolation + // resets the streak rather than pausing it, so a heal is followed by + // LEADER_CONFIRM_COUNT clean passes like any other blip. + const { connected } = liveness.localConnectivity(); + cache.leaderStreak = electedLeader && connected ? (cache.leaderStreak || 0) + 1 : 0; const isLeader = electedLeader && cache.leaderStreak >= LEADER_CONFIRM_COUNT; + // Withdrawn on every unpromoted pass, so a lost election drops the claim + // and the intent behind it. It is raised again only where the promotion is + // APPLIED - the state machine records intent at the last gate, and the + // monitor flips it once the folder batch lands in syncthing. + // masterSlaveApps reads designatedLeader to skip the primary-selection + // index stagger, so it has to mean "the folder is writable", not "won the + // vote" and not "promotion decided". + cache.designatedLeader = false; + cache.designationPending = false; // RESIDUAL LIMITATION (architectural - this election is a heuristic, not consensus): // a confirmed leader is the cold-start seed and flips to sendreceive WITHOUT a sync @@ -642,8 +905,39 @@ async function handleReceiveOnlyTransition(params) { // candidate over the on-chain confirmed node set + a data-aware quorum lease that // subsumes the data-version check) - a separate, proposed redesign, out of scope here. if (isLeader) { + // The seed flip below runs WITHOUT a sync check, and that is only sound when + // there is nothing to lose: an empty folder (the cold start this election + // exists for) or a fully synced copy (a survivor taking over). A node can + // reach a confirmed designation MID-SYNC - its source dropped out of the + // election as provably gone and the list collapsed to itself - and promoting + // there publishes a partial copy as the truth: the files it has not fetched + // yet become deletions on every peer the moment a source returns. A leader + // holding a partial copy therefore waits, receiveonly - either the sync + // completes against a returning source, or the stall ladder decides the data + // question. An unreadable status counts as partial: it cannot show there is + // nothing to lose. + if (!folderIsEmpty && !(syncStatus && syncStatus.isSynced)) { + log.info(`handleReceiveOnlyTransition - ${appId} is the confirmed designated leader but holds a partial copy (${syncStatus ? `${syncStatus.syncPercentage.toFixed(2)}% synced` : 'sync status unreadable'}); staying receiveonly until synced`); + syncthingFolder.type = 'receiveonly'; + return { syncthingFolder, cache }; + } log.info(`handleReceiveOnlyTransition - ${appId} is the designated leader (elected from ${runningAppList.length} peers, confirmed ${cache.leaderStreak}x), starting immediately`); + // Winning the election is not the same as being the first to win it. Each node + // decides from its own view of the holder list, and those views fill in at + // different moments: the first-placed node is briefly the only holder it knows + // of and seeds on that basis, which is correct - somebody has to seed an empty + // folder or the app never starts. A node that can see further then wins the + // tiebreak among the holders it can see and seeds too, and neither revisits it, + // because a promoted folder never re-enters this election. So the last check + // before promoting is whether somebody already has. + const blocker = await findPeerBlockingPromotion(appId, runningAppList, localSocketAddr, liveness); + if (blocker) { + log.info(`handleReceiveOnlyTransition - ${appId} won the election but ${blocker.ip} ${blocker.reason}; staying receiveonly`); + syncthingFolder.type = 'receiveonly'; + return { syncthingFolder, cache }; + } + // A folder must pass the sendreceive safety verification BEFORE it ever // flips - the seed included. An empty cold-start folder passes (empty index // over an empty disk); an unmounted dir, or a stale index claiming bytes @@ -656,6 +950,15 @@ async function handleReceiveOnlyTransition(params) { return { syncthingFolder, cache }; } + // Every gate passed - but deciding the promotion is not applying it. The + // designation masterSlaveApps reads has to mean "the folder IS writable", + // and the type below reaches syncthing only when the monitor applies this + // pass's folder batch - so the claim is recorded as intent here, and the + // monitor raises designatedLeader once the apply lands. Raising it now + // would let the container start against a folder still receiveonly for as + // long as the apply takes. + cache.designationPending = true; + // Fix permissions before changing to sendreceive - ensures correct ownership for synced data await fixAppdataPermissions(appId); @@ -670,6 +973,32 @@ async function handleReceiveOnlyTransition(params) { return { syncthingFolder, cache }; } + // WHY THIS NODE IS NOT THE SOURCE. Only the promoting path used to say + // anything, so every way of losing left the same picture - a receiveonly + // folder, a stopped container and no source - with nothing to tell them + // apart: a list this node is not in, an empty one (which reads as "wait for + // peers to broadcast"), a deferral to a peer that is serving, or a win that + // never confirms because connectivity keeps resetting the streak. On a cold + // start where every copy loses for one of these reasons the app never starts + // at all, and that is the case that most needs the reason recorded. + // + // Written on the EDGE - when the reason changes, and never again while it + // holds. That an app is still unpromoted is already in the cycle's own sync + // status line; what is missing is which gate it lost at, and that is a fact + // about a transition. Repeating it on a timer would be twice a minute per + // app for as long as the state lasts, which is unbounded in exactly the + // standoff this exists to explain, and every one of those lines would carry + // the same six values as the one before it. + const selfInElection = electionList.some((peer) => socketAddressesMatch(peer.ip, localSocketAddr)); + const reason = `candidates=${electionList.length} self=${selfInElection} ` + + `elected=${electedLeader} connected=${connected} ` + + `streak=${cache.leaderStreak}/${LEADER_CONFIRM_COUNT} ` + + `peerHasData=${aPeerHasData} folderEmpty=${folderIsEmpty}`; + if (reason !== cache.lastNotPromotedReason) { + log.info(`handleReceiveOnlyTransition - ${appId} not promoted: ${reason}`); + cache.lastNotPromotedReason = reason; + } + // Not the leader - syncStatus already read above syncthingFolder.type = 'receiveonly'; cache.numberOfExecutions = (cache.numberOfExecutions || 0) + 1; @@ -677,11 +1006,20 @@ async function handleReceiveOnlyTransition(params) { if (syncStatus) { cache.statusUnreadableSince = null; // status readable again - reset the unreadable timer - log.info( - `handleReceiveOnlyTransition - ${appId} sync status: ${syncStatus.syncPercentage.toFixed(2)}% ` - + `(${syncStatus.inSyncBytes}/${syncStatus.globalBytes} bytes), ` - + `state: ${syncStatus.state}, executions: ${cache.numberOfExecutions}`, - ); + // Edge, not cycle. A folder that is actually moving changes these numbers + // every pass and prints every pass; one parked at a value prints once and + // then stays quiet, however long it stays parked. The execution count is + // carried but not part of the comparison - it increments unconditionally, + // so keying on it would make every line unique and print forever. + const progress = `${syncStatus.syncPercentage.toFixed(2)}% ` + + `(${syncStatus.inSyncBytes}/${syncStatus.globalBytes} bytes), state: ${syncStatus.state}`; + if (progress !== cache.lastProgressLogged) { + log.info( + `handleReceiveOnlyTransition - ${appId} sync status: ${progress}, ` + + `executions: ${cache.numberOfExecutions}`, + ); + cache.lastProgressLogged = progress; + } // Synced -> candidate for sendreceive. But completion metrics only count CLUSTER // data: local additions in a receiveonly folder leave needBytes 0 / completion 100, @@ -692,7 +1030,9 @@ async function handleReceiveOnlyTransition(params) { if (syncStatus.isSynced && syncStatus.receiveOnlyChangedFiles > 0) { log.warn(`handleReceiveOnlyTransition - ${appId} is synced but the receive-only folder has ${syncStatus.receiveOnlyChangedFiles} locally changed item(s); reverting local changes instead of promoting (promotion would propagate them to the cluster)`); try { - await syncthingService.dbRevert(appId); + // dataOrThrow: dbRevert answers in-band; without it this catch is + // dead code and a failed revert reads as reverted + messageHelper.dataOrThrow(await syncthingService.dbRevert(appId)); } catch (error) { log.error(`handleReceiveOnlyTransition - revert of local changes for ${appId} failed: ${error.message}`); } @@ -738,6 +1078,10 @@ async function handleReceiveOnlyTransition(params) { cache.nudgeCount = 0; cache.evidenceSince = null; cache.lastNudgeAt = null; + // Cleared with the rest of the stall state, so a folder that stalls again + // after a source came and went announces the second stall as well as the + // first. A latch that is only ever set reports one stall per app lifetime. + cache.waitingForSourceLogged = false; return { syncthingFolder, cache }; } @@ -750,7 +1094,10 @@ async function handleReceiveOnlyTransition(params) { } if (!aPeerHasData) { - log.warn(`handleReceiveOnlyTransition - ${appId} idle with no sync progress and no CONNECTED synced peer; waiting (syncthing auto-resumes when a source returns)`); + if (!cache.waitingForSourceLogged) { + log.warn(`handleReceiveOnlyTransition - ${appId} idle with no sync progress and no CONNECTED synced peer; waiting (syncthing auto-resumes when a source returns)`); + cache.waitingForSourceLogged = true; + } return { syncthingFolder, cache }; } @@ -859,7 +1206,7 @@ async function manageFolderSyncState(params) { localSocketAddr, syncthingFolder, installedAppName, - mountVerifyNeeded = true, + liveness, } = params; // Check if folder already exists and is in sendreceive mode @@ -867,57 +1214,15 @@ async function manageFolderSyncState(params) { // If already syncing in sendreceive mode, ensure container is running if (folderAlreadySyncing) { - // Mount safety of a live sendreceive folder is verified at decision points - // (startup, FolderErrors from syncthing) - not per pass: the .stfolder - // marker inside the volume turns storage loss into FolderErrors, and the - // caller flags exactly those folders here - if (mountVerifyNeeded) { - const folderPath = syncFolder.path || `${appsFolder}${appId}/appdata`; - let mountSafety = await verifySendReceiveFolderSafety(appId, folderPath); - - if (!mountSafety.isSafe && !mountSafety.isMounted) { - // The detection is actionable: the backing image normally still exists, - // and FluxOS owns the mount - repair instead of just blocking. The - // re-verify still holds the folder back (receiveonly) if the freshly - // mounted volume disagrees with the index (phantom-index case). - const mountAttempt = await volumeService.ensureAppVolumeMounted(appId); - if (mountAttempt.mounted) { - log.info(`manageFolderSyncState - ${appId} volume was not mounted; mounted it, re-verifying folder safety`); - mountSafety = await verifySendReceiveFolderSafety(appId, folderPath); - } - } - - if (!mountSafety.isSafe) { - // DANGER: Mount not ready! Switch to receiveonly to prevent data propagation - log.error(`manageFolderSyncState - SAFETY BLOCK: ${appId} mount not safe (${mountSafety.reason}). Switching to receiveonly mode to prevent data loss.`); - log.error(`manageFolderSyncState - Mount status: mounted=${mountSafety.isMounted}, hasContent=${mountSafety.hasContent}, files=${mountSafety.fileCount}`); - - // Update folder to receiveonly mode to prevent this node from sending "empty" state to peers - syncthingFolder.type = 'receiveonly'; - const cache = { - numberOfExecutions: 0, - mountSafetyBlocked: true, - blockedReason: mountSafety.reason, - blockedAt: Date.now(), - }; - receiveOnlySyncthingAppsCache.set(appId, cache); - - // Hold the container too: its binds point at the same unsafe dir. The - // reconciler is the actuator; the receiveonly machinery flips the - // verdict back to running once the folder is verifiably synced. - appReconciler.setControllerDesired(appId, 'stopped', `mount safety block: ${mountSafety.reason}`); - - // Return with skipUpdate=false so the folder config gets updated to receiveonly - return { syncthingFolder, cache, skipUpdate: false }; - } - } - - // Mount is safe (verified) or not in question (steady state) + // The mount is sound by the time this runs: the pass verifies every folder + // it is going to act on before it acts, and holds out the ones that fail. + // Re-deriving that verdict here would cost a syncthing round trip and a + // directory walk per folder to answer a question already answered. await ensureContainerRunning(appId, containerDataFlags); // Ensure cache entry exists so health monitor can track this folder const existingCache = receiveOnlySyncthingAppsCache.get(appId); const cache = existingCache || { restarted: true }; - return { syncthingFolder, cache, skipUpdate: true }; + return { syncthingFolder, cache }; } // First run scenario @@ -953,6 +1258,7 @@ async function manageFolderSyncState(params) { localSocketAddr, containerDataFlags, syncthingFolder, + liveness, }); return result; } @@ -996,9 +1302,11 @@ async function manageFolderSyncState(params) { module.exports = { manageFolderSyncState, getFolderSyncCompletion, + probeFolderSyncCompletion, isDesignatedLeader, verifyFolderMountSafety, verifySendReceiveFolderSafety, + findSyncedPeer, isPathMounted, checkDirectoryHasContent, checkDirectoryHasSyncScopedContent, diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 5bd78e5075..43e8901704 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -6,7 +6,10 @@ const dbHelper = require('../dbHelper'); const serviceHelper = require('../serviceHelper'); const dockerService = require('../dockerService'); const fluxNetworkHelper = require('../fluxNetworkHelper'); +const messageHelper = require('../messageHelper'); const syncthingService = require('../syncthingService'); +const globalState = require('../utils/globalState'); +const fluxEventBus = require('../utils/fluxEventBus'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const log = require('../../lib/log'); const { @@ -19,11 +22,14 @@ const { EARLY_EVAL_MIN_GAP_MS, } = require('./syncthingMonitorConstants'); const { createMonitorAccelerator } = require('./syncthingMonitorAccelerator'); +const { createPeerFolderLiveness } = require('./peerFolderLiveness'); +const { socketAddressesMatch } = require('../utils/socketAddressUtils'); const { sortAndFilterLocations, buildDeviceConfiguration, createSyncthingFolderConfig, ensureStfolderExists, + ensureStignoreCovers, getContainerDataFlags, requiresSyncing, folderNeedsUpdate, @@ -52,60 +58,122 @@ const appsFolder = `${appsFolderPath}/`; * Verify one app folder's mount safety, repairing an unmounted volume on the * spot (FluxOS owns the mount - the backing image normally still exists, so * the actionable response is to mount it, not just to report it). + * + * A folder that is currently sendreceive is verified at the deeper level, which + * also rejects a stale index over an empty volume: sendreceive is the only mode + * that can broadcast the resulting deletions, so the check belongs exactly where + * that is possible and nowhere else - it costs a syncthing round trip and a + * scoped directory walk per folder. The repair runs first either way, so the + * index is judged against a mounted volume rather than against the absence of + * one. + * * @param {string} appId - Docker app identifier * @param {string} appFolder - App folder path + * @param {boolean} sending - Whether syncthing currently holds this folder sendreceive * @returns {Promise<{isSafe: boolean, reason: string}>} Result after any repair */ -async function verifyAppFolderMountWithRepair(appId, appFolder) { - let mountSafety = await verifyFolderMountSafety(appId, appFolder); +async function verifyAppFolderMountWithRepair(appId, appFolder, sending) { + const verify = sending ? verifySendReceiveFolderSafety : verifyFolderMountSafety; + let mountSafety = await verify(appId, appFolder); if (!mountSafety.isSafe && !mountSafety.isMounted) { const mountAttempt = await volumeService.ensureAppVolumeMounted(appId); if (mountAttempt.mounted) { log.info(`checkAppFolderMounts - ${appId} volume was not mounted; mounted it`); - mountSafety = await verifyFolderMountSafety(appId, appFolder); + mountSafety = await verify(appId, appFolder); } } return mountSafety; } /** - * Check if app folders are properly mounted - * Returns list of apps whose folders are not mounted yet - * Uses verifyFolderMountSafety to detect folders that exist but aren't properly mounted + * The components of one installed app, each as its docker app identifier (which + * IS its syncthing folder id) paired with the containerData that decides + * whether it syncs. A version <= 3 app is a single component - itself. + * @param {object} installedApp - Installed app specification + * @returns {Array<{appId: string, containerData: string}>} The app's components + */ +function appComponents(installedApp) { + if (installedApp.version <= 3) { + return [{ + appId: dockerService.getAppIdentifier(installedApp.name), + containerData: installedApp.containerData, + }]; + } + return (installedApp.compose || []).map((component) => ({ + appId: dockerService.getAppIdentifier(`${component.name}_${installedApp.name}`), + containerData: component.containerData, + })); +} + +/** + * The pass's single mount-safety authority: every component of every app it is + * given gets exactly one verdict, and every consumer of that verdict reads it + * from here. Nothing downstream re-derives it. * @param {Array} appsInstalled - List of installed apps - * @returns {Promise} List of apps with unmounted folders + * @param {Set} sendingFolderIds - Folder ids syncthing currently holds sendreceive + * @param {Array<{appId: string, appName: string}>} extraFolders - Folder entries + * verified by id alone, for folders whose owning app's spec cannot be read + * @returns {Promise<{unmountedApps: Array, verifiedSafeIds: string[]}>} Apps with + * unmounted folders, and the folder ids that verified safe (so a pending + * mount-verify flag on them can be resolved) */ -async function checkAppFolderMounts(appsInstalled) { +async function checkAppFolderMounts(appsInstalled, sendingFolderIds, extraFolders = []) { const unmountedApps = []; + const verifiedSafeIds = []; + + const verifyOne = async (appId, appName) => { + const appFolder = `${appsFolder}${appId}`; + const mountSafety = await verifyAppFolderMountWithRepair(appId, appFolder, sendingFolderIds.has(appId)); + if (mountSafety.isSafe) { + verifiedSafeIds.push(appId); + } else { + // Folder exists but mount is not safe (empty and not mounted - likely unmounted loop device) + unmountedApps.push({ appId, appName, reason: mountSafety.reason }); + } + }; // eslint-disable-next-line no-restricted-syntax for (const installedApp of appsInstalled) { - if (installedApp.version <= 3) { - // Legacy app - single folder - const appId = dockerService.getAppIdentifier(installedApp.name); - const appFolder = `${appsFolder}${appId}`; + // eslint-disable-next-line no-restricted-syntax + for (const { appId } of appComponents(installedApp)) { // eslint-disable-next-line no-await-in-loop - const mountSafety = await verifyAppFolderMountWithRepair(appId, appFolder); - if (!mountSafety.isSafe) { - // Folder exists but mount is not safe (empty and not mounted - likely unmounted loop device) - unmountedApps.push({ appId, appName: installedApp.name, reason: mountSafety.reason }); - } - } else { - // Newer app - check each component - // eslint-disable-next-line no-restricted-syntax - for (const component of installedApp.compose || []) { - const appId = dockerService.getAppIdentifier(`${component.name}_${installedApp.name}`); - const appFolder = `${appsFolder}${appId}`; - // eslint-disable-next-line no-await-in-loop - const mountSafety = await verifyAppFolderMountWithRepair(appId, appFolder); - if (!mountSafety.isSafe) { - unmountedApps.push({ appId, appName: installedApp.name, reason: mountSafety.reason }); - } - } + await verifyOne(appId, installedApp.name); } } - return unmountedApps; + // The verdict derives entirely from the folder id, so a folder whose owning + // app cannot be read this pass is verified all the same. + // eslint-disable-next-line no-restricted-syntax + for (const { appId, appName } of extraFolders) { + // eslint-disable-next-line no-await-in-loop + await verifyOne(appId, appName); + } + + return { unmountedApps, verifiedSafeIds }; +} + +/** + * The apps holding at least one syncing folder still awaiting a promotion + * decision. Only those folders ask a peer anything, so only their holders are + * worth asking about: a node whose synced apps are all running probes nothing, + * and must keep probing nothing. + * @param {Array} appsInstalled - List of installed apps (decrypted) + * @param {Set} suspendedAppNames - Apps under backup or restore + * @param {Map} receiveOnlySyncthingAppsCache - Per-folder transition state + * @returns {Set} App names + */ +function appsAwaitingPromotion(appsInstalled, suspendedAppNames, receiveOnlySyncthingAppsCache) { + const names = new Set(); + appsInstalled.forEach((installedApp) => { + if (suspendedAppNames.has(installedApp.name)) return; + appComponents(installedApp).forEach(({ appId, containerData }) => { + const primaryContainer = (containerData ?? '').split('|')[0]; + if (!requiresSyncing(getContainerDataFlags(primaryContainer))) return; + const cache = receiveOnlySyncthingAppsCache.get(appId); + if (cache && !cache.restarted) names.add(installedApp.name); + }); + }); + return names; } /** @@ -118,14 +186,36 @@ async function checkAppFolderMounts(appsInstalled) { function appsMatchingFolderIds(appsInstalled, folderIds) { if (folderIds.length === 0) return []; const wanted = new Set(folderIds); - return appsInstalled.filter((installedApp) => { - if (installedApp.version <= 3) { - return wanted.has(dockerService.getAppIdentifier(installedApp.name)); - } - return (installedApp.compose || []).some( - (component) => wanted.has(dockerService.getAppIdentifier(`${component.name}_${installedApp.name}`)), - ); + return appsInstalled.filter( + (installedApp) => appComponents(installedApp).some(({ appId }) => wanted.has(appId)), + ); +} + +/** + * The syncthing folder ids this node's installed apps own. A folder is owned + * when an installed component whose primary mount carries a sync flag (g:/r:/s:) + * maps to it - ownership is a property of the installed specification, not of + * what any one pass managed to process. + * + * An app under backup or restore owns its folders like any other. It used to be + * exempt, on the grounds that those flows deleted and rebuilt their own folder + * configs, so a folder had to be neither kept nor re-added underneath them. + * Neither flow deletes a folder any more - both pause it and resume it, and for + * a restore the resume IS the propagation. Sweeping it mid-operation takes the + * index, the peer devices and any standing safety demotion with it, and leaves + * the resume addressing a folder that no longer exists. + * @param {Array} appsInstalled - List of installed apps (decrypted) + * @returns {Set} Owned folder ids + */ +function syncingFolderOwnerIds(appsInstalled) { + const ownerIds = new Set(); + appsInstalled.forEach((installedApp) => { + appComponents(installedApp).forEach(({ appId, containerData }) => { + const primaryContainer = (containerData ?? '').split('|')[0]; + if (requiresSyncing(getContainerDataFlags(primaryContainer))) ownerIds.add(appId); + }); }); + return ownerIds; } // Helper function to get app locations @@ -158,14 +248,14 @@ async function processContainerData(params) { localSocketAddr, localDeviceId, state, - erroredFolderIds, - allFoldersResp, - allDevicesResp, + allFolders, + allDevices, devicesConfiguration, devicesIds, folderIds, foldersConfiguration, newFoldersConfiguration, + liveness, } = params; const containersData = containerData.split('|'); @@ -194,6 +284,19 @@ async function processContainerData(params) { return; } + const syncFolder = allFolders.find((x) => x.id === id); + + // Converge the FluxOS ignore policy through syncthing's own API, which owns + // and atomically writes .stignore. Only once syncthing knows the folder: a + // brand-new folder had its .stignore seeded at volume creation, and an + // existing one was configured in a prior pass and persists across restarts - + // so this reaches every folder whose ignores predate a policy line, and skips + // the one pass where a fresh install is not yet configured. A converged + // folder posts nothing and triggers no rescan. + if (syncFolder) { + await ensureStignoreCovers(id); + } + // Get and process app locations let locations = await appLocation(installedAppName); locations = sortAndFilterLocations(locations, localSocketAddr); @@ -206,12 +309,11 @@ async function processContainerData(params) { state.syncthingDevicesIDCache, devicesConfiguration, devicesIds, - allDevicesResp, + allDevices, ); // Create base folder configuration const syncthingFolder = createSyncthingFolderConfig(id, label, folder, devices); - const syncFolder = allFoldersResp.data.find((x) => x.id === id); // Handle receive-only or global sync flags if (primaryContainerDataFlags.includes('r') || primaryContainerDataFlags.includes('g')) { @@ -226,7 +328,7 @@ async function processContainerData(params) { localSocketAddr, syncthingFolder, installedAppName, - mountVerifyNeeded: state.syncthingAppsFirstRun || erroredFolderIds.has(appId), + liveness, }); // Update cache if provided @@ -269,28 +371,16 @@ async function logSyncState(foldersConfiguration) { // Get sync status for all folders in parallel const syncStatusPromises = foldersConfiguration.map(async (folder) => { try { - const statusResponse = await syncthingService.getDbStatus({ - query: { folder: folder.id }, - }, null); - - if (statusResponse && statusResponse.status === 'success') { - const { globalBytes = 0, inSyncBytes = 0, state: syncState } = statusResponse.data; - const syncPercentage = globalBytes > 0 ? (inSyncBytes / globalBytes) * 100 : 100; - - return { - id: folder.id, - type: folder.type, - syncPercentage, - globalBytes, - inSyncBytes, - state: syncState, - }; - } + const { globalBytes = 0, inSyncBytes = 0, state: syncState } = await syncthingService.getDbStatus(folder.id); + const syncPercentage = globalBytes > 0 ? (inSyncBytes / globalBytes) * 100 : 100; return { id: folder.id, type: folder.type, - error: 'Failed to get status', + syncPercentage, + globalBytes, + inSyncBytes, + state: syncState, }; } catch (error) { return { @@ -346,50 +436,33 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { return; } - // Decrypt enterprise apps (version 8 with encrypted content) - appsInstalled.data = await decryptEnterpriseApps(appsInstalled.data); - - // Mount safety is verified at decision points and in reaction to - // syncthing's own storage signal - never as a steady-state sweep. The full - // sweep runs on the first pass after process start (the reboot case: loop - // mounts may not be up yet, and the sweep repairs them). After that, a - // vanished mount takes the folder's .stfolder marker with it, syncthing - // halts the folder and raises FolderErrors, and only the flagged folders - // are verified here (checkAppFolderMounts repairs an unmounted volume - // itself when the backing image still exists, so a non-empty unmountedApps - // means repair failed too). - const erroredFolderIds = new Set(syncthingEventsConsumer.drainErroredFolderIds()); - const appsToVerify = state.syncthingAppsFirstRun - ? appsInstalled.data - : appsMatchingFolderIds(appsInstalled.data, [...erroredFolderIds]); - const unmountedApps = appsToVerify.length > 0 ? await checkAppFolderMounts(appsToVerify) : []; - if (unmountedApps.length > 0) { - const unmountedList = unmountedApps.map((app) => app.appId).join(', '); - log.warn(`syncthingAppsCore - Skipping processing: ${unmountedApps.length} app folders not mounted yet: ${unmountedList}`); - log.warn('syncthingAppsCore - Waiting for app folders to be mounted before syncthing processing'); - - // Never leave an unsafe-mount folder sendreceive while processing is - // skipped: the syncthing daemon keeps running as configured, so an - // un-demoted sendreceive folder over a bad mount can still broadcast its - // (leaked or missing) disk state to the healthy peers. Demote those - // folders and hold their containers before bailing - idempotent, and the - // normal receiveonly machinery re-promotes once the mount is healthy. - const foldersResp = await syncthingService.getConfigFolders(); - const folders = Array.isArray(foldersResp?.data) ? foldersResp.data : []; - // eslint-disable-next-line no-restricted-syntax - for (const { appId, reason } of unmountedApps) { - const folder = folders.find((f) => f.id === appId); - if (folder && folder.type === 'sendreceive') { - log.error(`syncthingAppsCore - SAFETY BLOCK: ${appId} folder is sendreceive over an unsafe mount (${reason}); switching to receiveonly and holding the container`); - // eslint-disable-next-line no-await-in-loop - await syncthingService.adjustConfigFolders('patch', { type: 'receiveonly' }, appId).catch((err) => { - log.error(`syncthingAppsCore - Failed to switch ${appId} to receiveonly: ${err.message}`); - }); - appReconciler.setControllerDesired(appId, 'stopped', `mount safety block: ${reason}`); - } - } - return; + // Decrypt enterprise apps (version 8 with encrypted content). This pass acts + // on the specification, and an app whose spec cannot be read tells us + // nothing about which folders it owns - so its folders are protected from + // the sweep and its safety flags are left standing, while every app that + // DID decrypt is managed normally. Aborting the whole pass instead would + // stop folder registration, mount safety, promotion and error draining for + // every app on the node, and stop publishing the writable-folder answer its + // peers block on - for as long as one app stays unreadable. + const decrypted = await decryptEnterpriseApps(appsInstalled.data); + appsInstalled.data = decrypted.readable; + const unreadableAppNames = new Set(decrypted.unreadable.map((app) => app.name)); + if (unreadableAppNames.size) { + log.warn(`syncthingAppsCore - folders of undecryptable apps are protected this pass: ${[...unreadableAppNames].join(', ')}`); } + // A folder id is the component identifier, which ends in _ - and an + // app name cannot contain an underscore - so a folder always names the app + // that owns it, even when that app's components are unreadable. + const ownedByUnreadableApp = (folderId) => { + const appName = folderId.slice(folderId.lastIndexOf('_') + 1); + return unreadableAppNames.has(appName); + }; + + // The folders installed apps own. Computed here, before any decision the + // pass makes: the skip-gate below tells "syncthing has no such folder + // because this component does not sync" from "an owned folder has gone + // missing" by it, and the sweep at the end deletes by it. + const ownerIds = syncingFolderOwnerIds(appsInstalled.data); // Get required IDs and configurations const localDeviceId = await syncthingService.getDeviceId(); @@ -404,72 +477,195 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { return; } - // Get current Syncthing configuration - const allFoldersResp = await syncthingService.getConfigFolders(); - const allDevicesResp = await syncthingService.getConfigDevices(); + // Get current Syncthing configuration. + // + // CRITICAL: validate it is loaded before proceeding. On system restart the + // Syncthing API can be up while the config is not fully loaded, and this is + // what stops data deletion in that window. An unreadable configuration now + // THROWS, so it can no longer be mistaken for an empty one - an EMPTY + // folders array is legal data, and a failed read is not an array at all. + // Read one at a time, and in this order. What this node holds writable is + // answered by the FOLDER configuration alone, and the peers that ask before + // promoting a folder of their own read that answer - so a device read this + // pass could not complete must not withhold a folder list it already has. + // Sharing one try did exactly that: the device read throws, the pass returns, + // and the folders published below never happen, so every peer asking is told + // to wait for as long as the device read keeps failing. + let allFolders; + try { + allFolders = await syncthingService.getConfigFolders(); + } catch (error) { + if (state.syncthingAppsFirstRun) { + log.warn('syncthingAppsCore - Syncthing configuration not ready yet on first run. Waiting for next cycle to avoid data loss.'); + } else { + log.error(`syncthingAppsCore - Failed to get Syncthing folder configuration: ${error.message}`); + } + return; + } - // CRITICAL: Validate Syncthing configuration is loaded before proceeding - // On system restart, Syncthing API might be available but config not fully loaded - // This prevents data deletion during the race condition window - if (!allFoldersResp || !allFoldersResp.data || !Array.isArray(allFoldersResp.data)) { + if (!Array.isArray(allFolders)) { if (state.syncthingAppsFirstRun) { log.warn('syncthingAppsCore - Syncthing folder configuration not ready yet on first run. Waiting for next cycle to avoid data loss.'); } else { - log.error('syncthingAppsCore - Failed to get Syncthing folders configuration'); + log.error('syncthingAppsCore - Failed to get Syncthing folders configuration: malformed response'); } return; } - if (!allDevicesResp || !allDevicesResp.data || !Array.isArray(allDevicesResp.data)) { + // Publish which folders this node holds writable, for the peers that ask before + // promoting one of their own. Recorded here rather than read on demand: the + // answer is a byproduct of a pass the monitor already makes, so serving it costs + // nothing, where an endpoint calling syncthing per request would be an + // unauthenticated amplifier into it. Replaced only by a validated response, so a + // failed read leaves the last good answer standing rather than momentarily + // claiming this node holds nothing writable. + const sendingFolderIds = new Set( + allFolders.filter((folder) => folder.type === 'sendreceive').map((folder) => folder.id), + ); + // Published as a COPY: the end-of-pass reconciliation mutates the published + // set as writes land (its job), while sendingFolderIds stays what this pass + // observed at scan time. Aliased, the local name silently changes meaning + // mid-pass and external readers (appQueryService) see a half-updated scan. + globalState.promotedFolderIds = new Set(sendingFolderIds); + + let allDevices; + try { + allDevices = await syncthingService.getConfigDevices(); + } catch (error) { if (state.syncthingAppsFirstRun) { log.warn('syncthingAppsCore - Syncthing device configuration not ready yet on first run. Waiting for next cycle to avoid data loss.'); } else { - log.error('syncthingAppsCore - Failed to get Syncthing devices configuration'); + log.error(`syncthingAppsCore - Failed to get Syncthing devices configuration: ${error.message}`); } return; } - // Mark that Syncthing is properly initialized - safe to clear first run flag - syncthingInitializedSuccessfully = true; + if (!Array.isArray(allDevices)) { + if (state.syncthingAppsFirstRun) { + log.warn('syncthingAppsCore - Syncthing device configuration not ready yet on first run. Waiting for next cycle to avoid data loss.'); + } else { + log.error('syncthingAppsCore - Failed to get Syncthing devices configuration: malformed response'); + } + return; + } - // CRITICAL STARTUP SAFETY CHECK: Verify all sendreceive folders have safe mounts - // This prevents data loss when loop mounts aren't ready after reboot - if (state.syncthingAppsFirstRun && allFoldersResp.data.length > 0) { - log.info('syncthingAppsCore - First run detected, performing mount safety verification on existing folders'); - let unsafeFoldersCount = 0; + // Syncthing itself is up and its configuration readable - that, and only + // that, is what the first-run flag gates. It must be set before any + // per-app work: a single app whose volume never mounts would otherwise + // hold the flag set forever, and the flag also gates the g: primary + // election node-wide (`masterSlaveApps`), so one broken app would stop + // every masterSlave app on the node from ever electing. + syncthingInitializedSuccessfully = true; + // Mount safety is verified at decision points and in reaction to + // syncthing's own storage signal - never as a steady-state sweep. The full + // sweep runs on the first pass after process start (the reboot case: loop + // mounts may not be up yet, and the sweep repairs them). After that, a + // vanished mount takes the folder's .stfolder marker with it, syncthing + // halts the folder and raises FolderErrors, and only the flagged folders + // are verified here (checkAppFolderMounts repairs an unmounted volume + // itself when the backing image still exists, so a non-empty unmountedApps + // means repair failed too). The flags are a durable level, resolved only + // by a completed outcome below - never consumed by being read - so a pass + // that fails mid-action leaves the flag standing and the next pass + // retries; the guard does not depend on FolderErrors ever re-firing. + const pendingFolderIds = syncthingEventsConsumer.mountVerifyPendingIds(); + const appsToVerify = state.syncthingAppsFirstRun + ? appsInstalled.data + : appsMatchingFolderIds(appsInstalled.data, pendingFolderIds); + // A flagged folder no installed app carries can never be acted on - + // resolve it rather than re-match it forever (the uninstall already + // removed whatever the flag was protecting). An app whose spec could not + // be read is not resolved here either: its folders still get their mount + // verdicts below, id-derived, and the flag resolves through a completed + // outcome like any other. + if (!state.syncthingAppsFirstRun && pendingFolderIds.length > 0) { + const matchable = new Set(); + appsToVerify.forEach((installedApp) => { + appComponents(installedApp).forEach(({ appId }) => matchable.add(appId)); + }); + pendingFolderIds.filter((id) => !matchable.has(id) && !ownedByUnreadableApp(id)) + .forEach((id) => syncthingEventsConsumer.resolveMountVerify(id)); + } + // An unreadable app's folders are protected from the sweep, not from the + // mount check: that verdict derives entirely from the folder id, and a + // sendreceive folder over a bad mount broadcasts its disk state whatever + // the spec says. The folders come from syncthing's own list - the spec is + // exactly what cannot be enumerated. The first pass verifies them all; + // after that, the flagged ones, the same trigger the readable apps get. + const unreadableFolderEntries = unreadableAppNames.size === 0 ? [] : allFolders + .filter((folder) => folder.type === 'sendreceive' && ownedByUnreadableApp(folder.id)) + .filter((folder) => state.syncthingAppsFirstRun || pendingFolderIds.includes(folder.id)) + .map((folder) => ({ appId: folder.id, appName: folder.id.slice(folder.id.lastIndexOf('_') + 1) })); + const { unmountedApps, verifiedSafeIds } = (appsToVerify.length > 0 || unreadableFolderEntries.length > 0) + ? await checkAppFolderMounts(appsToVerify, sendingFolderIds, unreadableFolderEntries) + : { unmountedApps: [], verifiedSafeIds: [] }; + // safe mount = the condition the flag exists for is gone + verifiedSafeIds.forEach((id) => syncthingEventsConsumer.resolveMountVerify(id)); + + // Folder ids held out of this pass. An unsafe mount is an app-level fault: + // that app is demoted and left alone while every other app is processed + // normally. Abandoning the whole pass instead would strand the node before + // it ever finished initialising - the flag set above gates the g: primary + // election node-wide, so one app whose volume can never mount would stop + // every masterSlave app on the node from electing. + const unsafeFolderIds = new Set(); + if (unmountedApps.length > 0) { + const unmountedList = unmountedApps.map((app) => app.appId).join(', '); + log.warn(`syncthingAppsCore - Holding ${unmountedApps.length} app folders out of this pass, not mounted: ${unmountedList}`); + + // Never leave an unsafe-mount folder sendreceive: the syncthing daemon + // keeps running as configured, so an un-demoted sendreceive folder over + // a bad mount can still broadcast its (leaked or missing) disk state to + // the healthy peers. The demotion is patched directly, with no config + // pre-read: a safety action must not be conditioned on a fallible read + // whose failure silently reads as "nothing to protect" (that exact + // silent no-op once cost a gate run). The patch is safe to repeat - + // syncthing restarts a folder only when its config actually changed + // (model.go CommitConfiguration diffs RequiresRestartOnly) - and a + // folder syncthing does not know answers 404, which means "not a + // syncthing app", not a failure. The normal receiveonly machinery + // re-promotes once the mount is healthy. // eslint-disable-next-line no-restricted-syntax - for (const folder of allFoldersResp.data) { - if (folder.type === 'sendreceive') { - // Extract appId from folder.id (e.g., fluxwp_myapp -> fluxwp_myapp) - const appId = folder.id; - const folderPath = folder.path; - - // eslint-disable-next-line no-await-in-loop - const mountSafety = await verifySendReceiveFolderSafety(appId, folderPath); - - if (!mountSafety.isSafe) { - unsafeFoldersCount += 1; - log.error(`syncthingAppsCore - STARTUP SAFETY: Folder ${appId} has unsafe mount (${mountSafety.reason}). Switching to receiveonly to prevent data loss.`); - - // Immediately switch to receiveonly mode - // eslint-disable-next-line no-await-in-loop - await syncthingService.adjustConfigFolders('patch', { type: 'receiveonly' }, folder.id).catch((err) => { - log.error(`syncthingAppsCore - Failed to switch ${folder.id} to receiveonly: ${err.message}`); - }); + for (const { appId, reason } of unmountedApps) { + unsafeFolderIds.add(appId); + // eslint-disable-next-line no-await-in-loop + const patchResponse = await syncthingService.adjustConfigFolders('patch', { type: 'receiveonly' }, appId); + if (patchResponse.status === 'success') { + log.error(`syncthingAppsCore - SAFETY BLOCK: ${appId} folder over an unsafe mount (${reason}); switched to receiveonly and holding the container`); + appReconciler.setControllerDesired(appId, 'stopped', `mount safety block: ${reason}`); + // A demoted folder re-enters the promotion machinery from the start. + // Leaving the count where it stood would let a folder that was + // moments from promotion resume there once the mount returns, on a + // sync state established before the volume went away. + state.receiveOnlySyncthingAppsCache.set(appId, { numberOfExecutions: 0 }); + syncthingEventsConsumer.resolveMountVerify(appId); + } else if (patchResponse.data?.code === 'ERR_BAD_REQUEST') { + // 4xx: syncthing has no such folder. What that means turns entirely on + // ownership. + if (ownerIds.has(appId)) { + // An installed syncing component owns this id, so "no such folder" + // is a contradiction, not an answer: the demotion could not be + // applied, so the flag stays standing for the next pass. The mount + // is unsafe either way, so the container is held now. Nothing is + // recreated from here - the level loop rebuilds the folder once the + // mount is healthy, under the normal receiveonly machinery. + log.error(`syncthingAppsCore - SAFETY BLOCK: ${appId} folder over an unsafe mount (${reason}) is unknown to syncthing though an installed component syncs it; holding the container, flag stands`); + appReconciler.setControllerDesired(appId, 'stopped', `mount safety block: ${reason}`); } else { - log.info(`syncthingAppsCore - Folder ${appId} mount is safe (mounted=${mountSafety.isMounted}, files=${mountSafety.fileCount})`); + // no installed component syncs this id - there is nothing to demote + // and nothing left to act on + syncthingEventsConsumer.resolveMountVerify(appId); } + } else { + // transient failure: the flag stays standing and the next pass + // retries the demotion - loudly, never silently + log.error(`syncthingAppsCore - SAFETY BLOCK FAILED for ${appId} (${reason}): ${patchResponse.data?.message || 'unknown error'}; retrying next pass`); } } - - if (unsafeFoldersCount > 0) { - // The receiveonly PATCH applies live (no restart needed on syncthing v2) - - // a process restart here would drop every folder's transfers node-wide. - log.error(`syncthingAppsCore - STARTUP WARNING: ${unsafeFoldersCount} folders had unsafe mounts and were switched to receiveonly mode. Check loop mounts!`); - } } + // Initialize tracking arrays const devicesIds = []; const devicesConfiguration = []; @@ -477,14 +673,39 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { const foldersConfiguration = []; const newFoldersConfiguration = []; + // Peer liveness, answered once for the whole pass. Both promotion decisions + // ask the same peers the same question, and the folder loop is sequential - + // asked inside it, one unreachable holder costs its full timeout again for + // every folder that elects it, and past three the pass outruns the interval + // and the next cycle is dropped for every folder on the node. Asking the + // whole set at once costs one timeout no matter how many folders wait on it. + // Nothing is probed unless a folder is actually awaiting promotion. + const liveness = createPeerFolderLiveness(); + const awaitingPromotion = appsAwaitingPromotion( + appsInstalled.data, + new Set([...state.backupInProgress, ...state.restoreInProgress]), + state.receiveOnlySyncthingAppsCache, + ); + if (awaitingPromotion.size) { + const peerLists = await Promise.all([...awaitingPromotion].map((name) => appLocation(name))); + await liveness.prewarm( + peerLists.flat() + .map((entry) => entry?.ip) + .filter((ip) => ip && !socketAddressesMatch(ip, localSocketAddr)), + ); + } + // Shared parameters for processing const sharedParams = { localSocketAddr, + liveness, localDeviceId, state, - erroredFolderIds, - allFoldersResp, - allDevicesResp, + // the folders flagged when the pass began: the state machine re-verifies + // exactly these on its own decision points (resolution of the flag by + // this pass does not retract the request to look) + allFolders, + allDevices, devicesConfiguration, devicesIds, folderIds, @@ -508,11 +729,16 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // Process based on app version if (installedApp.version <= 3) { // Legacy app (version <= 3) - single containerData + const identifier = installedApp.name; + if (unsafeFolderIds.has(dockerService.getAppIdentifier(identifier))) { + // eslint-disable-next-line no-continue + continue; + } // eslint-disable-next-line no-await-in-loop await processContainerData({ ...sharedParams, containerData: installedApp.containerData, - identifier: installedApp.name, + identifier, installedAppName: installedApp.name, }); } else { @@ -520,61 +746,150 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // eslint-disable-next-line no-restricted-syntax for (const installedComponent of installedApp.compose) { const identifier = `${installedComponent.name}_${installedApp.name}`; - // eslint-disable-next-line no-await-in-loop - await processContainerData({ - ...sharedParams, - containerData: installedComponent.containerData, - identifier, - installedAppName: installedApp.name, - }); + if (!unsafeFolderIds.has(dockerService.getAppIdentifier(identifier))) { + // eslint-disable-next-line no-await-in-loop + await processContainerData({ + ...sharedParams, + containerData: installedComponent.containerData, + identifier, + installedAppName: installedApp.name, + }); + } } } } - // Remove unused folders and devices (parallelized for better performance) - const nonUsedFolders = allFoldersResp.data.filter( - (syncthingFolder) => !folderIds.includes(syncthingFolder.id), + // Remove unused folders and devices (parallelized for better performance). + // A folder is unused when no installed syncing component owns it: the app + // was uninstalled, or dropped the g:/r:/s: flag from its primary mount. + // Whether this pass reached the component is a different question - a + // component skipped for an unmounted volume, deferred by the state machine, + // or held still for a backup or restore still owns its folder, and deleting + // it would take syncthing's index, peer devices and any standing safety + // demotion with it. + const nonUsedFolders = allFolders.filter( + (syncthingFolder) => !ownerIds.has(syncthingFolder.id) + && !ownedByUnreadableApp(syncthingFolder.id), ); - const nonUsedDevices = allDevicesResp.data.filter( + allFolders + .filter((syncthingFolder) => !ownerIds.has(syncthingFolder.id) + && ownedByUnreadableApp(syncthingFolder.id)) + .forEach((syncthingFolder) => log.warn(`syncthingAppsCore - keeping folder ${syncthingFolder.id}: its app could not be decrypted, so ownership is unknown`)); + // An owned folder the pass never registered means its component went + // unprocessed: the folder survives, but the skip is never silent - no + // configuration is being applied to it until the component is reached again. + const processedFolderIds = new Set(folderIds); + allFolders + .filter((syncthingFolder) => ownerIds.has(syncthingFolder.id) && !processedFolderIds.has(syncthingFolder.id)) + .forEach((syncthingFolder) => log.warn(`syncthingAppsCore - keeping folder ${syncthingFolder.id}: its component went unprocessed this pass`)); + // A peer device cannot be attributed to an app without doing that app's + // work, so while anything is held out the device sweep stands down entirely + // rather than delete a peer it simply never saw. + const nonUsedDevices = unsafeFolderIds.size > 0 ? [] : allDevices.filter( (syncthingDevice) => !devicesIds.includes(syncthingDevice.deviceID) && syncthingDevice.deviceID !== localDeviceId, ); // Parallelize cleanup operations const cleanupPromises = [ - ...nonUsedFolders.map((folder) => { + ...nonUsedFolders.map(async (folder) => { log.info(`syncthingAppsCore - Removing unused Syncthing folder ${folder.id}`); - return syncthingService.adjustConfigFolders('delete', undefined, folder.id).catch((err) => { - log.error(`Failed to remove folder ${folder.id}: ${err.message}`); - }); + const response = await syncthingService.adjustConfigFolders('delete', undefined, folder.id); + if (response?.status !== 'success') { + log.error(`Failed to remove folder ${folder.id}: ${response?.data?.message || 'unknown error'}`); + } }), - ...nonUsedDevices.map((device) => { + ...nonUsedDevices.map(async (device) => { log.info(`syncthingAppsCore - Removing unused Syncthing device ${device.deviceID}`); - return syncthingService.adjustConfigDevices('delete', undefined, device.deviceID).catch((err) => { - log.error(`Failed to remove device ${device.deviceID}: ${err.message}`); - }); + const response = await syncthingService.adjustConfigDevices('delete', undefined, device.deviceID); + if (response?.status !== 'success') { + log.error(`Failed to remove device ${device.deviceID}: ${response?.data?.message || 'unknown error'}`); + } }), ]; await Promise.all(cleanupPromises); - // Apply new configuration + // Apply new configuration. A failed apply aborts the pass loudly (outer + // catch): the steps below reason about the configuration this was meant + // to install, and the level loop reassembles everything next pass anyway. if (devicesConfiguration.length > 0) { - await syncthingService.adjustConfigDevices('put', devicesConfiguration); + messageHelper.dataOrThrow(await syncthingService.adjustConfigDevices('put', devicesConfiguration)); + } + // The config this pass computed sets paused:false on every folder it writes, + // and only a live backup or restore ever sets paused:true - so writing a + // busy app's folder here would un-pause the hold it took to protect its data + // mid-operation. The per-app skip above catches apps already busy when their + // turn came; an app that became busy DURING the pass was processed as free + // and its folder is in the batch. So the busy set is read again at the write + // - the guard belongs at the action it guards - and those folders are held + // back. The op holds its claim from a synchronous test-and-set before it + // pauses anything, so a folder read as free here is one whose pause has not + // happened yet; the op's own pause is the later write and wins. The monitor + // only ever DROPS work for a busy app, never waits on it, so it cannot block + // an operation. A folder a crashed op left paused holds no claim and is + // un-paused normally - the self-heal is untouched. + const busyAppNames = new Set([...state.backupInProgress, ...state.restoreInProgress]); + const busyFolderIds = new Set(); + if (busyAppNames.size) { + for (const installedApp of appsInstalled.data) { + if (busyAppNames.has(installedApp.name)) { + appComponents(installedApp).forEach(({ appId }) => busyFolderIds.add(appId)); + } + } + } + const foldersToWrite = busyFolderIds.size + ? newFoldersConfiguration.filter((folder) => !busyFolderIds.has(folder.id)) + : newFoldersConfiguration; + const heldForBusy = busyFolderIds.size + ? newFoldersConfiguration.filter((folder) => busyFolderIds.has(folder.id)).map((folder) => folder.id) + : []; + + // Inert in production; the harness waits on it to know a pass reached the + // folder write and to read what it wrote versus what it held for a live + // backup or restore. + fluxEventBus.publish('syncthing:passComplete', { + wrote: foldersToWrite.map((folder) => folder.id), + heldForBusy, + }); + + if (foldersToWrite.length > 0) { + messageHelper.dataOrThrow(await syncthingService.adjustConfigFolders('put', foldersToWrite)); + // The published set was built from the folder list this pass opened with, so + // a promotion applied on this line is absent from it until the next pass + // reads syncthing again - and findPeerBlockingPromotion asks a peer for + // exactly this set before promoting a folder of its own. Two nodes promoting + // in one cycle would each advertise nothing and neither would block, which is + // the collision that check exists to catch. Reconciled here instead, in both + // directions, so the answer is true from the moment it became true. + // eslint-disable-next-line no-restricted-syntax + for (const folder of foldersToWrite) { + if (folder.type === 'sendreceive') globalState.promotedFolderIds.add(folder.id); + else globalState.promotedFolderIds.delete(folder.id); + } } - if (newFoldersConfiguration.length > 0) { - await syncthingService.adjustConfigFolders('put', newFoldersConfiguration); + + // Promotions decided this pass are applied now, so the claims they made + // become true here and nowhere earlier: masterSlaveApps starts containers + // on designatedLeader, and a flag raised before the folder batch landed + // would start a primary against a folder still receiveonly. A failed + // apply threw above, so intent survives untouched for the retry pass. + // eslint-disable-next-line no-restricted-syntax + for (const [, folderCache] of state.receiveOnlySyncthingAppsCache) { + if (folderCache && folderCache.designationPending) { + folderCache.designationPending = false; + folderCache.designatedLeader = true; + } } // Check for folder errors in parallel const folderErrorChecks = await Promise.all( foldersConfiguration.map(async (folder) => { - try { - const folderError = await syncthingService.getFolderIdErrors(folder.id); - if (folderError?.status === 'success' && folderError.data.errors?.length > 0) { - return { folder, error: folderError }; - } - } catch (error) { - log.warn(`Failed to check errors for folder ${folder.id}: ${error.message}`); + const folderError = await syncthingService.getFolderIdErrors(folder.id); + if (folderError?.status === 'success' && folderError.data.errors?.length > 0) { + return { folder, error: folderError }; + } + if (folderError?.status !== 'success') { + log.warn(`Failed to check errors for folder ${folder.id}: ${folderError?.data?.message || 'malformed response'}`); } return null; }), @@ -625,12 +940,6 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { } } - // Check if Syncthing restart is needed - const restartRequired = await syncthingService.getConfigRestartRequired(); - if (restartRequired?.status === 'success' && restartRequired.data.requiresRestart === true) { - log.info('syncthingAppsCore - New configuration applied. Syncthing restart required, restarting...'); - await syncthingService.systemRestart(); - } } catch (error) { log.error(`syncthingAppsCore - Error in sync monitoring: ${error.message}`); log.error(error.stack); diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js b/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js index 6eb090ca0b..de2964a937 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js @@ -5,6 +5,8 @@ const path = require('node:path'); const log = require('../../lib/log'); const serviceHelper = require('../serviceHelper'); const volumeService = require('../utils/volumeService'); +const { SYNCTHING_IGNORE_LINES } = require('../appSystem/volumeReservedNames'); +const syncthingService = require('../syncthingService'); const { DEVICE_ID_REQUEST_TIMEOUT_MS, SYNCTHING_RESCAN_INTERVAL_SECONDS, @@ -104,7 +106,7 @@ function sortRunningAppList(runningAppList) { * @param {Map} deviceCache - Device ID cache * @param {Array} devicesConfiguration - Array to populate with devices * @param {Array} devicesIds - Array to populate with device IDs - * @param {Array} allDevicesResp - Existing syncthing devices + * @param {Array} allDevices - Existing syncthing devices * @returns {Promise} Array of device objects for folder configuration */ async function buildDeviceConfiguration( @@ -114,7 +116,7 @@ async function buildDeviceConfiguration( deviceCache, devicesConfiguration, devicesIds, - allDevicesResp, + allDevices, ) { const devices = [{ deviceID: myDeviceId }]; @@ -169,7 +171,7 @@ async function buildDeviceConfiguration( devicesIds.push(deviceID); if (deviceID !== myDeviceId) { - const syncthingDeviceExists = allDevicesResp.data.find((device) => device.name === name); + const syncthingDeviceExists = allDevices.find((device) => device.name === name); if (!syncthingDeviceExists) { devicesConfiguration.push(newDevice); } @@ -295,6 +297,58 @@ function folderNeedsUpdate(existingFolder, newFolder) { ); } +/** + * Ensure a folder's syncthing ignores carry every FluxOS policy line. + * + * .stignore is syncthing's own control file - it writes it atomically, runs as + * root so it lands on any legacy root-owned file, and never replicates it or + * its temp. So FluxOS sets the patterns through syncthing's API rather than + * writing the file: there is no temp, no ownership dance, and nothing on the + * volume to orphan on a powercut. Volume creation still seeds the file directly + * for a brand-new folder syncthing does not yet know; this converges every + * EXISTING folder whose ignores predate a policy line. + * + * Asserted by POSITION, not by presence. syncthing takes the FIRST pattern that + * matches, so a policy line sitting below anything is a policy line something + * else can answer for - an `!/backup` above it un-ignores the very directory + * this exists to keep off the network, and a presence test would call that + * converged. The policy lines therefore lead, and everything else follows in the + * order it already had. That is the same rule the v9 spec-driven writer states, + * where the owner supplies patterns of their own: they extend the set, they + * never un-exclude what FluxOS put there. + * + * Nothing is lost. syncthing's POST replaces the whole set, so the desired list + * is built FROM the current one and only duplicate copies of our own lines drop + * out. Nothing is posted when the folder already reads that way, so a converged + * folder is neither rewritten nor rescanned - which is what makes this safe on + * every monitor pass. Every syncthing call returns its outcome in-band and never + * throws, so status is checked rather than caught. + * + * Call only for a folder syncthing already knows (the caller checks); on an + * unknown folder the API would answer with an error and nothing would converge. + * + * @param {string} folderId - the syncthing folder id (the app identifier) + */ +async function ensureStignoreCovers(folderId) { + const read = await syncthingService.getFolderIgnores(folderId); + if (read.status !== 'success') { + log.error(`ensureStignoreCovers - could not read ignores for ${folderId}: ${read.data?.message ?? 'unknown error'}`); + return; + } + const current = Array.isArray(read.data?.ignore) ? read.data.ignore : []; + const rest = current.filter((line) => !SYNCTHING_IGNORE_LINES.includes(line)); + const desired = [...SYNCTHING_IGNORE_LINES, ...rest]; + const converged = desired.length === current.length + && desired.every((line, index) => line === current[index]); + if (converged) return; + const written = await syncthingService.setFolderIgnores(folderId, desired); + if (written.status !== 'success') { + log.error(`ensureStignoreCovers - could not set ignores for ${folderId}: ${written.data?.message ?? 'unknown error'}`); + return; + } + log.info(`ensureStignoreCovers - ${folderId} ignores now lead with ${SYNCTHING_IGNORE_LINES.join(', ')}`); +} + module.exports = { getDeviceID, getDeviceIDCached, @@ -303,6 +357,7 @@ module.exports = { buildDeviceConfiguration, createSyncthingFolderConfig, ensureStfolderExists, + ensureStignoreCovers, getContainerFolderPath, getContainerDataFlags, requiresSyncing, diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index f97eb5e710..2ab90fdc40 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -1,4 +1,5 @@ const config = require('config'); +const crypto = require('node:crypto'); const axios = require('axios'); const dbHelper = require('../dbHelper'); const fluxNetworkHelper = require('../fluxNetworkHelper'); @@ -8,14 +9,21 @@ const verificationHelper = require('../verificationHelper'); const log = require('../../lib/log'); const upnpService = require('../upnpService'); const serviceHelper = require('../serviceHelper'); +const messageHelper = require('../messageHelper'); const fluxHttpTestServer = require('../utils/fluxHttpTestServer'); -const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); -const { specificationFormatter } = require('../utils/appSpecHelpers'); const { localAppsInformation, globalAppsInformation } = require('../utils/appConstants'); +const appUtilities = require('../utils/appUtilities'); +const { Privilege, authOf } = require('../utils/privileges'); +const fluxCaching = require('../utils/cacheManager'); +const fluxEventBus = require('../utils/fluxEventBus'); +const { nodeSigner } = require('../utils/nodeSigner'); // Global cache for failed nodes const failedNodesTestPortsCache = new Map(); +// One entry: this answers what THIS node holds, so there is nothing to key on. +const PORTS_IN_USE_KEY = 'portsInUse'; + // A single UPnP map failure is routine on consumer routers (busy router, a // node network blip) and the app itself keeps running regardless - it must // NEVER escalate straight to a force-removal + network broadcast. Removal @@ -75,119 +83,114 @@ function ensureAppUniquePorts(appSpecFormatted) { } /** - * Get ports assigned by currently installed applications - * @returns {Promise} Array of objects with app names and their assigned ports + * The applications in a set of stored specifications, and the host ports each + * one declares. + * + * Enterprise specifications are decrypted before anything is read out of them. A + * version 8 specification seals `contacts` and `compose`, and every port an + * application holds lives inside `compose` - so a reader that skips the decrypt + * does not see an application it cannot read. It sees one holding no ports at + * all, and a hole in the answer reads as "those ports are free". + * + * Through the cached path rather than checkAndDecryptAppSpecs directly. That + * primitive holds no cache: it costs two globalAppsMessages queries and a benchd + * RSA decrypt per enterprise application on every call, and these lists are + * reached from an unauthenticated endpoint and from every spawn attempt. The + * wrapper answers from enterpriseAppDecryptionCache (keyed on spec.hash, seven + * days), shares one in-flight attempt between concurrent callers, and remembers + * a failure briefly. formatSpecs is false because the formatter strips the hash + * the cache keys on. + * + * The ports come from getAppPorts, which is the one place that derivation lives. + * What to do about a specification that would not open is left to the caller, + * and the two callers answer it differently - each says why. + * + * @param {Array} specs - stored application specifications + * @returns {Promise<{apps: Array<{name: string, ports: number[]}>, unreadable: Array}>} + */ +async function appsWithPorts(specs) { + // eslint-disable-next-line global-require + const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); + const { readable, unreadable } = await decryptEnterpriseApps(specs, { formatSpecs: false }); + + const apps = readable.map((app) => ({ + name: app.name, + ports: appUtilities.getAppPorts(app), + })); + + return { apps, unreadable }; +} + +/** + * The host ports the applications installed on this node hold. + * + * Refuses rather than answering short. This list is what portsInUse publishes to + * a Flux node sharing our public address, so a hole in it does not merely lead + * this node astray - it tells a sibling a port is free when it is not. + * + * A specification here that will not open is a genuine fault rather than a key + * this node was never meant to hold: an enterprise application only ever + * installs on ArcaneOS (appSpawner), so a node holding one can always read it. + * + * @returns {Promise>} the applications + * and the ports each holds */ async function assignedPortsInstalledApps() { - // construct object ob app name and ports array const dbopen = dbHelper.databaseConnection(); const database = dbopen.db(config.database.appslocal.database); const query = {}; const projection = { projection: { _id: 0 } }; const results = await dbHelper.findInDatabase(database, localAppsInformation, query, projection); - const decryptedApps = []; - // ToDo: move the functions around so we can remove no-use-before-define - // eslint-disable-next-line no-restricted-syntax - for (const spec of results) { - const isEnterprise = Boolean( - spec.version >= 8 && spec.enterprise, - ); - if (isEnterprise) { - // eslint-disable-next-line no-await-in-loop - const decrypted = await checkAndDecryptAppSpecs(spec); - const formatted = specificationFormatter(decrypted); - decryptedApps.push(formatted); - } else { - decryptedApps.push(spec); - } + + const { apps, unreadable } = await appsWithPorts(results); + + if (unreadable.length) { + throw new Error(`Cannot list ports in use: ${unreadable.length} of ${results.length} application specifications could not be read`); } - const apps = []; - decryptedApps.forEach((app) => { - // there is no app - if (app.version === 1) { - const appSpecs = { - name: app.name, - ports: [Number(app.port)], - }; - apps.push(appSpecs); - } else if (app.version <= 3) { - const appSpecs = { - name: app.name, - ports: [], - }; - app.ports.forEach((port) => { - appSpecs.ports.push(Number(port)); - }); - apps.push(appSpecs); - } else if (app.version >= 4) { - const appSpecs = { - name: app.name, - ports: [], - }; - app.compose.forEach((component) => { - component.ports.forEach((port) => { - appSpecs.ports.push(Number(port)); - }); - }); - apps.push(appSpecs); - } - }); + return apps; } /** - * Get ports assigned by global applications - * @param {string[]} appNames - Array of app names to check - * @returns {Promise} Array of objects with app names and their assigned ports + * The host ports named applications hold, read from the network-wide + * specifications. + * + * These are other nodes' applications - the ones the network reports as running + * at a public address this node shares - so their ports come from the broadcast + * specification, there being nothing about them installed here to read. + * + * Answers short rather than refusing, which is the opposite of the installed + * list above and deliberately so. Every node stores every global specification, + * including enterprise ones sealed to a key that a node not running ArcaneOS + * does not hold, and those are the majority of what cannot be opened here. + * Refusing would stop every installation on every such node over a specification + * it was never meant to read. The gap is said out loud and siblingHoldingPort + * covers it by asking the node that holds the port instead of reading its + * specification. + * + * @param {string[]} appNames - the applications to look up + * @returns {Promise>} the applications + * whose ports could be read, and the ports each holds */ async function assignedPortsGlobalApps(appNames) { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - if (!appNames || appNames.length === 0) { return []; } - const appsQuery = appNames.map((app) => ({ name: app })); - const query = { $or: appsQuery }; + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + const query = { $or: appNames.map((name) => ({ name })) }; const projection = { projection: { _id: 0 } }; const results = await dbHelper.findInDatabase(database, globalAppsInformation, query, projection); - const appsWithPorts = []; - - results.forEach((app) => { - const appPorts = []; + const { apps, unreadable } = await appsWithPorts(results); - if (app.version === 1) { - if (app.port) { - appPorts.push(Number(app.port)); - } - } else if (app.version <= 3) { - if (app.ports && Array.isArray(app.ports)) { - app.ports.forEach((port) => { - appPorts.push(Number(port)); - }); - } - } else if (app.version >= 4 && app.compose) { - // For compose applications, collect ports from all components - app.compose.forEach((component) => { - if (component.ports && Array.isArray(component.ports)) { - component.ports.forEach((port) => { - appPorts.push(Number(port)); - }); - } - }); - } - - if (appPorts.length > 0) { - appsWithPorts.push({ - name: app.name, - ports: appPorts, - }); - } - }); + if (unreadable.length) { + log.warn(`assignedPortsGlobalApps - ${unreadable.length} of ${results.length} specifications at this address could not be read; ` + + 'the ports those applications hold are not in this answer'); + } - return appsWithPorts; + return apps; } /** @@ -382,68 +385,422 @@ async function getAllUsedPorts() { } /** - * Check if a specific port is available - * @param {number} port - Port number to check - * @param {string} excludeApp - App name to exclude from check (for updates) - * @returns {Promise} True if port is available + * The host ports this node's applications hold. + * + * Answered from this node's own record of what it has installed rather than + * from its containers. That record covers an enterprise application like any + * other - a node decrypts its own specifications - and it covers an application + * that is installed but stopped, which still holds the router's forward. + * + * @returns {Promise} the ports, ascending */ -async function isPortAvailable(port, excludeApp = null) { - const usedPorts = await assignedPortsInstalledApps(); +async function portsInUse() { + // Cached as a VALUE rather than as a response. A route cache stores whatever + // the handler produced, so a transient failure answered with a 200 and an + // error body stays pinned for the rest of the window after the condition has + // cleared - and a sibling acts on this answer. A throw produces no value, so + // there is nothing here to remember. + const cache = fluxCaching.default.portsInUseCache; + + const cached = cache.get(PORTS_IN_USE_KEY); + if (cached) return cached; + + const ports = await getAllUsedPorts(); + const answer = ports.map(Number).filter(Number.isInteger).sort((a, b) => a - b); + + cache.set(PORTS_IN_USE_KEY, answer); + return answer; +} - // eslint-disable-next-line no-restricted-syntax - for (const app of usedPorts) { - if (excludeApp && app.name === excludeApp) { - continue; // eslint-disable-line no-continue +/** + * POST /flux/portsinuse - the ports this node holds, to a Fluxnode that signed + * the question. + * + * Asked today by another Flux node on the same public address, deciding whether + * a port it is about to install onto is already spoken for. The router forwards + * each port to exactly one node, so two applications wanting the same port at + * one address cannot both be reached, whichever applications they are - which + * is why the answer is ports alone and names no application. + * + * SIGNED, not open - and answered to any listed Fluxnode, not to siblings alone. + * Which ports a node holds is a fact about that node, and a sibling is one + * caller for it. What it discloses is small: port numbers, no application + * identity, and a port scan of the address finds most of it anyway. The reason + * for the signature is the other half. Answering means reading this node's own + * specifications, decrypting the enterprise ones, and signing the answer, and + * an anonymous caller could ask for that as often as it liked. It is the check + * /flux/checkappavailability makes - same list, same signature. + * + * An operator asking by hand is accepted on the usual privilege, so the endpoint + * stays usable directly. + * + * @param {object} req Request. + * @param {object} res Response. + * @returns {Promise} + */ +async function portsInUseApi(req, res) { + try { + // req.body, not the raw stream. express.json() is global (fluxServer.js), so + // for a JSON content type the body is already consumed by the time a handler + // runs and a stream read waits for an 'end' that has been and gone - the + // request then hangs until the caller times out, which is what it did. + // + // The older handlers on this path read the stream because the product posts + // JSON.stringify(...) as a STRING, which axios does not label as JSON, so + // the parser skips it and leaves the stream untouched. Both work; only one + // of them works for both kinds of caller. + const processedBody = serviceHelper.ensureObject(req.body); + + // A signed ask is good for its window and no longer, and the window is + // checked before the signature is: an ask outside it costs nothing to + // refuse, whoever sent it. Refused as STALE rather than as unauthentic, + // because the two want different fixes and a node whose clock has drifted + // should be able to read which from one line. + // + // Inside the window a captured ask is answered. What a replay yields is the + // port list below, signed to the ask's own time - and nothing else. + // + // Not asked of an operator: they authenticate as themselves, and a person + // asking by hand has no signature for anyone to capture. + const claimsSignature = Boolean(processedBody.pubKey && processedBody.signature); + const askedAt = claimsSignature ? Number(processedBody.timestamp) : null; + + if (claimsSignature) { + const drift = Math.abs(Date.now() - askedAt); + if (!Number.isFinite(askedAt) || drift > config.fluxapps.siblingAskValidityMs) { + throw new Error('Request is stale or carries no timestamp'); + } } - if (app.ports.includes(Number(port))) { - return false; + + const signed = await fluxNetworkHelper.verifySignedFluxnodeMessage(processedBody); + const authorized = signed + ? true + : await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); + + if (signed !== true && authorized !== true) { + throw new Error('Unable to verify request authenticity'); } + + const ports = await portsInUse(); + + // Signed, so the answer is bound to the Fluxnode that owns this address + // rather than to whatever is listening on it. A node's own record of what it + // has installed IS the truth about which ports are spoken for here, and that + // is the whole reason to act on the answer - but only once it is that node + // saying it. The request side has always established who is ASKING; without + // this the reply established nothing at all. + const signer = await nodeSigner(); + if (!signer) throw new Error('Unable to sign the answer'); + + // askedAt is the ASK's time signed back, not this answer's. It is what makes + // the answer good for one question rather than for every later one. + const answer = { pubKey: signer.pubKey, ports, askedAt }; + const signature = signer.sign(JSON.stringify(answer)); + + // A key this node has is not a signature it produced. An unsigned answer is + // discarded at the other end without a word; say what happened instead. + if (!signature) throw new Error('Unable to sign the answer'); + + res.json(messageHelper.createDataMessage({ ...answer, signature })); + } catch (error) { + log.error(error); + res.json(messageHelper.createErrorMessage( + error.message || error, + error.name, + error.code, + )); } +} - return true; + +/** + * The other Flux nodes sharing our public address. + * + * Taken from the node list rather than from app locations, so a node that is not + * running anything yet is still counted - that is the node most likely to be + * installing. + * + * @param {string} localSocketAddress - our own ip:port + * @returns {string[]|null} their socket addresses, or null when the node list is + * not known - which is an absence of information, not an absence of siblings + */ +function siblingSocketAddresses(localSocketAddress) { + const ip = extractIp(localSocketAddress); + if (!ip || !networkStateService.isReady()) return null; + + const ownPort = extractPort(localSocketAddress); + + return networkStateService.networkState() + .map((node) => node.ip) + .filter((address) => address + && extractIp(address) === ip + && extractPort(address) !== ownPort); } /** - * Find the next available port in a given range - * @param {number} startPort - Starting port to check - * @param {number} endPort - Ending port range - * @param {string} excludeApp - App name to exclude from check - * @returns {Promise} Next available port or null if none found + * The Flux node at our public address holding a port this specification wants, + * if there is one. + * + * Each node behind a shared address keeps its own database and its own docker, + * so every one of them binds the port and only the one the router forwards to + * ever receives traffic. The rest run unreachable while still being broadcast as + * live instances. + * + * Asked before the firewall is opened and before any mapping is attempted, so a + * refusal costs nothing to unwind. + * + * Answers rather than throws. This is the same class of fact as "the ports are + * not reachable from outside" and is handled on the same path. A port that + * belongs to a neighbour is an ordinary answer rather than a fault, and the + * spawner tells the two apart only by how they arrive: an answer is one log + * line and a published deferral, a throw is an error with a stack trace and + * no event. The spawn cache holds the app for the same time either way. + * + * Advisory. A sibling that is down, or answers nothing we can read, leaves us no + * wiser - and silence is never read as clearance, because the port test that + * follows is what decides. + * + * DEPENDS ON NAT HAIRPINNING. A sibling is addressed by the public address it + * shares with us, so asking it means leaving the router and being sent straight + * back in. Where a router does not do that, every sibling is silent and this + * check contributes nothing. What is lost is the only answer to "is this port + * already SPOKEN FOR here": the port test that follows answers a different + * question - whether the port reaches us right now - and so cannot see a port a + * neighbour has installed and is not currently serving. Asking over the LAN + * rather than through the router is the fix, and it belongs with the work that + * gives siblings an address that is not the shared one. + * + * @param {number[]} appPorts - the host ports this application wants + * @param {string} localSocketAddress - our own ip:port + * @returns {Promise<{address: string, port: number}|null>} the sibling and the + * port it holds, or null when none of them reported one */ -async function findNextAvailablePort(startPort, endPort, excludeApp = null) { - for (let port = startPort; port <= endPort; port += 1) { - // eslint-disable-next-line no-await-in-loop - const available = await isPortAvailable(port, excludeApp); - if (available) { - return port; +async function siblingHoldingPort(appPorts, localSocketAddress) { + try { + return await askSiblingsForHeldPort(appPorts, localSocketAddress); + } catch (error) { + // The contract above - answers, never throws - held here rather than line + // by line. Enforcing it a line at a time means guessing which line can + // fail, and a line that was not guessed leaves through the spawner's catch + // as an error against an application that did nothing wrong. Nothing about + // this question is the application's fault, and no future line added here + // can make it so. + log.warn(`siblingHoldingPort - the question could not be asked: ${error.message || error}; ` + + 'which ports this address holds is unknown to this node'); + return null; + } +} + +async function askSiblingsForHeldPort(appPorts, localSocketAddress) { + const wanted = new Set((appPorts || []).map(Number).filter(Number.isInteger)); + if (!wanted.size) return null; + + const siblings = siblingSocketAddresses(localSocketAddress); + if (!siblings || !siblings.length) return null; + + const timeout = config.fluxapps.siblingPortsTimeoutMs; + + // Signed for the sibling to verify, the same way the port test signs what it + // sends to /flux/checkappavailability. + const signer = await nodeSigner(); + + // An unsigned ask is refused by every sibling, and that reads back as "no + // sibling holds the port" - the advisory check failing open, and silently, on + // a node whose key is briefly unavailable. Answer no information instead. + if (!signer) { + log.warn('siblingHoldingPort - this node cannot sign the request; no sibling was asked'); + return null; + } + // Timestamped. Without it the body is constant - a key and a fixed word - so + // one captured signature is a bearer token for this endpoint on every node in + // the network, for ever. It is also what binds the ANSWER to this question: + // the sibling signs the time back, so a captured answer cannot be replayed + // into a later ask either. One field, both directions. + const ask = { pubKey: signer.pubKey, asking: 'portsInUse', timestamp: Date.now() }; + const signature = signer.sign(JSON.stringify(ask)); + + if (!signature) { + log.warn('siblingHoldingPort - could not sign the request; no sibling was asked'); + return null; + } + + const signedAsk = { ...ask, signature }; + + const answers = await Promise.all(siblings.map(async (address) => { + try { + const response = await axios.post( + `http://${extractIp(address)}:${extractPort(address)}/flux/portsinuse`, + signedAsk, + // A list of port numbers, so the ceiling is generous by orders of + // magnitude. Bounded at all because axios does not bound a response body + // by default, and this address is only as trustworthy as whatever is + // answering on it. + { timeout, maxContentLength: 64 * 1024, maxBodyLength: 64 * 1024 }, + ).catch(() => null); + + const body = response && response.data; + if (!body || body.status !== 'success') { + // Said out loud. A sibling refusing the question - a stale ask, a clock + // that has drifted, a key it will not accept - is a different thing from a + // sibling that holds nothing, and both used to be the same silence. + const refusal = body && body.data && body.data.message; + if (refusal) log.warn(`siblingHoldingPort - ${address} refused the question: ${refusal}`); + return null; + } + + const answer = body.data; + if (!answer || !Array.isArray(answer.ports)) return null; + + // The answer names the question it answers. An answer signed for some other + // ask is a recording, and a recording says what was true then. + if (answer.askedAt !== ask.timestamp) { + log.warn(`siblingHoldingPort - ${address} answered a question other than the one asked; ignoring it`); + return null; + } + + // Verified as the answer of the node AT THIS ADDRESS. A listed Fluxnode + // elsewhere signing a port list says nothing about what is installed here, + // and this address is only as trustworthy as whatever is answering on it. + // The body is passed through as it arrived, because that is what was signed. + // eslint-disable-next-line no-await-in-loop + const verified = await fluxNetworkHelper.verifySignedFluxnodeMessage(answer, { socketAddress: address }); + + if (!verified) { + log.warn(`siblingHoldingPort - ${address} did not answer as the Flux node at that address; ignoring it`); + return null; + } + + return { address, ports: answer.ports.map(Number) }; + } catch (error) { + // One sibling breaking is not the others breaking, so this loses that + // sibling and no more. A dial that fails is handled above and stays + // quiet - where the router does not hairpin that is every sibling on + // every cycle, and a line each would bury everything else. Anything + // reaching HERE is not the expected silence, so it says so. + log.warn(`siblingHoldingPort - ${address} could not be asked: ${error.message || error}`); + return null; } + })); + + const heard = answers.filter(Boolean); + + if (!heard.length) { + // Nothing was learned, which is not the same as there being nothing to + // learn - and the two used to be the same silence. Reaching a sibling means + // leaving the router and being sent straight back in, so where the router + // does not hairpin every sibling looks mute and this check quietly + // contributes nothing on the very topology it exists for. Worth a line: it + // is an anomaly rather than a routine state, and it cannot be told from an + // absence of siblings by anyone reading afterwards. + log.warn(`siblingHoldingPort - ${siblings.length} Flux node(s) share this address and none of them answered; ` + + 'which ports they hold is unknown to this node'); + return null; + } + + // eslint-disable-next-line no-restricted-syntax + for (const answer of heard) { + const held = answer.ports.find((port) => wanted.has(port)); + if (held) return { address: answer.address, port: held }; } + return null; } /** - * Sign application data for verification - * @param {string} message - Message to sign - * @returns {Promise} Signature + * How many independent peers must agree before a port test refuses an install. + * Not config: it is a property of what counts as evidence, not a tuning knob, + * and a fluxapps key has to be added in two places to be visible under test. */ -async function signCheckAppData(message) { - const privKey = await fluxNetworkHelper.getFluxNodePrivateKey(); - const signature = await verificationHelper.signMessage(message, privKey); - return signature; +const PORT_TEST_CORROBORATION = 2; + +/** + * The port to refuse on once enough independent readings agree the ports are + * not ours, or null while the evidence is still one peer's word. + * + * Proof and report are not the same kind of evidence. Our own token coming back + * PROVES the port reaches this node - a secret cannot be manufactured, so one + * peer settles it and a second adds nothing. Anything else is one observer's + * report about a third party, and a report can be wrong: a truncated read, + * something in the path, a peer having a bad moment. Refusing on the first of + * those stops the node installing anything at all, quietly, until somebody goes + * looking for it. + * + * A real collision corroborates itself for free. Behind a shared address the + * router forwards that port to one node, and every peer that reads it sees the + * same thing; an anomaly does not reproduce on a different peer. So: one witness + * to accept, two to refuse. + * + * Keyed by peer, because the draw is random and can return the same peer twice - + * one peer asked twice is one witness, whatever the count of readings says. + * + * @param {Map} disagreements Peer address -> the port its + * reading said was not ours. + * @param {number} corroboration How many distinct peers must agree. + * @returns {number | null} The first port to refuse on, or null. + */ +function refusedPort(disagreements, corroboration) { + if (disagreements.size < corroboration) return null; + return disagreements.values().next().value; +} + +/** + * The first port whose answer was not ours, or null when every one of them was. + * + * The peer read each port and handed back what it found; the comparison is + * HERE, against a secret the peer was never given. That direction is the whole + * design. A peer cannot tell our application from a neighbour's at the same + * address - that is why this check exists - so a peer is not in a position to + * judge, and one that is old, broken or lying cannot manufacture a token it + * never saw. + * + * Only ports the peer would have read are required to carry it: it skips any + * outside the app port range, and a port it never tried says nothing either + * way. + * + * @param {number[]} portsToTest - the ports asked about + * @param {object} answered - port -> what that port replied, from the peer + * @param {string} token - the secret this node published on its test servers + * @returns {number|null} the first port that was not ours, or null + */ +function portNotOurs(portsToTest, answered, token) { + if (!token) return null; + + const at = portsToTest.findIndex((port) => { + const probed = port >= config.fluxapps.portMin && port <= config.fluxapps.portMax; + if (!probed) return false; + + const reply = answered?.[port] ?? answered?.[String(port)]; + + // Substring rather than equality: the peer hands back the raw bytes the + // port produced, headers and all, capped. What matters is that our token is + // in there and could only have come from us. + return typeof reply !== 'string' || !reply.includes(token); + }); + + return at === -1 ? null : portsToTest[at]; } /** - * To check if app ports are available publicly before installation + * To check if app ports are available publicly before installation. + * + * Answers with the decision itself rather than a bare boolean: proceeding + * because our own token came back and proceeding because nothing could be + * learned are the same outcome for the caller and completely different facts + * about the network, and only one of them is evidence. The object returned is + * the one published to the harness bus, so the two cannot drift. + * * @param {Array} portsToTest Array of ports to test - * @returns {Promise} True if ports are available, false otherwise + * @returns {Promise<{ok: boolean, reason: string, port?: number, peers?: string[], + * readings?: object, asked?: string[], silent?: boolean}>} */ async function checkInstallingAppPortAvailable(portsToTest = []) { const beforeAppInstallTestingServers = []; + // One secret for this whole test run, published by our own test servers and + // never sent to the peer. The peer returns what it read; we compare. + const portTestToken = crypto.randomBytes(16).toString('hex'); const isUPNP = upnpService.isUPNP(); - let portsStatus = false; - const portsNotWorking = new Set(); - let originalPortFailed = null; - let nextTestingPort = 0; + // null until something decides; see the loop below. + let decision = null; try { const localSocketAddress = await fluxNetworkHelper.getLocalSocketAddress(); @@ -453,7 +810,9 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { const localIp = extractIp(localSocketAddress); const localPort = extractPort(localSocketAddress); - const pubKey = await fluxNetworkHelper.getFluxNodePublicKey(); + const signer = await nodeSigner(); + if (!signer) throw new Error('Unable to sign the port test'); + let somePortBanned = false; portsToTest.forEach((portToTest) => { const iBP = fluxNetworkHelper.isPortBanned(portToTest); @@ -462,7 +821,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { } }); if (somePortBanned) { - return false; + return { ok: false, reason: 'portBanned' }; } if (isUPNP) { somePortBanned = false; @@ -473,7 +832,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { } }); if (somePortBanned) { - return false; + return { ok: false, reason: 'portUpnpBanned' }; } } const firewallActive = await fluxNetworkHelper.isFirewallActive(); @@ -491,7 +850,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { throw new Error('Failed to create map UPNP port'); } } - const testHttpServer = new fluxHttpTestServer.FluxHttpTestServer(); + const testHttpServer = new fluxHttpTestServer.FluxHttpTestServer(portTestToken); // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(config.fluxapps.portTestBindDelayMs); @@ -531,28 +890,65 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { port: localPort, appname: 'appPortsTest', ports: portsToTest, - pubKey, + pubKey: signer.pubKey, + // Asks the peer to READ each port and hand back what it found. The token + // it should find is deliberately not here: a peer that never learns it + // cannot produce it without actually reaching us. + echo: true, }; const stringData = JSON.stringify(data); // eslint-disable-next-line no-await-in-loop - const signature = await signCheckAppData(stringData); + const signature = signer.sign(stringData); data.signature = signature; + // Every attempt does one of two things: it DECIDES, or it records what it + // learned and asks somebody else. Running out - of attempts, of peers, of + // anyone willing to answer - is resolved once, after the loop. + // + // It used to be resolved in three places inside it, each a variation on the + // same paragraph, and a fourth way of running out had no paragraph at all: a + // last attempt whose peer never answered fell out of the bottom onto the + // `false` the answer was initialised with, and refused the install having + // said nothing. A verdict and "nobody has decided yet" must never be the + // same value, or the next exit somebody adds inherits that too. let i = 0; - let finished = false; - while (!finished && i < config.fluxapps.portTestMaxAttempts) { + + // Peer address -> the port that peer said is not serving this node. Keyed by + // peer so that redrawing the same one does not read as a second opinion. + const disagreements = new Map(); + // Every peer already asked, so a redraw is ANOTHER peer rather than another + // draw. Without this the picker can hand back the one just asked - which on + // a budget of two attempts it often does - and a check counting distinct + // witnesses never reaches two however many times it tries. + const asked = []; + // Why the loop stopped learning, for the resolution below. There being + // nobody left outside this address is not the same as everyone who was there + // being unable or unwilling to answer, and a reader needs to know which. + let ranOutOfObservers = false; + let sawUnreadablePeer = false; + let sawSilentPeer = false; + + while (decision === null && i < config.fluxapps.portTestMaxAttempts) { i += 1; // eslint-disable-next-line no-await-in-loop - const randomSocketAddress = await networkStateService.getRandomSocketAddress( + const randomSocketAddress = await networkStateService.getRandomExternalObserver( localSocketAddress, + { exclude: asked }, ); - // this should never happen as the list should be populated here + // Nobody outside this address left to ask. A Flux node sharing our public + // address is not outside it: its packets never leave the router, so what + // it can reach says nothing about what the internet can reach. Answering + // nothing is honest, and the resolution below takes it the same way it + // takes every other "nothing was learned" - which is the check this node + // made before the token existed. if (!randomSocketAddress) { - throw new Error('Unable to get random test connection'); + ranOutOfObservers = true; + break; } const askingIP = extractIp(randomSocketAddress); const askingIpPort = extractPort(randomSocketAddress); + asked.push(randomSocketAddress); // first check against our IP address // eslint-disable-next-line no-await-in-loop @@ -560,26 +956,144 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { log.error(`${askingIP} for app availability is not reachable`); log.error(error); }); - if (resMyAppAvailability && resMyAppAvailability.data.status === 'error') { - if (resMyAppAvailability.data.data && resMyAppAvailability.data.data.message && resMyAppAvailability.data.data.message.includes('Failed port: ')) { - const portToRetest = serviceHelper.ensureNumber(resMyAppAvailability.data.data.message.split('Failed port: ')[1]); - if (portToRetest > 0) { - portsNotWorking.add(portToRetest); - // if we aren't already testing ports, we set it here, otherwise, just continue - if (!originalPortFailed) { - originalPortFailed = portToRetest; - // eslint-disable-next-line no-unused-vars - nextTestingPort = portToRetest < 65535 ? portToRetest + 1 : portToRetest - 1; - } - } + + // What this attempt learned about one of our ports, or null when it + // learned nothing at all. Only a port recorded here counts as a witness. + let reportedPort = null; + + if (!resMyAppAvailability) { + // The peer never answered us. That is a fact about the peer, not about + // our ports. + sawSilentPeer = true; + // eslint-disable-next-line no-continue + continue; + } + + if (resMyAppAvailability.data.status === 'error') { + // Two different things arrive here and only one of them is about our + // ports. A peer that NAMES the port it could not reach has read + // something at this address. A peer that rejected the request outright - + // a stale node list, a clock, a key briefly unavailable at its end - has + // told us nothing, and taking that for "your port is closed" refuses an + // install that was fine and records a cause that never happened. + const failure = resMyAppAvailability.data.data && resMyAppAvailability.data.data.message; + const failedPort = typeof failure === 'string' && failure.includes('Failed port: ') + ? serviceHelper.ensureNumber(failure.split('Failed port: ')[1]) + : null; + + if (!(failedPort > 0)) { + log.warn(`checkInstallingAppPortAvailable - ${askingIP} would not answer the question (${failure || 'no reason given'}); asking another peer`); + sawSilentPeer = true; + // eslint-disable-next-line no-continue + continue; + } + + // One peer's report that a port did not answer it. Evidence, and it goes + // to the same rule everything else goes to - not a verdict. Only our own + // token coming back settles a port on one peer's say-so, because only + // this node could have produced it. + log.warn(`checkInstallingAppPortAvailable - ${askingIP} could not reach port ${failedPort}`); + reportedPort = failedPort; + } else if (resMyAppAvailability.data.status === 'success') { + const { answered } = resMyAppAvailability.data.data || {}; + + if (!answered) { + // This peer is on older code: it reached the ports but did not read + // them, so it has told us nothing we can act on. Ask someone else. + log.info(`checkInstallingAppPortAvailable - ${askingIP} cannot read ports back, asking another peer`); + sawUnreadablePeer = true; + // eslint-disable-next-line no-continue + continue; } - portsStatus = false; - finished = true; - } else if (resMyAppAvailability && resMyAppAvailability.data.status === 'success') { - portsStatus = true; - finished = true; + + const notOurs = portNotOurs(portsToTest, answered, portTestToken); + + if (notOurs === null) { + // Our own token came back. Proof, not report - only this node could + // have produced it - so one peer settles it. + decision = { ok: true, reason: 'proven', port: null }; + break; + } + + log.info(`checkInstallingAppPortAvailable - ${askingIP} read something other than this node on port ${notOurs}`); + reportedPort = notOurs; + } else { + // An answer in a shape this node does not understand is not an answer. + sawSilentPeer = true; + // eslint-disable-next-line no-continue + continue; + } + + disagreements.set(askingIP, reportedPort); + + const refused = refusedPort(disagreements, PORT_TEST_CORROBORATION); + + if (refused === null) { + log.info(`checkInstallingAppPortAvailable - one peer so far says port ${reportedPort} is not ours; asking another before refusing`); + // eslint-disable-next-line no-continue + continue; } + + // Behind a shared address that is a neighbour's application holding the + // router's forward, which is exactly what this refuses - and now more than + // one peer has read it that way. Each peer's own reading is named, because + // two peers tripping on different ports is still corroboration and a line + // naming only the first port would be describing something else. + const readings = [...disagreements.entries()].map(([peer, port]) => `${peer} on port ${port}`); + log.warn(`checkInstallingAppPortAvailable - port ${refused} at this address is answered by something other than this node, as read by ` + + `${disagreements.size} peers (${readings.join(', ')}). Installation aborted.`); + decision = { + ok: false, + reason: 'notOurs', + port: refused, + peers: [...disagreements.keys()], + readings: Object.fromEntries(disagreements), + }; + fluxEventBus.publish('ports:notOurs', decision); + } + + if (decision === null) { + // Nothing proved and nothing corroborated. What was not learned does not + // refuse an install: this is the check this node made before the token + // existed, and it is what stops the first nodes to upgrade refusing + // everything while the rest of the network catches up. + let reason; + if (disagreements.size) { + reason = ranOutOfObservers ? 'noOtherObserver' : 'singleWitness'; + } else if (ranOutOfObservers) { + reason = 'noObserver'; + } else if (sawUnreadablePeer) { + reason = 'noReader'; + } else { + reason = 'noneAnswered'; + } + + const witnesses = [...disagreements.keys()]; + const disputed = disagreements.size ? [...disagreements.values()][0] : null; + + if (witnesses.length) { + log.warn(`checkInstallingAppPortAvailable - ${witnesses.length} peer(s) read a port that was not ours ` + + `(${witnesses.join(', ')}) and no other Flux node outside this address could be asked; ` + + 'proceeding on reachability alone'); + } else { + log.warn(`checkInstallingAppPortAvailable - nothing was learned about these ports from ${asked.length} peer(s) asked ` + + `(${reason}); proceeding on reachability alone`); + } + + decision = { + ok: true, + reason, + port: disputed, + // The peers that disagreed, where any did. `asked` carries everyone who + // was asked, which is the other question a reader of this wants + // answered. + peers: reason === 'noReader' ? asked.map(extractIp) : witnesses, + asked: asked.map(extractIp), + silent: sawSilentPeer, + }; + fluxEventBus.publish('ports:unproven', decision); } + // stop listening on the port, close the port // eslint-disable-next-line no-restricted-syntax for (const portToTest of portsToTest) { @@ -603,7 +1117,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { }); })), ); - return portsStatus; + return decision; } catch (error) { let firewallActive = true; firewallActive = await fluxNetworkHelper.isFirewallActive().catch((e) => log.error(e)); @@ -636,7 +1150,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { })), ); log.error(error); - return false; + return { ok: false, reason: 'error' }; } } @@ -653,18 +1167,20 @@ async function callOtherNodeToKeepUpnpPortsOpen() { return; } - const randomSocketAddress = await networkStateService.getRandomSocketAddress(localSocketAddr); + // An external observer, for the same reason the install-time port test wants + // one: a node behind our own router cannot tell us what our address looks + // like from outside it. This used to redraw by hand on a same-IP match; the + // picker guarantees it now, for every caller that asks this question. + const randomSocketAddress = await networkStateService.getRandomExternalObserver(localSocketAddr); if (!randomSocketAddress) return; const askingIP = extractIp(randomSocketAddress); const askingIpPort = extractPort(randomSocketAddress); + // Still needed below: this node's own address is what it asks the peer to + // keep open. It is no longer used to reject a same-address peer - the + // picker does that. const localIp = extractIp(localSocketAddr); - - if (localIp === askingIP) { - callOtherNodeToKeepUpnpPortsOpen(); - return; - } if (failedNodesTestPortsCache.has(askingIP)) { callOtherNodeToKeepUpnpPortsOpen(); return; @@ -678,7 +1194,9 @@ async function callOtherNodeToKeepUpnpPortsOpen() { return; } const apps = installedAppsRes.data; - const pubKey = await fluxNetworkHelper.getFluxNodePublicKey(); + const signer = await nodeSigner(); + if (!signer) throw new Error('Unable to sign the UPnP request'); + const ports = []; // eslint-disable-next-line no-restricted-syntax for (const app of apps) { @@ -718,12 +1236,12 @@ async function callOtherNodeToKeepUpnpPortsOpen() { ip: localIp, apiPort, ports, - pubKey, + pubKey: signer.pubKey, timestamp: Math.floor(Date.now() / 1000), }; const stringData = JSON.stringify(dataUPNP); - const signature = await signCheckAppData(stringData); + const signature = signer.sign(stringData); dataUPNP.signature = signature; const logMsg = `callOtherNodeToKeepUpnpPortsOpen - calling ${askingIP}:${askingIpPort} to test ports: ${ports}`; @@ -748,9 +1266,11 @@ module.exports = { restoreAppsPortsSupport, restorePortsSupport, getAllUsedPorts, - isPortAvailable, - findNextAvailablePort, - signCheckAppData, + portsInUse, + portsInUseApi, + portNotOurs, + refusedPort, + siblingHoldingPort, checkInstallingAppPortAvailable, callOtherNodeToKeepUpnpPortsOpen, failedNodesTestPortsCache, diff --git a/ZelBack/src/services/appPlacement/geolocationRule.js b/ZelBack/src/services/appPlacement/geolocationRule.js new file mode 100644 index 0000000000..4ade69524d --- /dev/null +++ b/ZelBack/src/services/appPlacement/geolocationRule.js @@ -0,0 +1,254 @@ +// The app geolocation rule, parsed once. +// +// A spec's geolocation array decides which nodes may run the app, and three +// places need that decision in different shapes: the candidate count asks it of +// every node on the list, the spawner asks it of this node against every app +// missing instances, and the installer asks it of this node against one app. +// Spelled out separately in each place the copies drift, so the meaning lives +// here and the callers only evaluate. +// +// parseGeolocation turns the entry array into terms. A term carries the +// granularity it applies at and, at region granularity, the proof it demands: a +// region pin is a promise, so it admits a node only where that node's region is +// known and equal, and a region ban excludes only where the same holds. Enforce +// on proof, ban on proof - a node the table cannot place at region granularity +// satisfies no region pin and is caught by no region deny. +// +// Parsing is separate from evaluating because the candidate count parses one +// spec and evaluates it against thousands of nodes; deriving the terms per node +// made that cost the product of the two. +// +// Allowed and forbidden entries are parsed by different functions, because the +// network's stored semantics differ: an allow of `_ALL` admits the whole +// continent, while the matching deny is compared whole against the node's own +// location string and so excludes nothing. That asymmetry is the wire format's, +// not a choice made here. + +// The table's own region vocabulary: a full ISO 3166-2 code belonging to the +// entry's country part. Anything else - ip-api region names, _NONE, _ALL, +// retired codes - is legacy-shaped and keeps country granularity, because +// matching it against the table's vocabulary would exclude nodes the installer +// accepts. +const TABLE_REGION = /^[A-Z]{2}-[A-Z0-9]{1,3}$/; + +/** + * @typedef {object} GeoTerm + * @property {'all'|'continent'|'country'|'region'|'never'} granularity What the + * term is capable of matching. 'never' is a term the wire format permits but + * which can match no node - a region part outside the table's vocabulary in a + * deny, where install-time compares the whole string and finds nothing. + * @property {string|null} continent + * @property {string|null} country + * @property {string|null} region + */ + +/** + * @typedef {object} GeoRule + * @property {boolean} unrestricted No entries at all - every node satisfies it + * @property {Array} allows + * @property {Array} denies + * @property {string|null} legacyCountry The first `b` entry's country + * @property {string|null} legacyContinent The first `a` entry's continent + */ + +/** + * Whether a region part is an ISO 3166-2 code belonging to that country. Half of + * the question a caller actually has - regionCodeOf is the whole of it, and is + * what everything outside this module asks. Answering "is this a table region?" + * without the vocabulary reports a name the table resolves as one it cannot. + * @param {string} part The entry's region part + * @param {string} countryPart The entry's country part + * @returns {boolean} + */ +function isTableRegionPart(part, countryPart) { + return TABLE_REGION.test(part ?? '') && part.slice(0, 2) === countryPart; +} + +/** + * @param {'all'|'continent'|'country'|'region'|'never'} granularity + * @param {string|null} [continent] + * @param {string|null} [country] + * @param {string|null} [region] + * @returns {GeoTerm} + */ +function term(granularity, continent = null, country = null, region = null) { + return { + granularity, continent, country, region, + }; +} + +/** + * The region code a third part names, in either vocabulary: an ISO 3166-2 code + * as written, or a region name resolved through the published vocabulary. Null + * when neither - the caller then answers the entry at country granularity, + * which counts more nodes rather than fewer. + * @param {string[]} parts The entry body, split on `_` + * @param {(countryCode: string, regionName: string) => string|null} resolveRegion + * @returns {string | null} + */ +function regionCodeOf(parts, resolveRegion = () => null) { + if (isTableRegionPart(parts[2], parts[1])) return parts[2]; + // a name may carry the separator, and install-time compares the whole tail + return resolveRegion(parts[1], parts.slice(2).join('_')); +} + +/** + * An allowed entry's body as a term. `ALL` admits everything and `_ALL` + * admits the continent; a third part naming a region in either vocabulary pins + * it, and one that resolves to neither admits the whole country. + * @param {string} body The entry with its `ac` prefix removed + * @param {(countryCode: string, regionName: string) => string|null} resolveRegion + * @returns {GeoTerm} + */ +function parseAllowedBody(body, resolveRegion) { + if (body === 'ALL') return term('all'); + const parts = body.split('_'); + if (parts.length === 1) return term('continent', parts[0]); + if (parts.length === 2) { + return parts[1] === 'ALL' ? term('continent', parts[0]) : term('country', parts[0], parts[1]); + } + const code = regionCodeOf(parts, resolveRegion); + if (code) return term('region', parts[0], parts[1], code); + return term('country', parts[0], parts[1]); +} + +/** + * A forbidden entry's body as a term. A deny excludes only at the granularities + * install-time resolves: continent, continent_country, and a region part in + * either vocabulary - the table's own codes, or an ip-api name the published + * artifact's region-name vocabulary maps onto one. Every other shape - `ALL`, + * `_ALL`, `_NONE`, a region part neither vocabulary resolves - bans + * nothing, which is why `_NONE` must never be stripped: doing so would ban a + * whole country the installer would accept. + * @param {string} body The entry with its `a!c` prefix removed + * @param {(countryCode: string, regionName: string) => string|null} resolveRegion + * @returns {GeoTerm} + */ +function parseForbiddenBody(body, resolveRegion) { + const parts = body.split('_'); + if (parts.length === 1) return term('continent', parts[0]); + if (parts.length === 2) return term('country', parts[0], parts[1]); + const code = regionCodeOf(parts, resolveRegion); + if (code) return term('region', parts[0], parts[1], code); + return term('never'); +} + +/** + * Parse a spec's geolocation array into the rule it expresses. Entries that are + * not strings carry no constraint and are skipped: a term cannot be derived + * from them, and failing the whole computation over one would refuse an app the + * installer has no objection to. + * @param {Array} entries A spec's geolocation array + * @param {(countryCode: string, regionName: string) => string|null} [resolveRegion] + * Resolves a region an app named the way ip-api does to its ISO 3166-2 code. + * Without one, such an entry is answered at country granularity - what a node + * holding no vocabulary can prove, and the direction that counts more nodes. + * @returns {GeoRule} + */ +function parseGeolocation(entries, resolveRegion = () => null) { + const list = entries ?? []; + const rule = { + unrestricted: list.length === 0, + allows: [], + denies: [], + legacyCountry: null, + legacyContinent: null, + }; + list.forEach((entry) => { + if (typeof entry !== 'string') return; + if (entry.startsWith('a!c')) { + rule.denies.push(parseForbiddenBody(entry.slice(3), resolveRegion)); + return; + } + if (entry.startsWith('ac')) { + rule.allows.push(parseAllowedBody(entry.slice(2), resolveRegion)); + return; + } + // legacy pins: the first of each wins, matching install-time's find() + if (entry.startsWith('b')) { + if (rule.legacyCountry === null) rule.legacyCountry = entry.slice(1); + return; + } + if (entry.startsWith('a') && rule.legacyContinent === null) { + rule.legacyContinent = entry.slice(1); + } + }); + return rule; +} + +/** + * Whether one term covers a node location. + * @param {GeoTerm} geoTerm + * @param {{continentCode: string|null, countryCode: string|null, + * region: string|null}} loc Node location + * @returns {boolean} + */ +function termCoversLocation(geoTerm, loc) { + switch (geoTerm.granularity) { + case 'all': + return true; + case 'continent': + return geoTerm.continent === loc.continentCode; + case 'country': + return geoTerm.continent === loc.continentCode && geoTerm.country === loc.countryCode; + case 'region': + // Proof, in both directions: a region term always carries an ISO 3166-2 + // code, and a node whose region the table does not carry holds null, so + // the equality admits it to no pin and exposes it to no ban. + return geoTerm.continent === loc.continentCode + && geoTerm.country === loc.countryCode + && geoTerm.region === loc.region; + default: + return false; + } +} + +/** + * Whether a node location satisfies a parsed rule. A location the table cannot + * resolve satisfies every rule: the rule cannot prove it ineligible, and only + * proven ineligibility may exclude a node from the candidate count. + * + * Denies are answered before allows, so a deny beats an allow. The legacy pins + * apply only in the absence of the current syntax, matching install-time: a + * `b` country pin applies whenever no allow entry exists, and an + * `a` continent pin only when there are no allow AND no deny entries. + * @param {GeoRule} rule A parseGeolocation() result + * @param {{continentCode: string|null, countryCode: string|null, + * region: string|null}} loc Node location + * @returns {boolean} + */ +function locationSatisfiesRule(rule, loc) { + if (rule.unrestricted) return true; + if (!loc || !loc.countryCode || !loc.continentCode) return true; + if (rule.denies.some((geoTerm) => termCoversLocation(geoTerm, loc))) return false; + if (rule.allows.length) { + return rule.allows.some((geoTerm) => termCoversLocation(geoTerm, loc)); + } + if (rule.legacyCountry !== null && rule.legacyCountry !== loc.countryCode) return false; + if (rule.denies.length === 0 + && rule.legacyContinent !== null && rule.legacyContinent !== loc.continentCode) { + return false; + } + return true; +} + +/** + * Parse and evaluate in one call, for the callers that answer about a single + * node. Callers evaluating many nodes against one spec must parse once and + * reuse the rule. + * @param {{continentCode: string|null, countryCode: string|null, + * region: string|null}} loc Node location + * @param {Array} entries A spec's geolocation array + * @param {(countryCode: string, regionName: string) => string|null} [resolveRegion] + * @returns {boolean} + */ +function locationSatisfiesGeolocation(loc, entries, resolveRegion) { + return locationSatisfiesRule(parseGeolocation(entries, resolveRegion), loc); +} + +module.exports = { + regionCodeOf, + parseGeolocation, + locationSatisfiesRule, + locationSatisfiesGeolocation, +}; diff --git a/ZelBack/src/services/appPlacement/ipLocationStore.js b/ZelBack/src/services/appPlacement/ipLocationStore.js new file mode 100644 index 0000000000..4a6089b571 --- /dev/null +++ b/ZelBack/src/services/appPlacement/ipLocationStore.js @@ -0,0 +1,836 @@ +// Mongo-backed store for the IP location baseline published in +// RunOnFlux/fluxos-network-policy (iplocation.bin.gz, format 2). +// +// The table maps every allocated IPv4 range to the organisation that holds it +// and where it is located (DB-IP City Lite country and region over RIR +// allocation boundaries). A range with no organisation carries no fault domain +// of its own and falls to the /16 rung placement computes. It is the input that +// makes placement fault domains computable locally - see placementFeasibility. +// +// The rows live in mongo, not in the process: two million ranges cost more +// resident memory than a node can spare, and the covering-row query answers in +// well under a millisecond. Ingest builds a fresh collection and swaps it in +// with a single rename, so a reader sees either the whole previous table or the +// whole new one, never a partial one. +// +// Decompressed artifact layout, integers little-endian: +// 0 6 magic 'FLXGEO' +// 6 1 format version, 0x02 +// 7 4 u32 header length +// 11 - header, UTF-8 JSON: generated, sources, countries, continents, orgs, regions +// - 4 u32 row count +// - - rows, each five unsigned LEB128 varints: +// gap (start - prevEnd - 1, prevEnd = -1 before the first row), +// len (end - start), org/cc/region (index + 1, 0 = none) +// +// This module holds no fetch logic. The artifact arrives via setArtifact() from +// whatever distribution layer feeds it; absence of a table is a valid state +// every consumer must handle. +// +// Two things outlive the process alongside the rows: an ingest marker naming +// the baseline the collection holds, so a boot adopts it instead of re-ingesting +// the same two million rows, and the per-node view (nodelocations) placement +// reads in one query rather than a lookup per node. + +const zlib = require('node:zlib'); +const util = require('node:util'); +const config = require('config'); +const log = require('../../lib/log'); +const dbHelper = require('../dbHelper'); +const cidrUtils = require('../utils/cidrUtils'); +const { bareIp } = require('../utils/socketAddressUtils'); + +const gunzip = util.promisify(zlib.gunzip); + +// Hand the thread back so queued I/O and timers run. setImmediate, not a +// resolved promise: a promise only drains microtasks, which is the same thread +// with extra steps. +const yieldToEventLoop = () => new Promise((resolve) => { setImmediate(resolve); }); + +const MAGIC = 'FLXGEO'; +const SUPPORTED_VERSION = 2; +// magic(6) + version(1) + u32 header length +const HEADER_OFFSET = 11; +const IPV4_MAX = 0xffffffff; +// A structurally valid but truncated generation must not replace a good +// table. A fleet-integrity invariant, deliberately NOT configuration: a knob +// would let a single node (or a config-generation defect) switch the +// protection off. The harness publishes padded real-scale artifacts instead. +const MIN_ROW_COUNT = 1500000; +const INSERT_BATCH_SIZE = 10000; +const MAX_BATCHES_IN_FLIGHT = 4; +// Rows decoded between handing the thread back. Small enough that no single +// stretch is noticeable (a few milliseconds), large enough that two million rows +// cost a couple of hundred yields rather than one per row. +const ROWS_PER_YIELD = 10000; +// Bounds what one artifact string can cost in a document. Not a vocabulary +// check - the vocabularies are the publisher's, and rejecting a token this +// build has not seen would take the whole fleet's table down with it. +const MAX_TOKEN_LENGTH = 64; +// A u32 needs at most five LEB128 bytes; every field of a row is a u32. +const MAX_VARINT_BYTES = 5; +const STORE_UNAVAILABLE = 'IPLOCATION_STORE_UNAVAILABLE'; +// The marker shares policyDocuments with the artifact record, under its own id. +const INGEST_MARKER_ID = 'ipLocationTableIngest'; +// A point lookup costs well under a millisecond; eight in flight fill a fleet's +// worth of node locations in about a second without crowding the API's queries. +const NODE_LOOKUP_CONCURRENCY = 8; + +const ipRangesCollection = config.database.local.collections.ipRanges; +const ipRangesNextCollection = `${ipRangesCollection}_next`; +const nodeLocationsCollection = config.database.local.collections.nodeLocations; +const policyDocumentsCollection = config.database.local.collections.policyDocuments; + +let status = { ready: false, generated: null, rowCount: 0 }; +// country -> continent, from the header of whatever baseline this node holds +let continentByCountry = new Map(); +// '|' -> ISO 3166-2, from the same header. The rows carry +// codes, while an app's geolocation may name a region the way ip-api does, and +// this is what connects the two. +let regionCodeByName = new Map(); + +// Organisation token -> network class, from the header's optional orgClasses. +// The classification is decided in the policy repo, where evidence a node cannot +// gather for itself (the registries' own record of what a block was assigned +// for) is available; here it is only read. +const NETWORK_CLASS_BY_CODE = Object.freeze({ 1: 'RESIDENTIAL', 2: 'DATACENTER' }); + +/** + * The class a wire code names, or null when this build does not know it. + * + * An explicit comparison rather than a property lookup: `NETWORK_CLASS_BY_CODE[code]` + * answers for every member of Object.prototype, so a header carrying + * `{"": "toString"}` would pass a truthiness check and store an inherited + * function as an organisation's network class. + * @param {*} code The code as it appears in the header. + * @returns {string|null} The class name, or null. + */ +function classForCode(code) { + if (code === 1) return NETWORK_CLASS_BY_CODE[1]; + if (code === 2) return NETWORK_CLASS_BY_CODE[2]; + return null; +} +let networkClassByOrg = new Map(); +let minimumRowCount = MIN_ROW_COUNT; +// The per-node view, resident. It is a decoration on the node list - one small +// fact per listed address - and the node list lives in this process, so keeping +// the view beside it removes both the per-computation read and the need to keep +// a collection in step with a list. Mongo still holds it, for restarts. +// Replaced whole on refresh, never mutated: see nodeLocationSnapshot. +let nodeView = new Map(); +let nodeViewLoaded = false; + +/** + * An artifact this build refuses. The caller keeps whatever it already holds. + * @param {string} message What is wrong with the bytes + * @returns {Error} + */ +function malformed(message) { + return new Error(`iplocation artifact: ${message}`); +} + +/** + * The store could not be reached. Tagged so a caller can tell "the table could + * not be read" from "no row covers this address" - see lookup(). + * @param {string} message Underlying reason + * @returns {Error} + */ +function unavailable(message) { + const error = new Error(`iplocation store unavailable: ${message}`); + error.code = STORE_UNAVAILABLE; + return error; +} + +/** + * Whether an error means the store could not be read. + * @param {Error} error Any error + * @returns {boolean} + */ +function isStoreUnavailable(error) { + return error?.code === STORE_UNAVAILABLE; +} + +/** + * The local apps database, or null when mongo is not connected. + * @returns {object|null} + */ +function db() { + const connection = dbHelper.databaseConnection(); + return connection ? connection.db(config.database.local.database) : null; +} + +/** + * Read one unsigned LEB128 varint and advance the cursor. Arithmetic rather + * than shifts: a u32 does not survive JavaScript's 32-bit signed shift. + * @param {Buffer} buf Decompressed artifact + * @param {{offset: number}} cursor Read position, advanced in place + * @returns {number} + */ +function readVarint(buf, cursor) { + let value = 0; + let scale = 1; + for (let i = 0; i < MAX_VARINT_BYTES; i += 1) { + if (cursor.offset >= buf.length) throw malformed('row stream ends mid-value'); + const byte = buf[cursor.offset]; + cursor.offset += 1; + value += (byte % 128) * scale; + if (byte < 128) { + if (value > IPV4_MAX) throw malformed('varint wider than 32 bits'); + return value; + } + scale *= 128; + } + throw malformed('varint wider than 32 bits'); +} + +/** + * Check one header vocabulary: a list of non-empty bounded strings. + * @param {*} list Candidate section + * @param {string} name Section name, for the error + * @returns {string[]} + */ +function assertTokenList(list, name) { + if (!Array.isArray(list)) throw malformed(`header section ${name} is missing`); + list.forEach((token, i) => { + if (typeof token !== 'string' || !token || token.length > MAX_TOKEN_LENGTH) { + throw malformed(`header ${name}[${i}] is not a token`); + } + }); + return list; +} + +/** + * Read and check the fixed prefix, the header JSON and the row count. + * @param {Buffer} buf Decompressed artifact + * @returns {{header: object, continents: Map, rowCount: number, rowsOffset: number}} + */ +function parseHeader(buf) { + if (buf.length < HEADER_OFFSET) throw malformed('shorter than the fixed header'); + if (buf.toString('latin1', 0, MAGIC.length) !== MAGIC) throw malformed('bad magic'); + if (buf[6] !== SUPPORTED_VERSION) throw malformed(`unsupported format version ${buf[6]}`); + const headerLength = buf.readUInt32LE(7); + const rowCountOffset = HEADER_OFFSET + headerLength; + // the u32 row count sits immediately after the header + if (rowCountOffset + 4 > buf.length) throw malformed('truncated header'); + let header; + try { + header = JSON.parse(buf.toString('utf8', HEADER_OFFSET, rowCountOffset)); + } catch (error) { + throw malformed(`header is not valid JSON: ${error.message}`); + } + if (!header || typeof header !== 'object' || Array.isArray(header)) { + throw malformed('header is not an object'); + } + if (typeof header.generated !== 'string' || !header.generated + || header.generated.length > MAX_TOKEN_LENGTH) { + throw malformed('header section generated is missing'); + } + if (!header.sources || typeof header.sources !== 'object' || Array.isArray(header.sources)) { + throw malformed('header section sources is missing'); + } + if (!header.continents || typeof header.continents !== 'object' || Array.isArray(header.continents)) { + throw malformed('header section continents is missing'); + } + assertTokenList(header.countries, 'countries'); + assertTokenList(header.orgs, 'orgs'); + assertTokenList(header.regions, 'regions'); + const continentEntries = Object.entries(header.continents); + continentEntries.forEach(([country, continent]) => { + if (typeof continent !== 'string' || !continent || continent.length > MAX_TOKEN_LENGTH) { + throw malformed(`header continents.${country} is not a token`); + } + }); + // The region-name vocabulary, keyed '|'. Optional because its + // absence degrades safely: a spec naming a region the way ip-api does is then + // answered at country granularity, the same place a name this vocabulary + // cannot resolve lands. Requiring it would trade that conservative fallback + // for rejecting the whole table - losing country, region and organisation with + // it - which is the worse failure, not the stricter one. + const regionNameEntries = Object.entries(header.regionNames ?? {}); + if (typeof (header.regionNames ?? {}) !== 'object' || Array.isArray(header.regionNames)) { + throw malformed('header section regionNames is not an object'); + } + regionNameEntries.forEach(([key, code]) => { + if (typeof code !== 'string' || !code || code.length > MAX_TOKEN_LENGTH + || key.length > MAX_TOKEN_LENGTH * 2) { + throw malformed(`header regionNames.${key} is not a token`); + } + }); + // Which organisations run access networks and which sell hosting. Optional for + // the same reason regionNames is: an organisation absent from the map has no + // verdict, and a consumer that acts against nodes must do nothing without one, + // so its absence costs enforcement rather than misdirecting it. + const orgClassEntries = Object.entries(header.orgClasses ?? {}); + if (typeof (header.orgClasses ?? {}) !== 'object' || Array.isArray(header.orgClasses)) { + throw malformed('header section orgClasses is not an object'); + } + const orgClasses = new Map(); + orgClassEntries.forEach(([token, code]) => { + // SKIPPED, not rejected. This vocabulary is a closed two-value enum here and + // a separate closed enum in the publisher's repo, and the artifact carries + // codes rather than names, so nothing binds them: adding a third class there + // and merging IS publishing - config.policy.baseUrl reads the branch head, + // with no FluxOS release in the loop. Throwing would take the next fetch on + // every node down with it, so a node holding no baseline yet would get none + // and every other node would silently stop updating - country, continent, + // region and organisation for two million rows, over one value in an + // optional section. + // + // That is the trade this file already refuses forty lines above, for + // regionNames: "the worse failure, not the stricter one". An organisation + // this build has no verdict for is a state lookup already returns + // (networkClass: null) and which enforces nothing, so an unreadable code + // costs exactly the enforcement it should and nothing else. + // + // Compared against the values rather than looked up, so a header saying + // "toString" cannot pass on an inherited function and store it as a class - + // the same hazard the Maps below are built to avoid, three lines away. + const known = classForCode(code); + if (!known) { + log.warn(`ipLocationStore - orgClasses.${token} carries class code ${JSON.stringify(code)},` + + ' which this build does not know; that organisation is left unclassified'); + return; + } + orgClasses.set(token, known); + }); + + return { + header, + // Maps, so a country named after an Object.prototype member reads as itself + continents: new Map(continentEntries), + regionNames: new Map(regionNameEntries), + orgClasses, + rowCount: buf.readUInt32LE(rowCountOffset), + rowsOffset: rowCountOffset + 4, + }; +} + +/** + * Walk the row stream, checking every row, and hand the caller completed + * batches of documents. With no callback nothing is materialised - that is the + * validation pass, which must complete before the first write so a malformed + * artifact never touches the database. + * + * The walk gives the thread back every ROWS_PER_YIELD rows. Two million rows of + * pure decoding is a fifth of a second on a developer machine and longer on a + * node, and for that whole time the process serves no request, reads no socket + * and fires no timer. The batch sink is not a substitute: it awaits real I/O + * only once MAX_BATCHES_IN_FLIGHT writes are outstanding, and awaiting an + * already-settled promise drains microtasks without letting the event loop run. + * @param {Buffer} buf Decompressed artifact + * @param {object} parsed parseHeader() result + * @param {(docs: Array) => Promise} [onBatch] Batch sink + */ +async function walkRows(buf, parsed, onBatch) { + const { header, continents, rowCount, rowsOffset } = parsed; + const cursor = { offset: rowsOffset }; + let previousEnd = -1; + let batch = onBatch ? [] : null; + for (let i = 0; i < rowCount; i += 1) { + const gap = readVarint(buf, cursor); + const len = readVarint(buf, cursor); + const org = readVarint(buf, cursor); + const cc = readVarint(buf, cursor); + const region = readVarint(buf, cursor); + const start = previousEnd + 1 + gap; + const end = start + len; + // Rows are sorted and non-overlapping by construction - a gap is unsigned, + // so a row that starts at or before the previous end is unrepresentable and + // shows up here as a range walking off the end of the address space. + if (end > IPV4_MAX) throw malformed(`row ${i} runs past the IPv4 address space`); + if (org > header.orgs.length) throw malformed(`org index out of range at row ${i}`); + if (cc > header.countries.length) throw malformed(`country index out of range at row ${i}`); + if (region > header.regions.length) throw malformed(`region index out of range at row ${i}`); + previousEnd = end; + if (batch) { + const countryCode = cc === 0 ? null : header.countries[cc - 1]; + batch.push({ + _id: start, + e: end, + o: org === 0 ? null : header.orgs[org - 1], + c: countryCode, + // denormalised at ingest so eligibility never joins + n: countryCode === null ? null : (continents.get(countryCode) ?? null), + r: region === 0 ? null : header.regions[region - 1], + }); + if (batch.length === INSERT_BATCH_SIZE) { + // eslint-disable-next-line no-await-in-loop + await onBatch(batch); + batch = []; + } + } + if ((i + 1) % ROWS_PER_YIELD === 0) { + // eslint-disable-next-line no-await-in-loop + await yieldToEventLoop(); + } + } + if (cursor.offset !== buf.length) throw malformed('row count disagrees with the byte stream'); + if (batch && batch.length) await onBatch(batch); +} + +/** + * Whether a mongo error only means the collection was not there. + * @param {Error} error Driver error + * @returns {boolean} + */ +function isNamespaceMissing(error) { + return error?.codeName === 'NamespaceNotFound' || error?.code === 26 + || /ns not found/i.test(error?.message ?? ''); +} + +/** + * Drop the staging collection a previous attempt may have left behind. + * @param {object} database Local apps database + */ +async function dropStagingCollection(database) { + try { + await dbHelper.dropCollection(database, ipRangesNextCollection); + } catch (error) { + if (!isNamespaceMissing(error)) throw error; + } +} + +/** + * Fill the staging collection, keeping a bounded number of batches in flight. + * Any failed batch fails the ingest: the live collection is not involved, so + * the previous table stays whole. + * @param {Buffer} buf Decompressed artifact + * @param {object} parsed parseHeader() result + * @param {object} database Local apps database + */ +async function fillStagingCollection(buf, parsed, database) { + const inFlight = new Set(); + let failure = null; + const track = (promise) => { + // the failure is captured here rather than at the await site, so a batch + // that fails while others are still running never surfaces as an unhandled + // rejection - the waiters below only ever see settled promises + const tracked = promise + .catch((error) => { failure = failure ?? error; }) + .finally(() => inFlight.delete(tracked)); + inFlight.add(tracked); + }; + + try { + await walkRows(buf, parsed, async (docs) => { + if (failure) throw failure; + if (inFlight.size >= MAX_BATCHES_IN_FLIGHT) await Promise.race(inFlight); + if (failure) throw failure; + // dbHelper.insertManyToDatabase reports a duplicate key as success; the + // count check turns it back into the ingest failure it is. + track(dbHelper.insertManyToDatabase(database, ipRangesNextCollection, docs, { ordered: false }) + .then((result) => { + if (result?.insertedCount !== docs.length) { + throw new Error(`batch inserted ${result?.insertedCount ?? 0} of ${docs.length} rows`); + } + })); + }); + } finally { + // no write outlives this call, so a rejected ingest is finished writing by + // the time the caller sees it + await Promise.all(inFlight); + } + if (failure) throw failure; +} + +/** + * Record which baseline the live collection holds. Written after the swap, so + * a marker never names rows that are not there; a failure to write it costs one + * re-ingest on the next boot and nothing else, which is why it does not fail + * the ingest. + * @param {object} database Local apps database + * @param {object} parsed parseHeader() result + */ +async function writeIngestMarker(database, parsed) { + await dbHelper.findOneAndUpdateInDatabase( + database, + policyDocumentsCollection, + { _id: INGEST_MARKER_ID }, + { + $set: { + generated: parsed.header.generated, + rowCount: parsed.rowCount, + // the header's vocabularies, so a boot that adopts the stored table can + // answer continentForCountry and resolve a region name without holding + // the artifact + continents: Object.fromEntries(parsed.continents), + regionNames: Object.fromEntries(parsed.regionNames), + orgClasses: Object.fromEntries(parsed.orgClasses), + ingestedAt: Date.now(), + }, + }, + { upsert: true }, + ).catch((error) => log.warn(`ipLocationStore - could not record the ingest marker: ${error.message}`)); +} + +/** + * Install a new baseline: validate the artifact end to end, build a fresh + * collection, then swap it in with one rename. Throws - and leaves both the + * live collection and the reported status exactly as they were - on a + * malformed artifact or on any database failure. + * @param {Buffer} bytes The gzipped artifact + * @returns {Promise<{generated: string, rowCount: number}>} + */ +async function setArtifact(bytes) { + if (!Buffer.isBuffer(bytes)) throw malformed('artifact bytes are not a buffer'); + let buf; + try { + buf = await gunzip(bytes); + } catch (error) { + throw malformed(`not a gzip stream: ${error.message}`); + } + const parsed = parseHeader(buf); + if (parsed.rowCount < minimumRowCount) { + throw malformed(`row count ${parsed.rowCount} is below the truncation floor ${minimumRowCount}`); + } + await walkRows(buf, parsed); + + const database = db(); + if (!database) throw unavailable('no database connection'); + await dropStagingCollection(database); + await fillStagingCollection(buf, parsed, database); + await database.renameCollection(ipRangesNextCollection, ipRangesCollection, { dropTarget: true }); + + status = { ready: true, generated: parsed.header.generated, rowCount: parsed.rowCount }; + continentByCountry = parsed.continents; + regionCodeByName = parsed.regionNames; + networkClassByOrg = parsed.orgClasses; + await writeIngestMarker(database, parsed); + log.info(`ipLocationStore - baseline installed: ${parsed.rowCount} ranges, generated ${parsed.header.generated}`); + return { generated: status.generated, rowCount: status.rowCount }; +} + +/** + * Adopt the baseline this node already holds. The rows survive a restart in + * mongo, so a boot that finds the marker is already serving the baseline it + * names - re-ingesting the same artifact would cost two million writes to + * arrive back exactly here. The marker is written only after the swap, and the + * swap is atomic, so nothing further needs verifying. + * @returns {Promise} true when a stored ingest was adopted + */ +async function adoptPersistedStatus() { + const database = db(); + if (!database) return false; + let marker; + try { + marker = await dbHelper.findOneInDatabase(database, policyDocumentsCollection, { _id: INGEST_MARKER_ID }); + } catch (error) { + log.warn(`ipLocationStore - could not read the ingest marker: ${error.message}`); + return false; + } + if (!marker || typeof marker.generated !== 'string' || !marker.generated) return false; + status = { ready: true, generated: marker.generated, rowCount: marker.rowCount ?? 0 }; + continentByCountry = new Map(Object.entries(marker.continents ?? {})); + regionCodeByName = new Map(Object.entries(marker.regionNames ?? {})); + networkClassByOrg = new Map(Object.entries(marker.orgClasses ?? {})); + log.info(`ipLocationStore - adopted the stored baseline: ${status.rowCount} ranges, generated ${status.generated}`); + return true; +} + +/** + * Continent code for an ISO 3166-1 country code, from the header of the + * baseline this node holds. + * @param {string} countryCode ISO 3166-1 alpha-2 code + * @returns {string | null} null without a table, or when the country is unknown + */ +function continentForCountry(countryCode) { + if (!status.ready) return null; + return continentByCountry.get(countryCode) ?? null; +} + +/** + * The ISO 3166-2 code for a region an app named the way ip-api does. + * + * A geolocation entry may carry either vocabulary - 'acEU_DE_DE-BY' or + * 'acEU_DE_Bavaria' - while the rows carry codes alone. Null when this node + * holds no vocabulary, or the name is not in it: the caller then answers that + * entry at country granularity, which counts more nodes rather than fewer. + * @param {string} countryCode ISO 3166-1 alpha-2 code + * @param {string} regionName The entry's region part + * @returns {string | null} + */ +function regionCodeForName(countryCode, regionName) { + if (!status.ready || !countryCode || !regionName) return null; + return regionCodeByName.get(`${countryCode}|${regionName}`) ?? null; +} + +/** + * The fault-domain key a stored location gives its address: the organisation + * holding the range. Null when the table names none, which leaves the address on + * the /16 rung its caller computes. Derived once, when the entry is built. + * + * Two rungs, not three. A row's own extent was once keyed between these as the + * range's "registry allocation block", but the artifact carries no allocation - + * a row is bounded by whichever of owner, country or region changes first, so + * its extent is a fragment of an allocation rather than one. It would also be + * the wrong direction: the median ownerless range is around 512 addresses, so + * keying on it splits one /16 into up to 128 domains and calls nodes diverse + * that the /16 rung holds together. + * @param {{o: string|null}} doc A stored location + * @returns {string | null} + */ +function domainKeyFor(doc) { + if (doc.o) return `org:${doc.o}`; + return null; +} + +/** + * One entry of the per-node view, in the shape its readers use: the fault + * domain already keyed, and the three location fields eligibility reads. + * @param {object} doc A stored nodelocations document + * @returns {{d: string|null, c: string|null, n: string|null, r: string|null, g: string|null}} + */ +function viewEntry(doc) { + return { + d: domainKeyFor(doc), + c: doc.c ?? null, + n: doc.n ?? null, + r: doc.r ?? null, + g: doc.g ?? null, + }; +} + +/** + * Load the stored view into the process. One read, at boot: the rows survive a + * restart in mongo, and re-deriving them would cost a lookup per node against + * the range table. + * @returns {Promise} + */ +async function loadNodeLocationView() { + const database = db(); + if (!database) throw unavailable('no database connection'); + let docs; + try { + docs = await dbHelper.findInDatabase(database, nodeLocationsCollection, {}); + } catch (error) { + throw unavailable(error.message); + } + const loaded = new Map(); + (docs ?? []).forEach((doc) => loaded.set(doc._id, viewEntry(doc))); + nodeView = loaded; + nodeViewLoaded = true; +} + +/** + * The per-node location view this process holds. + * + * The map is replaced whole on every refresh and never mutated in place, so a + * caller that takes it once answers every question from one consistent picture + * of the network - a spawn decision and the share it is measured against can + * never come from two different views. + * + * `ready` is what a caller must gate a location answer on: it means this + * process holds both a baseline and the view derived from it. Without it the + * map is empty, every address falls to /16 arithmetic, and that is exactly the + * posture a node with no table at all is in. + * @returns {{byIp: Map, ready: boolean, generated: string|null}} + */ +function nodeLocationSnapshot() { + return { + byIp: nodeView, + ready: status.ready && nodeViewLoaded, + generated: status.generated ?? null, + }; +} + +/** + * Locate an IP in the stored table. + * + * Resolves null when the address does not parse, is not IPv4 (the table is + * IPv4 only - no Flux node holds a v6 address), or no row covers it. Rejects + * with a store-unavailable error - isStoreUnavailable(error) - when mongo + * could not answer: callers must treat that exactly like "no table" and fall + * back to /16 arithmetic, never like "no covering row". + * @param {string} ip Bare IP address (no port) + * @returns {Promise<{org: string|null, countryCode: string|null, + * continentCode: string|null, region: string|null} | null>} + */ +async function lookup(ip) { + const parsed = cidrUtils.parseIp(ip); + if (!parsed || parsed.version !== 4) return null; + const needle = Number(parsed.value); + const database = db(); + if (!database) throw unavailable('no database connection'); + let rows; + try { + rows = await dbHelper.findInDatabase( + database, + ipRangesCollection, + { _id: { $lte: needle } }, + { sort: { _id: -1 }, limit: 1 }, + ); + } catch (error) { + throw unavailable(error.message); + } + const row = rows?.[0]; + if (!row || !Number.isInteger(row.e) || row.e < needle) return null; + return { + org: row.o ?? null, + countryCode: row.c ?? null, + continentCode: row.n ?? null, + region: row.r ?? null, + // null, never a third class value: an organisation with no published verdict + // is one nothing may act on, and that must not be confusable with a verdict. + networkClass: (row.o && networkClassByOrg.get(row.o)) || null, + }; +} + +/** + * Bring the per-node view in line with the node list. + * + * The list is what the view is FOR, so it is also what the view is bounded by: + * an address the list no longer carries is dropped, and an entry derived from + * an older baseline is re-derived. That leaves the view holding exactly the + * listed addresses, which is what lets a reader take absence at face value. + * + * A lookup that fails leaves that address out, which reads downstream as an + * unresolved location - the /16 rung - so one failure never fails the pass. The + * resident view is swapped in at the end, whole: a reader holding the previous + * one keeps a consistent picture rather than watching this pass rewrite it. + * @param {Array<{ip: string}>} nodeList The deterministic node list + * @returns {Promise<{refreshed: number, dropped: number}>} + */ +async function refreshNodeLocations(nodeList) { + // without a baseline there is nothing to derive a location from, and the + // entries already held stay as they are + if (!status.ready) return { refreshed: 0, dropped: 0 }; + // An empty list is not a fleet with no nodes, it is a list this node failed to + // obtain. Taken at face value it makes every held address "departed" below and + // deletes the entire view in one call. The caller's accessor is supposed to make + // that unreachable, and does today - but the cost of it ever not holding is the + // whole collection, and that is too much to rest on a contract kept in another + // file. + if (!nodeList?.length) { + // Said out loud, because the caller only logs a pass that changed something - + // so a silent refusal here is indistinguishable from a pass with nothing to do, + // and this one means the node list could not be obtained. + log.warn('ipLocationStore - node location refresh skipped: the node list came back empty, which is a failed fetch rather than a fleet with no nodes'); + return { refreshed: 0, dropped: 0 }; + } + const database = db(); + if (!database) throw unavailable('no database connection'); + if (!nodeViewLoaded) await loadNodeLocationView(); + + const listed = new Set(); + const missing = []; + (nodeList ?? []).forEach((node) => { + const ip = bareIp(node?.ip); + if (!ip || listed.has(ip)) return; + listed.add(ip); + const held = nodeView.get(ip); + // an entry from an older baseline says where the address used to resolve + if (!held || held.g !== status.generated) missing.push(ip); + }); + + const departed = [...nodeView.keys()].filter((ip) => !listed.has(ip)); + + let dropped = 0; + if (departed.length) { + try { + const removal = await dbHelper.removeDocumentsFromCollection( + database, + nodeLocationsCollection, + { _id: { $in: departed } }, + ); + dropped = removal?.deletedCount ?? 0; + } catch (error) { + throw unavailable(error.message); + } + } + + const next = new Map(); + nodeView.forEach((entry, ip) => { + if (listed.has(ip) && entry.g === status.generated) next.set(ip, entry); + }); + + let refreshed = 0; + let failure = null; + let cursor = 0; + const fill = async () => { + while (cursor < missing.length) { + const ip = missing[cursor]; + cursor += 1; + const doc = { + o: null, c: null, n: null, r: null, g: status.generated, + }; + try { + // eslint-disable-next-line no-await-in-loop + const hit = await lookup(ip); + doc.o = hit?.org ?? null; + doc.c = hit?.countryCode ?? null; + doc.n = hit?.continentCode ?? null; + doc.r = hit?.region ?? null; + // eslint-disable-next-line no-await-in-loop + await dbHelper.updateOneInDatabase( + database, + nodeLocationsCollection, + { _id: ip }, + { $set: doc }, + { upsert: true }, + ); + next.set(ip, viewEntry(doc)); + refreshed += 1; + } catch (error) { + failure = failure ?? error; + } + } + }; + await Promise.all(Array.from( + { length: Math.min(NODE_LOOKUP_CONCURRENCY, missing.length) }, + () => fill(), + )); + + nodeView = next; + nodeViewLoaded = true; + if (failure) log.warn(`ipLocationStore - some node locations could not be refreshed: ${failure.message}`); + return { refreshed, dropped }; +} + +/** + * What this process holds, from memory - never a database call. + * @returns {{ready: boolean, generated: string|null, rowCount: number}} + */ +function currentStatus() { + return { ...status }; +} + +/** + * Forget the installed baseline and restore the production row floor. Test + * support; the stored collection is untouched. + */ +function clear() { + status = { ready: false, generated: null, rowCount: 0 }; + continentByCountry = new Map(); + regionCodeByName = new Map(); + networkClassByOrg = new Map(); + minimumRowCount = MIN_ROW_COUNT; + nodeView = new Map(); + nodeViewLoaded = false; +} + +/** + * Lower the truncation floor so a fixture need not carry a real baseline's + * worth of rows. Test support - clear() restores MIN_ROW_COUNT. + * @param {number} rows Minimum accepted row count + */ +function setMinimumRowCount(rows) { + if (!Number.isInteger(rows) || rows < 1) throw new Error('ipLocationStore: row floor must be a positive integer'); + minimumRowCount = rows; +} + +module.exports = { + setArtifact, + adoptPersistedStatus, + continentForCountry, + regionCodeForName, + loadNodeLocationView, + nodeLocationSnapshot, + refreshNodeLocations, + lookup, + status: currentStatus, + clear, + setMinimumRowCount, + isStoreUnavailable, + MIN_ROW_COUNT, + STORE_UNAVAILABLE, +}; diff --git a/ZelBack/src/services/appPlacement/ipLocationSync.js b/ZelBack/src/services/appPlacement/ipLocationSync.js new file mode 100644 index 0000000000..8cb741ba66 --- /dev/null +++ b/ZelBack/src/services/appPlacement/ipLocationSync.js @@ -0,0 +1,231 @@ +// Interim fetch-and-restore for the iplocation artifact. +// +// This branch ships before the policy store (feat/userconfig-rearchitecture), +// which already registers this same artifact and takes over when it rebases +// onto this branch. To make that handover seamless, this module mirrors the +// store's artifact contract exactly: same registry key, same GridFS bucket and +// record shape (policyArtifactRepository, shared verbatim), same conditional +// requests, and the same rejection rule - bytes the reader throws on are never +// cached and never displace a good stored copy. The policy store will restore +// the cache this module populated; no node refetches across the transition. +// +// AT REBASE: delete this module and its serviceManager start call, and wire +// policyStore.onArtifact('ipLocationTable', (bytes) => ipLocationStore.setArtifact(bytes)); +// beside policyStore.startSync() instead. The rows live in mongo and the ingest +// marker names the baseline they came from, so their boot restore only +// re-ingests when the artifact's generated timestamp differs from the marker's. + +const config = require('config'); +const log = require('../../lib/log'); +const serviceHelper = require('../serviceHelper'); +const fluxCommunicationUtils = require('../fluxCommunicationUtils'); +const policyArtifactRepository = require('../appDatabase/policyArtifactRepository'); +const ipLocationStore = require('./ipLocationStore'); + +const ARTIFACT_NAME = 'ipLocationTable'; // registry key, shared with policyStore +const ARTIFACT_FILE = 'iplocation.bin.gz'; +const REFRESH_INTERVAL_MS = 24 * 60 * 60 * 1000; +const RETRY_INTERVAL_MS = 10 * 60 * 1000; // only while the node holds no table at all +const MAX_RETRY_ATTEMPTS = 5; // 10m, 20m, 40m, 80m, 160m - then the daily refresh +const FETCH_TIMEOUT_MS = 120 * 1000; // 4.2 MB over slow uplinks; never gates boot + +let etag = null; +let refreshInterval = null; +let retryTimer = null; +let retryAttempt = 0; +let started = false; +let restored = false; +let nodeLocationPass = null; + +/** + * Bring the per-node location view in line with the current node list. Several + * paths want this after they change something, and a single pass at a time is + * enough for all of them - a second concurrent pass would look up exactly the + * addresses the first is already writing. + * @returns {Promise} + */ +function refreshNodeLocations() { + if (nodeLocationPass) return nodeLocationPass; + nodeLocationPass = fluxCommunicationUtils.deterministicFluxList() + .then((nodeList) => ipLocationStore.refreshNodeLocations(nodeList)) + .then(({ refreshed, dropped }) => { + if (refreshed || dropped) log.info(`ipLocationSync - node locations refreshed: ${refreshed} written, ${dropped} dropped`); + }) + .catch((error) => log.warn(`ipLocationSync - node location refresh failed: ${error.message}`)) + .finally(() => { nodeLocationPass = null; }); + return nodeLocationPass; +} + +/** + * Fetch the artifact if it changed, install it, and cache it. A malformed + * response is rejected by the reader's parse and never written to the cache; + * an unchanged artifact costs a 304 and no body. + * @returns {Promise} true when a new table was installed. + */ +async function refresh() { + const url = `${config.policy.baseUrl}/${ARTIFACT_FILE}`; + try { + const options = { + timeout: FETCH_TIMEOUT_MS, + responseType: 'arraybuffer', + // axios rejects everything outside 2xx; 304 is the expected answer for + // an unchanged artifact and must come back as a response + validateStatus: (status) => (status >= 200 && status < 300) || status === 304, + }; + if (etag) options.headers = { 'If-None-Match': etag }; + const res = await serviceHelper.axiosGet(url, options); + if (res.status === 304) return false; + const bytes = Buffer.from(res.data); + const served = (res.headers && (res.headers.etag ?? res.headers.ETag)) ?? null; + try { + // before the cache write, so a malformed artifact never displaces a good stored copy + await ipLocationStore.setArtifact(bytes); + } catch (error) { + // Remember the etag of bytes this build cannot read, so the next attempt + // is a 304 rather than another full download of the same broken + // artifact. A corrected publication carries a new etag and is fetched. + etag = served; + throw error; + } + etag = served; + await policyArtifactRepository.writeArtifactBytes(ARTIFACT_NAME, bytes, etag) + .catch((error) => log.warn(`ipLocationSync - failed to cache artifact: ${error.message}`)); + log.info('ipLocationSync - iplocation table refreshed'); + // a new baseline invalidates every node location document + refreshNodeLocations(); + return true; + } catch (error) { + log.warn(`ipLocationSync - failed to refresh from ${url}, keeping current table: ${error.message}`); + return false; + } +} + +/** + * Run a refresh, and while the node holds NO table at all, retry on a short + * interval instead of waiting out the daily one. A node whose first fetch + * lands in a boot-time network gap would otherwise spend a full day computing + * /16 fault domains while the rest of the fleet uses organisations. + */ +function scheduleRefresh() { + refresh() + .then((installed) => { + // The node list drifts while the table does not, so a 304 still leaves + // nodes that joined since the last pass without a location document. + // An install has already asked for the pass this would repeat. + if (!installed) refreshNodeLocations(); + if (installed || ipLocationStore.status().ready) { + retryAttempt = 0; + return; + } + if (retryTimer || retryAttempt >= MAX_RETRY_ATTEMPTS) return; + // Exponential backoff with a cap on attempts: a boot-time network gap + // clears in minutes, while a published artifact this build cannot read + // never clears, and retrying it forever would have every node in the + // fleet re-downloading the same broken file on a fixed interval. After + // the attempts are spent the daily refresh is the only retry. + const delay = RETRY_INTERVAL_MS * 2 ** retryAttempt; + retryAttempt += 1; + retryTimer = setTimeout(() => { + retryTimer = null; + scheduleRefresh(); + }, delay); + if (retryTimer.unref) retryTimer.unref(); + }) + .catch((error) => log.error(`ipLocationSync - refresh error: ${error.message}`)); +} + +/** + * Bring back the table this node already holds: adopt the stored baseline, or + * restore the last-good artifact from GridFS. + * + * Separate from startSync, and started separately, because the two halves need + * different things and cost different amounts. This half needs MONGO ONLY - on a + * node that has run before it is a single marker read, and the two million rows + * are already in the collection - so it belongs as early as the database is up. + * Every consumer of the table then has it within milliseconds of boot instead of + * waiting on work it does not depend on. Idempotent. + * @returns {Promise} + */ +async function restoreCachedTable() { + if (restored) return; + restored = true; + // Best-effort: a database briefly unavailable at this moment must not cost + // this process its table for the rest of its life, so a failure here still + // leaves the fetch and the refresh loop armed. + try { + const adopted = await ipLocationStore.adoptPersistedStatus(); + await policyArtifactRepository.sweepOrphanedArtifacts(ARTIFACT_NAME); + const record = await policyArtifactRepository.getArtifactRecord(ARTIFACT_NAME); + if (adopted) { + // The rows are already in mongo under the marker's baseline; re-ingesting + // the same two million of them to learn what the marker already says buys + // nothing. The etag still comes from the record so the daily refresh is a + // conditional request rather than a full download. + etag = record?.etag ?? null; + refreshNodeLocations(); + } else { + const bytes = record ? await policyArtifactRepository.readArtifactBytes(record.fileId) : null; + if (bytes) { + try { + await ipLocationStore.setArtifact(bytes); + ({ etag } = record); + log.info('ipLocationSync - iplocation table restored from cache'); + refreshNodeLocations(); + } catch (error) { + // A stored copy this build cannot read must not leave the next + // refresh answering 304 for bytes we are not actually holding - drop + // the etag so the refetch is unconditional. This is also the upgrade + // path: a node that cached the previous JSON artifact holds bytes + // whose magic this reader rejects, and the unconditional refetch + // below is what brings it the binary one. + etag = null; + log.error(`ipLocationSync - stored iplocation table rejected, will refetch: ${error.message}`); + } + } + } + } catch (error) { + log.warn(`ipLocationSync - could not restore the cached table, fetching instead: ${error.message}`); + } +} + +/** + * Fetch the artifact if it has changed, and keep it fresh daily. + * + * The expensive half: a 4.2 MB download and, when the published baseline has + * moved, an ingest of two million rows. It is deliberately NOT started with the + * restore above - a node with no cache would otherwise run that ingest + * concurrently with the app-database rebuild, which is the busiest the database + * ever is. Placement needs no table to run - it degrades to status-quo /16 + * arithmetic - so nothing here gates boot either way. Restores first if that has + * not happened. Idempotent. + * @returns {Promise} + */ +async function startSync() { + if (started) return; + started = true; + await restoreCachedTable(); + scheduleRefresh(); + refreshInterval = setInterval(scheduleRefresh, REFRESH_INTERVAL_MS); + if (refreshInterval.unref) refreshInterval.unref(); +} + +/** + * Stop the refresh loop. Test support and shutdown. + */ +function stopSync() { + if (refreshInterval) clearInterval(refreshInterval); + if (retryTimer) clearTimeout(retryTimer); + refreshInterval = null; + retryTimer = null; + retryAttempt = 0; + started = false; + restored = false; + etag = null; +} + +module.exports = { + restoreCachedTable, + startSync, + stopSync, + refresh, +}; diff --git a/ZelBack/src/services/appPlacement/placementFeasibility.js b/ZelBack/src/services/appPlacement/placementFeasibility.js new file mode 100644 index 0000000000..810569010e --- /dev/null +++ b/ZelBack/src/services/appPlacement/placementFeasibility.js @@ -0,0 +1,817 @@ +// Placement feasibility for synced apps. +// +// A placement constraint may only reject a node when a better-placed candidate +// provably exists. This module supplies the proof: it computes, from the +// deterministic node list and the IP location table, how many distinct fault +// domains an app's eligible candidates span, and from that each domain's share +// of the app's instances - the smallest uniform level the domains can absorb. +// The spawner, the registration validator and the placement API all consume +// this one computation. +// +// Every approximation in here errs toward counting MORE candidates and MORE +// domains, which pushes the share toward 1 - i.e. toward the strict behaviour +// the network has today - never toward stacking instances. A missing table, an +// unresolvable location, a region-granularity pin the table cannot answer: all +// degrade to the status quo. +// +// The same principle at the entry level: an ALLOWED restriction the table +// cannot fully resolve over-includes (a region-level pin admits the whole +// country), while a FORBIDDEN restriction it cannot resolve is not applied at +// all. A constraint never strands an app on missing data, and a ban never +// applies to a node it cannot be proven to cover. + +const config = require('config'); +const log = require('../../lib/log'); +const fluxCommunicationUtils = require('../fluxCommunicationUtils'); +const networkStateService = require('../networkStateService'); +const generalService = require('../generalService'); +const messageHelper = require('../messageHelper'); +const serviceHelper = require('../serviceHelper'); +const cidrUtils = require('../utils/cidrUtils'); +const mountParser = require('../utils/mountParser'); +const verificationHelper = require('../verificationHelper'); +const { bareIp, socketAddressesMatch } = require('../utils/socketAddressUtils'); +const geolocationRule = require('./geolocationRule'); +const ipLocationStore = require('./ipLocationStore'); +const { Privilege, authOf } = require('../utils/privileges'); + + +// geonames/ip-api continent convention - the same vocabulary the location +// table's country -> continent map uses +const CONTINENT_CODES = new Set(['AF', 'AN', 'AS', 'EU', 'NA', 'OC', 'SA']); + +/** + * The bottom rung of the fault-domain ladder: /16 (v4) or /32 (v6) arithmetic, + * which is what the network used before the location table existed. + * @param {string} ip Bare IP address + * @returns {string | null} null when the address does not parse + */ +function netDomain(ip) { + const parsed = cidrUtils.parseIp(ip); + if (!parsed) return null; + return `net:${cidrUtils.prefixKey(ip, parsed.version === 4 ? 16 : 32)}`; +} + +/** + * The fault-domain function over one node location snapshot: the domain the + * view already keyed for that address - organisation, else registry allocation + * block - else /16 arithmetic. An address the snapshot does not carry falls to + * /16 as well, and over-approximating the domain count errs strict. + * @param {Map} byIp The resident node location view + * @returns {(address: string) => string | null} + */ +function domainFunction(byIp) { + return (address) => { + const ip = bareIp(address); + if (!ip) return null; + return byIp.get(ip)?.d ?? netDomain(ip); + }; +} + +/** + * The node location view plus what it says about the table behind it. A process + * that does not yet hold the view answers in the same direction as no table at + * all - an empty view, every node on /16 arithmetic - because the alternative + * reads as zero candidates, which is a proof this node does not have. + * @returns {{byIp: Map, tableAvailable: boolean, + * tableGenerated: string|null}} + */ +function nodeLocationView() { + const snapshot = ipLocationStore.nodeLocationSnapshot(); + return { + byIp: snapshot.byIp, + tableAvailable: snapshot.ready, + tableGenerated: snapshot.ready ? snapshot.generated : null, + }; +} + +/** + * The fault-domain key for a single address, straight from the stored table: + * organisation, else registry allocation block, else /16 (v4) / /32 (v6) + * arithmetic. A store that cannot be read falls to /16, exactly like no table. + * Prefer placementComputation's domainOf when several addresses are keyed at + * once - it answers from one snapshot instead of a lookup each. + * @param {string} address ip or ip:port + * @returns {Promise} null when the address does not parse + */ +async function faultDomain(address) { + const ip = bareIp(address); + if (!ip) return null; + let hit = null; + try { + hit = await ipLocationStore.lookup(ip); + } catch (error) { + log.warn(`placementFeasibility - location lookup unavailable for ${ip}, using /16: ${error.message}`); + } + if (hit?.org) return `org:${hit.org}`; + if (hit?.block) return `blk:${hit.block.start}-${hit.block.end}`; + return netDomain(ip); +} + +/** + * Whether a node location satisfies an app's geolocation specification, for + * the callers that answer about a single node. A candidate count parses the + * spec once through geolocationRule and reuses the rule instead - deriving the + * terms per node costs the product of the node count and the entry count. + * @param {{continentCode: string|null, countryCode: string|null, + * region: string|null}} loc Node location + * @param {string[]} geolocation App spec geolocation entries + * @returns {boolean} + */ +function nodeLocationMatchesGeolocation(loc, geolocation) { + return geolocationRule.locationSatisfiesGeolocation( + loc, geolocation, ipLocationStore.regionCodeForName, + ); +} + +/** + * The node-list entries an app may be placed on. A spec carrying a non-empty + * `nodes` list is a closed pool - v7 enforces it at install + * (checkAppNodesRequirements) and only enterprise owners may carry it from v8 + * on - so the candidate set IS that list. Counting the whole network for such + * an app computes a share against fault domains it can never use, which + * strands it below its instance count. + * @param {Array} nodeList The deterministic node list + * @param {string[]} pinned The spec's nodes entries (socket addresses or outpoints) + * @returns {Array} + */ +function pooledNodes(nodeList, pinned) { + if (!pinned.length) return nodeList; + const outpoints = new Set(pinned); + return nodeList.filter((node) => pinned.some((entry) => socketAddressesMatch(entry, node.ip)) + || outpoints.has(`${node.txhash}:${node.outidx}`)); +} + +/** + * The per-domain share: the smallest uniform level L at which the domains can + * absorb all instances, i.e. sum(min(candidatesInDomain, L)) >= instances. + * When every domain holds at least ceil(instances / domains) candidates this + * is exactly ceil(instances / domains); when shallow domains cannot absorb + * their share the level rises only as far as needed, so an app is never + * stranded by domains too small to take what the average assumes. + * @param {number[]} domainSizes Candidate count per fault domain + * @param {number} instances Required instance count + * @returns {number} + */ +function domainShareLevel(domainSizes, instances) { + if (domainSizes.length === 0) return instances; + let level = Math.ceil(instances / domainSizes.length); + const absorbed = (l) => domainSizes.reduce((sum, size) => sum + Math.min(size, l), 0); + while (level < instances && absorbed(level) < instances) level += 1; + return level; +} + +/** + * One placement computation over the current network: the feasibility numbers + * and the fault-domain function they were computed with. The node location view + * is read ONCE here, and every domain the caller keys afterwards comes from that + * same snapshot - so a spawn decision and the share it is measured against can + * never be answering from two different views of the network. + * @param {object} appSpecifications App specifications (geolocation, hw fields) + * @param {number} [minInstances] Required instance count; defaults to the spec's + * @returns {Promise<{feasibility: object, domainOf: (address: string) => string|null}>} + */ +async function placementComputation(appSpecifications, minInstances) { + const instances = minInstances ?? appSpecifications.instances ?? config.fluxapps.minimumInstances; + // Asked before the accessor rather than after: the accessor waits for the + // list, and this is reached from a request handler that cannot wait. + if (!networkStateService.isReady()) { + const error = new Error('Node list is not available yet'); + error.statusCode = 503; + throw error; + } + const nodeList = await fluxCommunicationUtils.deterministicFluxList(); + if (!Array.isArray(nodeList) || nodeList.length === 0) { + // an empty node list is missing data, not an empty network - reporting + // zero candidates would read as proven impossibility to the callers + const error = new Error('Node list is not available yet'); + error.statusCode = 503; + throw error; + } + const { byIp, tableAvailable, tableGenerated } = nodeLocationView(); + const domainOf = domainFunction(byIp); + // Parsed once, for every node below. The spec's entries decide the rule and + // the node decides nothing about it, so re-deriving the terms inside the loop + // made one answer cost the node count times the entry count - and a spec may + // carry two hundred entries against six thousand nodes. + const geoRule = geolocationRule.parseGeolocation( + appSpecifications.geolocation, ipLocationStore.regionCodeForName, + ); + const geoRestricted = !geoRule.unrestricted; + + const domains = new Map(); // fault domain -> candidate count + // Tier is deliberately NOT a filter. A tier is a collateral class, not a + // hardware guarantee: install-time sizes an app against the node's actual + // CPU, RAM and disk, so a node whose hardware exceeds its tier's nominal + // figure accepts apps this arithmetic would have ruled out. Excluding on + // the nominal figure therefore refuses deployable apps, and no bound this + // module can compute is a proof of unfitness. Install time enforces it. + const candidates = pooledNodes(nodeList, appSpecifications.nodes ?? []); + let candidateCount = 0; + // eslint-disable-next-line no-restricted-syntax + for (const node of candidates) { + const ip = bareIp(node.ip); + if (!ip) continue; // eslint-disable-line no-continue + if (geoRestricted && tableAvailable) { + const doc = byIp.get(ip); + // a node the view does not carry has no provable location, and an + // unprovable location counts + const loc = doc ? { continentCode: doc.n ?? null, countryCode: doc.c ?? null, region: doc.r ?? null } : null; + if (!geolocationRule.locationSatisfiesRule(geoRule, loc)) continue; // eslint-disable-line no-continue + } + const domain = domainOf(ip); + if (!domain) continue; // eslint-disable-line no-continue + candidateCount += 1; + domains.set(domain, (domains.get(domain) ?? 0) + 1); + } + + const domainCount = domains.size; + return { + feasibility: { + instances, + candidateCount, + domainCount, + maxPerDomain: domainShareLevel([...domains.values()], instances), + placeable: domainCount > 0, + tableAvailable, + tableGenerated, + }, + domainOf, + }; +} + +/** + * Compute the placement feasibility of an app over the current network. + * @param {object} appSpecifications App specifications (geolocation, hw fields) + * @param {number} [minInstances] Required instance count; defaults to the spec's + * @returns {Promise<{instances: number, candidateCount: number, domainCount: number, + * maxPerDomain: number, placeable: boolean, tableAvailable: boolean, + * tableGenerated: string|null}>} + */ +async function placementFeasibility(appSpecifications, minInstances) { + const { feasibility } = await placementComputation(appSpecifications, minInstances); + return feasibility; +} + +/** + * How many of the given app locations sit in a fault domain. + * @param {Array<{ip: string}>} locations Running or installing app locations + * @param {string} domainKey A fault domain key + * @param {(address: string) => string|null} [domainOf] A placementComputation + * domain function; without it each location costs a lookup of its own + * @returns {Promise} + */ +async function countHeldInDomain(locations, domainKey, domainOf) { + if (!domainKey) return 0; + const held = locations ?? []; + if (domainOf) return held.filter((location) => domainOf(location.ip) === domainKey).length; + const domains = await Promise.all(held.map((location) => faultDomain(location.ip))); + return domains.filter((domain) => domain === domainKey).length; +} + +/** + * Whether the app's spec names this node, by socket address or by collateral + * outpoint. Being named is the owner's own placement choice and bypasses the + * diversity share. + * @param {object} appSpecifications App specifications + * @param {string} localSocketAddr This node's ip:port + * @returns {Promise} + */ +async function specNamesThisNode(appSpecifications, localSocketAddr) { + const nodes = appSpecifications.nodes ?? []; + if (!nodes.length) return false; + if (nodes.some((node) => socketAddressesMatch(node, localSocketAddr))) return true; + try { + const collateral = await generalService.obtainNodeCollateralInformation(); + return nodes.includes(`${collateral.txhash}:${collateral.txindex}`); + } catch (error) { + log.warn(`placementFeasibility - could not resolve node collateral: ${error.message}`); + return false; + } +} + +/** + * The placement category of a computed feasibility - the availability promise + * the network can make for the spec: + * 'impossible' - fewer eligible nodes than instances, even though every + * approximation counts TOWARD eligibility, so the shortfall + * is proven. The spec can never reach its instance count; + * registration rejects it. + * 'constrained' - the instance count is reachable, but a synced app's + * instances outnumber its fault domains, so some instances + * must share a provider. Deliverable, with less resiliency + * than the instance count implies; registration warns. + * 'ok' - the requested count and diversity are both deliverable. + * @param {object} feasibility A placementFeasibility() result + * @param {boolean} syncedApp Whether the spec has synced components + * @returns {'impossible'|'constrained'|'ok'} + */ +function placementCategory(feasibility, syncedApp) { + if (feasibility.candidateCount < feasibility.instances) return 'impossible'; + if (syncedApp && feasibility.domainCount < feasibility.instances) return 'constrained'; + return 'ok'; +} + +/** + * The placement-relevant sizing of a spec: the fields that decide how many + * nodes could hold it. Compared between an update and the spec it replaces. + * @param {object} spec Formatted app specifications + * @returns {string} A comparable digest + */ +function placementShape(spec) { + const components = spec.version <= 3 + ? [{ cpu: spec.cpu, ram: spec.ram, hdd: spec.hdd, tiered: spec.tiered }] + : (spec.compose ?? []).map((c) => ({ + cpu: c.cpu, ram: c.ram, hdd: c.hdd, tiered: c.tiered, + })); + return JSON.stringify({ + instances: spec.instances ?? null, + geolocation: [...(spec.geolocation ?? [])].sort(), + components, + }); +} + +/** + * Whether an update changes anything placement depends on. An update that + * touches none of it - an expire-only renewal, a cancellation (expire: 1), a + * description or environment edit - must never be refused by the placement + * gate: the owner is not making placement worse, and refusing would strand + * them with an app they can neither renew nor cancel. + * @param {object} next The update's formatted specifications + * @param {object} previous The specifications it replaces + * @returns {boolean} + */ +function changesPlacement(next, previous) { + if (!previous) return true; // nothing to compare against - gate it + // An enterprise spec is stored with its compose stripped, and a previous + // spec that could not be decrypted arrives here in that stripped form. It + // is not comparable, and reading the difference as a placement change would + // gate exactly the renewals and cancellations this exists to let through. + const strippedPrevious = previous.version >= 8 + && (previous.compose ?? []).length === 0 + && (next.compose ?? []).length > 0; + if (strippedPrevious) return false; + return placementShape(next) !== placementShape(previous); +} + +/** + * Enforce placement feasibility on the user-facing registration and update + * paths: an impossible spec is rejected before it is paid for, a constrained + * synced spec is accepted with a warning. Called from the API front door + * only - never from p2p message verification, where nodes with different + * table versions must not disagree about message validity. A failure to + * COMPUTE feasibility never rejects: without the computation there is no + * proof, and only proven impossibility may refuse a registration. + * + * On an update path, pass the previous specifications: an update that does + * not change placement is never gated (see changesPlacement). + * @param {object} appSpecFormatted Formatted app specifications + * @param {string} caller Log prefix identifying the calling path + * @param {object} [previousSpec] The specifications an update replaces + * @returns {Promise} The feasibility, or null when it could not + * be computed or the check did not apply + * @throws When the spec provably cannot reach its instance count + */ +async function checkPlacementFeasibility(appSpecFormatted, caller, previousSpec) { + if (previousSpec && !changesPlacement(appSpecFormatted, previousSpec)) return null; + let synced; + let feasibility; + try { + synced = appSpecFormatted.version <= 3 + ? mountParser.isSyncedComponent(appSpecFormatted.containerData) + : (appSpecFormatted.compose ?? []).some((component) => mountParser.isSyncedComponent(component.containerData)); + feasibility = await placementFeasibility(appSpecFormatted); + } catch (error) { + log.warn(`${caller} - placement feasibility check failed: ${error.message}`); + return null; + } + const category = placementCategory(feasibility, synced); + const geoRestricted = (appSpecFormatted.geolocation ?? []).length > 0; + const pinned = appSpecFormatted.nodes ?? []; + if (category === 'impossible' && pinned.length >= feasibility.instances) { + // A pinned spec names the only machines it may ever use, and the owner + // holds them. Named enough of them and the shortfall is that some are not + // in the confirmed list at this moment - a node rebooting, one that missed + // a check-in, or one not yet installed. That resolves without touching the + // spec, and it is the owner's to resolve, so this reports rather than + // refuses. Naming FEWER machines than instances is the other thing entirely + // and still refuses below: no wait fixes arithmetic. + log.warn(`${caller} - App ${appSpecFormatted.name} requests ${feasibility.instances} instances and names ${pinned.length} node(s), of which ${feasibility.candidateCount} are in the confirmed node list right now; it will run below its instance count until the rest confirm`); + return feasibility; + } + if (category === 'impossible') { + // A geo-restricted request that resolves to NO candidate at all is a + // shortfall this node usually cannot stand behind. Candidate countries + // come from the published table while country-level install eligibility + // is decided by each node's own ip-api self-report, and the two disagree + // for some ranges - a total miss there is indistinguishable from the + // table mis-attributing that geography. Some candidates resolving proves + // the attribution works, so a shortfall above zero is real and refusable. + // + // The one exception is a spec whose every allow entry is a region pin in + // the table's own vocabulary: the installer resolves its region through + // the same table this count reads, so zero candidates means zero nodes + // whose installer would accept - registering it sells a deployment that + // provably cannot start. Same source on both ends turns the miss into + // proof, and proof rejects. + const { allows } = geolocationRule.parseGeolocation( + appSpecFormatted.geolocation, ipLocationStore.regionCodeForName, + ); + const allTableRegionPins = allows.length > 0 + && allows.every((term) => term.granularity === 'region'); + if (geoRestricted && feasibility.candidateCount === 0 && !allTableRegionPins) { + log.warn(`${caller} - App ${appSpecFormatted.name} resolves no eligible node for its geolocation; the location table may not cover it, so the registration is allowed`); + return feasibility; + } + // Two different shortfalls, and telling an owner the wrong one sends them to + // edit a field that was never the problem: a pinned spec has no allowed + // locations to widen, and the machines it may use are the ones it names. + if (pinned.length) { + throw new Error(`App ${appSpecFormatted.name} requests ${feasibility.instances} instances but names only ${pinned.length} node(s), so it can never reach that count. Name at least ${feasibility.instances} nodes or lower the instance count.`); + } + throw new Error(`App ${appSpecFormatted.name} requests ${feasibility.instances} instances but only ${feasibility.candidateCount} eligible nodes exist for its geolocation and tier requirements. Widen the allowed locations or lower the instance count.`); + } + if (category === 'constrained') { + log.warn(`${caller} - App ${appSpecFormatted.name} requests ${feasibility.instances} instances across ${feasibility.domainCount} fault domain(s); synced instances will co-locate up to ${feasibility.maxPerDomain} per domain`); + } + return feasibility; +} + +/** + * Normalise one structured geolocation entry to the spec-string form the + * network stores and matches (ac, ac_, ac__, + * a!c... when forbidden). Vocabulary is the location table's: two-letter + * continent codes, ISO 3166-1 alpha-2 countries, ISO 3166-2 regions. The + * continent may be omitted when the table can derive it from the country; + * a continent that contradicts the table's pairing is an error rather than + * a silent correction. + * @param {{continent?: string, country?: string, region?: string, + * forbidden?: boolean}} entry Structured geolocation entry + * @returns {string} Spec-string form + */ +function normalizeStructuredEntry(entry) { + if (entry.forbidden !== undefined && typeof entry.forbidden !== 'boolean') { + throw new Error('Invalid geolocation entry: forbidden must be a boolean'); + } + const field = (value, name) => { + if (value === undefined || value === null) return null; + if (typeof value !== 'string') throw new Error(`Invalid geolocation entry: ${name} must be a string`); + // capped before it can reach a rejection message: an unbounded value + // echoed into an error writes arbitrary volume into the node's logs, and + // holding a Flux ID is not a reason to be trusted with the length + if (value.length > 20) throw new Error(`Invalid geolocation entry: ${name} is too long`); + return value.trim().toUpperCase(); + }; + const continent = field(entry.continent, 'continent'); + const country = field(entry.country, 'country'); + const region = field(entry.region, 'region'); + if (region && !country) { + throw new Error(`Invalid geolocation entry: region ${region} requires its country`); + } + if (!continent && !country) { + throw new Error('Invalid geolocation entry: a continent or country is required'); + } + if (continent && !CONTINENT_CODES.has(continent)) { + throw new Error(`Invalid geolocation entry: unknown continent code ${continent}`); + } + if (country && !/^[A-Z]{2}$/.test(country)) { + throw new Error(`Invalid geolocation entry: ${country} is not an ISO 3166-1 alpha-2 country code`); + } + if (region && !/^[A-Z]{2}-[A-Z0-9]{1,3}$/.test(region)) { + throw new Error(`Invalid geolocation entry: ${region} is not an ISO 3166-2 region code`); + } + if (region && region.slice(0, 2) !== country) { + throw new Error(`Invalid geolocation entry: region ${region} does not belong to ${country}`); + } + const tableContinent = country ? ipLocationStore.continentForCountry(country) : null; + if (country && ipLocationStore.status().ready && !tableContinent) { + throw new Error(`Invalid geolocation entry: unknown country code ${country}`); + } + if (continent && tableContinent && continent !== tableContinent) { + throw new Error(`Invalid geolocation entry: country ${country} is in ${tableContinent}, not ${continent}`); + } + const resolvedContinent = continent ?? tableContinent; + if (!resolvedContinent) { + throw new Error(`Invalid geolocation entry: cannot derive the continent of ${country} without the location table - include continent`); + } + const parts = [resolvedContinent]; + if (country) parts.push(country); + // The region is emitted in the table's vocabulary (full ISO 3166-2, already + // validated to belong to its country above). Placement matches it at region + // granularity and the installer resolves its own region through the same + // table - one vocabulary end to end, enforced on proof in both directions: + // a node the table cannot place at region granularity satisfies no region + // pin and is caught by no region deny. + if (region) parts.push(region); + return `${entry.forbidden === true ? 'a!c' : 'ac'}${parts.join('_')}`; +} + +/** + * Normalise a mixed geolocation array: spec strings pass through verbatim, + * structured entries become spec strings. Also reports which normalised + * entries carry a region part placement can only honour at country + * granularity - which is a part the table can resolve NEITHER way: not an + * ISO 3166-2 code, and not a name the published vocabulary maps to one. + * Resolved through the same call the rule itself uses, because an entry the + * count honours exactly must never be reported as widened. Structured entries + * always emit table-vocabulary regions, so they are never coarsened. + * @param {Array} entries Geolocation entries, either syntax + * @returns {{normalized: string[], coarsened: string[]}} + */ +function normalizeGeolocation(entries) { + const coarsened = []; + const normalized = entries.map((entry) => { + if (typeof entry === 'string') { + if (entry.length > 50) throw new Error('Invalid geolocation specified'); + // a spec string the caller already holds keeps its region part - it is + // theirs to register - but a legacy-shaped part is answered at country + // granularity, so report it + const body = entry.startsWith('a!c') ? entry.slice(3) : (entry.startsWith('ac') ? entry.slice(2) : null); + const parts = body ? body.split('_') : []; + if (parts.length >= 3 && parts[2] !== 'ALL' && parts[2] !== 'NONE' + && !geolocationRule.regionCodeOf(parts, ipLocationStore.regionCodeForName)) { + coarsened.push(entry); + } + return entry; + } + if (entry && typeof entry === 'object' && !Array.isArray(entry)) { + return normalizeStructuredEntry(entry); + } + throw new Error('Invalid geolocation specified'); + }); + return { normalized, coarsened }; +} + +/** + * The geolocation input of a prospective spec, in either accepted shape: the + * spec's flat geolocation array (spec strings and/or structured entries), or + * the v9 placement shape - geoAllow/geoDeny arrays of structured entries, + * exactly what a v9 spec's placement carries - so the deploy form can pass + * one object to both this endpoint and the spec it registers. Returns the + * mixed entry list normalizeGeolocation consumes. + * @param {object} spec Request body + * @returns {Array} + */ +function geolocationEntries(spec) { + const hasPlacementShape = spec.geoAllow !== undefined || spec.geoDeny !== undefined; + if (!hasPlacementShape) { + const entries = spec.geolocation ?? []; + // 10 entries is the registration limit - nothing beyond it can be bought + if (!Array.isArray(entries) || entries.length > 10) { + throw new Error('Invalid geolocation specified'); + } + return entries; + } + if (spec.geolocation !== undefined) { + throw new Error('Provide either geolocation or geoAllow/geoDeny, not both'); + } + const entries = []; + // eslint-disable-next-line no-restricted-syntax + for (const [name, list, forbidden] of [['geoAllow', spec.geoAllow, false], ['geoDeny', spec.geoDeny, true]]) { + if (list === undefined || list === null) continue; // eslint-disable-line no-continue + // the v9 schema caps each list at 100 entries + if (!Array.isArray(list) || list.length > 100) { + throw new Error(`Invalid ${name} specified`); + } + // eslint-disable-next-line no-restricted-syntax + for (const entry of list) { + if (!entry || typeof entry !== 'object' || Array.isArray(entry) || entry.forbidden !== undefined) { + throw new Error(`Invalid ${name} specified`); + } + entries.push({ ...entry, forbidden }); + } + } + return entries; +} + +// Advice is computed on every request, deliberately. The answer is one pass +// over the resident node list with the rule already parsed - no I/O - so a memo +// would save a few milliseconds while introducing a staleness window on numbers +// the caller is about to spend money against. + +/** + * The placement advice for a prospective app spec, before payment: how many + * fault domains the requested geography spans, how many instances it can + * truly hold, and the spec-string geolocation that was evaluated. + * Geolocation entries are spec strings ('acEU_CZ', 'a!cEU', legacy + * 'aEU'/'bFR') or structured { continent?, country?, region?, forbidden? } + * objects; the v9 placement shape ({ geoAllow, geoDeny } arrays) is accepted + * in place of the flat array. The structured forms are normalised to spec + * strings, and normalizedGeolocation echoes exactly what was evaluated so + * the caller can register it verbatim. coarsenedEntries lists entries whose + * region part placement cannot honour at region granularity. Compose or + * top-level sizing narrows candidates to the tiers that can hold the app. + * Throws on invalid input. + * @param {object} spec Prospective spec { instances?, geolocation? | + * geoAllow?/geoDeny?, compose? | containerData?, cpu/ram/hdd? } + * @returns {Promise} Feasibility plus the advice fields + */ +async function placementAdvice(spec) { + // An unparsed body reaches here as {} - answering about a default spec would + // advise a purchase the caller never described. This endpoint requires + // Content-Type: application/json, which is what makes req.body exist. + if (!spec || typeof spec !== 'object' || Array.isArray(spec) || Object.keys(spec).length === 0) { + throw new Error('Empty or unparsed request body - send JSON with Content-Type: application/json'); + } + const instances = serviceHelper.ensureNumber(spec.instances ?? config.fluxapps.minimumInstances); + if (!Number.isInteger(instances) || instances < 1 || instances > config.fluxapps.maximumInstances) { + throw new Error('Invalid instances specified'); + } + const { normalized, coarsened } = normalizeGeolocation(geolocationEntries(spec)); + // advice differs from enforcement here: the registration gate stays + // permissive without a table (nothing is provable), but serving a + // geo-restricted ANSWER computed over the whole network would advise a + // purchase on numbers that mean nothing - say unavailable instead + if (normalized.some((value) => value !== '') && !ipLocationStore.status().ready) { + const error = new Error('The IP location table is not available yet - geolocation feasibility cannot be answered'); + error.statusCode = 503; + throw error; + } + let synced = true; + if (Array.isArray(spec.compose)) { + synced = spec.compose.some((component) => mountParser.isSyncedComponent(component?.containerData)); + } else if (typeof spec.containerData === 'string') { + synced = mountParser.isSyncedComponent(spec.containerData); + } + const feasibility = await placementFeasibility({ geolocation: normalized, instances }, instances); + // the availability gate above raced the computation: a store that became + // unreadable in between degrades the numbers to the /16 posture, which for + // a geo-restricted question is the whole network - unavailable, not advice + if (normalized.some((value) => value !== '') && !feasibility.tableAvailable) { + const error = new Error('The IP location table is not available yet - geolocation feasibility cannot be answered'); + error.statusCode = 503; + throw error; + } + return { + ...feasibility, + syncedApp: synced, + category: placementCategory(feasibility, synced), + // diversity below the requested count: some fault domain must hold more than one instance + constrained: synced && feasibility.domainCount < feasibility.instances, + // with the water-filled share, any count up to the candidate pool is reachable + satisfiable: feasibility.candidateCount >= feasibility.instances, + normalizedGeolocation: normalized, + coarsenedEntries: coarsened, + }; +} + +/** + * API handler: POST /apps/placementfeasibility. + * + * Requires a signed-in Flux ID, and the reason is compatibility rather than + * cost. This endpoint is new, so it could ask from the outset without breaking + * a caller, and whoever asks it is about to sign a registration anyway - the + * gate takes nothing the deploy path does not already hold. + * + * Cost is deliberately not the reason, because it does not survive contact with + * the neighbours: verifyAppRegistrationParameters and validateAppUpdate run the + * same pass over the node list and stay open, because tooling calls them to + * check a spec before there is a signature to gate on. Every caller does send a + * different spec, so no shared cache bounds this the way one bounds the + * placement geography - but that is a fact about caching, not a reason to gate. + * @param {object} req Request + * @param {object} res Response + */ +async function placementFeasibilityAPI(req, res) { + try { + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); + if (authorized !== true) { + res.json(messageHelper.errUnauthorizedMessage()); + return; + } + const response = messageHelper.createDataMessage(await placementAdvice(req.body ?? {})); + res.json(response); + } catch (error) { + // rejected input and unavailable data are both ordinary answers here - a + // stack per bad request would let a caller fill the error log + log.warn(`placementFeasibilityAPI - ${error.message}`); + const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); + if (error.statusCode) res.status(error.statusCode); + res.json(errorResponse); + } +} + +/** + * The live placement geography in one pass over the node list: node, fault + * domain and tier counts per continent and country, from the node's own + * location table. Nodes the table cannot resolve are counted in unresolved + * rather than guessed at; without a table the tree is empty and only the + * totals (with /16 fault domains) are served. + * @returns {Promise<{tableAvailable: boolean, tableGenerated: string|null, + * total: {nodes: number, domains: number}, unresolved: number, + * continents: object}>} + */ +async function placementLocations() { + // A request is waiting on this, so it may not block: the accessors below wait + // for the node list, and a client holding a connection through a boot is a + // worse answer than a plain "not yet". Its sibling placementComputation says + // the same thing the same way - without this the two would disagree, one + // refusing to answer and the other reporting that an app can be placed + // nowhere. + if (!networkStateService.isReady()) { + const error = new Error('Node list is not available yet'); + error.statusCode = 503; + throw error; + } + + const { byIp, tableAvailable, tableGenerated } = nodeLocationView(); + if (!tableAvailable) { + // the tree IS the product here - totals over /16 fallback domains are + // not the placement geography, so absence of the view is unavailability + const error = new Error('The IP location table is not available yet'); + error.statusCode = 503; + throw error; + } + const nodeList = await fluxCommunicationUtils.deterministicFluxList(); + const domainOf = domainFunction(byIp); + const totalDomains = new Set(); + let totalNodes = 0; + let unresolvedNodes = 0; + const continents = new Map(); + // eslint-disable-next-line no-restricted-syntax + for (const node of nodeList) { + const ip = bareIp(node.ip); + if (!ip) continue; // eslint-disable-line no-continue + totalNodes += 1; + const domain = domainOf(ip); + if (domain) totalDomains.add(domain); + const doc = byIp.get(ip); + if (!doc?.c || !doc.n) { + unresolvedNodes += 1; + continue; // eslint-disable-line no-continue + } + const tier = typeof node.tier === 'string' ? node.tier : 'UNKNOWN'; + let continent = continents.get(doc.n); + if (!continent) { + continent = { nodes: 0, domains: new Set(), tiers: {}, countries: new Map() }; + continents.set(doc.n, continent); + } + continent.nodes += 1; + if (domain) continent.domains.add(domain); + continent.tiers[tier] = (continent.tiers[tier] ?? 0) + 1; + let country = continent.countries.get(doc.c); + if (!country) { + country = { nodes: 0, domains: new Set(), tiers: {} }; + continent.countries.set(doc.c, country); + } + country.nodes += 1; + if (domain) country.domains.add(domain); + country.tiers[tier] = (country.tiers[tier] ?? 0) + 1; + } + const continentsOut = {}; + // eslint-disable-next-line no-restricted-syntax + for (const [code, continent] of continents) { + const countriesOut = {}; + // eslint-disable-next-line no-restricted-syntax + for (const [cc, country] of continent.countries) { + countriesOut[cc] = { nodes: country.nodes, domains: country.domains.size, tiers: country.tiers }; + } + continentsOut[code] = { + nodes: continent.nodes, + domains: continent.domains.size, + tiers: continent.tiers, + countries: countriesOut, + }; + } + return { + tableAvailable, + tableGenerated, + total: { nodes: totalNodes, domains: totalDomains.size }, + unresolved: unresolvedNodes, + continents: continentsOut, + }; +} + +/** + * API handler: GET /apps/placementlocations. + * @param {object} req Request + * @param {object} res Response + */ +async function placementLocationsAPI(req, res) { + try { + const response = messageHelper.createDataMessage(await placementLocations()); + res.json(response); + } catch (error) { + log.warn(`placementLocationsAPI - ${error.message}`); + const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); + if (error.statusCode) res.status(error.statusCode); + res.json(errorResponse); + } +} + +module.exports = { + faultDomain, + nodeLocationMatchesGeolocation, + placementComputation, + placementFeasibility, + placementCategory, + changesPlacement, + countHeldInDomain, + specNamesThisNode, + checkPlacementFeasibility, + normalizeGeolocation, + placementAdvice, + placementFeasibilityAPI, + placementLocations, + placementLocationsAPI, +}; diff --git a/ZelBack/src/services/appQuery/appQueryService.js b/ZelBack/src/services/appQuery/appQueryService.js index 717e4cfda0..41b0d9879b 100644 --- a/ZelBack/src/services/appQuery/appQueryService.js +++ b/ZelBack/src/services/appQuery/appQueryService.js @@ -4,6 +4,7 @@ const dbHelper = require('../dbHelper'); const messageHelper = require('../messageHelper'); const dockerService = require('../dockerService'); const registryManager = require('../appDatabase/registryManager'); +const appsRuntimeState = require('../appManagement/appsRuntimeState'); const appConstants = require('../utils/appConstants'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const { specificationFormatter } = require('../utils/appSpecHelpers'); @@ -71,16 +72,38 @@ async function decryptEnterpriseSpec(spec) { } /** - * Decrypt enterprise apps from a list of apps + * Decrypt enterprise apps, reporting the ones that could not be read. + * + * An enterprise spec carries its components inside the encrypted blob, so a + * failed decrypt leaves a spec with no components - which is not a valid app + * and must never be handed out as though it were. Every caller that enumerates + * components would act on it as "this app has none": the sweep would delete its + * folders, the blocked-image scan would find no images to block, the update + * check would find nothing to update. All silent. + * + * So the result offers two views of one decryption, and the caller says at the + * point of use which it needs: + * + * readable only the specs whose components can be read. What a caller that + * ACTS on components takes - it cannot be handed an invalid spec. + * unreadable the specs that did not decrypt, to report or defer. + * inPlace every spec in the order asked about, unreadable ones left as + * they arrived. What a caller that only DISPLAYS or counts takes - + * an app missing from that list would read as uninstalled. + * + * One name with named views rather than two near-identical exports: picking the + * wrong one of those is silent, and a caller that reaches for the whole result + * where an array is meant fails loudly on the first array method instead. * @param {Array} apps - Array of app specifications * @param {Object} options - Options for decryption * @param {boolean} options.formatSpecs - Whether to format specs (strips metadata like hash, height). Default: true - * @param {boolean} options.throwOnError - Rethrow a decrypt failure instead of returning the encrypted spec. Default: false - * @returns {Promise} Array of decrypted app specifications + * @returns {Promise<{readable: Array, unreadable: Array, inPlace: Array}>} */ async function decryptEnterpriseApps(apps, options = {}) { - const { formatSpecs = true, throwOnError = false } = options; - const decryptedApps = []; + const { formatSpecs = true } = options; + const readable = []; + const unreadable = []; + const inPlace = []; // eslint-disable-next-line no-restricted-syntax for (const spec of apps) { @@ -94,21 +117,19 @@ async function decryptEnterpriseApps(apps, options = {}) { // Apply formatting if requested const result = formatSpecs ? specificationFormatter(decrypted) : decrypted; - decryptedApps.push(result); + readable.push(result); + inPlace.push(result); } catch (error) { log.error(`Failed to decrypt enterprise app ${spec.name}: ${error.message}`); - // Display/listing callers (default) keep the lenient behavior: include the - // still-encrypted spec so the rest of the list isn't lost. Callers that act - // on the spec (the reconciler) pass throwOnError so they can defer rather - // than operate on undecrypted data (wrong containerData, mis-typed g:/r:). - if (throwOnError) throw error; - decryptedApps.push(spec); + unreadable.push(spec); + inPlace.push(spec); } } else { - decryptedApps.push(spec); + readable.push(spec); + inPlace.push(spec); } } - return decryptedApps; + return { readable, unreadable, inPlace }; } /** @@ -152,16 +173,54 @@ async function installedApps(req, res) { } /** - * To list running apps. - * @param {object} req Request. - * @param {object} res Response. - * @returns {object} Message. + * The container fields a public listing carries. + * + * An allowlist rather than a filter: the view is built from three of docker's + * own fields, keeping docker's names and forms, so it holds nothing else and a + * field docker adds in a future version is absent by default rather than + * carried until someone notices. A listing assembled this way cannot report an + * application's image, entrypoint, ports or build metadata; those belong to its + * owner, and are published from its specification when its specification is + * public. + * + * The same three fields for every application, not a smaller view for some. + * Choosing per application would put that choice at the exit, where getting it + * wrong is silent. + * + * Names stays docker's array, verbatim. FDM builds /flux{component}_{app} and + * matches Names[0] exactly, failing closed when it does not - so a bare name, + * or one with the prefix stripped, takes every g: app out of routing. + * + * @param {Array} containers - docker container objects + * @returns {Array} each container as {Names, State, Status} + */ +function publicContainerView(containers) { + return containers.map((container) => ({ + Names: container.Names, + State: container.State, + Status: container.Status, + })); +} + +/** + * The containers this node should be routed to: everything running, plus any + * container a backup or restore is holding stopped. + * + * Not a filter of listAllApps. A container stopped for a backup is still the + * application's, and leaving it out is FDM dropping that application from + * haproxy for the length of the backup - so the in-memory backup and restore + * sets put it back. + * + * Returns docker's container objects whole, for callers inside this process. + * The public route answers from listRunningAppsApi, which projects them. + * + * @returns {object} Message carrying the container objects. */ -async function listRunningApps(req, res) { +async function listRunningApps() { try { let apps = await dockerService.dockerListContainers(false); if (apps.length > 0) { - apps = apps.filter((app) => (app.Names[0].slice(1, 4) === 'zel' || app.Names[0].slice(1, 5) === 'flux')); + apps = apps.filter((app) => dockerService.isAppContainer(app)); } // Include apps that are in backup or restore as "running" even if container is stopped @@ -173,7 +232,7 @@ async function listRunningApps(req, res) { if (appsInBackupRestore.length > 0) { // Get all containers including stopped ones const allContainers = await dockerService.dockerListContainers(true); - const fluxContainers = allContainers.filter((app) => (app.Names[0].slice(1, 4) === 'zel' || app.Names[0].slice(1, 5) === 'flux')); + const fluxContainers = allContainers.filter((app) => dockerService.isAppContainer(app)); // Find stopped containers that are in backup/restore and add them to running list fluxContainers.forEach((container) => { @@ -206,7 +265,95 @@ async function listRunningApps(req, res) { modifiedApps.push(app); }); const appsResponse = messageHelper.createDataMessage(modifiedApps); - return res ? res.json(appsResponse) : appsResponse; + return appsResponse; + } catch (error) { + log.error(error); + const errorResponse = messageHelper.createErrorMessage( + error.message || error, + error.name, + error.code, + ); + return errorResponse; + } +} + +/** + * GET /apps/listrunningapps - the public view of the containers this node + * should be routed to. + * + * The response is the same for every caller, which is what lets the route keep + * its cache: apicache keys an entry on the request URL alone and answers from + * its store before the handler runs, so anything decided from who is asking is + * decided once and then served to everyone else. + * + * @param {object} req Request. + * @param {object} res Response. + * @returns {void} + */ +async function listRunningAppsApi(req, res) { + const response = await listRunningApps(); + if (response.status === 'error') { + res.json(response); + return; + } + res.json(messageHelper.createDataMessage(publicContainerView(response.data))); +} + +/** + * Component identifiers this node holds: running here, committed to running and + * not started yet, or deliberately stopped here by the operator. + * + * The question a primary election actually asks a peer, and the answer is about + * OWNERSHIP, not about what is up. Three sources, because no one of them answers + * it on its own: + * + * - Running containers miss the masterSlave primary path, which fixes ownership + * on the persistent data before it starts anything. For that whole window the + * node has decided but has no container. + * - committedIdentifiers covers that window, but it is in-memory and re-derived + * from live truth, so a FluxOS restart empties it. It is also only ever written + * at the moment a node wins an election, never re-asserted while it goes on + * being the primary. + * - The operator stop lock is the durable one. `appstop` writes it to hold the + * component down HERE - the election skips this node and the reconciler will + * not restart it - and until this endpoint reported it, that intent never left + * the node. A peer saw no container and no commitment, concluded the component + * was free, and elected a new primary over an owner who had stopped theirs to + * work on it. Whether that happened at all turned on whether this node's FluxOS + * had restarted since it was elected, which is not something an owner can see. + * + * Not filtered to g: components. The list answers "is this component mine", which + * is true of a stopped component whatever its storage mode, and the only caller + * asks about g: components alone - so filtering would cost a spec lookup to leave + * out entries nobody looks up. + * + * Cached for one second at the route, not the fifteen listrunningapps takes: long + * enough to bound an anonymous caller to one pass of this work per second, short + * enough to be meaningless against the tens of seconds the window it closes runs + * for. + * + * @param {object} req Request. + * @param {object} res Response. + * @returns {object} Message carrying an array of container-name identifiers. + */ +async function heldComponents(req, res) { + try { + const containers = await dockerService.dockerListContainers(false); + const running = containers + .map((container) => (container.Names?.[0] || '').replace(/^\//, '')) + .filter((name) => name.slice(0, 3) === 'zel' || name.slice(0, 4) === 'flux'); + + // eslint-disable-next-line global-require + const appReconciler = require('../appMonitoring/appReconciler'); + const committed = appReconciler.committedIdentifiers() + .map((identifier) => dockerService.getAppIdentifier(identifier)); + + const operatorStopped = (await appsRuntimeState.operatorStoppedIdentifiers()) + .map((identifier) => dockerService.getAppIdentifier(identifier)); + + const held = [...new Set([...running, ...committed, ...operatorStopped])]; + const response = messageHelper.createDataMessage(held); + return res ? res.json(response) : response; } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage( @@ -219,16 +366,68 @@ async function listRunningApps(req, res) { } /** - * List all apps (both running and installed) + * Syncthing folder ids this node has promoted to sendreceive - the folders it + * holds the writable copy of. + * + * Asked by a peer before it promotes a folder of its own. Promotion is decided + * from each node's own view of the holder list, and those views fill in at + * different moments, so two nodes can each conclude they are the one - the first + * while it is briefly the only holder it knows of, the second once it can see + * more and wins the tiebreak among them. Neither revisits the decision, because a + * promoted folder never re-enters the election. Nothing else carries this: folder + * type is local syncthing config, and at genesis the promoted node has no data + * yet, so the has-data signal is silent exactly when it is needed. + * + * Served from the set the syncthing monitor refreshes each pass, not by reading + * syncthing per request: the route is unauthenticated and reachable by any peer, + * so an on-demand read would be an amplifier into syncthing, and the API has no + * rate limiting of its own. It is also then O(1), so it needs no response cache + * and carries no staleness beyond one monitor pass. + * + * `ready` is what stops a booting node being read as a free one. Before the + * monitor's first pass this node cannot distinguish "I hold nothing" from "I have + * not looked", and answering the first would invite a peer to promote alongside a + * folder this node is already holding. The asker treats an unready peer as a + * reason to wait rather than a clearance. + * * @param {object} req Request. * @param {object} res Response. - * @returns {object} Message. + * @returns {object} Message carrying { ready, folders }. */ -async function listAllApps(req, res) { +async function promotedFolders(req, res) { + try { + // eslint-disable-next-line global-require + const globalState = require('../utils/globalState'); + const ids = globalState.promotedFolderIds; + const response = messageHelper.createDataMessage({ + ready: ids !== null, + folders: ids === null ? [] : [...ids], + }); + return res ? res.json(response) : response; + } catch (error) { + log.error(error); + const errorResponse = messageHelper.createErrorMessage( + error.message || error, + error.name, + error.code, + ); + return res ? res.json(errorResponse) : errorResponse; + } +} + +/** + * Every container of an application on this node, running or not. + * + * Returns docker's container objects whole, for callers inside this process. + * The public route answers from listAllAppsApi, which projects them. + * + * @returns {object} Message carrying the container objects. + */ +async function listAllApps() { try { let apps = await dockerService.dockerListContainers(true); if (apps.length > 0) { - apps = apps.filter((app) => (app.Names[0].slice(1, 4) === 'zel' || app.Names[0].slice(1, 5) === 'flux')); + apps = apps.filter((app) => dockerService.isAppContainer(app)); } const modifiedApps = []; apps.forEach((app) => { @@ -240,19 +439,33 @@ async function listAllApps(req, res) { delete app.Mounts; modifiedApps.push(app); }); - const appsResponse = messageHelper.createDataMessage(modifiedApps); - return res ? res.json(appsResponse) : appsResponse; + return messageHelper.createDataMessage(modifiedApps); } catch (error) { log.error(error); - const errorResponse = messageHelper.createErrorMessage( + return messageHelper.createErrorMessage( error.message || error, error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; } } +/** + * GET /apps/listallapps - the public view of every container on this node. + * + * @param {object} req Request. + * @param {object} res Response. + * @returns {void} + */ +async function listAllAppsApi(req, res) { + const response = await listAllApps(); + if (response.status === 'error') { + res.json(response); + return; + } + res.json(messageHelper.createDataMessage(publicContainerView(response.data))); +} + /** * To get latest application specification API version. * @param {object} req Request. @@ -360,8 +573,13 @@ async function getAppsMessagesCount(req, res) { module.exports = { installedApps, decryptEnterpriseApps, + publicContainerView, listRunningApps, + listRunningAppsApi, + heldComponents, + promotedFolders, listAllApps, + listAllAppsApi, getlatestApplicationSpecificationAPI, getApplicationOriginalOwner, getAppsInstallingLocations, diff --git a/ZelBack/src/services/appQuery/fileQueryService.js b/ZelBack/src/services/appQuery/fileQueryService.js index 5d6a48aff0..13df9e1d42 100644 --- a/ZelBack/src/services/appQuery/fileQueryService.js +++ b/ZelBack/src/services/appQuery/fileQueryService.js @@ -5,6 +5,8 @@ const verificationHelper = require('../verificationHelper'); const IOUtils = require('../IOUtils'); const log = require('../../lib/log'); const { sanitizePath, verifyRealPath } = require('../utils/pathSecurity'); +const { isReservedName } = require('../appSystem/volumeReservedNames'); +const { Privilege, authOf } = require('../utils/privileges'); /** * To get apps folder contents. @@ -15,7 +17,7 @@ async function getAppsFolder(req, res) { try { let { appname } = req.params; appname = appname || req.query.appname || ''; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (authorized) { let { folder } = req.params; folder = folder || req.query.folder || ''; @@ -25,23 +27,40 @@ async function getAppsFolder(req, res) { throw new Error('appname and component parameters are mandatory'); } let filepath; - const appVolumePath = await IOUtils.getVolumeInfo(appname, component, 'B', 'mount', 0); - if (appVolumePath.length > 0) { + const { mounts } = await IOUtils.getVolumeInfo(appname, component, 'B', 'mount', 0); + if (mounts.length > 0) { // Browse at appid level to show appdata and all other mount points // Sanitize folder path to prevent directory traversal attacks - filepath = sanitizePath(folder, appVolumePath[0].mount); + filepath = sanitizePath(folder, mounts[0].mount); // Verify resolved path stays within the allowed base directory - await verifyRealPath(filepath, appVolumePath[0].mount); + await verifyRealPath(filepath, mounts[0].mount); } else { throw new Error('Application volume not found'); } const options = { withFileTypes: false, }; - const files = await fs.readdir(filepath, options); + const listed = await fs.readdir(filepath, options); + + // The browser opens at the volume root so an app with several mounts + // shows them all, and that root also holds things that are not the + // owner's: syncthing's control files, the filesystem's recovery + // directory, and what an interrupted file operation left for the boot + // sweep. They are implementation detail, they cannot be written through + // any endpoint, and a listing that offers them invites an operation that + // will only be refused. + const atRoot = filepath === mounts[0].mount; + const files = atRoot ? listed.filter((name) => !isReservedName(name)) : listed; + const filesWithDetails = []; // eslint-disable-next-line no-restricted-syntax for (const file of files) { + // lstat, so an entry describes ITSELF. An application can put a link in + // its own volume pointing anywhere on the node, and stat would answer + // with the size and type of whatever it names. It also decides whether + // this descends: a linked directory is not a directory here, so the size + // walk below is never sent through one. See the rule at the top of + // appSystem/fileSystemManager. // eslint-disable-next-line no-await-in-loop const fileStats = await fs.lstat(`${filepath}/${file}`); const isDirectory = fileStats.isDirectory(); diff --git a/ZelBack/src/services/appQuery/resourceQueryService.js b/ZelBack/src/services/appQuery/resourceQueryService.js index ad8ceb23bf..3e797fd08c 100644 --- a/ZelBack/src/services/appQuery/resourceQueryService.js +++ b/ZelBack/src/services/appQuery/resourceQueryService.js @@ -49,19 +49,47 @@ async function fluxUsage(req, res) { } /** - * Get apps resource usage - * @param {object} req Request. - * @param {object} res Response. - * @returns {object} Message. + * What this node has committed to the applications it holds, and which of them it + * could not read. + * + * An enterprise app is stored locally with its components emptied - they are the + * customer's decrypted configuration and do not belong in a local database - so + * counting the stored row directly credits that app with no cpu, no memory and + * no disk at all, including the fixed overhead every component carries. It is + * not skipped by a rule: it takes the ordinary path, finds an empty list and + * adds nothing. That total is what the node subtracts from its own capacity + * before accepting another app, so a node holding enterprise apps believes it + * is emptier than it is and keeps taking work it cannot run. + * + * So the specifications are decrypted first, through the cached path: it keys on + * the spec hash, shares one attempt between concurrent callers and remembers a + * failure briefly, none of which the raw decrypt does - and this is reached from + * two public endpoints, so a decrypt per caller would be an amplifier into + * fluxbenchd. Unformatted, because the formatter strips the hash the cache keys + * on and nothing here reads a formatted field. + * + * `unreadable` is the names it could not decrypt, and it is the reason this + * returns two things rather than three numbers. An app whose components cannot + * be read contributes nothing, which is indistinguishable from an app that + * reserves nothing - the caller has to be able to tell those apart, because one + * of them means this node cannot account for itself. It does not leave the + * process: the API halves publish the totals alone. + * + * @returns {object} Message carrying {appsCpusLocked, appsRamLocked, appsHddLocked, unreadable} */ -async function appsResources(req, res) { +async function appsResources() { log.info('Checking appsResources'); try { const dbopen = dbHelper.databaseConnection(); const appsDatabase = dbopen.db(config.database.appslocal.database); const appsQuery = {}; const appsProjection = { projection: { _id: 0 } }; - const appsResult = await dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, appsQuery, appsProjection); + const stored = await dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, appsQuery, appsProjection); + + // eslint-disable-next-line global-require + const { decryptEnterpriseApps } = require('./appQueryService'); + const { inPlace: appsResult, unreadable } = await decryptEnterpriseApps(stored, { formatSpecs: false }); + let appsCpusLocked = 0; let appsRamLocked = 0; let appsHddLocked = 0; @@ -102,21 +130,81 @@ async function appsResources(req, res) { appsCpusLocked, appsRamLocked, appsHddLocked, + unreadable: unreadable.map((app) => app.name), }; - const response = messageHelper.createDataMessage(appsUsage); - return res ? res.json(response) : response; + return messageHelper.createDataMessage(appsUsage); } catch (error) { log.error(error); - const errorResponse = messageHelper.createErrorMessage( + return messageHelper.createErrorMessage( error.message || error, error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; } } +/** + * The applications this node holds but cannot size. + * + * One whose specification cannot be read contributes nothing to the totals, + * which is indistinguishable from one that reserves nothing - so a decision that + * subtracts those totals from the node's capacity believes space is free that is + * already spoken for. + * + * Offered rather than enforced here, because the answer differs by caller. + * Taking on NEW work while this is non-empty over-commits the node. Maintaining + * work it already holds does not: a redeploy of an app already counted adds + * nothing, and refusing there would freeze every other application on the node + * over one it cannot read. + * + * @param {object} response - what appsResources answered with + * @returns {string[]} the names, empty when the totals account for everything + */ +function unaccountedApps(response) { + if (response.status !== 'success') return []; + + return response.data.unreadable || []; +} + +/** + * The committed totals, and nothing else. + * + * Three numbers, node-wide, exactly as this has always answered. `unreadable` + * stops here: the names are already public on chain, but "this node is holding an + * application it cannot read" is a statement about the node's health that nobody + * outside it needs. + * + * @param {object} req Request. + * @param {object} res Response. + * @returns {Promise} + */ +async function appsResourcesApi(req, res) { + const response = await appsResources(); + + if (response.status === 'error') { + res.json(response); + return; + } + + const { appsCpusLocked, appsRamLocked, appsHddLocked } = response.data; + res.json(messageHelper.createDataMessage({ appsCpusLocked, appsRamLocked, appsHddLocked })); +} + +/** + * The same three numbers, for a caller inside this process that publishes them. + * @param {object} usage - what appsResources answered with + * @returns {object} {appsCpusLocked, appsRamLocked, appsHddLocked} + */ +function publicResourceView(usage) { + const { appsCpusLocked, appsRamLocked, appsHddLocked } = usage; + + return { appsCpusLocked, appsRamLocked, appsHddLocked }; +} + module.exports = { fluxUsage, appsResources, + unaccountedApps, + appsResourcesApi, + publicResourceView, }; diff --git a/ZelBack/src/services/appRequirements/appValidator.js b/ZelBack/src/services/appRequirements/appValidator.js index 96cbaba964..1ca458c1cf 100644 --- a/ZelBack/src/services/appRequirements/appValidator.js +++ b/ZelBack/src/services/appRequirements/appValidator.js @@ -8,6 +8,7 @@ const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const fluxCommunicationMessagesSender = require('../fluxCommunicationMessagesSender'); const registryManager = require('../appDatabase/registryManager'); const messageVerifier = require('../appMessaging/messageVerifier'); +const signatureVerifier = require('../signatureVerifier'); const imageManager = require('../appSecurity/imageManager'); // const advancedWorkflows = require('../appLifecycle/advancedWorkflows'); // Moved to dynamic require to avoid circular dependency // eslint-disable-next-line no-unused-vars @@ -16,9 +17,11 @@ const { } = require('../utils/appConstants'); const { specificationFormatter, findCommonArchitectures } = require('../utils/appUtilities'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); +const placementFeasibility = require('../appPlacement/placementFeasibility'); const enterpriseConfig = require('../utils/enterpriseConfig'); const portManager = require('../appNetwork/portManager'); const { peerManager } = require('../utils/peerState'); +const { Privilege, authOf } = require('../utils/privileges'); const isArcane = Boolean(process.env.FLUXOS_PATH); @@ -1216,11 +1219,12 @@ function checkComposeHWParameters(appSpecsComposed) { * Validates specs including hardware requirements, architecture compatibility, and Docker compliance * @param {object} appSpecifications - Application specifications to validate * @param {number} height - Block height for validation context - * @param {boolean} checkDockerAndWhitelist - Whether to check Docker, whitelist, and architecture requirements + * @param {boolean} liveSubmission - Whether the spec is being submitted now through the API, rather than + * replayed from a message already on chain. Gates the checks that only hold for a spec being accepted today. * @returns {Promise} True if validation passes * @throws {Error} If validation fails (e.g., incompatible architectures, missing requirements) */ -async function verifyAppSpecifications(appSpecifications, height, checkDockerAndWhitelist = false) { +async function verifyAppSpecifications(appSpecifications, height, liveSubmission = false) { if (!appSpecifications) { throw new Error('Invalid Flux App Specifications'); } @@ -1234,6 +1238,14 @@ async function verifyAppSpecifications(appSpecifications, height, checkDockerAnd // TYPE CHECKS verifyTypeCorrectnessOfApp(appSpecifications); + // OWNER IDENTITY + // Updates verify against the owner already on record, so the incoming owner is + // never used as a key and an owner that cannot be signed for is accepted. Held + // to live submissions only - messages already on chain replay unchanged. + if (liveSubmission && !signatureVerifier.isValidSigningIdentity(appSpecifications.owner)) { + throw new Error('Invalid Flux App owner. Must be a Flux ID or an Ethereum address'); + } + // RESTRICTION CHECKS verifyRestrictionCorrectnessOfApp(appSpecifications, height); @@ -1257,7 +1269,7 @@ async function verifyAppSpecifications(appSpecifications, height, checkDockerAnd } // Whitelist, repository checks - if (checkDockerAndWhitelist) { + if (liveSubmission) { // check blacklist await imageManager.checkApplicationImagesCompliance(appSpecifications); @@ -1376,6 +1388,11 @@ async function verifyAppRegistrationParameters(req, res) { // parameters are now proper format and assigned. Check for their validity, if they are within limits, have propper ports, repotag exists, string lengths, specs are ok await verifyAppSpecifications(appSpecFormatted, daemonHeight, true); + // placement feasibility at the front door, while the spec is still + // decrypted: an impossible spec is rejected before it is paid for, a + // diversity-constrained one is accepted with a warning + await placementFeasibility.checkPlacementFeasibility(appSpecFormatted, 'verifyAppRegistrationParameters'); + if (appSpecFormatted.version === 7 && appSpecFormatted.nodes.length > 0) { // eslint-disable-next-line no-restricted-syntax for (const appComponent of appSpecFormatted.compose) { @@ -1471,6 +1488,12 @@ async function validateAppUpdate(appSpecification) { await advancedWorkflows.validateApplicationUpdateCompatibility(appSpecFormatted, previousAppSpecs); + // placement feasibility applies to updates too: a narrowed geolocation, + // raised instance count or grown sizing must not buy a spec the network + // provably cannot satisfy. Passing the previous spec keeps an update that + // changes nothing placement-relevant - a renewal, a cancellation - unrefused. + await placementFeasibility.checkPlacementFeasibility(appSpecFormatted, 'validateAppUpdate', previousAppSpecs); + if (isEnterprise) { appSpecFormatted.contacts = []; appSpecFormatted.compose = []; @@ -1520,7 +1543,7 @@ async function registerAppGlobalyApi(req, res) { }); req.on('end', async () => { try { - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); diff --git a/ZelBack/src/services/appRequirements/hwRequirements.js b/ZelBack/src/services/appRequirements/hwRequirements.js index 29ab20b730..9615eae1f9 100644 --- a/ZelBack/src/services/appRequirements/hwRequirements.js +++ b/ZelBack/src/services/appRequirements/hwRequirements.js @@ -2,6 +2,8 @@ const os = require('os'); const config = require('config'); const generalService = require('../generalService'); const geolocationService = require('../geolocationService'); +const ipLocationStore = require('../appPlacement/ipLocationStore'); +const geolocationRule = require('../appPlacement/geolocationRule'); const benchmarkService = require('../benchmarkService'); const fluxNetworkHelper = require('../fluxNetworkHelper'); const { socketAddressesMatch } = require('../utils/socketAddressUtils'); @@ -122,7 +124,15 @@ async function nodeFullGeolocation() { } /** - * To check app requirements of staticip restrictions for a node + * To check app requirements of staticip restrictions for a node. + * + * `staticip` means the node is directly connected - a public address on its own + * interface - and has not been seen to move. Both halves are required: an app + * asking for this needs a fixed ip:port that stays REACHABLE, and a node + * reaching the world through a NAT port mapping promises neither, however + * stable its upstream address is. A range-level "this is a hosting network" + * verdict answers neither half, so it confers nothing here. The full rule, and + * the fleet census behind it, are in geolocationService's decision table. * @param {object} appSpecs App specifications. * @returns {boolean} True if all checks passed. */ @@ -169,30 +179,106 @@ async function checkAppGeolocationRequirements(appSpecs) { const geoC = appSpecs.geolocation.filter((x) => x.startsWith('ac')); // this ensures that new specs can only run on updated nodes. const geoCForbidden = appSpecs.geolocation.filter((x) => x.startsWith('a!c')); - const myNodeLocationContinent = nodeGeo.continentCode; - const myNodeLocationContCountry = `${nodeGeo.continentCode}_${nodeGeo.countryCode}`; - const myNodeLocationFull = `${nodeGeo.continentCode}_${nodeGeo.countryCode}_${nodeGeo.regionName}`; + // This node's own continent, country and region, resolved through the SAME + // published table the candidate count reads for every other node - so the + // two cannot disagree about where this node is. + // + // That matters in one direction. The count has no alternative source: it is + // answering for thousands of nodes it cannot ask. So if this check read the + // node's ip-api self-report instead, the two would differ wherever the table + // and the self-report differ, and the count would exclude nodes this check + // would have accepted - a candidate count below the instance count, and a + // registration refused for a spec that would have deployed. + // + // The self-report is the fallback, for a node the table cannot place at all. + // The count treats such a node as unprovable and includes it, so falling + // back here can only make this check stricter than the count, which is the + // safe direction. + let myContinent = nodeGeo.continentCode; + let myCountry = nodeGeo.countryCode; + let myTableRegion = null; + try { + const hit = nodeGeo.ip ? await ipLocationStore.lookup(nodeGeo.ip) : null; + if (hit?.countryCode && hit.continentCode) { + myContinent = hit.continentCode; + myCountry = hit.countryCode; + myTableRegion = hit.region ?? null; + } + } catch (error) { + log.warn(`checkAppGeolocationRequirements - node location unavailable from the table: ${error.message}`); + } + + const myNodeLocationContinent = myContinent; + const myNodeLocationContCountry = `${myContinent}_${myCountry}`; + // the region part of a legacy entry is an ip-api region NAME, which only + // this node knows about itself - the count cannot read it for other nodes + // and deliberately answers such entries at country granularity, so matching + // it here can only narrow what this node accepts + const myNodeLocationFull = `${myContinent}_${myCountry}_${nodeGeo.regionName}`; const myNodeLocationContinentALL = 'ALL'; - const myNodeLocationContCountryALL = `${nodeGeo.continentCode}_ALL`; - const myNodeLocationFullALL = `${nodeGeo.continentCode}_${nodeGeo.countryCode}_ALL`; + const myNodeLocationContCountryALL = `${myContinent}_ALL`; + const myNodeLocationFullALL = `${myContinent}_${myCountry}_ALL`; + // A region entry matches on proof only, in both directions: an allow is + // satisfied - and a deny applies - exactly when this node's table region is + // known and equal. An unknown region satisfies no region pin and is caught + // by no region deny. + // + // The entry may name its region in either vocabulary, an ISO 3166-2 code or + // the name ip-api uses, and the published vocabulary resolves the second to + // the first. Whichever it is, the count resolves it the same way and both + // sides compare against the same table region. + const entryRegionCode = (parts) => (parts.length < 3 + ? null + : geolocationRule.regionCodeOf(parts, ipLocationStore.regionCodeForName)); + const matchesTableRegionEntry = (value) => { + const parts = value.split('_'); + const code = entryRegionCode(parts); + if (!code || !myTableRegion) return false; + return `${parts[0]}_${parts[1]}` === myNodeLocationContCountry && code === myTableRegion; + }; + // This node's own ip-api region name. The count cannot read it for any other + // node, so an entry answered this way is answered by this node alone. + const matchesSelfReportedRegionName = (value) => value === myNodeLocationFull; + // For an ALLOW. A region part neither vocabulary resolves is left to the + // self-reported name, and matching it can only narrow what this node + // accepts. An entry the vocabulary DOES resolve must not also match by + // name: that would accept nodes the count excluded, and where the two + // sources disagree the table is the one to believe. + const matchesSelfReportedRegion = (value) => { + const parts = value.split('_'); + if (parts.length >= 3 && entryRegionCode(parts)) return false; + return value === myNodeLocationFull; + }; if (appContinent && !geoC.length && !geoCForbidden.length) { // backwards old style compatible. Can be removed after a month - if (appContinent.slice(1) !== nodeGeo.continentCode) { + if (appContinent.slice(1) !== myContinent) { throw new Error('App specs with continents geolocation set not matching node geolocation. Aborting.'); } } if (appCountry) { - if (appCountry.slice(1) !== nodeGeo.countryCode) { + if (appCountry.slice(1) !== myCountry) { throw new Error('App specs with countries geolocation set not matching node geolocation. Aborting.'); } } + // A DENY takes either source, and this is the one place the self-reported + // name still counts for an entry the vocabulary resolves. The rule an allow + // uses would be the permissive choice here: declining to match by name lets + // a node the table places one region over run an app that named its own + // region, so a ban would catch fewer nodes than the owner wrote. Taking both + // keeps the self-report able only to STRENGTHEN a ban, never to grant + // eligibility - the table remains the only thing that can say yes. A ban is + // usually written for a reason the network cannot see, so the error worth + // making is excluding a node that would have been fine. geoCForbidden.forEach((locationNotAllowed) => { - if (locationNotAllowed.slice(3) === myNodeLocationContinent || locationNotAllowed.slice(3) === myNodeLocationContCountry || locationNotAllowed.slice(3) === myNodeLocationFull) { + const v = locationNotAllowed.slice(3); + if (v === myNodeLocationContinent || v === myNodeLocationContCountry + || matchesSelfReportedRegionName(v) || matchesTableRegionEntry(v)) { throw new Error('App specs of geolocation set is forbidden to run on node geolocation. Aborting.'); } }); if (geoC.length) { - const nodeLocationOK = geoC.find((locationAllowed) => locationAllowed.slice(2) === myNodeLocationContinent || locationAllowed.slice(2) === myNodeLocationContCountry || locationAllowed.slice(2) === myNodeLocationFull - || locationAllowed.slice(2) === myNodeLocationContinentALL || locationAllowed.slice(2) === myNodeLocationContCountryALL || locationAllowed.slice(2) === myNodeLocationFullALL); + const nodeLocationOK = geoC.find((locationAllowed) => locationAllowed.slice(2) === myNodeLocationContinent || locationAllowed.slice(2) === myNodeLocationContCountry + || locationAllowed.slice(2) === myNodeLocationContinentALL || locationAllowed.slice(2) === myNodeLocationContCountryALL || locationAllowed.slice(2) === myNodeLocationFullALL + || matchesSelfReportedRegion(locationAllowed.slice(2)) || matchesTableRegionEntry(locationAllowed.slice(2))); if (!nodeLocationOK) { throw new Error('App specs of geolocation set is not matching to run on node geolocation. Aborting.'); } diff --git a/ZelBack/src/services/appSecurity/imageManager.js b/ZelBack/src/services/appSecurity/imageManager.js index 1cb28aacab..b757a1ac69 100644 --- a/ZelBack/src/services/appSecurity/imageManager.js +++ b/ZelBack/src/services/appSecurity/imageManager.js @@ -2,8 +2,6 @@ const config = require('config'); const axios = require('axios'); const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); -// eslint-disable-next-line no-unused-vars -const pgpService = require('../pgpService'); const registryCredentialHelper = require('../utils/registryCredentialHelper'); const imageVerifier = require('../utils/imageVerifier'); const dbHelper = require('../dbHelper'); @@ -12,6 +10,7 @@ const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const log = require('../../lib/log'); const { supportedArchitectures, globalAppsMessages, globalAppsInformation } = require('../utils/appConstants'); const fluxCaching = require('../utils/cacheManager').default; +const { Privilege, authOf } = require('../utils/privileges'); // Cache for blocked repositories let cacheUserBlockedRepos = null; @@ -36,11 +35,9 @@ function classifyVerificationError(error, errorMeta) { return { ttlMs: 2 * FluxCacheManager.oneHour, reason: 'Rate limiting (429)' }; case 'server_error': return { ttlMs: 3 * FluxCacheManager.oneHour, reason: 'Server error (5xx)' }; - case 'whitelist_fetch_error': case 'auth_unavailable': return { ttlMs: 2 * FluxCacheManager.oneHour, reason: 'Temporary service issue' }; // Permanent errors - longer cache - case 'not_whitelisted': case 'invalid_format': case 'unsupported_architecture': case 'unsupported_media_type': @@ -182,7 +179,7 @@ async function getBlockedRepositores() { if (cachedResponse) { return cachedResponse; } - const resBlockedRepo = await serviceHelper.axiosGet(`${config.github.rawBaseUrl}/helpers/blockedrepositories.json`); + const resBlockedRepo = await serviceHelper.axiosGet(`${config.policy.baseUrl}/blockedrepositories.json`); if (resBlockedRepo.data) { fluxCaching.blockedRepositoriesCache.set('blockedRepositories', resBlockedRepo.data); return resBlockedRepo.data; @@ -205,7 +202,7 @@ async function getVettedRepositories() { if (cachedResponse) { return cachedResponse; } - const resVettedRepo = await serviceHelper.axiosGet(`${config.github.rawBaseUrl}/helpers/vettedrepositories.json`); + const resVettedRepo = await serviceHelper.axiosGet(`${config.policy.baseUrl}/vettedrepositories.json`); if (resVettedRepo.data) { fluxCaching.blockedRepositoriesCache.set('vettedRepositories', resVettedRepo.data); return resVettedRepo.data; @@ -288,13 +285,13 @@ async function getUserBlockedRepositores() { return cacheUserBlockedRepos; } - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const userBlockedRepos = userconfig.initial.blockedRepositories || []; if (userBlockedRepos.length === 0) { return userBlockedRepos; } const usableUserBlockedRepos = []; - const marketPlaceUrl = 'https://stats.runonflux.io/marketplace/listapps'; + const marketPlaceUrl = `${config.stats.baseUrl}/marketplace/listapps`; const response = await axios.get(marketPlaceUrl); console.log(response); if (response && response.data && response.data.status === 'success') { @@ -591,7 +588,7 @@ async function checkDockerAccessibility(req, res) { }); req.on('end', async () => { try { - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -633,8 +630,12 @@ async function checkApplicationsCompliance(installedApps, removeAppLocally) { throw new Error('Failed to get installed Apps'); } // Decrypt enterprise apps (version 8 with encrypted content) - installedAppsRes.data = await decryptEnterpriseApps(installedAppsRes.data); - const appsInstalled = installedAppsRes.data; + const { readable: appsInstalled, unreadable } = await decryptEnterpriseApps(installedAppsRes.data); + if (unreadable.length) { + // their repotags are inside the blob, so a blocked image in one cannot be + // seen here - it is not cleared, it is unexamined + log.warn(`Cannot check blocked images for undecryptable apps: ${unreadable.map((app) => app.name).join(', ')}`); + } const appsToRemoveNames = []; // eslint-disable-next-line no-restricted-syntax for (const app of appsInstalled) { diff --git a/ZelBack/src/services/appSystem/fileOperationRecovery.js b/ZelBack/src/services/appSystem/fileOperationRecovery.js new file mode 100644 index 0000000000..22ef2996ee --- /dev/null +++ b/ZelBack/src/services/appSystem/fileOperationRecovery.js @@ -0,0 +1,82 @@ +const deviceHelper = require('../deviceHelper'); +const log = require('../../lib/log'); +const { appsFolder } = require('../utils/appConstants'); +const executor = require('./volumeExecutor'); +const { sessionForMountedVolume } = require('./volumeSession'); +const fluxEventBus = require('../utils/fluxEventBus'); + +/** + * Reclaim what a FluxOS restart left behind from in-flight file operations. + * + * An operation's container is detached from the process that started it, so a + * restart leaves it running with nobody waiting for its exit code, and its + * staging directory sitting on the volume. Neither is visible at a destination + * path: a publish is one atomic exchange, so a destination always holds + * something complete and nothing the user can see is left inconsistent. This is + * about not accumulating debris. + * + * Runs at startup, after app volumes are mounted - but the API is already + * answering by then, so an operation of THIS process can be in flight when it + * runs. It removes only what no live operation owns: the executor records each + * running operation's container and staging directory, and reap and sweep skip + * those, so anything they reclaim belonged to a PREVIOUS process. Safe to run + * more than once for the same reason, which matters because a startup that throws + * is retried. + * + * @returns {Promise<{containers: number, removed: number}>} + */ +async function recoverInterruptedFileOperations() { + // Published on every path that ENDS the pass - the one that found nothing, + // and the one that threw. "The sweep ran and had nothing to do" is a + // different fact from "the sweep has not run yet", and the log line below + // cannot express the first because it only fires when there was something to + // report. Anything that restarts a node to exercise boot recovery needs to + // know the pass is over: without a signal it can only guess, and a pass that + // lands after the guess reaches into whatever is running by then. A throw + // still propagates - a startup that throws is retried, and the retry + // publishes again, which is safe for the same reason the sweep is. + let result = { containers: 0, removed: 0 }; + try { + result = await sweepEveryMountedVolume(); + } finally { + fluxEventBus.publish('fileops:recovered', result); + } + return result; +} + +async function sweepEveryMountedVolume() { + const containers = await executor.reapOrphanedContainers(); + + let mounts = []; + try { + mounts = await deviceHelper.listMountedFilesystems(); + } catch (error) { + log.error(`fileOperationRecovery - could not read the mount table: ${error.message}`); + return { containers, removed: 0 }; + } + + // Only mounted app volumes. A staging directory can only exist on one, and + // reading an unmounted mountpoint would walk the bare host directory + // underneath it instead. + const volumes = mounts.filter((mount) => mount.target.startsWith(appsFolder)); + + let removed = 0; + // eslint-disable-next-line no-restricted-syntax + for (const volume of volumes) { + try { + // eslint-disable-next-line no-await-in-loop + const result = await executor.sweepStagingDirectories(sessionForMountedVolume(volume)); + removed += result.removed.length; + } catch (error) { + // One unreadable volume must not strand the debris on every other app. + log.error(`fileOperationRecovery - could not sweep ${volume.target}: ${error.message}`); + } + } + + if (containers || removed) { + log.info(`fileOperationRecovery - reaped ${containers} container(s), removed ${removed} artefact(s)`); + } + return { containers, removed }; +} + +module.exports = { recoverInterruptedFileOperations }; diff --git a/ZelBack/src/services/appSystem/fileSystemManager.js b/ZelBack/src/services/appSystem/fileSystemManager.js index 7d80c49978..f1ef65be49 100644 --- a/ZelBack/src/services/appSystem/fileSystemManager.js +++ b/ZelBack/src/services/appSystem/fileSystemManager.js @@ -1,14 +1,181 @@ // File System Manager - Manages filesystem operations for FluxOS applications +// +// Every mutating endpoint here runs its work in a throwaway container with only +// the target app's volume mounted (see volumeExecutor), and reaches that volume +// through a VolumeSession (see volumeSession) rather than by building paths of +// its own. A handler that skips a check does not produce an unsafe endpoint - it +// produces code that does not run. +// +// The two download endpoints are the exception, and deliberately so. They are +// reads: opening a file has no side effects, so the file can be opened and the +// DESCRIPTOR checked before a byte is sent, which is a stronger guarantee than +// re-checking a name that may since have come to mean something else. A write +// commits the moment it opens, which is why an upload cannot be made safe that +// way and runs in the container like everything else. +// +// A LINK ON A VOLUME IS ORDINARY CONTENT, AND NOTHING HERE FOLLOWS ONE. An +// application has its own volume mounted and can create a link in it at any +// moment, pointing anywhere on the node - no endpoint is involved in that, so +// refusing links at an endpoint secures nothing. What holds is the reading side, +// and anything added later has to hold it too: the downloads open with +// O_NOFOLLOW (see utils/fileTransfer), every walk of a volume lstats and does +// not descend into a linked directory (see utils/treeSize, appQuery's listing, +// IOUtils.getPathFileList), and the folder download stores a link rather than +// what it points at. An extraction does not classify the links an archive +// carries either: what bounds an archive this node cannot vouch for is the +// container it is unpacked in, which mounts that one volume and nothing else. +// +// HOW A NAME COLLISION AT THE DESTINATION RESOLVES. `overwrite` is opt-in: a +// caller that does not pass it has a taken name refused ("Destination already +// exists") and confirms before retrying. What `overwrite` then allows depends on +// what the two entries ARE, and is identical on every endpoint because they share +// one publish (see volumeExecutor, and flux-op's publish for where it is decided): +// +// a file over a file replaced, atomically. +// a directory over a directory MERGED - the source is overlaid onto the +// destination, a name in both is overwritten, and +// everything the source does not name is kept. +// copy, move and extract do this; it is what cp -T, +// tar and unzip do. A directory is never replaced +// wholesale, which would delete what the caller +// never named but that sat beside what they did. +// a file over a directory, or refused. A file cannot stand in for a tree, and +// a directory over a file seating it there would delete the tree. +// the same entry under two refused. A symlink in the volume can make two +// names (via a symlink) paths name one file; an exchange would then move +// nothing and the cleanup would delete the file. +// +// Upload carries no overwrite flag - it has always meant replace-a-file - so it +// replaces a file, refuses a directory by the rule above, and creates when the +// name is free. Compress writes a single file, so its overwrite is a file-over- +// file replace. A merge is not atomic (it is a sequence of renames), which is the +// trade for overlaying rather than replacing an occupied directory. +const config = require('config'); const archiver = require('archiver'); const { PassThrough } = require('stream'); const path = require('path'); +const { formidable } = require('formidable'); const messageHelper = require('../messageHelper'); const verificationHelper = require('../verificationHelper'); const serviceHelper = require('../serviceHelper'); const IOUtils = require('../IOUtils'); -const fs = require('fs').promises; const log = require('../../lib/log'); -const { sanitizePath, verifyRealPath, verifyRealPathOfExistingPath } = require('../utils/pathSecurity'); +const { sanitizePath, verifyRealPath, validateFilename } = require('../utils/pathSecurity'); +const { openVolume, SPACE_HEADROOM } = require('./volumeSession'); +const { sendFile } = require('../utils/fileTransfer'); +const executor = require('./volumeExecutor'); +const jobRegistry = require('../utils/jobRegistry'); +const operationsController = require('../appManagement/operationsController'); +const { Privilege, authOf } = require('../utils/privileges'); + +/** + * Reclaim an upload's operation slot if the request stops sending bytes at the + * floor rate. + * + * The executor's rate floor governs a file part's BODY, reached only once a part + * begins streaming. The slot, though, is taken the instant the request arrives - + * so a request that sends no file part (or dribbles the multipart preamble) held + * the slot until server.requestTimeout (2h) with the body floor never engaging, + * and maxConcurrentPerNode such requests 503 every file operation for every app + * on the node. This applies the SAME floor to the whole request: each window, + * the parser must have received at least the floor's worth of bytes, or the slot + * is released. The window is the interval's own cadence, so nothing here depends + * on a clock the tests would have to fake. + * + * @param {object} params + * @param {number} params.minBitsPerSecond - the floor, in bits per second + * @param {number} params.windowMs - how often the floor is checked + * @param {() => number} params.getBytes - cumulative bytes the parser has received + * @param {() => void} params.onStall - called once, when a window falls short + * @returns {() => void} stop the watchdog + */ +function startSlotFloor({ + minBitsPerSecond, windowMs, getBytes, onStall, +}) { + if (!(minBitsPerSecond > 0) || !(windowMs > 0)) return () => {}; + const floorBytesPerWindow = (minBitsPerSecond / 8) * (windowMs / 1000); + let windowBytes = getBytes(); + const timer = setInterval(() => { + const seen = getBytes(); + if (seen - windowBytes < floorBytesPerWindow) { + onStall(); + return; + } + windowBytes = seen; + }, windowMs); + if (timer.unref) timer.unref(); + return () => clearInterval(timer); +} + +/** + * Report a failure. + * + * A refusal to START - the node or the app already has its allowance of + * concurrent operations - answers 503 with a Retry-After rather than a generic + * error, so a caller turned away learns it immediately instead of registering + * an operation and polling to discover it never began. + */ +function respondError(res, error) { + log.error(error); + const errorResponse = messageHelper.createErrorMessage( + error.message || error, + error.name, + error.code, + ); + if (error.kind === 'busy') { + if (error.retryAfterMs) { + res.setHeader('Retry-After', String(Math.ceil(error.retryAfterMs / 1000))); + } + // The operation being waited on travels in the body rather than only in the + // message, so a client can link to it, poll it or cancel it instead of + // retrying until it happens to succeed. + if (error.operation) { + errorResponse.data.operation = error.operation; + } + res.status(503).json(errorResponse); + return; + } + res.json(errorResponse); +} + +function respondSuccess(res, message) { + res.json(messageHelper.createSuccessMessage(message)); +} + +/** A required parameter, from the path, the query string or a JSON body. */ +function requiredParam(req, name) { + const body = serviceHelper.ensureObject(req.body) || {}; + const value = req.params[name] || req.query[name] || body[name]; + if (!value) throw new Error(`${name} parameter is mandatory`); + return value; +} + +/** + * WHICH SHAPE AN OPERATION ANSWERS IN + * + * An operation bounded by construction answers inline: a mkdir and a rename are + * one syscall whatever the tree holds, so making a caller poll for them is + * ceremony. An operation whose duration scales with the data registers a job + * and answers 202 - a copy, a move, a compression, an extraction. The rule is + * about the WORST case rather than the common one: `moveobject` is a job + * because overwriting does an unbounded rm of what it displaced, not because + * moving is usually slow. + * + * `removeobject` is a job by the same rule - `rm -rf` scales with the tree - + * but it answered inline before jobs existed, and two dashboards call it. So it + * keeps answering inline WHEN IT IS QUICK and becomes a job only when it + * outlives its deadline. + * + * THAT CLAUSE IS COMPATIBILITY, NOT DESIGN, AND HERE IS HOW TO KNOW IT CAN GO. + * The callers are fluxos-frontend's VolumeBrowser and the palworld dashboard's + * ModManager, and today neither polls: a delete answering 202 would be read as + * finished the moment it was accepted. Check them - if both poll a job through + * to a terminal state, the only callers still needing an inline answer are ones + * nobody here can see, and taking it away becomes a version boundary rather + * than a surprise. Then delete REMOVE_INLINE_DEADLINE_MS, pass no + * inlineDeadlineMs, and remove is a job like the others - along with the option + * itself, which exists for nothing else. + */ /** * To create a folder in app's volume. Only accessible by app owners and above. @@ -17,224 +184,128 @@ const { sanitizePath, verifyRealPath, verifyRealPathOfExistingPath } = require(' */ async function createAppsFolder(req, res) { try { - let { appname } = req.params; - appname = appname || req.query.appname || ''; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); - if (authorized) { - let { folder } = req.params; - folder = folder || req.query.folder || ''; - let { component } = req.params; - component = component || req.query.component || ''; - if (!appname || !component) { - throw new Error('appname and component parameters are mandatory'); - } - let filepath; - const appVolumePath = await IOUtils.getVolumeInfo(appname, component, 'B', 'mount', 0); - if (appVolumePath.length > 0) { - // Use appid level to access appdata and all other mount points - // Sanitize folder path to prevent directory traversal attacks - filepath = sanitizePath(folder, appVolumePath[0].mount); - // Verify resolved path stays within the allowed base directory - await verifyRealPathOfExistingPath(filepath, appVolumePath[0].mount); - } else { - throw new Error('Application volume not found'); - } - const mkdirResult = await serviceHelper.runCommand('mkdir', { runAsRoot: true, params: [filepath] }); - if (mkdirResult.error) { - throw mkdirResult.error; - } - const resultsResponse = messageHelper.createSuccessMessage('Folder Created'); - res.json(resultsResponse); - } else { - const errMessage = messageHelper.errUnauthorizedMessage(); - res.json(errMessage); - } + const volume = await openVolume(req); + const folder = requiredParam(req, 'folder'); + const target = await volume.resolve(folder); + + // No command: the folder is created as staging and published under the name + // the caller asked for, which is the publish every other operation here + // already uses. `mkdir` ran directly before, and the difference is what a + // failure can say - a command reports one by exiting non-zero, so "that name + // is taken" arrived as a status of 1 and a sentence worded by whichever + // build of mkdir the image carries. The dashboard shows an owner a different + // message for a name in use than for a folder that could not be made, and + // noReplace is what lets it tell the two apart. + const staging = volume.staging(); + await executor.run(volume, [], { + publish: { staging, destination: target }, + mkdirStaging: true, + noReplace: true, + }); + respondSuccess(res, 'Folder Created'); } catch (error) { - log.error(error); - const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); - res.json(errMessage); + respondError(res, error); } } /** - * To rename a file or folder. Oldpath is relative path to default fluxshare directory; newname is just a new name of folder/file. Only accessible by admins. + * To rename a file or folder WITHIN its current directory. + * + * Kept for existing clients. moveAppsObject is the general form and handles + * this case too; the difference is only that a new name here may not contain a + * path separator, which is why this endpoint could never move anything. + * * @param {object} req Request. * @param {object} res Response. */ async function renameAppsObject(req, res) { try { - let { appname } = req.params; - appname = appname || req.query.appname || ''; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); - if (authorized) { - let { oldpath } = req.params; - let { component } = req.params; - component = component || req.query.component || ''; - if (!appname || !component) { - throw new Error('appname and component parameters are mandatory'); - } - oldpath = oldpath || req.query.oldpath; - if (!oldpath) { - throw new Error('No file nor folder to rename specified'); - } - let { newname } = req.params; - newname = newname || req.query.newname; - if (!newname) { - throw new Error('No new name specified'); - } - if (newname.includes('/')) { - throw new Error('New name is invalid'); - } - // stop sharing of ALL files that start with the path - const fileURI = encodeURIComponent(oldpath); - let oldfullpath; - let newfullpath; - const appVolumePath = await IOUtils.getVolumeInfo(appname, component, 'B', 'mount', 0); - if (appVolumePath.length > 0) { - // Use appid level to access appdata and all other mount points - // Sanitize paths to prevent directory traversal attacks - oldfullpath = sanitizePath(oldpath, appVolumePath[0].mount); - newfullpath = sanitizePath(newname, appVolumePath[0].mount); - } else { - throw new Error('Application volume not found'); - } - const fileURIArray = fileURI.split('%2F'); - fileURIArray.pop(); - if (fileURIArray.length > 0) { - const renamingFolder = fileURIArray.join('/'); - // Sanitize the combined path as well - newfullpath = sanitizePath(`${renamingFolder}/${newname}`, appVolumePath[0].mount); - } - // Verify parent directories resolve within the allowed base directory to prevent symlink escapes. - await verifyRealPathOfExistingPath(path.dirname(oldfullpath), appVolumePath[0].mount); - await verifyRealPathOfExistingPath(path.dirname(newfullpath), appVolumePath[0].mount); + const volume = await openVolume(req); + const oldpath = req.params.oldpath || req.query.oldpath; + if (!oldpath) throw new Error('No file nor folder to rename specified'); + const newname = req.params.newname || req.query.newname; + if (!newname) throw new Error('No new name specified'); + if (newname.includes('/')) throw new Error('New name is invalid'); - // Allow renaming symlinks directly (mv renames the link itself, not the target). - // For non-symlink targets, enforce full real path containment. - let isSymbolicLink = false; - try { - const stats = await fs.lstat(oldfullpath); - isSymbolicLink = stats.isSymbolicLink(); - } catch (error) { - if (error.code !== 'ENOENT') { - throw error; - } - } - if (!isSymbolicLink) { - await verifyRealPath(oldfullpath, appVolumePath[0].mount); - } - const mvResult = await serviceHelper.runCommand('mv', { runAsRoot: true, params: ['-T', oldfullpath, newfullpath] }); - if (mvResult.error) { - throw mvResult.error; - } - const response = messageHelper.createSuccessMessage('Rename successful'); - res.json(response); - } else { - const errMessage = messageHelper.errUnauthorizedMessage(); - res.json(errMessage); - } + // The new name lands beside the old one, so the destination is built from + // the SOURCE's directory rather than from anything else the caller sent. + const destination = path.posix.join(path.posix.dirname(oldpath), newname); + + const { source, destination: target } = await volume.pair(oldpath, destination); + + // No command, and `source` rather than `staging`: a rename publishes the + // caller's own entry where it stands, so there is nothing to run and + // nothing a failure may throw away. + // + // Never overwrites, and takes no flag to say otherwise. Publishing over the + // destination exchanges the two entries and removes what was displaced, and + // that removal is unbounded - which is why moveAppsObject answers 202 and + // runs as a job. This endpoint answers inline, so allowing an overwrite + // would put an unbounded delete inside a held request. A caller that means + // to replace something uses moveAppsObject, which is the general form and + // handles this case too. + await executor.run(volume, [], { publish: { source, destination: target }, noReplace: true }); + respondSuccess(res, 'Rename successful'); } catch (error) { - log.error(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - try { - res.write(serviceHelper.ensureString(errorResponse)); - res.end(); - } catch (e) { - log.error(e); - } + respondError(res, error); } } /** - * To remove a specified shared file. Only accessible by admins. + * To remove a file or folder from an app's volume. Only accessible by app + * owners and above. * @param {object} req Request. * @param {object} res Response. */ +/** + * How long a remove may take before it becomes something to come back for. + * + * Deleting an ordinary folder is sub-second, so a caller written before jobs + * existed keeps getting the completed answer it has always had. A tree big + * enough to outlive this is the case where holding a request open was already + * wrong. + * + * Temporary: see the shape rule at the top of this file for what has to be true + * before this goes, and what goes with it. + */ +const REMOVE_INLINE_DEADLINE_MS = 10000; + async function removeAppsObject(req, res) { try { - let { appname } = req.params; - appname = appname || req.query.appname || ''; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); - if (authorized) { - let { object } = req.params; - object = object || req.query.object; - let { component } = req.params; - component = component || req.query.component || ''; - if (!component) { - throw new Error('component parameter is mandatory'); - } - if (!object) { - throw new Error('No object specified'); - } - let filepath; - const appVolumePath = await IOUtils.getVolumeInfo(appname, component, 'B', 'mount', 0); - if (appVolumePath.length > 0) { - // Use appid level to access appdata and all other mount points - // Sanitize object path to prevent directory traversal attacks - filepath = sanitizePath(object, appVolumePath[0].mount); - } else { - throw new Error('Application volume not found'); - } - // Verify parent directories resolve within the allowed base directory to prevent symlink escapes. - await verifyRealPathOfExistingPath(path.dirname(filepath), appVolumePath[0].mount); + const volume = await openVolume(req); + const object = requiredParam(req, 'object'); + // No mustExist: a delete is idempotent. rm -rf exits 0 on a path that is + // already gone, so removing something twice - a client retrying after a + // timeout, above all - answers success rather than "does not exist", which + // is how this behaved before it became a job. Containment and the + // reserved-name guard hold either way; only the existence requirement, + // which a delete does not need, is dropped. + const target = await volume.resolve(object); - // Allow removing symlinks directly (rm removes the link itself, not the target). - // For non-symlink targets (or symlinks in parent components), enforce real path containment. - let isSymbolicLink = false; - try { - const stats = await fs.lstat(filepath); - isSymbolicLink = stats.isSymbolicLink(); - } catch (error) { - if (error.code !== 'ENOENT') { - throw error; - } - } - if (!isSymbolicLink) { - // Verify resolved path stays within the allowed base directory - await verifyRealPathOfExistingPath(filepath, appVolumePath[0].mount); - } - const rmResult = await serviceHelper.runCommand('rm', { runAsRoot: true, params: ['-rf', filepath] }); - if (rmResult.error) { - throw rmResult.error; - } - const response = messageHelper.createSuccessMessage('File Removed'); - res.json(response); - } else { - const errMessage = messageHelper.errUnauthorizedMessage(); - res.json(errMessage); - } - } catch (error) { - log.error(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, + return startOperation( + res, + volume, + { kind: 'fileoperation.remove', status: 'Removing...', owner: volume.owner }, + (progress) => executor.run(volume, ['rm', '-rf', target], progress), + { inlineDeadlineMs: REMOVE_INLINE_DEADLINE_MS }, ); - try { - res.write(serviceHelper.ensureString(errorResponse)); - res.end(); - } catch (e) { - log.error(e); - } + } catch (error) { + return respondError(res, error); } } /** - * To download a zip folder for a specified directory. Only accessible by admins. + * To download a zip folder for a specified directory. The app owner or the flux team. * @param {object} req Request. * @param {object} res Response. - * @param {boolean} authorized False until verified as an admin. + * @param {boolean} authorized False until the caller is verified. * @returns {void} Return statement is only used here to interrupt the function and nothing is returned. */ async function downloadAppsFolder(req, res) { try { let { appname } = req.params; appname = appname || req.query.appname || ''; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (authorized) { let { folder } = req.params; folder = folder || req.query.folder; @@ -246,13 +317,13 @@ async function downloadAppsFolder(req, res) { return; } let folderpath; - const appVolumePath = await IOUtils.getVolumeInfo(appname, component, 'B', 'mount', 0); - if (appVolumePath.length > 0) { + const { mounts } = await IOUtils.getVolumeInfo(appname, component, 'B', 'mount', 0); + if (mounts.length > 0) { // Use appid level to access appdata and all other mount points // Sanitize folder path to prevent directory traversal attacks - folderpath = sanitizePath(folder, appVolumePath[0].mount); + folderpath = sanitizePath(folder, mounts[0].mount); // Verify real path after symlink resolution to prevent symlink escape attacks - await verifyRealPath(folderpath, appVolumePath[0].mount); + await verifyRealPath(folderpath, mounts[0].mount); } else { throw new Error('Application volume not found'); } @@ -300,7 +371,7 @@ async function downloadAppsFolder(req, res) { } /** - * To download a specified file. Only accessible by admins. + * To download a specified file. The app owner or the flux team. * @param {object} req Request. * @param {object} res Response. * @returns {void} Return statement is only used here to interrupt the function and nothing is returned. @@ -309,7 +380,7 @@ async function downloadAppsFile(req, res) { try { let { appname } = req.params; appname = appname || req.query.appname || ''; - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (authorized) { let { file } = req.params; file = file || req.query.file; @@ -321,24 +392,18 @@ async function downloadAppsFile(req, res) { return; } let filepath; - const appVolumePath = await IOUtils.getVolumeInfo(appname, component, 'B', 'mount', 0); - if (appVolumePath.length > 0) { + const { mounts } = await IOUtils.getVolumeInfo(appname, component, 'B', 'mount', 0); + if (mounts.length > 0) { // Use appid level to access appdata and all other mount points // Sanitize file path to prevent directory traversal attacks - filepath = sanitizePath(file, appVolumePath[0].mount); + filepath = sanitizePath(file, mounts[0].mount); // Verify real path after symlink resolution to prevent symlink escape attacks - await verifyRealPath(filepath, appVolumePath[0].mount); + await verifyRealPath(filepath, mounts[0].mount); } else { throw new Error('Application volume not found'); } - const chmodResult = await serviceHelper.runCommand('chmod', { runAsRoot: true, params: ['777', filepath] }); - if (chmodResult.error) { - throw chmodResult.error; - } - // beautify name - const fileNameArray = filepath.split('/'); - const fileName = fileNameArray[fileNameArray.length - 1]; - res.download(filepath, fileName, { dotfiles: 'allow' }); + const fileName = path.basename(filepath); + await sendFile(res, filepath, fileName); } else { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -359,10 +424,641 @@ async function downloadAppsFile(req, res) { } } +/** + * Read the operands the two-operand endpoints share. + * + * `destination` is the full target path INCLUDING the new name, not the parent + * directory. That keeps -T semantics identical between copy and move and + * removes the "paste into this directory" versus "paste as this name" + * ambiguity - the client appends the basename itself. + * + * `overwrite` is opt-in. A client calls without it, and turns the resulting + * 'Destination already exists' into a confirmation before retrying with it. + */ +async function resolveOperands(req, volume) { + const source = requiredParam(req, 'source'); + const destination = requiredParam(req, 'destination'); + // A real boolean from a JSON body, or the string a form-encoded caller sends. + // Anything else is false: overwrite has to be asked for, so an unparseable + // value must not be read as consent to destroy something. + const raw = serviceHelper.ensureObject(req.body)?.overwrite ?? req.query.overwrite; + const overwrite = raw === true || raw === 'true'; + const pair = await volume.pair(source, destination); + + // Carried to the publish rather than settled here. What "the destination is + // taken" means is decided by the rename that acts on it, in one step, on the + // volume of an application that is writing to it the whole time - a look taken + // now answers for a moment that has passed by the time the container runs. + return { ...pair, noReplace: !overwrite }; +} + +/** + * Start a file operation and answer 202 with a job to poll. + * + * All four long operations go through here, move included. Its visible part is + * a rename, but paste is ONE gesture in a file browser: cut-paste returning a + * result while copy-paste returned a job would put two response shapes inside a + * single user action, and the way that gets misread is treating the 202 body's + * status: 'Running' as terminal success. A move also does an unbounded rm -rf + * of the displaced copy when it overwrites, so "instant" was a claim about the + * common case rather than about the operation. + * + * The job is registered LAST, after every synchronous refusal has been decided, + * so a caller turned away never sees an operation that existed briefly and then + * failed for a reason it could have been told up front. + * + * @param {object} res + * @param {VolumeSession} volume + * @param {{kind: string, status: string, owner: string|null, + * trackBytes?: boolean, bytesTotal?: number}} meta - `trackBytes` for an + * operation that WRITES into staging, so its size means progress; `bytesTotal` + * only where a denominator is genuinely knowable + * @param {function(object): Promise} work - receives the executor options + * carrying progress, cancellation and byte reporting, and runs the operation + * @param {{inlineDeadlineMs?: number}} [options] - answer inline if the work + * finishes within this, rather than 202 immediately. Compatibility for an + * operation that answered inline before jobs existed, and temporary: the + * shape rule at the top of this file records what removes it + */ +function startOperation(res, volume, meta, work, { inlineDeadlineMs = 0 } = {}) { + // Before the job exists, so a caller with no slot gets 503 + Retry-After now + // rather than an operation that is only ever going to report that it never + // started. + executor.assertCapacity(volume); + + // detail() is read when a client polls, so anything here costs nothing while + // nobody is watching - which is also why the byte figures belong here rather + // than in `progress`, which is append-only and returned whole. + // + // bytesDone is read from the STAGING path, not from the copying process. No + // counter on the source side sees the bytes: since coreutils 9.0 `cp` uses + // copy_file_range(2) and the kernel moves them, so /proc//io stays flat + // and the fdinfo offset does not advance until the end. What grows the whole + // way through is the destination inode. + // + // A denominator is only offered where one is real. A copy knows it, from the + // measurement the capacity check already made. An extraction does not: the + // only figure available is the archive's own declared uncompressed size, + // which is written by whoever built the archive and is exactly what the size + // ceiling refuses to believe. Nor does a compression, whose ratio is not + // knowable in advance. Those two report bytes written and no percentage, + // rather than a percentage derived from a number that can lie. + let bytesDone = null; + const handle = jobRegistry.start({ + kind: meta.kind, + owner: meta.owner, + detail: () => ({ + app: volume.identifier, + operation: meta.kind, + // Capped at the total while one is known. The running figure is what the + // whole volume has consumed, and the application is writing to it too, so + // its own activity would otherwise push a copy past 100% - which reads as + // a broken bar rather than as the estimate it is. The figure a completed + // operation reports is measured from what it published, so the cap only + // ever applies mid-flight. + ...(bytesDone === null ? {} : { + bytesDone: meta.bytesTotal === undefined ? bytesDone : Math.min(bytesDone, meta.bytesTotal), + }), + ...(meta.bytesTotal === undefined ? {} : { bytesTotal: meta.bytesTotal }), + }), + }); + + // Deliberately not awaited: the response goes back now and the work reports + // itself into the registry. + // + // Wrapped in a resolved promise so a SYNCHRONOUS throw from work() settles the + // job too. Without it such a throw escapes past these handlers, and the job it + // left behind stays Running - which never expires, because only terminal jobs + // are retained on a clock. + const running = Promise.resolve() + .then(() => work({ + status: meta.status, + onProgress: (message) => jobRegistry.progress(handle.jobId, message), + isCanceled: () => jobRegistry.isCanceled(handle.jobId), + ...(meta.trackBytes ? { onBytes: (bytes) => { bytesDone = bytes; } } : {}), + })) + // Succeeded even if a cancel was asked for. Reaching here means the command + // exited 0, which means it published - so the cancel lost the race, and + // saying Canceled would tell the caller nothing happened while their + // destination has in fact been replaced. For a move it would be worse + // still: the source is gone, and the answer says it was not touched. + // Cancellation is cooperative, so "we stopped if we could" is the promise, + // and a stop that arrived too late is not a stop. + .then(() => jobRegistry.succeed(handle.jobId)) + // A cancel that DID take effect lands here instead: flux-op traps the + // signal and exits 143, so the executor throws rather than resolving. + .catch((error) => { + if (jobRegistry.isCanceled(handle.jobId)) jobRegistry.cancelled(handle.jobId); + else jobRegistry.fail(handle.jobId, error); + }); + + if (!inlineDeadlineMs) return operationsController.accepted(res, handle); + + // An operation that used to answer inline still does when it is quick, so a + // client written before jobs existed is never worse off: a delete of an + // ordinary folder is sub-second and answers 200 exactly as it always has. + // Only one that outlives the deadline becomes something to come back for - + // and that is the case where holding the request was already wrong, since an + // unbounded one is held open until an intermediate proxy kills it. + return Promise.race([running.then(() => true), serviceHelper.delay(inlineDeadlineMs)]) + .then((finished) => (finished + ? operationsController.completed(res, handle, meta.owner) + : operationsController.accepted(res, handle))); +} + +/** + * Which archive tool handles this name, or null if we do not handle it. + * + * @param {string} name + * @returns {'zip'|'tar'|null} + */ +function archiveFormat(name) { + // Case-insensitive: an extension is a convention, not an identifier, and + // plenty of software writes BACKUP.ZIP. Refusing to extract one with a + // message listing the extension it plainly has reads as a broken endpoint. + const lower = name.toLowerCase(); + if (lower.endsWith('.zip')) return 'zip'; + if (lower.endsWith('.tar.gz') || lower.endsWith('.tgz')) return 'tar'; + return null; +} + +/** + * Move a file or folder anywhere within the app's volume. + * + * This is what renameAppsObject could never do: that one rejects any new name + * containing a path separator, so it can rename in place but not relocate. + * + * No capacity check - a rename within one filesystem moves no bytes. + * + * @param {object} req Request. + * @param {object} res Response. + */ +async function moveAppsObject(req, res) { + try { + const volume = await openVolume(req); + const { source, destination, noReplace } = await resolveOperands(req, volume); + + // No command: the source IS the result, so publishing it is the whole + // operation. Going through publish rather than a bare `mv` is what handles + // an existing destination - rename(2) refuses a non-empty directory target + // and cannot replace a file with a directory at all. + return startOperation(res, volume, { kind: 'fileoperation.move', status: 'Moving...', owner: volume.owner }, (progress) => executor.run(volume, [], { ...progress, publish: { source, destination }, noReplace, merge: true })); + } catch (error) { + respondError(res, error); + } +} + +/** + * Copy a file or folder within the app's volume. + * + * @param {object} req Request. + * @param {object} res Response. + */ +async function copyAppsObject(req, res) { + try { + const volume = await openVolume(req); + const { source, destination, noReplace } = await resolveOperands(req, volume); + + // The same measurement serves both the capacity check and the progress + // denominator - a copy writes as many bytes as it reads, so the figure the + // one needs is exactly the figure the other reports against. + const bytesTotal = await volume.measure(source); + volume.requireSpace(bytesTotal); + + const staging = volume.staging(); + // -a preserves ownership, timestamps and symlinks and implies -r; -T stops + // cp copying INTO the staging directory instead of becoming it. + return startOperation(res, volume, { + kind: 'fileoperation.copy', status: 'Copying...', owner: volume.owner, trackBytes: true, bytesTotal, + }, (progress) => executor.run(volume, ['cp', '-a', '-T', source, staging], { + ...progress, + publish: { staging, destination }, + noReplace, + // A directory copied onto an existing directory overlays it rather than + // replacing it wholesale; cp -T merges the same way. A file over a file is + // still replaced, and a file over a directory is refused. + merge: true, + // The measurement above is what refuses this early and with a sentence. + // It is not what makes it safe: it is taken by the FluxOS process, which + // is root on ArcaneOS but an ordinary user elsewhere, and a directory the + // app made private is one it cannot open. measureTree skips what it + // cannot read, so the figure can be low - silently, and in the direction + // that admits a copy which does not fit. The ceiling is applied to what + // actually lands, by the container, which can read all of it. + maxBytes: volume.availableBytes / SPACE_HEADROOM, + })); + } catch (error) { + respondError(res, error); + } +} + +/** + * Archive a file or folder, leaving the archive on the volume. + * + * Unlike downloadAppsFolder, which zips only to stream the result to the + * browser, this produces an archive the app keeps - the thing you want before a + * risky upgrade. + * + * @param {object} req Request. + * @param {object} res Response. + */ +async function compressAppsObject(req, res) { + try { + const volume = await openVolume(req); + const { source, destination, noReplace } = await resolveOperands(req, volume); + + const format = archiveFormat(destination.relative); + if (!format) { + throw new Error('Destination must end in .zip, .tar.gz or .tgz'); + } + + // The archive cannot be larger than what goes into it by enough to matter, + // and compressed output is normally far smaller - so the source size is a + // safe over-estimate rather than a guess. + volume.requireSpace(await volume.measure(source)); + + // An archive holds the source's CONTENTS at its top level, so extracting it + // to a destination reproduces the source under that name - compress then + // extract returns what went in. + // + // Both archivers decide their layout from where they run and what they are + // handed, and neither infers anything useful from an absolute operand: zip + // stores the whole path minus its leading slash, which would put an + // internal mount directory the user has never seen inside their archive, + // and tar's -C cannot be pointed at a file at all. Running in the right + // directory and passing a bare operand is what makes the two agree, and is + // the only form that works for a single file. + const sourceIsDirectory = await volume.isDirectory(source); + const workingDir = sourceIsDirectory ? source : volume.parent(source); + const operand = sourceIsDirectory ? '.' : path.basename(source.relative); + + // The archive goes inside a minted DIRECTORY rather than at the root, + // because the tool's scratch follows its output: Info-ZIP builds the + // archive in a temp file in the output's directory, and at the volume root + // that temp sat outside the shape the sweep may delete. In here, the temp, + // a partial archive and the entry are one reclaim. The executor creates + // the directory and reclaims it whole. + const { entry: staging } = volume.stagingDir(destination); + // `--` before the operand, because a name is not an option. A file may + // legitimately begin with a dash - the component rule rejects only the + // separators and the control characters - and both archivers would read one + // as a flag and refuse the request. Ending option parsing is what makes the + // operand a filename whatever it starts with, and unlike a `./` prefix it + // leaves the name stored in the archive alone. + const argv = format === 'zip' + // -r recurses, -q keeps the per-file listing out of the container's + // output, -y stores a symlink as a symlink instead of the file it points + // at, which is what tar and cp -a already do. + ? ['zip', '-r', '-q', '-y', staging, '--', operand] + : ['tar', '-czf', staging, '--', operand]; + + // Bytes written to the archive, with no total: how far a source of a known + // size compresses is not knowable until it has. + return startOperation(res, volume, { + kind: 'fileoperation.compress', status: 'Compressing...', owner: volume.owner, trackBytes: true, + }, (progress) => executor.run(volume, argv, { + ...progress, + workingDir, + publish: { staging, destination }, + noReplace, + // As for copy: the measurement above refuses this early, the ceiling is + // what makes it safe. A source measured by a process that cannot open + // every directory in it reads low, and an archive is written by one that + // can read all of them. + maxBytes: volume.availableBytes / SPACE_HEADROOM, + })); + } catch (error) { + respondError(res, error); + } +} + +/** + * Unpack an archive already on the volume into a directory. + * + * The only endpoint here whose CONTENT is attacker-supplied, so it is the one + * the container configuration matters most for. Three things bound what a hostile archive + * can do, none of which depend on inspecting it first: + * + * a member named ../../etc/passwd resolves inside a container whose rootfs is + * read-only and which is discarded either way; + * + * --no-same-owner ignores the uids the archive claims, and + * --no-same-permissions its modes, so it cannot plant a setuid binary - and + * the volume is mounted nosuid, so one would be inert even if it did; + * + * a zip bomb fills the app's own volume, which is a fixed-size loop file, so + * it cannot reach the host disk. + * + * @param {object} req Request. + * @param {object} res Response. + */ +async function extractAppsObject(req, res) { + try { + const volume = await openVolume(req); + const { source, destination, noReplace } = await resolveOperands(req, volume); + + // Refused by extension rather than by sniffing the content: a caller who + // has to name what they uploaded cannot have it interpreted as something + // else. + const format = archiveFormat(source.relative); + if (!format) { + throw new Error('Source must be a .zip, .tar.gz or .tgz archive'); + } + + // How much this will write cannot be measured up front, so the ceiling below + // is the only bound - and on a full volume that ceiling is zero, which is + // how "no ceiling" is spelled. Refused here rather than expressed as a limit + // nothing enforces. + volume.requireCapacity(); + + const staging = volume.staging(); + const argv = format === 'zip' + ? ['unzip', '-q', source, '-d', staging] + : ['tar', '-xzf', source, '-C', staging, '--no-same-owner', '--no-same-permissions']; + + // Bytes unpacked so far, with no total: the only figure that could serve as + // one is the archive's own account of itself, which is precisely what the + // ceiling below refuses to take on trust. + return startOperation(res, volume, { + kind: 'fileoperation.extract', status: 'Extracting...', owner: volume.owner, trackBytes: true, + }, (progress) => executor.run(volume, argv, { + ...progress, + publish: { staging, destination }, + noReplace, + // Extracting over an existing folder overlays it rather than replacing it + // wholesale, which is what every archiver and untarFile already do - a + // three-file patch over mods/ keeps the rest of mods/. + merge: true, + // tar -C and unzip -d both need the directory to exist already. + mkdirStaging: true, + // The capacity check the other operations make up front cannot be made + // here: an archive's declared uncompressed size is written by whoever + // built it, so a bomb simply understates itself. The ceiling is applied + // to what actually lands instead, and it is the free space on the volume, + // so an extraction can fill what is available and no more. + maxBytes: volume.availableBytes / SPACE_HEADROOM, + // A FIFO, socket or device node in the result is refused: none of them is + // data, and whatever opens a FIFO without O_NONBLOCK waits for a writer + // that is never coming, so one published here is a reader that hangs. tar + // both carries and recreates a FIFO, so an archive is all it takes. + // + // Links pass. What bounds an archive this node cannot vouch for is the + // container it is unpacked in - one volume mounted, a read-only rootfs - + // and what bounds a link left in the result is the reader: every walk of a + // volume here lstats, and the downloads open with O_NOFOLLOW. + dataOnly: true, + })); + } catch (error) { + respondError(res, error); + } +} + +/** + * Where an upload's files land, relative to the volume root. + * + * The restore flow uploads an archive to a fixed place; everything else lands + * where the file browser is pointed, which may be the volume root. + */ +function uploadFolder(req) { + const type = req.params.type || req.query.type || ''; + if (type === 'backup') return 'backup/upload'; + return req.params.folder || req.query.folder || ''; +} + +/** + * Receive uploaded files onto an app's volume. + * + * The bytes go from the request straight into a container that writes them, so + * they never touch the node's own filesystem. That is the whole reason this + * moved: a write commits the moment it opens - it creates or truncates whatever + * the name pointed at - so checking a path first and writing to it afterwards + * cannot be made safe, and node has no way to say "open this only if it is + * inside that directory". The container has nothing else mounted, so the + * question does not arise. + * + * Each file is published atomically on its own, so one that fails leaves the + * others alone and leaves nothing half-written at its destination. + * + * The request holds ONE operation slot however many files it carries, and the + * files are handled one at a time inside it. A slot per file would refuse the + * second one; a slot per file with no serialisation would put an unbounded + * number of containers on the node for a single request. + * + * The response is a stream of progress figures, then each file's name as it + * lands, and its shape is unchanged - a client reads bytes received against + * bytes expected while the upload runs. A failure is written into it as the + * standard error envelope, because by then the status line has long gone. + * + * @param {object} req Request. + * @param {object} res Response. + */ +async function uploadAppsFiles(req, res) { + const folder = uploadFolder(req); + let volume = null; + let releaseSlot = null; + + const fail = (error) => { + log.error(error); + const envelope = messageHelper.createErrorMessage(error.message || error, error.name, error.code); + // Before anything has been written the status line is still ours, so a + // refusal can be answered as one. Once the body has started it cannot, and + // the envelope goes into the stream where a client parses it out. + if (res.headersSent) { + try { + res.write(serviceHelper.ensureString(envelope)); + res.end(); + } catch (writeError) { + log.error(writeError); + } + return; + } + respondError(res, error); + }; + + try { + volume = await openVolume(req); + // Resolved once, before anything is received, so a folder that resolves + // outside the volume is refused while the caller can still be told. + const target = await volume.resolve(folder, { allowRoot: true }); + + // Before a byte is read. What arrives is bounded only by the ceiling below, + // and on a full volume that ceiling is zero - which is how "no ceiling" is + // spelled, so the upload would run unbounded until a write failed. + volume.requireCapacity(); + + // One slot for the request. Taken before the first byte is read, so a + // caller with no slot is refused with a 503 and a Retry-After rather than + // after uploading a gigabyte. + releaseSlot = executor.acquireSlot(volume.identifier); + + // Created if it is not there. The restore flow uploads into backup/upload + // on volumes that have never held a restore, so an upload has always + // brought its own destination into existence. + const present = await volume.isDirectory(target).catch(() => null); + if (present === false) { + throw new Error('Upload destination is not a folder'); + } + if (present === null) { + await executor.run(volume, ['mkdir', '-p', target], { slotHeld: true }); + } + } catch (error) { + if (releaseSlot) releaseSlot(); + fail(error); + return; + } + + // The most that may be written, from the volume itself rather than a figure + // chosen here. flux-op enforces the same number as the bytes arrive, so an + // upload that would fill the volume is refused at the limit instead of + // filling it and being refused afterwards. + const ceiling = Math.floor(volume.availableBytes / SPACE_HEADROOM); + + // Files are handled strictly in turn. A multipart body delivers its parts in + // order anyway; this makes the operations follow them, so a request never has + // two containers of its own running at once. + let queue = Promise.resolve(); + let failure = null; + // The stream currently being received, so a request that fails or goes away + // can settle the operation waiting on it rather than leaving a container + // holding an input nothing will ever close. + let receiving = null; + + const receiveOne = async (file, incoming) => { + if (failure) { + // Drained rather than left: formidable is writing into it, and a stream + // nobody reads holds the request open behind a file that is not going to + // be stored. + incoming.resume(); + return; + } + receiving = incoming; + try { + const name = validateFilename(uploadNames.get(file)); + const destination = await volume.resolve(path.posix.join(folder, name)); + const staging = volume.staging(); + + await executor.run(volume, [], { + input: incoming, + publish: { staging, destination }, + maxBytes: ceiling, + slotHeld: true, + status: 'Uploading...', + }); + + res.write(serviceHelper.ensureString(name)); + if (res.flush) res.flush(); + } catch (error) { + failure = error; + incoming.resume(); + } finally { + receiving = null; + } + }; + + const form = formidable({ + multiples: true, + hashAlgorithm: false, + // No parser limit. The container is the only ceiling, and it is the only + // one that can be exact: it refuses AS the bytes arrive, where a parser + // that gives up mid-request leaves the caller a broken connection instead + // of a reason. What may be written is capped either way. + maxFileSize: Infinity, + fileWriteStreamHandler: (file) => { + const incoming = new PassThrough(); + // Nothing reads this until its turn comes, so formidable is held at the + // buffer's own high-water mark rather than racing ahead of a container + // that does not exist yet. + queue = queue.then(() => receiveOne(file, incoming)); + return incoming; + }, + }); + + // An explicit filename parameter wins over the form field name, which is what + // the restore flow relies on to name the archive it uploads. Read here rather + // than from the file object because formidable does not put the field name on + // one - fileBegin is the only place it is available, and it fires immediately + // before the write stream is asked for. + const requestedFilename = req.params.filename || req.query.filename || ''; + const uploadNames = new WeakMap(); + + // Reached from three directions - the body ended, the parser gave up, or the + // client went away - and it has to run exactly once from whichever arrives + // first. Missing one of them leaks the app's operation slot for as long as + // FluxOS runs, which takes that app's whole file browser with it. + let settled = false; + // Cumulative bytes the parser has received, and the floor watching them. A + // held slot must always see bytes at the floor rate - see startSlotFloor. + let bytesReceivedTotal = 0; + let stopSlotFloor = () => {}; + const finish = () => { + if (settled) return; + settled = true; + stopSlotFloor(); + // After the queue, not before: the last container is still running long + // after the parser has finished with the request. + queue + .then(() => { + releaseSlot(); + if (failure) fail(failure); + else res.end(); + }) + .catch((error) => { + releaseSlot(); + fail(error); + }); + }; + + const abandon = (error) => { + failure = failure || error; + // Settles whatever operation is waiting on this stream. Without it the + // container sits on an input that will never close, until the stall check + // notices minutes later. + if (receiving) receiving.destroy(error); + finish(); + }; + + form + .on('fileBegin', (name, file) => { + uploadNames.set(file, requestedFilename || name); + }) + .on('progress', (bytesReceived, bytesExpected) => { + bytesReceivedTotal = bytesReceived; + try { + res.write(serviceHelper.ensureString([bytesReceived, bytesExpected])); + if (res.flush) res.flush(); + } catch (error) { + log.error(error); + } + }) + .on('error', (error) => abandon(error)) + .on('end', finish); + + req.on('aborted', () => abandon(new Error('The upload did not complete'))); + + // The floor covers the whole request, not just a part body: the executor's + // floor never sees a request that sends no file part, and that request held + // the slot until the 2h request timeout. + const { minUploadBitsPerSecond, stallTimeoutMs } = config.fluxapps.volumeOperations; + stopSlotFloor = startSlotFloor({ + minBitsPerSecond: minUploadBitsPerSecond, + windowMs: stallTimeoutMs, + getBytes: () => bytesReceivedTotal, + onStall: () => abandon(new Error(`The upload held a slot without sending the ${minUploadBitsPerSecond} bit/s a transfer has to keep`)), + }); + + form.parse(req); +} + module.exports = { createAppsFolder, renameAppsObject, removeAppsObject, + uploadAppsFiles, + moveAppsObject, + copyAppsObject, + compressAppsObject, + extractAppsObject, downloadAppsFolder, downloadAppsFile, + startSlotFloor, }; diff --git a/ZelBack/src/services/appSystem/networkRecovery.js b/ZelBack/src/services/appSystem/networkRecovery.js new file mode 100644 index 0000000000..0f4883ec1f --- /dev/null +++ b/ZelBack/src/services/appSystem/networkRecovery.js @@ -0,0 +1,52 @@ +const dockerService = require('../dockerService'); +const appQueryService = require('../appQuery/appQueryService'); +const log = require('../../lib/log'); + +/** + * Remove app networks no installed app accounts for. + * + * A network is created per app and removed by the uninstaller, and by nothing + * else. An uninstall interrupted between the container going and the network + * going - a reboot, a crash, a removal that failed both its retries - leaves + * one behind for ever, because nothing looks again. + * + * Each holds an explicitly assigned `172.23..0/24`, and the octet is + * taken from a walk of 1..255 for one nothing is using. A leaked network keeps + * its octet permanently, so the cost is not untidiness: when the last octet is + * gone, no app can be installed on the node again. + * + * Runs at startup, before anything installs. A sweep must never meet an install + * in progress - there is a moment in one where the network exists and the + * database record does not - and at boot no such moment exists. + * + * The names are built from the app records rather than parsed back out of the + * networks: an app name can only be recovered from `fluxDockerNetwork_` + * by assuming what a name may contain, and being wrong there removes a live + * app's network. + * + * A list that cannot be read means every network looks unowned, so nothing is + * removed rather than everything. + * + * @returns {Promise} the networks reclaimed + */ +async function reclaimOrphanedAppNetworks() { + try { + const installed = await appQueryService.installedApps(); + if (!installed || installed.status !== 'success' || !Array.isArray(installed.data)) { + log.warn('networkRecovery - the installed app list could not be read; no network was reclaimed'); + return []; + } + + const expected = new Set(installed.data.map((app) => `fluxDockerNetwork_${app.name}`)); + const reclaimed = await dockerService.reclaimAppNetworks(expected); + if (reclaimed.length) { + log.info(`networkRecovery - reclaimed ${reclaimed.length} app network(s) no installed app owns: ${reclaimed.join(', ')}`); + } + return reclaimed; + } catch (error) { + log.error(`networkRecovery - could not reclaim app networks: ${error.message}`); + return []; + } +} + +module.exports = { reclaimOrphanedAppNetworks }; diff --git a/ZelBack/src/services/appSystem/systemIntegration.js b/ZelBack/src/services/appSystem/systemIntegration.js index fb6e343d75..399731ea02 100644 --- a/ZelBack/src/services/appSystem/systemIntegration.js +++ b/ZelBack/src/services/appSystem/systemIntegration.js @@ -1,75 +1,18 @@ -const os = require('os'); -const config = require('config'); +// What the app layer asks of the host system, where the answer is not a +// requirements check. +// +// The requirements checks live in appRequirements/hwRequirements - hardware, +// static ip, datacenter, nodes and geolocation - and are reached through +// appInstaller.checkAppRequirements. Parameter validation lives in +// appRequirements/appValidator, and the app monitoring lifecycle in +// appMonitoring/monitoringOrchestrator. + const log = require('../../lib/log'); const messageHelper = require('../messageHelper'); -// eslint-disable-next-line no-unused-vars -const serviceHelper = require('../serviceHelper'); const verificationHelper = require('../verificationHelper'); const dockerService = require('../dockerService'); -// eslint-disable-next-line no-unused-vars -const daemonServiceFluxnodeRpcs = require('../daemonService/daemonServiceFluxnodeRpcs'); -const fluxNetworkHelper = require('../fluxNetworkHelper'); const benchmarkService = require('../benchmarkService'); -const { socketAddressesMatch } = require('../utils/socketAddressUtils'); -const hwRequirements = require('../appRequirements/hwRequirements'); -const daemonServiceBenchmarkRpcs = require('../daemonService/daemonServiceBenchmarkRpcs'); -const generalService = require('../generalService'); - -// Node specifications cache -const nodeSpecs = { - cpuCores: 0, - ram: 0, - ssdStorage: 0, -}; - -/** - * Get node specifications (CPU, RAM, Storage) and cache them - * @returns {Promise} - */ -async function getNodeSpecs() { - try { - if (nodeSpecs.cpuCores === 0) { - nodeSpecs.cpuCores = os.cpus().length; - } - if (nodeSpecs.ram === 0) { - nodeSpecs.ram = os.totalmem() / 1024 / 1024; // Convert to MB - } - if (nodeSpecs.ssdStorage === 0) { - // get my external IP and check that it is longer than 5 in length. - const benchmarkResponse = await daemonServiceBenchmarkRpcs.getBenchmarks(); - if (benchmarkResponse.status === 'success') { - const benchmarkResponseData = JSON.parse(benchmarkResponse.data); - log.info(`Gathered ssdstorage ${benchmarkResponseData.ssd}`); - nodeSpecs.ssdStorage = benchmarkResponseData.ssd; - } else { - throw new Error('Error getting ssdstorage from benchmarks'); - } - } - } catch (error) { - log.error(error); - } -} - -/** - * Set node specifications manually - * @param {number} cores - Number of CPU cores - * @param {number} ram - RAM in MB - * @param {number} ssdStorage - SSD storage in GB - */ -function setNodeSpecs(cores, ram, ssdStorage) { - nodeSpecs.cpuCores = cores || nodeSpecs.cpuCores; - nodeSpecs.ram = ram || nodeSpecs.ram; - nodeSpecs.ssdStorage = ssdStorage || nodeSpecs.ssdStorage; - log.info(`Node specs updated: CPU: ${nodeSpecs.cpuCores}, RAM: ${nodeSpecs.ram}MB, SSD: ${nodeSpecs.ssdStorage}GB`); -} - -/** - * Return current node specifications - * @returns {object} Current node specs - */ -function returnNodeSpecs() { - return { ...nodeSpecs }; -} +const { Privilege, authOf } = require('../utils/privileges'); /** * To get system architecture type (ARM64 or AMD64). @@ -84,118 +27,6 @@ async function systemArchitecture() { return benchmarkBenchRes.data.architecture; } -/** - * To check app requirements of staticip restrictions for a node - * @param {object} appSpecs App specifications. - * @returns {boolean} True if all checks passed. - */ -function checkAppStaticIpRequirements(appSpecs) { - if (appSpecs.version >= 7 && appSpecs.staticip) { - // Import locally to avoid circular dependency - // eslint-disable-next-line global-require - const geolocationService = require('../geolocationService'); - const isMyNodeStaticIP = geolocationService.isStaticIP(); - if (isMyNodeStaticIP !== appSpecs.staticip) { - throw new Error(`Application ${appSpecs.name} requires static IP address to run. Aborting.`); - } - } - return true; -} - -/** - * To check app requirements of datacenter restrictions for a node - * @param {object} appSpecs App specifications. - * @returns {boolean} True if all checks passed. - */ -function checkAppDataCenterRequirements(appSpecs) { - if (appSpecs.version >= 8 && appSpecs.datacenter === true) { - // Import locally to avoid circular dependency - // eslint-disable-next-line global-require - const geolocationService = require('../geolocationService'); - const isMyNodeDataCenter = geolocationService.isDataCenter(); - if (!isMyNodeDataCenter) { - throw new Error(`Application ${appSpecs.name} requires data center node to run. Aborting.`); - } - } - return true; -} - -/** - * To check app satisfaction of nodes restrictions for a node - * @param {object} appSpecs App specifications. - * @returns {boolean} True if all checks passed. - */ -async function checkAppNodesRequirements(appSpecs) { - if (appSpecs.version === 7 && appSpecs.nodes && appSpecs.nodes.length) { - const myCollateral = await generalService.obtainNodeCollateralInformation(); - const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); - if (!localSocketAddr) { - throw new Error('Unable to detect Flux IP address'); - } - - if (appSpecs.nodes.find((node) => socketAddressesMatch(node, localSocketAddr)) || appSpecs.nodes.includes(`${myCollateral.txhash}:${myCollateral.txindex}`)) { - return true; - } - throw new Error(`Application ${appSpecs.name} is not allowed to run on this node. Aborting.`); - } - - return true; -} - -/** - * To check app requirements of geolocation restrictions for a node - * @param {object} appSpecs App specifications. - * @returns {boolean} True if all checks passed. - */ -async function checkAppGeolocationRequirements(appSpecs) { - if (appSpecs.version >= 5 && appSpecs.geolocation && appSpecs.geolocation.length > 0) { - // Import locally to avoid circular dependency - // eslint-disable-next-line global-require - const geolocationService = require('../geolocationService'); - const nodeGeo = await geolocationService.getNodeGeolocation(); - if (!nodeGeo) { - throw new Error('Node Geolocation not set. Aborting.'); - } - // previous geolocation specification version (a, b) [aEU, bFR] - // current geolocation style [acEU], [acEU_CZ], [acEU_CZ_PRG], [a!cEU], [a!cEU_CZ], [a!cEU_CZ_PRG] - const appContinent = appSpecs.geolocation.find((x) => x.startsWith('a')); - const appCountry = appSpecs.geolocation.find((x) => x.startsWith('b')); - const geoC = appSpecs.geolocation.filter((x) => x.startsWith('ac')); // this ensures that new specs can only run on updated nodes. - const geoCForbidden = appSpecs.geolocation.filter((x) => x.startsWith('a!c')); - - const myNodeLocationContinent = nodeGeo.continentCode; - const myNodeLocationContCountry = `${nodeGeo.continentCode}_${nodeGeo.countryCode}`; - const myNodeLocationFull = `${nodeGeo.continentCode}_${nodeGeo.countryCode}_${nodeGeo.regionName}`; - const myNodeLocationContinentALL = 'ALL'; - const myNodeLocationContCountryALL = `${nodeGeo.continentCode}_ALL`; - const myNodeLocationFullALL = `${nodeGeo.continentCode}_${nodeGeo.countryCode}_ALL`; - if (appContinent && !geoC.length && !geoCForbidden.length) { // backwards old style compatible. Can be removed after a month - if (appContinent.slice(1) !== nodeGeo.continentCode) { - throw new Error('App specs with continents geolocation set not matching node geolocation. Aborting.'); - } - } - if (appCountry) { - if (appCountry.slice(1) !== nodeGeo.countryCode) { - throw new Error('App specs with countries geolocation set not matching node geolocation. Aborting.'); - } - } - geoCForbidden.forEach((locationNotAllowed) => { - if (locationNotAllowed.slice(3) === myNodeLocationContinent || locationNotAllowed.slice(3) === myNodeLocationContCountry || locationNotAllowed.slice(3) === myNodeLocationFull) { - throw new Error('App specs of geolocation set is forbidden to run on node geolocation. Aborting.'); - } - }); - if (geoC.length) { - const nodeLocationOK = geoC.find((locationAllowed) => locationAllowed.slice(2) === myNodeLocationContinent || locationAllowed.slice(2) === myNodeLocationContCountry || locationAllowed.slice(2) === myNodeLocationFull - || locationAllowed.slice(2) === myNodeLocationContinentALL || locationAllowed.slice(2) === myNodeLocationContCountryALL || locationAllowed.slice(2) === myNodeLocationFullALL); - if (!nodeLocationOK) { - throw new Error('App specs of geolocation set is not matching to run on node geolocation. Aborting.'); - } - } - } - - return true; -} - /** * Get full node geolocation string * @returns {Promise} Full geolocation string @@ -211,200 +42,6 @@ async function nodeFullGeolocation() { return `${nodeGeo.continentCode}_${nodeGeo.countryCode}_${nodeGeo.regionName}`; } -/** - * To check app requirements of HW for a node - * @param {object} appSpecs App specifications. - * @returns {boolean} True if all checks passed. - */ -async function checkAppHWRequirements(appSpecs) { - // Import locally to avoid circular dependency - // eslint-disable-next-line global-require - const appController = require('../appManagement/appController'); - - // appSpecs has hdd, cpu and ram assigned to correct tier - const tier = await generalService.nodeTier(); - const resourcesLocked = await appController.appsResources(); - if (resourcesLocked.status !== 'success') { - throw new Error('Unable to obtain locked system resources by Flux Apps. Aborting.'); - } - - const appHWrequirements = hwRequirements.totalAppHWRequirements(appSpecs, tier); - await getNodeSpecs(); - const totalSpaceOnNode = nodeSpecs.ssdStorage; - if (totalSpaceOnNode === 0) { - throw new Error('Insufficient space on Flux Node to spawn an application'); - } - const useableSpaceOnNode = totalSpaceOnNode * 0.95 - config.lockedSystemResources.hdd - config.lockedSystemResources.extrahdd; - const hddLockedByApps = resourcesLocked.data.appsHddLocked; - const availableSpaceForApps = useableSpaceOnNode - hddLockedByApps; - // bigger or equal so we have the 1 gb free... - if (appHWrequirements.hdd > availableSpaceForApps) { - throw new Error('Insufficient space on Flux Node to spawn an application'); - } - - const totalCpuOnNode = nodeSpecs.cpuCores * 10; - const useableCpuOnNode = totalCpuOnNode - config.lockedSystemResources.cpu; - const cpuLockedByApps = resourcesLocked.data.appsCpusLocked * 10; - const adjustedAppCpu = appHWrequirements.cpu * 10; - const availableCpuForApps = useableCpuOnNode - cpuLockedByApps; - if (adjustedAppCpu > availableCpuForApps) { - throw new Error('Insufficient CPU power on Flux Node to spawn an application'); - } - - const totalRamOnNode = nodeSpecs.ram; - const useableRamOnNode = totalRamOnNode - config.lockedSystemResources.ram; - const ramLockedByApps = resourcesLocked.data.appsRamLocked; - const availableRamForApps = useableRamOnNode - ramLockedByApps; - if (appHWrequirements.ram > availableRamForApps) { - throw new Error('Insufficient RAM on Flux Node to spawn an application'); - } - - return true; -} - -/** - * To check app requirements to include HDD space, CPU power, RAM and GEO for a node - * @param {object} appSpecs App specifications. - * @returns {boolean} True if all checks passed. - */ -async function checkAppRequirements(appSpecs) { - // appSpecs has hdd, cpu and ram assigned to correct tier - await checkAppHWRequirements(appSpecs); - // check geolocation - checkAppStaticIpRequirements(appSpecs); - checkAppDataCenterRequirements(appSpecs); - await checkAppNodesRequirements(appSpecs); - await checkAppGeolocationRequirements(appSpecs); - return true; -} - -/** - * Check hardware parameters for legacy apps - * @param {object} appSpecs - App specifications - * @returns {boolean} True if parameters are valid - */ -function checkHWParameters(appSpecs) { - // check specs parameters. JS precision - if ((appSpecs.cpu * 10) % 1 !== 0 || (appSpecs.cpu * 10) > (config.fluxSpecifics.cpu.stratus - config.lockedSystemResources.cpu) || appSpecs.cpu < 0.1) { - throw new Error(`CPU badly assigned for ${appSpecs.name}`); - } - if (appSpecs.ram % 100 !== 0 || appSpecs.ram > (config.fluxSpecifics.ram.stratus - config.lockedSystemResources.ram) || appSpecs.ram < 100) { - throw new Error(`RAM badly assigned for ${appSpecs.name}`); - } - if (appSpecs.hdd % 1 !== 0 || appSpecs.hdd > (config.fluxSpecifics.hdd.stratus - config.lockedSystemResources.hdd) || appSpecs.hdd < 1) { - throw new Error(`SSD badly assigned for ${appSpecs.name}`); - } - if (appSpecs.tiered) { - if ((appSpecs.cpubasic * 10) % 1 !== 0 || (appSpecs.cpubasic * 10) > (config.fluxSpecifics.cpu.cumulus - config.lockedSystemResources.cpu) || appSpecs.cpubasic < 0.1) { - throw new Error(`CPU for Cumulus badly assigned for ${appSpecs.name}`); - } - if (appSpecs.rambasic % 100 !== 0 || appSpecs.rambasic > (config.fluxSpecifics.ram.cumulus - config.lockedSystemResources.ram) || appSpecs.rambasic < 100) { - throw new Error(`RAM for Cumulus badly assigned for ${appSpecs.name}`); - } - if (appSpecs.hddbasic % 1 !== 0 || appSpecs.hddbasic > (config.fluxSpecifics.hdd.cumulus - config.lockedSystemResources.hdd) || appSpecs.hddbasic < 1) { - throw new Error(`SSD for Cumulus badly assigned for ${appSpecs.name}`); - } - if ((appSpecs.cpusuper * 10) % 1 !== 0 || (appSpecs.cpusuper * 10) > (config.fluxSpecifics.cpu.nimbus - config.lockedSystemResources.cpu) || appSpecs.cpusuper < 0.1) { - throw new Error(`CPU for Nimbus badly assigned for ${appSpecs.name}`); - } - if (appSpecs.ramsuper % 100 !== 0 || appSpecs.ramsuper > (config.fluxSpecifics.ram.nimbus - config.lockedSystemResources.ram) || appSpecs.ramsuper < 100) { - throw new Error(`RAM for Nimbus badly assigned for ${appSpecs.name}`); - } - if (appSpecs.hddsuper % 1 !== 0 || appSpecs.hddsuper > (config.fluxSpecifics.hdd.nimbus - config.lockedSystemResources.hdd) || appSpecs.hddsuper < 1) { - throw new Error(`SSD for Nimbus badly assigned for ${appSpecs.name}`); - } - if ((appSpecs.cpubamf * 10) % 1 !== 0 || (appSpecs.cpubamf * 10) > (config.fluxSpecifics.cpu.stratus - config.lockedSystemResources.cpu) || appSpecs.cpubamf < 0.1) { - throw new Error(`CPU for Stratus badly assigned for ${appSpecs.name}`); - } - if (appSpecs.rambamf % 100 !== 0 || appSpecs.rambamf > (config.fluxSpecifics.ram.stratus - config.lockedSystemResources.ram) || appSpecs.rambamf < 100) { - throw new Error(`RAM for Stratus badly assigned for ${appSpecs.name}`); - } - if (appSpecs.hddbamf % 1 !== 0 || appSpecs.hddbamf > (config.fluxSpecifics.hdd.stratus - config.lockedSystemResources.hdd) || appSpecs.hddbamf < 1) { - throw new Error(`SSD for Stratus badly assigned for ${appSpecs.name}`); - } - } - return true; -} - -/** - * Check hardware parameters for compose apps - * @param {object} appSpecsComposed - Composed app specifications - * @returns {boolean} True if parameters are valid - */ -function checkComposeHWParameters(appSpecsComposed) { - // calculate total HW assigned - let totalCpu = 0; - let totalRam = 0; - let totalHdd = 0; - let totalCpuBasic = 0; - let totalCpuSuper = 0; - let totalCpuBamf = 0; - let totalRamBasic = 0; - let totalRamSuper = 0; - let totalRamBamf = 0; - let totalHddBasic = 0; - let totalHddSuper = 0; - let totalHddBamf = 0; - const isTiered = appSpecsComposed.compose.find((appComponent) => appComponent.tiered === true); - appSpecsComposed.compose.forEach((appComponent) => { - if (isTiered) { - totalCpuBamf += ((appComponent.cpubamf || appComponent.cpu) * 10); - totalRamBamf += appComponent.rambamf || appComponent.ram; - totalHddBamf += appComponent.hddbamf || appComponent.hdd; - totalCpuSuper += ((appComponent.cpusuper || appComponent.cpu) * 10); - totalRamSuper += appComponent.ramsuper || appComponent.ram; - totalHddSuper += appComponent.hddsuper || appComponent.hdd; - totalCpuBasic += ((appComponent.cpubasic || appComponent.cpu) * 10); - totalRamBasic += appComponent.rambasic || appComponent.ram; - totalHddBasic += appComponent.hddbasic || appComponent.hdd; - } else { - totalCpu += (appComponent.cpu * 10); - totalRam += appComponent.ram; - totalHdd += appComponent.hdd; - } - }); - // check specs parameters. JS precision - if (totalCpu > (config.fluxSpecifics.cpu.stratus - config.lockedSystemResources.cpu)) { - throw new Error(`Too much CPU resources assigned for ${appSpecsComposed.name}`); - } - if (totalRam > (config.fluxSpecifics.ram.stratus - config.lockedSystemResources.ram)) { - throw new Error(`Too much RAM resources assigned for ${appSpecsComposed.name}`); - } - if (totalHdd > (config.fluxSpecifics.hdd.stratus - config.lockedSystemResources.hdd)) { - throw new Error(`Too much SSD resources assigned for ${appSpecsComposed.name}`); - } - if (isTiered) { - if (totalCpuBasic > (config.fluxSpecifics.cpu.cumulus - config.lockedSystemResources.cpu)) { - throw new Error(`Too much CPU for Cumulus resources assigned for ${appSpecsComposed.name}`); - } - if (totalRamBasic > (config.fluxSpecifics.ram.cumulus - config.lockedSystemResources.ram)) { - throw new Error(`Too much RAM for Cumulus resources assigned for ${appSpecsComposed.name}`); - } - if (totalHddBasic > (config.fluxSpecifics.hdd.cumulus - config.lockedSystemResources.hdd)) { - throw new Error(`Too much SSD for Cumulus resources assigned for ${appSpecsComposed.name}`); - } - if (totalCpuSuper > (config.fluxSpecifics.cpu.nimbus - config.lockedSystemResources.cpu)) { - throw new Error(`Too much CPU for Nimbus resources assigned for ${appSpecsComposed.name}`); - } - if (totalRamSuper > (config.fluxSpecifics.ram.nimbus - config.lockedSystemResources.ram)) { - throw new Error(`Too much RAM for Nimbus resources assigned for ${appSpecsComposed.name}`); - } - if (totalHddSuper > (config.fluxSpecifics.hdd.nimbus - config.lockedSystemResources.hdd)) { - throw new Error(`Too much SSD for Nimbus resources assigned for ${appSpecsComposed.name}`); - } - if (totalCpuBamf > (config.fluxSpecifics.cpu.stratus - config.lockedSystemResources.cpu)) { - throw new Error(`Too much CPU for Stratus resources assigned for ${appSpecsComposed.name}`); - } - if (totalRamBamf > (config.fluxSpecifics.ram.stratus - config.lockedSystemResources.ram)) { - throw new Error(`Too much RAM for Stratus resources assigned for ${appSpecsComposed.name}`); - } - if (totalHddBamf > (config.fluxSpecifics.hdd.stratus - config.lockedSystemResources.hdd)) { - throw new Error(`Too much SSD for Stratus resources assigned for ${appSpecsComposed.name}`); - } - } - return true; -} - /** * Create Flux network via API * @param {object} req - Request object @@ -412,7 +49,7 @@ function checkComposeHWParameters(appSpecsComposed) { */ async function createFluxNetworkAPI(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (!authorized) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -431,74 +68,8 @@ async function createFluxNetworkAPI(req, res) { } } -/** - * Start monitoring of apps - * @param {object[]} appSpecsToMonitor - Array of app specifications to monitor - * @returns {Promise} - */ -async function startMonitoringOfApps(appSpecsToMonitor) { - try { - if (!appSpecsToMonitor || appSpecsToMonitor.length === 0) { - return; - } - - log.info(`Starting monitoring for ${appSpecsToMonitor.length} apps`); - - // eslint-disable-next-line no-restricted-syntax - for (const appSpec of appSpecsToMonitor) { - // Initialize monitoring for each app - log.info(`Monitoring started for ${appSpec.name}`); - } - } catch (error) { - log.error(`Error starting app monitoring: ${error.message}`); - throw error; - } -} - -/** - * Stop monitoring of apps - * @param {object[]} appSpecsToMonitor - Array of app specifications to stop monitoring - * @param {boolean} [deleteData=false] - Whether to delete monitoring data - * @returns {Promise} - */ -async function stopMonitoringOfApps(appSpecsToMonitor, deleteData = false) { - try { - if (!appSpecsToMonitor || appSpecsToMonitor.length === 0) { - return; - } - - log.info(`Stopping monitoring for ${appSpecsToMonitor.length} apps`); - - // eslint-disable-next-line no-restricted-syntax - for (const appSpec of appSpecsToMonitor) { - // Stop monitoring for each app - log.info(`Monitoring stopped for ${appSpec.name}`); - - if (deleteData) { - log.info(`Monitoring data deleted for ${appSpec.name}`); - } - } - } catch (error) { - log.error(`Error stopping app monitoring: ${error.message}`); - throw error; - } -} - module.exports = { - getNodeSpecs, - setNodeSpecs, - returnNodeSpecs, systemArchitecture, - checkAppStaticIpRequirements, - checkAppDataCenterRequirements, - checkAppNodesRequirements, - checkAppGeolocationRequirements, nodeFullGeolocation, - checkAppHWRequirements, - checkAppRequirements, - checkHWParameters, - checkComposeHWParameters, createFluxNetworkAPI, - startMonitoringOfApps, - stopMonitoringOfApps, }; diff --git a/ZelBack/src/services/appSystem/volumeExecutor.js b/ZelBack/src/services/appSystem/volumeExecutor.js new file mode 100644 index 0000000000..2b54f86189 --- /dev/null +++ b/ZelBack/src/services/appSystem/volumeExecutor.js @@ -0,0 +1,2082 @@ +const config = require('config'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const os = require('node:os'); +const fs = require('node:fs/promises'); +const dockerService = require('../dockerService'); +const deviceHelper = require('../deviceHelper'); +const serviceHelper = require('../serviceHelper'); +const networkStateService = require('../networkStateService'); +const { bareIp, extractPort } = require('../utils/socketAddressUtils'); +const fluxNetworkHelper = require('../fluxNetworkHelper'); +const jobRegistry = require('../utils/jobRegistry'); +const fluxEventBus = require('../utils/fluxEventBus'); +const log = require('../../lib/log'); +const { Writable, pipeline } = require('node:stream'); +const { createWriteStream, createReadStream } = require('node:fs'); +const { AsyncLock } = require('../utils/asyncLock'); +const { measureTree } = require('../utils/treeSize'); +const { appsFolder } = require('../utils/appConstants'); +const { + VolumePath, VolumeSession, WORK_ROOT, +} = require('./volumeSession'); +const { + isStagingName, +} = require('./volumeReservedNames'); + +const settings = () => config.fluxapps.volumeOperations; + + + +/** + * Labels every executor container carries. + * + * `role` is what keeps these out of the app sweeps: forceAppRemovals derives an + * app name from a container name and hands it to removeAppLocally, so a + * container it does not recognise produces a plausible-looking wrong name. The + * label answers the question directly instead. + */ +const EXECUTOR_LABELS = { 'runonflux.role': 'fileop' }; + +// One slot per concurrent operation. Refusing rather than queueing is +// deliberate: a queued request holds its connection open behind someone else's +// long copy until an intermediate proxy kills it, which reads to the user as a +// failure with no explanation. +const nodeLock = new AsyncLock(Number.MAX_SAFE_INTEGER); +const appLocks = new Map(); + +// Container ids and staging paths of the operations THIS process has in flight. +// Boot recovery removes file-operation containers and staging directories left by +// a PREVIOUS process; consulting these keeps it from reaping one that belongs to an +// operation running right now. Populated by run() for the life of each operation and +// cleared in its finally, so "orphaned" means exactly "owned by no live operation" +// rather than "carries our label" - the API answers requests before recovery runs, +// so an operation can be in flight when it does. +const liveContainerIds = new Set(); +const liveStagingPaths = new Set(); + +function lockForApp(identifier) { + if (!appLocks.has(identifier)) appLocks.set(identifier, new AsyncLock(Number.MAX_SAFE_INTEGER)); + return appLocks.get(identifier); +} + +/** + * How long to tell a refused caller to wait. + * + * Derived from the operation in the way only it is measured: a copy that has + * moved a known fraction of a known total in a known time says when it will be + * done. Anything else gets the default: a denominator is only offered where one + * is real, and an invented wait is worse than an honest shrug. + * + * Capped, because an estimate of hours belongs in the job a caller can watch + * rather than in a header telling it to sleep. + * + * @param {{startedAt: number, detail: object}|null} operation + * @returns {number} milliseconds + */ +function retryAfterFor(operation) { + const detail = operation && operation.detail; + if (!detail || !detail.bytesTotal || !detail.bytesDone) return BUSY_RETRY_AFTER_MS; + + const elapsed = Date.now() - operation.startedAt; + if (elapsed <= 0 || detail.bytesDone >= detail.bytesTotal) return BUSY_RETRY_AFTER_MS; + + const remaining = ((detail.bytesTotal - detail.bytesDone) / detail.bytesDone) * elapsed; + return Math.min(Math.max(Math.round(remaining), BUSY_RETRY_AFTER_MS), BUSY_RETRY_AFTER_CEILING_MS); +} + +/** + * Take a slot for this app, or throw. + * + * The read of activeCount and the register() that follows are not separated by + * an await, so nothing can interleave between them. It reads like a + * check-then-act race and is not one - do not "fix" it by adding a lock. + * + * @param {string} identifier + * @returns {function(): void} release + */ +function acquireSlot(identifier) { + const { maxConcurrentPerApp, maxConcurrentPerNode } = settings(); + const appLock = lockForApp(identifier); + + // Marked `busy` so the HTTP layer answers 503 with a Retry-After rather than + // a generic failure: a caller turned away before any work started should + // learn that immediately, not by registering an operation and polling to + // discover it was refused. + const busy = (message, operation = null) => { + const error = new Error(message); + error.kind = 'busy'; + error.retryAfterMs = retryAfterFor(operation); + // What the caller is waiting behind, so a refusal is something to watch or + // cancel rather than an invitation to guess. A client with nothing to name + // can only retry blindly, which behind a four hour copy is a thousand + // refusals telling it nothing it did not already know. + if (operation) { + error.operation = { + jobId: operation.jobId, + kind: operation.kind, + statusUrl: operation.statusUrl, + }; + } + return error; + }; + + if (appLock.activeCount >= maxConcurrentPerApp) { + const running = jobRegistry.runningForApp(identifier); + throw busy( + running + ? `${running.kind} is already running for ${identifier}` + : `Another file operation is already running for ${identifier}`, + running, + ); + } + if (nodeLock.activeCount >= maxConcurrentPerNode) { + throw busy('This node is running its maximum number of file operations; try again shortly'); + } + + appLock.register(); + nodeLock.register(); + + let released = false; + return () => { + if (released) return; + released = true; + appLock.disable(); + nodeLock.disable(); + if (!appLock.activeCount) appLocks.delete(identifier); + }; +} + +/** + * Throw `busy` if this app or the node has no free slot, WITHOUT taking one. + * + * Lets a caller refuse before it registers an operation, so the common case is + * a clean 503 rather than a job that exists only to report that it never began. + * The real limit is still enforced by acquireSlot; losing the race between the + * two just means the refusal is recorded against a job instead of a response. + * + * @param {VolumeSession} session + */ +function assertCapacity(session) { + const release = acquireSlot(session.identifier); + release(); +} + +/** + * Confirm the session's mount is a filesystem the kernel currently reports. + * + * FluxOS holds the docker socket, so whatever decides a bind source decides + * host access - a wrong path here is not a containment bug, it is a host + * compromise. The mount was already SELECTED from the mount table when the + * session was opened; this re-reads it immediately before the bind so a volume + * unmounted in between cannot be bound as a plain host directory, which is what + * would happen if the mountpoint were bound while empty. + * + * @param {VolumeSession} session + */ +async function assertMountIsLive(session) { + if (!session.mount.startsWith(appsFolder)) { + throw new Error('Application volume is not under the apps folder'); + } + const mounts = await deviceHelper.listMountedFilesystems(); + if (!mounts.some((mount) => mount.target === session.mount)) { + throw new Error('Application volume is no longer mounted'); + } +} + +/** + * The identifiers that name the image this node must run - any one of which is + * proof, because every one of them is derived from the bytes. + * + * A tag says which image to fetch; it does not say what is in it. One at a + * registry can be moved, and one inside an archive a peer hands over is + * whatever that peer wrote there - so the tag is the name and these are the + * proof. + * + * There are two of them because docker files an image under different content + * digests depending on how it stores images, and a published image is not one + * blob: it is an INDEX naming one image per architecture. + * + * classic store this architecture's own config digest + * containerd store the digest of the index covering every architecture + * + * The containerd store is the default from Docker 29, which on 2026-08-14 was + * 5,587 of the fleet's 6,066 nodes. Pinning only the config digest left every + * one of them pulling the image successfully and then refusing it - not as a + * transient failure but for good, since no retry makes two different numbers + * agree - and taking the whole file browser down with it, `createfolder`, + * `renameobject` and `removeobject` included. + * + * Accepting either weakens nothing. Both are content digests over the same + * bytes: the index covers the per-architecture manifests, which cover their + * configs and layers, so neither can be forged without breaking the other. + * + * ONCE PRE-29 DOCKER IS OFF THE NETWORK this collapses to the index digest + * alone - one identifier, no architecture in it. `fluxos-docker-versions` is + * the survey that says when. Until then the config digest is what the 479 + * remaining older daemons answer with. + * + * @returns {Array} most specific first; a held image matching any is + * the pinned image + */ +function expectedImageIds() { + const { image, imageIds, indexId } = settings(); + const architecture = os.arch() === 'x64' ? 'amd64' : os.arch(); + const forArchitecture = imageIds && imageIds[architecture]; + if (!forArchitecture && !indexId) { + throw new Error(`No file operation image is pinned for ${architecture}, so ${image} cannot be verified`); + } + return [forArchitecture, indexId].filter(Boolean); +} + +/** + * Which of them this node actually holds, or null. + * + * The answer is the id to RUN as well as the answer to "is it here": a + * container has to be created from the identifier the local daemon resolves, + * which is the one that just answered. + * + * @returns {Promise} + */ +async function heldImageId() { + // eslint-disable-next-line no-restricted-syntax + for (const candidate of expectedImageIds()) { + // eslint-disable-next-line no-await-in-loop + if (await dockerService.imageExists(candidate)) return candidate; + } + return null; +} + +/** + * Monotonic milliseconds. A backoff measured against the wall clock expires + * early or never when the clock is corrected. + * @returns {number} + */ +function monotonicMs() { + return Number(process.hrtime.bigint() / 1000000n); +} + +/** How long a caller waits for an image already on its way before being told to come back. */ +const IMAGE_WAIT_MS = 10000; + +/** + * How long a failed attempt answers callers without searching again. + * + * Without it every click repeats the whole search - the registry and four peers + * - so a node that cannot get the image makes every file operation cost minutes + * rather than telling the user in milliseconds. The background loop owns + * retrying; a caller only needs the answer. + */ +const IMAGE_FAILURE_SILENCE_MS = 60000; + +/** One registry attempt loses a transient error - a DNS blip, a single 503 - to bad luck. */ +const REGISTRY_ATTEMPTS = 2; +const REGISTRY_RETRY_MS = 3000; + +/** Between cycles: soon enough that a node whose network returns is not stuck for an hour. */ +const ACQUIRE_BACKOFF_MS = 30000; +const ACQUIRE_BACKOFF_CEILING_MS = 60 * 60 * 1000; + +let acquiring = null; +let acquiringUsedRegistry = false; +let failedAt = 0; +let backoffMs = 0; +let backoffUntil = 0; +let acquireTimer = null; + +/** + * A refusal the HTTP layer answers 503 to, carrying how long to wait. + * + * The node is not broken and the request is not wrong: what it needs is on its + * way, which is a different thing from a failure and reads differently to + * whoever is looking at it. + * + * @param {number} retryAfterMs + * @returns {Error} + */ +function imageComing(retryAfterMs) { + const error = new Error('The file operation image is not on this node yet and is being fetched; try again shortly'); + error.kind = 'busy'; + error.retryAfterMs = Math.min(Math.max(retryAfterMs || IMAGE_WAIT_MS, 5000), 300000); + return error; +} + +/** + * Where in the window this node asks the REGISTRY, derived from its own address. + * + * Derived rather than drawn at random so it is the same slot on every restart: + * a node that re-rolls picks a new one each boot, which turns a restarting + * fleet back into the burst the window exists to spread. Hashed because + * addresses are not evenly distributed and their low bits least of all. + * + * @returns {number} + */ +function prefetchDelayMs() { + const { prefetchWindowMs } = settings(); + const identity = userconfig.initial.ipaddress; + if (!identity) return Math.floor(prefetchWindowMs / 2); + const digest = crypto.createHash('sha256').update(identity).digest(); + return digest.readUInt32BE(0) % prefetchWindowMs; +} + +/** How many peers are asked for the image before the attempt is given up. */ +const PEER_IMAGE_ATTEMPTS = 4; + +/** What a refused caller waits when nothing better can be said. */ +const BUSY_RETRY_AFTER_MS = 5000; + +/** The most a refusal asks anyone to sleep, however long the operation has left. */ +const BUSY_RETRY_AFTER_CEILING_MS = 5 * 60 * 1000; + +/** How many draws that costs at most, since the same peer can come up twice. */ +const PEER_IMAGE_DRAWS = 20; + +/** How long a peer has to answer at all. */ +const PEER_IMAGE_TIMEOUT_MS = 120000; + +/** + * How long the transfer may go with no bytes arriving. + * + * PEER_IMAGE_TIMEOUT_MS does NOT cover this and cannot be made to: axios settles + * a stream request when the response HEADERS arrive, and its timer is spent by + * then. A peer that answers and then goes quiet - a hostile one, or an ordinary + * NAT dropping the connection mid-archive - therefore left `docker load` waiting + * on a body that never ended, so the shared acquisition promise never settled, + * no retry was ever scheduled, the registry was never reached, and every file + * operation on the node was refused until FluxOS restarted. + * + * Measured against arrival rather than total time, so a genuinely slow link + * still finishes: the same reasoning as the upload rate floor, in the other + * direction. + */ +const PEER_IMAGE_STALL_MS = 30000; + +/** + * The most this node will take from a peer before it has verified anything. + * + * Nothing is known about the bytes until they have all arrived, so this is what + * stops a peer writing an unbounded amount into a node - a check that ran + * afterwards ran too late. + * + * The pinned image saves to 5.9 MiB, so 32 MiB is over five times what an + * honest archive weighs. It can be that tight because the image is pinned in + * THIS repository: it cannot grow past the ceiling without a release that can + * raise the ceiling with it, which would not be true of an image a node merely + * pulled. And the failure is benign - a refused archive falls through to the + * registry, so the operation is slower rather than lost. + * + * Read it as a RAM ceiling rather than a disk one. The archive goes to + * os.tmpdir(), which is tmpfs on ArcaneOS and on any node whose systemd puts + * /tmp there, and tmpfs pages are reclaimable only to swap. A CUMULUS has 8GB + * for the node and every application on it, so what a peer can make it hold + * matters more than the free space suggests. TMPDIR moves this onto disk and + * needs no change here - os.tmpdir() reads it. + */ +const PEER_IMAGE_MAX_BYTES = 32 * 1024 * 1024; + +/** + * How many peers this node hands the image to at once. + * + * The archive is around thirteen megabytes and the endpoint answers anyone the + * network state recognises, so without a ceiling a node can be asked to spend + * its bandwidth by whoever asks most often. + */ +const PEER_IMAGE_SERVE_LIMIT = 4; + +/** + * How long a serve may go with the caller taking nothing. + * + * A slot came back only when the caller DISCONNECTED, and a caller that neither + * disconnects nor reads does neither: the export simply blocks on backpressure + * and the slot is held for as long as the socket is open. Two such connections + * took every slot on a node until FluxOS restarted, and nothing said so - the + * node's own operations kept working, so it looked healthy while quietly + * serving no peer at all. Cheap to do to every node in the fleet at once, which + * pushes all of them onto the registry: the load peer serving exists to avoid. + * + * Reachable by more than node operators, because an application's container + * egresses under its host node's address. + * + * Measured against bytes taken rather than total time, so a slow but honest + * caller still completes. The ingress side already reasons this way; this is the + * same rule pointed the other direction, and there was no two-hour backstop + * here as there is there. + */ +const PEER_IMAGE_SERVE_STALL_MS = 30000; + +let peerImageServes = 0; + +/** + * How long the set of fleet addresses is reused for. + * + * Deciding whether a caller is a node meant copying the whole network state - + * around 13,000 entries - and splitting a string per entry, BEFORE the caller + * had been shown to be anyone. Measured at ~2.4ms of the event loop per + * request, so roughly 400 requests a second saturate the one core FluxOS has + * and stall everything else on it: app installs, operation polling, peer + * messaging. The route is unauthenticated and the image id it needs is public + * config, so that was reachable by anyone. + * + * A set is built once per window instead, however many callers arrive, which is + * the same answer at a constant cost. Held only while the endpoint is being + * used: nothing builds it on a node no peer ever asks. + * + * The convention this follows is already in the tree - the one other + * unauthenticated route that walks the fleet list is wrapped in + * `cache('30 seconds')`. A window is fine here for the same reason it is there: + * a node that joined seconds ago can wait, and one that left is refused a + * little late. + */ +const FLEET_ADDRESS_WINDOW_MS = 30000; + +let fleetAddresses = null; +let fleetAddressesAt = 0; + +/** + * Whether an address belongs to a node in the fleet. + * + * By address rather than by socketAddress: a node's API port cannot be read off + * an inbound connection, whose source port is ephemeral, and the fleet does not + * all run on the default one. + * + * @param {string} remote + * @returns {boolean} + */ +function fleetHolds(remote) { + if (!fleetAddresses || monotonicMs() - fleetAddressesAt >= FLEET_ADDRESS_WINDOW_MS) { + fleetAddresses = new Set(networkStateService.networkState().map((node) => bareIp(node.ip))); + fleetAddressesAt = monotonicMs(); + } + return fleetAddresses.has(remote); +} + +/** + * Take a peer's archive onto the disk, bounded, before anything reads it. + * + * To a file and not to memory: the ceiling has to be generous enough that an + * honest archive is never refused, and holding that much heap on a node whose + * memory is the constraint would trade one denial of service for another. + * + * Three things end the transfer: the ceiling, the stall window, and the caller + * giving up. Each destroys the response, so a peer cannot hold the socket after + * this returns, and the partial file is removed by the caller's finally. + * + * @param {NodeJS.ReadableStream} body - the peer's response + * @param {string} destination - where to put it + * @returns {Promise} bytes taken + */ +async function receivePeerArchive(body, destination) { + let taken = 0; + let stalled = null; + let failure = null; + + const sink = createWriteStream(destination); + + const stopWith = (error) => { + failure = failure || error; + body.destroy(error); + }; + + const watchdog = setInterval(function noticeSilence() { + if (stalled === taken) { + stopWith(new Error(`sent nothing for ${PEER_IMAGE_STALL_MS}ms`)); + return; + } + stalled = taken; + }, PEER_IMAGE_STALL_MS); + if (watchdog.unref) watchdog.unref(); + + body.on('data', function count(chunk) { + taken += chunk.length; + if (taken > PEER_IMAGE_MAX_BYTES) { + stopWith(new Error(`sent more than ${PEER_IMAGE_MAX_BYTES} bytes`)); + } + }); + + try { + await new Promise((resolve, reject) => { + pipeline(body, sink, (error) => (error || failure ? reject(failure || error) : resolve())); + }); + } finally { + clearInterval(watchdog); + } + + return taken; +} + +/** + * Remove everything an archive brought that was not the image being fetched. + * + * A peer answers with a tar of its own making. The wanted id is kept and + * everything else goes - extra images are not free, and one carrying a name the + * sender chose is worse than not free: nothing else on this node would ever + * look at it again. + * + * A tag is resolved before it is removed, because an archive may perfectly well + * deliver the wanted image WITH a tag on it, and removing that tag would delete + * the image this just went and fetched. A tag that cannot be resolved is left + * alone rather than guessed at. + * + * Best effort throughout: failing to tidy up is not a reason to fail an + * acquisition that otherwise worked. + * + * @param {{ids: Array, tags: Array}} loaded + * @param {Array} accepted - the ids that may be kept: the pin carries a + * config id and an index id, and which one the daemon files the image under + * depends on its image store, so either answers + * @param {string} socketAddress - the peer, for the log + * @returns {Promise} + */ +async function discardUnwantedImages(loaded, accepted, socketAddress) { + const wanted = new Set(accepted); + const unwanted = loaded.ids.filter((id) => !wanted.has(id)); + + // eslint-disable-next-line no-restricted-syntax + for (const tag of loaded.tags) { + // eslint-disable-next-line no-await-in-loop + const id = await dockerService.getImageId(tag).catch(() => null); + if (id && !wanted.has(id)) unwanted.push(tag); + } + + if (!unwanted.length) return; + + log.warn(`volumeExecutor - ${socketAddress} sent ${unwanted.length} image(s) that were not asked for; removing them`); + fluxEventBus.publish('fileoperation:imageDiscarded', { peer: socketAddress, count: unwanted.length }); + // eslint-disable-next-line no-restricted-syntax + for (const reference of unwanted) { + // eslint-disable-next-line no-await-in-loop + await dockerService.appDockerImageRemove(reference).catch(() => {}); + } +} + +/** + * Take the image from another Flux node. + * + * The registry is one place, and a node that cannot reach it has no file + * browser at all - `mkdir` included, which used to be a local call. Every other + * node that has ever run a file operation holds the same image, so the fleet is + * the second place. + * + * A peer is not trusted for what it sends. The archive names itself, so the ids + * the daemon reports loading are checked against the one this node is pinned to + * and anything else is removed again rather than left on the disk. + * + * @param {string} expected - the image id this node must end up holding + * @returns {Promise<{peer: string, asked: number}>} which peer provided it, and + * how many were asked to get there + * @throws {Error} if no peer provided it + */ +async function fetchImageFromPeer(expected) { + const asked = new Set(); + const localAddress = await fluxNetworkHelper.getLocalSocketAddress().catch(() => null); + + // Bounded by peers CONTACTED, not by draws. The draw is random, so counting + // draws lets a repeat stand in for a peer: on a small fleet that is the + // difference between asking the node that has the image and never reaching + // it. The draw ceiling is what stops a fleet of one from looping. + for (let draw = 0; asked.size < PEER_IMAGE_ATTEMPTS && draw < PEER_IMAGE_DRAWS; draw += 1) { + // eslint-disable-next-line no-await-in-loop + // This node's own address, so a draw cannot come back as itself. Every other + // caller of this passes one; passing null never matched, so a node could + // spend one of only four attempts asking itself for an image it has already + // established it does not hold. Negligible across the fleet, one draw in + // three on a three-node one. + // + // Not fatal if it cannot be determined - a draw that might waste an attempt + // is better than no peer search at all. + // eslint-disable-next-line no-await-in-loop + const socketAddress = await networkStateService.getRandomSocketAddress(localAddress); + if (!socketAddress || asked.has(socketAddress)) { + // eslint-disable-next-line no-continue + continue; + } + asked.add(socketAddress); + + // Through the shared parser rather than split on a colon. An address may be + // ip, ip:port, an IPv4-mapped ::ffff:1.2.3.4 - which the serve side strips + // for exactly this reason, so the shape occurs - or an IPv6 literal, and + // splitting on the first colon reads the mapped form as an EMPTY host and + // an IPv6 literal as its first group. The serve side takes care over this + // and the fetch side did not; two halves of one feature disagreeing about + // what an address is, is the trap. + const ip = bareIp(socketAddress); + const port = extractPort(socketAddress); + // A literal has to be bracketed to sit in a URL at all. + const host = ip && ip.includes(':') ? `[${ip}]` : ip; + let archiveDir = null; + let archivePath = null; + try { + // eslint-disable-next-line no-await-in-loop + const response = await serviceHelper.axiosGet( + `http://${host}:${port}/apps/fileoperationimage/${expected}`, + { responseType: 'stream', timeout: PEER_IMAGE_TIMEOUT_MS }, + ); + // Out of memory first, bounded and with a stall window, so an archive + // this node knows nothing about cannot be unbounded in size or in time. + // + // Into a directory of its own rather than loose in a shared one: + // os.tmpdir() is world-writable and holds everything else on the box, and + // this is an untrusted peer's bytes. mkdtemp creates it 0700 and creates + // it atomically, so nothing else can be sitting at the name first. + // + // The base is created if it is missing, which is what lets a deployment + // point TMPDIR at a path of its own - on ArcaneOS, at a directory on /dat + // rather than the tmpfs /tmp is - without having to create it in the same + // change, or at all. + // eslint-disable-next-line no-await-in-loop + await fs.mkdir(os.tmpdir(), { recursive: true }).catch(() => {}); + // eslint-disable-next-line no-await-in-loop + archiveDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flux-op-image-')); + archivePath = path.join(archiveDir, 'image.tar'); + // eslint-disable-next-line no-await-in-loop + await receivePeerArchive(response.data, archivePath); + + // And looked at before the daemon is allowed near it. `docker load` + // APPLIES the names an archive declares, which moves them off whatever + // this node had under them - so a peer could rename this node's own app + // images by packing their names in, and the cleanup afterwards removes + // the stolen name rather than giving it back. This node's own serve path + // exports by id and so declares no names at all; an archive that declares + // any is doing something we never do, and is refused rather than + // repaired. + // eslint-disable-next-line no-await-in-loop + const declared = await dockerService.archiveNames(archivePath); + if (declared.length) { + throw new Error(`archive names ${declared.join(', ')}, which this node does not accept from a peer`); + } + + // eslint-disable-next-line no-await-in-loop + const loaded = await dockerService.loadImage(createReadStream(archivePath)); + // Whatever else came in the archive goes, whether or not the wanted image + // was in it. Returning on success first - which is what this did - left a + // peer able to put images of its choosing on this node permanently, since + // nothing else ever looks at them. + // eslint-disable-next-line no-await-in-loop + await discardUnwantedImages(loaded, expectedImageIds(), socketAddress); + + // The identifier the archive actually delivered, which is the only one + // this daemon can act on: a containerd store files the image under the + // index digest and knows nothing about the config digest, so naming it by + // the first pinned id 404s and leaves the image nameless - which is the + // very state the naming exists to prevent. + const matched = loaded.ids.find((id) => expectedImageIds().includes(id)); + if (matched) { + // Named only now that the id has been checked, and after the discard + // above, so the name goes on bytes this node has verified rather than + // on the sender's claim about them. A peer serves the archive by id and + // the daemon writes no names for a reference that carries none, so what + // arrives is nameless - and a nameless image is a dangling one, which + // the prune before every app install takes. Without this the node loses + // the image it just fetched at the next install and asks a peer again, + // forever, on exactly the nodes that took the peer path because they + // cannot reach the registry. + // + // A failure here is not a failure of the fetch: the image is present + // and usable, it is only unprotected from the prune, which is where + // this path stood before. + // eslint-disable-next-line no-await-in-loop + await dockerService.tagImage(matched, settings().image).catch((error) => { + log.warn(`volumeExecutor - the file operation image could not be named, so a prune will take it: ${error.message}`); + }); + log.info(`volumeExecutor - took the file operation image from ${socketAddress}`); + return { peer: socketAddress, asked: asked.size }; + } + + log.warn(`volumeExecutor - ${socketAddress} sent ${loaded.ids.length} image(s), none of them ${expected}`); + } catch (error) { + log.warn(`volumeExecutor - ${socketAddress} could not provide the file operation image: ${error.message}`); + } finally { + // Whatever ended the transfer, the directory and the partial file in it + // go: a peer that dies mid-archive must not leave this node accumulating + // debris nothing will ever look at again. The directory rather than the + // file, so nothing is left behind if the name inside it ever changes. + if (archiveDir) { + // eslint-disable-next-line no-await-in-loop + await fs.rm(archiveDir, { recursive: true, force: true }).catch(() => {}); + } + } + } + + return { peer: null, asked: asked.size }; +} + +/** + * Hand the image to another Flux node that cannot reach the registry. + * + * Narrow on purpose. The caller names the id it wants and gets it only if that + * is the id THIS node is pinned to, so there is no "send me image X" here and + * it cannot become one: a node of another architecture asks for an id this one + * does not have and is refused, which is the right answer rather than a case to + * handle. Only an address the network state recognises is answered, and only a + * couple at a time. + * + * @param {object} req + * @param {object} res + * @returns {Promise} + */ +async function serveImageToPeer(req, res) { + let serving = false; + + try { + // The remote address rather than a forwarded header: this answers other + // nodes directly, and a header is written by whoever sends it. + // + // Only the IPv4-mapped prefix is stripped. Cutting to the LAST colon, which + // is the usual spelling of this, turns 2001:db8::1 into "1" - so a genuine + // IPv6 caller could never match the network state whatever it held, and the + // 403 it received said nothing about why. Stripping just the prefix leaves + // such an address intact to be compared as itself. + const remote = (req.socket.remoteAddress || '').replace(/^::ffff:/i, ''); + if (!remote) { + res.status(400).end(); + return; + } + + const asked = req.params.imageid; + // Either identifier is a fair way to ask, because a caller names the image + // by whatever ITS daemon files this image under, and that need not be what + // ours does. Still only this image: the id is compared, never used to look + // one up, so there is no "send me image X" here and there cannot become one. + if (!expectedImageIds().includes(asked)) { + res.status(404).end(); + return; + } + + // HEAD reaches this handler too - express answers it from the GET route when + // no HEAD route exists - and node discards the body of a HEAD response + // without ever applying backpressure. So a HEAD cost this node a full + // export, read off the disk and packed by docker, while costing the caller + // one packet and no bandwidth at all. Nothing here has a body worth + // describing, so there is nothing to answer. + if (req.method === 'HEAD') { + res.status(405).end(); + return; + } + + // The ceiling before the fleet lookup, because it is a comparison and the + // lookup is a set membership that may have to rebuild the set: a node with + // no capacity should not pay to find out who is asking. + // + // Taken in the SAME TICK it is tested, before any await. Testing it, then + // awaiting, then taking it - which is what this did - lets every request + // that arrives together read the same count and all pass, so the ceiling + // bounded nothing at exactly the moment it was needed. + if (peerImageServes >= PEER_IMAGE_SERVE_LIMIT) { + log.info(`volumeExecutor - refused ${remote} the file operation image: already serving ${peerImageServes}`); + res.set('Retry-After', '30'); + res.status(503).end(); + return; + } + serving = true; + peerImageServes += 1; + + if (!fleetHolds(remote)) { + res.status(403).end(); + return; + } + + // What THIS node holds it as, which is what it can export. A caller asking + // under the other identifier still gets the same bytes, and checks them + // against its own. + const held = await heldImageId(); + if (!held) { + res.status(404).end(); + return; + } + + // Subscribed BEFORE the export, because the caller can hang up while docker + // is still packing the archive. `close` fires once, so a listener attached + // after it has already happened never sees it: the wait at the end would + // never settle, the slot would never be given back, and two of those leave + // this node serving no peer at all until FluxOS restarts - which pushes + // everyone who asks it back onto the registry, the load peer serving exists + // to avoid. Same reason collectOutput subscribes before the container + // starts. + let archive = null; + let callerGone = false; + let noteCallerGone = null; + const disconnected = new Promise((resolve) => { noteCallerGone = resolve; }); + res.on('close', function callerWentAway() { + callerGone = true; + if (archive) archive.destroy(); + noteCallerGone(); + }); + + archive = await dockerService.exportImage(held); + if (callerGone) { + // Nobody to send it to, and the export is this node's to close. Returning + // here still gives the slot back, because that happens in the finally. + archive.destroy(); + return; + } + + res.set('Content-Type', 'application/x-tar'); + archive.on('error', function exportFailed() { res.destroy(); }); + + // Bytes LEAVING the export, which is what a caller taking nothing stops: + // pipe holds the archive against backpressure, so no data event fires while + // the socket is not being drained. Counting them is therefore counting the + // caller's progress, without needing anything from the socket. + let sent = 0; + let sentAtLastLook = null; + archive.on('data', function count(chunk) { sent += chunk.length; }); + + const watchdog = setInterval(function noticeIdleCaller() { + if (sentAtLastLook === sent) { + log.warn(`volumeExecutor - stopped serving the file operation image to ${remote}: took nothing for ${PEER_IMAGE_SERVE_STALL_MS}ms`); + archive.destroy(); + res.destroy(); + noteCallerGone(); + return; + } + sentAtLastLook = sent; + }, PEER_IMAGE_SERVE_STALL_MS); + if (watchdog.unref) watchdog.unref(); + + try { + archive.pipe(res); + await disconnected; + } finally { + clearInterval(watchdog); + } + } catch (error) { + log.error(`volumeExecutor - could not hand over the file operation image: ${error.message}`); + if (!res.headersSent) res.status(500).end(); + } finally { + // Only the request that took a slot gives one back: an early refusal that + // decremented would release somebody else's. + if (serving) peerImageServes -= 1; + } +} + +/** + * One attempt at getting the image: peers, then the registry if it is allowed. + * + * Peers first, everywhere. Once the fleet holds the image they are the faster + * answer and they cost one central place nothing; a peer that does not have it + * refuses at once rather than timing out, so asking is cheap even when it is + * futile. + * + * @param {string} expected + * @param {{registry: boolean}} sources + * @returns {Promise} whether the node now holds it + */ +async function acquisitionCycle(expected, sources) { + const fromPeer = await fetchImageFromPeer(expected).catch((error) => { + log.error(`volumeExecutor - the peer search failed: ${error.message}`); + return { peer: null, asked: 0 }; + }); + if (!fromPeer.peer) { + log.info(`volumeExecutor - no peer provided the file operation image: asked ${fromPeer.asked}`); + } + + if (await heldImageId()) { + // Where it came from, not just that it is here. The store is asked either + // way, so "the node holds it" cannot tell a transfer that was read + // correctly from one that was not - a peer whose archive was misread still + // leaves the image on the disk, and the node goes on asking other peers for + // something it already has. `unrecognised` is that case, and is the only + // way it is visible from outside. + fluxEventBus.publish('fileoperation:imageAcquired', fromPeer.peer + ? { source: 'peer', peer: fromPeer.peer, asked: fromPeer.asked } + : { source: 'unrecognised', asked: fromPeer.asked }); + return true; + } + + if (!sources.registry) return false; + + const { image } = settings(); + for (let attempt = 0; attempt < REGISTRY_ATTEMPTS; attempt += 1) { + try { + // eslint-disable-next-line no-await-in-loop + await dockerService.pullImage({ repoTag: image }); + // A pull can end on an error event and still call back without one, so + // the store is asked rather than the pull taken at its word. + // eslint-disable-next-line no-await-in-loop + if (await heldImageId()) { + // `asked` is peers asked, on every branch. It used to be re-used here for + // registry attempts, so one key on one event meant two different things + // and anything adding them up was adding different denominators - and + // the peers this node asked first were invisible whenever the registry + // won, which is most of the time. + fluxEventBus.publish('fileoperation:imageAcquired', { + source: 'registry', asked: fromPeer.asked, attempts: attempt + 1, + }); + return true; + } + log.warn(`volumeExecutor - ${image} resolved to an image this node is not pinned to`); + return false; + } catch (error) { + log.info(`volumeExecutor - the registry did not provide ${image}: ${error.message}`); + } + // eslint-disable-next-line no-await-in-loop + if (attempt + 1 < REGISTRY_ATTEMPTS) await serviceHelper.delay(REGISTRY_RETRY_MS); + } + + return false; +} + +/** + * Arm the next acquisition, replacing any already armed. + * + * The ONLY place this timer is set, as stopImagePrefetch is the only place it + * is cleared. Two paths schedule one - the boot prefetch's wait before it asks + * the registry, and the retry backoff - and nothing about either says they + * cannot both be live. + * + * Today they cannot: startImagePrefetch returns early when a timer is already + * armed, and acquireImage dedupes, so an operation arriving mid-prefetch joins + * that round rather than racing it and cannot arm before the prefetch does. A + * timer left referenced by nothing would survive stopImagePrefetch and fire on + * a node that asked not to fetch, so this does not rest on either of those + * holding: there is one way to arm, and it clears what is already armed. + * + * @param {string} expected + * @param {number} wait + * @returns {void} + */ +function armAcquire(expected, wait) { + if (acquireTimer) clearTimeout(acquireTimer); + // eslint-disable-next-line no-use-before-define + acquireTimer = setTimeout(() => { acquireImage(expected, { registry: true, thenRetry: true }); }, wait); + if (acquireTimer.unref) acquireTimer.unref(); +} + +/** + * Try again later, doubling the wait to a ceiling and jittered so nodes that + * failed together do not return together. + * + * @param {string} expected + * @returns {void} + */ +function scheduleAcquisition(expected) { + backoffMs = backoffMs + ? Math.min(backoffMs * 2, ACQUIRE_BACKOFF_CEILING_MS) + : ACQUIRE_BACKOFF_MS; + const wait = Math.round(backoffMs * (0.8 + (crypto.randomInt(0, 400) / 1000))); + backoffUntil = monotonicMs() + wait; + + armAcquire(expected, wait); +} + +/** + * Run a cycle, sharing one between everything waiting on it, and decide what + * happens when it comes back empty. + * + * @param {string} expected + * @param {{registry: boolean, thenRetry: boolean}} options + * @returns {Promise} + */ +function acquireImage(expected, options) { + // Sharing a round is right when it is looking where this caller needs it to. + // It is not when the round in flight is the prefetch's peers-only one and the + // caller was allowed the registry: joining it inherited a refusal from + // sources that were never tried, and since the round was also `thenRetry: + // false`, nothing was scheduled afterwards either. The caller was simply told + // no while the source that would have answered went unasked - and the window + // is not short, since four peers that blackhole take minutes. + if (acquiring && (acquiringUsedRegistry || !options.registry)) return acquiring; + + if (acquiring) { + return acquiring.then((held) => (held ? true : acquireImage(expected, options))); + } + + acquiringUsedRegistry = options.registry; + + // Only a round that was allowed every source can say the search failed. The + // prefetch's first round asks peers alone, so it has learned nothing about + // the registry - and recording it as a failure would make it answer for one, + // silencing callers for a minute over a source that was never tried. On a + // cold fleet, where no peer holds the image either, that is every operation + // on the node while the prefetch's own registry attempt is still hours away. + const searched = (at) => { if (options.registry) failedAt = at; }; + + acquiring = acquisitionCycle(expected, options) + .then((held) => { + if (held) { + failedAt = 0; + backoffMs = 0; + backoffUntil = 0; + log.info('volumeExecutor - the file operation image is on this node'); + return true; + } + searched(monotonicMs()); + if (options.thenRetry) scheduleAcquisition(expected); + return false; + }) + .catch((error) => { + searched(monotonicMs()); + log.error(`volumeExecutor - could not fetch the file operation image: ${error.message}`); + if (options.thenRetry) scheduleAcquisition(expected); + return false; + }) + .finally(() => { acquiring = null; }); + + return acquiring; +} + +/** + * Take the image before anything asks for it. + * + * Peers immediately, because once the fleet holds the image that is the whole + * job and it costs nobody anything - a node that reboots into a fleet that has + * it is done in a moment. Only when no peer has it does this node need the + * registry, and that is the fetch worth spreading, so it waits for its own + * point in the window before going there. + * + * @returns {Promise} + */ +async function prefetchImage() { + const [expected] = expectedImageIds(); + if (await heldImageId()) return; + + await acquireImage(expected, { registry: false, thenRetry: false }); + if (await heldImageId()) return; + + const wait = prefetchDelayMs(); + log.info(`volumeExecutor - no peer had the file operation image; the registry will be asked in ${Math.round(wait / 60000)} minute(s)`); + armAcquire(expected, wait); +} + +/** + * Start this node's fetch. + * + * Returns the first attempt so a caller can wait for it. Nothing in the boot + * path does - it is deliberately not awaited there - but a test that asserts + * what was scheduled has to know the scheduling has happened. + * + * @returns {Promise} + */ +function startImagePrefetch() { + if (acquireTimer) return Promise.resolve(); + return prefetchImage().catch((error) => { + log.error(`volumeExecutor - could not start the image fetch: ${error.message}`); + }); +} + +/** + * Stop a scheduled fetch. + * @returns {void} + */ +function stopImagePrefetch() { + if (acquireTimer) clearTimeout(acquireTimer); + acquireTimer = null; + backoffMs = 0; + backoffUntil = 0; + failedAt = 0; +} + +/** + * Make sure the executor image is on this node. + * + * Creating a container does not pull - docker answers 404 for an image it does + * not hold - so without this the first file operation on a node fails with an + * opaque docker error, and so does every one after it. + * + * Checked before EVERY operation rather than once at startup, because nothing + * guarantees the image is still there. An operator prunes, a disk fills, a + * dockerd is replaced; the check is one inspect when it is present, so paying + * it every time costs nothing and removes a whole class of "worked yesterday". + * + * `performDockerCleanup` is NOT one of the things that removes it, despite + * running before every app install: `pruneImages` filters on dangling, and a + * tagged image is not dangling. That holds for BOTH routes only because the + * peer route names what it took - an archive addressed by id carries no names, + * so an untagged arrival would be dangling and this would be false for exactly + * the nodes that cannot reach the registry. + * + * A caller waits a short while and is then told to come back. It does not wait + * for the fetch's own patience - a peer has two minutes to hand over thirteen + * megabytes and nobody clicking a button has two minutes - and the fetch is not + * abandoned because a caller stopped waiting. A cycle that has just failed + * answers immediately, so a node that cannot get the image costs a click + * milliseconds rather than minutes. + * + * @param {function(string): void} [onProgress] + * @returns {Promise} the id of the image to run + */ +async function ensureImage(onProgress = null) { + // However it got here, and whichever identifier this daemon files it under. + // An image carrying one of them is the image, whether it came from the + // registry, from a peer, or was already on the node - and the one that + // answered is the one the container is created from, because it is the only + // one this daemon can resolve. + const held = await heldImageId(); + if (held) return held; + + // How long this node will ACTUALLY refuse for, which is the longer of the two + // things that make it refuse: the silence window it is inside, and whatever + // the backoff has climbed to. Reporting only the backoff told a caller to come + // back in the five-second floor while the node went on refusing for a minute, + // so a client that honours the header retried about twelve times for nothing. + const silenceLeft = () => (failedAt ? IMAGE_FAILURE_SILENCE_MS - (monotonicMs() - failedAt) : 0); + const comeBackIn = () => Math.max(silenceLeft(), backoffUntil - monotonicMs()); + + if (failedAt && monotonicMs() - failedAt < IMAGE_FAILURE_SILENCE_MS) { + throw imageComing(comeBackIn()); + } + + if (onProgress) onProgress('Fetching the file operation image...'); + const [expected] = expectedImageIds(); + const cycle = acquireImage(expected, { registry: true, thenRetry: true }); + await Promise.race([cycle, serviceHelper.delay(IMAGE_WAIT_MS)]); + + const arrived = await heldImageId(); + if (arrived) return arrived; + throw imageComing(comeBackIn()); +} + +/** + * How much of a failed operation's own output is kept. + * + * Bounded because the output is produced inside the container and a runaway + * command could otherwise fill this process's memory with it. The TAIL is kept + * rather than the head: a tool that fails says why on its last line, after + * however much routine chatter came first. + */ +const OUTPUT_TAIL_BYTES = 2000; + +/** + * The exit statuses flux-op uses to name a refusal, and what each means here. + * + * A status is the one part of a failure that does not depend on wording. The + * commands that run inside the image are busybox's, and busybox words the same + * condition differently from coreutils - so a caller reading the reason out of + * the output would be matching on which build of a tool the image happens to + * carry. Anything not listed is the command's own status, which says no more + * than that it failed. + */ +const REFUSAL_BY_STATUS = new Map([ + [5, { code: 'EEXIST', message: 'Destination already exists' }], + // flux-op refused because the only way to carry the request out was to delete + // data it never named - a file put where a directory is, an entry moved onto + // itself under another name. A distinct code so a caller answers it specifically + // rather than as a generic failure. + [6, { code: 'EDESTRUCTIVE', message: 'The destination could only be replaced by deleting data that was not part of this request' }], +]); + +/** + * The error a non-zero exit becomes. + * + * A named refusal keeps its code, because a caller acts on it: the dashboard + * tells an app owner a folder is already there rather than that their request + * failed. Everything else keeps the output, which is all there is to say. + * + * @param {number} status + * @param {string} said - the tail of what the container wrote + * @returns {Error} + */ +function failureFor(status, said) { + const refusal = REFUSAL_BY_STATUS.get(status); + if (refusal) { + const error = new Error(refusal.message); + error.code = refusal.code; + return error; + } + return new Error(said + ? `File operation failed (exit ${status}): ${said}` + : `File operation failed with exit code ${status}`); +} + +/** + * Collect what the command writes, so a failure can say what went wrong. + * + * Without this, `AutoRemove` takes the container and its logs the moment it + * exits and the caller is handed an exit code. "The archive is corrupt", "it + * expands past the volume" and "it holds something that is not data" are three + * different problems with three different answers, and they arrived as the same + * number - to the user, and to whoever they then asked for help. + * + * Attached BEFORE start, for the same reason the exit subscription is: a fast + * command can finish and be reaped before a later attach lands, and its output + * is then gone. + * + * Never fatal. Losing the explanation is worse than an exit code alone, but + * failing the operation over it would be worse still. + * + * @param {object} container - dockerode container + * @returns {Promise<{text: string}>} filled in as the command writes + */ +async function collectOutput(container) { + const captured = { text: '' }; + + const sink = new Writable({ + write(chunk, encoding, callback) { + captured.text = (captured.text + chunk.toString('utf8')).slice(-OUTPUT_TAIL_BYTES); + callback(); + }, + }); + + try { + const stream = await container.attach({ stream: true, stdout: true, stderr: true }); + // stdout and stderr into one buffer: the caller wants to know what + // happened, not which descriptor it arrived on. + container.modem.demuxStream(stream, sink, sink); + } catch (error) { + log.warn(`volumeExecutor - could not capture operation output: ${error.message}`); + } + + return captured; +} + +/** + * Bytes in use on a volume, from the filesystem itself. + * + * One syscall, whatever the tree looks like. The alternative - walking the + * staging directory on every tick - costs 179ms per 20,000 files, which for an + * app with 30,000 of them is a tenth of a core burned continuously to draw a + * progress bar. Nothing else reports progress that way: rsync counts what it + * writes because it does the writing, and we do not. + * + * Byte progress is readable here and nowhere else. Since coreutils 9.0 `cp` + * copies with copy_file_range(2) - the kernel moves the bytes, so no counter on + * the SOURCE side sees them: /proc//io stays flat and the file offset in + * /proc//fdinfo does not advance until the end, which is why progress(1) + * stopped working on cp. What the destination filesystem has consumed is a + * different question, and it answers steadily the whole way through. + * + * It counts everything written to the volume, not only this operation's share, + * because the application keeps running throughout. That is the right figure + * for how full a volume is getting and an approximate one for how far a copy + * has got; the caller clamps it, and the figure a completed operation reports + * is taken from what it actually published rather than from here. + * + * @param {string} mount - host path of the app volume + * @param {object} fsPromises + * @returns {Promise} bytes in use, or null if it cannot be read + */ +async function volumeUsedBytes(mount, fsPromises) { + const stats = await fsPromises.statfs(mount).catch(() => null); + if (!stats) return null; + return (Number(stats.blocks) - Number(stats.bfree)) * Number(stats.bsize); +} + +/** + * The container an operation runs in. + * + * Containment comes from the container having nowhere to escape TO, rather than + * from a sequence of path checks being correct: + * + * the app's volume and nothing else a path that escapes it lands nowhere + * ReadonlyRootfs writes outside the volume fail + * NetworkMode none a hostile archive cannot phone home + * no-new-privileges a setuid file cannot escalate + * CapDrop ALL + three see below + * pids and memory limits bound a runaway archive + * AutoRemove no stopped container for a prune to find + * + * Three capabilities are added back out of docker's default fourteen. cp -a + * cannot restore ownership without CAP_CHOWN and does not fail when it can't - + * it exits 0 having written root-owned files, and an app running as a non-root + * user then silently loses access to its own data. FOWNER and DAC_OVERRIDE are + * needed to read and re-stamp files the container does not own. Everything else + * stays dropped, including MKNOD, so an archive cannot create device nodes. + */ +function containerOptions(session, argv, image, workingDir = WORK_ROOT, withInput = false) { + const { memoryBytes, pidsLimit, cpuCores } = settings(); + + return { + Image: image, + Cmd: argv, + WorkingDir: workingDir, + Labels: { ...EXECUTOR_LABELS, 'runonflux.app': session.identifier }, + AttachStdout: true, + AttachStderr: true, + // Only an upload opens stdin. StdinOnce closes it once the attach that + // wrote it disconnects, so the container cannot sit waiting on a descriptor + // nobody holds any more. + ...(withInput ? { OpenStdin: true, StdinOnce: true, AttachStdin: true } : {}), + HostConfig: { + Binds: [`${session.mount}:${WORK_ROOT}`], + ReadonlyRootfs: true, + NetworkMode: 'none', + AutoRemove: true, + CapDrop: ['ALL'], + CapAdd: ['CHOWN', 'FOWNER', 'DAC_OVERRIDE'], + SecurityOpt: ['no-new-privileges'], + Memory: memoryBytes, + // Equal to Memory: this field is memory plus swap in one figure, so equal + // means no swap. Left unset, docker grants the same amount again in swap, + // and memoryBytes was chosen as the bound on a runaway archive, not half + // of one. + MemorySwap: memoryBytes, + PidsLimit: pidsLimit, + NanoCPUs: Math.round(cpuCores * 1e9), + // A quarter of the default weight: under contention the applications win, + // because they are what the node is for and their own CPU quotas were + // priced. On an idle core this costs a file operation nothing - shares + // only decide who yields when someone has to. + CpuShares: 256, + // Docker's default seccomp and apparmor profiles apply because nothing + // here disables them. Never pass seccomp=unconfined - it is the change + // that gets made to "fix" a mystery permissions error and it removes the + // syscall filter for every operation. + }, + }; +} + +/** + * Open the container's standard input. + * + * `hijack` gives a real duplex socket; without it the attach returns a + * half-closed response stream and nothing can be written to the container at + * all. Opened BEFORE the container starts, for the same reason the exit + * subscription is: StdinOnce closes the descriptor once the attach that wrote it + * disconnects, and a container that starts with nobody attached can reach that + * point before the attach lands. + * + * @param {object} container - dockerode container, not yet started + * @returns {Promise} the duplex socket + */ +function attachInput(container) { + return container.attach({ + stream: true, stdin: true, hijack: true, + }); +} + +/** + * Watch how the caller's stream ends, from before anything else is awaited. + * + * Subscribed immediately rather than when the pipe is set up, because an + * `error` event with nobody listening is what node ends the PROCESS over - and + * there is real time between receiving this stream and having a container to + * feed it to, during which the client can disconnect. + * + * @param {import('node:stream').Readable} input + * @returns {Promise<{complete: boolean, reason: string|null}>} + */ +function watchInput(input) { + return new Promise((resolve) => { + input.on('end', () => resolve({ complete: true, reason: null })); + input.on('error', (error) => resolve({ + complete: false, + reason: error.message || 'the connection ended early', + })); + }); +} + +/** + * Count what arrives from the caller. + * + * The stall check reads the volume, which answers for a command writing into + * staging and does not answer for an upload: what a client sends lands in + * filesystem blocks, so a slow one moves nothing measurable for minutes and + * reads as a container getting nowhere. Bytes arriving are the direct evidence + * that something is happening, and they are exact where a block is rounded. + * + * Attached where the pipe is, not before: a `data` listener starts the stream + * flowing, and one added before the destination exists loses what arrives in + * between. + * + * @param {import('node:stream').Readable} input + * @param {{bytes: number}} received - updated in place + * @returns {void} + */ +function countInput(input, received) { + input.on('data', (chunk) => { + received.bytes += chunk.length; + }); +} + +/** + * Feed the caller's own bytes to a container that is writing them into staging, + * and decide how the transfer ended. + * + * Three things here are load-bearing and none of them are obvious. + * + * The stream is piped with `end: false`, never through stream.pipeline. Pipeline + * destroys its destination when the source errors, and destroying this socket is + * indistinguishable to the container from the clean end-of-input that means "you + * have everything". A browser that goes away mid-upload would look exactly like + * one that finished, and half a file would be published as though it were whole. + * Closing stdin is the only signal that the transfer completed, so it is sent + * only when it did. + * + * An upload that did not complete stops the container instead. The command + * cannot exit before its input closes, and flux-op cannot publish before the + * command exits, so there is no race to lose: the stop always arrives first, and + * flux-op reclaims staging on its way out. + * + * The pipe is raced against the container's exit because a container that has + * stopped reading never drains the socket and never errors - measured, the + * writer stalls at around 448KB and stays there indefinitely. Every refusal an + * upload can produce arrives that way: too large, no space, a volume that filled + * while it ran. Without the race each of them hangs the caller's request until + * something else times it out. + * + * @param {object} stdin - the hijacked duplex socket from attachInput + * @param {import('node:stream').Readable} input + * @param {Promise<{complete: boolean, reason: string|null}>} transferred - from + * watchInput, subscribed before any of this was awaited + * @param {Promise} exited + * @param {function(): void} stopContainer + * @param {{bytes: number}} [received] - counted as it arrives, so a caller + * sending slowly is not mistaken for a container getting nowhere + * @returns {Promise<{delivered: boolean, reason: string|null}>} + */ +async function feedContainer(stdin, input, transferred, exited, stopContainer, received = null) { + input.pipe(stdin, { end: false }); + if (received) countInput(input, received); + + const ended = exited.then(() => 'exited', () => 'exited'); + const outcome = await Promise.race([transferred, ended]); + + if (outcome !== 'exited' && outcome.complete) { + stdin.end(); + return { delivered: true, reason: null }; + } + + if (outcome !== 'exited') { + stopContainer(); + stdin.destroy(); + return { delivered: false, reason: outcome.reason }; + } + + // The container gave up on its own - it has already decided, and its exit + // status carries the reason. + // + // Unpiped, NOT destroyed. This stream belongs to the caller, and for an + // upload it is the multipart parser's: destroying it stops the parser + // consuming the request, so a client still sending cannot finish and can + // never read the refusal it is being sent. The caller drains what remains. + input.unpipe(stdin); + stdin.destroy(); + return { delivered: false, reason: null }; +} + +/** + * Run one file operation on an app's volume. + * + * @param {VolumeSession} session + * @param {Array} argv - operands must be VolumePath; a + * string operand is refused, which is what makes the session's checks + * unskippable rather than merely conventional + * @param {object} [options] + * @param {function(string): void} [options.onProgress] - called with each + * status line. The caller decides where it goes; for the HTTP endpoints that + * is jobRegistry.progress, so a client polls for the whole list rather than + * holding a connection open to receive it. + * @param {function(): boolean} [options.isCanceled] - polled while the + * operation runs; when it returns true the container is killed. Cancellation + * is cooperative, so status stays Running until the work actually stops. + * @param {string} [options.status] - the line reported while it runs + * @param {function(number|null): void} [options.onBytes] - called with the bytes + * published so far, or null once measuring them stops being affordable. Only + * meaningful alongside `publish`, and only for operations that WRITE into + * staging: a move publishes the source where it stands, so its staging size is + * the whole operation from the first tick and says nothing about progress. + * + * Called once more on success, with the size of what was actually published, + * so a finished operation reports what it finished rather than whichever tick + * completed last. + * @param {{staging?: VolumePath, source?: VolumePath, destination: VolumePath}} + * [options.publish] - run the command into `staging` and move the result to + * `destination` only if it succeeds. Wrapping this here rather than leaving it + * to the caller is what stops an endpoint writing to a destination directly + * and losing the guarantee that a failure changes nothing. + * + * Exactly one of `staging` and `source`. `staging` is scratch this operation + * created, so a failure may throw it away; `source` is the caller's own data, + * published where it stands, which is how a move is expressed - there is no + * command, because the source already IS the result. Naming them differently + * is what stops the discard applying to somebody's only copy: the difference + * has to be stated to be used, rather than remembered. + * @param {boolean} [options.mkdirStaging] - create the staging directory first, + * for commands like `tar -C` that need it to exist. A file copy must NOT ask + * for it: cp -T refuses to overwrite a directory with a non-directory. + * @param {number} [options.maxBytes] - ceiling on what the command may leave in + * staging. Enforced on the RESULT rather than on what the input claims about + * itself, because an archive's declared sizes are written by whoever built it. + * ZERO MEANS NO CEILING, here and in the image alike. An operation whose + * ceiling IS the volume's free space therefore has to establish that there is + * some before it asks - on a full volume the figure is zero, and the only + * bound it has would read as none. requireSpace and requireCapacity are how a + * caller does that. + * @param {boolean} [options.dataOnly] - refuse a result holding a FIFO, a socket + * or a device node. None of them is data, and whatever opens a FIFO without + * O_NONBLOCK waits for a writer that never comes. Links are content and pass. + * @param {boolean} [options.noReplace] - publish only onto a free name, and fail + * with EEXIST rather than replacing what is there. The refusal is the rename's + * own, so it answers for the instant nothing was written rather than for a + * look taken beforehand - the app is writing to this volume throughout. + * @param {boolean} [options.merge] - overlay a directory result onto an existing + * directory at the destination rather than replacing it wholesale. Only acts + * when both are directories: a file over a file is still replaced, and a file + * over a directory (or the reverse) is refused either way. Without it a + * directory is never replaced wholesale, since that deletes every entry the + * caller did not name but that sat beside one they did. + * @param {VolumePath} [options.workingDir] - the directory the command runs in, + * defaulting to the volume root. An archiver decides its stored layout from + * where it is run and what it is handed, and zip has no equivalent of tar's + * -C, so this is the only way to make the two agree. + * @param {import('node:stream').Readable} [options.input] - the caller's own + * bytes, streamed into the container, which writes them into staging itself. + * There is no command on this path and that is the point of it: a command + * reading a stream cannot tell a truncated one from a complete one, so it + * would exit successfully on half a file. Requires `publish.staging`, and + * `maxBytes` is enforced as the bytes arrive rather than on what was left + * behind, because here we are the writer. + * @param {boolean} [options.slotHeld] - the caller already holds this app's + * operation slot and will release it. For a request carrying several files: + * they are one operation from the caller's point of view, and taking a slot + * per file would refuse the second one. + * @returns {Promise} resolves when the operation succeeded + */ +async function run(session, argv, options = {}) { + const { + onProgress = null, isCanceled = null, status = 'Working...', + publish = null, mkdirStaging = false, maxBytes = 0, dataOnly = false, + noReplace = false, merge = false, onBytes = null, workingDir = null, input = null, + slotHeld = false, + } = options; + + if (!(session instanceof VolumeSession)) { + throw new Error('run requires a VolumeSession'); + } + + if (workingDir && !(workingDir instanceof VolumePath)) { + throw new Error('workingDir must be a VolumePath'); + } + + const toParam = (arg) => { + if (arg instanceof VolumePath) return arg.containerPath; + if (typeof arg !== 'string') throw new Error('Command arguments must be strings or VolumePath'); + // A string that looks like a host path never belongs in argv: operands are + // expressed relative to the container's view of the volume, and a caller + // passing an absolute path has bypassed the session. + if (path.isAbsolute(arg) && !arg.startsWith(`${WORK_ROOT}/`) && arg !== WORK_ROOT) { + throw new Error(`Refusing an absolute path operand outside ${WORK_ROOT}: ${arg}`); + } + return arg; + }; + + let params = argv.map(toParam); + + if (input) { + if (!publish || !publish.staging) { + throw new Error('input requires publishing through staging'); + } + if (params.length) { + throw new Error('input takes no command - flux-op writes the stream itself'); + } + } + + if (publish) { + if (Boolean(publish.staging) === Boolean(publish.source)) { + throw new Error('publish requires exactly one of staging and source'); + } + const target = publish.staging || publish.source; + if (!(target instanceof VolumePath) || !(publish.destination instanceof VolumePath)) { + throw new Error('publish requires VolumePath operands'); + } + params = [ + 'flux-op', + // Names what an interrupted publish leaves behind, and where. Both are + // given rather than derived from the operand: a move's operand is the + // caller's own path at whatever depth they keep it, so a name derived + // from it collides with what a user might call a folder, and a location + // derived from it lands outside the one directory the sweep reads. + '--id', crypto.randomUUID(), + '--root', WORK_ROOT, + ...(publish.staging ? ['--discard-staging'] : []), + ...(mkdirStaging ? ['--mkdir'] : []), + ...(maxBytes > 0 ? ['--max-bytes', String(Math.floor(maxBytes))] : []), + ...(dataOnly ? ['--data-only'] : []), + ...(noReplace ? ['--no-replace'] : []), + ...(merge ? ['--merge'] : []), + ...(input ? ['--from-stdin'] : []), + toParam(target), + toParam(publish.destination), + '--', + ...params, + ]; + } + + // Before any await. The client can disconnect while the image is being + // fetched or the container created, and an error event with no listener + // ends the process. + const transferred = input ? watchInput(input) : null; + // What the caller has sent. The volume answers for a command writing into + // staging; it does not answer for an upload, where a slow client moves no + // whole block for minutes and reads as a container doing nothing. + const received = { bytes: 0 }; + + // Marked live so a restart mid-operation does not reap this container or sweep + // this staging directory out from under it. A move or a rename publishes the + // caller's own source rather than a staging directory, so there is nothing to + // guard for those - only the operations that write into staging. What is + // registered - and later reclaimed - is the minted ROOT: for an entry nested + // in a staging directory that is the directory, so the scratch a tool wrote + // beside its output goes with it. + const registeredStaging = publish && publish.staging + ? stagingRootOf(publish.staging.hostPath, session.mount) + : null; + const release = slotHeld ? () => {} : acquireSlot(session.identifier); + let container = null; + let registeredContainerId = null; + if (registeredStaging) liveStagingPaths.add(registeredStaging); + let ticker = null; + // Hoisted so the `finally` can reach them: everything this function opens is + // closed there, and a handle declared inside the `try` is out of scope. + let stdin = null; + let exited = null; + // The container's own exit has been observed, so it is reaping itself and + // must not be stopped from here. + let settled = false; + // stop, not kill: this sends SIGTERM first and only escalates to SIGKILL + // after the grace period. flux-op traps the TERM, stops the command and + // reclaims its staging directory - a SIGKILL reaches neither, which is what + // reclaimStaging in the finally is for. + const stopContainer = () => { + if (!container) return; + container.stop({ t: settings().cancelGraceSeconds }).catch(() => {}); + }; + let measuring = false; + // Only scratch we created grows as the work proceeds. A move's operand is + // whole from the first tick, so measuring it would report 100% throughout. + let measurable = Boolean(onBytes && publish && publish.staging); + const stopMeasuring = () => { measurable = false; }; + // What the volume held before this operation wrote anything. + let baseline = null; + // Liveness, kept separately from progress: the last figure the volume + // reported and when it last CHANGED. A delete moves it down and a write moves + // it up; either counts as the operation still doing something. + let lastUsed = null; + let lastChangeAt = process.hrtime.bigint(); + let stalled = false; + let stallReason = null; + // What the caller had sent when the current liveness window opened. Bytes are + // measured against this as a RATE, because "has a byte arrived" is a question + // one byte per window answers forever. + let receivedAtWindowStart = 0; + // Opened wherever progress is seen, so that the next window has to earn its + // own. Kept together because a window that moved its clock without moving its + // byte mark would measure the new window against the old one's total. + const openLivenessWindow = () => { + lastChangeAt = process.hrtime.bigint(); + receivedAtWindowStart = received.bytes; + }; + // Closed once the final figure is in, so a read that started before the + // operation ended cannot report over it. + let reportsClosed = false; + + try { + // Before the mount check, not after: fetching can take seconds, and the + // mount is re-read immediately before the bind on purpose. + // By id rather than by tag: the id is what was verified, and a tag is a + // local name that anything with docker access can move. + const image = await ensureImage(onProgress); + await assertMountIsLive(session); + + // A nested staging entry's directory has to exist before the command does: + // zip cannot create its output's parent. Host-side for the same reasons + // the sweep is, and after the mount check for the same reason everything + // else here is. + if (registeredStaging && registeredStaging !== publish.staging.hostPath) { + const made = await serviceHelper.runCommand('mkdir', { runAsRoot: true, params: ['-p', registeredStaging] }); + if (made.error) throw made.error; + } + + container = await dockerService.createContainer( + containerOptions( + session, + params, + image, + workingDir ? workingDir.containerPath : undefined, + Boolean(input), + ), + ); + registeredContainerId = container.id; + liveContainerIds.add(registeredContainerId); + + // Opened BEFORE start, and on next-exit rather than the default. The + // default condition is "not-running", which a created container already + // satisfies - so a naive wait-before-start returns 0 immediately. Asking + // after start instead would race: a fast command can finish and be reaped + // by AutoRemove before the request arrives, and the exit status is then + // unknowable. + exited = container.wait({ condition: 'next-exit' }); + const output = await collectOutput(container); + stdin = input ? await attachInput(container) : null; + + try { + await container.start(); + } catch (error) { + // AutoRemove only fires for a container that RAN, so one that never + // started stays on the node - stopped, invisible to the app sweeps + // because it is correctly labelled as ours, and holding a reference to + // the executor image that stops anything reclaiming it. Cleared so the + // handler below does not then try to stop a container that is gone. + await container.remove({ force: true }).catch(() => {}); + container = null; + throw error; + } + + if (onProgress) onProgress(status); + + // Everything the volume held before this operation wrote anything. Progress + // is the difference from here, so the app's existing data is not counted as + // this copy's work. + if (measurable) baseline = await volumeUsedBytes(session.mount, fs); + if (baseline === null) stopMeasuring(); + // Timed from here, not from when run() was entered: fetching the image can + // take a minute on a cold node, and that is not the operation making no + // progress. + openLivenessWindow(); + + // One timer serves four jobs: report that the operation is still alive, + // notice a cancellation, read how far it has got, and notice that it has + // stopped getting anywhere. A cancel only sets a flag - the work is not + // interrupted where it stands - so something has to look, and this is + // already looking. + // + // The read is async and the timer is not, so one still in flight when the + // next tick arrives is skipped rather than stacked. + const readVolume = () => { + if (measuring) return; + measuring = true; + volumeUsedBytes(session.mount, fs) + .then((used) => { + if (used === null) return; + if (used !== lastUsed) { + lastUsed = used; + openLivenessWindow(); + } + if (reportsClosed || !measurable) return; + // Never negative: the application is writing to this volume too, and + // deleting something of its own would otherwise send a progress bar + // backwards. + onBytes(Math.max(0, used - baseline)); + }) + .catch(() => {}) + .finally(() => { measuring = false; }); + }; + + ticker = setInterval(() => { + if (isCanceled && isCanceled()) { + log.info(`volumeExecutor - cancel requested, stopping ${session.identifier} operation`); + stopContainer(); + return; + } + + // Stopped because it is getting NOWHERE, not because it has taken a + // while. A wall clock cannot tell a wedged container from a large copy: + // moving 100 GB legitimately outruns any limit short enough to be useful, + // and the 15 minutes this replaced was borrowed from the ceiling on short + // shell commands. The volume's own usage is the honest signal, and it is + // already being read - if it has not moved in either direction for this + // long, nothing is happening. + const { stallTimeoutMs, minUploadBitsPerSecond } = settings(); + const idleMs = Number(process.hrtime.bigint() - lastChangeAt) / 1e6; + if (!stalled && stallTimeoutMs > 0 && idleMs > stallTimeoutMs) { + // The volume has not moved for a whole window. For an upload that is + // not yet an answer: a slow caller fills no whole filesystem block for + // minutes, so the only evidence it is alive is the bytes it has sent - + // asked as a rate over the window that just elapsed, never as "did any + // byte arrive", which one byte per window satisfies until the request + // itself times out hours later. + // + // Only for an operation with a caller attached. A copy or a move sends + // nothing, so measuring it against a floor would stop every one of them + // on the first window. + const carried = received.bytes - receivedAtWindowStart; + const floorBytes = (minUploadBitsPerSecond / 8) * (idleMs / 1000); + if (input && carried >= floorBytes) { + openLivenessWindow(); + } else { + const seconds = Math.round(idleMs / 1000); + stalled = true; + stallReason = input + ? `The upload sent ${carried} bytes in ${seconds}s, under the ${minUploadBitsPerSecond} bit/s a transfer has to keep` + : 'File operation stopped after making no progress'; + log.error(`volumeExecutor - ${session.identifier} ${stallReason}; stopping it`); + stopContainer(); + return; + } + } + + if (onProgress) onProgress(status); + readVolume(); + }, settings().progressIntervalMs); + + // Started only once the ticker is running, so a transfer that stalls is + // still subject to the same "has this got anywhere" check as everything + // else - a client that opens an upload and then sends nothing holds a + // container open otherwise. + const transfer = input ? await feedContainer(stdin, input, transferred, exited, stopContainer, received) : null; + + const result = await exited; + settled = true; + if (stalled) { + throw new Error(stallReason); + } + // An upload that did not arrive is not a failure of the operation - the + // container did exactly what it was told. Reported as itself, because + // "exit 143" tells the caller nothing about their own connection dropping. + if (transfer && !transfer.delivered && transfer.reason) { + throw new Error(`The upload did not complete: ${transfer.reason}`); + } + if (result.StatusCode !== 0) { + throw failureFor(result.StatusCode, output.text.trim()); + } + + // The operation succeeded, so everything it was going to publish IS + // published - and the running figure is whatever the last tick happened to + // read, short of the truth by however much was written after it. Without a + // final reading a completed copy reports some fraction of its own total and + // stays there: the job says Succeeded while the bytes say 87%, which is the + // one moment a progress figure is read most carefully. + // + // Measured at the DESTINATION, because that is where the result now is and + // it is the only exact answer available: publishing is a rename, so staging + // no longer exists, and the volume's own usage includes whatever the + // application wrote alongside us. One walk, once, at the end. + if (measurable) { + if (ticker) { + clearInterval(ticker); + ticker = null; + } + // Closed BEFORE the measurement, not after: a read already in flight + // would otherwise land between the two and report over the final figure. + reportsClosed = true; + stopMeasuring(); + + // Occupied, because every other figure this progress bar is built from + // is: the running one is the volume's own used-bytes, read through + // statfs. A final reading in apparent bytes would make the bar jump at + // the last tick - downwards, and by orders of magnitude for a tree of + // small files - which is the very thing this reading exists to prevent. + const published = await measureTree(publish.destination.hostPath, fs, { occupied: true }) + .catch(() => null); + if (published !== null) onBytes(published); + } + } finally { + if (ticker) clearInterval(ticker); + // A walk still in flight when the operation ends must not report after it: + // its figure is from part-way through, and landing it on a job already + // marked Succeeded would show a finished copy stuck short of its total. + stopMeasuring(); + // The hijacked socket to dockerd. feedContainer closes it on the paths it + // owns, and destroy() is idempotent, so closing it here covers every other + // way out at no cost to those. + if (stdin) stdin.destroy(); + // An unsettled promise with no handler is what node ends the PROCESS over, + // and a container can fail long after the wait was opened. + if (exited) exited.catch(() => {}); + // Left running, it keeps writing to the volume with nobody waiting for it, + // while the slot released below lets another operation start on the same + // app. A container that has already exited reaps itself. + if (container && !settled) stopContainer(); + if (registeredContainerId) liveContainerIds.delete(registeredContainerId); + // Deregistered by the reclaim once it has run, not here: a sweep running + // while the reclaim waits out the container must still skip this path. + if (registeredStaging) reclaimStaging(registeredStaging, exited); + release(); + } +} + +/** + * Remove one staging entry, on the host. + * + * Host-side rather than in a container, for the same reasons the sweep is: the + * path is the mount plus one minted component with nothing to traverse, `rm + * -rf` removes a symlink rather than following it, and a node that cannot + * fetch the executor image still reclaims its debris. Root, because the + * container wrote into it as root and the FluxOS process is not root + * everywhere. + * + * @param {string} hostPath absolute path of the staging entry + */ +async function removeStagingPath(hostPath) { + const result = await serviceHelper.runCommand('rm', { runAsRoot: true, params: ['-rf', hostPath] }); + if (result.error) throw result.error; +} + +/** + * The minted root an operation's staging entry lives under. + * + * What the live registry holds and the reclaim removes: for a plain staging + * entry that is the entry itself, and for one nested in a staging DIRECTORY + * (a compress archive, with the tool's scratch beside it) it is the + * directory, so the scratch goes with the entry. The root must carry the + * minted shape, because deriving a DIFFERENT path than the caller handed over + * and then rm -rf'ing it deserves proof it is ours. + * + * @param {string} hostPath the staging entry's host path + * @param {string} mount the volume root + * @returns {string} the root-level path to register and reclaim + */ +function stagingRootOf(hostPath, mount) { + const [top, ...rest] = path.relative(mount, hostPath).split(path.sep); + if (!rest.length) return hostPath; + if (!isStagingName(top)) { + throw new Error('A nested staging entry must live under a minted staging directory'); + } + return path.join(mount, top); +} + +/** + * Reclaim an operation's staging path once its container is gone. + * + * flux-op removes its own staging on every exit it is allowed to see - but + * SIGKILL is not one of those: the memory cgroup OOM-killing PID 1, a cancel + * whose grace expired mid-removal, a dockerd restart. The path then holds + * whatever was staged - up to the volume's whole free space - under a name the + * owner can neither see (filtered from the listing) nor delete (refused), and + * it used to stay that way until the next FluxOS restart. FluxOS minted the + * name, so FluxOS ends it: once the container's exit has been observed + * (bounded by the cancel grace plus a margin, for a wait whose connection died + * with dockerd), whatever is left at the path is removed. Ordinarily nothing + * is - the publish renamed it away or flux-op removed it - and the rm is a + * no-op. + * + * Deliberately not awaited by run(): the slot is released the moment the + * operation ends, and this finishes on its own clock. The path stays in + * liveStagingPaths until it is done, so a sweep running meanwhile still skips + * it. + * + * @param {string} hostPath the operation's registered staging path + * @param {Promise|null} exited the container's exit subscription, if one was opened + */ +async function reclaimStaging(hostPath, exited) { + try { + if (exited) { + const graceMs = (settings().cancelGraceSeconds * 1000) + 10000; + let deadline; + const deadlinePassed = new Promise((resolve) => { deadline = setTimeout(resolve, graceMs); }); + await Promise.race([exited.catch(() => {}), deadlinePassed]); + clearTimeout(deadline); + } + await removeStagingPath(hostPath); + } catch (error) { + log.warn(`volumeExecutor - could not reclaim ${hostPath}: ${error.message}`); + } finally { + liveStagingPaths.delete(hostPath); + } +} + +/** + * Remove executor containers left running by a FluxOS restart. + * + * A container is detached from the process that started it, so a restart leaves + * one running with nobody waiting for its result. Its staging directory is + * reclaimed separately by sweepStagingDirectories; nothing it wrote is visible + * at a destination path, because publishing is the last thing flux-op does. + * + * Selection is by LABEL, less what a live operation owns. This is the + * ownership-scoped removal that replaced the blanket container prune: it removes + * what FluxOS knows it started and is NOT still running, rather than everything + * docker currently considers unused. The API answers before this runs, so a + * container this process created moments ago is skipped by its id. + * + * @returns {Promise} how many were removed + */ +async function reapOrphanedContainers() { + let containers; + try { + containers = await dockerService.dockerListContainers(true); + } catch (error) { + log.error(`volumeExecutor - could not list containers to reap: ${error.message}`); + return 0; + } + + const orphans = (containers || []).filter( + (container) => container.Labels + && container.Labels['runonflux.role'] === 'fileop' + && !liveContainerIds.has(container.Id), + ); + + let removed = 0; + // eslint-disable-next-line no-restricted-syntax + for (const orphan of orphans) { + try { + // eslint-disable-next-line no-await-in-loop + await dockerService.appDockerForceRemove(orphan.Id, false); + removed += 1; + } catch (error) { + log.warn(`volumeExecutor - could not remove orphaned container ${orphan.Id}: ${error.message}`); + } + } + if (removed) log.info(`volumeExecutor - reaped ${removed} orphaned file-operation container(s)`); + return removed; +} + +/** + * Reclaim what an interrupted operation left on a volume. + * + * One kind of entry, and one rule. flux-op works in `.flux-op-` and + * publishes by exchanging it with the destination in a single atomic step, so a + * crash lands on one side of that step or the other: + * + * before the destination is the caller's data, untouched, and the staging + * entry holds a result that was never published + * after the destination is the result, and the staging entry holds the + * caller's superseded data + * + * In both, the destination is complete and the staging entry is disposable - so + * this deletes staging entries and decides nothing else. It used to have to + * decide: the publish was two renames, and between them the caller's only copy + * sat under a second name with a marker beside it saying where it belonged. + * Working out whether the second rename had happened meant comparing an inode + * number and a timestamp, neither of which is unique, and the cost of being + * wrong was deleting that copy. + * + * Matched against a real identifier shape rather than by prefix, and skipping any + * a live operation is still writing into, because this DELETES what it matches in a + * directory the app owner also writes to: `.flux-op-backups` is a name somebody may + * legitimately have chosen, and a `.flux-op-` an operation of this process + * minted is one it still needs. + * + * On the host rather than in a container: the name came from readdir, so it is + * one component with nothing to traverse, and `rm -rf` unlinks a symlink rather + * than following it. That also means a node which cannot fetch the executor + * image still reclaims its debris. + * + * @param {VolumeSession} session + * @returns {Promise<{removed: Array}>} + */ +async function sweepStagingDirectories(session) { + const { mount } = session; + const entries = await fs.readdir(mount).catch((error) => { + log.warn(`volumeExecutor - could not read ${mount} to sweep: ${error.message}`); + return null; + }); + if (!entries) return { removed: [] }; + + const removed = []; + + const remove = async (name) => { + await removeStagingPath(path.join(mount, name)); + removed.push(name); + }; + + // eslint-disable-next-line no-restricted-syntax + for (const entry of entries) { + try { + if (isStagingName(entry) && !liveStagingPaths.has(path.join(mount, entry))) { + // eslint-disable-next-line no-await-in-loop + await remove(entry); + } + } catch (error) { + log.warn(`volumeExecutor - could not sweep ${entry} in ${mount}: ${error.message}`); + } + } + + if (removed.length) { + log.info(`volumeExecutor - swept ${removed.length} interrupted operation artefact(s) from ${mount}`); + } + return { removed }; +} + +module.exports = { + run, + ensureImage, + serveImageToPeer, + startImagePrefetch, + stopImagePrefetch, + assertCapacity, + reapOrphanedContainers, + sweepStagingDirectories, + acquireSlot, + EXECUTOR_LABELS, +}; diff --git a/ZelBack/src/services/appSystem/volumeReservedNames.js b/ZelBack/src/services/appSystem/volumeReservedNames.js new file mode 100644 index 0000000000..a6620690b0 --- /dev/null +++ b/ZelBack/src/services/appSystem/volumeReservedNames.js @@ -0,0 +1,101 @@ +/** + * The names an app volume's root does not belong to its owner. + * + * Three kinds live there and none of them are the owner's data: what an + * interrupted file operation leaves behind, what syncthing needs in the folder + * it replicates, and what the filesystem keeps for its own recovery. The + * browser reaches that root deliberately - it is how an app with several mounts + * shows them - so every one of these was listable, downloadable, renameable and + * deletable by whoever owns the app. + * + * Removing `.stfolder` stops syncthing replicating the folder at all, and + * replacing `.stignore` changes what leaves the node. An operation's staging + * directory is reserved for a different reason: the boot sweep deletes whatever + * carries that name, so a folder an owner created and called one would be + * deleted out from under them. + * + * ROOT ONLY, deliberately. `.stignore` means something to syncthing at the + * folder root and nowhere else, and the sweep reads only the root - so + * reserving these further down would take names away from the owner inside + * their own data for no benefit, and leave a `photos/.stignore` they could + * create and never manage. + */ + +/** What a staging directory is called while an operation runs. */ +const STAGING_PREFIX = '.flux-op-'; + +/** + * The identifier flux-op names a staging directory with. A randomUUID, so the + * shape is exact. + * + * Names are matched against this rather than by prefix alone because the sweep + * DELETES what it matches, in a directory the app owner can also write to. + * Nothing reserves these prefixes at creation time, so a folder called + * `.flux-op-backups` is a name a user can legitimately choose - and would lose + * on the next restart if a prefix test were the whole rule. + */ +const OPERATION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +const isStagingName = (name) => name.startsWith(STAGING_PREFIX) + && OPERATION_ID.test(name.slice(STAGING_PREFIX.length)); + +/** + * Names something other than FluxOS puts in the volume root and depends on. + * + * `.stfolder` is how syncthing knows the folder is really mounted: without it + * the folder is unhealthy and stops replicating. `.stignore` is what keeps the + * backup directory from being replicated to every other node running the app. + * + * `lost+found` is ext4's, and reserved for the owner's own sake rather than + * ours - fsck puts orphaned inodes there after an unclean shutdown, so a volume + * without one recovers worse. + * + * `.stversions` is deliberately absent: file versioning is not configured on + * any folder FluxOS creates, so that name never appears and reserving it would + * be reserving a name we do not use. + * + * `backup` is deliberately absent for the opposite reason. It sits in the same + * root and FluxOS writes it, but what it holds is the owner's own archives: + * the upload path creates it when a restore needs one, and the backup + * interface lists it through its own endpoint rather than this browser. Hiding + * it would take away something they have a reason to reach, and refusing to + * write it would break the restore that puts files there. + */ +const SYNCTHING_FOLDER_MARKER = '.stfolder'; +const SYNCTHING_IGNORE_FILE = '.stignore'; + +/** + * The .stignore lines FluxOS asserts on every folder it replicates. + * + * `/backup` keeps the owner's local archives off the network. The staging + * pattern keeps an operation's scratch off it: every byte a copy, extract or + * upload stages would otherwise replicate to every peer only to be deleted + * again on publish, and a peer's boot sweep could delete a replicated staging + * directory a live operation on another node still needs. Both are anchored to + * the folder root, so neither takes a name from the owner deeper in their own + * tree. Derived from STAGING_PREFIX so the pattern cannot drift from the names + * the sweep owns. + */ +const SYNCTHING_IGNORE_LINES = ['/backup', `/${STAGING_PREFIX}*`]; + +const FOREIGN_NAMES = new Set([SYNCTHING_FOLDER_MARKER, SYNCTHING_IGNORE_FILE, 'lost+found']); + +/** + * Whether a name in the volume root belongs to something other than the owner. + * @param {string} name - a single path component, not a path + * @returns {boolean} + */ +function isReservedName(name) { + if (typeof name !== 'string' || !name) return false; + return FOREIGN_NAMES.has(name) + || isStagingName(name); +} + +module.exports = { + STAGING_PREFIX, + SYNCTHING_FOLDER_MARKER, + SYNCTHING_IGNORE_FILE, + SYNCTHING_IGNORE_LINES, + isStagingName, + isReservedName, +}; diff --git a/ZelBack/src/services/appSystem/volumeSession.js b/ZelBack/src/services/appSystem/volumeSession.js new file mode 100644 index 0000000000..fa16881117 --- /dev/null +++ b/ZelBack/src/services/appSystem/volumeSession.js @@ -0,0 +1,614 @@ +const path = require('node:path'); +const crypto = require('node:crypto'); +const fs = require('node:fs/promises'); +const deviceHelper = require('../deviceHelper'); +const serviceHelper = require('../serviceHelper'); +const verificationHelper = require('../verificationHelper'); +const { + sanitizePath, verifyRealPath, verifyRealPathOfExistingPath, +} = require('../utils/pathSecurity'); +const { appsFolder, APP_NAME_REGEX, APP_NAME_REGEX_LEGACY } = require('../utils/appConstants'); +const { STAGING_PREFIX, isReservedName } = require('./volumeReservedNames'); +const { measureTree, BLOCK_UNIT } = require('../utils/treeSize'); +const { Privilege, authOf } = require('../utils/privileges'); + +/** + * Where an app's volume is mounted inside the executor container. Operands in + * argv are always expressed relative to this, never as host paths. + */ +const WORK_ROOT = '/work'; + +/** + * Fraction of the required bytes held back on a capacity check. The measurement + * races with whatever the application is writing, and a copy rounds every file + * up to a filesystem block, so an exact fit is not a fit. + */ +const SPACE_HEADROOM = 1.05; + +/** + * A path inside one app's volume that has passed every containment check. + * + * Deliberately not a string. The executor accepts only these, so a handler that + * skips the guards produces code that does not run rather than an endpoint that + * is quietly unsafe - the checks are structural instead of a convention each + * handler has to remember. + * + * Instances come only from VolumeSession. The constructor is not exported. + */ +class VolumePath { + #hostPath; + + #relative; + + constructor(hostPath, relative, brand) { + if (brand !== VolumePath) { + throw new Error('VolumePath cannot be constructed directly - use VolumeSession.resolve'); + } + this.#hostPath = hostPath; + this.#relative = relative; + } + + /** + * The path as the executor container sees it. This is what goes into argv. + * + * Host paths never appear in a command: the container binds the volume at + * WORK_ROOT and has nothing else mounted, so a path that somehow escaped the + * checks would name a file that does not exist in there. That is a second + * barrier which does not depend on the checks being right. + */ + get containerPath() { + return this.#relative === '' ? WORK_ROOT : path.posix.join(WORK_ROOT, this.#relative); + } + + /** The resolved host path. For stat/realpath only - never for argv. */ + get hostPath() { + return this.#hostPath; + } + + /** Path relative to the mount root; '' is the root itself. */ + get relative() { + return this.#relative; + } +} + +/** + * Resolve which volume an (app, component) pair names, WITHOUT authorising. + * + * Internal callers - backup, restore, the reconciler, the boot sweep - act with + * no request and no user, so they need this. Request paths must use openVolume + * instead; see the note there for why that split exists rather than an + * authorise-or-not flag on one function. + * + * @param {string} appname + * @param {string} component - the component, or 'null' for the flat + * single-component form whose identifier is the bare app name + * @returns {Promise<{mount: string, availableBytes: number, identifier: string}>} + */ +async function resolveVolumeMount(appname, component) { + if (!appname) throw new Error('appname parameter is mandatory'); + if (!component) throw new Error('component parameter is mandatory'); + + // Validated before either value reaches a comparison or a path. The charsets + // happen to make the identifier unambiguous - neither may contain the + // underscore that separates them - but that is a property to assert, not one + // to rely on silently. + if (!APP_NAME_REGEX.test(appname)) { + throw new Error('appname contains disallowed characters'); + } + if (component !== 'null' && !APP_NAME_REGEX_LEGACY.test(component)) { + throw new Error('component contains disallowed characters'); + } + + const identifier = component === 'null' ? `flux${appname}` : `flux${component}_${appname}`; + + // SELECTED from the kernel's mount table, never built with path.join. FluxOS + // holds the docker socket, so whatever decides a bind source decides host + // access; sourcing the candidates from findmnt means a request can only ever + // name a filesystem that is already mounted as an app volume. The worst a + // hostile appname achieves is matching nothing. + const mounts = await deviceHelper.listMountedFilesystems(); + const matched = mounts.filter((mount) => path.basename(mount.target) === identifier); + + if (!matched.length) throw new Error('Application volume not found'); + // Never [0]. One identifier resolving to several mounts means the assumption + // behind this lookup no longer holds, and picking one silently operates on + // arbitrary data - a restore into the wrong one overwrites what is live. + if (matched.length > 1) { + throw new Error(`${identifier} resolves to ${matched.length} mounts; refusing to guess`); + } + + const [volume] = matched; + // A mount table row that is not under the apps folder is not an app volume, + // whatever its basename looks like. + if (!volume.target.startsWith(appsFolder)) { + throw new Error(`${identifier} is mounted outside the apps folder; refusing to use it`); + } + + return { mount: volume.target, availableBytes: volume.availableBytes, identifier }; +} + +/** + * A resolved, authorised handle on one app's volume. + * + * Obtain one with openVolume. Every path that reaches the executor comes from + * resolve() or staging() on one of these. + */ +class VolumeSession { + #mount; + + #availableBytes; + + #identifier; + + #owner; + + constructor(mount, availableBytes, identifier, owner, brand) { + if (brand !== VolumeSession) { + throw new Error('VolumeSession cannot be constructed directly - use openVolume'); + } + this.#mount = mount; + this.#availableBytes = availableBytes; + this.#identifier = identifier; + this.#owner = owner; + } + + /** Host mount path. This, and only this, is the executor's bind source. */ + get mount() { + return this.#mount; + } + + /** Free bytes on the volume, from the mount table row that resolved it. */ + get availableBytes() { + return this.#availableBytes; + } + + get identifier() { + return this.#identifier; + } + + /** + * The FluxID this session was opened by, or null. + * + * Carried so an operation started from it is registered against the same + * identity the status resource checks on a poll - otherwise a caller could + * not read back the job they just started. + */ + get owner() { + return this.#owner; + } + + /** + * Turn a caller-supplied relative path into a VolumePath, or throw. + * + * Runs the lexical checks (null bytes, backslashes, absolute paths, traversal, + * the character allowlist) and then the symlink-resolved containment check. A + * directory inside the mount can itself be a symlink pointing anywhere on the + * host, so the string check alone is not sufficient. + * + * @param {string} userPath - relative to the mount root + * @param {{mustExist?: boolean, allowRoot?: boolean, allowReserved?: boolean}} [options] + * @returns {Promise} + */ + async resolve(userPath, options = {}) { + const { mustExist = false, allowRoot = false, allowReserved = false } = options; + + const hostPath = sanitizePath(userPath, this.#mount); + const relative = path.relative(this.#mount, hostPath); + + if (!allowRoot && relative === '') { + throw new Error('Refusing to operate on the volume root'); + } + + // The parent is checked because a directory inside the mount can itself be + // a symlink pointing anywhere on the host. The volume root is the exception: + // its parent is outside the mount by definition, and the mount IS the trust + // boundary, so it is verified directly instead. + const parent = relative === '' ? hostPath : path.dirname(hostPath); + await verifyRealPathOfExistingPath(parent, this.#mount); + + // Names in the volume root that are not the owner's: syncthing's control + // files, the filesystem's own recovery directory, and what an interrupted + // operation leaves for the boot sweep. Refused here rather than at each + // endpoint because every one of them arrives through this method, and being + // able to write one of these is being able to stop a folder replicating or + // to hand the sweep its input. + // + // Root only: these mean something to the reader that looks for them there + // and nowhere else, and reserving them deeper would take names away from + // the owner inside their own data. + // + // Which root is decided from where the path LANDS, not from how it was + // spelled. An app can `ln -s . here` inside its own volume and reach the + // root as `here/.stignore`: that carries a separator, so a test on the + // string never fires, and it resolves inside the mount, so containment is + // satisfied. Both are true, and neither is the question being asked. + // + // Only for a name that is reserved at all, so the ordinary path costs + // nothing. verifyRealPath returns a path it cannot resolve unchanged, so a + // name under a directory that does not exist yet compares as itself and + // stays the owner's. + if (!allowReserved && relative !== '' && isReservedName(path.basename(hostPath))) { + const [realParent, realMount] = await Promise.all([ + verifyRealPath(parent, this.#mount), + verifyRealPath(this.#mount, this.#mount), + ]); + if (realParent === realMount) { + throw new Error(`${relative} is not an application's to write`); + } + } + + // Operations act on a link rather than through it (mv, rm and cp -a all + // do), so verifying a link's TARGET would reject legitimate work on a + // dangling one. The parent check above still holds. + let isSymbolicLink = false; + try { + const stats = await fs.lstat(hostPath); + isSymbolicLink = stats.isSymbolicLink(); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + if (mustExist) throw new Error('Source does not exist'); + } + if (!isSymbolicLink) { + await verifyRealPath(hostPath, this.#mount); + } + + return new VolumePath(hostPath, relative, VolumePath); + } + + /** + * Resolve a source and destination together, applying every guard that only + * makes sense for a pair. + * + * All the two-operand endpoints go through here so the guard set stays in one + * reviewable place rather than being re-inlined per endpoint. + * + * `destination` is the full target path INCLUDING the new name, not the + * parent directory - which is what keeps -T semantics identical between copy + * and move and removes the paste-into versus paste-as ambiguity. + * + * Whether an occupied destination is refused is NOT decided here. It is the + * publish's `noReplace`, which refuses as part of the rename rather than from + * a look taken beforehand - the application whose volume this is writes to it + * throughout, so a verdict reached here is about a moment that has passed by + * the time the container runs. + * + * @param {string} source + * @param {string} destination + * @returns {Promise<{source: VolumePath, destination: VolumePath}>} + */ + async pair(source, destination) { + const from = await this.resolve(source, { mustExist: true }); + const to = await this.resolve(destination); + + if (from.hostPath === to.hostPath) { + throw new Error('Source and destination are the same'); + } + + // '' means identical; a '..'-prefixed or absolute result means the two sit + // on separate branches. Anything else means one holds the other. + // + // Compared this way rather than by string prefix, which reads photos-2024 as + // living inside photos. + const holds = (ancestor, descendant) => { + const within = path.relative(ancestor.hostPath, descendant.hostPath); + return Boolean(within) && !within.startsWith('..') && !path.isAbsolute(within); + }; + + // For a directory copy this recurses until the volume fills. + if (holds(from, to)) { + throw new Error('Destination is inside the source'); + } + + // The other direction, which only overwrite lets through: replacing photos + // with photos/2024. The executor cannot carry it out - displacing the + // destination takes the source away inside it, so the publish stops between + // its two renames and the caller's whole folder is parked under a name the + // reserved names hide from them until the next boot sweep. Completing it + // instead would delete everything else in photos, which they never named. + // + // Refused here as well as in the image so the caller is told in a sentence + // rather than through a container's exit code. The image refuses it too, + // because that invariant is not one it should hold on trust from a caller. + if (holds(to, from)) { + throw new Error('Destination contains the source'); + } + + return { source: from, destination: to }; + } + + /** + * A fresh staging directory inside the volume. + * + * Everything is written here and renamed into place only on success, which is + * what makes the guarantees the endpoints advertise true: a cancelled copy + * and an extraction that trips its size cap on entry 900 of 1000 both leave + * nothing the user can see, and an operation abandoned by a FluxOS restart + * leaves a directory a boot sweep can recognise and reclaim. + * + * @returns {VolumePath} + */ + staging() { + const name = `${STAGING_PREFIX}${crypto.randomUUID()}`; + return new VolumePath(path.join(this.#mount, name), name, VolumePath); + } + + /** + * A staging DIRECTORY with the operation's result entry inside it. + * + * For an operation whose tool writes scratch beside its output: Info-ZIP + * builds an archive in a temp file in the output's directory, and at the + * volume root that temp - ziXXXXXX, outside the shape the sweep may delete - + * survived a SIGKILL forever, replicated mid-write, and sat in the owner's + * listing. Inside the minted directory, the temp, a partial result and the + * entry are one reclaim. The executor creates the directory and reclaims it + * whole. + * + * The entry is named after the destination it will become, because the tool + * writes THAT name and flux-op inspects it by name afterwards: zip appends + * .zip to a name carrying no extension, so an entry called `result` is + * written as `result.zip` and the inspection then lstats a path that does not + * exist. Using the destination's own basename - which archiveFormat has + * already required to be a valid archive extension - means the tool is given + * the exact name it writes, and nothing is mutated. + * + * Takes the destination as a VolumePath, not a string: the name comes from a + * path resolve() already produced, so this is not a second place a caller + * string becomes a trusted path, and a basename cannot traverse. + * + * @param {VolumePath} destination the resolved path this result becomes + * @returns {{directory: VolumePath, entry: VolumePath}} + */ + stagingDir(destination) { + if (!(destination instanceof VolumePath)) { + throw new Error('stagingDir must be given the destination VolumePath to name its result after'); + } + const name = `${STAGING_PREFIX}${crypto.randomUUID()}`; + const relative = path.posix.join(name, path.posix.basename(destination.relative)); + return { + directory: new VolumePath(path.join(this.#mount, name), name, VolumePath), + entry: new VolumePath(path.join(this.#mount, relative), relative, VolumePath), + }; + } + + /** + * Whether this path is a directory, following nothing. + * + * A symlink answers false however it resolves, which is what the callers + * want: an archiver is given the link itself, not the tree behind it. + * + * @param {VolumePath} volumePath + * @returns {Promise} + */ + // eslint-disable-next-line class-methods-use-this + async isDirectory(volumePath) { + if (!(volumePath instanceof VolumePath)) { + throw new Error('isDirectory requires a VolumePath'); + } + const stats = await fs.lstat(volumePath.hostPath).catch((error) => { + if (error.code === 'ENOENT') throw new Error('Source does not exist'); + throw error; + }); + return stats.isDirectory(); + } + + /** + * The directory containing this path. + * + * Built rather than resolved: the path it derives from has already passed + * every guard, and its parent was itself checked for containment on the way + * through. Re-resolving would re-run those checks against a volume the app + * can change underneath us, which is a second answer to a settled question + * rather than a stronger one. + * + * @param {VolumePath} volumePath + * @returns {VolumePath} + */ + parent(volumePath) { + if (!(volumePath instanceof VolumePath)) { + throw new Error('parent requires a VolumePath'); + } + const relative = path.dirname(volumePath.relative); + // dirname of a top-level entry is '.', which as a relative path means the + // mount root - the form VolumePath spells ''. + const normalised = relative === '.' ? '' : relative; + return new VolumePath(path.join(this.#mount, normalised), normalised, VolumePath); + } + + /** + * Byte cost of an operation whose source is this path. + * + * Symlinks measure zero - cp -a and the archivers copy the link, not what it + * points at. + * + * NOTE: this reads the volume from the FluxOS process. When FluxOS is demoted + * to an unprivileged system user it will no longer be able to, and this moves + * into the executor alongside everything else that touches app data. + * + * @param {VolumePath} volumePath + * @returns {Promise} bytes + */ + // eslint-disable-next-line class-methods-use-this + async measure(volumePath) { + if (!(volumePath instanceof VolumePath)) { + throw new Error('measure requires a VolumePath'); + } + const stats = await fs.lstat(volumePath.hostPath).catch((error) => { + if (error.code === 'ENOENT') throw new Error('Source does not exist'); + throw error; + }); + if (stats.isSymbolicLink()) return 0; + + // What it OCCUPIES, not what it says. This figure is handed to + // requireSpace, which compares it against the volume's free space - a count + // of blocks - and a file occupies whole blocks. A directory of ten thousand + // one-byte files reports ten kilobytes for itself and costs forty megabytes, + // so measuring what the files say passes a copy that cannot fit. + const size = stats.isDirectory() + ? await measureTree(volumePath.hostPath, fs, { occupied: true }) + : stats.blocks * BLOCK_UNIT; + + // An ESTIMATE, and low rather than high when it is wrong. The lstat above + // throws on a source that cannot be reached at all, but measureTree skips + // an entry it cannot stat and walks nothing under a directory it cannot + // open - and this runs in the FluxOS process, root on ArcaneOS and an + // ordinary user elsewhere, so a directory the app made private to its own + // uid is exactly that case. + // + // Which is what it is for: refusing an operation early and in a sentence, + // before anything starts. What makes the operation SAFE is the byte ceiling + // its executor run carries - applied to what actually lands, by a container + // that can read every part of the volume. + return size; + } + + /** + * Throw unless `requiredBytes` plus headroom fits in the volume. + * + * Fails closed: a capacity that cannot be established is a refusal, because + * an operation that runs out of space partway leaves a partial tree the user + * has to identify and clean up, having consumed the space it failed to need. + * + * @param {number} requiredBytes + */ + requireSpace(requiredBytes) { + if (!Number.isFinite(this.#availableBytes)) { + throw new Error('Unable to determine free space on the application volume'); + } + const needed = Math.ceil(requiredBytes * SPACE_HEADROOM); + if (needed > this.#availableBytes) { + throw new Error(`Not enough free space: ${needed} bytes required, ${this.#availableBytes} bytes available`); + } + } + + /** + * Throw unless the volume has room for anything at all. + * + * For the operations whose size cannot be known in advance - an extraction, + * an upload - where the byte ceiling is the only bound and IS the volume's + * free space. A full volume makes that ceiling zero, and a ceiling of zero is + * how the executor and the image both spell "no ceiling was asked for", so + * the one operation with nothing else protecting it would run unbounded until + * the filesystem refused a write. Refused here instead, before a container + * starts, which is also where the caller gets a sentence rather than whatever + * tar says about ENOSPC. + */ + requireCapacity() { + if (!Number.isFinite(this.#availableBytes)) { + throw new Error('Unable to determine free space on the application volume'); + } + if (this.#availableBytes <= 0) { + throw new Error('No free space on the application volume'); + } + } +} + +/** + * The authorised way to reach an app's volume from a request. + * + * --- Why authorisation lives here and not in each handler --- + * + * The check this makes is OBJECT-level - "is this caller the owner of THIS + * app" - not route-level - "is this caller logged in". The industry splits + * those deliberately, and puts them in different places: + * + * route-level middleware, above the handler (express middleware, rails + * before_action, spring security filters) + * object-level fused with the lookup that fetches the object, so the + * unauthorised object cannot be obtained at all (django's + * get_object_or_404 over a user-scoped queryset, rails + * pundit's policy_scope) + * + * Object-level checks are fused because the failure mode of scattering them is + * that one gets forgotten, and that omission is invisible - the endpoint works + * perfectly for its author. It is the most commonly missed check in the field: + * Broken Object Level Authorization is number one on the OWASP API Security + * Top 10. + * + * --- Why two functions rather than one with a flag --- + * + * resolveVolumeMount exists unauthorised for callers that genuinely have no + * user: backup, restore, the reconciler, the boot sweep. Serving both from one + * function would mean a `skipAuth` argument, and a guarantee that can be + * switched off by a boolean is weaker than no guarantee at all, because it + * reads as safe. Two names, one of which is the only thing request paths may + * use, is the same shape django and rails settled on. + * + * @param {object} req - express request. appname and component are read from + * the JSON body for the endpoints that POST one, and from params or query for + * the older GET endpoints that still take them there. + * @param {{privilege?: string}} [options] + * @returns {Promise} + */ +async function openVolume(req, options = {}) { + // This default is the gate on eight endpoints that write to a customer's app + // volume - create, rename, move, copy, compress, extract, upload and remove - + // and no caller overrides it. appownerorfluxteam refuses the node operator, + // because uploading into, rewriting or deleting the data of an app you only + // host is not the node operator's to do. + const { privilege = Privilege.APP_OWNER_OR_FLUX_TEAM } = options; + + // ensureObject for url-encoded parity: express.json() populates req.body for + // application/json, and a form-encoded caller arrives as a string. + const body = serviceHelper.ensureObject(req.body) || {}; + const appname = req.params.appname || req.query.appname || body.appname || ''; + const component = req.params.component || req.query.component || body.component || ''; + + const authorized = await verificationHelper.verifyPrivilege(privilege, authOf(req), { appName: appname }); + if (!authorized) { + // Carries the code so this reaches a client as the body + // messageHelper.errUnauthorizedMessage() has always produced. Handlers used + // to call that directly; they now throw, and dropping the 401 here would + // silently change what every existing caller reads. + const error = new Error('Unauthorized. Access denied.'); + error.name = 'Unauthorized'; + error.code = 401; + throw error; + } + + const { mount, availableBytes, identifier } = await resolveVolumeMount(appname, component); + // Read after authorisation succeeded, so this is the identity that passed it. + const auth = serviceHelper.ensureObject(authOf(req)); + const owner = (auth && auth.zelid) || null; + return new VolumeSession(mount, availableBytes, identifier, owner, VolumeSession); +} + +/** + * A session on a volume named by the mount table rather than by a request. + * + * The boot sweep has no user to authorise and no app name to look up - it walks + * the mount table and acts on whatever app volumes are mounted. It still needs a + * session, because every path the executor accepts comes from resolve() on one + * of these: a caller with no user is exactly the caller that must not be given a + * way around the containment checks. + * + * @param {{target: string, availableBytes: number}} mountRow - a row from + * deviceHelper.listMountedFilesystems + * @returns {VolumeSession} + */ +function sessionForMountedVolume(mountRow) { + const target = mountRow && mountRow.target; + // The same rule resolveVolumeMount applies to what it selected: a mount that + // is not under the apps folder is not an app volume, whatever it is called. + if (!target || !target.startsWith(appsFolder)) { + throw new Error(`${target} is not an app volume mount; refusing to use it`); + } + return new VolumeSession( + target, + mountRow.availableBytes, + path.basename(target), + // No user. The sweep acts for the node, and openVolume is the only path + // that may record an owner, because it is the only one that checked. + null, + VolumeSession, + ); +} + +module.exports = { + openVolume, + resolveVolumeMount, + sessionForMountedVolume, + VolumePath, + VolumeSession, + WORK_ROOT, + SPACE_HEADROOM, +}; diff --git a/ZelBack/src/services/appTamperingBlocklistService.js b/ZelBack/src/services/appTamperingBlocklistService.js index 39bc5d2300..655191536f 100644 --- a/ZelBack/src/services/appTamperingBlocklistService.js +++ b/ZelBack/src/services/appTamperingBlocklistService.js @@ -7,8 +7,13 @@ const generalService = require('./generalService'); const daemonServiceMiscRpcs = require('./daemonService/daemonServiceMiscRpcs'); const benchmarkService = require('./benchmarkService'); -const BLOCKLIST_URL = `${config.github.rawBaseUrl}/helpers/tamperingblockednodes.json`; +const BLOCKLIST_URL = `${config.policy.baseUrl}/tamperingblockednodes.json`; const CHECK_INTERVAL_MS = 12 * 60 * 60 * 1000; // 12 hours +// How often to look at the DOS slot while waiting for another owner to let go +// of it. Purely local - it reads the slot and nothing else, so it costs no +// blocklist fetch and no benchmark call, which is why it can run this often +// against a 12-hourly enforcement cadence. +const SLOT_WATCH_MS = 60 * 1000; // 60s const SYNC_POLL_MS = 60 * 1000; // 60s while waiting for daemon sync const TAMPER_SCORE_THRESHOLD = 10; const DOS_MESSAGE_PREFIX = 'Node flagged via tampering blocklist'; @@ -16,6 +21,14 @@ const DOS_MESSAGE_PREFIX = 'Node flagged via tampering blocklist'; const tamperingEventsCollection = config.database.local.collections.appTamperingEvents; let intervalHandle = null; +let slotWatchHandle = null; +// The DOS this node should be under, held here while another owner has the +// single sticky slot. Enforcement runs every 12 hours, so without this a +// blocklisted node that the other owner later RELEASES - a residential verdict +// flipping to datacenter clears its own sticky - sits at DOS 0 taking apps +// until the next 12-hourly tick. The slot is watched instead, and claimed +// within a minute of it coming free. +let deferredDosMessage = null; let ourDosActive = false; let stopping = false; let syncWaitTimer = null; @@ -31,18 +44,40 @@ function isOurStickyDos() { } /** - * Fetch the manually-curated txhash blocklist from the RunOnFlux repo. - * Returns [] on any failure so the caller never crashes the enforcer loop. + * Give up the DOS this service is holding. The slot is only cleared when the + * message in it is still ours: the slot holds one message and has more than one + * enforcer writing to it, so clearing on our own `ourDosActive` alone would drop + * another owner's DOS on the floor. Dropping our claim is all we are entitled to + * do once the slot has changed hands. + * @param {string} reason Logged context for the release. + */ +function releaseOurDos(reason) { + if (isOurStickyDos()) { + log.info(`appTamperingBlocklist - clearing sticky DOS (${reason})`); + fluxNetworkHelper.clearStickyDosMessage(); + ourDosActive = false; + return; + } + if (ourDosActive) { + log.info(`appTamperingBlocklist - our DOS was replaced by another owner, releasing our claim only (${reason})`); + ourDosActive = false; + } +} + +/** + * Fetch the manually-curated txhash blocklist from the policy repo. + * Returns null on any failure - could-not-fetch is not an empty list, and the + * enforcer must distinguish them or an outage clears an active DOS. */ async function fetchBlocklist() { try { const res = await serviceHelper.axiosGet(BLOCKLIST_URL); if (res && Array.isArray(res.data)) return res.data; log.warn('appTamperingBlocklist - unexpected response shape from blocklist URL'); - return []; + return null; } catch (error) { log.warn(`appTamperingBlocklist - failed to fetch blocklist: ${error.message}`); - return []; + return null; } } @@ -83,7 +118,11 @@ async function isArcaneOs() { async function computeTamperScore() { try { const db = dbHelper.databaseConnection(); - if (!db) return 0; + // null, never 0: a score this node could not read is not a score of zero, + // and returning zero would take the clear branch and release a node this + // service had deliberately DOSed - the same distinction the blocklist + // fetch makes between could-not-ask and nothing-listed + if (!db) return null; const database = db.db(config.database.local.database); const pipeline = [ { $match: { schemaVersion: { $gte: 1 } } }, @@ -93,7 +132,7 @@ async function computeTamperScore() { return incidents.reduce((score, incident) => score + (incident.severity ?? 0), 0); } catch (error) { log.warn(`appTamperingBlocklist - failed to compute tamper score: ${error.message}`); - return 0; + return null; } } @@ -164,14 +203,45 @@ async function enforceBlocklist() { return; } - const listed = Array.isArray(blocklist) && blocklist.includes(myTxhash); + // An unreadable blocklist is not an empty one. Falling through on null would + // take the clear branch below and release a node this service had already + // DOSed - an outage would undo enforcement rather than postpone it. + if (blocklist === null) { + log.warn('appTamperingBlocklist - blocklist unavailable, skipping this tick'); + return; + } + + // Same rule for the other input to the decision: an unreadable score cannot + // clear an active DOS. + if (tamperScore === null) { + log.warn('appTamperingBlocklist - tamper score unavailable, skipping this tick'); + return; + } + + const listed = blocklist.includes(myTxhash); const exceedsThreshold = tamperScore > TAMPER_SCORE_THRESHOLD; const shouldDos = listed && exceedsThreshold; log.info(`appTamperingBlocklist - txhash=${myTxhash} listed=${listed} score=${tamperScore} shouldDos=${shouldDos}`); if (shouldDos) { + // Another owner's DOS already has this node out of service for its own + // reason. Taking the single slot from it would leave that owner unable to + // recognise or release its own state, and the node is DOSed either way - + // so leave it and re-check next tick. const message = `${DOS_MESSAGE_PREFIX}: tamper score ${tamperScore}, txhash ${myTxhash}`; + const sticky = fluxNetworkHelper.getStickyDosMessage(); + if (sticky && !isOurStickyDos()) { + log.info('appTamperingBlocklist - another sticky DOS is active, not overwriting it; watching for the slot'); + // Remembered, and the slot watched. The score in it can be a few hours + // stale by the time the slot frees, which is the right trade: the next + // full tick refreshes the message, and the node is one this build has + // already determined should be out of service. + deferredDosMessage = message; + startSlotWatch(); + return; + } + stopSlotWatch(); fluxNetworkHelper.setStickyDosMessage(message); fluxNetworkHelper.setStickyDosStateValue(100); ourDosActive = true; @@ -179,10 +249,42 @@ async function enforceBlocklist() { return; } - if (ourDosActive || isOurStickyDos()) { - log.info(`appTamperingBlocklist - clearing sticky DOS (listed=${listed}, score=${tamperScore})`); - fluxNetworkHelper.clearStickyDosMessage(); - ourDosActive = false; + stopSlotWatch(); + releaseOurDos(`listed=${listed}, score=${tamperScore}`); +} + +/** + * Claim the DOS slot the moment the owner holding it lets go. + * + * Local only: it reads the sticky message and nothing else, so it costs no + * blocklist fetch, no benchmark call and no RPC. + */ +function claimSlotIfFree() { + if (!deferredDosMessage) { + stopSlotWatch(); + return; + } + const sticky = fluxNetworkHelper.getStickyDosMessage(); + if (sticky && !isOurStickyDos()) return; + fluxNetworkHelper.setStickyDosMessage(deferredDosMessage); + fluxNetworkHelper.setStickyDosStateValue(100); + ourDosActive = true; + log.error(`${deferredDosMessage} (claimed after another owner released the slot)`); + deferredDosMessage = null; + stopSlotWatch(); +} + +function startSlotWatch() { + if (slotWatchHandle || stopping) return; + slotWatchHandle = setInterval(claimSlotIfFree, SLOT_WATCH_MS); + if (slotWatchHandle.unref) slotWatchHandle.unref(); +} + +function stopSlotWatch() { + deferredDosMessage = null; + if (slotWatchHandle) { + clearInterval(slotWatchHandle); + slotWatchHandle = null; } } @@ -224,6 +326,7 @@ async function start() { function stop() { stopping = true; + stopSlotWatch(); if (syncWaitTimer) { clearTimeout(syncWaitTimer); syncWaitTimer = null; @@ -247,6 +350,8 @@ module.exports = { start, stop, enforceBlocklist, + // Test seam: the slot watch is otherwise only driven by its own timer. + claimSlotIfFree, fetchBlocklist, computeTamperScore, getMyTxhash, diff --git a/ZelBack/src/services/appTamperingDetectionService.js b/ZelBack/src/services/appTamperingDetectionService.js index bde2b290c6..2704067250 100644 --- a/ZelBack/src/services/appTamperingDetectionService.js +++ b/ZelBack/src/services/appTamperingDetectionService.js @@ -461,11 +461,81 @@ async function checkNodeReboot() { } } +/** + * Merge duplicate incident rollups, then assert the unique index they need. + * + * The rollup keys on (appName, eventType, incidentKey) and each doc carries a + * `count` recordEvent increments. A node upgrading from before the unique index + * - or two recorders racing on a fresh key before it existed - can hold two + * rollups for one incident, and the unique build then fails on that dirty data. + * Dropping one would UNDERCOUNT the incident, so this sums the counts and keeps + * the widest firstSeen/lastSeen window rather than deduping blindly - which is + * why it lives here, with the collection, instead of as a generic ensureIndex + * recovery. Built before the boot sweeps record their own incidents, so the + * unique index the upsert relies on is already in place. + * + * Low-stakes local telemetry on a 30-day TTL, so a failure logs and is left for + * the next boot rather than wedging startup - the disposition its owner chooses, + * the same one appsRuntimeState.prepareCollection takes. + */ +async function prepareIncidentRollup() { + try { + const db = dbHelper.databaseConnection(); + if (!db) { + log.warn('appTamperingDetection - DB not available, skipping incident rollup preparation'); + return; + } + const database = db.db(config.database.local.database); + const rollups = await dbHelper.findInDatabase(database, tamperingEventsCollection, { incidentKey: { $exists: true } }); + + const byKey = new Map(); + // eslint-disable-next-line no-restricted-syntax + for (const doc of rollups) { + // NUL as the separator, written as an escape: it cannot occur in any of the + // three parts, so no combination of them can collide on one key. The raw + // byte in the source classifies the whole file as binary, and grep then + // finds nothing in it while reporting success. + const key = `${doc.appName}\x00${doc.eventType}\x00${doc.incidentKey}`; + const group = byKey.get(key) || []; + group.push(doc); + byKey.set(key, group); + } + + // eslint-disable-next-line no-restricted-syntax + for (const [, group] of byKey) { + if (group.length <= 1) continue; + const query = { appName: group[0].appName, eventType: group[0].eventType, incidentKey: group[0].incidentKey }; + const newest = group.slice().sort((a, b) => new Date(b.lastSeen || 0).getTime() - new Date(a.lastSeen || 0).getTime())[0]; + const earliest = group.reduce((min, d) => (new Date(d.firstSeen || 0) < new Date(min) ? d.firstSeen : min), newest.firstSeen); + const merged = { + ...newest, + count: group.reduce((sum, d) => sum + (d.count || 0), 0), + firstSeen: earliest, + lastSeen: newest.lastSeen, + }; + delete merged._id; + log.warn(`appTamperingDetection - merged ${group.length} duplicate incident rollups for ${query.appName}/${query.eventType} (count=${merged.count})`); + // eslint-disable-next-line no-await-in-loop + await dbHelper.removeDocumentsFromCollection(database, tamperingEventsCollection, query); + // eslint-disable-next-line no-await-in-loop + await dbHelper.updateOneInDatabase(database, tamperingEventsCollection, query, { $set: merged }, { upsert: true }); + } + + await database.collection(tamperingEventsCollection).createIndex( + { appName: 1, eventType: 1, incidentKey: 1 }, + { unique: true, partialFilterExpression: { incidentKey: { $exists: true } }, name: 'incident_upsert' }, + ); + } catch (error) { + log.error(`appTamperingDetection - failed to prepare incident rollup: ${error.message}`); + } +} + module.exports = { recordEvent, getEvents, isNetworkMissingError, checkNodeReboot, + prepareIncidentRollup, startIdentityBackfill, deriveMainAppName, EVENT_SEVERITY, diff --git a/ZelBack/src/services/backupRestoreService.js b/ZelBack/src/services/backupRestoreService.js index e412a37408..b863e1b843 100644 --- a/ZelBack/src/services/backupRestoreService.js +++ b/ZelBack/src/services/backupRestoreService.js @@ -2,10 +2,11 @@ const log = require('../lib/log'); const path = require('path'); const messageHelper = require('./messageHelper'); const verificationHelper = require('./verificationHelper'); -const serviceHelper = require('./serviceHelper'); +const { sendFile } = require('./utils/fileTransfer'); const IOUtils = require('./IOUtils'); const fs = require('fs').promises; const { sanitizePath, verifyRealPath } = require('./utils/pathSecurity'); +const { Privilege, authOf } = require('./utils/privileges'); const fluxDirPath = process.env.FLUXOS_PATH || path.join(process.env.HOME, 'zelflux'); // ToDo: Fix all the string concatenation in this file and use path.join() @@ -82,18 +83,20 @@ async function getVolumeDataOfComponent(req, res) { if (!appname || !component) { throw new Error('Both the appname and component parameters are required'); } - const authorized = res ? await verificationHelper.verifyPrivilege('appownerabove', req, appname) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (authorized === true) { - const dfInfoData = await IOUtils.getVolumeInfo(appname, component, multiplier, decimal, fields); - if (dfInfoData === null) { + const { error, mounts } = await IOUtils.getVolumeInfo(appname, component, multiplier, decimal, fields); + // A mount table that could not be read and a volume that is not mounted + // are both "no data to report" to this endpoint. + if (error || !mounts.length) { throw new Error('No matching mount found'); } - const response = messageHelper.createDataMessage(dfInfoData[0]); - return res ? res.json(response) : response; + const response = messageHelper.createDataMessage(mounts[0]); + return res.json(response); // eslint-disable-next-line no-else-return } else { const errorResponse = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } catch (error) { log.error(error); @@ -102,7 +105,7 @@ async function getVolumeDataOfComponent(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -130,7 +133,7 @@ async function getLocalBackupList(req, res) { if (!path) { throw new Error('path and appname parameters are required'); } - const authorized = res ? await verificationHelper.verifyPrivilege('appownerabove', req, appname) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (authorized === true) { if (!pathValidation(vPath)) { throw new Error('Path validation failed..'); @@ -140,11 +143,11 @@ async function getLocalBackupList(req, res) { throw new Error('No matching mount found'); } const response = messageHelper.createDataMessage(listData); - return res ? res.json(response) : response; + return res.json(response); // eslint-disable-next-line no-else-return } else { const errorResponse = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } catch (error) { log.error(error); @@ -153,7 +156,7 @@ async function getLocalBackupList(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -180,18 +183,18 @@ async function getRemoteFileSize(req, res) { if (!fileurl || !appname) { throw new Error('fileurl and appname parameters are mandatory'); } - const authorized = res ? await verificationHelper.verifyPrivilege('appownerabove', req, appname) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (authorized === true) { const fileSize = await IOUtils.getRemoteFileSize(fileurl, multiplier, decimal, number); if (fileSize === false) { throw new Error('Error fetching file size'); } const response = messageHelper.createDataMessage(fileSize); - return res ? res.json(response) : response; + return res.json(response); // eslint-disable-next-line no-else-return } else { const errorResponse = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } catch (error) { log.error(error); @@ -200,7 +203,7 @@ async function getRemoteFileSize(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -221,7 +224,7 @@ async function removeBackupFile(req, res) { if (!filepath || !appname) { throw new Error('filepath and appname parameters are mandatory'); } - const authorized = res ? await verificationHelper.verifyPrivilege('appownerabove', req, appname) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (authorized === true) { if (!pathValidation(filepath)) { throw new Error('Path validation failed..'); @@ -241,7 +244,7 @@ async function removeBackupFile(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -262,7 +265,7 @@ async function downloadLocalFile(req, res) { if (!filepath || !appname) { throw new Error('filepath and appname parameters are mandatory'); } - const authorized = await verificationHelper.verifyPrivilege('appownerabove', req, appname); + const authorized = await verificationHelper.verifyPrivilege(Privilege.APP_OWNER_OR_FLUX_TEAM, authOf(req), { appName: appname }); if (authorized) { if (!pathValidation(filepath)) { throw new Error('Path validation failed..'); @@ -271,11 +274,7 @@ async function downloadLocalFile(req, res) { await verifyRealPath(filepath, appsFolder); const fileNameArray = filepath.split('/'); const fileName = fileNameArray[fileNameArray.length - 1]; - const chmodResult = await serviceHelper.runCommand('chmod', { runAsRoot: true, params: ['777', filepath] }); - if (chmodResult.error) { - throw chmodResult.error; - } - return res.download(filepath, fileName); + return await sendFile(res, filepath, fileName); // eslint-disable-next-line no-else-return } else { const errMessage = messageHelper.errUnauthorizedMessage(); @@ -288,7 +287,8 @@ async function downloadLocalFile(req, res) { error.name, error.code, ); - return res ? res.json(errorResponse) : errorResponse; + // The route is its only caller, so there is always a response to write to. + return res.json(errorResponse); } } diff --git a/ZelBack/src/services/benchmarkService.js b/ZelBack/src/services/benchmarkService.js index 0b8494b0e0..11e136b410 100644 --- a/ZelBack/src/services/benchmarkService.js +++ b/ZelBack/src/services/benchmarkService.js @@ -11,6 +11,7 @@ const generalService = require('./generalService'); const upnpService = require('./upnpService'); const fluxRpc = require('./utils/fluxRpc'); const dbHelper = require('./dbHelper'); +const { Privilege, authOf } = require('./utils/privileges'); // eslint-disable-next-line no-unused-vars const isArcane = Boolean(process.env.FLUXOS_PATH); @@ -183,7 +184,7 @@ async function getStatus(req, res) { * @returns {object} Message. */ async function restartNodeBenchmarks(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response; @@ -205,7 +206,7 @@ async function restartNodeBenchmarks(req, res) { * @returns {object} Message. */ async function signFluxTransaction(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); let { hexstring } = req.params; hexstring = hexstring || req.query.hexstring; @@ -223,7 +224,7 @@ async function signFluxTransaction(req, res) { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -239,7 +240,7 @@ async function signFluxTransactionPost(req, res) { req.on('end', async () => { const processedBody = serviceHelper.ensureObject(body); const { hexstring } = processedBody; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); let response; @@ -313,7 +314,7 @@ async function help(req, res) { const response = await executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -323,7 +324,7 @@ async function help(req, res) { * @returns {object} Message. */ async function stop(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); let response; diff --git a/ZelBack/src/services/cloudUIUpdateService.js b/ZelBack/src/services/cloudUIUpdateService.js index 5c17b080d4..aae5bd7a5e 100644 --- a/ZelBack/src/services/cloudUIUpdateService.js +++ b/ZelBack/src/services/cloudUIUpdateService.js @@ -1,7 +1,7 @@ const config = require('config'); const fs = require('fs'); const path = require('path'); -const { exec } = require('child_process'); +const { execFile } = require('child_process'); const axios = require('axios'); const log = require('../lib/log'); @@ -14,6 +14,14 @@ const VERSION_FILE = path.join(CLOUDUI_DIR, 'version'); // ArcaneOS nodes have watchdog handling CloudUI updates const isArcaneOS = Boolean(process.env.FLUXOS_PATH); +// The run in progress, if there is one. The script removes the served directory +// before it copies the new one into place, so two runs at once means the second +// one's removal lands inside the first one's copy - and the run that wrote the +// version file is then not the run that wrote the files beside it. A second +// caller joins the run already going rather than starting another, which is +// also the honest answer: the rebuild it asked for is happening. +let updateInFlight = null; + /** * Checks if CloudUI folder exists and has content * @returns {boolean} @@ -101,15 +109,48 @@ async function getRemoteVersionInfo() { } } +/** + * Whether CloudUI on this node belongs to something else. + * + * On ArcaneOS the watchdog installs and updates it, so nothing here may touch it - + * neither the periodic check nor the operator's rebuild. One definition, because two + * paths reach the script and a rule only half of them consult is not a rule. + * + * @returns {boolean} + */ +function watchdogManagesCloudUI() { + return isArcaneOS; +} + /** * Executes the update:cloudui script * @returns {Promise} */ function runUpdateScript() { + if (updateInFlight) { + log.info('CloudUI: an update is already running, waiting for it rather than starting another'); + return updateInFlight; + } + + updateInFlight = startUpdateScript().finally(() => { updateInFlight = null; }); + + return updateInFlight; +} + +/** + * Runs the update:cloudui script, unconditionally. Reached only through + * runUpdateScript, which is what keeps two of them from overlapping. + * @returns {Promise} + */ +function startUpdateScript() { return new Promise((resolve) => { log.info('CloudUI: Running update script...'); - exec('npm run update:cloudui', { cwd: PROJECT_ROOT, timeout: 300000 }, (error, stdout, stderr) => { + // The script is handed the API host rather than knowing one. Config is the single + // source of truth for every endpoint a node reaches, so a hardcoded URL in the script, + // or one taken from the environment, would sit outside it and be unreachable by the + // harness. + execFile('npm', ['run', 'update:cloudui', '--', config.github.apiBaseUrl], { cwd: PROJECT_ROOT, timeout: 300000 }, (error, stdout, stderr) => { if (error) { log.error(`CloudUI update script error: ${error.message}`); if (stderr) { @@ -138,7 +179,7 @@ function runUpdateScript() { async function checkAndUpdateCloudUI() { try { // Skip on ArcaneOS - watchdog handles CloudUI updates - if (isArcaneOS) { + if (watchdogManagesCloudUI()) { log.info('CloudUI: Running on ArcaneOS, skipping update check (handled by watchdog)'); return; } @@ -211,6 +252,8 @@ async function checkAndUpdateCloudUI() { module.exports = { checkAndUpdateCloudUI, cloudUIExists, + runUpdateScript, + watchdogManagesCloudUI, getLocalVersionHash, getRemoteVersionInfo, // Exported for testing diff --git a/ZelBack/src/services/daemonService/daemonServiceAddressRpcs.js b/ZelBack/src/services/daemonService/daemonServiceAddressRpcs.js index 710889e07f..541f416eb8 100644 --- a/ZelBack/src/services/daemonService/daemonServiceAddressRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceAddressRpcs.js @@ -199,7 +199,7 @@ async function getSingleAddressDeltas(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -307,7 +307,7 @@ async function getSingleAddressMempool(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } module.exports = { diff --git a/ZelBack/src/services/daemonService/daemonServiceBenchmarkRpcs.js b/ZelBack/src/services/daemonService/daemonServiceBenchmarkRpcs.js index 5642de50de..dcc15deda8 100644 --- a/ZelBack/src/services/daemonService/daemonServiceBenchmarkRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceBenchmarkRpcs.js @@ -1,6 +1,7 @@ const messageHelper = require('../messageHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); const verificationHelper = require('../verificationHelper'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -39,7 +40,7 @@ async function getBenchStatus(req, res) { * @returns {object} Message. */ async function startBenchmarkD(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -57,7 +58,7 @@ async function startBenchmarkD(req, res) { * @returns {object} Message. */ async function stopBenchmarkD(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; diff --git a/ZelBack/src/services/daemonService/daemonServiceBlockchainRpcs.js b/ZelBack/src/services/daemonService/daemonServiceBlockchainRpcs.js index dffba051ee..daa73ea529 100644 --- a/ZelBack/src/services/daemonService/daemonServiceBlockchainRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceBlockchainRpcs.js @@ -2,6 +2,7 @@ const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); const verificationHelper = require('../verificationHelper'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -87,7 +88,7 @@ async function getBlockHash(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -108,7 +109,7 @@ async function getBlockDeltas(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -149,7 +150,7 @@ async function getBlockHashes(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -199,7 +200,7 @@ async function getBlockHeader(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -270,7 +271,7 @@ async function getRawMemPool(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -297,7 +298,7 @@ async function getTxOut(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -323,7 +324,7 @@ async function getTxOutProof(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -350,10 +351,10 @@ async function verifyChain(req, res) { let { checklevel, numblocks } = req.params; checklevel = checklevel || req.query.checklevel || 3; numblocks = numblocks || req.query.numblocks || 288; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } checklevel = serviceHelper.ensureNumber(checklevel); numblocks = serviceHelper.ensureNumber(numblocks); @@ -362,7 +363,7 @@ async function verifyChain(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -383,7 +384,7 @@ async function verifyTxOutProof(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -406,7 +407,7 @@ async function getSpentInfo(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** diff --git a/ZelBack/src/services/daemonService/daemonServiceControlRpcs.js b/ZelBack/src/services/daemonService/daemonServiceControlRpcs.js index 5dd3be507b..a7aadee821 100644 --- a/ZelBack/src/services/daemonService/daemonServiceControlRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceControlRpcs.js @@ -1,6 +1,7 @@ const messageHelper = require('../messageHelper'); const verificationHelper = require('../verificationHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -19,7 +20,7 @@ async function help(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -36,7 +37,7 @@ async function getInfo(req, res) { delete response.data.balance; return response; } - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { delete response.data.balance; } @@ -51,7 +52,7 @@ async function getInfo(req, res) { * @returns {object} Message. */ async function stop(req, res) { // practically useless - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; diff --git a/ZelBack/src/services/daemonService/daemonServiceFluxnodeRpcs.js b/ZelBack/src/services/daemonService/daemonServiceFluxnodeRpcs.js index 14af678928..bb948e8b8c 100644 --- a/ZelBack/src/services/daemonService/daemonServiceFluxnodeRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceFluxnodeRpcs.js @@ -2,6 +2,7 @@ const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); const verificationHelper = require('../verificationHelper'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -53,7 +54,7 @@ async function listFluxNodes(req, res) { response.data = response.data.slice(0, limit); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -63,12 +64,12 @@ async function listFluxNodes(req, res) { * @returns {object} Message. */ async function listFluxNodeConf(req, res) { // practically useless - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); let { filter } = req.params; filter = filter || req.query.filter; if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'listzelnodeconf'; // listfluxnodeconf const rpcparameters = []; @@ -78,7 +79,7 @@ async function listFluxNodeConf(req, res) { // practically useless response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -88,7 +89,7 @@ async function listFluxNodeConf(req, res) { // practically useless * @returns {object} Message. */ async function createFluxNodeKey(req, res) { // practically useless - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { const rpccall = 'createzelnodekey'; // createfluxnodekey @@ -149,7 +150,7 @@ async function getStartList(req, res) { * @returns {object} Message. */ async function getFluxNodeOutputs(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -171,7 +172,7 @@ async function startDeterministicFluxNode(req, res) { alias = alias || req.query.alias; lockwallet = lockwallet ?? req.query.lockwallet ?? false; lockwallet = serviceHelper.ensureBoolean(lockwallet); - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { const rpccall = 'startdeterministiczelnode'; // startdeterministicfluxnode const rpcparameters = []; @@ -183,7 +184,7 @@ async function startDeterministicFluxNode(req, res) { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -197,7 +198,7 @@ async function startFluxNode(req, res) { set = set || req.query.set; lockwallet = lockwallet ?? req.query.lockwallet; alias = alias || req.query.alias; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { const rpccall = 'startzelnode'; // startfluxnode const rpcparameters = []; @@ -212,7 +213,7 @@ async function startFluxNode(req, res) { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** diff --git a/ZelBack/src/services/daemonService/daemonServiceMiningRpcs.js b/ZelBack/src/services/daemonService/daemonServiceMiningRpcs.js index 29a3e048a0..bd3b66d5b3 100644 --- a/ZelBack/src/services/daemonService/daemonServiceMiningRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceMiningRpcs.js @@ -2,6 +2,7 @@ const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); const verificationHelper = require('../verificationHelper'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -24,7 +25,7 @@ async function getBlockSubsidy(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -46,7 +47,7 @@ async function getBlockTemplate(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -96,7 +97,7 @@ async function getNetworkHashPs(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -117,7 +118,7 @@ async function getNetworkSolPs(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -131,10 +132,10 @@ async function prioritiseTransaction(req, res) { txid = txid || req.query.txid; prioritydelta = prioritydelta || req.query.prioritydelta; feedelta = feedelta || req.query.feedelta; - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'prioritiseTransaction'; let rpcparameters = []; @@ -146,7 +147,7 @@ async function prioritiseTransaction(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -159,7 +160,7 @@ async function submitBlock(req, res) { let { hexdata, jsonparametersobject } = req.params; hexdata = hexdata || req.query.hexdata; jsonparametersobject = jsonparametersobject || req.query.jsonparametersobject; - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (authorized === true) { const rpccall = 'submitBlock'; let rpcparameters = []; @@ -175,7 +176,7 @@ async function submitBlock(req, res) { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -194,7 +195,7 @@ async function submitBlockPost(req, res) { const { hexdata } = processedBody; let { jsonparametersobject } = processedBody; - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (authorized === true) { const rpccall = 'submitBlock'; let rpcparameters = []; diff --git a/ZelBack/src/services/daemonService/daemonServiceNetworkRpcs.js b/ZelBack/src/services/daemonService/daemonServiceNetworkRpcs.js index b4aee68025..4e98b6154d 100644 --- a/ZelBack/src/services/daemonService/daemonServiceNetworkRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceNetworkRpcs.js @@ -2,6 +2,7 @@ const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); const verificationHelper = require('../verificationHelper'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -15,11 +16,11 @@ async function addNode(req, res) { let { node, command } = req.params; node = node || req.query.node; command = command || req.query.command; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'addNode'; let rpcparameters = []; @@ -28,7 +29,7 @@ async function addNode(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -38,7 +39,7 @@ async function addNode(req, res) { * @returns {object} Message. */ async function clearBanned(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -58,7 +59,7 @@ async function clearBanned(req, res) { async function disconnectNode(req, res) { let { node } = req.params; node = node || req.query.node; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { const rpccall = 'disconnectNode'; let rpcparameters = []; @@ -71,7 +72,7 @@ async function disconnectNode(req, res) { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -84,7 +85,7 @@ async function getAddedNodeInfo(req, res) { let { dns, node } = req.params; dns = dns ?? req.query.dns; node = node || req.query.node; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { const rpccall = 'getAddedNodeInfo'; const rpcparameters = []; @@ -101,7 +102,7 @@ async function getAddedNodeInfo(req, res) { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -195,7 +196,7 @@ async function listBanned(req, res) { * @returns {object} Message. */ async function ping(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -220,7 +221,7 @@ async function setBan(req, res) { command = command || req.query.command; bantime = bantime || req.query.bantime; absolute = absolute ?? req.query.absolute; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { const rpccall = 'setBan'; const rpcparameters = []; @@ -242,7 +243,7 @@ async function setBan(req, res) { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } module.exports = { diff --git a/ZelBack/src/services/daemonService/daemonServiceTransactionRpcs.js b/ZelBack/src/services/daemonService/daemonServiceTransactionRpcs.js index c33a7d7ecc..b2559102b1 100644 --- a/ZelBack/src/services/daemonService/daemonServiceTransactionRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceTransactionRpcs.js @@ -3,6 +3,7 @@ const messageHelper = require('../messageHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); const verificationHelper = require('../verificationHelper'); const daemonServiceBlockchainRpcs = require('./daemonServiceBlockchainRpcs'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -26,7 +27,7 @@ async function createRawTransaction(req, res) { }); if (!blockcount) { // getBlockCount rejected the promise - return error message - return res ? res.json(response) : response; + return res.json(response); } const defaultExpiryHeight = blockcount + 20; let { expiryheight } = req.params; @@ -43,7 +44,7 @@ async function createRawTransaction(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -107,7 +108,7 @@ async function decodeRawTransaction(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -153,7 +154,7 @@ async function decodeScript(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -199,7 +200,7 @@ async function fundRawTransaction(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -271,7 +272,7 @@ async function sendRawTransaction(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -320,10 +321,10 @@ async function signRawTransaction(req, res) { sighashtype = sighashtype || req.query.sighashtype || 'ALL'; branchid = branchid || req.query.branchid; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'signRawTransaction'; const rpcparameters = []; @@ -343,7 +344,7 @@ async function signRawTransaction(req, res) { } } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -363,7 +364,7 @@ async function signRawTransactionPost(req, res) { const { hexstring, branchid } = processedBody; let { prevtxs, privatekeys, sighashtype } = processedBody; sighashtype = sighashtype || 'ALL'; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res.json(response); diff --git a/ZelBack/src/services/daemonService/daemonServiceUtilityRpcs.js b/ZelBack/src/services/daemonService/daemonServiceUtilityRpcs.js index a537648b4b..78a17bb26d 100644 --- a/ZelBack/src/services/daemonService/daemonServiceUtilityRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceUtilityRpcs.js @@ -2,6 +2,7 @@ const messageHelper = require('../messageHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); const serviceHelper = require('../serviceHelper'); const verificationHelper = require('../verificationHelper'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -25,7 +26,7 @@ async function createMultiSig(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -74,7 +75,7 @@ async function estimateFee(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -95,7 +96,7 @@ async function estimatePriority(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -122,7 +123,7 @@ async function validateAddress(req, res) { return response; } - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { delete response.data.ismine; delete response.data.iswatchonly; @@ -150,7 +151,7 @@ async function verifyMessage(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -196,7 +197,7 @@ async function zValidateAddress(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } module.exports = { diff --git a/ZelBack/src/services/daemonService/daemonServiceWalletRpcs.js b/ZelBack/src/services/daemonService/daemonServiceWalletRpcs.js index b2aeac075f..e4c60dc392 100644 --- a/ZelBack/src/services/daemonService/daemonServiceWalletRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceWalletRpcs.js @@ -2,6 +2,7 @@ const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); const verificationHelper = require('../verificationHelper'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -16,10 +17,10 @@ async function addMultiSigAddress(req, res) { let { n, keysobject } = req.params; n = n || req.query.n; keysobject = keysobject || req.query.keysobject; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'addMultiSigAddress'; let rpcparameters = []; @@ -30,7 +31,7 @@ async function addMultiSigAddress(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -48,7 +49,7 @@ async function addMultiSigAddressPost(req, res) { const processedBody = serviceHelper.ensureObject(body); let { n } = processedBody; let { keysobject } = processedBody; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res.json(response); @@ -75,10 +76,10 @@ async function addMultiSigAddressPost(req, res) { async function backupWallet(req, res) { let { destination } = req.params; destination = destination || req.query.destination; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'backupWallet'; let rpcparameters = []; @@ -87,7 +88,7 @@ async function backupWallet(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -99,10 +100,10 @@ async function backupWallet(req, res) { async function dumpPrivKey(req, res) { let { taddr } = req.params; taddr = taddr || req.query.taddr; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'dumpPrivKey'; let rpcparameters = []; @@ -111,7 +112,7 @@ async function dumpPrivKey(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -124,10 +125,10 @@ async function getBalance(req, res) { let { minconf, includewatchonly } = req.params; minconf = minconf || req.query.minconf || 1; includewatchonly = includewatchonly ?? req.query.includewatchonly ?? false; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'getBalance'; minconf = serviceHelper.ensureNumber(minconf); @@ -135,7 +136,7 @@ async function getBalance(req, res) { const rpcparameters = ['', minconf, includewatchonly]; response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -145,7 +146,7 @@ async function getBalance(req, res) { * @returns {object} Message. */ async function getNewAddress(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -163,7 +164,7 @@ async function getNewAddress(req, res) { * @returns {object} Message. */ async function getRawChangeAddress(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -184,10 +185,10 @@ async function getReceivedByAddress(req, res) { let { fluxaddress, minconf } = req.params; fluxaddress = fluxaddress || req.query.fluxaddress; minconf = minconf || req.query.minconf || 1; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'getReceivedByAddress'; let rpcparameters = []; @@ -197,7 +198,7 @@ async function getReceivedByAddress(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -219,7 +220,7 @@ async function getTransaction(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -229,7 +230,7 @@ async function getTransaction(req, res) { * @returns {object} Message. */ async function getUnconfirmedBalance(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -247,7 +248,7 @@ async function getUnconfirmedBalance(req, res) { * @returns {object} Message. */ async function getWalletInfo(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -270,10 +271,10 @@ async function importAddress(req, res) { label = label || req.query.label || ''; rescan = rescan ?? req.query.rescan ?? true; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'importAddress'; let rpcparameters = []; @@ -283,7 +284,7 @@ async function importAddress(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -297,10 +298,10 @@ async function importPrivKey(req, res) { fluxprivkey = fluxprivkey || req.query.fluxprivkey; label = label || req.query.label || ''; rescan = rescan ?? req.query.rescan ?? true; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'importPrivKey'; let rpcparameters = []; @@ -310,7 +311,7 @@ async function importPrivKey(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -322,10 +323,10 @@ async function importPrivKey(req, res) { async function importWallet(req, res) { let { filename } = req.params; filename = filename || req.query.filename; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'importWallet'; let rpcparameters = []; @@ -334,7 +335,7 @@ async function importWallet(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -346,17 +347,17 @@ async function importWallet(req, res) { async function keyPoolRefill(req, res) { let { newsize } = req.params; newsize = newsize || req.query.newsize || 100; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'keyPoolRefill'; newsize = serviceHelper.ensureNumber(newsize); const rpcparameters = [newsize]; response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -366,7 +367,7 @@ async function keyPoolRefill(req, res) { * @returns {object} Message. */ async function listAddressGroupings(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -384,7 +385,7 @@ async function listAddressGroupings(req, res) { * @returns {object} Message. */ async function listLockUnspent(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -404,17 +405,17 @@ async function listLockUnspent(req, res) { async function rescanBlockchain(req, res) { let { startheight } = req.params; startheight = startheight || req.query.startheight || 0; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } startheight = serviceHelper.ensureNumber(startheight); const rpccall = 'rescanblockchain'; const rpcparameters = [startheight]; response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -428,10 +429,10 @@ async function listReceivedByAddress(req, res) { minconf = minconf || req.query.minconf || 1; includeempty = includeempty ?? req.query.includeempty ?? false; includewatchonly = includewatchonly ?? req.query.includewatchonly ?? false; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } minconf = serviceHelper.ensureNumber(minconf); includeempty = serviceHelper.ensureBoolean(includeempty); @@ -440,7 +441,7 @@ async function listReceivedByAddress(req, res) { const rpcparameters = [minconf, includeempty, includewatchonly]; response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -454,10 +455,10 @@ async function listSinceBlock(req, res) { blockhash = blockhash || req.query.blockhash || ''; targetconfirmations = targetconfirmations || req.query.targetconfirmations || 1; includewatchonly = includewatchonly ?? req.query.includewatchonly ?? false; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } targetconfirmations = serviceHelper.ensureNumber(targetconfirmations); includewatchonly = serviceHelper.ensureBoolean(includewatchonly); @@ -465,7 +466,7 @@ async function listSinceBlock(req, res) { const rpcparameters = [blockhash, targetconfirmations, includewatchonly]; response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -480,10 +481,10 @@ async function listTransactions(req, res) { count = count || req.query.count || 10; from = from || req.query.from || 0; includewatchonly = includewatchonly ?? req.query.includewatchonly ?? false; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } count = serviceHelper.ensureNumber(count); from = serviceHelper.ensureNumber(from); @@ -492,7 +493,7 @@ async function listTransactions(req, res) { const rpcparameters = [account, count, from, includewatchonly]; response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -506,10 +507,10 @@ async function listUnspent(req, res) { minconf = minconf || req.query.minconf || 1; maxconf = maxconf || req.query.maxconf || 9999999; addresses = addresses || req.query.addresses; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } minconf = serviceHelper.ensureNumber(minconf); maxconf = serviceHelper.ensureNumber(maxconf); @@ -522,7 +523,7 @@ async function listUnspent(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -535,10 +536,10 @@ async function lockUnspent(req, res) { let { unlock, transactions } = req.params; unlock = unlock ?? req.query.unlock; transactions = transactions || req.query.transactions; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'lockUnspent'; let rpcparameters = []; @@ -550,7 +551,7 @@ async function lockUnspent(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -569,10 +570,10 @@ async function sendFrom(req, res) { minconf = minconf || req.query.minconf || 1; comment = comment || req.query.comment || ''; commentto = commentto || req.query.commentto || ''; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'sendFrom'; let rpcparameters = []; @@ -584,7 +585,7 @@ async function sendFrom(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -608,7 +609,7 @@ async function sendFromPost(req, res) { minconf = minconf || 1; comment = comment || ''; commentto = commentto || ''; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res.json(response); @@ -642,16 +643,16 @@ async function sendMany(req, res) { minconf = minconf || req.query.minconf || 1; comment = comment || req.query.comment || ''; substractfeefromamount = substractfeefromamount || req.query.substractfeefromamount; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'sendMany'; let rpcparameters = []; if (!amounts) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } amounts = serviceHelper.ensureObject(amounts); minconf = serviceHelper.ensureNumber(minconf); @@ -663,7 +664,7 @@ async function sendMany(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -685,7 +686,7 @@ async function sendManyPost(req, res) { const fromaccount = ''; minconf = minconf || 1; comment = comment || ''; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res.json(response); @@ -725,10 +726,10 @@ async function sendToAddress(req, res) { comment = comment || req.query.comment || ''; commentto = commentto || req.query.commentto || ''; substractfeefromamount = substractfeefromamount ?? req.query.substractfeefromamount ?? false; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'sendToAddress'; let rpcparameters = []; @@ -740,7 +741,7 @@ async function sendToAddress(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -764,7 +765,7 @@ async function sendToAddressPost(req, res) { comment = comment || ''; commentto = commentto || ''; substractfeefromamount = substractfeefromamount ?? false; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res.json(response); @@ -792,10 +793,10 @@ async function sendToAddressPost(req, res) { async function setTxFee(req, res) { let { amount } = req.params; amount = amount || req.query.amount; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'setTxFee'; let rpcparameters = []; @@ -806,7 +807,7 @@ async function setTxFee(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -819,10 +820,10 @@ async function signMessage(req, res) { let { taddr, message } = req.params; taddr = taddr || req.query.taddr; message = message || req.query.message; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'signMessage'; let rpcparameters = []; @@ -832,7 +833,7 @@ async function signMessage(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -851,7 +852,7 @@ async function signMessagePost(req, res) { const { taddr } = processedBody; const { message } = processedBody; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res.json(response); diff --git a/ZelBack/src/services/daemonService/daemonServiceZcashRpcs.js b/ZelBack/src/services/daemonService/daemonServiceZcashRpcs.js index 1eb0a4677a..2d8accce71 100644 --- a/ZelBack/src/services/daemonService/daemonServiceZcashRpcs.js +++ b/ZelBack/src/services/daemonService/daemonServiceZcashRpcs.js @@ -2,6 +2,7 @@ const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const daemonServiceUtils = require('./daemonServiceUtils'); const verificationHelper = require('../verificationHelper'); +const { Privilege, authOf } = require('../utils/privileges'); let response = messageHelper.createErrorMessage(); @@ -14,10 +15,10 @@ let response = messageHelper.createErrorMessage(); async function zExportKey(req, res) { let { zaddr } = req.params; zaddr = zaddr || req.query.zaddr; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_exportkey'; let rpcparameters = []; @@ -27,7 +28,7 @@ async function zExportKey(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -39,10 +40,10 @@ async function zExportKey(req, res) { async function zExportViewingKey(req, res) { let { zaddr } = req.params; zaddr = zaddr || req.query.zaddr; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_exportviewingkey'; let rpcparameters = []; @@ -52,7 +53,7 @@ async function zExportViewingKey(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -65,10 +66,10 @@ async function zGetBalance(req, res) { let { address, minconf } = req.params; address = address || req.query.address; minconf = minconf ?? req.query.minconf ?? 1; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_getbalance'; let rpcparameters = []; @@ -79,7 +80,7 @@ async function zGetBalance(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -89,7 +90,7 @@ async function zGetBalance(req, res) { * @returns {object} Message. */ async function zGetMigrationStatus(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -110,17 +111,17 @@ async function zGetMigrationStatus(req, res) { async function zGetNewAddress(req, res) { let { type } = req.params; type = type || req.query.type || 'sapling'; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_getnewaddress'; const rpcparameters = [type]; response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -132,10 +133,10 @@ async function zGetNewAddress(req, res) { async function zGetOperationResult(req, res) { let { operationid } = req.params; operationid = operationid || req.query.operationid || []; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } operationid = serviceHelper.ensureObject(operationid); const rpccall = 'z_getoperationresult'; @@ -143,7 +144,7 @@ async function zGetOperationResult(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -155,10 +156,10 @@ async function zGetOperationResult(req, res) { async function zGetOperationStatus(req, res) { let { operationid } = req.params; operationid = operationid || req.query.operationid || []; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } operationid = serviceHelper.ensureObject(operationid); const rpccall = 'z_getoperationstatus'; @@ -166,7 +167,7 @@ async function zGetOperationStatus(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -179,10 +180,10 @@ async function zGetTotalBalance(req, res) { let { minconf, includewatchonly } = req.params; minconf = minconf ?? req.query.minconf ?? 1; includewatchonly = includewatchonly ?? req.query.includewatchonly ?? false; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } minconf = serviceHelper.ensureNumber(minconf); includewatchonly = serviceHelper.ensureBoolean(includewatchonly); @@ -191,7 +192,7 @@ async function zGetTotalBalance(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -205,10 +206,10 @@ async function zImportKey(req, res) { zkey = zkey || req.query.zkey; rescan = rescan || req.query.rescan || 'whenkeyisnew'; startheight = startheight || req.query.startheight || 0; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_importkey'; let rpcparameters = []; @@ -219,7 +220,7 @@ async function zImportKey(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -233,10 +234,10 @@ async function zImportViewingKey(req, res) { vkey = vkey || req.query.vkey; rescan = rescan || req.query.rescan || 'whenkeyisnew'; startheight = startheight || req.query.startheight || 0; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_importviewingkey'; let rpcparameters = []; @@ -247,7 +248,7 @@ async function zImportViewingKey(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -259,10 +260,10 @@ async function zImportViewingKey(req, res) { async function zImportWallet(req, res) { let { filename } = req.params; filename = filename || req.query.filename; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_importwallet'; let rpcparameters = []; @@ -272,7 +273,7 @@ async function zImportWallet(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -284,10 +285,10 @@ async function zImportWallet(req, res) { async function zListAddresses(req, res) { let { includewatchonly } = req.params; includewatchonly = includewatchonly ?? req.query.includewatchonly ?? false; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } includewatchonly = serviceHelper.ensureBoolean(includewatchonly); const rpccall = 'z_listaddresses'; @@ -295,7 +296,7 @@ async function zListAddresses(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -305,7 +306,7 @@ async function zListAddresses(req, res) { * @returns {object} Message. */ async function zListOperationIds(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -327,10 +328,10 @@ async function zListReceivedByAddress(req, res) { let { address, minconf } = req.params; address = address || req.query.address; minconf = minconf ?? req.query.minconf ?? 1; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_listreceivedbyaddress'; let rpcparameters = []; @@ -341,7 +342,7 @@ async function zListReceivedByAddress(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -359,10 +360,10 @@ async function zListUnspent(req, res) { includewatchonly = includewatchonly ?? req.query.includewatchonly ?? false; addresses = addresses || req.query.addresses; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_listunspent'; minconf = serviceHelper.ensureNumber(minconf); @@ -376,7 +377,7 @@ async function zListUnspent(req, res) { response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -395,10 +396,10 @@ async function zMergeToAddress(req, res) { transparentlimit = transparentlimit ?? req.query.transparentlimit ?? 50; // 0 for as many as can fit shieldedlimit = shieldedlimit ?? req.query.shieldedlimit ?? 20; // 0 for as many as can fit memo = memo || req.query.memo || ''; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_mergetoaddress'; let rpcparameters = []; @@ -411,7 +412,7 @@ async function zMergeToAddress(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -428,10 +429,10 @@ async function zSendMany(req, res) { amounts = amounts || req.query.amounts; minconf = minconf ?? req.query.minconf ?? 1; fee = fee || req.query.fee || 0.0001; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_sendmany'; let rpcparameters = []; @@ -443,7 +444,7 @@ async function zSendMany(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -463,7 +464,7 @@ async function zSendManyPost(req, res) { let { amounts, minconf, fee } = processedBody; minconf = minconf || 1; fee = fee || 0.0001; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res.json(response); @@ -491,10 +492,10 @@ async function zSendManyPost(req, res) { async function zSetMigration(req, res) { let { enabled } = req.params; enabled = enabled ?? req.query.enabled; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_setmigration'; let rpcparameters = []; @@ -504,7 +505,7 @@ async function zSetMigration(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -521,10 +522,10 @@ async function zShieldCoinBase(req, res) { toaddress = toaddress || req.query.toaddress; fee = fee || req.query.fee || 0.0001; limit = limit ?? req.query.limit ?? 50; // 0 for as many as can fit - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'z_shieldcoinbase'; let rpcparameters = []; @@ -535,7 +536,7 @@ async function zShieldCoinBase(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -548,10 +549,10 @@ async function zcBenchmark(req, res) { let { benchmarktype, samplecount } = req.params; benchmarktype = benchmarktype || req.query.benchmarktype; samplecount = samplecount || req.query.samplecount; - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'zcbenchmark'; let rpcparameters = []; @@ -561,7 +562,7 @@ async function zcBenchmark(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -579,10 +580,10 @@ async function zcRawJoinSplit(req, res) { outputs = outputs || req.query.outputs; vpubold = vpubold || req.query.vpubold; vpubnew = vpubnew || req.query.vpubnew; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'zcrawjoinsplit'; let rpcparameters = []; @@ -593,7 +594,7 @@ async function zcRawJoinSplit(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -612,7 +613,7 @@ async function zcRawJoinSplitPost(req, res) { const { rawtx, vpubold, vpubnew } = processedBody; let { inputs, outputs } = processedBody; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res.json(response); @@ -637,7 +638,7 @@ async function zcRawJoinSplitPost(req, res) { * @returns {object} Message. */ async function zcRawKeygen(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; @@ -658,10 +659,10 @@ async function zcRawReceive(req, res) { let { zcsecretkey, encryptednote } = req.params; zcsecretkey = zcsecretkey || req.query.zcsecretkey; encryptednote = encryptednote || req.query.encryptednote; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } const rpccall = 'zcrawreceive'; let rpcparameters = []; @@ -670,7 +671,7 @@ async function zcRawReceive(req, res) { } response = await daemonServiceUtils.executeCall(rpccall, rpcparameters); - return res ? res.json(response) : response; + return res.json(response); } /** @@ -689,7 +690,7 @@ async function zcRawReceivePost(req, res) { const { zcsecretkey } = processedBody; const { encryptednote } = processedBody; - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res.json(response); @@ -712,7 +713,7 @@ async function zcRawReceivePost(req, res) { * @returns {object} Message. */ async function zcSampleJoinSplit(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { response = messageHelper.errUnauthorizedMessage(); return res ? res.json(response) : response; diff --git a/ZelBack/src/services/deviceHelper.js b/ZelBack/src/services/deviceHelper.js index c0019c6535..76617ead6c 100644 --- a/ZelBack/src/services/deviceHelper.js +++ b/ZelBack/src/services/deviceHelper.js @@ -21,6 +21,43 @@ const serviceHelper = require('./serviceHelper'); // return ''; // } +/** + * Every mounted real (block-backed) filesystem with its byte-level usage. + * + * This is the `df` view, sourced from `findmnt --real --list`: one flat row per + * mount, no mount-tree nesting. `--real` drops the pseudo filesystems + * (proc/sysfs/cgroup/tmpfs); loop devices ARE real, so an app's loop-mounted + * FLUXFSVOL appears here. + * + * Byte counts come from `--bytes`, so they need no unit conversion. + * + * Throws on findmnt failure rather than returning [], so a caller cannot read + * "no disks" as "no space" and act on it. + * + * @returns {Promise>} + */ +async function listMountedFilesystems() { + const res = await serviceHelper.runCommand('findmnt', { + logError: false, + params: ['--real', '--list', '--bytes', '--json', '--output', 'SOURCE,TARGET,FSTYPE,SIZE,USED,AVAIL,USE%'], + }); + if (res.error) { + throw new Error(`findmnt --real --list failed: ${res.error.message || res.error}`); + } + const filesystems = JSON.parse(res.stdout || '{}').filesystems || []; + return filesystems.map((entry) => ({ + source: entry.source, + target: entry.target, + fstype: entry.fstype, + sizeBytes: Number(entry.size), + usedBytes: Number(entry.used), + availableBytes: Number(entry.avail), + usePercent: Number(String(entry['use%'] || '').replace('%', '')), + })); +} + /** * Determines if mount target has a filesystem quota * @param {string} target The mount target @@ -64,4 +101,5 @@ if (require.main === module) { module.exports = { hasQuotaOptionForMountTarget, + listMountedFilesystems, }; diff --git a/ZelBack/src/services/dockerService.js b/ZelBack/src/services/dockerService.js index 047d4d0719..df930da035 100644 --- a/ZelBack/src/services/dockerService.js +++ b/ZelBack/src/services/dockerService.js @@ -1,5 +1,6 @@ const config = require('config'); -const stream = require('stream'); +const fs = require('fs').promises; +const tar = require('tar'); const Docker = require('dockerode'); const path = require('path'); const serviceHelper = require('./serviceHelper'); @@ -11,6 +12,7 @@ const fluxNetworkHelper = require('./fluxNetworkHelper'); const { extractIp } = require('./utils/socketAddressUtils'); const log = require('../lib/log'); const cpuBurstHelper = require('./utils/cpuBurstHelper'); +const LogFrameDecoder = require('./utils/logFrameDecoder'); const globalState = require('./utils/globalState'); @@ -183,9 +185,11 @@ async function dockerListImages() { async function getDockerContainerOnly(idOrName) { const containers = await dockerListContainers(true); const myContainer = containers.find((container) => (container.Names[0] === getAppDockerNameIdentifier(idOrName) || container.Id === idOrName)); - if (!myContainer) { - log.error(`Container ${idOrName} not found`); - } + // Absence is not logged here. The two direct callers probe with this + // deliberately - the reconciler asks whether a container exists at all - so a + // miss is an answer, not an incident, and logging it buried the journal in + // errors from a healthy node. getDockerContainerByIdOrName, whose contract IS + // that the container exists, throws with this same text. return myContainer; } @@ -196,11 +200,44 @@ async function getDockerContainerOnly(idOrName) { * @returns {object} dockerContainer */ async function getDockerContainerByIdOrName(idOrName) { - const myContainer = await getDockerContainerOnly(idOrName); - // Don't throw error here, let it fail with property access error - // to match test expectations - const dockerContainer = docker.getContainer(myContainer.Id); - return dockerContainer; + // Docker filters server-side, so this asks about ONE container whatever the + // node is running. It used to list every container, `all: true`, and scan the + // result - for every start, stop, remove, inspect, exec, stats and log poll. + // On a node running twenty apps that is twenty records to answer a question + // about one, and the log endpoint pays it on a timer for as long as a browser + // is left open. + // + // The name filter is a REGEX match (docker runs it through regexp.MatchString), + // so it is at least a substring match and the exact comparison below is what + // decides: `fluxweb` must not answer for `fluxwebsite`. The filter narrows what + // comes back; it does not choose. + const dockerName = getAppDockerNameIdentifier(idOrName); + let containers = await docker.listContainers({ + all: true, + filters: JSON.stringify({ name: [getAppIdentifier(idOrName)] }), + }); + let myContainer = containers.find((container) => container.Names[0] === dockerName); + + // Only the reconciler passes a raw docker id, and a name filter cannot match + // one, so it costs a second request rather than making every other caller pay + // for a listing. + if (!myContainer && /^[0-9a-f]{12,64}$/.test(idOrName)) { + containers = await docker.listContainers({ + all: true, + filters: JSON.stringify({ id: [idOrName] }), + }); + myContainer = containers.find((container) => container.Id === idOrName); + } + + // A container that is not there is an expected outcome, not an accident: an + // app is removed or redeployed while something else still holds its name. + // Dereferencing undefined instead raised `Cannot read properties of undefined + // (reading 'Id')` - a message that describes the mistake rather than the + // condition, and that callers had to pattern-match on to recognise it. + if (!myContainer) { + throw new Error(`Container ${idOrName} not found`); + } + return docker.getContainer(myContainer.Id); } /** * Returns low-level information about a container. @@ -233,43 +270,6 @@ async function dockerContainerStats(idOrName) { return response; } -/** - * Take stats from docker container and follow progress of the stream. - * @param {string} repoTag Docker Hub repo/image tag. - * @param {object} res Response. - * @param {function} callback Callback. - */ -async function dockerContainerStatsStream(idOrName, req, res, callback) { - // container ID or name - const dockerContainer = await getDockerContainerByIdOrName(idOrName); - - dockerContainer.stats(idOrName, (err, mystream) => { - function onFinished(error, output) { - if (error) { - callback(err); - } else { - callback(null, output); - } - mystream.destroy(); - } - function onProgress(event) { - if (res) { - res.write(serviceHelper.ensureString(event)); - if (res.flush) res.flush(); - } - log.info(event); - } - if (err) { - callback(err); - } else { - docker.modem.followProgress(mystream, onFinished, onProgress); - } - req.on('close', () => { - mystream.destroy(); - }); - }); -} - /** * Returns changes on a container’s filesystem. * @@ -359,8 +359,14 @@ async function dockerContainerExec(container, cmd, env, res, callback) { let resultString = ''; const exec = await container.exec(options); exec.start(optionsExecStart, (err, mystream) => { - if (err) { - callback(err); + // The container can stop between the exec being created and started, and + // docker then answers 404 "no such exec" with a NULL stream rather than no + // callback at all. This callback is dockerode's, not ours, so the try + // around it does not catch a throw here - dereferencing that null reached + // apiServer's uncaughtException handler and exited the node. + if (err || !mystream) { + callback(err || new Error('Exec started with no stream')); + return; } mystream.on('data', (data) => { resultString = serviceHelper.dockerBufferToString(data); @@ -374,55 +380,6 @@ async function dockerContainerExec(container, cmd, env, res, callback) { } } -/** - * Subscribes to logs stream. - * - * @param {string} idOrName - * @param {object} res - * @param {function} callback - */ -async function dockerContainerLogsStream(idOrName, res, callback) { - try { - // container ID or name - const containers = await dockerListContainers(true); - const myContainer = containers.find((container) => (container.Names[0] === getAppDockerNameIdentifier(idOrName) || container.Id === idOrName)); - const dockerContainer = docker.getContainer(myContainer.Id); - const logStream = new stream.PassThrough(); - logStream.on('data', (chunk) => { - res.write(serviceHelper.ensureString(chunk.toString('utf8'))); - if (res.flush) res.flush(); - }); - - dockerContainer.logs( - { - follow: true, - stdout: true, - stderr: true, - }, - (err, mystream) => { - if (err) { - callback(err); - } else { - try { - dockerContainer.modem.demuxStream(mystream, logStream, logStream); - mystream.on('end', () => { - logStream.end(); - callback(null); - }); - - setTimeout(() => { - mystream.destroy(); - }, 2000); - } catch (error) { - throw new Error('An error obtaining log data of an application has occured'); - } - } - }, - ); - } catch (error) { - callback(error); - } -} /** * Returns requested number of lines of logs from the container. @@ -446,94 +403,267 @@ async function dockerContainerLogs(idOrName, lines) { return logs; } -async function dockerContainerLogsPolling(idOrName, lineCount, sinceTimestamp, callback) { - try { - const dockerContainer = await getDockerContainerByIdOrName(idOrName); - const logStream = new stream.PassThrough(); - let logBuffer = ''; +/** + * How many of `lines` docker will hand back again when asked from `ms`, so that + * many can be skipped next time. + * + * Everything stamped at or after `ms` comes back, and so does anything earlier + * that sits after the first such line in the file - docker stops filtering on + * `since` once it has found its starting point, which is what makes an + * out-of-order line reappear. Counting from the first line at or after `ms` to + * the end of what was delivered is exactly that set. + * + * `ms` is the NEWEST timestamp delivered, so the first line at or after it may + * sit anywhere in the block, and everything from there on was delivered. + * + * @param {string[]} lines - timestamped lines, in docker's order + * @param {number} ms + * @returns {number} + */ +function countFrom(lines, ms) { + const start = lines.findIndex((line) => Math.floor(Date.parse(line.split(' ')[0])) >= ms); + return start === -1 ? 0 : lines.length - start; +} - logStream.on('data', (chunk) => { - logBuffer += chunk.toString('utf8'); - const lines = logBuffer.split('\n'); - logBuffer = lines.pop(); - // eslint-disable-next-line no-restricted-syntax - for (const line of lines) { - if (line.trim()) { - if (callback) { - callback(null, line); - } - } - } - }); +/** + * How much of a log payload is decoded before the event loop is released. + * + * Roughly one socket read, so the work between yields is the size the streaming + * path already handles per chunk. Smaller yields more often and costs more + * scheduling; larger holds the loop for longer. At 64KB an 8.47MB log decodes + * in 130 slices and the worst slip measured on a node was 0.4ms. + */ +const LOG_DECODE_CHUNK_BYTES = 65536; - logStream.on('error', (error) => { - log.error('Log stream encountered an error:', error); - if (callback) { - callback(error); - } - }); +/** + * The whole of a container's log, as `appDockerCreate` configures the daemon to + * keep it. Named here rather than written into the create call, because a read + * has to be sized against what a log can hold and the two must be the same fact. + */ +const LOG_MAX_FILES = 4; +const LOG_MAX_FILE_MB = 5; - logStream.on('end', () => { - if (callback) { - callback(null, 'Stream ended'); // Notify end of logs - } - }); +/** + * The most lines that log can hold. The smallest thing docker can return is one + * frame carrying an empty message: its 8-byte header, an RFC3339Nano timestamp, + * the space before the message and the newline after it. + */ +const MIN_LOG_LINE_BYTES = 8 + 30 + 1 + 1; +const MAX_RETAINED_LINES = Math.ceil( + (LOG_MAX_FILES * LOG_MAX_FILE_MB * 1024 * 1024) / MIN_LOG_LINE_BYTES, +); - const logOptions = { - follow: true, - stdout: true, - stderr: true, - tail: lineCount, - timestamps: true, - }; +/** + * The lines a reader has not seen yet, and the position it has reached. + * + * A poll, not a subscription: docker is asked for what it has and closes the + * connection itself, so the read costs what the read costs. `follow: true` was + * asked for instead, which never closes - the only way out was a 1500ms timer, + * so every poll took 1500ms to answer whether one line was waiting or none. + * + * `since` is inclusive and millisecond-resolved, so a reader is always handed + * back lines it already holds. `position.count` says how many of them, and they + * are dropped here rather than by the reader - the overlap is the proof there is + * no gap between two polls, and dropping it exactly is what the count is for. + * + * `tail` is not a companion to a position: docker applies it AFTER `since`, so + * `tail: 100` over a burst of 500 answers a reader asking for everything since T + * with the last 100 and no indication the rest existed. + * + * A positioned read is ONE read, and it always carries a `tail`. That bound is + * what keeps the cost flat: a read with `since` and no `tail` has no index to + * seek with, so docker decodes forward from the oldest file and this function + * then walks every frame of it. Measured on a live node against a 5MB log, + * 97,510 lines: 1099ms to fetch and 349ms of BLOCKING decode, against 104ms and + * 1ms for the bounded read - and the event loop is the whole node, so those + * 349ms answer no peer and no other request. `position.ms` is a value the caller + * supplies, so a millisecond older than the log takes that path on every poll. + * + * @param {string} idOrName + * @param {{position: {ms: number, count: number}|null, lineCount: number|'all', maxLines: number}} options + * @returns {Promise<{lines: string[], position: {ms: number, count: number}|null, rolledOver: boolean, truncated: boolean, skipped: boolean}>} + */ +async function dockerContainerLogsPolling(idOrName, options = {}) { + const { + position = null, since = null, lineCount = 'all', maxLines = 5000, + } = options; - if (sinceTimestamp) { - logOptions.since = new Date(sinceTimestamp).getTime() / 1000; - } - await new Promise((resolve, reject) => { - // eslint-disable-next-line consistent-return - dockerContainer.logs(logOptions, (err, mystream) => { - if (err) { - log.error('Error fetching logs:', err); - if (callback) { - callback(err); - } - return reject(err); - } - try { - dockerContainer.modem.demuxStream(mystream, logStream, logStream); - setTimeout(() => { - logStream.end(); - }, 1500); - mystream.on('end', () => { - logStream.end(); - resolve(); - }); - - mystream.on('error', (error) => { - log.error('Stream error:', error); - logStream.end(); - if (callback) { - callback(error); - } - reject(error); - }); - } catch (error) { - log.error('Error during stream processing:', error); - if (callback) { - callback(new Error('An error occurred while processing the log stream')); - } - reject(error); - } - }); - }); - } catch (error) { - log.error('Error in dockerContainerLogsPolling:', error); - if (callback) { - callback(error); + const dockerContainer = await getDockerContainerByIdOrName(idOrName); + + const logOptions = { + follow: false, + stdout: true, + stderr: true, + timestamps: true, + }; + + // A `since` timestamp is a FILTER; a position is a receipt for lines the reader + // already holds. Only the second one earns the behaviour below - dropping the + // line limit, capping, and reporting rolled-over - because only the second one + // is a reader walking forward who will come back for the rest. Treating a + // typed-in date as a position removed its line limit and answered it with a + // data-loss warning for a line it never claimed to have. + if (since !== null) { + logOptions.since = since / 1000; + if (lineCount && lineCount !== 'all') logOptions.tail = lineCount; + } else if (position) { + logOptions.since = position.ms / 1000; + // `since` on its own has no index to seek with: docker reads from the start + // of the oldest file and decodes forward until it finds the first matching + // line, so it re-reads the whole retained log on every poll and gets slower + // as that log grows - 74ms at 3.5MB, 273ms at 14MB, measured. With `tail` + // present it opens at the END and works backwards applying `since` as it + // goes, which answers byte-for-byte the same in ~4ms and stays flat however + // big the log is. + // + // `tail` bounds the window over the FILE and `since` is applied to that + // window afterwards - measured on a live daemon 2026-09-07: asked for the + // last 1884 lines at-or-after a timestamp, it answered 1883, having dropped + // a leading line stamped before it. So the window has to hold the overlap + // this reader already acknowledged as well as the page it is owed, or the + // filter trims the front of the window and `count` - a place in the sequence + // docker returns - is measured from a different first line than the one it + // was taken from. That loses the lines in between with nothing to report it: + // the answer comes back shorter than a page, which reads as "the whole set + // fitted". + // + // Capped at what the log can hold, because `position.count` is a value the + // caller writes and this is the bound that keeps the read cheap: uncapped, a + // crafted position sizes the window itself and asks for the whole retained + // log. Capped by the page instead it would be too SMALL - `count` grows past + // a page whenever a burst lands inside one millisecond - and a window that + // does not cover the overlap answers a reader with nothing, forever, and + // says nothing about it. + logOptions.tail = Math.min(maxLines + 1 + position.count, MAX_RETAINED_LINES); + } else if (lineCount && lineCount !== 'all') { + logOptions.tail = lineCount; + } + + const payload = await dockerContainer.logs(logOptions); + + // Every app container is created with Tty false (appDockerCreate), so docker + // frames each write with an 8-byte header carrying the stream id and length. + // + // Decoded a slice at a time with the event loop released between slices, and + // through the decoder the follow stream already uses - it carries a partial + // frame AND a partial line across a boundary, which is what makes an + // arbitrary slice safe to hand it. What this replaced walked every frame in + // one synchronous pass and then joined every body into a single string. + // + // Measured on a node against an 8.47MB log, 163,417 lines: that pass held the + // event loop for 40ms on a clean heap and 564ms on a warm one, against a + // 0.3ms idle baseline - and the event loop is the whole node, so those + // milliseconds answer no peer and no other request. This holds it for 0.4ms. + // Peak RSS halves as well, because the joined 8MB string is never built. The + // answer is identical line for line; only who else gets to run changes. + const decoder = new LogFrameDecoder({ timestamped: true }); + let lines = []; + for (let at = 0; at < payload.length; at += LOG_DECODE_CHUNK_BYTES) { + const decoded = decoder.push(payload.subarray(at, Math.min(at + LOG_DECODE_CHUNK_BYTES, payload.length))); + // Appended rather than spread: a slice can finish tens of thousands of + // lines, and push(...lines) at that width is an argument list long enough + // to overflow the stack. + for (let i = 0; i < decoded.length; i += 1) lines.push(decoded[i]); + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setImmediate(resolve); }); + } + const held = decoder.flush(); + for (let i = 0; i < held.length; i += 1) lines.push(held[i]); + + // The window came back full, so more is waiting than this read can see. The + // reader is moved to the end of the log rather than walked to it: seeing past + // the window means a read with no `tail`, which is the unbounded decode the + // docblock measures, and a reader writing faster than a page per poll would + // pay it on every poll forever without ever arriving. + // + // Moving them is also the answer they want. A viewer more than a page behind + // is asking to see what is happening now, which is what `docker logs`, + // kubectl and journalctl all answer by default. What it must not do is stay + // silent about the gap, so `skipped` says it outright. + // + // Counted in lines, which is why it sits after the decode: `tail` bounds the + // read in lines and a frame is a write, so docker splitting a write larger + // than its buffer put several frames behind one line. Counting those declared + // an overflow that had not happened - which reported a gap the reader never + // had and, because an overflow skips the overlap drop below, handed back the + // lines it had just acknowledged. The decode below the old count ran + // unconditionally, so reading the bodies never cost anything to avoid. + // Against the NEW lines, not the window: the window is a page plus the + // overlap, so comparing its whole length would call a reader that is up to + // date more than a page behind and resync it. + const skipped = position !== null && lines.length - position.count > maxLines; + + // The line asked from is absent, so docker rotated it away between polls and + // what sat between it and the oldest line here is gone. Reporting it is the + // only honest answer: nothing can recover those lines, and a reader that is + // not told sees a silent gap it has no way to notice. + let rolledOver = false; + if (position && !skipped) { + const firstMs = lines.length ? Date.parse(lines[0].split(' ')[0]) : position.ms; + if (firstMs > position.ms) { + rolledOver = true; + } else { + lines = lines.slice(position.count); } - throw error; } + + // A resync keeps the NEWEST of the window: the reader is being moved to where + // the log is now, and the lines below are the ones `skipped` accounts for. + if (skipped) lines = lines.slice(-maxLines); + + // There is no "more is waiting" answer, because the window read and the page + // returned are the same size: a positioned reader is either answered + // completely or resynced. A reader up to a page behind gets all of it in this + // one response, which is better than being handed it a page at a time, and + // past that there is nothing to page towards - the lines it would walk are the + // ones `skipped` accounts for. + // + // `truncated` is a different question and a live one: what lies BEHIND a line + // limit, which is the answer this endpoint has always given a caller that + // asked for the last N lines and got N. Nothing can walk backwards to fetch + // the rest, so a positioned reader is never told it - the only thing it could + // do about it is a poll that returns nothing. A caller without a position is + // not capped at all: "all logs" is a download, not a page, and the retention + // config already bounds what a container can hold. + const truncated = position === null && lineCount !== 'all' && lines.length >= lineCount; + + // The position is the last line handed over, never the newest line seen: a + // reader that is behind must come back for the rest, and it comes back from + // where it actually got to. + // + // `count` is how many lines have been delivered FROM THE FRONT of what docker + // returns for `ms` - a place in that sequence, not a property of a timestamp. + // The distinction is the whole correctness of the skip. A container writing to + // stdout and stderr has two writers that each stamp a line before it is + // serialised into the file, so the file is NOT in timestamp order: measured on + // a real daemon, 3,304 backwards steps in 40,000 lines. Counting "lines whose + // millisecond equals the last one's" therefore missed the already-delivered + // lines stamped just before `ms`, the skip came up short, and the tail of each + // page was delivered twice. Counting position in the returned sequence is + // exact whatever order docker returns, because it counts the same sequence + // docker returns again. + let nextPosition = position; + if (lines.length) { + // The NEWEST timestamp delivered, not the last line's. Out of order they are + // different, and only the newest is monotonic: taking the last line's would + // drag the window backwards and re-read everything after it. + const carried = position && !rolledOver && !skipped ? position : null; + const newestMs = lines.reduce( + (max, line) => Math.max(max, Math.floor(Date.parse(line.split(' ')[0]))), + carried ? carried.ms : 0, + ); + // When nothing newer arrived, the line docker will start from is one an + // earlier page already delivered, so that page's count still applies and + // this page's lines are added to it. When something newer did arrive, the + // starting line is in this page and the count is measured from there. + nextPosition = carried && newestMs === carried.ms + ? { ms: newestMs, count: carried.count + lines.length } + : { ms: newestMs, count: countFrom(lines, newestMs) }; + } + + return { + lines, position: nextPosition, rolledOver, truncated, skipped, + }; } async function obtainPayloadFromStorage(url, appName) { @@ -547,6 +677,7 @@ async function obtainPayloadFromStorage(url, appName) { const timestamp = Date.now(); const message = version + url + timestamp; const signature = await fluxCommunicationMessagesSender.getFluxMessageSignature(message); + if (!signature) throw new Error('This node cannot sign the request as itself'); const axiosConfig = { headers: { 'flux-message': message, @@ -721,6 +852,98 @@ const getContainerIP = async (containerName) => { } }; +/** + * Create a container from a fully-formed options object and return its handle. + * + * The thin wrapper appDockerCreate does not provide: that one assembles an + * application's container from a spec. Callers that run a short-lived container + * of their own build their own options and need the handle back to wait on it. + * + * @param {object} options - docker create options + * @returns {Promise} dockerode container + */ +async function createContainer(options) { + return docker.createContainer(options); +} + +/** + * Whether a container summary is one of this node's APPLICATION containers. + * + * The `runonflux.role` label is authoritative when present: FluxOS runs + * containers for its own purposes too, and those must stay invisible to + * anything that reclaims apps. `forceAppRemovals` derives an app name from a + * container name by slicing a prefix and splitting on the underscore, so a + * container that is not shaped `flux_` yields a + * plausible-looking wrong name which is then handed to removeAppLocally. + * + * The name-prefix test remains for containers created before the labels shipped + * and not recreated since. + * + * @param {object} container - a container summary from dockerListContainers + * @returns {boolean} + */ +function isAppContainer(container) { + const role = container.Labels && container.Labels['runonflux.role']; + if (role) return role === 'app'; + + const name = (container.Names && container.Names[0]) || ''; + return name.slice(1, 4) === 'zel' || name.slice(1, 5) === 'flux'; +} + +/** + * Whether a container summary is one FluxOS put there, in ANY role. + * + * The broader question than isAppContainer, and the one a sweep that stops + * foreign containers has to ask. A file-operation container is emphatically not + * an application, so isAppContainer answers no about it - and a sweep phrased + * as "stop everything that is not an app" would therefore stop the node's own + * work mid-copy. + * + * A container FluxOS runs for itself is created with no name, because a name it + * does not need is one more thing that can collide with a tenant's. Docker then + * assigns a random one, which no prefix test can recognise - so the label is + * the only thing that can answer this question at all. + * + * @param {object} container - a container summary from dockerListContainers + * @returns {boolean} + */ +function isFluxOwnedContainer(container) { + if (container.Labels && container.Labels['runonflux.role']) return true; + + const name = (container.Names && container.Names[0]) || ''; + return name.slice(1, 4) === 'zel' || name.slice(1, 5) === 'flux'; +} + +/** + * The identity of a container, stamped as docker labels when it is created. + * + * The container NAME already encodes this (`flux_`), but a name + * is a string every caller has to re-parse, and the parsers have drifted - some + * slice a fixed prefix width, some split on the underscore, and a container + * whose name does not fit that shape yields a plausible-looking wrong answer + * rather than an error. A label is read back verbatim. + * + * `role` separates an application container from one FluxOS runs for its own + * purposes, so a sweep that reclaims orphaned apps can select what it owns + * instead of inferring it from a name prefix. + * + * @param {string} appName + * @param {string} componentName - the component, or the app name for the + * single-component flat form which has no separate component + * @param {string|null} owner - the app owner's id, when the caller has the + * full spec in scope + * @returns {Object} + */ +function componentIdentityLabels(appName, componentName, owner) { + const labels = { + 'runonflux.app': appName, + 'runonflux.component': componentName, + 'runonflux.role': 'app', + }; + if (owner) labels['runonflux.owner'] = owner; + return labels; +} + /** * Creates an app container. * @@ -918,8 +1141,14 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS : { Type: 'json-file', Config: { - 'max-file': '1', - 'max-size': '20m', + // Same 20MB of disk as one 20MB file, and a floor instead of none. Docker + // does not trim a full log file, it discards it: with one file the history + // an operator can read swings between 20MB and NOTHING, and the wipe takes + // everything with it. Split into four, only the oldest quarter is dropped + // per rotation, so at least 15MB is always readable. `docker logs` reads + // across the set, so nothing that reads logs needs to know. + 'max-file': `${LOG_MAX_FILES}`, + 'max-size': `${LOG_MAX_FILE_MB}m`, }, }; const autoAssignedIP = await getNextAvailableIPForApp(appName); @@ -928,9 +1157,9 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS // scope, and stamped onto the container as docker labels. Every subsequent // start of this container (initial start, restart, recovery) reads the // labels in appDockerStart and reapplies burst — no per-caller plumbing. - const burstOwner = fullAppSpecs?.owner || null; - const burstEligible = burstOwner - && cpuBurstHelper.isEnterpriseOwner(burstOwner) + const appOwner = fullAppSpecs?.owner || null; + const burstEligible = appOwner + && cpuBurstHelper.isEnterpriseOwner(appOwner) && await cpuBurstHelper.isCpuBurstSupported(); const burstLabels = burstEligible ? { @@ -938,9 +1167,14 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS 'flux.burst.cores': String(appSpecifications.cpu), } : null; - const containerLabels = (labels || burstLabels) - ? { ...(labels || {}), ...(burstLabels || {}) } - : null; + const identityLabels = componentIdentityLabels( + appName, + isComponent ? appSpecifications.name : appName, + appOwner, + ); + const containerLabels = { + ...identityLabels, ...(labels || {}), ...(burstLabels || {}), + }; if (burstEligible) { log.info(`CPU burst: marking ${identifier} as burst-eligible (cores=${appSpecifications.cpu})`); } @@ -956,8 +1190,7 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS Env: envParams, Tty: false, ExposedPorts: exposedPorts, - // Conditionally include Labels only if it's not null - ...(containerLabels && { Labels: containerLabels }), + Labels: containerLabels, HostConfig: { NanoCPUs: Math.round(appSpecifications.cpu * 1e9), Memory: Math.round(appSpecifications.ram * 1024 * 1024), @@ -1075,6 +1308,11 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS throw error; }); + // The container exists, so there is no absence of it to attribute to anyone. + // The other half of the removal funnels' record: while an entry stands, this + // container is missing because FluxOS took it. + globalState.fluxRemovedContainers.delete(getDockerName(identifier)); + return app; } @@ -1231,6 +1469,24 @@ async function appDockerKill(idOrName) { return `Flux App ${idOrName} successfully killed.`; } +/** + * Whether a removal is worth recording as FluxOS's own. + * + * The record answers one question, asked by the reconciler: is this container + * missing because FluxOS removed it, or because something else did? It only ever + * asks about app containers, by identifier. A container addressed by raw docker + * id is an orphan the reconciler never asks about - so the entry can never be + * read, and it can never be dropped either, because clearFluxRemovedContainers + * matches on the app name a hex id does not carry. Not writing it is the whole + * fix; there is nothing about it worth remembering. + * + * @param {string} idOrName + * @returns {boolean} + */ +function removalIsWorthRecording(idOrName) { + return !/^[0-9a-f]{12,64}$/.test(idOrName); +} + /** * Removes app's docker. * @@ -1243,6 +1499,9 @@ async function appDockerRemove(idOrName) { globalState.stoppingContainers.delete(getDockerName(idOrName)); await dockerContainer.remove(); + // Recorded only once the container is actually gone - this is a record of what + // FluxOS removed, and a remove that threw removed nothing. + if (removalIsWorthRecording(idOrName)) globalState.fluxRemovedContainers.add(getDockerName(idOrName)); return `Flux App ${idOrName} successfully removed.`; } @@ -1259,9 +1518,303 @@ async function appDockerForceRemove(idOrName, removeVolumes = true) { globalState.stoppingContainers.delete(getDockerName(idOrName)); await dockerContainer.remove({ force: true, v: removeVolumes }); + if (removalIsWorthRecording(idOrName)) globalState.fluxRemovedContainers.add(getDockerName(idOrName)); return `Flux App ${idOrName} successfully force removed.`; } +/** + * Drop every fluxRemovedContainers entry belonging to an app. Called when the + * app's local row goes: nothing reconciles an app with no row, so there is no + * absence left to attribute, and an entry with no reader would otherwise outlive + * the app for the life of the process. + * + * Every entry belongs to an app, because a removal addressed by raw docker id is + * not recorded at all - it would carry no app name for this to match, and no + * reader to want it. + * + * Lives here because the entries are keyed by docker name and this module owns + * that naming: a component is `flux_`, a v<=3 app is `flux`, + * and an app name never contains an underscore (the codebase splits component + * identifiers on it throughout). + * + * @param {string} appName - bare app name + */ +function clearFluxRemovedContainers(appName) { + const appDockerName = getDockerName(appName); + for (const container of globalState.fluxRemovedContainers) { + if (container === appDockerName || container.endsWith(`_${appName}`)) { + globalState.fluxRemovedContainers.delete(container); + } + } +} + +/** + * Whether an image is in this node's local store. + * + * Creating a container does NOT pull. Docker answers 404 for an image it does + * not hold, so anything running a pinned image has to ask this first and fetch + * it itself. + * + * @param {string} reference - repo:tag or repo@digest + * @returns {Promise} + */ +async function imageExists(reference) { + try { + await docker.getImage(reference).inspect(); + return true; + } catch (error) { + if (error.statusCode === 404) return false; + throw error; + } +} + +/** + * The id an image reference currently resolves to, or null if there is none. + * + * A tag says nothing about WHICH image it names - it is moved by whoever loads + * or pulls one next - so anything deciding what to do with a reference someone + * else chose has to ask what it points at now. + * + * @param {string} reference - repo:tag, an id, or repo@digest + * @returns {Promise} + */ +async function getImageId(reference) { + try { + const info = await docker.getImage(reference).inspect(); + return info.Id; + } catch (error) { + if (error.statusCode === 404) return null; + throw error; + } +} + +/** + * Pull an image, resolving when the pull has finished. + * + * dockerPullStream reports progress through a callback, so a caller that wants + * to await it has to wrap it - and wrapping it at the call site, as two of them + * did, captures this function when the CALLING module loads. That pins whichever + * version was in place at that moment, which is a detail of this module that has + * no business reaching callers, and it makes the wrapped copy unreachable to + * anything that replaces this one afterwards. + * + * @param {object} pullConfig - repoTag and optional auth + * @param {object} [res] - response to stream progress to, if any + * @returns {Promise<*>} + */ +function pullImage(pullConfig, res = null) { + return new Promise((resolve, reject) => { + dockerPullStream(pullConfig, res, (error, result) => { + if (error) reject(error); + else resolve(result); + }); + }); +} + +/** + * Stream an image out of the local store as a tar archive. + * + * The archive carries the image's config and layers, which is what makes an id + * checkable at the other end: the id IS the digest of that config, so a receiver + * can tell what it was sent from the bytes rather than from the sender. + * + * @param {string} reference - repo:tag, repo@digest or an image id + * @returns {Promise} + */ +async function exportImage(reference) { + return docker.getImage(reference).get(); +} + +/** + * Load images from a tar archive, answering what arrived. + * + * The ids are what identifies the image. An archive names itself - its tags are + * whatever the sender wrote - so a caller taking one from anywhere it does not + * control decides from the ids, and removes whatever it did not want. + * + * Tags are reported as well, and separately. They cannot say WHICH image + * something is, but an archive can carry them, and something loaded onto this + * node under a name of the sender's choosing is exactly what a caller has to be + * able to remove again. Reporting only the ids left those on the disk with + * nothing that could name them. + * + * The daemon narrates the load as a JSON stream and dockerode's reader frames + * it, the same way pullImage does. Concatenating chunks and matching a regex + * over them - which this did - loses any line that falls across two network + * writes, so an archive that did contain the wanted image reported that it did + * not. + * + * @param {NodeJS.ReadableStream} stream - a docker image archive + * @returns {Promise<{ids: Array, tags: Array}>} what the daemon + * reports loading + */ +async function loadImage(stream) { + const progress = await docker.loadImage(stream); + + const events = await new Promise((resolve, reject) => { + docker.modem.followProgress(progress, (error, output) => ( + error ? reject(error) : resolve(output || []) + )); + }); + + const ids = new Set(); + const tags = new Set(); + + for (const event of events) { + const narration = (event && event.stream) || ''; + + const id = narration.match(/Loaded image ID: (sha256:[0-9a-f]{64})/); + if (id) { + ids.add(id[1]); + } else { + const tag = narration.match(/Loaded image: (\S+)/); + if (tag) tags.add(tag[1]); + } + } + + return { ids: [...ids], tags: [...tags] }; +} + +/** + * What a manifest can legitimately weigh. + * + * The archive holds ONE image - a peer packs a single id - and docker refuses + * to build deeper than 125 layers, each listed as a path of about 80 bytes. + * That is ~10KB of Layers, plus a config path and any tags; a real one measures + * 1.2KB. 64KB is several times the format's own ceiling. + * + * It needs a ceiling at all because this entry is read into memory while the + * archive around it is a file, bounded at PEER_IMAGE_MAX_BYTES. Nothing stops a + * peer making the manifest the whole of that, and the reason the archive goes + * to disk in the first place is not holding it in heap. + */ +const MANIFEST_MAX_BYTES = 64 * 1024; + +/** What a gzip stream starts with, and the only two bytes needed to know. */ +const GZIP_MAGIC = Buffer.from([0x1f, 0x8b]); + +/** + * Refuse an archive that arrives compressed. + * + * The ceiling a peer's archive is taken under counts the bytes on the wire, and + * the reader below is node-tar, which inflates gzip transparently - so a + * compressed archive is bounded at what it weighs rather than at what it + * becomes. Measured at level 9 on zeros, that is 1029:1: the 32MB a peer may + * send expands to about 34GB, which is a minute of inflate on a laptop and + * longer on a node, on a threadpool thread the filesystem also wants. + * + * Refused by FORMAT rather than bounded by size, because this node never sends + * one: the serve path exports with `docker save`, which writes a plain tar. An + * archive that arrives compressed is doing something we do not do, so there is + * nothing to weigh and no limit to choose - and the next peer is asked instead. + * + * @param {string} archivePath + */ +async function refuseCompressedArchive(archivePath) { + const handle = await fs.open(archivePath, 'r'); + try { + const head = Buffer.alloc(GZIP_MAGIC.length); + const { bytesRead } = await handle.read(head, 0, head.length, 0); + if (bytesRead === head.length && head.equals(GZIP_MAGIC)) { + throw new Error('the archive is compressed, which this node does not accept from a peer'); + } + } finally { + await handle.close(); + } +} + +/** + * The names a docker image archive declares for what it carries. + * + * `docker load` applies these: an archive naming `some/app:v1` MOVES that name + * onto whatever the archive holds, taking it off whatever the node had under it. + * From a source this node does not control that is not a detail - it is the + * sender choosing what this node's own images are called - so a caller has to be + * able to look before loading rather than repair afterwards. Repairing is too + * late: removing the stolen name does not give it back to the image that had it, + * and that image is then nameless, which is to say dangling, which is to say the + * next prune deletes it. + * + * Read from the archive rather than from the sender's word about it. Only + * manifest.json is parsed; the layers are not touched, so this costs a scan of + * the tar's headers. + * + * @param {string} archivePath - a docker image archive on disk + * @returns {Promise>} every name the archive declares + */ +async function archiveNames(archivePath) { + await refuseCompressedArchive(archivePath); + + let manifest = ''; + let taken = 0; + let found = false; + let oversize = false; + + await tar.t({ + file: archivePath, + onReadEntry(entry) { + if (entry.path !== 'manifest.json') { + entry.resume(); + return; + } + found = true; + entry.on('data', (chunk) => { + taken += chunk.length; + // Past the ceiling the bytes are drained and dropped rather than kept: + // the entry still has to be walked to finish the scan, but nothing more + // of it is held. + if (taken > MANIFEST_MAX_BYTES) { + oversize = true; + return; + } + manifest += chunk.toString(); + }); + entry.resume(); + }, + }); + + // An archive with no manifest is not a docker image archive. Saying so here + // is better than letting the daemon report it as an empty load, which reads + // as "the peer did not have it" and sends the caller to another peer. + if (!found) throw new Error('the archive carries no manifest'); + + if (oversize) { + throw new Error(`the archive's manifest is over ${MANIFEST_MAX_BYTES} bytes, so it is not describing one image`); + } + + const entries = JSON.parse(manifest); + return entries.flatMap((entry) => (entry && entry.RepoTags) || []); +} + +/** + * Give a loaded image the name it is pinned under. + * + * An archive addressed by id carries no names - the daemon writes RepoTags only + * for a reference that has one - so an image taken from a peer arrives nameless. + * A nameless image is a DANGLING image, and the prune that runs before every app + * install takes dangling images. Naming it is what leaves the peer path in the + * same state a registry pull leaves, so the image survives to be used. + * + * Named after the id has been checked, never before: the name is this node's + * word for what it verified, not the sender's word for what it sent. + * + * @param {string} id - the image id, already verified + * @param {string} reference - the repo:tag to name it with + * @returns {Promise} + */ +async function tagImage(id, reference) { + // The tag is what follows the last colon, and only when no slash follows it: + // a registry host names its port with a colon too, so cutting at the first + // one turns `fluxregistry:5000/x` into the repository `fluxregistry`. + const cut = reference.lastIndexOf(':'); + const tagged = cut > 0 && !reference.slice(cut).includes('/'); + + await docker.getImage(id).tag({ + repo: tagged ? reference.slice(0, cut) : reference, + tag: tagged ? reference.slice(cut + 1) : 'latest', + }); +} + /** * Removes app's docker image. * @@ -1359,34 +1912,6 @@ async function dockerNetworkState(networkName) { } } -/** - * Pauses app's docker. - * - * @param {string} idOrName - * @returns {string} message - */ -async function appDockerPause(idOrName) { - // container ID or name - const dockerContainer = await getDockerContainerByIdOrName(idOrName); - - await dockerContainer.pause(); - return `Flux App ${idOrName} successfully paused.`; -} - -/** - * Unpauses app's docker. - * - * @param {string} idOrName - * @returns {string} message - */ -async function appDockerUnpause(idOrName) { - // container ID or name - const dockerContainer = await getDockerContainerByIdOrName(idOrName); - - await dockerContainer.unpause(); - return `Flux App ${idOrName} successfully unpaused.`; -} - /** * Returns app's docker's active processes. * @@ -1500,6 +2025,66 @@ async function getFreeFluxAppNetworkOctet(excludeOctets = new Set()) { return null; } +/** + * Remove app networks that no installed app owns. + * + * A network is created per app and removed by the uninstaller, and by nothing + * else. An uninstall interrupted between the container going and the network + * going - a reboot, a crash, a removal that failed both its retries - leaves + * one behind for ever, because nothing looks again. + * + * That is not free. Each carries an explicitly assigned `172.23..0/24`, + * and getFreeFluxAppNetworkOctet walks 1..255 for one nothing is using: a + * leaked network holds its octet permanently, and when the last one goes the + * answer is null and no app can be installed on the node again. Rare, never + * self-healing, and terminal when it arrives. + * + * The caller supplies the names it expects, rather than this deriving them: an + * app name can be recovered from a network name only by assuming what is in it, + * and being wrong there deletes a live app's network. + * + * A network with anything attached is left alone whatever the caller said, + * because something is using it and this cannot be the thing that decides + * otherwise. + * + * @param {Set} expected - network names installed apps account for + * @returns {Promise} what was reclaimed + */ +async function reclaimAppNetworks(expected) { + const reclaimed = []; + const networks = await getFluxDockerNetworks(); + + for (const summary of networks) { + const name = summary.Name; + if (!name || !name.startsWith('fluxDockerNetwork_') || expected.has(name)) { + // eslint-disable-next-line no-continue + continue; + } + + const network = docker.getNetwork(name); + // eslint-disable-next-line no-await-in-loop + const detail = await dockerNetworkInspect(network).catch(() => null); + if (!detail) { + // eslint-disable-next-line no-continue + continue; + } + if (Object.keys(detail.Containers || {}).length) { + log.info(`reclaimAppNetworks - ${name} has no installed app but something is attached; leaving it`); + // eslint-disable-next-line no-continue + continue; + } + + // eslint-disable-next-line no-await-in-loop + const removed = await dockerRemoveNetwork(network).then(() => true).catch((error) => { + log.warn(`reclaimAppNetworks - could not remove ${name}: ${error.message}`); + return false; + }); + if (removed) reclaimed.push(name); + } + + return reclaimed; +} + /** * Creates flux application docker network if doesn't exist * @@ -1693,26 +2278,15 @@ async function getAppContainerNames(appName) { return names; } -/** - * Remove all unused containers. Unused contaienrs are those wich are not running - */ -async function pruneContainers() { - return docker.pruneContainers(); -} - -/** - * Remove all unused networks. Unused networks are those which are not referenced by any running containers - */ -async function pruneNetworks() { - return docker.pruneNetworks(); -} - -/** - * Remove all unused Volumes. Unused Volumes are those which are not referenced by any containers - */ -async function pruneVolumes() { - return docker.pruneVolumes(); -} +// No blanket container/network/volume prune primitive is exposed, deliberately. +// Docker's "unused" is a runtime predicate - nothing attached right now - which +// is true of every healthy app whose container is momentarily down, of every +// container FluxOS runs for its own purposes between exiting and being reaped, +// and of anything the node operator left stopped on their own machine. A prune +// keyed on it deletes all three. Removal of flux objects is scoped by OWNERSHIP +// instead: appUninstaller for an app's containers and volumes, appNetwork for +// its networks, and the identity labels stamped by componentIdentityLabels for +// everything else. /** * Remove all unused Images. Unused Images are those which are not referenced by any containers @@ -1868,15 +2442,21 @@ module.exports = { appDockerCreate, appDockerUpdateCpu, appDockerImageRemove, + imageExists, + getImageId, + pullImage, + exportImage, + loadImage, + archiveNames, + tagImage, appDockerKill, - appDockerPause, appDockerRemove, appDockerForceRemove, + clearFluxRemovedContainers, appDockerRestart, appDockerStart, appDockerStop, appDockerTop, - appDockerUnpause, createFluxAppDockerNetwork, createFluxDockerNetwork, dockerContainerChanges, @@ -1884,9 +2464,7 @@ module.exports = { dockerContainerInspect, dockerContainerLogs, dockerContainerLogsPolling, - dockerContainerLogsStream, dockerContainerStats, - dockerContainerStatsStream, dockerCreateNetwork, dockerGetEvents, dockerGetUsage, @@ -1897,6 +2475,7 @@ module.exports = { dockerNetworkInspect, dockerPullStream, dockerRemoveNetwork, + reclaimAppNetworks, dockerVersion, getAppDockerNameIdentifier, getAppIdentifier, @@ -1908,15 +2487,15 @@ module.exports = { getFluxDockerNetworkSubnets, getFreeFluxAppNetworkOctet, migrateContainerRestartPolicies, - pruneContainers, pruneImages, - pruneNetworks, - pruneVolumes, removeFluxAppDockerNetwork, forceRemoveFluxAppDockerNetwork, appDockerNetworkConnect, getAppContainerNames, getAppContainerObjects, + isAppContainer, + isFluxOwnedContainer, + createContainer, getAppNameByContainerIp, classifyContainerNetworkAttachment, isContainerDetachedFromNetwork, diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index 137b6e5aa3..10241717fc 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -22,6 +22,7 @@ const { extractIp } = require('./utils/socketAddressUtils'); const fluxEventBus = require('./utils/fluxEventBus'); const globalState = require('./utils/globalState'); const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('./utils/appSyncEvents'); +const { Privilege, authOf } = require('./utils/privileges'); const coinbaseFusionIndexCollection = config.database.daemon.collections.coinbaseFusionIndex; // fusion const utxoIndexCollection = config.database.daemon.collections.utxoIndex; @@ -626,7 +627,20 @@ async function processBlock(blockHeight, isInsightExplorer) { const options = { upsert: true, }; - // this should run only when node is synced + // True when this block was still the chain tip at the moment it was fetched, + // which is NOT the same as "the node is synced". The tip is read from a + // cache refreshed every daemonInfoIntervalMs, and the branch at the end of + // this function chains straight into the next block whenever it is behind - + // so a burst of blocks arriving between two refreshes is processed back to + // back and only the LAST of them qualifies. Everything gated below is + // skipped for the others, and skipped silently: expiring global app + // records, trimming surplus instances, reinstalling outdated apps. + // + // Post-PON that is a 30s block against a 30s refresh, and the refresh + // reschedules only after awaiting its RPC, so it drifts behind the chain and + // the bursts recur. v9 removes the race rather than tuning it - fluxd pushes + // hashblockheight to chainTipSource, which drives both the cached tip and + // this scan, so a block is processed while it is still the tip. isSynced = !(blockDataVerbose.confirmations >= 2); if (isSynced) { blockEmitter.emit('blocksProcessed', scannedHeight); @@ -1104,12 +1118,22 @@ async function checkAndHandleReorgs(database, scannedBlockHeight) { return height; } +// How long the explorer waits before asking again whether the chain has moved. +// +// This is the floor on how fast a node can process blocks: a block is not looked +// at until the next poll, so nothing downstream of block processing - expiring +// app records, trimming surplus instances, the give-up pass - can run more often +// than this however fast blocks are produced. Production wants 5s against a 30s +// block; a harness driving its own chain wants it far shorter, and had no way to +// say so. +const POLL_INTERVAL_MS = config.fluxapps.explorerPollIntervalMs ?? 5000; + async function pollForNewBlocks() { if (!blockProccessingCanContinue) return; try { const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); if (!syncStatus.data.synced) { - pollTimeout = setTimeout(pollForNewBlocks, 5000); + pollTimeout = setTimeout(pollForNewBlocks, POLL_INTERVAL_MS); return; } @@ -1131,10 +1155,10 @@ async function pollForNewBlocks() { return; } - pollTimeout = setTimeout(pollForNewBlocks, 5000); + pollTimeout = setTimeout(pollForNewBlocks, POLL_INTERVAL_MS); } catch (error) { log.error(`Explorer poll error: ${error.message}`); - pollTimeout = setTimeout(pollForNewBlocks, 5000); + pollTimeout = setTimeout(pollForNewBlocks, POLL_INTERVAL_MS); } } @@ -1725,7 +1749,7 @@ async function checkBlockProcessingStopped(i, callback) { * @param {object} res Response. */ async function stopBlockProcessing(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { const i = 0; checkBlockProcessingStopped(i, async (response) => { @@ -1744,7 +1768,7 @@ async function stopBlockProcessing(req, res) { * @param {object} res Response. */ async function restartBlockProcessing(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { const i = 0; checkBlockProcessingStopped(i, async () => { @@ -1764,7 +1788,7 @@ async function restartBlockProcessing(req, res) { * @param {object} res Response. */ async function reindexExplorer(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { // stop block processing const i = 0; @@ -1806,53 +1830,6 @@ async function reindexExplorer(req, res) { } } -async function fixExplorer(height = 1670000, rescanApps = true) { - try { - const dbopen = dbHelper.databaseConnection(); - const blockheight = serviceHelper.ensureNumber(height); - const database = dbopen.db(config.database.daemon.database); - const query = { generalScannedHeight: { $gte: 0 } }; - const projection = { - projection: { - _id: 0, - generalScannedHeight: 1, - }, - }; - const currentHeight = await dbHelper.findOneInDatabase(database, scannedHeightCollection, query, projection); - if (!currentHeight) { - throw new Error('No scanned height found'); - } - if (currentHeight.generalScannedHeight <= blockheight) { - throw new Error('Block height shall be lower than currently scanned'); - } - if (blockheight < 0) { - throw new Error('BlockHeight lower than 0'); - } - const rescanapps = serviceHelper.ensureBoolean(rescanApps); - if (blockheight === 0) { - await dbHelper.dropCollection(database, scannedHeightCollection).catch((error) => { - if (error.message !== 'ns not found') { - log.error(error); - } - }); - } else { - // stop block processing - const update = { $set: { generalScannedHeight: blockheight } }; - const options = { - upsert: true, - }; - // update scanned Height in scannedBlockHeightCollection - await dbHelper.updateOneInDatabase(database, scannedHeightCollection, query, update, options); - } - initiateBlockProcessor(true, false, rescanapps); // restore database and possibly do rescan of apps - const message = messageHelper.createSuccessMessage(`Explorer rescan from blockheight ${blockheight} initiated`); - log.info(message); - } catch (error) { - log.warn(error); - initiateBlockProcessor(true, true); - } -} - /** * To rescan Flux explorer database from a specific block height. Only accessible by admins and Flux team members. * @param {object} req Request. @@ -1860,7 +1837,7 @@ async function fixExplorer(height = 1670000, rescanApps = true) { */ async function rescanExplorer(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { // since what blockheight let { blockheight } = req?.params || {}; // we accept both help/command and help?command=getinfo diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index c1f459d0f8..16709f6e11 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -13,7 +13,7 @@ const fluxNetworkHelper = require('./fluxNetworkHelper'); const messageHelper = require('./messageHelper'); const dbHelper = require('./dbHelper'); const { peerManager, PEER_SOURCE } = require('./utils/peerState'); -const { SIGTERM_EXPIRY_MS } = require('./utils/appConstants'); +const { SIGTERM_EXPIRY_MS, RUNNING_EXPIRY_MS } = require('./utils/appConstants'); const cacheManager = require('./utils/cacheManager').default; const networkStateService = require('./networkStateService'); const nodeConfirmationService = require('./nodeConfirmationService'); @@ -37,6 +37,7 @@ const { FluxPeerManager, DIRECTION, FLUX_VERSION, FLUX_CAPABILITIES } = require( const { NAK_REASON, buildSyncSignatureMessage } = require('./utils/peerCodec'); const { networkHealthMonitor } = require('./utils/NetworkHealthMonitor'); const verifyPool = require('./utils/verifyPool'); +const { Privilege, authOf } = require('./utils/privileges'); const DISCOVERY = { maxOutbound: 14, @@ -47,6 +48,12 @@ const DISCOVERY = { connectionDelayMs: config.fluxapps.discoveryConnectionDelayMs ?? 500, }; +// How many events of a sync response are carried through verification and +// storage together. A response holds up to 2500, and each event fans out into a +// database operation per app it reports, so the whole response is never held at +// once. +const SYNC_EVENTS_PER_SLICE = 250; + /** * To handle temporary app messages. * @param {object} message Message. @@ -109,8 +116,9 @@ async function batchVerifyBroadcasts(broadcasts, label) { if (items.length === 0) return []; - const workerItems = items.map((it) => ({ messageToVerify: it.messageToVerify, pubKey: it.pubKey, signature: it.signature })); - const cryptoResults = await verifyPool.verify(workerItems); + // The worker reads only the three fields it needs, so items go across as they + // are rather than being copied into a narrower shape first + const cryptoResults = await verifyPool.verify(items); const verified = []; for (let i = 0; i < items.length; i++) { @@ -123,11 +131,22 @@ async function batchVerifyBroadcasts(broadcasts, label) { return verified; } -async function handleTempSyncResponse(message, peerKey) { +async function handleTempSyncResponse(message, peerSocket) { try { - if (!peerManager.isSyncRequested(peerKey)) return; + if (!peerManager.isSyncResponseWanted(peerSocket)) return; + const peerKey = peerSocket.key; if (!message.data || message.data.type !== 'fluxapptempsync') return; - const { messages, done } = message.data; + const { messages, done, refused } = message.data; + // A peer whose own app state is not authoritative holds an unknown fraction + // of the network's pending registrations, and says so rather than sending + // the fraction. A refusal is an answer and not a completion - see + // handleAppRunningSyncResponse - and a peer refuses all four streams or + // none, so whichever refusal arrives first ends the request. + if (refused) { + log.info(`handleTempSyncResponse - ${peerKey} declined: its app state is not authoritative yet`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_REFUSED, 'apptemp', peerKey); + return; + } if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleTempSyncResponse - Received ${messages.length} temp messages from ${peerKey} (done: ${!!done})`); let stored = 0; @@ -140,65 +159,130 @@ async function handleTempSyncResponse(message, peerKey) { } } log.info(`handleTempSyncResponse - Processed ${stored} of ${messages.length} messages`); + // COUNTED LIKE THE REST. A peer is credited once it has delivered every + // stream it was asked for, so the record that admits its responses stays + // open until this one has ended too - the three surveys finishing first no + // longer cuts off what is still arriving here. + if (done) { + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apptemp', peerKey); + log.info('handleTempSyncResponse - Sync complete'); + } } catch (error) { log.error(error); } } -async function handleAppRunningSyncResponse(message, peerKey) { +async function handleAppRunningSyncResponse(message, peerSocket) { try { if (!message.data || message.data.type !== 'fluxapprunningsync') return; - if (!peerManager.isSyncRequested(peerKey)) return; - const { messages, done } = message.data; + if (!peerManager.isSyncResponseWanted(peerSocket)) return; + const peerKey = peerSocket.key; + const { messages, done, refused } = message.data; + // A peer that says its own app state is not worth surveying has ANSWERED, + // and the answer is not a completion. Marking it declined stops it being + // offered again on this connection, which opens a deficit in the pool of + // outstanding requests and gets another peer asked. + if (refused) { + log.info(`handleAppRunningSyncResponse - ${peerKey} declined: its app state is not authoritative yet`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_REFUSED, 'apprunning', peerKey); + return; + } if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleAppRunningSyncResponse - Received ${messages.length} events from ${peerKey} (done: ${!!done})`); - const appRunningBroadcasts = []; - const otherBroadcasts = []; - const evictedEvents = []; - for (const event of messages) { - if (event.envelope && event.type === 'apprunning') { - appRunningBroadcasts.push({ ...event.envelope, data: event.data }); - } else if (event.type === 'evicted') { - // Evicted events lack per-event signatures because they are generated - // locally by nodeStatusMonitor, which makes non-deterministic HTTP - // probe decisions about whether a remote node is alive. The - // isSyncRequested check above ensures only solicited responses are - // processed, but a compromised confirmed peer we sync from could still - // include fake evictions. Impact is limited: only affects this node's - // view and self-heals on the next apprunning broadcast (≤60 min). - // - // The root cause is nodeStatusMonitor itself — it will be replaced by - // a peer quorum approach where eviction is determined by consensus of - // signed "peer unreachable" events (3 missed pongs on the WebSocket - // layer). Once that lands, evicted events will carry verifiable - // signatures and this path will verify them like all other event types. - evictedEvents.push(event); - } else if (event.envelope) { - otherBroadcasts.push(event); + // A sync response is processed a slice at a time. Verifying and storing the + // whole response at once holds the events, their verification copies and the + // database encoding of every location update in memory together, which is + // what made a single response cost hundreds of megabytes that were never + // returned to the OS. + // Evictions are applied ahead of the other state events, as they were + // before this response was processed in slices. + const evictions = []; + const stateEvents = []; + // Which apps a node still runs is only known from its newest broadcast in + // the whole response, so pruning cannot be decided from inside a slice. Only + // verified broadcasts count - pruning deletes rows, and an unsigned event + // must never be able to do that. + const newestByIp = new Map(); + let locationWriteFailed = false; + + await serviceHelper.processInSlices(messages, SYNC_EVENTS_PER_SLICE, async (slice) => { + const appRunningBroadcasts = []; + const otherBroadcasts = []; + const evictedEvents = []; + for (const event of slice) { + if (event.envelope && event.type === 'apprunning') { + appRunningBroadcasts.push({ ...event.envelope, data: event.data }); + } else if (event.type === 'evicted') { + // Evicted events lack per-event signatures because they are generated + // locally by nodeStatusMonitor, which makes non-deterministic HTTP + // probe decisions about whether a remote node is alive. The + // isSyncResponseWanted check above ensures only solicited responses are + // processed, but a compromised confirmed peer we sync from could still + // include fake evictions. Impact is limited: only affects this node's + // view and self-heals on the next apprunning broadcast (≤60 min). + // + // The root cause is nodeStatusMonitor itself — it will be replaced by + // a peer quorum approach where eviction is determined by consensus of + // signed "peer unreachable" events (3 missed pongs on the WebSocket + // layer). Once that lands, evicted events will carry verifiable + // signatures and this path will verify them like all other event types. + evictedEvents.push(event); + } else if (event.envelope) { + otherBroadcasts.push(event); + } } - } - const verifiedAppRunning = await batchVerifyBroadcasts(appRunningBroadcasts, 'handleAppRunningSyncResponse'); + const verifiedAppRunning = await batchVerifyBroadcasts(appRunningBroadcasts, 'handleAppRunningSyncResponse'); - const otherToVerify = otherBroadcasts.map((e) => ({ ...e.envelope, data: e.data })); - const verifiedOther = await batchVerifyBroadcasts(otherToVerify, 'handleAppRunningSyncResponse'); - const verifiedOtherSet = new Set(verifiedOther); - const otherEvents = [...evictedEvents]; - for (let i = 0; i < otherBroadcasts.length; i++) { - if (verifiedOtherSet.has(otherToVerify[i])) { - otherEvents.push(otherBroadcasts[i]); + const otherToVerify = otherBroadcasts.map((e) => ({ ...e.envelope, data: e.data })); + const verifiedOther = await batchVerifyBroadcasts(otherToVerify, 'handleAppRunningSyncResponse'); + const verifiedOtherSet = new Set(verifiedOther); + evictions.push(...evictedEvents); + for (let i = 0; i < otherBroadcasts.length; i++) { + if (verifiedOtherSet.has(otherToVerify[i])) { + stateEvents.push(otherBroadcasts[i]); + } } - } - if (verifiedAppRunning.length > 0) { - const { stored } = await messageStore.storeBatchAppRunningMessages(verifiedAppRunning); - log.info(`handleAppRunningSyncResponse - Stored ${stored} of ${verifiedAppRunning.length} verified apprunning events`); - fluxEventBus.publish('sync:chunkVerified', { syncType: 'apprunning', peer: peerKey, verified: verifiedAppRunning.length, stored }); + for (const broadcast of verifiedAppRunning) { + const { data } = broadcast; + if (!data || data.version !== 2 || !Array.isArray(data.apps) || !data.apps.length) continue; + // Skipped for the same reason messageStore skips it when it builds this + // map itself: an expired broadcast is not evidence of what an IP is + // running now. The prune it feeds only deletes rows at or below the + // broadcast's own timestamp, so an expired one could only ever take + // already-expired rows - but that is a bound to be read out of another + // file, and two builders of one input should not need reconciling. + if (data.broadcastedAt + RUNNING_EXPIRY_MS < Date.now()) continue; + const seen = newestByIp.get(data.ip); + if (!seen || data.broadcastedAt > seen.broadcastedAt) { + newestByIp.set(data.ip, { names: data.apps.map((a) => a.name), broadcastedAt: data.broadcastedAt }); + } + } + + if (verifiedAppRunning.length > 0) { + const { stored, writeFailed } = await messageStore.storeBatchAppRunningMessages(verifiedAppRunning); + if (writeFailed) locationWriteFailed = true; + log.info(`handleAppRunningSyncResponse - Stored ${stored} of ${verifiedAppRunning.length} verified apprunning events`); + fluxEventBus.publish('sync:chunkVerified', { syncType: 'apprunning', peer: peerKey, verified: verifiedAppRunning.length, stored }); + } + }); + + if (locationWriteFailed) { + log.warn('handleAppRunningSyncResponse - skipping location pruning, a location write failed'); + } else { + await messageStore.pruneAppRunningLocations(newestByIp); } + + // Applied after every slice, never inside one. An eviction clears a node's + // locations outright, so a slice storing that node's apprunning events + // afterwards would put them straight back - and evictions carry no + // broadcastedAt, so the sender's timestamp sort puts them in the earliest + // slice every time. const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); - for (const event of otherEvents) { + for (const event of [...evictions, ...stateEvents]) { if (event.type === 'sigterm') { await messageStore.storeAppStateEvent(event.type, { message: event.data, envelope: event.envelope }); const newExpireAt = new Date(event.data.broadcastedAt + SIGTERM_EXPIRY_MS); @@ -213,8 +297,9 @@ async function handleAppRunningSyncResponse(message, peerKey) { await messageStore.storeAppStateEvent(event.type, { message: event.data, envelope: event.envelope }); } } + if (done) { - appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apprunning'); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apprunning', peerKey); log.info('handleAppRunningSyncResponse - Sync complete'); } } catch (error) { @@ -222,21 +307,29 @@ async function handleAppRunningSyncResponse(message, peerKey) { } } -async function handleAppInstallingSyncResponse(message, peerKey) { +async function handleAppInstallingSyncResponse(message, peerSocket) { try { - if (!peerManager.isSyncRequested(peerKey)) return; + if (!peerManager.isSyncResponseWanted(peerSocket)) return; + const peerKey = peerSocket.key; if (!message.data || message.data.type !== 'fluxappinstallingsync') return; - const { messages, done } = message.data; + const { messages, done, refused } = message.data; + // A refusal is an answer and not a completion - see handleAppRunningSyncResponse. + if (refused) { + log.info(`handleAppInstallingSyncResponse - ${peerKey} declined: its app state is not authoritative yet`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_REFUSED, 'appinstalling', peerKey); + return; + } if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleAppInstallingSyncResponse - Received ${messages.length} broadcasts from ${peerKey} (done: ${!!done})`); - const verified = await batchVerifyBroadcasts(messages, 'handleAppInstallingSyncResponse'); - if (verified.length > 0) { + await serviceHelper.processInSlices(messages, SYNC_EVENTS_PER_SLICE, async (slice) => { + const verified = await batchVerifyBroadcasts(slice, 'handleAppInstallingSyncResponse'); + if (verified.length === 0) return; const { stored } = await messageStore.storeBatchAppInstallingMessages(verified); log.info(`handleAppInstallingSyncResponse - Stored ${stored} of ${verified.length} verified broadcasts`); fluxEventBus.publish('sync:chunkVerified', { syncType: 'appinstalling', peer: peerKey, verified: verified.length, stored }); - } + }); if (done) { - appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'appinstalling'); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'appinstalling', peerKey); log.info('handleAppInstallingSyncResponse - Sync complete'); } } catch (error) { @@ -244,21 +337,29 @@ async function handleAppInstallingSyncResponse(message, peerKey) { } } -async function handleAppInstallingErrorsSyncResponse(message, peerKey) { +async function handleAppInstallingErrorsSyncResponse(message, peerSocket) { try { - if (!peerManager.isSyncRequested(peerKey)) return; + if (!peerManager.isSyncResponseWanted(peerSocket)) return; + const peerKey = peerSocket.key; if (!message.data || message.data.type !== 'fluxappinstallingerrorssync') return; - const { messages, done } = message.data; + const { messages, done, refused } = message.data; + // A refusal is an answer and not a completion - see handleAppRunningSyncResponse. + if (refused) { + log.info(`handleAppInstallingErrorsSyncResponse - ${peerKey} declined: its app state is not authoritative yet`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_REFUSED, 'apperrors', peerKey); + return; + } if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleAppInstallingErrorsSyncResponse - Received ${messages.length} broadcasts from ${peerKey} (done: ${!!done})`); - const verified = await batchVerifyBroadcasts(messages, 'handleAppInstallingErrorsSyncResponse'); - if (verified.length > 0) { + await serviceHelper.processInSlices(messages, SYNC_EVENTS_PER_SLICE, async (slice) => { + const verified = await batchVerifyBroadcasts(slice, 'handleAppInstallingErrorsSyncResponse'); + if (verified.length === 0) return; const { stored } = await messageStore.storeBatchAppInstallingErrorMessages(verified); log.info(`handleAppInstallingErrorsSyncResponse - Stored ${stored} of ${verified.length} verified broadcasts`); fluxEventBus.publish('sync:chunkVerified', { syncType: 'apperrors', peer: peerKey, verified: verified.length, stored }); - } + }); if (done) { - appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apperrors'); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apperrors', peerKey); log.info('handleAppInstallingErrorsSyncResponse - Sync complete'); } } catch (error) { @@ -339,7 +440,13 @@ async function handleAppInstallingMessage(message, fromIP, port) { try { const rebroadcastToPeers = await messageStore.storeAppInstallingMessage(message.data); if (rebroadcastToPeers === true) { - fluxEventBus.publish('network:appinstalling', { ip: message.data.ip, name: message.data.name }); + // Version 2 withdraws the sender's claim rather than recording one and + // arrives through this handler too, so the event names which it was. + fluxEventBus.publish('network:appinstalling', { + ip: message.data.ip, + name: message.data.name, + withdrawn: message.data.withdrawn === true, + }); } messageStore.storeSignedAppInstallingBroadcast(message); const currentTimeStamp = Date.now(); @@ -509,7 +616,6 @@ async function handleNodeSigtermMessage(message, fromIP, port) { * @param {import('./utils/FluxPeerSocket').FluxPeerSocket} peerSocket FluxPeerSocket instance. */ async function dispatchFluxMessage(msgObj, peerSocket) { - const isOutbound = peerSocket.direction === DIRECTION.OUTBOUND; const codes = peerSocket.closeCodes; const { pubKey, timestamp, signature, version, data, @@ -638,49 +744,125 @@ async function dispatchFluxMessage(msgObj, peerSocket) { const syncChunkQueues = new Map(); -async function processSyncChunk(msgObj, peerKey) { - const result = await fluxCommunicationUtils.verifyFluxBroadcast(msgObj); - if (result !== fluxCommunicationUtils.VerifyResult.OK) { - log.warn(`Sync response from ${peerKey} failed envelope verification: ${result}`); - return; - } - +// Verified by dispatchSyncResponse before the chunk was queued, because whether +// a peer signed what it sent is a statement about the peer and the deadline +// waiting on it needs the answer at arrival, not at the back of a queue. +async function processSyncChunk(msgObj, peerSocket) { const { type } = msgObj.data; switch (type) { case 'fluxapptempsync': - await handleTempSyncResponse(msgObj, peerKey); + await handleTempSyncResponse(msgObj, peerSocket); break; case 'fluxapprunningsync': - await handleAppRunningSyncResponse(msgObj, peerKey); + await handleAppRunningSyncResponse(msgObj, peerSocket); break; case 'fluxappinstallingsync': - await handleAppInstallingSyncResponse(msgObj, peerKey); + await handleAppInstallingSyncResponse(msgObj, peerSocket); break; case 'fluxappinstallingerrorssync': - await handleAppInstallingErrorsSyncResponse(msgObj, peerKey); + await handleAppInstallingErrorsSyncResponse(msgObj, peerSocket); break; default: log.warn(`Unknown sync response type: ${type}`); } } +/** + * The envelope check, as a verdict that never rejects. + * + * A chunk's verdict is read twice - once by the arrival that queued it and once + * by whichever arrival is draining - so a rejection would surface in two places + * and one of them is holding a queue for a peer it does not own. A failure to + * decide is not a pass: it answers null, which is not OK, and the stream ends + * on it like any other envelope this node cannot stand behind. + * @param {object} msgObj + * @returns {Promise} + */ +async function verifySyncEnvelope(msgObj) { + try { + return await fluxCommunicationUtils.verifyFluxBroadcast(msgObj); + } catch (error) { + log.error(error); + return null; + } +} + async function dispatchSyncResponse(msgObj, peerSocket) { try { const peerKey = peerSocket.key; - if (!peerManager.isSyncRequested(peerKey)) return; - + if (!peerManager.isSyncResponseWanted(peerSocket)) return; + + // THE QUEUE IS THE ORDER, so nothing that can reorder may sit in front of + // it. Chunks carry meaning by position: the sender sorts by timestamp and + // an eviction has none, so evictions land in the FIRST chunk and clear a + // node's locations outright - a later chunk processed ahead of them has its + // rows deleted by an eviction that came before them. `done` is positional + // too, and a stream marked finished early loses whatever was still coming. + // + // So the chunk takes its place here, in the same synchronous step as its + // arrival, and the envelope check is STARTED rather than waited for. Four + // chunks arriving together are queued in arrival order and their checks + // race each other with no say in it. if (!syncChunkQueues.has(peerKey)) { syncChunkQueues.set(peerKey, { queue: [], processing: false }); } const state = syncChunkQueues.get(peerKey); - state.queue.push(msgObj); - - if (state.processing) return; - state.processing = true; + const chunk = { msgObj, verdict: verifySyncEnvelope(msgObj) }; + state.queue.push(chunk); + // Decided here for the same reason: read after an await, two arrivals both + // find a queue nobody is draining and both start draining it. + const drainer = !state.processing; + if (drainer) state.processing = true; + + // THE PEER SPOKE, AND IT REALLY WAS THE PEER. Two different questions used + // to be split at the wrong seam: arrival on one side, everything else on + // the other. The seam that matters is whose statement it is. + // + // Whether a peer signed what it sent is about the PEER, and it is a + // signature check. Storing two thousand messages, or re-verifying every + // pending registration, is about US. So the envelope is answered here, as + // soon as the check comes back and whatever the queue is doing, and the + // payload work stays in the drain below. + // + // Announced at arrival rather than after that work, because a peer that + // answered all four requests correctly and instantly went unheard for as + // long as WE took on the first of them - and was recorded as having said + // nothing and set aside, with the three answers queued behind it discarded + // on the way out. + // + // And unverifiable bytes are not the peer speaking. Credited as progress + // they renewed the deadline that exists to take the slot back, so a peer + // streaming rubbish inside every stall window held one of the answers this + // node needs for the whole attempt while never answering at all. + const verdict = await chunk.verdict; + if (verdict === fluxCommunicationUtils.VerifyResult.OK && !msgObj?.data?.refused) { + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_PROGRESS, peerKey); + } + + if (!drainer) return; while (state.queue.length > 0) { - const chunk = state.queue.shift(); - await processSyncChunk(chunk, peerKey); + const next = state.queue.shift(); + const nextVerdict = await next.verdict; + // A HOLE IS NOT A SURVEY. Stepping over the chunk and carrying on left + // this node counting a peer as having surveyed the network out of an + // answer it knows part of is missing - and it cannot know which part, + // because the chunk it could not attribute is the one it cannot read. + // The stream ends the way every other failure in this design ends: the + // request is over, and another peer is asked. + if (nextVerdict !== fluxCommunicationUtils.VerifyResult.OK) { + log.warn(`Sync response from ${peerKey} failed envelope verification: ${nextVerdict}, ending its request`); + syncChunkQueues.delete(peerKey); + // Only about the connection this drain serves. A backlog left by a + // connection that has since been replaced still drains, and ending a + // request on its behalf would close the one belonging to the peer that + // dialled back in - which has answered nothing wrong. + if (peerManager.isSyncResponseWanted(peerSocket)) { + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_UNVERIFIED, peerKey); + } + return; + } + await processSyncChunk(next.msgObj, peerSocket); } state.processing = false; @@ -795,6 +977,35 @@ function connectedPeersInfo(req, res) { return res ? res.json(message) : message; } +/** + * How many peers answered our most recent ping, out of how many we hold. + * + * The freshest liveness signal this node has about the network, and the one that + * tells it whether IT is the thing that has gone quiet. Peer COUNT alone cannot: a + * socket survives wsMaxMissedPongs rounds before it is dropped, so a node that has + * just been cut off keeps a full peer list for ~45s and reads as perfectly healthy + * throughout - after the point a decision gated on two 30s monitor passes could + * already have been made. missedPongs moves on the first missed round instead + * (~15s), which lands before that. + * + * Reported as a ratio rather than a bare count because the useful question is + * proportional: one silent peer among many is that peer's problem, while all of + * them going quiet at once is this node's. An absolute floor would also be a fleet + * size in disguise - a node configured with two peers can never reach a threshold + * written for a node with twelve. + * + * @returns {{responding: number, total: number}} Peers with no missed pong, and all peers + */ +function peerResponsiveness() { + let responding = 0; + let total = 0; + for (const peer of peerManager.allValues()) { + total += 1; + if (peer.missedPongs === 0) responding += 1; + } + return { responding, total }; +} + /** * To keep connections alive by pinging all outgoing and incoming peers. */ @@ -817,7 +1028,7 @@ async function removePeer(req, res) { let { ip } = req.params; ip = ip || req.query.ip; - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const message = messageHelper.errUnauthorizedMessage(); @@ -861,7 +1072,7 @@ async function removeIncomingPeer(req, res) { let { ip } = req.params; ip = ip || req.query.ip; - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const message = messageHelper.errUnauthorizedMessage(); @@ -947,13 +1158,48 @@ async function initiateAndHandleConnection(connection, source = PEER_SOURCE.RAND const key = `${ip}:${port}`; if (peerManager.has(key) || peerManager.isPending(key)) return; peerManager.markPending(key); - if (!myPort) { - const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); - if (!localSocketAddr) { - peerManager.clearPending(key); - return; - } - myPort = extractPort(localSocketAddr); + + // This node's own address, asked of the peer manager rather than of + // benchmark. That is where the fact lives - fluxNetworkHelper pushes every + // refresh into it from the one place the node learns what it is - and the + // peer manager already answers this exact question for the sync draw. + // Asking benchmark is an uncached RPC (executeCall), and this runs on every + // dial: discovery's deterministic loop, the reconnect queue, the random + // draw, the manual add and /flux/addpeer. It also made all five hard + // dependent on benchd answering, which only discovery already was. + // + // Fresh enough by construction: fluxDiscovery refreshes it once per cycle + // before it dials anything, and so do the availability checker and the + // address-change handler. That is a bound the form this replaces did not + // have - it read the port once per process and never again, so after an + // address change a node announced a stale one for as long as it ran. + let localSocketAddr = peerManager.getOwnSocketAddress?.() || null; + + if (!localSocketAddr) { + // Never been told, which a dial arriving before the first refresh + // genuinely is. Ask once - the answer fills the peer manager's copy on its + // way through, so this costs one call rather than one per dial. + localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); + } + + if (!localSocketAddr) { + peerManager.clearPending(key); + return; + } + myPort = extractPort(localSocketAddr); + + // Never ourselves, and refused HERE rather than by each caller. fluxDiscovery + // filters its own address before dialling, but it is one of four ways in - + // manual, deterministic, reconnect and random all arrive through this + // function, and the reconnect queue in particular re-dials whatever it holds + // without asking whose address it is. A self-connection is not merely a + // wasted socket: it occupies a peer slot, is offered back as a peer to + // gossip and to sync from, and answers every question with what this node + // already knows. + if (socketAddressesMatch(key, localSocketAddr)) { + log.warn(`initiateAndHandleConnection - refusing to connect to ourselves at ${key} (source ${source})`); + peerManager.clearPending(key); + return; } const options = { handshakeTimeout: config.fluxapps.wsHandshakeTimeoutMs ?? 10000, @@ -961,7 +1207,11 @@ async function initiateAndHandleConnection(connection, source = PEER_SOURCE.RAND zlibDeflateOptions: { // See zlib defaults. chunkSize: 1024, - memLevel: 9, + // No-context-takeover resets the stream after every message, so the + // window only ever matches within one. Gossip messages are a few KB + // and compress to the same bytes on an 8KB window as on a 32KB one, + // for a third of the memory - and a context is held per peer socket. + memLevel: 8, level: 9, }, zlibInflateOptions: { @@ -970,8 +1220,10 @@ async function initiateAndHandleConnection(connection, source = PEER_SOURCE.RAND // Other options settable: clientNoContextTakeover: true, // Defaults to negotiated value. serverNoContextTakeover: true, // Defaults to negotiated value. - serverMaxWindowBits: 15, // Defaults to negotiated value. - clientMaxWindowBits: 15, // Defaults to negotiated value. + // This socket only ever talks to another node, so both directions size + // down; a peer on an older build negotiates back up to 15. + serverMaxWindowBits: 13, + clientMaxWindowBits: 13, // Below options specified as default values. concurrencyLimit: 2, // Limits zlib concurrency for perf. threshold: 128, // Size (in bytes) below which messages @@ -1099,8 +1351,8 @@ async function addPeer(req, res) { ip = ip || req.query.ip; const authorized = await verificationHelper.verifyPrivilege( - 'adminandfluxteam', - req, + Privilege.NODE_OPERATOR_OR_FLUX_TEAM, + authOf(req), ); if (authorized !== true) { @@ -1204,12 +1456,16 @@ async function addOutgoingPeer(req, res) { function startDiscovery() { if (discoveryRunning) return; discoveryRunning = true; - fluxDiscovery(); + // Driven by the node list arriving rather than by a retry that happens to + // land after it. A peer holds one socket in either direction, so peers that + // dial us while we are waiting take the very sockets we would have dialled + // them on, and a late first pass then has nothing left to connect to. + networkStateService.onReady(fluxDiscovery); } async function startDiscoveryApi(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('fluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); if (authorized !== true) { return res.json(messageHelper.errUnauthorizedMessage()); } @@ -1241,6 +1497,13 @@ async function fluxDiscovery() { throw new Error('Flux IP not detected. Flux discovery is awaiting.'); } + // An unknown node list and an empty one are the same value below, and acting + // on the second when it is really the first sizes both deterministic loops + // to zero - the node then connects to nobody and reports no error. + if (!networkStateService.isReady()) { + throw new Error('Network state not yet known. Flux discovery is awaiting.'); + } + const sortedNodeList = await fluxCommunicationUtils.deterministicFluxList({ sort: true, addressOnly: true, @@ -1394,12 +1657,29 @@ async function fluxDiscovery() { function initializeDiscovery() { nodeConfirmationService.onConfirmationChange((confirmed) => { - if (confirmed) { - peerManager.allowConnections(); - } else { + if (!confirmed) { log.info('fluxDiscovery - Confirmation lost, disconnecting all peers'); peerManager.disconnectAll(); + return; } + + // Confirmed is not the same as ready to peer. Every message an inbound peer + // sends is checked against the node list, so a peer that arrives before the + // list does is refused however legitimate it is - there is nothing to + // validate it against. The two facts come from different calls to the same + // daemon, one carrying a single record and one carrying every node, so the + // list lands well after the confirmation and that gap is the whole of the + // window peers were being turned away in. + // + // Only the first open waits: once the list is here isReady() is true and + // this runs inline, so regaining confirmation reconnects immediately. + networkStateService.onReady(() => { + // The wait is not instant, and confirmation can be lost inside it. Without + // this the callback would re-open the door straight after disconnectAll(). + if (!nodeConfirmationService.isConfirmed()) return; + + peerManager.allowConnections(); + }); }); } @@ -1503,7 +1783,7 @@ function getUnstableNodes(req, res) { * @param {object} res Response. */ async function getPeerHistory(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { return res.json(messageHelper.errUnauthorizedMessage()); } @@ -1590,6 +1870,7 @@ module.exports = { removePeer, removeIncomingPeer, connectedPeersInfo, + peerResponsiveness, keepConnectionsAlive, fluxDiscovery, startDiscovery, @@ -1599,6 +1880,11 @@ module.exports = { addPeer, logSocketsEvery, handleAppRunningMessage, + handleAppInstallingMessage, + handleTempSyncResponse, + handleAppRunningSyncResponse, + handleAppInstallingSyncResponse, + handleAppInstallingErrorsSyncResponse, handleIPChangedMessage, handleAppRemovedMessage, handleNodeSigtermMessage, diff --git a/ZelBack/src/services/fluxCommunicationMessagesSender.js b/ZelBack/src/services/fluxCommunicationMessagesSender.js index 5e54fd883a..29d997d39a 100644 --- a/ZelBack/src/services/fluxCommunicationMessagesSender.js +++ b/ZelBack/src/services/fluxCommunicationMessagesSender.js @@ -9,6 +9,8 @@ const { peerManager } = require('./utils/peerState'); const cacheManager = require('./utils/cacheManager').default; const { serialiseAndSignFluxBroadcast, getFluxMessageSignature } = require('./utils/fluxBroadcastHelper'); const fluxEventBus = require('./utils/fluxEventBus'); +const globalState = require('./utils/globalState'); +const { Privilege, authOf } = require('./utils/privileges'); const myMessageCache = cacheManager.tempMessageCache; @@ -22,6 +24,7 @@ const myMessageCache = cacheManager.tempMessageCache; async function sendSignedMessage(message, peer, options = {}) { try { const messageSigned = await serialiseAndSignFluxBroadcast(message); + if (!messageSigned) return; if (options.awaitDrain) { await peer.sendAsync(messageSigned); } else { @@ -135,6 +138,7 @@ async function relay(data, excludeKey) { */ async function broadcastMessageToAll(dataToBroadcast) { const serialisedData = await serialiseAndSignFluxBroadcast(dataToBroadcast); + if (!serialisedData) return null; await relay(serialisedData); return JSON.parse(serialisedData); } @@ -145,6 +149,7 @@ async function broadcastMessageToAll(dataToBroadcast) { */ async function broadcastMessageToRandomOutgoing(dataToBroadcast) { const serialisedData = await serialiseAndSignFluxBroadcast(dataToBroadcast); + if (!serialisedData) return; const peer = peerManager.getRandomPeer('outbound'); if (peer) peer.send(serialisedData); } @@ -155,6 +160,7 @@ async function broadcastMessageToRandomOutgoing(dataToBroadcast) { */ async function broadcastMessageToRandomIncoming(dataToBroadcast) { const serialisedData = await serialiseAndSignFluxBroadcast(dataToBroadcast); + if (!serialisedData) return; const peer = peerManager.getRandomPeer('inbound'); if (peer) peer.send(serialisedData); } @@ -203,7 +209,7 @@ async function broadcastMessageFromUser(req, res) { if (data === undefined || data === null) { throw new Error('No message to broadcast attached.'); } - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let message; @@ -241,7 +247,7 @@ async function broadcastMessageFromUserPost(req, res) { throw new Error('No message to broadcast attached.'); } const processedBody = serviceHelper.ensureObject(body); - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let message; @@ -288,6 +294,24 @@ async function broadcastTemporaryAppMessage(message) { async function respondWithTempMessages(peer, sinceTimestamp = 0) { try { + // THE SAME INCOMPLETENESS, ON A DIFFERENT COLLECTION. A node still catching + // up holds SOME of the network's pending registrations and cannot say which + // ones it is missing, so handing them over is exactly the partial answer the + // three app-state types refuse to give: the asker has no way to tell a short + // set from the whole one, and every message in it verifying says nothing + // about the ones that are absent. + // + // Refusing costs a booting fleet nothing it was going to get: a node + // refuses all four streams or none, so a peer that would have been credited + // for this one was never going to be credited for the other three either, + // and readiness still arrives by the block fallback exactly as before. + if (!globalState.appStateAuthoritative) { + log.info(`respondWithTempMessages - Refusing ${peer.key}: this node's app state is not authoritative yet`); + await sendSignedMessage({ type: 'fluxapptempsync', version: 1, messages: [], done: true, refused: true }, peer, { awaitDrain: true }); + fluxEventBus.publish('sync:refused', { syncType: 'fluxapptempsync', peer: peer.key }); + return; + } + const globalAppsTempMessages = config.database.appsglobal.collections.appsTemporaryMessages; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); @@ -325,6 +349,23 @@ async function respondWithTempMessages(peer, sinceTimestamp = 0) { async function streamBatchedSync(peer, { sinceTimestamp, collectionName, validityMs, query, projection, messageType, label }) { try { + // A NODE THAT DOES NOT KNOW YET SAYS SO, rather than sending what it + // happens to hold. An empty response and a complete one are the same three + // fields on the wire, so the asker counted a booting node's nothing as one + // of the three surveys it needs - and three booting nodes could tell it the + // network was empty without one of them saying anything false. + // + // Refusing is cheaper than answering badly and it is decided here, at the + // moment of asking, so there is nothing cached to go stale. Older nodes + // send no such field, which reads as a refusal of nothing and leaves them + // behaving exactly as they do now. + if (!globalState.appStateAuthoritative) { + log.info(`${label} - Refusing ${peer.key}: this node's app state is not authoritative yet`); + await sendSignedMessage({ type: messageType, messages: [], done: true, refused: true }, peer, { awaitDrain: true }); + fluxEventBus.publish('sync:refused', { syncType: messageType, peer: peer.key }); + return; + } + const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); diff --git a/ZelBack/src/services/fluxCommunicationUtils.js b/ZelBack/src/services/fluxCommunicationUtils.js index 8c32f9ed8d..14fa0f975d 100644 --- a/ZelBack/src/services/fluxCommunicationUtils.js +++ b/ZelBack/src/services/fluxCommunicationUtils.js @@ -24,6 +24,14 @@ async function deterministicFluxList(options = {}) { const sort = options.sort || false; const addressOnly = options.addressOnly || false; + // The node list takes a moment to arrive, and every accessor here answers an + // unknown state and a genuinely empty one identically - an empty list, a zero, + // a false. Waiting is the safe default, so the twelve callers that cannot tell + // those apart get the right answer without each having to remember to ask. + // Anything on a repeating schedule must NOT reach here unready: it checks + // networkStateService.isReady() and re-arms, the way it already does for the + // daemon. See fluxDiscovery, checkDeterministicNodesCollisions, + // monitorNodeStatus. await networkStateService.waitStarted(); if (!filter) { @@ -49,7 +57,6 @@ async function deterministicFluxList(options = {}) { async function getNodeCount() { await networkStateService.waitStarted(); - const count = networkStateService.nodeCount(); return count; @@ -62,7 +69,6 @@ async function getNodeCount() { */ async function getFluxnodeFromFluxList(socketAddress) { await networkStateService.waitStarted(); - const node = await networkStateService.getFluxnodeBySocketAddress(socketAddress); return node; @@ -75,7 +81,6 @@ async function getFluxnodeFromFluxList(socketAddress) { */ async function socketAddressInFluxList(socketAddress) { await networkStateService.waitStarted(); - const found = await networkStateService.socketAddressInNetworkState(socketAddress); return found; diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 2e5e9b36d8..6ed6780e53 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -25,11 +25,25 @@ const cacheManager = require('./utils/cacheManager').default; const networkStateService = require('./networkStateService'); const fluxEventBus = require('./utils/fluxEventBus'); const { - normalizeSocketAddress, extractIp, extractPort, socketAddressesMatch, parseSocketAddress, + normalizeSocketAddress, extractIp, extractPort, socketAddressesMatch, parseSocketAddress, ipsMatch, } = require('./utils/socketAddressUtils'); const isArcane = Boolean(process.env.FLUXOS_PATH); +// Fired once, with the apps that survived an address change, after the node's +// public IP has moved. serviceManager wires it to appReconciler.requestRestartOf +// (mirrors appUninstaller.setOnComponentRemoved). +// +// A seam rather than a require, and not for tidiness: appUninstaller requires +// THIS module and appReconciler requires appUninstaller, so reaching upward from +// here for either closes a cycle. Twenty-odd app-layer modules import this one - +// it sits underneath them and stays there. What it knows is that the address +// moved and which apps it kept; what restarting one involves is not its business. +let onAddressChanged = null; +function setOnAddressChanged(callback) { + onAddressChanged = callback; +} + let dosState = 0; // we can start at bigger number later let dosMessage = null; @@ -40,6 +54,29 @@ let dosMessage = null; let stickyDosState = 0; let stickyDosMessage = null; +// Who may hold this node back from placement. An owner is an IDENTITY, not a +// message: the reason is what an operator reads, and the owner is what a release +// is checked against. Adding a feature that holds placement means adding a value +// here, which is the point - an unknown owner is refused rather than accepted as +// a new one. +const PlacementHoldOwner = Object.freeze({ + RESIDENTIAL_DOS: 'residentialDos', +}); + +// Stops the node taking on NEW apps without declaring it unfit for the ones it +// already runs. DOS conflates those: isNodeDos() makes appSpawner refuse +// installs AND makes nodeStatusMonitor and appStartupManager delete every app on +// the box. A node that must stop growing but keep its customer volumes needs +// only the first, so this is a separate flag whose only consumer is the spawner. +// +// owner -> reason, rather than one slot. A single slot cannot express two +// owners: a second hold OVERWROTE the first, and whoever cleared next released +// both - so the node resumed taking apps for a condition that had not lifted. +// Checking ownership on a single slot only moves the failure, because the +// overwritten owner can then never clear and the node stays held forever. The +// node is held while any owner holds it. +const placementHolds = new Map(); + let storedFluxBenchAllowed = null; let ipChangeData = null; let dosTooManyIpChanges = false; @@ -47,6 +84,7 @@ let maxNumberOfIpChanges = 0; const myCache = cacheManager.ipCache; const { lruRateLimit } = require('./utils/rateLimit'); +const { Privilege, authOf } = require('./utils/privileges'); // This node's socket address (ip:port) from benchmark let localSocketAddress = null; @@ -82,9 +120,12 @@ async function isInterfaceUp(interfaceName) { } /** - * Gets the IP address assigned to a specific network interface. + * Gets the first routable IPv4 address assigned to a network interface. An + * interface can carry a private primary and a public secondary; stopping at + * the first non-internal address would answer for whichever the kernel lists + * first rather than for the interface. * @param {string} interfaceName - The name of the network interface - * @returns {string|null} The IPv4 address or null if not found + * @returns {string|null} The routable IPv4 address or null if none is bound */ function getInterfaceIp(interfaceName) { const interfaces = os.networkInterfaces(); @@ -92,7 +133,7 @@ function getInterfaceIp(interfaceName) { if (!iface) return null; for (const addr of iface) { - if (addr.family === 'IPv4' && !addr.internal) { + if (addr.family === 'IPv4' && !addr.internal && !serviceHelper.isNonRoutableAddress(addr.address)) { return addr.address; } } @@ -104,7 +145,9 @@ function getInterfaceIp(interfaceName) { * This is a strong indicator of a static IP (data center/VPS/dedicated server). * Uses the Linux routing table to find the default route interface, then checks * if that interface has a public IP assigned. - * @returns {Promise} True if a public IP is configured on the default route interface + * @returns {Promise} True if a public IP is configured on the + * default route interface, false if none is, null if the routing table could + * not be read - which is not the same answer as "there is none". */ async function hasPublicIpOnInterface() { try { @@ -157,7 +200,7 @@ async function hasPublicIpOnInterface() { const isUp = await isInterfaceUp(route.iface); if (isUp) { const ip = getInterfaceIp(route.iface); - if (ip && !serviceHelper.isNonRoutableAddress(ip)) { + if (ip) { log.info(`Public IP ${ip} found on default route interface ${route.iface}`); return true; } @@ -166,8 +209,13 @@ async function hasPublicIpOnInterface() { return false; } catch (error) { + // Null, not false. "There is no public address on any interface" is a fact + // about the node; "I could not read the routing table" is a fact about this + // process, and answering the second with the first asserts NAT on a node + // that may well hold a public address. The one caller that decides anything + // on this treats null as unknown. log.error(`Failed to check network interfaces via routing table: ${error.message}`); - return false; + return null; } } @@ -257,6 +305,73 @@ function isPortUPNPBanned(port) { return portBanned; } +/** + * The most a peer will hand back from a port it was asked to read. + * + * A DISCLOSURE bound, and only that. The port may be forwarded to a neighbour at + * the same public address, so what comes back can be a stranger's response, and + * nobody should be askable to shuttle a payload. Choose it on that question + * alone. + * + * It is NOT what makes the proof survive. The test server writes its secret into + * the first response header, so the thing the requester has to find sits at byte + * 67 of an answer that server authors in full - inside this prefix however large + * the rest of what a port says turns out to be. The two were entangled once: the + * token was last in the body, and any 48 bytes appearing before it silently + * turned every install on the network into "a neighbour holds this port". + */ +const MAX_ECHO_BYTES = 256; + +/** + * What a port answered, capped, for the requester to judge. + * + * The requester published a secret on its own test server and did NOT tell us + * what it is: we fetch whatever is on that port and hand it back verbatim, and + * the requester decides. That direction is the point. This check exists because + * a peer cannot tell the requester's application from a neighbour's at the same + * address - so a peer is not in a position to judge, and one that is old, + * broken or lying cannot manufacture a secret it was never given. + * + * Bounded by MAX_ECHO_BYTES, because this relays bytes read from a stranger's + * port and nobody can be asked to shuttle a payload. That bound cannot cost the + * requester its answer: the secret is in the first header of a reply the test + * server writes in full, so it is inside any prefix this returns. + * + * @param {string} ip - the requester's address + * @param {number} port - the port to read + * @param {object} options - { timeout } + * @returns {Promise} what it answered, or null if nothing did + */ +async function portAnswered(ip, port, options = {}) { + const timeout = options.timeout || 5_000; + + return new Promise((resolve) => { + const socket = new net.Socket(); + let received = ''; + let settled = false; + + const done = (answer) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + resolve(answer); + }; + + const timer = setTimeout(() => done(received || null), timeout); + + socket.connect(port, ip, () => { + socket.write(`GET / HTTP/1.1\r\nHost: ${ip}:${port}\r\nConnection: close\r\n\r\n`); + }); + socket.on('data', (chunk) => { + received += chunk.toString('utf8'); + if (received.length >= MAX_ECHO_BYTES) done(received.slice(0, MAX_ECHO_BYTES)); + }); + socket.on('end', () => done(received || null)); + socket.on('error', () => done(null)); + }); +} + /** * To perform a basic check if TCP port on an ip is open. I.e. that we receive a * SYN-ACK in response to a SYN. If connected, we send an RST and close the port. @@ -389,6 +504,87 @@ async function checkFluxAvailability(req, res) { * @param {object} res Response. * @returns {object} Message. */ +/** + * Whether a signed body - asked or answered - carries a signature from a + * Fluxnode on the deterministic list, over its own contents. + * + * Extracted rather than written twice. Two copies of a signature check is the + * one duplication that must not drift - whichever copy is corrected, the other + * keeps accepting what it always did, and nothing points at it. + * + * The body is verified as it arrived minus the signature itself, which is how + * the sender built the message it signed. + * + * `socketAddress` binds the signer to a place, and the two directions need + * different answers. For a request, "some listed Fluxnode signed this" is the + * whole question - any node on the list may ask. For an ANSWER it is not enough: + * we dialled one address, and what comes back has to be from the node that lives + * there rather than a signature made by, or relayed from, somewhere else. + * + * @param {object} processedBody The parsed body, carrying pubKey and signature + * @param {{socketAddress?: string}} [options] The address the signer must hold + * @returns {Promise} True when a listed Fluxnode signed this body + */ +async function verifySignedFluxnodeMessage(processedBody, options = {}) { + if (!processedBody || !processedBody.pubKey || !processedBody.signature) return false; + + const { pubKey, signature } = processedBody; + + const nodes = await fluxCommunicationUtils.deterministicFluxList({ filter: pubKey }); + if (!nodes.length) return false; + + const { socketAddress = null } = options; + if (socketAddress && !nodes.some((node) => socketAddressesMatch(node.ip, socketAddress))) return false; + + const dataToVerify = { ...processedBody }; + delete dataToVerify.signature; + + return verificationHelper.verifyMessage(JSON.stringify(dataToVerify), pubKey, signature) === true; +} + +/** + * The most ports one request may ask this node to test. + * + * An application is capped at maxComponents (10) components of ports (5) each in + * appValidator, so 50 is the largest honest ask. The bound exists because the + * ports are tested one after another, each for up to a full connect timeout. + */ +const MAX_TESTABLE_PORTS = 50; + +// Every installed application's ports plus the four node service ports the +// keep-alive caller adds - maxAppsPerNode x MAX_TESTABLE_PORTS + 4. A bound on +// how long one request can keep this node poking, and nothing tighter: the +// honest list really can be that long. +const NODE_SERVICE_PORTS_KEPT_ALIVE = 4; +const MAX_KEEPALIVE_PORTS = config.fluxapps.maxAppsPerNode * MAX_TESTABLE_PORTS + NODE_SERVICE_PORTS_KEPT_ALIVE; + +/** + * The address a peer endpoint acts on: the one the caller connected from, or the + * one it named when it holds the privilege to name one. + * + * Never an input otherwise. A caller that names the address chooses where this + * node connects, which makes the endpoint a probe aimed at anything the caller + * likes - the loopback and the RFC1918 side of its own router included. The + * honest caller never needed it: it is asking about ITS OWN ports, so the + * address it means is the one it is connecting from. That is the rule + * /flux/addpeer already applies, and every inbound peer is already identified by + * its socket address - a Fluxnode whose egress differed from its declared + * address could not hold a peer slot anywhere on the network, so nothing + * legitimate is lost by insisting on it. + * + * An IPv4 connection to a dual-stack listener arrives as ::ffff:a.b.c.d, and an + * address that does not match itself would refuse every honest caller. + * + * @param {object} req + * @param {string|undefined} namedAddress - what the body says, if anything + * @param {boolean} mayName - whether this caller may choose the address + * @returns {string} the address, or '' when there is none to act on + */ +function addressToProbe(req, namedAddress, mayName) { + const remoteIp = (req.socket.remoteAddress || '').replace(/^::ffff:/i, ''); + return mayName === true ? (namedAddress || remoteIp) : remoteIp; +} + async function checkAppAvailability(req, res) { let body = ''; req.on('data', (data) => { @@ -396,28 +592,66 @@ async function checkAppAvailability(req, res) { }); req.on('end', async () => { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + // The other way in. This endpoint's caller is a Fluxnode proving itself by + // signature; this is for a PERSON driving it by hand instead, and the only + // thing a person gets out of it is naming the address to probe below - + // asking whether the ports are open on the machine they happen to be + // sitting at answers nothing anyone wants. Skipping the signature and + // choosing the address are therefore one question, and it is asked once. + // + // Not the node operator. NODE_OPERATOR_OR_FLUX_TEAM reads node-local and is + // not: it is thousands of separate credentials, one per node, holding what + // is a Flux team diagnostic. An operator wanting to dial out of their own + // box has a shell on it, and their node still reaches this endpoint the way + // every node does - by signing. + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); const processedBody = serviceHelper.ensureObject(body); - const { - ip, ports, pubKey, signature, - } = processedBody; + const { ports } = processedBody; const ipPort = processedBody.port; // pubkey of the message has to be on the list - const nodes = await fluxCommunicationUtils.deterministicFluxList({ filter: pubKey }); - const dataToVerify = processedBody; - delete dataToVerify.signature; - const messageToVerify = JSON.stringify(dataToVerify); - const verified = verificationHelper.verifyMessage(messageToVerify, pubKey, signature); - if ((verified !== true || !nodes.length) && authorized !== true) { + const verified = await verifySignedFluxnodeMessage(processedBody); + if (!verified && authorized !== true) { throw new Error('Unable to verify request authenticity'); } + // The address to probe is NOT an input - see addressToProbe. Here the + // stakes are highest: the echo below hands back the first bytes of what + // answered, so a body-supplied address turns every Flux node into a fetch + // primitive aimed at anything a signed peer likes. The range guard does + // not help: portMin is 1 and portMax is 65535, and bannedPorts names this + // node's own services rather than a database's. + // + // Flux team may still name one, which is the whole of what the privilege + // above is for: see it. + const ip = addressToProbe(req, processedBody.ip, authorized); + + if (!ip) { + throw new Error('Unable to determine which address to test'); + } + + if (!Array.isArray(ports)) { + throw new Error('No ports to test'); + } + + // A valid application cannot hold more ports than the specification allows + // it: maxComponents (10) x ports per component (5) = 50, both in + // appValidator. Bounded at all because each port below costs up to a full + // connect timeout and they are tested in sequence. + if (ports.length > MAX_TESTABLE_PORTS) { + throw new Error(`Too many ports to test. Maximum of ${MAX_TESTABLE_PORTS} allowed.`); + } + const { fluxapps: { portMin: minPort, portMax: maxPort } } = config; + // A requester that wants proof asks for it. One that does not - an older + // node - gets exactly the check it always got. + const echo = processedBody.echo === true; + const answered = {}; + // eslint-disable-next-line no-restricted-syntax for (const port of ports) { const iBP = isPortBanned(+port); @@ -425,16 +659,32 @@ async function checkAppAvailability(req, res) { const withinRange = portNum >= minPort && portNum <= maxPort; if (withinRange && !iBP) { - // eslint-disable-next-line no-await-in-loop - const isOpen = await isPortOpen(ip, port); - if (!isOpen) { - throw new Error(`Flux Applications on ${ip}:${ipPort} are not available. Failed port: ${port}`); + if (echo) { + // Read rather than merely reached. The requester compares. + // eslint-disable-next-line no-await-in-loop + const answer = await portAnswered(ip, port); + if (answer === null) { + throw new Error(`Flux Applications on ${ip}:${ipPort} are not available. Failed port: ${port}`); + } + answered[port] = answer; + } else { + // eslint-disable-next-line no-await-in-loop + const isOpen = await isPortOpen(ip, port); + if (!isOpen) { + throw new Error(`Flux Applications on ${ip}:${ipPort} are not available. Failed port: ${port}`); + } } } else { log.error(`Flux App port ${port} is outside allowed range. minPort: ${minPort}, maxPort: ${maxPort}, isBanned: ${iBP}`); } } const successResponse = messageHelper.createSuccessMessage(`Flux Applications on ${ip}:${ipPort} are available.`); + // `answered` present at all is how the requester knows this peer READ the + // ports rather than merely reaching them. Absent means no proof is + // available from this peer, which is not the same as the ports being bad. + // Added as a field rather than through createSuccessMessage, whose second + // and third parameters are name and code. + if (echo) successResponse.data.answered = answered; res.json(successResponse); } catch (error) { const errorResponse = messageHelper.createErrorMessage( @@ -489,16 +739,33 @@ function tcpConnectAndDestroy(host, port, timeout) { * @param {object} res Response * @returns {Promise} */ +/** + * POST /flux/keepupnpportsopen - poke the caller's ports so its router keeps the + * UPnP mappings for them alive. + * + * The address poked is the one the caller connected from, never one it names - + * the rule /flux/checkappavailability applies, for the reason on addressToProbe. + * Naming one by hand is the same single privilege as skipping the signature: + * Flux team, not one every node operator holds. The API port stays the caller's + * to name, being a port on the address just bound. + * + * NO range or banned-port filter on the ports, deliberately, and unlike the + * availability endpoint beside this one. The caller sends its own service ports + * alongside its application ports - the API port minus one, minus five, plus one + * and plus two - and every one of those sits inside the banned 16100-16299 + * block. The filter that is right there would silently end the keep-alive here. + * + * @param {object} req + * @param {object} res + */ async function keepUPNPPortsOpen(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); const { body } = req; const processedBody = serviceHelper.ensureObject(body); - const { - ip, apiPort, ports, pubKey, timestamp, signature, - } = processedBody; + const { apiPort, ports, timestamp } = processedBody; const now = Math.floor(Date.now() / 1000); @@ -508,12 +775,12 @@ async function keepUPNPPortsOpen(req, res) { return; } - if (!ip || !apiPort || !pubKey || !signature) { + if (!apiPort) { res.status(422).end(); return; } - if (!Array.isArray(ports)) { + if (!Array.isArray(ports) || ports.length > MAX_KEEPALIVE_PORTS) { res.status(422).end(); return; } @@ -527,16 +794,18 @@ async function keepUPNPPortsOpen(req, res) { } // pubkey of the message has to be on the list - const nodes = await fluxCommunicationUtils.deterministicFluxList({ filter: pubKey }); - const dataToVerify = processedBody; - delete dataToVerify.signature; - const messageToVerify = JSON.stringify(dataToVerify); - const verified = verificationHelper.verifyMessage(messageToVerify, pubKey, signature); - if ((verified !== true || !nodes.length) && authorized !== true) { + const verified = await verifySignedFluxnodeMessage(processedBody); + if (!verified && authorized !== true) { res.status(401).end(); throw new Error('Unable to verify request authenticity'); } + const ip = addressToProbe(req, processedBody.ip, authorized); + if (!ip) { + res.status(422).end(); + return; + } + // make sure that we can reach the api port first. This is in case of nodes that // are able to receive communcation from another node, but because of routing issues, // can connect back the other way. This has a timeout of 3 seconds, whereas the other end @@ -575,6 +844,11 @@ async function keepUPNPPortsOpen(req, res) { */ function setLocalSocketAddress(value) { localSocketAddress = value ? normalizeSocketAddress(value) : null; + // Told here because this is the one place the node learns what it is. The + // peer manager needs it to keep this node out of its own peer draws - a node + // that syncs from itself learns nothing, and it spends one of very few + // attempts doing so. Optional because the unit suite stubs peerState. + peerManager.setOwnSocketAddress?.(localSocketAddress); } /** @@ -606,6 +880,7 @@ function getDosMessage() { */ function setStickyDosMessage(message) { stickyDosMessage = message; + publishEffectiveDosState(); } /** @@ -622,6 +897,30 @@ function getStickyDosMessage() { function clearStickyDosMessage() { stickyDosMessage = null; stickyDosState = 0; + publishEffectiveDosState(); +} + +/** + * Publish the DOS state a reader would actually see. + * + * isNodeDos() answers on `stickyDosMessage ? stickyDosState : dosState`, so the + * effective value moves when EITHER half of the sticky pair moves, and neither + * setter emitted anything. A consumer therefore had to poll /flux/info, and a + * poll cannot order a DOS against anything else: the installed-apps record + * outlives the removal it follows by ~20s, so two polls of two sources disagree + * about what happened first. On one event stream the ids settle it. + * + * Inert in production - fluxEventBus.publish returns immediately unless + * config.testEventStream is set, which it is only under the harness. This is + * not the 21-site refactor noted below getDosStateValue; that one is about the + * product's own scattered dosState mutations. + */ +function publishEffectiveDosState() { + const effectiveDosState = stickyDosMessage ? stickyDosState : dosState; + fluxEventBus.publish('dos:changed', { + dosState: effectiveDosState, + dosMessage: stickyDosMessage || dosMessage, + }); } /** @@ -630,6 +929,7 @@ function clearStickyDosMessage() { */ function setStickyDosStateValue(value) { stickyDosState = value; + publishEffectiveDosState(); } /** @@ -662,6 +962,53 @@ function isNodeDos() { return effectiveState >= 100; } +/** + * Hold this node back from new placements. Idempotent per owner. + * @param {string} owner A PlacementHoldOwner value. An unknown one throws: it is + * a caller that was never given an identity, and accepting it would create an + * owner nothing can ever release. + * @param {string} reason Logged and reported. + */ +function setPlacementHold(owner, reason) { + if (!Object.values(PlacementHoldOwner).includes(owner)) { + throw new Error(`setPlacementHold: unknown owner ${owner}`); + } + if (placementHolds.get(owner) === reason) return; + placementHolds.set(owner, reason); + log.info(`Placement hold set by ${owner}: ${reason}`); +} + +/** + * Release one owner's hold. Any other owner's hold stands, and the node stays + * held until every one of them has released - so a feature clearing its own + * condition can never speak for a condition it knows nothing about. + * @param {string} owner A PlacementHoldOwner value. + */ +function clearPlacementHold(owner) { + const reason = placementHolds.get(owner); + if (reason === undefined) return; + placementHolds.delete(owner); + log.info(`Placement hold cleared by ${owner} (was: ${reason})`); +} + +/** + * @returns {string|null} Why the node is held, or null when it is not. + */ +function getPlacementHold() { + if (!placementHolds.size) return null; + // Every reason, not an arbitrary one: the spawner logs this to say why the + // node is not installing, and naming one of two holds would send an operator + // to lift a condition that would not release the node. + return [...placementHolds.values()].join('; '); +} + +/** + * @returns {boolean} True when this node must not take on new apps. + */ +function isPlacementHeld() { + return placementHolds.size > 0; +} + /** * Get this node's socket address (ip:port). * @returns {Promise} Normalized socket address (always ip:port) or null. @@ -705,7 +1052,14 @@ async function getFluxNodePublicKey(privatekey) { const pubKey = privKeyToPubKey(privateKey, isCompressed); return pubKey; } catch (error) { - return error; + // Null, not the Error. An Error here is the worst of both: truthy, so a + // guard on the value passes; not a string, so nothing type-based notices; + // and `{}` once JSON.stringify reaches it - which is how a node with a + // briefly unavailable key went on broadcasting messages that every peer + // silently refused. Said out loud too, because it was silent at the point + // it happened and loud only at the far end. + log.error(`getFluxNodePublicKey - unable to derive this node's public key: ${error.message || error}`); + return null; } } @@ -722,7 +1076,10 @@ async function closeConnection(ip, port) { if (!peer || peer.direction !== DIRECTION.OUTBOUND) { return messageHelper.createWarningMessage(`Connection to ${ip}:${port} does not exists.`); } - peer.close(CLOSE_CODES.CLOSED_OUTBOUND, 'purposefully closed'); + // Evicted rather than closed: the caller asked for this peer to be gone, and + // until it leaves the map it still fills a slot no reconnect is dialled for + // and is still offered as a sync source. + peerManager.evict(key, CLOSE_CODES.CLOSED_OUTBOUND, 'purposefully closed'); log.info(`Connection to ${ip}:${port} closed with code ${CLOSE_CODES.CLOSED_OUTBOUND}`); return messageHelper.createSuccessMessage(`Outgoing connection to ${ip}:${port} closed`); } @@ -742,7 +1099,7 @@ async function closeIncomingConnection(ip, port) { if (!peer || peer.direction !== DIRECTION.INBOUND) { return messageHelper.createWarningMessage(`Connection from ${ip}:${port} does not exists.`); } - peer.close(CLOSE_CODES.CLOSED_INBOUND, 'purposefully closed'); + peerManager.evict(key, CLOSE_CODES.CLOSED_INBOUND, 'purposefully closed'); log.info(`Connection from ${ip}:${port} closed with code ${CLOSE_CODES.CLOSED_INBOUND}`); return messageHelper.createSuccessMessage(`Incoming connection to ${ip}:${port} closed`); } @@ -1022,8 +1379,7 @@ async function clockDrift(req, res) { * @param {object} res Response. */ function isCommunicationEstablished(req, res) { - const outboundCount = peerManager.outboundCount; - const inboundCount = peerManager.inboundCount; + const { outboundCount, inboundCount } = peerManager; let message; if (outboundCount < config.fluxapps.minOutgoing) { // easier to establish message = messageHelper.createErrorMessage(`Not enough outgoing connections established to Flux network. Minimum required ${config.fluxapps.minOutgoing} found ${outboundCount}`); @@ -1117,6 +1473,23 @@ async function adjustExternalIP(ip) { if (ip === userconfig.initial.ipaddress) { return; } + // Everything below needs to know which node this is: whose registration among + // the ones found at the new address is our own, which apps are ours to hand + // over, and what address the fluxipchanged broadcast is moving FROM. + // localSocketAddress is cleared whenever benchmark hiccups, and a comparison + // against nothing matches nothing - so acting here would read our own rows as + // strangers' and uninstall the apps they belong to. + // + // Return BEFORE the userconfig write, which is what makes this a deferral + // rather than a silent drop: the write is what marks the change handled, so + // leaving it unwritten leaves the change pending. checkMyFluxAvailability + // already refuses to run while the address is unknown, so nothing reaches here + // again until benchmark answers - and then this runs with the node knowing + // itself, exactly once, as designed. + if (!localSocketAddress) { + log.warn(`adjustExternalIP - own address unknown, deferring the change to ${ip} until benchmark answers`); + return; + } const oldUserConfigIp = userconfig.initial.ipaddress; log.info(`Adjusting External IP from ${userconfig.initial.ipaddress} to ${ip}`); const dataToWrite = `module.exports = { @@ -1156,13 +1529,15 @@ async function adjustExternalIP(ip) { // eslint-disable-next-line global-require const appUninstaller = require('./appLifecycle/appUninstaller'); // eslint-disable-next-line global-require - const appController = require('./appManagement/appController'); - // eslint-disable-next-line global-require const enterpriseHelper = require('./utils/enterpriseHelper'); let apps = await appQueryService.installedApps(); if (apps.status === 'success' && apps.data.length > 0) { apps = apps.data; let appsRemoved = 0; + // The apps still installed once the loop has removed the ones that cannot + // stay. Handed to whoever registered for an address change; nothing here + // knows what bringing them back involves. + const staying = []; // eslint-disable-next-line no-restricted-syntax for (const app of apps) { // Check if app requires static IP - if so, uninstall it since IP changed @@ -1193,7 +1568,21 @@ async function adjustExternalIP(ip) { // eslint-disable-next-line no-await-in-loop const runningAppList = await registryManager.appLocation(app.name); - const duplicateInstance = runningAppList.find((instance) => extractIp(instance.ip) === ip); + // An instance at this address means the ports are taken and this node + // cannot run the app: one instance per IP is enforced by the host port + // mapping, so a UPnP sibling on another port holds them just as surely + // as a node that owns the address alone. That is why the address is + // compared at IP granularity. + // + // The node's OWN registration is not that. It stores its own running-app + // row locally, at the address benchmark reports, so the row sitting at + // this address is most often itself - and removing on that is a node + // deleting an app that is exactly where it belongs, then telling the + // network it is gone. Own-ness is the full socket address, which is what + // separates it from the sibling that shares only the IP. + const duplicateInstance = runningAppList.find( + (instance) => ipsMatch(instance.ip, ip) && !socketAddressesMatch(instance.ip, localSocketAddress), + ); if (duplicateInstance) { log.info(`Aplication: ${app.name}, was found on the network already running under the same ip, uninstalling app`); log.warn(`REMOVAL REASON: Duplicate IP detected - ${app.name} already running on network with IP ${ip} (after IP change)`); @@ -1201,11 +1590,20 @@ async function adjustExternalIP(ip) { await appUninstaller.removeAppLocally(app.name, null, true, null, true).catch((error) => log.error(error)); appsRemoved += 1; } else { - // once app specs v8 is done we check if app have specs that is using fluxnode service. - // eslint-disable-next-line no-await-in-loop - await appController.appDockerRestart(app.name); + staying.push(app); } } + // One handover for the whole set, not a call per app: what an app is made + // of - a composed one's containers are `_`, and an + // enterprise one's names are inside a blob this layer cannot read - is + // knowledge the reconciler already holds. Failures stay inside it too, so + // one app that cannot be asked costs the others nothing and leaves the + // broadcast, the confirmation transaction and the geolocation update below + // reachable. + if (staying.length && onAddressChanged) { + await onAddressChanged(staying, `node ip changed to ${ip}`) + .catch((error) => log.error(`adjustExternalIP - restart request failed: ${error.message}`)); + } if (apps.length > appsRemoved) { const broadcastedAt = Date.now(); const newIpChangedMessage = { @@ -1270,11 +1668,22 @@ async function checkMyFluxAvailability(retryNumber = 0) { return false; } - const randomSocketAddress = await networkStateService.getRandomSocketAddress( + // An external observer. This asks a peer whether it can reach US, and a Flux + // node sharing our public address cannot answer: reaching us means leaving the + // router and being sent straight back in, which most consumer routers do not + // do. Asking one produced a false "unreachable" and two points of dosState, + // on exactly the shared-address topology this release is about. + const randomSocketAddress = await networkStateService.getRandomExternalObserver( localSocketAddress, ); - if (!randomSocketAddress) return false; + // Nobody outside this address to ask, so nothing is learned and nothing is + // concluded - dosState is deliberately untouched here, unlike every failure + // path below it. The next cycle asks again. + if (!randomSocketAddress) { + log.warn('checkMyFluxAvailability - no Flux node outside this address could be asked; skipping this pass'); + return false; + } const remoteIp = extractIp(randomSocketAddress); const remotePort = extractPort(randomSocketAddress); @@ -1406,6 +1815,20 @@ async function checkDeterministicNodesCollisions() { }, 120 * 1000); return; } + // Same shape as the daemon check above, for the same reason. The list + // accessors wait for the list to arrive, and this loop only re-arms once + // it has finished - so awaiting in here would retire it for the life of + // the process rather than delay it, and this is the only thing that ever + // clears this node's DOS state. Reading an unknown list instead is no + // better: it makes every branch below conclude this node is not in the + // confirmed list, log that as the reason, and skip the availability check + // that would have cleared the DOS. + if (!networkStateService.isReady()) { + setTimeout(() => { + checkDeterministicNodesCollisions(); + }, 120 * 1000); + return; + } const nodeList = await fluxCommunicationUtils.deterministicFluxList(); const result = nodeList.filter((node) => socketAddressesMatch(node.ip, localSocketAddr)); const nodeStatus = await daemonServiceFluxnodeRpcs.getFluxNodeStatus(); @@ -1562,12 +1985,12 @@ async function setDOSStateApi(req, res) { if (!config.has('testEventStream') || config.get('testEventStream') !== true) { return res.status(404).json({ status: 'error', data: { message: 'Not available' } }); } - const authorized = await verificationHelper.verifyPrivilege('fluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); } - let body = req.body; + let { body } = req; if (typeof body !== 'object') { try { body = JSON.parse(body); } catch { body = {}; } } @@ -1779,7 +2202,7 @@ async function allowPortApi(req, res) { const errMessage = messageHelper.createErrorMessage('No Port address specified.'); return res.json(errMessage); } - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let message; @@ -2228,6 +2651,7 @@ module.exports = { getLocalSocketAddress, getFluxNodePrivateKey, getFluxNodePublicKey, + MAX_KEEPALIVE_PORTS, checkDeterministicNodesCollisions, getIncomingConnections, getIncomingConnectionsInfo, @@ -2246,6 +2670,7 @@ module.exports = { checkFluxbenchVersionAllowed, checkMyFluxAvailability, adjustExternalIP, + setOnAddressChanged, allowPort, allowOutPort, isFirewallActive, @@ -2261,6 +2686,11 @@ module.exports = { setDosStateValue, getDosStateValue, isNodeDos, + PlacementHoldOwner, + setPlacementHold, + clearPlacementHold, + getPlacementHold, + isPlacementHeld, setStickyDosMessage, getStickyDosMessage, clearStickyDosMessage, @@ -2270,7 +2700,10 @@ module.exports = { isCommunicationEstablished, lruRateLimit, isPortOpen, + portAnswered, + MAX_ECHO_BYTES, checkAppAvailability, + verifySignedFluxnodeMessage, isPortEnterprise, isPortBanned, isPortUPNPBanned, diff --git a/ZelBack/src/services/fluxService.js b/ZelBack/src/services/fluxService.js index 94ba6cb013..d89103d3e2 100644 --- a/ZelBack/src/services/fluxService.js +++ b/ZelBack/src/services/fluxService.js @@ -10,6 +10,7 @@ const log = require('../lib/log'); const packageJson = require('../../../package.json'); const serviceHelper = require('./serviceHelper'); const verificationHelper = require('./verificationHelper'); +const verificationHelperUtils = require('./verificationHelperUtils'); const messageHelper = require('./messageHelper'); const dbHelper = require('./dbHelper'); const daemonServiceUtils = require('./daemonService/daemonServiceUtils'); @@ -17,6 +18,7 @@ const daemonServiceBlockchainRpcs = require('./daemonService/daemonServiceBlockc const daemonServiceFluxnodeRpcs = require('./daemonService/daemonServiceFluxnodeRpcs'); const daemonServiceControlRpcs = require('./daemonService/daemonServiceControlRpcs'); const benchmarkService = require('./benchmarkService'); +const cloudUIUpdateService = require('./cloudUIUpdateService'); const generalService = require('./generalService'); const explorerService = require('./explorerService'); const fluxCommunication = require('./fluxCommunication'); @@ -33,9 +35,16 @@ const tar = require('tar/create'); // use non promises stream for node 14.x compatibility // const stream = require('node:stream/promises'); const stream = require('node:stream'); +const { Privilege, authOf } = require('./utils/privileges'); const isArcane = Boolean(process.env.FLUXOS_PATH); +// Where this node's checkout is, named once. Every command below that reads or +// writes the repository is told it, rather than inheriting whatever directory +// the process happens to be running in: `git checkout` and `git fetch` write, +// and a write is not something to leave to the launcher's habits. +const REPO_ROOT = path.join(__dirname, '../../../'); + // Cache for OS distribution information let cachedOSDistInfo = null; @@ -143,31 +152,39 @@ async function fluxBackendFolder(req, res) { * @param {object} res Response. * @returns {Promise} Message. */ -async function getCurrentCommitId(req, res) { +async function getCurrentCommitId() { // Fix - this breaks if head in detached state? (or something, can't remember) - if (req) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); - if (authorized !== true) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - } - const { stdout: commitId, error } = await serviceHelper.runCommand('git', { + cwd: REPO_ROOT, logError: false, params: ['rev-parse', '--short', 'HEAD'], }); - if (error) { - const errMsg = messageHelper.createErrorMessage( + if (error) throw error; + + return commitId.trim(); +} + +/** + * To show the current short commit id. Flux team only: which code a node runs is not the operator's to choose or to read. + * @param {object} req Request. + * @param {object} res Response. + * @returns {Promise} Message. + */ +async function getCurrentCommitIdApi(req, res) { + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); + if (authorized !== true) { + return res.json(messageHelper.errUnauthorizedMessage()); + } + + try { + return res.json(messageHelper.createSuccessMessage(await getCurrentCommitId())); + } catch (error) { + return res.json(messageHelper.createErrorMessage( `Error getting current commit id of Flux: ${error.message}`, error.name, error.code, - ); - return res ? res.json(errMsg) : errMsg; + )); } - - const successMsg = messageHelper.createSuccessMessage(commitId.trim()); - return res ? res.json(successMsg) : successMsg; } /** @@ -176,116 +193,225 @@ async function getCurrentCommitId(req, res) { * @param {object} res Response. * @returns {Promise} Message. */ -async function getCurrentBranch(req, res) { +async function getCurrentBranch() { // ToDo: Fix - this breaks if head in detached state (or something similar) - if (req) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); - if (authorized !== true) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - } - - const { stdout: commitId, error } = await serviceHelper.runCommand('git', { + const { stdout: branch, error } = await serviceHelper.runCommand('git', { + cwd: REPO_ROOT, logError: false, params: ['rev-parse', '--abbrev-ref', 'HEAD'], }); - if (error) { - const errMsg = messageHelper.createErrorMessage( + if (error) throw error; + + return branch.trim(); +} + +/** + * To show the currently selected branch. Flux team only: which code a node runs is not the operator's to choose or to read. + * @param {object} req Request. + * @param {object} res Response. + * @returns {Promise} Message. + */ +async function getCurrentBranchApi(req, res) { + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); + if (authorized !== true) { + return res.json(messageHelper.errUnauthorizedMessage()); + } + + try { + return res.json(messageHelper.createSuccessMessage(await getCurrentBranch())); + } catch (error) { + return res.json(messageHelper.createErrorMessage( `Error getting current branch of Flux: ${error.message}`, error.name, error.code, - ); - return res ? res.json(errMsg) : errMsg; + )); } +} - const successMsg = messageHelper.createSuccessMessage(commitId.trim()); - return res ? res.json(successMsg) : successMsg; +/** + * Bring a branch onto a node that has no reference to it, without widening what the + * clone tracks. + * + * The installer clones `--depth 1 --single-branch`, so a node carries exactly the branch + * it was installed on: no local branch for any other, and no remote-tracking ref either, + * because `remote.origin.fetch` maps only the one. Nothing on such a node can check out + * another branch, and that is not a state a switch should refuse - it is the ordinary + * state of every node. + * + * Fetched into `refs/heads/` rather than a tracking ref, because git decides what + * counts as a remote branch from the configured refspec: with a single-branch mapping it + * refuses to check out or track a `refs/remotes/origin/` this fetch created, + * saying it "is not a branch". Fetching the local branch directly sidesteps that, and + * leaves `remote.origin.fetch` exactly as the installer set it - the clone stays + * single-branch and stays shallow. + * + * The upstream is then written by hand for the same reason, and it is not optional: the + * update paths run `git pull`, which has nothing to pull without it. + * + * The depth is conditional. Passing `--depth` to a fetch on a FULL clone makes that + * repository shallow - a legacy node would be quietly truncated by a branch switch - so + * it is passed only where the repository already is. + * + * @param {string} branch The branch to bring down + * @returns {Promise} + */ +async function fetchBranch(branch) { + const { stdout: shallow } = await serviceHelper.runCommand('git', { + cwd: REPO_ROOT, + params: ['rev-parse', '--is-shallow-repository'], + }); + + const depth = String(shallow).trim() === 'true' ? ['--depth', '1'] : []; + + const { error: fetchError } = await serviceHelper.runCommand('git', { + cwd: REPO_ROOT, + params: ['fetch', ...depth, 'origin', `${branch}:refs/heads/${branch}`], + }); + + if (fetchError) throw new Error(`Branch ${branch} is not on this node and could not be fetched: ${fetchError.message}`); + + const { error: remoteError } = await serviceHelper.runCommand('git', { + cwd: REPO_ROOT, + params: ['config', `branch.${branch}.remote`, 'origin'], + }); + const { error: mergeError } = await serviceHelper.runCommand('git', { + cwd: REPO_ROOT, + params: ['config', `branch.${branch}.merge`, `refs/heads/${branch}`], + }); + + if (remoteError || mergeError) throw new Error(`Fetched ${branch} but could not set it to track origin`); } /** - * Check out branch if it exists locally + * Check out a branch this node can reach. + * + * Each step names itself when it fails, because the caller reports the reason to whoever + * asked and "could not switch branch" does not distinguish a branch this node has never + * fetched from a working tree with changes in it. + * + * The branch is looked for where `git checkout` looks. A node carries only the branch it + * was installed on and remote-tracking refs for the rest - the installer clones shallow + * but tracks every head - and checkout creates the local branch from origin/ when + * there is no local one. `rev-parse --verify ` never resolves a remote-tracking + * ref, so asking only that refuses a switch the node can perfectly well make: on an + * Arcane node sitting on master, `origin/development` resolves and `development` does not. + * * @param {string} branch The branch to checkout * @param {{pull?: Boolean}} options - * @returns {Promise} + * @returns {Promise} */ async function checkoutBranch(branch, options = {}) { // ToDo: this will break if multiple remotes - const { error: verifyError } = await serviceHelper.runCommand('git', { - params: ['rev-parse', '--verify', branch], + const { error: localMissing } = await serviceHelper.runCommand('git', { + cwd: REPO_ROOT, + params: ['rev-parse', '--verify', '--quiet', `refs/heads/${branch}`], }); - if (verifyError) return false; + if (localMissing) { + const { error: trackingMissing } = await serviceHelper.runCommand('git', { + cwd: REPO_ROOT, + params: ['rev-parse', '--verify', '--quiet', `refs/remotes/origin/${branch}`], + }); + + if (trackingMissing) await fetchBranch(branch); + } const { error: checkoutError } = await serviceHelper.runCommand('git', { + cwd: REPO_ROOT, params: ['checkout', branch], }); - if (checkoutError) return false; + if (checkoutError) throw new Error(`Could not check out ${branch}: ${checkoutError.message}`); if (options.pull) { - const { error: pullError } = await serviceHelper.runCommand('git', { params: ['pull'] }); - if (pullError) return false; + const { error: pullError } = await serviceHelper.runCommand('git', { cwd: REPO_ROOT, params: ['pull'] }); + if (pullError) throw new Error(`Checked out ${branch} but could not pull it: ${pullError.message}`); } +} - return true; +/** + * Where the working tree actually sits, read back from git. + * + * Returns null rather than throwing. This describes an operation that has already + * succeeded, so a tree it cannot name - a detached HEAD, or a node not deployed from a + * checkout at all - must not turn a completed switch into a reported failure. + * + * @returns {Promise} + */ +async function currentCheckout() { + try { + const [branch, commitId] = await Promise.all([getCurrentBranch(), getCurrentCommitId()]); + return `${branch} at ${commitId}`; + } catch (error) { + log.warn(`Could not read the current checkout: ${error.message}`); + return null; + } } /** - * To switch to master branch of FluxOS. Only accessible by admins and Flux team members. + * To switch to master branch of FluxOS. Flux team only: an operator updates along the branch their node is on, and does not choose a different one. * @param {object} req Request. * @param {object} res Response. * @returns {Promise} Message. */ -// eslint-disable-next-line consistent-return -async function enterMaster(req, res) { - // why use npm for this? - if (req) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); - if (authorized !== true) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - } - const cwd = path.join(__dirname, '../../../'); +async function enterMaster() { + await checkoutBranch('master'); +} - const { error } = await serviceHelper.runCommand('npm', { cwd, params: ['run', 'entermaster'] }); +/** + * To switch to master branch of FluxOS. Flux team only: an operator updates along the branch their node is on, and does not choose a different one. + * @param {object} req Request. + * @param {object} res Response. + * @returns {Promise} Message. + */ +async function enterMasterApi(req, res) { + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); + if (authorized !== true) { + return res.json(messageHelper.errUnauthorizedMessage()); + } - if (error) { - const errMessage = messageHelper.createErrorMessage(`Error entering master branch of Flux: ${error.message}`, error.name, error.code); - return res ? res.json(errMessage) : errMessage; + try { + await enterMaster(); + const at = await currentCheckout(); + return res.json(messageHelper.createSuccessMessage( + at ? `Master branch successfully entered, now on ${at}` : 'Master branch successfully entered', + )); + } catch (error) { + return res.json(messageHelper.createErrorMessage(`Error entering master branch of Flux: ${error.message}`, error.name, error.code)); } +} - const message = messageHelper.createSuccessMessage('Master branch successfully entered'); - return res ? res.json(message) : message; +/** + * To switch to development branch of FluxOS. Flux team only: an operator updates along the branch their node is on, and does not choose a different one. + * @param {object} req Request. + * @param {object} res Response. + * @returns {Promise} Message. + */ +async function enterDevelopment() { + await checkoutBranch('development'); } /** - * To switch to development branch of FluxOS. Only accessible by admins and Flux team members. + * To switch to development branch of FluxOS. Flux team only: an operator updates along the branch their node is on, and does not choose a different one. * @param {object} req Request. * @param {object} res Response. * @returns {Promise} Message. */ -// eslint-disable-next-line consistent-return -async function enterDevelopment(req, res) { - if (req) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); - if (authorized !== true) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } +async function enterDevelopmentApi(req, res) { + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); + if (authorized !== true) { + return res.json(messageHelper.errUnauthorizedMessage()); } - const cwd = path.join(__dirname, '../../../'); - - const { error } = await serviceHelper.runCommand('npm', { cwd, params: ['run', 'enterdevelopment'] }); - if (error) { - const errMessage = messageHelper.createErrorMessage(`Error entering development branch of Flux: ${error.message}`, error.name, error.code); - return res ? res.json(errMessage) : errMessage; + try { + await enterDevelopment(); + const at = await currentCheckout(); + return res.json(messageHelper.createSuccessMessage( + at ? `Development branch successfully entered, now on ${at}` : 'Development branch successfully entered', + )); + } catch (error) { + return res.json(messageHelper.createErrorMessage(`Error entering development branch of Flux: ${error.message}`, error.name, error.code)); } - - const message = messageHelper.createSuccessMessage('Development branch successfully entered'); - return res ? res.json(message) : message; } /** @@ -296,23 +422,21 @@ async function enterDevelopment(req, res) { */ // eslint-disable-next-line consistent-return async function updateFlux(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); } - const cwd = path.join(__dirname, '../../../'); - - const { error } = await serviceHelper.runCommand('npm', { cwd, params: ['run', 'updateflux'] }); + const { error } = await serviceHelper.runCommand('npm', { cwd: REPO_ROOT, params: ['run', 'updateflux'] }); if (error) { const errMessage = messageHelper.createErrorMessage(`Error updating Flux: ${error.message}`, error.name, error.code); - return res ? res.json(errMessage) : errMessage; + return res.json(errMessage); } const message = messageHelper.createSuccessMessage('Flux successfully updated'); - return res ? res.json(message) : message; + return res.json(message); } /** @@ -321,27 +445,31 @@ async function updateFlux(req, res) { * @param {object} res Response. * @returns {Promise} Message. */ -// eslint-disable-next-line consistent-return -async function softUpdateFlux(req, res) { - if (req) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); - if (authorized !== true) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - } +async function softUpdateFlux() { - const cwd = path.join(__dirname, '../../../'); + const { error } = await serviceHelper.runCommand('npm', { cwd: REPO_ROOT, params: ['run', 'softupdate'] }); - const { error } = await serviceHelper.runCommand('npm', { cwd, params: ['run', 'softupdate'] }); + if (error) throw error; +} - if (error) { - const errMessage = messageHelper.createErrorMessage(`Error soft updating Flux: ${error.message}`, error.name, error.code); - return res ? res.json(errMessage) : errMessage; +/** + * To soft update FluxOS version (executes the command `npm run softupdate` on the node machine). Only accessible by admins and Flux team members. + * @param {object} req Request. + * @param {object} res Response. + * @returns {Promise} Message. + */ +async function softUpdateFluxApi(req, res) { + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); + if (authorized !== true) { + return res.json(messageHelper.errUnauthorizedMessage()); } - const message = messageHelper.createSuccessMessage('Flux successfully soft updated'); - return res ? res.json(message) : message; + try { + await softUpdateFlux(); + return res.json(messageHelper.createSuccessMessage('Flux successfully soft updated')); + } catch (error) { + return res.json(messageHelper.createErrorMessage(`Error soft updating Flux: ${error.message}`, error.name, error.code)); + } } /** @@ -350,27 +478,31 @@ async function softUpdateFlux(req, res) { * @param {object} res Response. * @returns {Promise} Message. */ -// eslint-disable-next-line consistent-return -async function softUpdateFluxInstall(req, res) { - if (req) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); - if (authorized !== true) { - const errMessage = messageHelper.errUnauthorizedMessage(); - return res ? res.json(errMessage) : errMessage; - } - } +async function softUpdateFluxInstall() { - const cwd = path.join(__dirname, '../../../'); + const { error } = await serviceHelper.runCommand('npm', { cwd: REPO_ROOT, params: ['run', 'softupdateinstall'] }); - const { error } = await serviceHelper.runCommand('npm', { cwd, params: ['run', 'softupdateinstall'] }); + if (error) throw error; +} - if (error) { - const errMessage = messageHelper.createErrorMessage(`Error soft updating Flux with installation: ${error.message}`, error.name, error.code); - return res ? res.json(errMessage) : errMessage; +/** + * To install the soft update of FluxOS (executes the command `npm run softupdateinstall` on the node machine). Only accessible by admins and Flux team members. + * @param {object} req Request. + * @param {object} res Response. + * @returns {Promise} Message. + */ +async function softUpdateFluxInstallApi(req, res) { + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); + if (authorized !== true) { + return res.json(messageHelper.errUnauthorizedMessage()); } - const message = messageHelper.createSuccessMessage('Flux successfully soft updated with installation'); - return res ? res.json(message) : message; + try { + await softUpdateFluxInstall(); + return res.json(messageHelper.createSuccessMessage('Flux successfully soft updated with installation')); + } catch (error) { + return res.json(messageHelper.createErrorMessage(`Error soft updating Flux with installation: ${error.message}`, error.name, error.code)); + } } /** @@ -381,15 +513,14 @@ async function softUpdateFluxInstall(req, res) { */ // eslint-disable-next-line consistent-return async function hardUpdateFlux(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); } - const cwd = path.join(__dirname, '../../../'); - const { error } = await serviceHelper.runCommand('npm', { cwd, params: ['run', 'hardupdateflux'] }); + const { error } = await serviceHelper.runCommand('npm', { cwd: REPO_ROOT, params: ['run', 'hardupdateflux'] }); if (error) { const errMessage = messageHelper.createErrorMessage(`Error hard updating Flux: ${error.message}`, error.name, error.code); @@ -401,30 +532,45 @@ async function hardUpdateFlux(req, res) { } /** - * To rebuild FluxOS (executes the command `npm run homebuild` on the node machine). Only accessible by admins and Flux team members. + * To rebuild the Flux UI by fetching the published CloudUI release again. Only accessible by admins and Flux team members. * @param {object} req Request. * @param {object} res Response. * @returns {Promise} Message. */ // eslint-disable-next-line consistent-return -async function rebuildHome(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); +async function rebuildUi(req, res) { + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); } - const cwd = path.join(__dirname, '../../../'); + // Refused on ArcaneOS, where the watchdog owns CloudUI: the periodic check + // stands aside there for that reason, and this must not walk under that by + // reaching the script directly. Answered rather than done quietly, so the + // caller learns which component to ask. + if (cloudUIUpdateService.watchdogManagesCloudUI()) { + const errMessage = messageHelper.createErrorMessage('CloudUI is managed by the watchdog on ArcaneOS, so it is not rebuilt from here'); + return res.json(errMessage); + } - const { error } = await serviceHelper.runCommand('npm', { cwd, params: ['run', 'homebuild'] }); + // The UI is fetched, not built here. It is a published release of a separate + // repository, so rebuilding it means taking that release again through the one path + // that knows which host to ask - the same path the periodic check uses. + // + // Unconditionally, unlike the periodic check: that one stands down when the + // installed hash already matches the release, and a CloudUI which is damaged + // rather than out of date matches all the same. Repairing one is what this is + // for, so it takes the release again whatever is on disk. + const rebuilt = await cloudUIUpdateService.runUpdateScript(); - if (error) { - const errMessage = messageHelper.createErrorMessage(`Error rebuilding Flux UI: ${error.message}`, error.name, error.code); - return res ? res.json(errMessage) : errMessage; + if (!rebuilt) { + const errMessage = messageHelper.createErrorMessage('Error rebuilding Flux UI, see the node log for what the fetch reported'); + return res.json(errMessage); } const message = messageHelper.createSuccessMessage('Flux UI successfully rebuilt'); - return res ? res.json(message) : message; + return res.json(message); } /** @@ -435,7 +581,7 @@ async function rebuildHome(req, res) { */ // eslint-disable-next-line consistent-return async function updateDaemon(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -463,7 +609,7 @@ async function updateDaemon(req, res) { */ // eslint-disable-next-line consistent-return async function updateBenchmark(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -491,7 +637,7 @@ async function updateBenchmark(req, res) { */ // eslint-disable-next-line consistent-return async function startBenchmark(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -521,7 +667,7 @@ async function startBenchmark(req, res) { */ // eslint-disable-next-line consistent-return async function restartBenchmark(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -549,7 +695,7 @@ async function restartBenchmark(req, res) { */ // eslint-disable-next-line consistent-return async function startDaemon(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -579,7 +725,7 @@ async function startDaemon(req, res) { */ // eslint-disable-next-line consistent-return async function restartDaemon(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -607,7 +753,7 @@ async function restartDaemon(req, res) { */ // eslint-disable-next-line consistent-return async function reindexDaemon(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -670,7 +816,7 @@ async function getFluxIP(req, res) { * @returns {object} Message. */ function getFluxZelID(req, res) { - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const zelID = userconfig.initial.zelid; const message = messageHelper.createDataMessage(zelID); return res ? res.json(message) : message; @@ -685,7 +831,9 @@ function getFluxZelID(req, res) { function getFluxIds(req, res) { const fluxConfig = { fluxTeamFluxID: configDefault.fluxTeamFluxID, - fluxSupportTeamFluxID: configDefault.fluxSupportTeamFluxID, + // An array. Read through the helper so a node whose local config still holds + // the single string this used to be reports the same shape as one that does not. + fluxSupportTeamFluxID: verificationHelperUtils.fluxSupportTeamZelids(), }; const message = messageHelper.createDataMessage(fluxConfig); @@ -723,7 +871,7 @@ async function getFluxGeolocation(req, res) { * @returns {object} Message. */ function getFluxPGPidentity(req, res) { - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const pgp = userconfig.initial.pgpPublicKey; const message = messageHelper.createDataMessage(pgp); return res ? res.json(message) : message; @@ -736,7 +884,7 @@ function getFluxPGPidentity(req, res) { * @returns {object} Message. */ function getFluxKadena(req, res) { - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const kadena = userconfig.initial.kadena || null; const message = messageHelper.createDataMessage(kadena); return res ? res.json(message) : message; @@ -749,7 +897,7 @@ function getFluxKadena(req, res) { * @returns {object} Message. */ function getRouterIP(req, res) { - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const routerIP = userconfig.initial.routerIP || ''; const message = messageHelper.createDataMessage(routerIP); return res ? res.json(message) : message; @@ -762,7 +910,7 @@ function getRouterIP(req, res) { * @returns {object} Message. */ function getBlockedPorts(req, res) { - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const blockedPorts = userconfig.initial.blockedPorts || []; const message = messageHelper.createDataMessage(blockedPorts); return res ? res.json(message) : message; @@ -775,7 +923,7 @@ function getBlockedPorts(req, res) { * @returns {object} Message. */ function getAPIPort(req, res) { - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const routerIP = userconfig.initial.apiport || '16127'; const message = messageHelper.createDataMessage(routerIP); return res ? res.json(message) : message; @@ -788,7 +936,7 @@ function getAPIPort(req, res) { * @returns {object} Message. */ function getBlockedRepositories(req, res) { - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const blockedPorts = userconfig.initial.blockedRepositories || []; const message = messageHelper.createDataMessage(blockedPorts); return res ? res.json(message) : message; @@ -813,11 +961,11 @@ function getEnterpriseAppOwners(req, res) { * @returns {object} Message. */ function getMarketplaceURL(req, res) { - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const development = userconfig.initial.development || false; - let marketPlaceUrl = 'https://stats.runonflux.io/marketplace/listapps'; + let marketPlaceUrl = `${config.stats.baseUrl}/marketplace/listapps`; if (development) { - marketPlaceUrl = 'https://stats.runonflux.io/marketplace/listdevapps'; + marketPlaceUrl = `${config.stats.baseUrl}/marketplace/listdevapps`; } const message = messageHelper.createDataMessage(marketPlaceUrl); return res ? res.json(message) : message; @@ -830,7 +978,7 @@ function getMarketplaceURL(req, res) { * @returns {Promise} Debug.log file for Flux daemon. */ async function daemonDebug(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -850,7 +998,7 @@ async function daemonDebug(req, res) { * @returns {Promise} Debug.log file for Flux benchmark. */ async function benchmarkDebug(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); return res.json(errMessage); @@ -874,7 +1022,7 @@ async function benchmarkDebug(req, res) { * @param {object} res Response. */ async function tailDaemonDebug(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -905,7 +1053,7 @@ async function tailDaemonDebug(req, res) { * @param {object} res Response. */ async function tailBenchmarkDebug(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -957,7 +1105,7 @@ async function fluxLog(res, filelog) { */ async function fluxErrorLog(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -977,7 +1125,7 @@ async function fluxErrorLog(req, res) { */ async function fluxWarnLog(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -997,7 +1145,7 @@ async function fluxWarnLog(req, res) { */ async function fluxInfoLog(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -1017,7 +1165,7 @@ async function fluxInfoLog(req, res) { */ async function fluxDebugLog(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -1036,7 +1184,7 @@ async function fluxDebugLog(req, res) { * @param {Promise} logfile Log file name (excluding `.log`). */ async function tailFluxLog(req, res, logfile) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -1067,7 +1215,7 @@ async function tailFluxLog(req, res, logfile) { */ async function tailFluxErrorLog(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { await tailFluxLog(req, res, 'error'); } else { @@ -1086,7 +1234,7 @@ async function tailFluxErrorLog(req, res) { */ async function tailFluxWarnLog(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { await tailFluxLog(req, res, 'warn'); } else { @@ -1105,7 +1253,7 @@ async function tailFluxWarnLog(req, res) { */ async function tailFluxInfoLog(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { await tailFluxLog(req, res, 'info'); } else { @@ -1124,7 +1272,7 @@ async function tailFluxInfoLog(req, res) { */ async function tailFluxDebugLog(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized === true) { await tailFluxLog(req, res, 'debug'); } else { @@ -1229,10 +1377,7 @@ async function getFluxInfo(req, res) { } info.flux.nodeJsVersion = nodeJsVersionsRes.data.node; const syncthingVersion = await syncthingService.systemVersion(); - if (syncthingVersion.status === 'error') { - throw syncthingVersion.data; - } - info.flux.syncthingVersion = syncthingVersion.data.version; + info.flux.syncthingVersion = syncthingVersion.version; const dockerVersion = await dockerService.dockerVersion(); info.flux.dockerVersion = dockerVersion.Version; info.flux.mongoDbVersion = await dbHelper.getMongoDbVersion(); @@ -1246,6 +1391,20 @@ async function getFluxInfo(req, res) { } info.flux.ip = ipRes.data; info.flux.staticIp = geolocationService.isStaticIP(); + // How far this node is through being staged out of service, or null. Read + // beside `dos` below, which reports the last stage: the three read together + // as HOLD, then EVACUATE, then EVACUATE with a DOS. + // + // Reported at all because the first stage is otherwise invisible. A held + // node stops taking new apps and looks in every other way like one that has + // simply not been given work - and that stage is the whole settling window, + // the only part of this where an operator can still put the node right and + // lose nothing. + // + // Lazily required: this service is loaded by half the tree and the enforcer + // reaches back into fluxNetworkHelper, which reaches here. + // eslint-disable-next-line global-require + info.flux.dosStaging = require('./residentialNodeDosService').getDosStaging(); info.flux.upnp = upnpService.isUPNP(); info.flux.maxNumberOfIpChanges = fluxNetworkHelper.getMaxNumberOfIpChanges(); const zelidRes = await getFluxZelID(); @@ -1282,7 +1441,7 @@ async function getFluxInfo(req, res) { info.flux.arcaneHumanVersion = arcaneHumanVersion; } info.flux.appsDos = dosAppsResult.data; - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; info.flux.development = userconfig.initial.development || false; const daemonInfoRes = await daemonServiceControlRpcs.getInfo(); if (daemonInfoRes.status === 'error') { @@ -1325,12 +1484,18 @@ async function getFluxInfo(req, res) { if (appsRunning.status === 'error') { throw appsRunning.data; } - info.apps.runningapps = appsRunning.data; + // The same public view /apps/listrunningapps serves, from the same function. + // This endpoint carries the container listing too, and a projection applied + // at one exit and not the other is the shape it exists to avoid. + info.apps.runningapps = appQueryService.publicContainerView(appsRunning.data); const appsResources = await resourceQueryService.appsResources(); if (appsResources.status === 'error') { throw appsResources.data; } - info.apps.resources = appsResources.data; + // The same three numbers /apps/appsresources publishes. This endpoint embeds + // them, and a field kept from one exit and not the other is the shape that + // catches this file out. + info.apps.resources = resourceQueryService.publicResourceView(appsResources.data); // eslint-disable-next-line global-require const registryManager = require('./appDatabase/registryManager'); const appHashes = await registryManager.getAppHashes(); @@ -1377,7 +1542,7 @@ async function getFluxInfo(req, res) { */ async function adjustKadenaAccount(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { let { account } = req.params; account = account || req.query.account; @@ -1394,7 +1559,7 @@ async function adjustKadenaAccount(req, res) { throw new Error(`Invalid Chain ID ${chainid} provided.`); } const kadenaURI = `kadena:${account}?chainid=${chainid}`; - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const fluxDirPath = path.join(__dirname, '../../../config/userconfig.js'); const dataToWrite = `module.exports = { initial: { @@ -1434,12 +1599,12 @@ async function adjustKadenaAccount(req, res) { */ async function adjustRouterIP(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { let { routerip } = req.params; routerip = routerip || req.query.routerip || ''; - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const dataToWrite = `module.exports = { initial: { ipaddress: '${userconfig.initial.ipaddress || '127.0.0.1'}', @@ -1477,7 +1642,7 @@ async function adjustRouterIP(req, res) { * @param {object} res Response. */ async function adjustBlockedPorts(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); @@ -1495,7 +1660,7 @@ async function adjustBlockedPorts(req, res) { if (!Array.isArray(blockedPorts)) { throw new Error('Blocked Ports is not a valid array'); } - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const dataToWrite = `module.exports = { initial: { ipaddress: '${userconfig.initial.ipaddress || '127.0.0.1'}', @@ -1533,7 +1698,7 @@ async function adjustBlockedPorts(req, res) { */ async function adjustAPIPort(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { let { apiport } = req.params; apiport = apiport || req.query.apiport || ''; @@ -1545,7 +1710,7 @@ async function adjustAPIPort(req, res) { return; } - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const dataToWrite = `module.exports = { initial: { ipaddress: '${userconfig.initial.ipaddress || '127.0.0.1'}', @@ -1583,7 +1748,7 @@ async function adjustAPIPort(req, res) { * @param {object} res Response. */ async function adjustBlockedRepositories(req, res) { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); @@ -1609,7 +1774,7 @@ async function adjustBlockedRepositories(req, res) { } }); - const userconfig = globalThis.userconfig; + const { userconfig } = globalThis; const dataToWrite = `module.exports = { initial: { ipaddress: '${userconfig.initial.ipaddress || '127.0.0.1'}', @@ -1680,7 +1845,7 @@ async function getNodeTier(req, res) { * @param {object} res Response. */ async function restartFluxOS(req, res) { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized !== true) { const errMessage = messageHelper.errUnauthorizedMessage(); res.json(errMessage); @@ -2113,7 +2278,9 @@ module.exports = { checkoutBranch, daemonDebug, enterDevelopment, + enterDevelopmentApi, enterMaster, + enterMasterApi, fluxBackendFolder, fluxDebugLog, fluxErrorLog, @@ -2124,7 +2291,9 @@ module.exports = { getBlockedRepositories, getEnterpriseAppOwners, getCurrentBranch, + getCurrentBranchApi, getCurrentCommitId, + getCurrentCommitIdApi, getFluxGeolocation, getFluxInfo, getFluxIP, @@ -2140,13 +2309,15 @@ module.exports = { getRouterIP, hardUpdateFlux, isStaticIPapi, - rebuildHome, + rebuildUi, reindexDaemon, restartBenchmark, restartDaemon, restartFluxOS, softUpdateFlux, + softUpdateFluxApi, softUpdateFluxInstall, + softUpdateFluxInstallApi, startBenchmark, startDaemon, streamChainPreparation, diff --git a/ZelBack/src/services/fluxshareService.js b/ZelBack/src/services/fluxshareService.js index 52c6cf902c..7ddd70d1fb 100644 --- a/ZelBack/src/services/fluxshareService.js +++ b/ZelBack/src/services/fluxshareService.js @@ -1,13 +1,11 @@ const config = require('config'); const crypto = require('crypto'); const path = require('path'); -const df = require('node-df'); const fs = require('fs'); const { formidable } = require('formidable'); const archiver = require('archiver'); -// eslint-disable-next-line import/no-extraneous-dependencies -const util = require('util'); const serviceHelper = require('./serviceHelper'); +const volumeService = require('./utils/volumeService'); const messageHelper = require('./messageHelper'); const dbHelper = require('./dbHelper'); const verificationHelper = require('./verificationHelper'); @@ -15,6 +13,7 @@ const generalService = require('./generalService'); const log = require('../lib/log'); const IOUtils = require('./IOUtils'); const { sanitizePath } = require('./utils/pathSecurity'); +const { Privilege, authOf } = require('./utils/privileges'); const dirpath = path.join(__dirname, '../../../'); const appsFolder = process.env.FLUX_APPS_FOLDER || path.join(dirpath, 'ZelApps'); @@ -184,7 +183,7 @@ async function fluxShareSharedFiles() { */ async function fluxShareGetSharedFiles(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { const files = await fluxShareSharedFiles(); const resultsResponse = messageHelper.createDataMessage(files); @@ -216,7 +215,7 @@ async function fluxShareGetSharedFiles(req, res) { */ async function fluxShareUnshareFile(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { let { file } = req.params; file = file || req.query.file; @@ -251,7 +250,7 @@ async function fluxShareUnshareFile(req, res) { */ async function fluxShareShareFile(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { let { file } = req.params; file = file || req.query.file; @@ -290,7 +289,7 @@ async function fluxShareDownloadFolder(req, res, authorized = false) { try { let auth = authorized; if (!auth) { - auth = await verificationHelper.verifyPrivilege('admin', req); + auth = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); } if (auth) { @@ -355,7 +354,7 @@ async function fluxShareDownloadFile(req, res) { try { // Define base path for sanitization const zelShareBase = path.join(appsFolder, 'ZelShare'); - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { let { file } = req.params; file = file || req.query.file; @@ -439,7 +438,7 @@ async function fluxShareDownloadFile(req, res) { */ async function fluxShareRename(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { let { oldpath } = req.params; oldpath = oldpath || req.query.oldpath; @@ -500,7 +499,7 @@ async function fluxShareRename(req, res) { */ async function fluxShareRemoveFile(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { let { file } = req.params; file = file || req.query.file; @@ -546,7 +545,7 @@ async function fluxShareRemoveFile(req, res) { */ async function fluxShareRemoveFolder(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { let { folder } = req.params; folder = folder || req.query.folder; @@ -588,7 +587,7 @@ async function fluxShareRemoveFolder(req, res) { */ async function fluxShareGetFolder(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { let { folder } = req.params; folder = folder || req.query.folder || ''; @@ -660,7 +659,7 @@ async function fluxShareGetFolder(req, res) { */ async function fluxShareCreateFolder(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { let { folder } = req.params; folder = folder || req.query.folder || ''; @@ -691,7 +690,7 @@ async function fluxShareCreateFolder(req, res) { */ async function fluxShareFileExists(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { let { file } = req.params; file = file || req.query.file; @@ -739,23 +738,7 @@ async function fluxShareFileExists(req, res) { * @returns {number} The quantity of space available (GB). */ async function getSpaceAvailableForFluxShare() { - const dfAsync = util.promisify(df); - // we want whole numbers in GB - const options = { - prefixMultiplier: 'GB', - isDisplayPrefixMultiplier: false, - precision: 0, - }; - - const dfres = await dfAsync(options); - const okVolumes = []; - dfres.forEach((volume) => { - if (volume.filesystem.includes('/dev/') && !volume.filesystem.includes('loop') && !volume.mount.includes('boot')) { - okVolumes.push(volume); - } else if (volume.filesystem.includes('loop') && volume.mount === '/') { - okVolumes.push(volume); - } - }); + const okVolumes = await volumeService.capacityVolumesInGib(); // now we know that most likely there is a space available. IF user does not have his own stuff on the node or space may be sharded accross hdds. let totalSpace = 0; @@ -779,7 +762,7 @@ async function getSpaceAvailableForFluxShare() { */ async function fluxShareStorageStats(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized) { const spaceAvailableForFluxShare = await getSpaceAvailableForFluxShare(); let spaceUsedByFluxShare = getFluxShareSize(); @@ -809,7 +792,7 @@ async function fluxShareStorageStats(req, res) { */ async function fluxShareUpload(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (!authorized) { throw new Error('Unauthorized. Access denied.'); } diff --git a/ZelBack/src/services/generalService.js b/ZelBack/src/services/generalService.js index 0890f77dd3..979815beb7 100644 --- a/ZelBack/src/services/generalService.js +++ b/ZelBack/src/services/generalService.js @@ -213,20 +213,16 @@ async function checkSynced() { } /** - * To create a JSON response showing a list of whitelisted Github repositories. + * @deprecated The image whitelist is retired - nothing has enforced it since + * July 2024 and the network accepts any image (the blocklist still governs). + * Kept returning an empty list so existing callers get a valid answer instead + * of a 404. Remove once no supported release could still call it. * @param {object} req Request. * @param {object} res Response. */ async function whitelistedRepositories(req, res) { - try { - const whitelisted = await serviceHelper.axiosGet(`${config.github.rawBaseUrl}/helpers/repositories.json`); - const resultsResponse = messageHelper.createDataMessage(whitelisted.data); - res.json(resultsResponse); - } catch (error) { - log.error(error); - const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); - res.json(errMessage); - } + const resultsResponse = messageHelper.createDataMessage([]); + res.json(resultsResponse); } /** diff --git a/ZelBack/src/services/geolocationService.js b/ZelBack/src/services/geolocationService.js index a60a316d21..09c1db868e 100644 --- a/ZelBack/src/services/geolocationService.js +++ b/ZelBack/src/services/geolocationService.js @@ -1,9 +1,11 @@ const config = require('config'); +const dns = require('node:dns').promises; const log = require('../lib/log'); const fluxNetworkHelper = require('./fluxNetworkHelper'); const { extractIp } = require('./utils/socketAddressUtils'); const serviceHelper = require('./serviceHelper'); const dbHelper = require('./dbHelper'); +const networkClassifier = require('./utils/networkClassifier'); const { geolocation: geolocationCollection } = config.database.local.collections; @@ -13,7 +15,30 @@ let staticIp = false; let dataCenter = false; let lastIpChangeDate = null; let execution = 1; -const staticIpOrgs = ['hetzner', 'ovh', 'netcup', 'hostnodes', 'contabo', 'hostslim', 'zayo', 'cogent', 'lumen']; +// What this node observed about its own address - never the verdict drawn from +// it. Gathering costs an ip-api call and a PTR lookup, so it happens on the slow +// pass; the verdict also needs the published table, which is cheap to consult +// and arrives later, so it is reached on demand. +// +// Null means nothing has been gathered yet, which readers must distinguish from +// a verdict of UNKNOWN: "not observed" against "observed, and the evidence does +// not decide". +let networkEvidence = null; + +/** + * Whether this address stays put. Three-state, because "we have not observed it + * long enough" is a different answer from "it moves", and only one of them + * should keep a node away from apps that need a stable address. + */ +const STATIC_IP_STATE = Object.freeze({ + STATIC: 'STATIC', + DYNAMIC: 'DYNAMIC', + UNKNOWN: 'UNKNOWN', +}); +let staticIpState = STATIC_IP_STATE.UNKNOWN; +// The stability window is measured from here, not from lastIpChangeDate, which +// stays null until a change is actually seen. +let ipFirstSeenAt = null; const staticIpStabilityDays = 10; /** @@ -23,7 +48,7 @@ const staticIpStabilityDays = 10; * @param {boolean} isDataCenter - Whether the node is in a data center * @param {number|null} ipChangeDate - Timestamp of when the IP last changed */ -async function storeGeolocationToDb(geolocation, isStaticIp, isDataCenter, ipChangeDate) { +async function storeGeolocationToDb(geolocation, isStaticIp, isDataCenter, ipChangeDate, observations = {}) { try { const dbClient = dbHelper.databaseConnection(); if (!dbClient) { @@ -38,6 +63,11 @@ async function storeGeolocationToDb(geolocation, isStaticIp, isDataCenter, ipCha staticIp: isStaticIp, dataCenter: isDataCenter, lastIpChangeDate: ipChangeDate, + // The window is an observation, so it outlives the process: a restart + // that reset it would hold every node at UNKNOWN for another ten days. + ipFirstSeenAt: observations.ipFirstSeenAt ?? null, + staticIpState: observations.staticIpState ?? null, + networkEvidence: observations.networkEvidence ?? null, updatedAt: Date.now(), }, }; @@ -68,31 +98,200 @@ async function getGeolocationFromDb() { staticIp: result.staticIp || false, dataCenter: result.dataCenter || false, lastIpChangeDate: result.lastIpChangeDate || null, + ipFirstSeenAt: result.ipFirstSeenAt || null, + // `?? null`: a document written before these fields existed carries no + // observation, and saying so lets the next pass start the window. + staticIpState: result.staticIpState ?? null, + networkEvidence: result.networkEvidence ?? null, }; } - return { geolocation: null, staticIp: false, dataCenter: false, lastIpChangeDate: null }; + return { + geolocation: null, + staticIp: false, + dataCenter: false, + lastIpChangeDate: null, + ipFirstSeenAt: null, + staticIpState: null, + networkEvidence: null, + }; } catch (error) { log.error(`Failed to retrieve geolocation from database: ${error.message}`); - return { geolocation: null, staticIp: false, dataCenter: false, lastIpChangeDate: null }; + return { + geolocation: null, + staticIp: false, + dataCenter: false, + lastIpChangeDate: null, + ipFirstSeenAt: null, + staticIpState: null, + networkEvidence: null, + }; } } /** - * Method responsable for setting node geolocation information + * Reverse DNS for this node's own address - the strongest signal about the + * network that does not come from a vendor. + * @param {string} ip The node's public address. + * @returns {Promise} The first PTR name, or null when there is none. */ -async function setNodeGeolocation() { +async function resolvePtr(ip) { + try { + const names = await dns.reverse(ip); + return (names && names.length) ? names[0] : null; + } catch (error) { + // No PTR is ordinary - roughly a quarter of fleet hosts have none. It costs + // one signal, and the classifier decides on what remains. + return null; + } +} + +/** + * What the published location table says about an address. + * + * The table is the authority. It is built in the policy repo from evidence a + * node cannot gather for itself - chiefly the registries' own record of what a + * block was assigned for, which six thousand nodes cannot each go and fetch - + * and it is reviewed there with its reasons rather than derived here. + * + * TWO OUTCOMES, AND THEY MUST NOT BE CONFUSED: + * + * consulted: false the table could not be asked - none has been ingested + * yet, or the store could not be read. Nothing is known, + * and a node must reach no verdict at all. + * consulted: true the table answered. `classification` is its verdict, or + * null where it holds none for this address: no covering + * row, or an organisation the policy repo deliberately + * left unclassified. THAT null is an answer, and it is what + * hands the decision to the node's own evidence. + * + * Collapsing the two is the very mistake this whole classifier exists to + * avoid, one level up: "I have not asked" is not "there is no verdict". A node + * boots, fetches a 4.6 MB artifact, and ingests two million rows, while a + * single ip-api call answers in milliseconds - so the table is reliably absent + * at the moment a booting node would otherwise decide, and treating that as an + * abstention lets the node act on its own guess against a verdict the table + * was about to give it. + * @param {string} ip The node's public address. + * @returns {Promise<{consulted: boolean, classification: string|null}>} + */ +async function publishedClassification(ip) { + // Lazily required, like the benchmark service below: the location store + // pulls in the database layer, and geolocation is read on paths that must + // not depend on it being up. + // eslint-disable-next-line global-require + const ipLocationStore = require('./appPlacement/ipLocationStore'); + if (!ipLocationStore.status().ready) { + return { consulted: false, classification: null }; + } + try { + const hit = await ipLocationStore.lookup(ip); + return { consulted: true, classification: hit?.networkClass ?? null }; + } catch (error) { + // A table that cannot be read is a table that was not asked. Same as above: + // not evidence about the address, and not an abstention either. + log.info(`Location table could not answer for ${ip}: ${error.message}`); + return { consulted: false, classification: null }; + } +} + +/** + * Bench upload/download, for the link-asymmetry signal. Required lazily to keep + * geolocation off the benchmark service's load path. + * @returns {Promise<{uploadSpeed: number, downloadSpeed: number}>} Zeroes when + * bench cannot be read, which the classifier reads as no signal rather than as a + * symmetric link. + */ +async function benchLinkSpeeds() { + try { + // eslint-disable-next-line global-require + const benchmarkService = require('./benchmarkService'); + const response = await benchmarkService.getBenchmarks(); + if (!response || response.status !== 'success' || !response.data) { + return { uploadSpeed: 0, downloadSpeed: 0 }; + } + return { + uploadSpeed: response.data.upload_speed || 0, + downloadSpeed: response.data.download_speed || 0, + }; + } catch (error) { + return { uploadSpeed: 0, downloadSpeed: 0 }; + } +} + +// How long until the next pass, by outcome. +const AWAITING_IP_RETRY_MS = 10 * 1000; +const REFRESH_INTERVAL_MS = 3 * 24 * 60 * 60 * 1000; +const FAILURE_RETRY_MS = 5 * 60 * 1000; + +// THE LOOP IS A SINGLETON, because it has two callers and it reschedules itself. +// serviceManager starts it at boot and fluxNetworkHelper restarts it on EVERY IP +// change, and each pass used to arm a bare setTimeout whose handle nobody kept. +// So a second caller did not resume the loop, it forked another one, and no +// chain could ever be stopped: a node that changed IP three times ran four +// independent loops for the life of the process, each hitting the geolocation +// API and writing the same record, and on a node that never detected its IP each +// one logged an error every ten seconds. Only restarting FluxOS ended any of it. +// +// One handle, cancelled before it is re-armed, is what makes a second caller +// resume the loop instead of adding one. +let scheduledRun = null; +// The pass currently executing, and whether a caller arrived while it ran. That +// is the normal case rather than the exotic one - an IP change lands while the +// previous pass is still awaiting the geolocation API - and letting the two +// interleave would settle lastIpChangeDate on whichever finished last rather +// than on the order the addresses actually changed. The late caller is not +// dropped; it runs once the pass in flight is done. +let runInFlight = null; +let rerunRequested = false; + +/** + * Arms the next pass, cancelling any pass already pending. Every exit from + * runGeolocationPass goes through here, so there is at most one at any moment. + * @param {number} delayMs + */ +function scheduleNext(delayMs) { + if (scheduledRun) clearTimeout(scheduledRun); + scheduledRun = setTimeout(() => { + scheduledRun = null; + setNodeGeolocation(); + }, delayMs); +} + +/** + * Ends the geolocation loop. Exported because a timer nobody can stop is a leak + * wherever it runs - a node that wants to shut down cleanly, and a test that + * would otherwise leave a three-day timer armed for the rest of the run. + */ +function stopNodeGeolocation() { + if (scheduledRun) clearTimeout(scheduledRun); + scheduledRun = null; + rerunRequested = false; +} + +/** + * One pass: read the address, classify it, persist it, and arm the next. + */ +async function runGeolocationPass() { try { const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); if (!localSocketAddr) { log.error('Flux IP not detected. Flux geolocation service is awaiting'); - setTimeout(() => { - setNodeGeolocation(); - }, 10 * 1000); + scheduleNext(AWAITING_IP_RETRY_MS); return; } const localIp = extractIp(localSocketAddr); + // Restore what this node already observed before deciding anything from it. + // serviceManager calls this function directly at boot, with nothing having + // read the collection, so every module variable below starts null: the + // address the node held reads as "no previous address", a change across the + // restart is therefore invisible, and the write at the end of this pass + // persists lastIpChangeDate: null over a record that may go back years. + // getNodeGeolocation returns immediately once storedGeolocation is set, so + // this costs one read on the first pass and nothing after it. + await getNodeGeolocation(); + // Store previous IP to detect changes const previousIp = storedGeolocation ? storedGeolocation.ip : null; @@ -100,7 +299,11 @@ async function setNodeGeolocation() { log.info(`Checking geolocation of ${localIp}`); storedIp = localSocketAddr; // consider another service failover or stats db - const ipApiUrl = `${config.geolocation.ipApiBaseUrl}/json/${localIp}?fields=status,continent,continentCode,country,countryCode,region,regionName,lat,lon,query,org,isp,proxy,hosting`; + // `as` names the operator's own autonomous system, which does not vary + // with who registered a /29: across the fleet 227 ASNs separate hosting + // from access networks with only 8 carrying both, against 87 distinct + // `org` strings for the 329 nodes this decides about. + const ipApiUrl = `${config.geolocation.ipApiBaseUrl}/json/${localIp}?fields=status,continent,continentCode,country,countryCode,region,regionName,lat,lon,query,org,isp,as,proxy,hosting,mobile`; const ipRes = await serviceHelper.axiosGet(ipApiUrl); if (ipRes.data.status === 'success' && ipRes.data.query !== '') { storedGeolocation = { @@ -114,11 +317,16 @@ async function setNodeGeolocation() { lat: ipRes.data.lat, lon: ipRes.data.lon, org: ipRes.data.org || ipRes.data.isp, + isp: ipRes.data.isp, + asn: ipRes.data.as, + mobile: ipRes.data.mobile, + proxy: ipRes.data.proxy, + hosting: ipRes.data.hosting, static: ipRes.data.proxy || ipRes.data.hosting, dataCenter: ipRes.data.hosting, }; } else { - const statsApiUrl = `${config.geolocation.statsApiBaseUrl}/fluxlocation/${localIp}`; + const statsApiUrl = `${config.stats.baseUrl}/fluxlocation/${localIp}`; const statsRes = await serviceHelper.axiosGet(statsApiUrl); if (statsRes.data.status === 'success' && statsRes.data.data) { storedGeolocation = { @@ -142,101 +350,207 @@ async function setNodeGeolocation() { } log.info(`Geolocation of ${localIp} is ${JSON.stringify(storedGeolocation)}`); - // Check if IP has changed + // Static IP is observed, never inferred from the operator. A public address + // is trusted until this node WATCHES it move; the stability window is how + // one that moved earns the trust back, not a probation every node serves + // once. The whole rule is the table below. const currentIp = storedGeolocation.ip; const ipChanged = previousIp && previousIp !== currentIp; + const now = Date.now(); + const stabilityThreshold = staticIpStabilityDays * 24 * 60 * 60 * 1000; if (ipChanged) { - // IP changed - set static to false and record the change date - staticIp = false; - lastIpChangeDate = Date.now(); - log.info(`IP changed from ${previousIp} to ${currentIp}. Setting staticIp to false.`); - } else { - // IP has not changed - check static IP conditions - const hasPublicIp = await fluxNetworkHelper.hasPublicIpOnInterface(); - const now = Date.now(); - const stabilityThreshold = staticIpStabilityDays * 24 * 60 * 60 * 1000; - - // If lastIpChangeDate is null (never recorded), consider IP as stable for more than 10 days - const effectiveLastIpChangeDate = lastIpChangeDate || (now - stabilityThreshold - 1); - const daysSinceChange = (now - effectiveLastIpChangeDate) / (24 * 60 * 60 * 1000); - - // Determine static IP status based on multiple signals - if (hasPublicIp) { - // Has public IP on interface - strong indicator of static IP - if (now - effectiveLastIpChangeDate >= stabilityThreshold) { - // IP stable for 10+ days with public IP on interface - definitely static - staticIp = true; - if (lastIpChangeDate) { - log.info(`Node has public IP on interface and IP stable for ${daysSinceChange.toFixed(1)} days. Setting staticIp to true.`); - } else { - log.info('Node has public IP on interface and no IP change recorded. Assuming stable IP. Setting staticIp to true.'); - } - } else { - // Has public IP but hasn't been stable long enough yet - // Check other signals (API and org-based) - staticIp = false; - if (storedGeolocation.static) { - staticIp = true; - } else if (storedGeolocation.org) { - for (let i = 0; i < staticIpOrgs.length; i += 1) { - const org = staticIpOrgs[i]; - if (storedGeolocation.org.toLowerCase().includes(org)) { - staticIp = true; - break; - } - } - } - log.info(`Node has public IP on interface, IP stable for ${daysSinceChange.toFixed(1)} days (need ${staticIpStabilityDays}). staticIp=${staticIp}`); - } - } else { - // No public IP on interface - use API and org-based detection only - staticIp = false; - if (storedGeolocation.static) { - staticIp = true; - } else if (storedGeolocation.org) { - for (let i = 0; i < staticIpOrgs.length; i += 1) { - const org = staticIpOrgs[i]; - if (storedGeolocation.org.toLowerCase().includes(org)) { - staticIp = true; - break; - } - } - } - } + lastIpChangeDate = now; + ipFirstSeenAt = now; + log.info(`IP changed from ${previousIp} to ${currentIp}. Static IP observation restarts.`); + } else if (!ipFirstSeenAt) { + // Seeded from the last change on record rather than from now: an address + // held since a change 400 days ago has been held for 400 days, and that + // record is persisted and restored. Without this, an in-place upgrade + // introducing ipFirstSeenAt restarts the window on a node that has held + // its address for years. + ipFirstSeenAt = lastIpChangeDate ?? now; + log.info(`First observation of ${currentIp} by this build` + + `${lastIpChangeDate ? `, held since the change recorded ${new Date(lastIpChangeDate).toISOString()}` : ', with no change ever recorded'}.`); } - // Data center detection (unchanged logic) - if (storedGeolocation.dataCenter) { - dataCenter = true; + const hasPublicIp = await fluxNetworkHelper.hasPublicIpOnInterface(); + const heldForMs = now - ipFirstSeenAt; + const heldDays = heldForMs / (24 * 60 * 60 * 1000); + + // THE WHOLE DECISION, in one table, in the order it is asked. + // + // this node WATCHED public IP on verdict + // it change <10d ago the interface + // ------------------ ------------- ------- + // yes - UNKNOWN + // no yes STATIC + // no no (NAT) DYNAMIC + // no unreadable UNKNOWN + // + // A NODE IS STATIC ONLY IF IT HOLDS A PUBLIC ADDRESS ON AN INTERFACE. + // Nothing else confers it - not the published table, not an ip-api flag, + // not the operator's name. Every input here is something this node observed + // about itself. + // + // Only STATIC satisfies an app's `staticip` requirement; UNKNOWN and + // DYNAMIC both fail it, and are kept apart so a node that could not answer + // does not read as one that answered "behind NAT". + // + // `staticip` IS TWO PROMISES, AND BOTH ARE ANSWERED ABOVE. The apps that + // ask for it - VPN endpoints, chain nodes, bootstrap peers - need an + // endpoint that stays reachable at a fixed ip:port, because clients and + // peers hold that pair written down. So the node must be DIRECTLY CONNECTED + // (hasPublicIpOnInterface) and its address must not have been seen to MOVE + // (the watched-change window). A node reaching the world through a NAT port + // mapping keeps neither promise on its own account: 76% of fleet slots sit + // behind a UPnP router, and a mapping lapses on a router reboot, a lease + // expiry, or a firmware quirk while the address itself never moves - the + // fleet UPnP survey found 135 router models and defects in several. + // + // A watched change is asked FIRST and nothing overrides it: this node saw + // this address move, and that outranks seeing it on an interface now. + // + // Row 2 is the common case: a node has WATCHED no change because it started + // watching after the fact, which every node does exactly once. That is not + // evidence the address moves. Measured on the fleet an address changes on + // 0.29% of node-days, so treating not-yet-watched as suspect is wrong about + // ~97% of nodes for the whole ten days it lasts - and it lands on all of + // them together, because they upgrade together. An observed change is what + // withdraws the trust, and the window is how the address earns it back. + // + // A RANGE-LEVEL VERDICT CANNOT ANSWER EITHER PROMISE, which is why the + // published table is not consulted here even though it is loaded and + // authoritative elsewhere in this file. + // - It cannot see whether this host is directly connected. Across a + // random sample of 442 live slots, 126 of the 228 holding a static + // address by range verdict - 55% - are UPnP nodes on RFC1918 + // addresses. 112 qualify on ip-api `hosting`, 12 on `proxy`, which is + // a VPN artefact rather than an address that stays put. + // - It does not imply a fixed address either. "This block is assigned to + // a hosting company" states what the block is FOR. An operator + // reassigns on rebuild, migration, or a released elastic address, and + // the node returns on a different one - same hosting range, same + // DATACENTER verdict, different address. + // Both promises are properties of this host, so only this host can attest + // to them. The table stays authoritative for the residential verdict, which + // IS a question about the range. + // + // THE CASE THIS RULE KNOWINGLY REFUSES: genuine 1:1 NAT, where a fixed + // address is mapped to a host that never sees it locally. The same sample + // holds 4 such nodes (~49 slots fleet-wide) and NONE claims a static + // address; the fleet's entire hyperscaler population is one Oracle slot. + // Real, and priced at nil. + // + // Every figure above was measured on the live fleet by reproducing + // hasPublicIpOnInterface() across a random sample of slots; the census and + // its bounds are recorded with the change that introduced this rule. + const watchedItChange = Boolean(lastIpChangeDate) && heldForMs < stabilityThreshold; + + if (watchedItChange) { + // Asked first, and nothing overrides it: this node saw this address move, + // and it has not been held long enough since to say it has settled. + staticIpState = STATIC_IP_STATE.UNKNOWN; + } else if (hasPublicIp === true) { + staticIpState = STATIC_IP_STATE.STATIC; + } else if (hasPublicIp === false) { + // Behind NAT. The address may well be fixed upstream, but this node + // cannot see it, cannot attest to it, and reaches the world through a + // port mapping it does not control. + staticIpState = STATIC_IP_STATE.DYNAMIC; } else { - dataCenter = false; - if (storedGeolocation.org) { - for (let i = 0; i < staticIpOrgs.length; i += 1) { - const org = staticIpOrgs[i]; - if (storedGeolocation.org.toLowerCase().includes(org)) { - dataCenter = true; - break; - } - } - } + // The routing table could not be read. Not evidence of anything about the + // address, and in particular not evidence of NAT. + staticIpState = STATIC_IP_STATE.UNKNOWN; } + staticIp = staticIpState === STATIC_IP_STATE.STATIC; + log.info(`Static IP: ${staticIpState} (public IP on interface: ${hasPublicIp}` + + `, address held ${heldDays.toFixed(1)} of ${staticIpStabilityDays} days)`); + + // Whether the address is held (above) and what network it sits on (here) are + // separate questions, answered from separate evidence. + // + // This pass gathers the EVIDENCE and stops there. Reaching a verdict also + // needs the published table, and the table is not this pass's to wait for: + // it arrives in a 4.6 MB artifact the node is still ingesting while this + // runs, and a pass that recorded its own conclusion here would be recording + // "the table said nothing" and standing by it until the next pass, three + // days later. getNetworkClassification() reaches the verdict when asked, + // which is what every other consumer of that table already does. + const [ptr, linkSpeeds] = await Promise.all([ + resolvePtr(currentIp), + benchLinkSpeeds(), + ]); + const classified = networkClassifier.classifyNetwork({ + ptr, + hosting: storedGeolocation.hosting, + proxy: storedGeolocation.proxy, + mobile: storedGeolocation.mobile, + isp: storedGeolocation.isp, + asn: storedGeolocation.asn, + uploadSpeed: linkSpeeds.uploadSpeed, + downloadSpeed: linkSpeeds.downloadSpeed, + }); + + // One whole object, assigned once every input has settled: a reader must + // never combine evidence from a half-finished pass. + networkEvidence = Object.freeze({ + ip: currentIp, + classification: classified.classification, + evidenceFor: Object.freeze([...classified.evidenceFor]), + evidenceAgainst: Object.freeze([...classified.evidenceAgainst]), + ptr: ptr || null, + // Whether this pass had the signals that can contradict a residential + // reading. False on the stats.runonflux.io fallback, which carries none + // of them - and an empty evidenceAgainst from that path is nobody having + // looked, not nothing having been found. + contradictionSignalsGathered: classified.contradictionSignalsGathered, + gatheredAt: now, + }); + dataCenter = classified.classification === networkClassifier.CLASSIFICATION.DATACENTER; + log.info(`Network evidence for ${currentIp}: the node's own reading is ${classified.classification}` + + ` (for: ${classified.evidenceFor.join(', ') || 'none'};` + + ` against: ${classified.evidenceAgainst.join(', ') || 'none'}` + + `${classified.contradictionSignalsGathered ? '' : '; hosting/proxy/operator signals were NOT gathered'})`); // Store geolocation to database for persistence across restarts - await storeGeolocationToDb(storedGeolocation, staticIp, dataCenter, lastIpChangeDate); + await storeGeolocationToDb(storedGeolocation, staticIp, dataCenter, lastIpChangeDate, { + ipFirstSeenAt, + staticIpState, + networkEvidence, + }); execution += 1; - setTimeout(() => { // executes again in 3 days - setNodeGeolocation(); - }, 3 * 24 * 60 * 60 * 1000); + scheduleNext(REFRESH_INTERVAL_MS); } catch (error) { log.error(`Failed to get Geolocation with ${error}`); log.error(error); - setTimeout(() => { - setNodeGeolocation(); - }, 5 * 60 * 1000); + scheduleNext(FAILURE_RETRY_MS); } } +/** + * Method responsable for setting node geolocation information. Safe to call from + * anywhere, any number of times: it resumes the one loop rather than starting + * another, and never runs two passes at once. + */ +async function setNodeGeolocation() { + if (runInFlight) { + rerunRequested = true; + return runInFlight; + } + runInFlight = runGeolocationPass(); + try { + await runInFlight; + } finally { + runInFlight = null; + } + if (rerunRequested) { + rerunRequested = false; + return setNodeGeolocation(); + } + return undefined; +} + /** * Method responsible for getting stored node geolocation information. * If not available in memory, attempts to retrieve from database. @@ -249,29 +563,138 @@ async function getNodeGeolocation() { // Try to get from database if not in memory const dbData = await getGeolocationFromDb(); if (dbData.geolocation) { - storedGeolocation = dbData.geolocation; - staticIp = dbData.staticIp; - dataCenter = dbData.dataCenter; - lastIpChangeDate = dbData.lastIpChangeDate; + ({ + geolocation: storedGeolocation, + staticIp, + dataCenter, + lastIpChangeDate, + ipFirstSeenAt, + networkEvidence, + } = dbData); + // A record from before the state machine carries only the boolean, and the + // state must agree with it: a node that restores as not-static fails + // checkAppStaticIpRequirements in the redeploy paths, which remove the app + // when it throws. STATIC here is a carried prior the first refresh pass + // replaces; a stored false restores as UNKNOWN because an old record + // cannot tell dynamic from never-checked. + staticIpState = dbData.staticIpState + ?? (staticIp ? STATIC_IP_STATE.STATIC : STATIC_IP_STATE.UNKNOWN); log.info('Geolocation restored from database'); } return storedGeolocation; } /** - * Method responsible for returning if node ip is static based on IP org. + * Whether this node's address is known to stay put. True only when the address + * is bound to a local interface and has been held for the stability window; an + * address not yet observed that long is not static, so apps that require one are + * never placed on evidence the node does not have. + * @returns {boolean} */ function isStaticIP() { return staticIp; } /** - * Method responsible for returning if node is in a data center based on IP org. + * The three-state form of isStaticIP, for callers that need to tell "we have not + * watched it long enough" apart from "it moves". + * @returns {('STATIC'|'DYNAMIC'|'UNKNOWN')} + */ +function getStaticIpState() { + return staticIpState; +} + +/** + * Whether this node sits in a data centre. True only on a positive verdict from + * networkClassifier - CONFLICTED and UNKNOWN are both false, because neither is + * evidence of a data centre. + * @returns {boolean} */ function isDataCenter() { return dataCenter; } +/** + * The node's access-network verdict, reached now from the evidence it gathered + * and the table it currently holds. + * + * Reached here rather than stored because its two halves arrive at different + * times and on different clocks. The evidence is expensive and refreshed every + * three days; the published table is a local read that a booting node does not + * have yet and will have shortly. Deciding once, at the moment only one half + * exists, is how a node ends up acting for three days on a verdict the table + * would have overruled. + * + * NO TABLE MEANS NO VERDICT. Not a fallback to the node's own reading - the + * fallback belongs to a table that WAS consulted and holds nothing for this + * address. Until one has been ingested, this node knows nothing about which + * kind of network it is on, and null is the only honest answer. Nothing + * enforces on null. + * @returns {Promise<{classification: string, source: string, + * evidenceFor: string[], evidenceAgainst: string[], ptr: string|null, + * gatheredAt: number}|null>} Null when nothing has been gathered yet, when no + * location table has been consulted, or when the table holds no verdict for + * this address. `source` names the authority: 'published-table', or + * 'node-veto' where the node declined a published RESIDENTIAL on evidence + * about its own address. There is no 'node' source - a table carrying no + * verdict yields null, it does not hand the decision back to the node. + */ +async function getNetworkClassification() { + if (!networkEvidence) return null; + + // Nothing usable was gathered about what kind of network this is, so this + // node has no verdict - not even the table's. The veto below is the only + // thing that can decline a published RESIDENTIAL, and it fires on local + // evidence AGAINST; on a pass that never obtained any, it cannot fire, so a + // published verdict would go unchallenged precisely where challenging it + // matters. Returning null leaves the node unclassified, which enforces + // nothing and re-derives on the next pass that reaches ip-api. + if (!networkEvidence.contradictionSignalsGathered) return null; + + const published = await publishedClassification(networkEvidence.ip); + if (!published.consulted) return null; + + // THE TABLE DECIDES, OR NOBODY DOES. Where it carries no verdict this returns + // null and the node is simply not classified, which enforces nothing. + // + // It used to fall back to the node's own reading. That rule is the published + // rule with its strongest signal removed - six thousand nodes cannot each + // query the RIRs, so registration data belongs in the table - and its error + // rate has never been measured: 0.13% is reverse DNS alone and 0.00% is the + // combined rule WITH registration, and neither is what a node runs. It decided + // for the 13% of the enforceable population - 70 of the 541 slots whose bench + // confirms they are not ArcaneOS - whose organisation carries no published + // verdict, on the one path that deletes customer data. + // + // Tuning belongs in fluxos-network-policy, where a verdict is evidence-backed, + // auditable by anyone, correctable by hand through + // data/orgclass-overrides.json, and fixable without a FluxOS release. + if (!published.classification) return null; + + // The table decides, but a node that can see hosting evidence about its OWN + // address exempts itself. An organisation is decided by 80% of its hosts + // agreeing, so a minority tail of the other kind is guaranteed by + // construction - and this is how one of them declines a verdict meant for its + // neighbours, until someone adjudicates the range. The veto only ever removes + // a node from enforcement; local evidence can never impose one the table did + // not give. + const vetoed = published.classification === networkClassifier.CLASSIFICATION.RESIDENTIAL + && networkEvidence.evidenceAgainst.length > 0; + + return Object.freeze({ + classification: vetoed + ? networkClassifier.CLASSIFICATION.CONFLICTED + : published.classification, + // Which authority decided, so a verdict can be traced to the table that + // carried it or to the node declining one. + source: vetoed ? 'node-veto' : 'published-table', + evidenceFor: networkEvidence.evidenceFor, + evidenceAgainst: networkEvidence.evidenceAgainst, + ptr: networkEvidence.ptr, + gatheredAt: networkEvidence.gatheredAt, + }); +} + /** * Method responsible for returning the timestamp of when the IP last changed. * @returns {number|null} Timestamp of last IP change or null if not tracked yet @@ -290,9 +713,13 @@ async function hasPublicIp() { module.exports = { setNodeGeolocation, + stopNodeGeolocation, getNodeGeolocation, isStaticIP, + getStaticIpState, isDataCenter, + getNetworkClassification, getLastIpChangeDate, hasPublicIp, + STATIC_IP_STATE, }; diff --git a/ZelBack/src/services/idService.js b/ZelBack/src/services/idService.js index eca658f2dc..a850d1e6bc 100644 --- a/ZelBack/src/services/idService.js +++ b/ZelBack/src/services/idService.js @@ -6,14 +6,30 @@ const serviceHelper = require('./serviceHelper'); const messageHelper = require('./messageHelper'); const dbHelper = require('./dbHelper'); const verificationHelper = require('./verificationHelper'); +const verificationHelperUtils = require('./verificationHelperUtils'); const generalService = require('./generalService'); const dockerService = require('./dockerService'); const syncthingService = require('./syncthingService'); const fluxNetworkHelper = require('./fluxNetworkHelper'); const appInspector = require('./appManagement/appInspector'); const signatureVerifier = require('./signatureVerifier'); +const { Privilege, authOf } = require('./utils/privileges'); -const goodchars = /^[1-9a-km-zA-HJ-NP-Z]+$/; +/** + * What /id/checkprivilege answers. + * + * A published contract, not an internal name: the frontend branches on these + * four strings in fourteen places, in a separately deployed repo, and reads them + * out of the response body whether the status says success or error. They are + * deliberately not Privilege's values - that enum is what a route requires + * internally, and the two are free to diverge. + */ +const PRIVILEGE_RESPONSE = Object.freeze({ + NODE_OPERATOR: 'admin', + FLUX_TEAM: 'fluxteam', + USER: 'user', + NONE: 'none', +}); async function deleteLoginPhrase(phrase) { try { @@ -27,7 +43,6 @@ async function deleteLoginPhrase(phrase) { log.error(error); } } -const ethRegex = /^0x[a-fA-F0-9]{40}$/; let syncthingWorking = false; @@ -255,18 +270,7 @@ async function verifyLogin(req, res) { throw new Error('No Flux ID is specified'); } - if (address[0] !== '1' && address[0] !== '0') { - throw new Error('Flux ID is not valid'); - } - - if (address[0] === '1') { - if (!goodchars.test(address)) { - throw new Error('Flux ID is not valid'); - } - if (address.length > 34 || address.length < 25) { - throw new Error('Flux ID is not valid'); - } - } else if (!ethRegex.test(address)) { + if (!signatureVerifier.isValidSigningIdentity(address)) { throw new Error('Flux ID is not valid'); } @@ -316,12 +320,19 @@ async function verifyLogin(req, res) { createdAt, expireAt, }; - const userconfig = globalThis.userconfig; - let privilage = 'user'; - if (address === config.fluxTeamFluxID || address === config.fluxSupportTeamFluxID) { - privilage = 'fluxteam'; - } else if (address === userconfig.initial.zelid) { - privilage = 'admin'; + const adminZelid = verificationHelperUtils.nodeOperatorZelid(); + if (!adminZelid) { + // The node answers HTTP before it has read its own configuration, so + // this window is reachable on any restart. Granting 'user' here would + // hand the operator a session with their own rights stripped and no + // indication why; refusing says what is true and costs one retry. + throw new Error('Node is still starting and cannot establish privileges yet'); + } + let privilage = PRIVILEGE_RESPONSE.USER; + if (address === config.fluxTeamFluxID || verificationHelperUtils.isFluxSupportTeamZelid(address)) { + privilage = PRIVILEGE_RESPONSE.FLUX_TEAM; + } else if (address === adminZelid) { + privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; } const loggedUsersCollection = config.database.local.collections.loggedUsers; const value = newLogin; @@ -387,18 +398,7 @@ async function provideSign(req, res) { throw new Error('No Flux ID is specified'); } - if (address[0] !== '1' && address[0] !== '0') { - throw new Error('Flux ID is not valid'); - } - - if (address[0] === '1') { - if (!goodchars.test(address)) { - throw new Error('Flux ID is not valid'); - } - if (address.length > 34 || address.length < 25) { - throw new Error('Flux ID is not valid'); - } - } else if (!ethRegex.test(address)) { + if (!signatureVerifier.isValidSigningIdentity(address)) { throw new Error('Flux ID is not valid'); } @@ -446,7 +446,7 @@ async function provideSign(req, res) { */ async function activeLoginPhrases(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { const db = dbHelper.databaseConnection(); @@ -479,7 +479,7 @@ async function activeLoginPhrases(req, res) { */ async function loggedUsers(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); @@ -511,11 +511,11 @@ async function loggedUsers(req, res) { */ async function loggedSessions(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (authorized === true) { const db = dbHelper.databaseConnection(); - const auth = serviceHelper.ensureObject(req.headers.zelidauth); + const auth = serviceHelper.ensureObject(authOf(req)); const queryFluxID = auth.zelid; const database = db.db(config.database.local.database); const collection = config.database.local.collections.loggedUsers; @@ -546,9 +546,9 @@ async function loggedSessions(req, res) { */ async function logoutCurrentSession(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (authorized === true) { - const auth = serviceHelper.ensureObject(req.headers.zelidauth); + const auth = serviceHelper.ensureObject(authOf(req)); const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); const collection = config.database.local.collections.loggedUsers; @@ -581,7 +581,7 @@ async function logoutSpecificSession(req, res) { }); req.on('end', async () => { try { - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (authorized === true) { const processedBody = serviceHelper.ensureObject(body); const obtainedLoginPhrase = processedBody.loginPhrase; @@ -616,9 +616,9 @@ async function logoutSpecificSession(req, res) { */ async function logoutAllSessions(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('user', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); if (authorized === true) { - const auth = serviceHelper.ensureObject(req.headers.zelidauth); + const auth = serviceHelper.ensureObject(authOf(req)); const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); const collection = config.database.local.collections.loggedUsers; @@ -645,7 +645,7 @@ async function logoutAllSessions(req, res) { */ async function logoutAllUsers(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('admin', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, authOf(req)); if (authorized === true) { const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); @@ -701,12 +701,15 @@ async function wsRespondLoginPhrase(ws, loginphrase) { }); if (result) { // user is logged, all ok - const userconfig = globalThis.userconfig; - let privilage = 'user'; - if (result.zelid === config.fluxTeamFluxID || result.zelid === config.fluxSupportTeamFluxID) { - privilage = 'fluxteam'; - } else if (result.zelid === userconfig.initial.zelid) { - privilage = 'admin'; + const adminZelid = verificationHelperUtils.nodeOperatorZelid(); + if (!adminZelid) { + throw new Error('Node is still starting and cannot establish privileges yet'); + } + let privilage = PRIVILEGE_RESPONSE.USER; + if (result.zelid === config.fluxTeamFluxID || verificationHelperUtils.isFluxSupportTeamZelid(result.zelid)) { + privilage = PRIVILEGE_RESPONSE.FLUX_TEAM; + } else if (result.zelid === adminZelid) { + privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; } const resData = { message: 'Successfully logged in', @@ -825,6 +828,7 @@ async function wsRespondSignature(ws, message) { * @param {object} req Request. * @param {object} res Response. */ + async function checkLoggedUser(req, res) { let body = ''; req.on('data', (data) => { @@ -844,44 +848,51 @@ async function checkLoggedUser(req, res) { if (!signature) { throw new Error('No user Flux ID signature specificed'); } - const request = { - headers: { - zelidauth: { - zelid, - loginPhrase: loggedPhrase, - signature, - }, - }, - }; - const isAdmin = await verificationHelper.verifyPrivilege('admin', request); + // Serialised because a privilege check takes the header's value, and a + // header value is a string. ensureObject parses it back on the other side. + const zelidauth = JSON.stringify({ zelid, loginPhrase: loggedPhrase, signature }); + const isAdmin = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR, zelidauth); if (isAdmin) { - const message = messageHelper.createSuccessMessage('admin'); + const message = messageHelper.createSuccessMessage(PRIVILEGE_RESPONSE.NODE_OPERATOR); res.json(message); return; } - const isFluxTeam = await verificationHelper.verifyPrivilege('fluxteam', request); + const isFluxTeam = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, zelidauth); if (isFluxTeam) { - const message = messageHelper.createSuccessMessage('fluxteam'); + const message = messageHelper.createSuccessMessage(PRIVILEGE_RESPONSE.FLUX_TEAM); res.json(message); return; } - const isUser = await verificationHelper.verifyPrivilege('user', request); + const isUser = await verificationHelper.verifyPrivilege(Privilege.USER, zelidauth); if (isUser) { - const message = messageHelper.createSuccessMessage('user'); + const message = messageHelper.createSuccessMessage(PRIVILEGE_RESPONSE.USER); res.json(message); return; } - const message = messageHelper.createErrorMessage('none'); + const message = messageHelper.createErrorMessage(PRIVILEGE_RESPONSE.NONE); res.json(message); } catch (error) { log.error(error); - const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); + // `message` stays inside the PRIVILEGE_RESPONSE contract even here, because + // the frontend reads it AS the privilege whether the status says success or + // error, and logs the user out on exactly 'none' (fluxos-frontend + // guards.js). A thrown error's text landing in that field set the privilege + // to something like 'No user Flux ID specificed' - not 'none' - so the stale + // zelidauth stayed in localStorage instead of being cleared, and every later + // navigation repeated the same failure. Access still failed closed, since no + // route's privilege list contains an error string, but the session never + // ended. + // + // The error is not lost. It is logged above, and `name`/`code` still + // separate a failure from a plain refusal, which carries neither. + const errMessage = messageHelper.createErrorMessage(PRIVILEGE_RESPONSE.NONE, error.name, error.code); res.json(errMessage); } }); } module.exports = { + PRIVILEGE_RESPONSE, loginPhrase, emergencyPhrase, verifyLogin, diff --git a/ZelBack/src/services/imageUpdateService.js b/ZelBack/src/services/imageUpdateService.js index 1f1dee4144..f3180cd795 100644 --- a/ZelBack/src/services/imageUpdateService.js +++ b/ZelBack/src/services/imageUpdateService.js @@ -185,7 +185,7 @@ async function getRemoteManifestDigest(repotag, repoauth, specVersion, appName) const digest = await verifier.fetchManifestDigestOnly(); if (verifier.error) { - const errorMeta = verifier.errorMeta; + const { errorMeta } = verifier; if (errorMeta && errorMeta.errorType === 'rate_limit') { log.warn(`Rate limited while checking ${repotag}`); return { error: 'rate_limited', digest: null }; @@ -364,7 +364,12 @@ async function checkForImageUpdates() { } // Decrypt enterprise apps (version 8 with encrypted content) - const apps = await appQueryService.decryptEnterpriseApps(installedAppsResponse.data, { formatSpecs: false }); + const { readable: apps, unreadable } = await appQueryService.decryptEnterpriseApps(installedAppsResponse.data, { formatSpecs: false }); + if (unreadable.length) { + // their images are inside the blob - checking them would find none and + // read as up to date + log.warn(`Skipping image update checks for undecryptable apps: ${unreadable.map((app) => app.name).join(', ')}`); + } log.info(`Checking ${apps.length} installed apps for image updates`); let updatesTriggered = 0; diff --git a/ZelBack/src/services/messageHelper.js b/ZelBack/src/services/messageHelper.js index c4d97343d9..fe84893772 100644 --- a/ZelBack/src/services/messageHelper.js +++ b/ZelBack/src/services/messageHelper.js @@ -1,3 +1,34 @@ +// Which layer answers, and with what. +// +// A request that REACHED a handler is answered in the body, at HTTP 200, in the +// shapes below - including when the answer is a failure. A refusal that happens +// BEFORE a handler runs is answered with a wire status: the middlewares +// (requireHttps 403, routeGuards 503/400), a request too large to accept +// (paymentService 413), a resource that does not exist to be addressed +// (fluxEventBus 404). +// +// This is the rule to write NEW code to, not a description of what is already +// here. The codebase is not uniform: there are ~50 res.status( sites and a good +// share of them are inside handlers that ran, so they answer a service failure +// with a wire status. operationsController is both at once - it writes +// `res.status(200).json(createErrorMessage(...))` explicitly, refusing to let +// the two blur, and a few lines later answers 400/404/500 from inside a handler. +// +// The earlier version of this note put a count here - "roughly 315 in band +// against 27 that set a status, and the 27 are all of the second kind" - which +// was wrong, and wrong in the way its own last line warns about. +// +// The reason is the one dataOrThrow states below: the in-band shape exists so a +// transport failure cannot impersonate a service answer. An error carried at a +// non-200 status would put the two on the same channel again, which is what the +// separation is for. `code` inside an error message is therefore a FluxOS code +// and not a wire status - a deprecated endpoint answers 200 with code 410, and a +// caller reading response.ok sees success and has to read the body, which is +// exactly what every FluxOS client already does. +// +// Written down here because it was not written down anywhere, and counting the +// call sites without reading what they were gives the wrong answer. + /** * Creates a message object. * @@ -93,10 +124,30 @@ function errUnauthorizedMessage() { return errMessage; } +/** + * The data of an in-band service response, or a throw for an error message. + * Service-to-service consumers read through this so a transport failure + * converted to an in-band error shape can never impersonate an empty result: + * internal code reasons about data or an exception, never a shape union. The + * HTTP handler surface keeps returning the messages themselves. + * @param {object} response A createDataMessage/createErrorMessage-shaped object + * @returns {*} The success message's data + * @throws {Error} Carrying the error message's message, name and code + */ +function dataOrThrow(response) { + if (response && response.status === 'success') return response.data; + const details = (response && typeof response.data === 'object' && response.data) || {}; + const error = new Error(details.message || 'service request failed'); + if (details.name) error.name = details.name; + if (details.code !== undefined) error.code = details.code; + throw error; +} + module.exports = { createDataMessage, createErrorMessage, createSuccessMessage, createWarningMessage, + dataOrThrow, errUnauthorizedMessage, }; diff --git a/ZelBack/src/services/networkStateService.js b/ZelBack/src/services/networkStateService.js index 2d5a09fe9f..c5eb5b8892 100644 --- a/ZelBack/src/services/networkStateService.js +++ b/ZelBack/src/services/networkStateService.js @@ -1,5 +1,6 @@ const daemonServiceFluxnodeRpcs = require('./daemonService/daemonServiceFluxnodeRpcs'); const networkStateManager = require('./utils/networkStateManager'); +const fluxEventBus = require('./utils/fluxEventBus'); /** * @typedef {import('./utils/networkStateManager').Fluxnode} Fluxnode @@ -13,6 +14,14 @@ const networkStateManager = require('./utils/networkStateManager'); */ let stateManager = null; +/** + * Resolves once the node list has been fetched and indexed. It exists before + * start() is called, so a caller that arrives early waits for the state rather + * than being handed an empty list and reading it as a network with no nodes. + */ +let resolveStarted; +let started = new Promise((resolve) => { resolveStarted = resolve; }); + /** * Throttle state for daemon RPC calls */ @@ -70,9 +79,20 @@ async function start(options = {}) { stateManager.once('populated', () => { clearTimeout(timeout); + resolveStarted(); resolve(); }); + // Every refresh, with what the node now believes the fleet to be. Nothing in + // production consumes it - the bus is test-only - and it exists because a + // test that changes the node list otherwise has no way to know when this + // node has read the change: the list is polled on a timer, and the only + // endpoints that report one ask the daemon rather than this cache. Sleeping + // long enough instead is the thing that turns into a flaky suite. + stateManager.on('updated', () => { + fluxEventBus.publish('networkstate:updated', { nodes: stateManager.nodeCount }); + }); + setImmediate(() => stateManager.start()); }); } @@ -86,6 +106,7 @@ async function stop() { await stateManager.stop(); stateManager = null; + started = new Promise((resolve) => { resolveStarted = resolve; }); } /** @@ -103,10 +124,48 @@ function networkState(options = {}) { return state; } -async function waitStarted() { - if (!stateManager) return; +/** + * Whether the node list has been fetched and indexed. + * + * The bulk accessors - networkState(), nodeCount() - answer an unknown state + * and a genuinely empty one with the same value, so a caller that would read + * those differently asks this first rather than guessing. The lookups that + * answer a question about one node do not: they wait for the list rather than + * report it absent from a state that has never held it. + * @returns {boolean} + */ +function isReady() { + return Boolean(stateManager && stateManager.started); +} - await stateManager.waitStarted; +/** + * Runs a callback once the network state is known, immediately if it already is. + * + * This is for work that cannot begin without the node list, so that it starts + * when the list arrives rather than whenever the next poll happens to come + * round. Callers on a schedule of their own should use isReady() instead. + * @param {Function} callback + * @returns {void} + */ +function onReady(callback) { + if (isReady()) { + callback(); + return; + } + + started.then(callback); +} + +/** + * Waits until the network state is known. + * + * Only for one-shot startup gates. Anything on a repeating schedule must use + * isReady() - several of those only re-arm once they have finished, so awaiting + * here would retire them for the life of the process rather than delay them. + * @returns {Promise} + */ +async function waitStarted() { + await started; } function nodeCount() { @@ -167,6 +226,19 @@ async function getRandomSocketAddress(socketAddress) { return random; } +/** + * A random node that can observe this one from outside its address - i.e. not a + * Flux node sharing our public address. Null when there is no such node. + * + * @param {string} socketAddress + * @returns {Promise} + */ +async function getRandomExternalObserver(socketAddress, options = {}) { + if (!stateManager) return null; + + return stateManager.getRandomExternalObserver(socketAddress, options); +} + /** * * @param {string} socketAddress @@ -200,8 +272,11 @@ module.exports = { getFluxnodeBySocketAddress, getFluxnodesByPubkey, getRandomSocketAddress, + getRandomExternalObserver, + isReady, networkState, nodeCount, + onReady, pubkeyInNetworkState, socketAddressInNetworkState, start, diff --git a/ZelBack/src/services/pgpService.js b/ZelBack/src/services/pgpService.js index 9e0da4e16a..15c2040d4f 100644 --- a/ZelBack/src/services/pgpService.js +++ b/ZelBack/src/services/pgpService.js @@ -1,24 +1,30 @@ const config = require('config'); const path = require('path'); const fs = require('fs').promises; -const openpgp = require('openpgp'); const generalService = require('./generalService'); +const workerRunner = require('./utils/workerRunner'); +const configManager = require('./utils/configManager'); const log = require('../lib/log'); +const runPgp = (operation, params) => workerRunner.runInWorker('pgpWorker', { operation, params }); + /** - * To adjust PGP identity + * To adjust PGP identity. The file is this node's only record of its keypair, + * so the in-process copy is refreshed from it here. A caller that stores an + * identity and then reads the previous one back cannot tell that the write + * happened, and every other writer of this file rebuilds it from the same + * in-process copy - so a stale one puts the replaced keypair straight back. * @param {string} privateKey Armored version of private key * @param {string} publicKey Armored version of public key - * @returns {void} Return statement is only used here to interrupt the function and nothing is returned. + * @returns {Promise} Rejects if the identity could not be stored. */ async function adjustPGPidentity(privateKey, publicKey) { - try { - const fluxDirPath = path.join(__dirname, '../../../config/userconfig.js'); - if (publicKey === userconfig.initial.pgpPublicKey && privateKey === userconfig.initial.pgpPrivateKey) { - return; - } - log.info(`Adjusting Identity to ${publicKey}`); - const dataToWrite = `module.exports = { + const fluxDirPath = path.join(__dirname, '../../../config/userconfig.js'); + if (publicKey === userconfig.initial.pgpPublicKey && privateKey === userconfig.initial.pgpPrivateKey) { + return; + } + log.info(`Adjusting Identity to ${publicKey}`); + const dataToWrite = `module.exports = { initial: { ipaddress: '${userconfig.initial.ipaddress || '127.0.0.1'}', zelid: '${userconfig.initial.zelid || config.fluxTeamFluxID}', @@ -34,62 +40,100 @@ async function adjustPGPidentity(privateKey, publicKey) { } }`; - await fs.writeFile(fluxDirPath, dataToWrite); - } catch (error) { - log.error(error); - } + await fs.writeFile(fluxDirPath, dataToWrite); + configManager.reloadConfig(); +} + +/** + * The private key this node has already been shown to hold the public half of. + * @type {string | null} + */ +let verifiedPrivateKey = null; + +/** + * The identity repair that is running, if any. Every caller relying on the + * private key meets the same corrupt pair, and a repair rewrites + * config/userconfig.js whole - a file that is emptied before it is written, and + * that a reader landing mid-write finds carrying no identity at all. Callers + * share one repair rather than each running theirs over the top of the others. + * @type {Promise | null} + */ +let repairInFlight = null; + +/** + * @returns {void} + */ +function clearRepairInFlight() { + repairInFlight = null; +} + +/** + * To generate a keypair and store it as this node's identity + * @returns {Promise} + */ +async function createIdentity() { + const collateralInfo = await generalService.obtainNodeCollateralInformation(); + // userId name is our txid:outputid + // userId email is our zelid@runonflux.io + const email = `${userconfig.initial.zelid}@runonflux.io`; // 1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC@runonflux.io + const name = `${collateralInfo.txhash}:${collateralInfo.txindex}`; // '0000000567ad22d02e3fc7631d94eb0dac5f1d5eb4adbd63349766f2665640c6:0' + const keypair = await runPgp('generateKey', { name, email }); + await adjustPGPidentity(keypair.privateKey, keypair.publicKey); + // the halves were generated together, so the check this saves has one answer + verifiedPrivateKey = keypair.privateKey; + log.info('PGP identity generated'); } /** - * To check if correct pgp identity exists + * To replace the stored keypair if the public key does not belong to the + * private key. Anything encrypted to a public key whose private half we do not + * hold is unreadable, so a mismatch is repaired rather than reported. + * + * A pair is written by adjustPGPidentity in one go and does not drift, so this + * guards against the config file having been corrupted or hand-edited. It runs + * when the private key is first relied on rather than at boot, because every + * openpgp operation costs a worker and a fresh load of the library - the entire + * cost of the check - and the answer for a key that has not changed is known. + * @returns {Promise} */ -async function identityExists() { +async function ensureIdentityVerified() { try { - // only generate new identity if private key or public key is missing, do not match - const existingPrivateKey = userconfig.initial.pgpPrivateKey; - const existingPublicKey = userconfig.initial.pgpPublicKey; - if (existingPrivateKey && existingPublicKey) { - // check if public key belongs to our private key - const privateKey = await openpgp.readPrivateKey({ armoredKey: existingPrivateKey }); - const publicKey = privateKey.toPublic().armor(); - if (publicKey !== existingPublicKey) { - log.warn('Existing PGP identity is corrupted. Generating new identity'); - return false; - } - return true; + const privateKey = userconfig.initial.pgpPrivateKey; + const publicKey = userconfig.initial.pgpPublicKey; + if (!privateKey || !publicKey || verifiedPrivateKey === privateKey) return; + + const derived = await runPgp('derivePublicKey', { armoredPrivateKey: privateKey }); + if (derived === publicKey) { + verifiedPrivateKey = privateKey; + return; } - log.info('PGP identity does not exist. Proceeding with generation'); - return false; + + if (!repairInFlight) { + log.warn('Existing PGP identity is corrupted. Generating new identity'); + log.warn('Whatever was sealed to the previous public key was never readable here and has to be published again'); + repairInFlight = createIdentity().finally(clearRepairInFlight); + } + await repairInFlight; } catch (error) { + // an identity that could not be checked is not a reason to refuse the work + // that prompted the check - a genuinely broken key fails the decrypt itself log.error(error); - log.info('PGP identity error. Generating new identity'); - return false; } } /** - * To generate and store new identity + * To give the node a PGP identity if it does not have one. A node that already + * carries a keypair needs nothing here - the pair is verified by + * ensureIdentityVerified when something first relies on it, so boot neither + * loads openpgp nor waits for it. + * @returns {Promise} */ async function generateIdentity() { try { - const currentIdentityExists = await identityExists(); - if (currentIdentityExists) { - return; - } - const collateralInfo = await generalService.obtainNodeCollateralInformation(); - // userId name is our txid:outputid - // userId email is our zelid@runonflux.io - const email = `${userconfig.initial.zelid}@runonflux.io`; // 1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC@runonflux.io - const name = `${collateralInfo.txhash}:${collateralInfo.txindex}`; // '0000000567ad22d02e3fc7631d94eb0dac5f1d5eb4adbd63349766f2665640c6:0' - const keypair = await openpgp.generateKey({ - type: 'ecc', // Type of the key, defaults to ECC - curve: 'curve25519', // ECC curve name, defaults to curve25519 - userIDs: [{ name, email }], // you can pass multiple user IDs - passphrase: '', // no password - format: 'armored', // output key format, defaults to 'armored' (other options: 'binary' or 'object') - }); - await adjustPGPidentity(keypair.privateKey, keypair.publicKey); - log.info('PGP identity generated'); + if (userconfig.initial.pgpPrivateKey && userconfig.initial.pgpPublicKey) return; + + log.info('PGP identity does not exist. Proceeding with generation'); + await createIdentity(); } catch (error) { log.error('Identity generation error'); log.error(error); @@ -104,15 +148,8 @@ async function generateIdentity() { */ async function encryptMessage(message, encryptionKeys) { try { - const publicKeys = await Promise.all(encryptionKeys.map((armoredKey) => openpgp.readKey({ armoredKey }))); - - const pgpMessage = await openpgp.createMessage({ text: message }); - const encryptedMessage = await openpgp.encrypt({ - message: pgpMessage, // input as Message object - encryptionKeys: publicKeys, - }); // '-----BEGIN PGP MESSAGE ... END PGP MESSAGE-----' - return encryptedMessage; + return await runPgp('encrypt', { message, encryptionKeys }); } catch (error) { log.error(error); return null; @@ -125,17 +162,15 @@ async function encryptMessage(message, encryptionKeys) { * @param {string} decryptionKey Armored version of private key * @returns {Promise} Return plain text message */ -async function decryptMessage(encryptedMessage, decryptionKey = userconfig.initial.pgpPrivateKey) { +async function decryptMessage(encryptedMessage, decryptionKey = null) { try { - const messageEncrypted = await openpgp.readMessage({ - armoredMessage: encryptedMessage, // parse armored message - }); - const privateKey = await openpgp.readPrivateKey({ armoredKey: decryptionKey }); - const decryptedMessage = await openpgp.decrypt({ - message: messageEncrypted, - decryptionKeys: privateKey, - }); - return decryptedMessage.data; + // this node's own key is the one that could be corrupt, so it is checked + // before it is used; a caller supplying a key has vouched for it already + if (!decryptionKey) await ensureIdentityVerified(); + + const key = decryptionKey ?? userconfig.initial.pgpPrivateKey; + + return await runPgp('decrypt', { encryptedMessage, decryptionKey: key }); } catch (error) { log.error(error); return null; diff --git a/ZelBack/src/services/registryAuth/providers/awsEcrAuthProvider.js b/ZelBack/src/services/registryAuth/providers/awsEcrAuthProvider.js index 2bc4415a85..b4e2f8c463 100644 --- a/ZelBack/src/services/registryAuth/providers/awsEcrAuthProvider.js +++ b/ZelBack/src/services/registryAuth/providers/awsEcrAuthProvider.js @@ -6,14 +6,13 @@ * AWS credentials and environment variable/IAM role authentication. */ -// eslint-disable-next-line import/no-unresolved -const { ECRClient, DescribeRepositoriesCommand, GetAuthorizationTokenCommand } = require('@aws-sdk/client-ecr'); +const workerRunner = require('../../utils/workerRunner'); const { RegistryAuthProvider } = require('./base/registryAuthProvider'); class AwsEcrAuthProvider extends RegistryAuthProvider { constructor(config, appName) { super(config, appName); - this.ecrClient = null; + this.clientConfig = null; this.ecrRegion = config.region || process.env.AWS_DEFAULT_REGION; // Initialize ECR client @@ -43,7 +42,7 @@ class AwsEcrAuthProvider extends RegistryAuthProvider { clientConfig.credentials.sessionToken = this.config.sessionToken; } - this.ecrClient = new ECRClient(clientConfig); + this.clientConfig = clientConfig; } catch (error) { const wrappedError = new Error(`Failed to initialize AWS ECR client: ${error.message}`); this.recordError(wrappedError); @@ -51,6 +50,21 @@ class AwsEcrAuthProvider extends RegistryAuthProvider { } } + /** + * Run a single ECR call against the configured registry. + * + * @param {string} operation ECR operation name. + * @param {object} params Parameters for that operation. + * @returns {Promise} The fields of the ECR response the provider uses. + */ + async runEcrCommand(operation, params) { + return workerRunner.runInWorker('awsEcrAuthWorker', { + operation, + clientConfig: this.clientConfig, + params, + }); + } + /** * Get ECR authentication credentials * Returns cached token if valid, otherwise fetches a new one @@ -81,7 +95,7 @@ class AwsEcrAuthProvider extends RegistryAuthProvider { * @returns {Promise} Fresh ECR credentials */ async refreshCredentials() { - if (!this.ecrClient) { + if (!this.clientConfig) { const error = new Error('ECR client not initialized'); this.recordError(error); throw error; @@ -93,9 +107,7 @@ class AwsEcrAuthProvider extends RegistryAuthProvider { commandParams.registryIds = this.config.registryIds; } - const command = new GetAuthorizationTokenCommand(commandParams); - - const response = await this.ecrClient.send(command); + const response = await this.runEcrCommand('getAuthorizationToken', commandParams); if (!response.authorizationData || response.authorizationData.length === 0) { throw new Error('No authorization data received from ECR'); @@ -274,7 +286,7 @@ class AwsEcrAuthProvider extends RegistryAuthProvider { return { ...baseError, region: this.ecrRegion, - clientInitialized: Boolean(this.ecrClient), + clientInitialized: Boolean(this.clientConfig), configurationValid: this.validateConfiguration(), credentialSources: { explicit: Boolean(this.config.accessKeyId && this.config.secretAccessKey), @@ -297,11 +309,7 @@ class AwsEcrAuthProvider extends RegistryAuthProvider { } try { - const command = new DescribeRepositoriesCommand({ - maxResults: 1, // Minimal request - }); - - await this.ecrClient.send(command); + await this.runEcrCommand('describeRepositories', { maxResults: 1 }); return true; } catch (error) { this.recordError(error); diff --git a/ZelBack/src/services/registryAuth/providers/azureAcrAuthProvider.js b/ZelBack/src/services/registryAuth/providers/azureAcrAuthProvider.js index 3ba1dd7cae..6167483718 100644 --- a/ZelBack/src/services/registryAuth/providers/azureAcrAuthProvider.js +++ b/ZelBack/src/services/registryAuth/providers/azureAcrAuthProvider.js @@ -10,14 +10,15 @@ * This matches the authentication pattern used by AWS ECR and Google GAR providers. */ -// eslint-disable-next-line import/no-unresolved -const { ClientSecretCredential } = require('@azure/identity'); +const workerRunner = require('../../utils/workerRunner'); const { RegistryAuthProvider } = require('./base/registryAuthProvider'); +const AAD_SCOPES = ['https://containerregistry.azure.net/.default']; + class AzureAcrAuthProvider extends RegistryAuthProvider { constructor(config, appName) { super(config, appName); - this.azureCredential = null; + this.clientInitialized = false; // Extract registry name from config if provided // Config can have: registryName (explicit) or registry (URL to parse) @@ -29,7 +30,8 @@ class AzureAcrAuthProvider extends RegistryAuthProvider { } /** - * Initialize the Azure Identity client with service principal credentials + * Confirm the service principal is complete enough to authenticate with. + * The Azure client itself is built inside the worker that uses it. */ initializeClient() { try { @@ -37,12 +39,7 @@ class AzureAcrAuthProvider extends RegistryAuthProvider { throw new Error('Service principal credentials (clientId, clientSecret, tenantId) are required'); } - // Create Azure ClientSecretCredential with service principal - this.azureCredential = new ClientSecretCredential( - this.config.tenantId, - this.config.clientId, - this.config.clientSecret, - ); + this.clientInitialized = true; } catch (error) { const wrappedError = new Error(`Failed to initialize Azure ACR client: ${error.message}`); // Only record error if provider name is set (avoid error in tests) @@ -53,6 +50,20 @@ class AzureAcrAuthProvider extends RegistryAuthProvider { } } + /** + * Obtain an Azure AD access token scoped to container registry access. + * + * @returns {Promise<{token: string, expiresOnTimestamp: number}|null>} Token response, or null if Azure returned none. + */ + async fetchAadToken() { + return workerRunner.runInWorker('azureAcrAuthWorker', { + tenantId: this.config.tenantId, + clientId: this.config.clientId, + clientSecret: this.config.clientSecret, + scopes: AAD_SCOPES, + }); + } + /** * Get ACR authentication credentials * Returns cached token if valid, otherwise fetches a new one @@ -199,7 +210,7 @@ class AzureAcrAuthProvider extends RegistryAuthProvider { * @returns {Promise} Fresh ACR credentials */ async refreshCredentials() { - if (!this.azureCredential) { + if (!this.clientInitialized) { const error = new Error('Azure credential not initialized'); this.recordError(error); throw error; @@ -208,9 +219,7 @@ class AzureAcrAuthProvider extends RegistryAuthProvider { try { // Step 1: Get Azure AD access token with correct scope for container registry // This is the CORRECT scope - not management.azure.com! - const tokenResponse = await this.azureCredential.getToken([ - 'https://containerregistry.azure.net/.default', - ]); + const tokenResponse = await this.fetchAadToken(); if (!tokenResponse || !tokenResponse.token) { throw new Error('No access token received from Azure Identity'); @@ -396,7 +405,7 @@ class AzureAcrAuthProvider extends RegistryAuthProvider { tenantId: this.config.tenantId, clientId: this.config.clientId, registryName: this.registryName, - azureCredentialInitialized: Boolean(this.azureCredential), + azureCredentialInitialized: Boolean(this.clientInitialized), configurationValid: this.validateConfiguration(), credentialSources: { hasClientId: Boolean(this.config.clientId), @@ -419,9 +428,7 @@ class AzureAcrAuthProvider extends RegistryAuthProvider { try { // Test by attempting to get an access token with correct scope - const tokenResponse = await this.azureCredential.getToken([ - 'https://containerregistry.azure.net/.default', - ]); + const tokenResponse = await this.fetchAadToken(); return Boolean(tokenResponse && tokenResponse.token); } catch (error) { this.recordError(error); diff --git a/ZelBack/src/services/registryAuth/providers/googleGarAuthProvider.js b/ZelBack/src/services/registryAuth/providers/googleGarAuthProvider.js index 2622f17a71..f3a4ed20ed 100644 --- a/ZelBack/src/services/registryAuth/providers/googleGarAuthProvider.js +++ b/ZelBack/src/services/registryAuth/providers/googleGarAuthProvider.js @@ -6,14 +6,15 @@ * to generate short-lived tokens for enhanced security over static JSON keys. */ -// eslint-disable-next-line import/no-unresolved -const { JWT } = require('google-auth-library'); +const workerRunner = require('../../utils/workerRunner'); const { RegistryAuthProvider } = require('./base/registryAuthProvider'); +const GAR_SCOPES = ['https://www.googleapis.com/auth/cloud-platform']; + class GoogleGarAuthProvider extends RegistryAuthProvider { constructor(config, appName) { super(config, appName); - this.jwtClient = null; + this.clientInitialized = false; // Initialize JWT client this.initializeClient(); @@ -33,12 +34,7 @@ class GoogleGarAuthProvider extends RegistryAuthProvider { throw new Error('Service account credentials are required. Provide keyFile (base64-encoded JSON)'); } - // Create JWT client with service account credentials - this.jwtClient = new JWT({ - email: this.config.clientEmail, - key: this.config.privateKey, - scopes: ['https://www.googleapis.com/auth/cloud-platform'], - }); + this.clientInitialized = true; } catch (error) { const wrappedError = new Error(`Failed to initialize Google GAR client: ${error.message}`); // Only record error if provider name is set (avoid error in tests) @@ -114,13 +110,26 @@ class GoogleGarAuthProvider extends RegistryAuthProvider { return credentials; } + /** + * Mint an OAuth access token for the configured service account. + * + * @returns {Promise<{token: string|null, expiryDate: number|null}>} Token and its expiry. + */ + async fetchAccessToken() { + return workerRunner.runInWorker('googleGarAuthWorker', { + clientEmail: this.config.clientEmail, + privateKey: this.config.privateKey, + scopes: GAR_SCOPES, + }); + } + /** * Refresh GAR OAuth access token from Google * * @returns {Promise} Fresh GAR credentials */ async refreshCredentials() { - if (!this.jwtClient) { + if (!this.clientInitialized) { const error = new Error('JWT client not initialized'); this.recordError(error); throw error; @@ -128,19 +137,19 @@ class GoogleGarAuthProvider extends RegistryAuthProvider { try { // Get access token from Google - const tokens = await this.jwtClient.getAccessToken(); + const tokens = await this.fetchAccessToken(); if (!tokens.token) { throw new Error('No access token received from Google Auth'); } // Validate that expiry time is provided by Google Auth Library - if (!this.jwtClient.credentials.expiry_date) { + if (!tokens.expiryDate) { throw new Error('Google Auth Library did not provide token expiry time'); } // Use only the actual expiry time from Google Auth Library - const expiryTime = this.jwtClient.credentials.expiry_date; + const expiryTime = tokens.expiryDate; // Create standardized credentials for Docker authentication // Google GAR expects: username = "oauth2accesstoken", password = access_token @@ -294,7 +303,7 @@ class GoogleGarAuthProvider extends RegistryAuthProvider { return { ...baseError, clientEmail: this.config.clientEmail, - jwtClientInitialized: Boolean(this.jwtClient), + jwtClientInitialized: Boolean(this.clientInitialized), configurationValid: this.validateConfiguration(), credentialSources: { hasPrivateKey: Boolean(this.config.privateKey), @@ -316,7 +325,7 @@ class GoogleGarAuthProvider extends RegistryAuthProvider { try { // Test by attempting to get an access token - const tokens = await this.jwtClient.getAccessToken(); + const tokens = await this.fetchAccessToken(); return Boolean(tokens.token); } catch (error) { this.recordError(error); diff --git a/ZelBack/src/services/residentialNodeDosService.js b/ZelBack/src/services/residentialNodeDosService.js new file mode 100644 index 0000000000..ee28d779b1 --- /dev/null +++ b/ZelBack/src/services/residentialNodeDosService.js @@ -0,0 +1,890 @@ +// A node on a residential connection is only fit to serve the network when it +// runs ArcaneOS. This service moves such a node off the network in three stages: +// +// HOLD it stops accepting NEW apps. Immediate, and it deletes nothing. +// EVACUATE it gives up one app at a time, and only ones another host +// demonstrably holds. Each departure leaves the app one short, the +// spawner replaces it on a node that is not held, and that is what +// releases the next holder. The removing is done by the single +// give-up-an-app pass in advancedWorkflows; this service owns only the +// policy and the pacing. +// DOS once the node runs nothing, the sticky DOS goes on. By then +// removeAllAppsLocally has nothing to find. +// +// DOS >= 100 is not a mark: it makes nodeStatusMonitor and appStartupManager +// `rm -rf` every app directory and volume on the box. Reaching that state only +// on an empty node is the whole point of the staging. +// +// Nothing here enforces against a node that is not PROVABLY residential. +// geolocationService's classification is four-state and only RESIDENTIAL acts: +// CONFLICTED and UNKNOWN are left alone, as is a node whose bench cannot be read. + +const config = require('config'); +const log = require('../lib/log'); +const dbHelper = require('./dbHelper'); +const fluxNetworkHelper = require('./fluxNetworkHelper'); +const geolocationService = require('./geolocationService'); +const benchmarkService = require('./benchmarkService'); +const { CLASSIFICATION } = require('./utils/networkClassifier'); +const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('./utils/appSyncEvents'); +const globalState = require('./utils/globalState'); +const { compareInstanceSeniority } = require('./utils/instanceOrdering'); +const { socketAddressesMatch } = require('./utils/socketAddressUtils'); +const fluxEventBus = require('./utils/fluxEventBus'); + +const DOS_MESSAGE_PREFIX = 'Residential node not running ArcaneOS'; +const HOLD_REASON = 'residential node not running ArcaneOS'; + +const CHECK_INTERVAL_MS = config.fluxapps.residentialCheckIntervalMs; +const RETRY_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes +// Before the first app is given up, the verdict must have held this long. +// The placement hold deletes nothing and needs no window; this paces only the +// part that moves customer data, so a momentary misread can correct itself. +const SETTLE_MS = config.fluxapps.residentialSettleMs; +// The window counts time this node OBSERVED the verdict, not time that passed. +// A tick that cannot decide - an unreadable bench, a table that will not load - +// returns without touching state, so measuring from the first verdict alone let +// two evaluations 24h apart satisfy a window meant to prove the verdict held +// throughout. Silence is not agreement, and the node that cannot read its own +// hardware is the one to be most careful with. +// +// Each confirming tick credits the time since the last one, capped at a single +// check interval so a long-ish gap cannot buy more than a tick's worth, and +// credited as nothing at all once the gap is long enough that this node +// plainly stopped watching. Derived from the check cadence rather than written +// as its own number, so it holds the same proportion at any scale the harness +// compresses that cadence to. +const MAX_CONFIRMATION_GAP_MS = CHECK_INTERVAL_MS * 2; +// A node may act on an app only once it has seen that app at full strength for +// base + position * step. Position in the shared instance order is a DELAY, +// never a veto: a rule of "only the most junior may leave" deadlocks, because +// the replacement is itself the most junior and does not want to leave. +const QUEUE_BASE_MS = config.fluxapps.residentialQueueBaseMs; +const QUEUE_STEP_MS = config.fluxapps.residentialQueueStepMs; +// The ticket is served against an UNINTERRUPTED observation - the wait means +// nothing if it can be accumulated across periods this node was not watching - +// and this is what counts as the interruption. A gap longer than one queue step +// means at least one whole give-up pass went by without this node evaluating +// the app, so the observation starts again rather than carrying over. +// +// It is what stops the departure interval leaking into the queue. A node inside +// its interval returns ABOVE the accounting below, so it records nothing for +// the whole block; the first pass after the block clears sees one six-hour gap +// and every ticket starts again. Without that, position separates the FIRST +// departure and nothing after it: every ticket matures untouched during the +// block, and the node is instantly ready for all of them the moment it clears. +// Two nodes whose blocks expire in the same pass then hand back the same app +// together, which is the defect the step was widened to 40 minutes to close - +// reached by the other door. +// +// Derived from the step rather than written as its own number, because the step +// is ALREADY required to outlast the pass: that inequality is asserted against +// production's config in the unit tests and re-derived per fleet by +// test-infra's coupled knobs. A literal here would be a third place that has to +// be kept in step with the block time, and the last hand-written number in this +// neighbourhood inverted the property it was meant to enforce. +// +// TWICE the step, not once. A gap has to mean "a pass did not happen", and one +// step is 1.82 passes - so a single pass running late trips it. Production +// hardly notices that (40 minutes of lateness on a 22-minute pass is an +// incident in itself), but the harness compresses the same ratio to about 30 +// seconds, where six fleets booting at once make a late pass ordinary. The +// ticket then restarts every few passes and never matures at all: on chud it +// counted 2m, 1m, 0m, and jumped back to 2m, three times over, and the suite +// waiting on it timed out. +// +// The ABSOLUTE jitter does not compress with the clocks, which is why a ratio +// that is comfortable at production scale is not comfortable at harness scale. +// Two steps is ~3.6 passes: a pass has to be missed outright, not merely be +// slow. The departure interval is what must still exceed it - see the coupled +// knob that enforces exactly that. +const MAX_TICKET_GAP_MS = QUEUE_STEP_MS * 2; +// Minimum gap between this node's departures. The give-up-an-app pass runs every +// 11 blocks (~22 min), which unpaced would empty the busiest node in the fleet in +// about four hours; there is no deadline here and slower is strictly safer. +const EVACUATION_INTERVAL_MS = config.fluxapps.residentialEvacuationIntervalMs; + +const startupCollection = config.database.local.collections.nodeStartupTracker; +const SETTLE_MARKER_KEY = 'residentialDos'; + +let timerHandle = null; +let started = false; +let stopping = false; +let ourDosActive = false; +let inconclusiveStreak = 0; +// appName -> { since, lastSeenAt } on the MONOTONIC clock, for an app this node +// has seen at full strength on every pass since `since`. Process lifetime only: +// losing it costs a queue wait, never a premature removal. +// +// Two fields rather than one, because the ticket is an uninterrupted observation +// and not an elapsed time. `since` is what the wait is measured from; +// `lastSeenAt` is the only way an interruption can be detected at all - without +// it a node that stopped looking, or one that was not allowed to act, goes on +// accruing credit for time it never spent watching. +const wholeObservation = new Map(); +// Whether the settling window has elapsed and departures may begin. +let evacuating = false; +// What /flux/info reports, and DELIBERATELY not derived from `evacuating`. +// +// `evacuating` is a per-tick permission for the give-up pass: any tick that +// cannot read something turns it off, correctly, because a node that cannot +// establish its own state must not hand an app back. It is not a description of +// how far through the staging the node is, and reporting it as one would show +// the node moving BACKWARDS through a staging it never moved backwards through. +// +// The window is what actually progresses. `observedWindowMs` only ever grows +// while the verdict holds - a tick that cannot decide leaves it alone rather +// than resetting it - and both are cleared together the moment the node stops +// being enforced, which is the one transition that really is a reversal. +let enforcing = false; +let observedWindowMs = 0; +// The network verdict behind the most recent isResidential(), carried into the +// decision event so a consumer can tell the three nulls apart. +let lastVerdict = { classification: null, source: null }; +// What the last evaluation concluded, in full. The tick answers with it, so a +// caller that needs to know WHICH decision was reached does not have to read +// the harness event stream to find out. +let lastDecision = null; +// Paces departures, on the MONOTONIC clock, and PERSISTED in the settle marker +// rather than held for the process lifetime. Same reasoning as the settling +// clock - a counter a restart resets makes restarting the way to go faster: a +// node restarting on a cron, crash-looping, or taking the ~4h auto-update shed +// an app every queue wait instead of every departure interval, and the busiest +// node in the fleet finished in hours rather than the three days the cadence is +// set for. +// +// `null`, not 0, for "no departure recorded". The gate used to read `now - 0` +// and get about 1.7e12 ms, which is open by arithmetic - but 0 on the monotonic +// clock is the start of THIS process, so the same expression would hold the +// gate SHUT on a freshly booted node for the first six hours. The state is +// named rather than encoded in a magic origin. +let lastEvacuationAt = null; + +// Whether this node yet knows what it is running. Starts false and is only +// raised by the orchestrator's own signal - the same one appSpawner waits on. +// globalState.spawnerPaused is NOT this signal: it initialises to false, so +// reading it would report a freshly booted node as ready, and an app list read +// then is not evidence of an empty node. +let nodeReady = false; +appSyncEvents.on(SYNC_EVENTS.SPAWNER_READY, () => { nodeReady = true; }); +appSyncEvents.on(SYNC_EVENTS.READINESS_LOST, () => { nodeReady = false; }); + +/** + * Milliseconds on the monotonic clock. Every elapsed-time decision the queue + * ticket makes reads this rather than the wall clock, so an NTP step cannot + * mature a ticket against time this node never spent watching. The backward + * step matters too, and more here than in most places: this runs on a box its + * own operator administers, and the operator has an incentive to stall the + * drain. + * + * Only values that must survive a restart stay on the wall clock, because a + * monotonic reading means nothing to the next process. + * @returns {number} Milliseconds since an arbitrary fixed origin. + */ +function monotonicMs() { + return Number(process.hrtime.bigint() / 1000000n); +} + +/** + * True when the current sticky DOS message was set by this service. + * Identified by the DOS_MESSAGE_PREFIX we always prepend when we set it. + */ +function isOurStickyDos() { + const msg = fluxNetworkHelper.getStickyDosMessage(); + return typeof msg === 'string' && msg.startsWith(DOS_MESSAGE_PREFIX); +} + +/** + * Three-state ArcaneOS check via fluxbenchd. + * true - confirmed ArcaneOS, nothing to enforce + * false - confirmed NOT ArcaneOS + * null - fluxbenchd unreachable or malformed, decide nothing + * + * Read from bench rather than `process.env.FLUXOS_PATH` because the env var is + * set by whoever launches FluxOS, and this check is exactly what a residential + * operator has an incentive to fake. + * @returns {Promise} + */ +async function isArcaneOs() { + try { + const benchmarkResponse = await benchmarkService.getBenchmarks(); + if (!benchmarkResponse || benchmarkResponse.status !== 'success' || !benchmarkResponse.data) { + return null; + } + const { systemsecure } = benchmarkResponse.data; + if (typeof systemsecure !== 'boolean') return null; + return systemsecure; + } catch (error) { + log.warn(`residentialNodeDos - benchmark check failed: ${error.message}`); + return null; + } +} + +/** + * Three-state residential check. + * true - RESIDENTIAL: positive evidence with no contradiction + * false - DATACENTER + * null - decide nothing. Either CONFLICTED/UNKNOWN, or there is no verdict + * to be had: nothing observed yet, or no published location table has + * been consulted. A node that has not read the table does not know + * what kind of network it is on, and enforcing on its own reading + * alone is what this whole staging exists to avoid. + * + * Awaiting getNodeGeolocation() first is what restores the observations from the + * db after a restart. + * @returns {Promise} + */ +async function isResidential() { + try { + await geolocationService.getNodeGeolocation(); + const verdict = await geolocationService.getNetworkClassification(); + // Kept for the decision event. The boolean below collapses CONFLICTED, + // UNKNOWN and no-table-consulted into one null, which is right for the + // enforcement decision and useless to anything trying to understand it - a + // node declining a published verdict about its own address and a node that + // has not read the table yet are the same answer here and nothing alike. + lastVerdict = verdict + ? { classification: verdict.classification, source: verdict.source } + : { classification: null, source: null }; + if (!verdict) return null; + if (verdict.classification === CLASSIFICATION.RESIDENTIAL) return true; + if (verdict.classification === CLASSIFICATION.DATACENTER) return false; + return null; + } catch (error) { + lastVerdict = { classification: null, source: null }; + log.warn(`residentialNodeDos - geolocation check failed: ${error.message}`); + return null; + } +} + +/** + * The settling marker as stored, or null when it cannot be read. + * + * Persisted rather than held in memory: a counter of consecutive evaluations in + * memory is reset by restarting FluxOS, which would make restarting on a timer a + * way to postpone the drain indefinitely. + * @returns {Promise} The marker document, or null. + */ +async function getSettleMarker() { + try { + const db = dbHelper.databaseConnection(); + if (!db) return null; + const database = db.db(config.database.local.database); + const marker = await dbHelper.findOneInDatabase(database, startupCollection, { _id: SETTLE_MARKER_KEY }); + return marker || null; + } catch (error) { + log.warn(`residentialNodeDos - could not read settle marker: ${error.message}`); + return null; + } +} + +/** + * A finite number, or null. + * + * Everything the window is computed from comes off a stored document, and a + * value that is not a finite number poisons every comparison downstream: NaN is + * neither `< SETTLE_MS` nor `>= SETTLE_MS`, so a gate written the obvious way + * around falls through to draining on it. Rejected at the read instead, where + * the alternative is "we have no record", which starts the window rather than + * ending it. + * @param {*} value The stored value. + * @returns {number|null} The number, or null. + */ +function numberOrNull(value) { + return Number.isFinite(value) ? value : null; +} + +/** + * How much of the gap since the last confirmation counts towards the window. + * + * Nothing once the gap is long enough that this node plainly stopped watching - + * that is the whole point, and it is what stops a day of silence reading as a + * day of agreement. Otherwise the gap itself, capped at one check interval so a + * long-ish gap cannot buy more than a tick's worth of credit. + * @param {number} gapMs Time since the last confirmation. + * @returns {number} Milliseconds to credit. + */ +function creditForGap(gapMs) { + if (gapMs > MAX_CONFIRMATION_GAP_MS) return 0; + return Math.min(gapMs, CHECK_INTERVAL_MS); +} + +/** + * Record that this tick confirmed the verdict, and return the observed time the + * verdict has now held for. + * + * Keyed on the VERDICT, not on the address: residential lines get dynamic + * addresses, so restarting the clock on an IP change would make power-cycling + * the router the way to postpone enforcement. + * @param {number} now Epoch ms. + * @returns {Promise} Observed milliseconds, or null when the marker + * cannot be written. + */ +async function noteVerdictConfirmed(now) { + const marker = await getSettleMarker(); + // A marker written before this node kept an observed total has no record of + // what was watched, only of when the clock started - and that is exactly the + // measure being replaced. It starts accumulating from here rather than being + // credited for time nobody can vouch for. + const previousConfirmedAt = numberOrNull(marker && marker.lastConfirmedAt); + const observedMs = numberOrNull(marker && marker.observedMs) ?? 0; + // Restored here rather than at boot: this is the pass that reads the marker, + // and it runs before any departure can be considered. + // + // CONVERTED, not read. The marker holds wall-clock ms and the gate runs on the + // monotonic clock, so what survives a restart is how long AGO the last + // departure was, not the instant it happened. A marker stamped in the future - + // the clock moved back between processes - would otherwise restore an origin + // ahead of now and hold the gate shut; it is floored at no elapsed time + // instead, which is the same answer as a departure that just happened. + const persistedEvacuation = numberOrNull(marker && marker.lastEvacuationAt); + if (persistedEvacuation) { + const restored = monotonicMs() - Math.max(0, now - persistedEvacuation); + if (lastEvacuationAt === null || restored > lastEvacuationAt) { + lastEvacuationAt = restored; + } + } + const credited = previousConfirmedAt ? creditForGap(now - previousConfirmedAt) : 0; + const totalObservedMs = observedMs + credited; + + try { + const db = dbHelper.databaseConnection(); + if (!db) return null; + const database = db.db(config.database.local.database); + await dbHelper.findOneAndUpdateInDatabase( + database, + startupCollection, + { _id: SETTLE_MARKER_KEY }, + { + $set: { + // Kept for the operator reading the record, not for the gate: it is + // when the verdict was FIRST seen, which is not what the window + // measures. + residentialSince: (marker && marker.residentialSince) || now, + lastConfirmedAt: now, + observedMs: totalObservedMs, + }, + }, + { upsert: true }, + ); + if (!marker) log.info('residentialNodeDos - settling window started'); + return totalObservedMs; + } catch (error) { + log.warn(`residentialNodeDos - could not write settle marker: ${error.message}`); + return null; + } +} + +/** + * Clear the settling window. Only a verdict flip does this - the node is no + * longer residential, or is now ArcaneOS. + */ +async function clearSettleMarker() { + try { + const db = dbHelper.databaseConnection(); + if (!db) return; + const database = db.db(config.database.local.database); + await dbHelper.findOneAndDeleteInDatabase(database, startupCollection, { _id: SETTLE_MARKER_KEY }, {}); + } catch (error) { + log.warn(`residentialNodeDos - could not clear settle marker: ${error.message}`); + } +} + +/** + * The apps installed on this node, or null when that cannot be established. + * + * The null is load-bearing. Setting the DOS on a node believed empty that is not + * makes nodeStatusMonitor delete every app it holds - the exact outcome this + * service exists to avoid - so "could not read" must never arrive here as an + * empty list. + * @param {Function} installedAppsFn Injected app lister. + * @returns {Promise} + */ +async function listInstalledApps(installedAppsFn) { + try { + const response = await installedAppsFn(); + if (!response || response.status !== 'success' || !Array.isArray(response.data)) return null; + return response.data.map((app) => app.name); + } catch (error) { + log.warn(`residentialNodeDos - could not list installed apps: ${error.message}`); + return null; + } +} + +/** + * How long this node must have seen an app whole before it may act on it. + * @param {object[]} locations Instance locations, any order. + * @param {string} localSocketAddr This node's socket address. + * @returns {number} Milliseconds. + */ +function queueDelayMs(locations, localSocketAddr) { + // Junior end first - the same order reasonToGiveUpApp ranks SURPLUS by, and + // for the same reason: the newest copy stands aside and the senior one goes + // on holding the data. Ranking the senior end first also put the ONE case + // that cannot simply leave at the front of the queue, because the elected + // primary is the senior instance: the node needing a stand-down was asked to + // go before any of the nodes that could just go, and every node behind it + // waited out its step while it negotiated. Senior last is both orders + // agreeing and the cheap departures happening first. + const ordered = [...locations].sort((a, b) => compareInstanceSeniority(b, a)); + const index = ordered.findIndex((entry) => socketAddressesMatch(entry.ip, localSocketAddr)); + // An instance this node cannot find in the list waits longest. For an EMPTY + // list `ordered.length` is 0, which is the front of the queue - the shortest + // wait of all, inverting the rule. An empty list is the ordinary result of + // expired location records, so it is not an exotic input. + const position = index < 0 ? Math.max(ordered.length, 1) : index; + return QUEUE_BASE_MS + (position * QUEUE_STEP_MS); +} + +/** + * How far this node is through being staged out of service, for /flux/info. + * + * null nothing is being enforced against this node + * HOLD enforced: it takes no NEW apps, and nothing has been deleted + * EVACUATE the settling window is served and it is handing apps back + * + * HOLD is the state worth reporting. It is the longest one - a full settling + * window - it is the only one where the operator can still fix the node and + * lose nothing, and without it a held node is indistinguishable from an + * ordinary one that happens not to have been given work. + * + * The DOS itself is already reported beside this as `dos`, so the three stages + * read together: HOLD with no DOS, EVACUATE with no DOS, then EVACUATE with + * one. A node EMPTY FROM THE START reports HOLD with a DOS, having skipped the + * window it had no data to need; one that DRAINED to empty has served the whole + * window by the time it gets there, so it reports EVACUATE with a DOS. + * @returns {('HOLD'|'EVACUATE'|null)} + */ +function getDosStaging() { + if (!enforcing) return null; + return observedWindowMs >= SETTLE_MS ? 'EVACUATE' : 'HOLD'; +} + +/** + * Whether this node is currently shedding the apps it holds. + * + * True only once the verdict has held for the settling window - the placement + * hold starts immediately, but nothing that moves customer data does. + * @returns {boolean} + */ +function isEvacuating() { + return evacuating; +} + +/** + * May this node give up this particular app right now? + * + * This is the pacing half of the decision and says nothing about safety; the + * give-up-an-app pass asks appEvacuationSafety separately, and both must agree. + * @param {string} appName Global app name. + * @param {object[]} locations Instance locations for the app. + * @param {string} localSocketAddr This node's socket address. + * @param {number} minInstances How many instances this app is meant to have. + * Passed in rather than derived here, so the pacing half of the decision and + * the SURPLUS half are ranked by one number instead of two. It is the LOCAL + * record's count; appEvacuationSafety re-derives from the global spec, which + * is the authority at the moment of removal. They can differ while an owner's + * instance-count change propagates, and either way round is safe: too low + * here lets a ticket run that the safety gate then refuses as short, which + * restarts the observation, and too high only makes the turn wait longer. + * @param {number} [now] Monotonic ms, injectable for tests. + * @returns {{ok: boolean, code: string, reason: string}} `code` is the + * machine-readable half. A caller that has to tell "waiting its turn" from + * "the app is short" cannot do it by matching prose, and the difference + * matters: the first is this working, the second is a node that wants to + * leave and cannot, which is the thing worth escalating. BELOW_INSTANCE_COUNT + * is deliberately the name appEvacuationSafety already uses for the same + * fact, because it IS the same fact asked one layer earlier. + */ +function mayEvacuateApp(appName, locations, localSocketAddr, minInstances, now = monotonicMs()) { + if (!evacuating) return { ok: false, code: 'NOT_EVACUATING', reason: 'node is not evacuating' }; + if (lastEvacuationAt !== null && now - lastEvacuationAt < EVACUATION_INTERVAL_MS) { + const wait = Math.round((EVACUATION_INTERVAL_MS - (now - lastEvacuationAt)) / 60000); + return { ok: false, code: 'DEPARTURE_INTERVAL', reason: `next departure in ${wait}m` }; + } + + // Everything below is the ticket, and it sits BELOW the interval gate on + // purpose: a blocked node records nothing, so its own block reads as one long + // gap and the ticket starts again. See MAX_TICKET_GAP_MS. + const previous = wholeObservation.get(appName); + const gap = previous ? now - previous.lastSeenAt : Infinity; + // Strength is tested HERE, where the clock is stamped, rather than left to + // the safety gate. The wait is "seen at full strength for base + position x + // step"; a clock that starts on the first ASK measures something else, and an + // app short of its count is one the spawner is part way through replacing. + // Time spent watching that is not time spent watching it whole. + const whole = locations.length >= minInstances; + const since = (!whole || gap > MAX_TICKET_GAP_MS) ? now : previous.since; + wholeObservation.set(appName, { since, lastSeenAt: now }); + if (!whole) { + return { + ok: false, + code: 'BELOW_INSTANCE_COUNT', + reason: `app is below its instance count (${locations.length}/${minInstances}); its turn starts again`, + }; + } + const wait = queueDelayMs(locations, localSocketAddr); + const observed = now - since; + if (observed < wait) { + return { ok: false, code: 'AWAITING_TURN', reason: `its turn is in ${Math.round((wait - observed) / 60000)}m` }; + } + return { ok: true, code: 'READY', reason: 'ready' }; +} + +/** + * Record that this app has gone, so the interval before the next one starts. + * @param {string} appName Global app name. + * @param {number} [now] Epoch ms, injectable for tests. + */ +function noteEvacuated(appName, now = monotonicMs()) { + lastEvacuationAt = now; + wholeObservation.delete(appName); + // Written through to the marker, so a restart does not re-open the gate. Best + // effort: the in-memory value already paces this process, and the persisted + // one only has to survive into the next. + // + // The WALL clock is what gets written. The next process cannot read this + // one's monotonic origin, so the only thing worth recording is an instant it + // can convert back into "how long ago". + persistLastEvacuationAt(Date.now()).catch(() => {}); + log.info(`residentialNodeDos - ${appName} handed back; next departure no sooner than ${EVACUATION_INTERVAL_MS / 3600000}h`); +} + +/** + * Record when this node last handed an app back. + * @param {number} now Epoch ms. + * @returns {Promise} + */ +async function persistLastEvacuationAt(now) { + try { + const db = dbHelper.databaseConnection(); + if (!db) return; + const database = db.db(config.database.local.database); + await dbHelper.findOneAndUpdateInDatabase( + database, + startupCollection, + { _id: SETTLE_MARKER_KEY }, + { $set: { lastEvacuationAt: now } }, + { upsert: true }, + ); + } catch (error) { + log.warn(`residentialNodeDos - could not record the departure time: ${error.message}`); + } +} + +/** + * Forget an app's queue observation. Called when it stops being safe to give up, + * so the wait is served against an uninterrupted observation rather than + * accumulated across a gap. + * @param {string} appName Global app name. + */ +function forgetAppObservation(appName) { + wholeObservation.delete(appName); +} + +/** + * Give up the DOS this service is holding. The slot is only cleared when the + * message in it is still ours: another owner may have taken it since we wrote, + * and clearing that would drop their DOS on the floor. + * @param {string} reason Logged context for the release. + */ +function releaseOurDos(reason) { + if (isOurStickyDos()) { + log.info(`residentialNodeDos - clearing sticky DOS (${reason})`); + fluxNetworkHelper.clearStickyDosMessage(); + ourDosActive = false; + return; + } + if (ourDosActive) { + log.info(`residentialNodeDos - our DOS was replaced by another owner, releasing our claim only (${reason})`); + ourDosActive = false; + } +} + +/** + * Put the node fully out of service. Only ever reached once it holds no apps. + */ +function applyDos() { + const sticky = fluxNetworkHelper.getStickyDosMessage(); + if (sticky && !isOurStickyDos()) { + // Another owner's DOS already has this node out of service for its own + // reason, and taking the single slot would leave it unable to recognise or + // release its own state. + log.info('residentialNodeDos - another sticky DOS is active, not overwriting it'); + return; + } + if (isOurStickyDos()) return; + const message = `${DOS_MESSAGE_PREFIX}. Migrate this node to ArcaneOS or move it to a data center connection.`; + fluxNetworkHelper.setStickyDosMessage(message); + fluxNetworkHelper.setStickyDosStateValue(100); + ourDosActive = true; + log.error(message); +} + +/** + * What this tick concluded, whichever way it went. + * + * Deciding NOT to enforce leaves no trace: no placement hold, no settle marker, + * no DOS - the outcome IS the absence of all three. So a caller with no event + * here can only wait out a duration and infer from nothing having happened, + * which is indistinguishable from the tick never having run. That is what the + * harness was doing, at thirty seconds a test. + * + * `enforce: null` is a tick that could not decide, with `undecidedBecause` + * naming the input that was missing. Kept apart from `false` because they are + * opposite states: one says this node is fit to serve, the other says nobody + * knows yet. + * + * fluxEventBus is inert on a real node - config.testEventStream is false - so + * this exists for the harness and costs production nothing. + * @param {{residential: boolean|null, arcaneOs: boolean|null, + * enforce: boolean|null, undecidedBecause: string|null}} verdict + */ +function publishDecision(verdict) { + lastDecision = { + ...verdict, + // Which network verdict produced this, and which authority reached it - + // 'published-table' or 'node-veto'. Without these, enforce: null covers + // CONFLICTED, UNKNOWN and "no table consulted" alike, and a suite waiting + // on a veto cannot tell it from a node that has read nothing. + classification: lastVerdict.classification, + source: lastVerdict.source, + }; + // One object, answered to the caller and published to the harness. Two copies + // of a decision are two things that can disagree. + fluxEventBus.publish('residential:decided', lastDecision); +} + +/** + * One evaluation of the policy. + * + * @param {object} deps Injected collaborators. + * @param {Function} deps.installedAppsFn Lists apps installed on this node. + * @returns {Promise} True when the tick reached a decision, false when + * an input was unavailable and the caller should retry sooner. + */ +async function runResidentialPolicy(deps) { + const { installedAppsFn } = deps; + + const [arcane, residential] = await Promise.all([isArcaneOs(), isResidential()]); + + // A tick that cannot decide stops the DRAIN but leaves the placement hold. + // The two fail in opposite directions on purpose: holding placement costs the + // node nothing it already has, so leaving it on through an unreadable tick is + // safe, while continuing to hand back an app every departure interval with no + // current verdict is not. `evacuating` was latched, so a node already draining + // whose bench or classification became unreadable kept going for as long as + // the input stayed unavailable. + if (arcane === null) { + evacuating = false; + log.info('residentialNodeDos - benchmark unreachable, skipping this tick'); + publishDecision({ residential, arcaneOs: arcane, enforce: null, undecidedBecause: 'benchmark' }); + return false; + } + if (residential === null) { + evacuating = false; + log.info('residentialNodeDos - no network verdict to act on yet, skipping this tick'); + publishDecision({ residential, arcaneOs: arcane, enforce: null, undecidedBecause: 'classification' }); + return false; + } + + const shouldEnforce = residential && !arcane; + log.info(`residentialNodeDos - residential=${residential} arcaneOs=${arcane} enforce=${shouldEnforce}`); + publishDecision({ residential, arcaneOs: arcane, enforce: shouldEnforce, undecidedBecause: null }); + + if (!shouldEnforce) { + fluxNetworkHelper.clearPlacementHold(fluxNetworkHelper.PlacementHoldOwner.RESIDENTIAL_DOS); + releaseOurDos(`residential=${residential}, arcaneOs=${arcane}`); + await clearSettleMarker(); + evacuating = false; + enforcing = false; + observedWindowMs = 0; + wholeObservation.clear(); + return true; + } + + // Costs the node nothing it already holds, so it needs no settling period. + fluxNetworkHelper.setPlacementHold(fluxNetworkHelper.PlacementHoldOwner.RESIDENTIAL_DOS, HOLD_REASON); + // Set with the hold and cleared with it: the two are the same fact, and the + // hold is the first thing that happens to an enforced node. + enforcing = true; + + if (!nodeReady) { + evacuating = false; + log.info('residentialNodeDos - node not ready yet, holding placement only this tick'); + return false; + } + + const installed = await listInstalledApps(installedAppsFn); + if (installed === null) { + evacuating = false; + log.info('residentialNodeDos - installed app list unavailable, holding placement only this tick'); + return false; + } + + if (!installed.length) { + // The placement hold above stops the spawner taking anything NEW, but an + // install already running is not stopped by it - and an install is real + // from appInstaller's installationInProgress flag, some way before its + // database record exists for listInstalledApps to see. A tick landing in + // that gap reads an empty node, DOSes it, and nodeStatusMonitor tears the + // arriving app down on its next loop. + // + // Undecided rather than "not empty": the retry backoff re-asks in minutes + // instead of deferring the DOS for a full check interval, and an install + // resolves on that timescale. Bounded today by appInstaller writing its + // database entry before creating the container, so no volume exists in the + // window - nothing here referenced that ordering, and nothing tested it. + if (globalState.installationInProgress) { + log.info('residentialNodeDos - an install is in flight, not treating this node as empty yet'); + return false; + } + applyDos(); + return true; + } + + const now = Date.now(); + const observedMs = await noteVerdictConfirmed(now); + if (observedMs !== null) observedWindowMs = observedMs; + if (observedMs === null) { + evacuating = false; + log.info('residentialNodeDos - settle marker unavailable, evacuation stays off'); + return false; + } + // Negated rather than written as `<`, so a value that is not a number refuses + // rather than reading as elapsed. numberOrNull above is the guard that + // actually stands between a stored value and this comparison; this form is + // the backstop for a path that ever reaches it another way, and the two are + // only distinguishable on an input like Infinity, which numberOrNull rejects + // and `>=` would accept. It is reachable without malformed data: the + // clock is Date.now(), and a node whose clock is behind when the marker is + // written - no RTC, a VM restored from snapshot, timesyncd not yet stepped - + // and is then corrected FORWARD reads the whole window as served. A backwards + // step fails safe; a forward step failed toward moving customer data. + if (!(observedMs >= SETTLE_MS)) { + evacuating = false; + // Hours of the verdict actually WATCHED, so a node whose checks keep coming + // back inconclusive sees this figure stall rather than count down - which + // is the difference the window is there to make. + const remaining = Math.round((SETTLE_MS - observedMs) / (60 * 60 * 1000)); + log.info(`residentialNodeDos - held, evacuation begins after about ${remaining}h more of confirmed verdict (${installed.length} app(s) installed)`); + return true; + } + + // The give-up-an-app pass reads this and does the removing; it asks + // mayEvacuateApp for the pacing and appEvacuationSafety for the safety. + if (!evacuating) log.warn(`residentialNodeDos - evacuation begins, ${installed.length} app(s) to hand back`); + evacuating = true; + return true; +} + +/** + * Delay before the next tick. A decided tick waits out the full interval; an + * inconclusive one comes back on the short retry, doubling each time it stays + * inconclusive, so a node that can never decide stops saying so 288 times a day. + * @param {boolean} decided Whether the tick reached a decision. + * @param {number} streak Consecutive inconclusive ticks, this one included. + * @returns {number} Milliseconds until the next tick. + */ +function nextDelay(decided, streak) { + if (decided) return CHECK_INTERVAL_MS; + return Math.min(RETRY_INTERVAL_MS * 2 ** (streak - 1), CHECK_INTERVAL_MS); +} + +/** + * One evaluation of the policy, answered as what it concluded. + * + * `decided` is the caller's question - was every input available, or should the + * next tick come sooner. `decision` is what was actually concluded, and it is + * the same object published to the harness: enforce false and enforce null are + * opposite claims, and nothing outside this module could previously tell them + * apart without reading the event stream. + * + * @param {object} deps Injected collaborators. + * @returns {Promise<{decided: boolean, decision: object|null}>} + */ +async function enforceResidentialPolicy(deps) { + lastDecision = null; + const decided = await runResidentialPolicy(deps); + return { decided, decision: lastDecision }; +} + +/** + * Run one tick and schedule the next one. + * @param {object} deps Injected collaborators, as enforceResidentialPolicy. + */ +async function tick(deps) { + let decided = false; + try { + ({ decided } = await enforceResidentialPolicy(deps)); + } catch (error) { + log.error(`residentialNodeDos - tick error: ${error.message}`); + } + inconclusiveStreak = decided ? 0 : inconclusiveStreak + 1; + if (stopping) return; + timerHandle = setTimeout(() => tick(deps), nextDelay(decided, inconclusiveStreak)); +} + +/** + * Start the enforcer. Performs the first check immediately, then reschedules + * itself. Safe to call multiple times. + * @param {object} deps Injected collaborators, as enforceResidentialPolicy. + */ +async function start(deps) { + // The guard is `started`, not `timerHandle`: the first tick is awaited before + // any timer exists, so a second start() landing inside it would run a second + // self-rescheduling chain. + if (started) return; + started = true; + stopping = false; + inconclusiveStreak = 0; + log.info('residentialNodeDos - enforcer starting'); + await tick(deps); +} + +function stop() { + stopping = true; + started = false; + // Cleared with the timer: a later start() must not inherit a claim from the + // previous run and skip the read-back that decides whether the slot is ours. + ourDosActive = false; + evacuating = false; + enforcing = false; + observedWindowMs = 0; + wholeObservation.clear(); + if (timerHandle) { + clearTimeout(timerHandle); + timerHandle = null; + } +} + +function isDosActive() { + return ourDosActive; +} + +module.exports = { + start, + stop, + enforceResidentialPolicy, + isArcaneOs, + isResidential, + isDosActive, + queueDelayMs, + listInstalledApps, + isEvacuating, + getDosStaging, + mayEvacuateApp, + noteEvacuated, + forgetAppObservation, + // Test seam for the readiness gate, which is otherwise only moved by events. + setNodeReadyForTests: (value) => { nodeReady = value; }, + DOS_MESSAGE_PREFIX, + HOLD_REASON, + CHECK_INTERVAL_MS, + RETRY_INTERVAL_MS, + SETTLE_MS, + QUEUE_BASE_MS, + QUEUE_STEP_MS, + MAX_TICKET_GAP_MS, + EVACUATION_INTERVAL_MS, +}; diff --git a/ZelBack/src/services/serviceHelper.js b/ZelBack/src/services/serviceHelper.js index 70f446eafe..b90576ddc9 100644 --- a/ZelBack/src/services/serviceHelper.js +++ b/ZelBack/src/services/serviceHelper.js @@ -4,9 +4,9 @@ const util = require('node:util'); const path = require('node:path'); const fs = require('node:fs/promises'); const execFile = util.promisify(require('node:child_process').execFile); +const { spawn } = require('node:child_process'); const axios = require('axios').default; -const config = require('config'); const qs = require('qs'); const asyncLock = require('./utils/asyncLock'); @@ -505,6 +505,152 @@ function ipInSubnet(ip, subnet) { return (ipAsInt & maskAsInt) === (networkAsInt & maskAsInt); } +/** + * Runs a command whose output is consumed as it arrives, rather than collected + * and returned. + * + * Two things follow from streaming, and both are the point. The output is never + * held whole, so a listing of any length costs the same and there is no buffer + * ceiling to breach. And every chunk that arrives is evidence the command is + * still working, which is what makes `idleTimeout` mean "stopped doing + * anything" rather than "taking a while" - the distinction that matters for + * work whose duration is set by how much data it was given. A total timeout can + * only kill the largest inputs, which are the ones the caller most needs to + * succeed. + * + * Root is the only reason these run as a child process at all: app data is + * written by containers as root and this process is not root. That needs sudo, + * not a shell - so the command is argv throughout and a path is never parsed as + * syntax. + * + * @param {string} userCmd - Command to run + * @param {object} options - runAsRoot, params, onLine, idleTimeout, logError + * @returns {Promise<{error: Error|null, stderr: string}>} + */ +async function runStreamingCommand(userCmd, options = {}) { + const { + runAsRoot = false, params = [], onLine = null, idleTimeout = 0, logError, + } = options; + + const res = { error: null, stderr: '', idleKilled: false }; + + if (!userCmd) { + res.error = new Error('Command must be present'); + return res; + } + + if (!Array.isArray(params) || !params.every((p) => typeof p === 'string' || typeof p === 'number')) { + res.error = new Error('Invalid params for command, must be an Array of strings'); + return res; + } + + const args = params.map(String); + let cmd = userCmd; + if (runAsRoot) { + args.unshift(userCmd); + cmd = 'sudo'; + } + + log.debug(`Run Cmd (streaming): ${cmd} ${args.join(' ')}`); + + return new Promise((resolve) => { + const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + + let idleTimer = null; + let idleKilled = false; + let settled = false; + let remainder = ''; + // Enough of stderr to explain a failure, and no more: a command that fails + // on every entry must not be able to grow this without limit. + const STDERR_CAP = 8192; + + // A root child needs a root signal; an unprivileged FluxOS cannot send one + // directly. runCommand prefixes sudo and catches its own spawn failures, so + // a kill that cannot fork under memory pressure resolves an error rather + // than throwing an unhandled 'error' event out of a bare spawn. Not awaited. + const killChild = () => { + if (runAsRoot) { + runCommand('kill', { runAsRoot: true, params: ['-TERM', String(child.pid)], logError: false }); + } else { + child.kill(); + } + }; + + const bump = () => { + if (!idleTimeout || idleKilled) return; + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout(() => { + idleKilled = true; + killChild(); + }, idleTimeout); + }; + + const finish = (error) => { + if (settled) return; + settled = true; + if (idleTimer) clearTimeout(idleTimer); + // Two facts can be true at once - the idle timer fired AND something else + // failed - and they must not compete for one slot: a real error (a + // consumer throw, a spawn failure) always wins the error field, with the + // idle kill carried separately on res.idleKilled. The idle message is the + // error only when the kill is the sole cause (close() passes no error for + // the exit the kill itself provoked). + if (idleKilled) res.idleKilled = true; + if (error) { + res.error = error; + } else if (idleKilled) { + res.error = new Error(`command produced no output for ${idleTimeout}ms and was stopped`); + } + if (res.error && logError !== false) log.error(res.error); + resolve(res); + }; + + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + if (settled) return; + bump(); + if (!onLine) return; + remainder += chunk; + const lines = remainder.split('\n'); + remainder = lines.pop(); + try { + lines.forEach((line) => { if (line) onLine(line); }); + } catch (err) { + // A consumer that cannot take the output ends the run: swallowed, the + // caller would read success off output nothing consumed; unhandled, + // the promise never settles and the operation hangs on it. + killChild(); + finish(err); + } + }); + + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk) => { + if (settled) return; + bump(); + if (res.stderr.length < STDERR_CAP) res.stderr += chunk; + }); + + child.on('error', (err) => finish(err)); + + child.on('close', (code) => { + if (remainder && onLine && !idleKilled && !settled) { + try { + onLine(remainder); + } catch (err) { + finish(err); + return; + } + } + // After an idle kill the non-zero exit is the kill's own consequence, not + // a second fact - the idle cause is the truthful report, so no error here. + finish(code === 0 || idleKilled ? null : new Error(`command exited with code ${code}`)); + }); + + bump(); + }); +} + /** * Runs a command as a child process, without a shell by default. * Using a shell is possible with the `shell` option. @@ -689,6 +835,32 @@ async function dirInfo(dir, options = {}) { return response; } +/** + * Carry a large collection through a handler a slice at a time. + * + * Sync responses fan out heavily - each event becomes a database operation per + * app it reports - so processing a whole response at once holds the events, + * their derived copies and the driver's encoding of every write in memory + * together. Slicing bounds all of that to one slice's worth. Slices run in + * order and one at a time, so each is released before the next is built. + * + * @param {Array} items Collection to process. + * @param {number} sliceSize Maximum items handed over at once. + * @param {Function} handler Async callback receiving each slice. + * @returns {Promise} + */ +async function processInSlices(items, sliceSize, handler) { + if (!Array.isArray(items) || items.length === 0) return; + if (!Number.isInteger(sliceSize) || sliceSize < 1) { + throw new Error('processInSlices requires a positive integer slice size'); + } + + for (let offset = 0; offset < items.length; offset += sliceSize) { + // eslint-disable-next-line no-await-in-loop + await handler(items.slice(offset, offset + sliceSize)); + } +} + module.exports = { axiosGet, axiosPost, @@ -711,5 +883,7 @@ module.exports = { parseInterval, randomDelayMs, runCommand, + runStreamingCommand, validIpv4Address, + processInSlices, }; diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index f1d631e97f..1ceb2214ec 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -30,6 +30,9 @@ const appSpawner = require('./appLifecycle/appSpawner'); const { AppSyncOrchestrator } = require('./appMessaging/appSyncOrchestrator'); const crontabAndMountsCleanup = require('./appLifecycle/crontabAndMountsCleanup'); const containerMountRecovery = require('./appLifecycle/containerMountRecovery'); +const fileOperationRecovery = require('./appSystem/fileOperationRecovery'); +const networkRecovery = require('./appSystem/networkRecovery'); +const volumeExecutor = require('./appSystem/volumeExecutor'); const appStartupManager = require('./appLifecycle/appStartupManager'); const hardwareValidationService = require('./appLifecycle/hardwareValidationService'); const globalState = require('./utils/globalState'); @@ -41,6 +44,7 @@ const daemonServiceMiscRpcs = require('./daemonService/daemonServiceMiscRpcs'); const daemonServiceUtils = require('./daemonService/daemonServiceUtils'); const fluxService = require('./fluxService'); const geolocationService = require('./geolocationService'); +const ipLocationSync = require('./appPlacement/ipLocationSync'); const upnpService = require('./upnpService'); const syncthingService = require('./syncthingService'); const pgpService = require('./pgpService'); @@ -52,6 +56,7 @@ const volumeValidationService = require('./volumeValidationService'); const watchdogService = require('./watchdogService'); const cloudUIUpdateService = require('./cloudUIUpdateService'); const appTamperingBlocklistService = require('./appTamperingBlocklistService'); +const residentialNodeDosService = require('./residentialNodeDosService'); const nodeConfirmationService = require('./nodeConfirmationService'); const appTamperingDetectionService = require('./appTamperingDetectionService'); const appsRuntimeState = require('./appManagement/appsRuntimeState'); @@ -93,26 +98,154 @@ const portsNotWorking = new Set(); const appsStorageViolations = []; /** - * createIndex that tolerates a pre-existing index with conflicting options - * (IndexOptionsConflict / IndexKeySpecsConflict) by finding the conflicting - * index via listIndexes, dropping it by its actual name, and recreating. - * Every other error bubbles up. + * Remove rows that duplicate a would-be-unique key, keeping the newest of each. + * + * A recovery strategy for ensureIndex: when a unique build fails because the + * collection already holds rows that violate it, this makes the data conform to + * the invariant the index DECLARES - it deletes duplicates on the key the index + * says must be unique, which is enforcing a contract rather than losing data. + * Only safe where the key IS the row's identity, so it is passed in per build by + * the caller that knows the collection, never applied by default. A rollup whose + * duplicates must be summed rather than dropped (the tampering incident count) + * belongs to its owning service instead - see the note on ensureIndex. + * + * Keeps the newest per group (rows sort by _id, which is time-ordered), honours + * the index's partialFilterExpression so it only touches rows the index covers, + * and returns how many it removed. + * + * @param {object} collection - a mongo collection handle + * @param {object} spec - the index key, e.g. { hash: 1 } + * @param {object} options - the index options (read for partialFilterExpression) + * @returns {Promise} rows removed */ -async function ensureIndex(collection, spec, options = {}) { +async function dedupeByKey(collection, spec, options = {}) { + const groupId = {}; + Object.keys(spec).forEach((key, i) => { groupId[`k${i}`] = `$${key}`; }); + // Held in memory deliberately, and measured rather than assumed: run against a + // live node's collection unioned with itself until every key appeared 16 times + // - 1,030,208 rows, the duplicate state this exists to repair - it finished in + // under 2s without spilling. The $sort adds nothing on top while it stays an + // index walk on _id, which it is for a spec with no partialFilterExpression; + // the first partial index to use this wants re-measuring, because the $match + // ahead of the sort is what would make the sort blocking. allowDiskUse is not + // set: it would take a mongo below 6.0, where the cap errors instead of + // spilling, and the network floor is moving past that. + // + // ids[0] rather than $max: $group does not document that it carries a + // preceding sort into an accumulator, and that non-guarantee is about results + // merged from several sources - this is one standalone mongod. Checked against + // 64,388 real duplicate groups, ids[0] was the newest in all 64,388. + const pipeline = [ + ...(options.partialFilterExpression ? [{ $match: options.partialFilterExpression }] : []), + { $sort: { _id: -1 } }, + { $group: { _id: groupId, ids: { $push: '$_id' } } }, + { $match: { 'ids.1': { $exists: true } } }, + ]; + const groups = await collection.aggregate(pipeline).toArray(); + const toRemove = groups.flatMap((group) => group.ids.slice(1)); + if (!toRemove.length) return 0; + await collection.deleteMany({ _id: { $in: toRemove } }); + return toRemove.length; +} + +/** + * Assert one index, healing the failures that are recoverable. + * + * - a pre-existing index with conflicting OPTIONS (IndexOptionsConflict / + * IndexKeySpecsConflict) is dropped by its real name and recreated; + * - a unique build blocked by DUPLICATE ROWS runs the caller's `recover` + * strategy (see dedupeByKey) and rebuilds, so the node ends up WITH the + * index rather than running degraded without it; + * - anything else rethrows. + * + * The rethrow is deliberate and is NOT the blanket swallow it replaced. Index + * setup runs before any service or interval starts, so the 15s startFluxFunctions + * retry re-runs it safely: a TRANSIENT failure (mongo mid-election, a slow-disk + * blip) heals on the next pass instead of being skipped until the next reboot, + * and a genuinely UNRECOVERABLE database wedges loudly - which is correct, since + * a node whose DB cannot hold its schema cannot serve apps and appremove would + * not rescue it. The realistic wedge that finding motivated - a unique index + * over rows that already violate it - is repaired above, not hidden. + * + * TRUE NORTH: eventually every collection owns its own schema-prepare - its + * index spec plus whatever dedupe or merge its data needs - the way + * appsRuntimeState.prepareCollection and appTamperingDetectionService already + * do, and boot just invokes those prepare functions. That turns this ~40-call + * imperative block into a set of owned, individually testable units. This + * function is the increment toward it, not the destination; a full move of the + * remaining builds is a separate refactor, out of scope for the PR that added it. + * + * @param {object} collection - a mongo collection handle + * @param {object} spec - the index key + * @param {object} [options] - the index options + * @param {(collection: object, spec: object, options: object) => Promise} [recover] + * run when a unique build is blocked by existing duplicate rows + */ +async function ensureIndex(collection, spec, options = {}, recover = null) { try { await collection.createIndex(spec, options); } catch (err) { const conflict = err && (err.codeName === 'IndexOptionsConflict' || err.codeName === 'IndexKeySpecsConflict'); - if (!conflict) throw err; - const specKeys = JSON.stringify(spec); - const indexes = await collection.listIndexes().toArray(); - const match = indexes.find((idx) => JSON.stringify(idx.key) === specKeys); - const dropName = match?.name; - if (dropName) { - log.warn(`ensureIndex - conflicting index '${dropName}' on ${collection.collectionName} (key: ${specKeys}), dropping and recreating`); - await collection.dropIndex(dropName); + if (conflict) { + const specKeys = JSON.stringify(spec); + const indexes = await collection.listIndexes().toArray(); + const match = indexes.find((idx) => JSON.stringify(idx.key) === specKeys); + if (match?.name) { + log.warn(`ensureIndex - conflicting index '${match.name}' on ${collection.collectionName} (key: ${specKeys}), dropping and recreating`); + await collection.dropIndex(match.name); + } + await collection.createIndex(spec, options); + return; } - await collection.createIndex(spec, options); + const duplicate = err && (err.code === 11000 || err.codeName === 'DuplicateKey'); + if (duplicate && recover) { + const removed = await recover(collection, spec, options); + log.warn(`ensureIndex - ${collection.collectionName} (key: ${JSON.stringify(spec)}) held ${removed} row(s) violating a unique index; removed and rebuilding`); + await collection.createIndex(spec, options); + return; + } + throw err; + } +} + +/** + * Assert every index one collection needs, in a single command. + * + * mongo's index build protocol has a fixed cost per BUILD - register, start, + * scan, wait for commit quorum, commit, log - and it does not care that the + * collection is empty. A node asserting its schema one index at a time pays + * that cost 34 times over 14 collections; createIndexes pays it once per + * collection. A node boots faster for it, and the integration harness, where + * ten nodes share one mongod and every database is new, feels it ten times over + * (measured: 938 concurrent builds per ten-node fleet). + * + * The batch is the fast path, not the only one. ensureIndex heals two failures + * that need to be attributed to a single index - an options conflict it drops + * and rebuilds, and a unique build blocked by duplicate rows it repairs through + * the caller's strategy - and a batch rejection does not say which member + * failed. So any error falls back to asserting them one at a time, which is + * exactly the behaviour that existed before this function. + * + * @param {object} collection - a mongo collection handle + * @param {Array} specs - `{ key, ...indexOptions, recover }` per index, + * where `recover` is ours and never reaches mongo + */ +async function ensureIndexes(collection, specs) { + try { + await collection.createIndexes(specs.map((spec) => { + // `recover` is a FluxOS concern; mongo is handed the index model alone + const model = { ...spec }; + delete model.recover; + return model; + })); + return; + } catch (error) { + log.warn(`ensureIndexes - batch of ${specs.length} on ${collection.collectionName} failed (${error.codeName || error.message}); asserting one at a time`); + } + // eslint-disable-next-line no-restricted-syntax + for (const { key, recover = null, ...options } of specs) { + // eslint-disable-next-line no-await-in-loop + await ensureIndex(collection, key, options, recover); } } @@ -179,44 +312,37 @@ async function startFluxFunctions() { log.error(error); } }); - await ensureIndex(database.collection(config.database.local.collections.loggedUsers), { createdAt: 1 }, { expireAfterSeconds: 14 * 24 * 60 * 60 }); - await ensureIndex(database.collection(config.database.local.collections.activeLoginPhrases), { createdAt: 1 }, { expireAfterSeconds: 900 }); - await ensureIndex(database.collection(config.database.local.collections.activeSignatures), { createdAt: 1 }, { expireAfterSeconds: 900 }); - await ensureIndex(database.collection(config.database.local.collections.activePaymentRequests), { createdAt: 1 }, { expireAfterSeconds: 3600 }); - await ensureIndex(database.collection(config.database.local.collections.completedPayments), { paymentId: 1 }); - await ensureIndex(database.collection(config.database.local.collections.completedPayments), { createdAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); + await ensureIndexes(database.collection(config.database.local.collections.loggedUsers), [ + { key: { createdAt: 1 }, expireAfterSeconds: 14 * 24 * 60 * 60 }, + ]); + await ensureIndexes(database.collection(config.database.local.collections.activeLoginPhrases), [ + { key: { createdAt: 1 }, expireAfterSeconds: 900 }, + ]); + await ensureIndexes(database.collection(config.database.local.collections.activeSignatures), [ + { key: { createdAt: 1 }, expireAfterSeconds: 900 }, + ]); + await ensureIndexes(database.collection(config.database.local.collections.activePaymentRequests), [ + { key: { createdAt: 1 }, expireAfterSeconds: 3600 }, + ]); + await ensureIndexes(database.collection(config.database.local.collections.completedPayments), [ + { key: { paymentId: 1 } }, + { key: { createdAt: 1 }, expireAfterSeconds: 7 * 24 * 60 * 60 }, + ]); // legacy pre-incident-schema rows expire via detectedAt; current incident // documents expire via lastSeen. The tamper service purges pre-schema // rows at startup, so the detectedAt pair only matters where old code // still writes; drop it once the fleet is past the incident schema. - await ensureIndex( - database.collection(config.database.local.collections.appTamperingEvents), - { detectedAt: 1 }, - { expireAfterSeconds: 30 * 24 * 60 * 60, name: 'detectedAt_ttl' }, // 30 days - ); - await ensureIndex( - database.collection(config.database.local.collections.appTamperingEvents), - { appName: 1, detectedAt: -1 }, - { name: 'appName_detectedAt' }, - ); - await ensureIndex( - database.collection(config.database.local.collections.appTamperingEvents), - { lastSeen: 1 }, - { expireAfterSeconds: 30 * 24 * 60 * 60, name: 'lastSeen_ttl' }, // 30 days - ); - // upsert key of the incident rollup; unique so concurrent recorders - // cannot double-insert an incident. Partial: legacy rows lack incidentKey - // and would otherwise collide on null. - await ensureIndex( - database.collection(config.database.local.collections.appTamperingEvents), - { appName: 1, eventType: 1, incidentKey: 1 }, - { unique: true, partialFilterExpression: { incidentKey: { $exists: true } }, name: 'incident_upsert' }, - ); - await ensureIndex( - database.collection(config.database.local.collections.appTamperingEvents), - { appName: 1, eventType: 1, lastSeen: -1 }, - { name: 'appName_eventType_lastSeen' }, - ); + await ensureIndexes(database.collection(config.database.local.collections.appTamperingEvents), [ + { key: { detectedAt: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'detectedAt_ttl' }, // 30 days + { key: { appName: 1, detectedAt: -1 }, name: 'appName_detectedAt' }, + { key: { lastSeen: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'lastSeen_ttl' }, // 30 days + { key: { appName: 1, eventType: 1, lastSeen: -1 }, name: 'appName_eventType_lastSeen' }, + ]); + // The unique incident-rollup index lives with its owner: duplicate rollups + // must have their counts SUMMED, not one dropped, so the merge needs the + // collection's own knowledge rather than a generic dedupe. See + // prepareIncidentRollup. + await appTamperingDetectionService.prepareIncidentRollup(); await appTamperingDetectionService.checkNodeReboot(); // appsRuntimeState (localzelapps): merge any pre-unique-index duplicate docs, // then enforce one doc per component identifier @@ -225,7 +351,9 @@ async function startFluxFunctions() { log.info('Preparing temporary database...'); // no need to drop temporary messages const databaseTemp = db.db(config.database.appsglobal.database); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsTemporaryMessages), { receivedAt: 1 }, { expireAfterSeconds: tempMsgTtlS }); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsTemporaryMessages), [ + { key: { receivedAt: 1 }, expireAfterSeconds: tempMsgTtlS }, + ]); log.info('Temporary database prepared'); log.info('Preparing Flux Apps locations'); @@ -239,47 +367,71 @@ async function startFluxFunctions() { // we have to create this index again here, as we need it to repair the db. As we were deleting this on every reboot (and it was only created when scannedHeight was 0) // Creating an index that already exists is a no-op - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), { hash: 1 }, { name: 'query for getting zelapp message based on hash', unique: true }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), { 'appSpecifications.version': 1 }, { name: 'query for getting app message based on version' }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), { 'appSpecifications.nodes': 1 }, { name: 'query for getting app message based on nodes' }); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), [ + { key: { hash: 1 }, name: 'query for getting zelapp message based on hash', unique: true, recover: dedupeByKey }, + { key: { 'appSpecifications.version': 1 }, name: 'query for getting app message based on version' }, + { key: { 'appSpecifications.nodes': 1 }, name: 'query for getting app message based on nodes' }, + ]); // TTL is driven by expireAt (set per-document by store functions). Migrate from old broadcastedAt-based TTL. await databaseTemp.collection(config.database.appsglobal.collections.appsLocations).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), { expireAt: 1 }, { expireAfterSeconds: 0 }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), { name: 1 }, { name: 'query for getting zelapp location based on zelapp specs name' }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), { ip: 1, name: 1 }); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'query for getting zelapp location based on zelapp specs name' }, + { key: { ip: 1, name: 1 } }, + ]); log.info('Flux Apps locations prepared'); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), { expireAt: 1 }, { expireAfterSeconds: 0 }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), { ip: 1, type: 1, dedupKey: 1 }, { unique: true }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), { broadcastedAt: 1 }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), { createdAt: 1 }); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { ip: 1, type: 1, dedupKey: 1 }, unique: true, recover: dedupeByKey }, + { key: { broadcastedAt: 1 } }, + { key: { createdAt: 1 } }, + ]); log.info('App state events collection prepared'); await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), { expireAt: 1 }, { expireAfterSeconds: 0 }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), { broadcastedAt: 1 }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), { 'data.name': 1, 'data.ip': 1 }, { unique: true }); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { broadcastedAt: 1 } }, + { key: { 'data.name': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, + ]); log.info('Signed appinstalling broadcasts collection prepared'); await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), { expireAt: 1 }, { expireAfterSeconds: 0 }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), { name: 1 }, { name: 'query for getting flux app install location based on specs name' }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), { name: 1, ip: 1 }, { name: 'query for getting flux app install location based on specs name and node ip' }); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'query for getting flux app install location based on specs name' }, + { key: { name: 1, ip: 1 }, name: 'query for getting flux app install location based on specs name and node ip' }, + ]); log.info('Flux Apps installing locations prepared'); await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).dropIndex('cachedAt_1').catch(() => {}); await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), { expireAt: 1 }, { expireAfterSeconds: 0 }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), { name: 1 }, { name: 'query for getting flux app install errors location based on specs name' }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), { name: 1, hash: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash' }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), { name: 1, hash: 1, ip: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash and node ip' }); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'query for getting flux app install errors location based on specs name' }, + { key: { name: 1, hash: 1 }, name: 'query for getting flux app install errors location based on specs name and hash' }, + { key: { name: 1, hash: 1, ip: 1 }, name: 'query for getting flux app install errors location based on specs name and hash and node ip' }, + ]); log.info('App installing errors locations prepared'); await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), { expireAt: 1 }, { expireAfterSeconds: 0 }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), { broadcastedAt: 1 }); - await ensureIndex(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), { 'data.name': 1, 'data.hash': 1, 'data.ip': 1 }, { unique: true }); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { broadcastedAt: 1 } }, + { key: { 'data.name': 1, 'data.hash': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, + ]); log.info('Signed app installing errors broadcasts collection prepared'); // This fixes an issue where the appsMessage db has NaN for valueSat. Once db is repaired on all nodes, // we can remove this. await dbHelper.repairNanInAppsMessagesDb(); + // The location table this node already holds, brought back as soon as the + // database is up. Detached and best-effort - every consumer degrades safely + // without it. On a node that has run before this is a single marker read + // against rows already in mongo, so the residential verdict and placement's + // fault domains hold their table within milliseconds of boot rather than + // behind the app-database rebuild, which neither depends on. Fetching a NEW + // baseline is the expensive half and stays in startDbDependentServices, + // where its two-million-row ingest cannot land on top of that rebuild. + ipLocationSync.restoreCachedTable().catch((err) => log.error(`ipLocationSync restore error: ${err.message}`)); + // Check for apps with incorrect volume mounts (containing /flux/ path) log.info('Checking for apps with incorrect volume mounts...'); setTimeout(() => { @@ -330,17 +482,18 @@ async function startFluxFunctions() { // Initialize app sync orchestrator and spawner const orchestrator = new AppSyncOrchestrator({ blockEmitter: explorerService.getBlockEmitter(), - getEligibleSyncPeers: (minUptime) => peerManager.getEligibleSyncPeers(minUptime) - .map((p) => ({ key: p.key, send: (msg) => p.send(msg) })), + getEligibleSyncPeers: () => peerManager.getEligibleSyncPeers() + .map((p) => ({ key: p.key, connectionId: p.connectionId, send: (msg) => p.send(msg) })), onPeerEvent: (event, cb) => peerManager.on(event, cb), offPeerEvent: (event, cb) => peerManager.removeListener(event, cb), peerCountIfAboveThreshold: () => peerManager.peerCountIfAboveThreshold(), - markSyncRequested: (key) => peerManager.markSyncRequested(key), - clearSyncRequested: () => peerManager.clearSyncRequested(), - isEnterprise: () => enterpriseNetwork.getCachedEnterpriseIdentity(), networkStateReady: () => networkStateService.waitStarted(), fluxVersion, }); + // The orchestrator issues the sync requests and holds their deadlines, so + // it is the only thing that knows whether an arriving answer is still + // wanted. The peer manager asks rather than keeping its own copy. + peerManager.syncResponseWanted = (peerSocket) => orchestrator.isSyncResponseWanted(peerSocket); nodeConfirmationService.onMessageCapabilityChange((capable) => orchestrator.onMessageCapabilityChange(capable)); peerNotification.initialize(); appSpawner.initialize(); @@ -351,7 +504,11 @@ async function startFluxFunctions() { appReconciler.setOnContainerStarted(() => peerNotification.checkAndNotifyPeersOfRunningApps()); // a removed component's in-memory controller verdict dies with it - a // reinstalled g:/r: app must await a fresh election, not inherit a stale one - appUninstaller.setOnComponentRemoved((id) => appReconciler.clearControllerDesired(id)); + appUninstaller.setOnComponentRemoved((id) => appReconciler.forgetDesiredState(id)); + // the node's address moved, so every app that survived it has to come up on + // the new one - asked for durably here rather than driven from the network + // layer, which sits underneath the reconciler and cannot require it + fluxNetworkHelper.setOnAddressChanged((apps, reason) => appReconciler.requestRestartOf(apps, reason)); log.info('App Spawner initialized'); fluxNetworkHelper.adjustFirewall(); @@ -373,6 +530,20 @@ async function startFluxFunctions() { appTamperingBlocklistService.start().catch((err) => { log.error(`appTamperingBlocklist start error: ${err.message}`); }); + // Not awaited, and started ahead of setNodeGeolocation below on purpose: the + // first tick reads geolocation from the db when there is one, and otherwise + // decides nothing and retries until the lookup this boot has landed. + // + // Injected the same way nodeStatusMonitor is, and for the same reason: the + // app list is read from a query service deep enough in the lifecycle graph + // that requiring it here would put geolocation and the network helper on + // that load path. Removing the app is not this service's job - the single + // give-up-an-app pass in advancedWorkflows does that. + residentialNodeDosService.start({ + installedAppsFn: appQueryService.installedApps, + }).catch((err) => { + log.error(`residentialNodeDos start error: ${err.message}`); + }); log.info('Flux checks operational'); fluxCommunication.initializeDiscovery(); await nodeConfirmationService.start(); @@ -391,8 +562,43 @@ async function startFluxFunctions() { await containerMountRecovery.performContainerMountRecovery().catch((error) => { log.error(`Container mount recovery service error: ${error.message}`); }); + // A file operation's container is detached from the process that started + // it, so a FluxOS restart leaves one running with nobody waiting for its + // result, and its staging directory on the volume. The recovery below + // reclaims both, after the volumes above are mounted, since it reads them. + // + // The fetch starts early so the image is in hand before the first file + // operation arrives, rather than being pulled while an owner waits on a + // request. The recovery does not depend on it: that is a host rm over names + // readdir returned, and runs on a node that can reach nothing. + // + // Not awaited: the node takes the image at its own place in a window, so + // the fleet ends up holding it without every node fetching at the same + // moment. A node that cannot reach the registry takes it from one that did, + // which only works if they have it. + volumeExecutor.startImagePrefetch(); + + log.info('Reclaiming interrupted file operations...'); + await fileOperationRecovery.recoverInterruptedFileOperations().catch((error) => { + log.error(`File operation recovery error: ${error.message}`); + }); + + // At boot, before anything installs: an app network is created per app and + // removed only by the uninstaller, so an uninstall interrupted between the + // container going and the network going leaves one behind for ever. Each + // holds an octet that getFreeFluxAppNetworkOctet cannot hand out again, and + // when the last of 255 is gone nothing can be installed on the node. + // + // Here rather than on a schedule because a sweep must not meet an install + // in progress: at boot the expected names are simply what the database + // holds, with no window in which an app has a network and no record yet. + await networkRecovery.reclaimOrphanedAppNetworks(); syncthingService.startSyncthingSentinel(); log.info('Syncthing service started'); + // Awaited: generating an identity rewrites config/userconfig.js, and that + // write is not atomic - a reload landing inside it leaves the process with + // no userconfig.initial at all. A node that already has an identity returns + // from here immediately, so this costs the fleet nothing. await pgpService.generateIdentity(); log.info('PGP service initiated'); // Ensure watchdog is installed and running on legacy OS (non-ArcaneOS) nodes @@ -433,7 +639,8 @@ async function startFluxFunctions() { }, bootDelay(30 * 1000)); setTimeout(() => { appController.stopAllNonFluxRunningApps(); - monitoringOrchestrator.startMonitoringOfApps(null, globalState.appsMonitored, appQueryService.installedApps); + // Best effort during boot — the reconciler starts monitoring per app as it settles. + monitoringOrchestrator.startMonitoringOfApps(null).catch((error) => log.error(error)); portManager.restoreAppsPortsSupport(); }, bootDelay(1 * 60 * 1000)); // Resolve this node's enterprise identity once, up front. Self-reschedules @@ -447,6 +654,13 @@ async function startFluxFunctions() { const startDbDependentServices = async () => { await globalState.waitForDbReady(); log.info('DB ready - starting db-dependent services'); + // Interim until policyStore supersedes it at the userconfig rebase (see the + // module header): keep the iplocation table fresh. The cached copy is + // already back - restoreCachedTable ran with the schema prep above - so + // what starts here is the fetch loop, whose ingest is the half worth + // keeping clear of the rebuild that just finished. Detached; placement + // degrades to /16 arithmetic without a table. + ipLocationSync.startSync().catch((err) => log.error(`ipLocationSync start error: ${err.message}`)); advancedWorkflows.checkAndRemoveEnterpriseAppsOnNonArcane(); await identityReady; try { @@ -510,13 +724,15 @@ async function startFluxFunctions() { // masterSlave self-gates on syncthingAppsFirstRun (the syncthing monitor's // first-run mount-safety must complete before any g: election), so it starts // concurrently rather than after a timed offset. + // The election reads the busy lists and the receive-only cache off + // globalState itself at each decision; they are not parameters. The + // getters return snapshots and masterSlaveApps re-invokes itself forever, + // so anything captured at this call is frozen at boot and goes quietly + // stale - which is exactly how the backup/restore guard once died. advancedWorkflows.masterSlaveApps( globalState, appQueryService.installedApps, appQueryService.listRunningApps, - globalState.receiveOnlySyncthingAppsCache, - globalState.backupInProgress, - globalState.restoreInProgress, https, ); // stops and starts g: syncthing apps when a new master is required or changed. setTimeout(() => { @@ -573,4 +789,7 @@ async function startFluxFunctions() { module.exports = { startFluxFunctions, + ensureIndex, + ensureIndexes, + dedupeByKey, }; diff --git a/ZelBack/src/services/signatureVerifier.js b/ZelBack/src/services/signatureVerifier.js index e1de891ca3..991d6a9e51 100644 --- a/ZelBack/src/services/signatureVerifier.js +++ b/ZelBack/src/services/signatureVerifier.js @@ -1,8 +1,49 @@ +const bs58check = require('bs58check'); const { pubKeyToAddr } = require('./utils/fluxCryptoUtils'); const bitcoinMessage = require('bitcoinjs-message'); const ethereumHelper = require('./ethereumHelper'); const log = require('../lib/log'); +const base58Chars = /^[1-9a-km-zA-HJ-NP-Z]+$/; +const ethAddress = /^0x[a-fA-F0-9]{40}$/; + +/** + * Whether an identity is one a signature can be verified against - a Flux ID + * (base58check P2PKH) or an Ethereum address. + * + * Login identities and app owners are both held to this. An app owner that is + * neither can never be signed for, which leaves the app unmanageable by anyone. + * + * @param {string} identity + * + * @returns {bool} isValid + */ +function isValidSigningIdentity(identity) { + if (!identity || typeof identity !== 'string') { + return false; + } + + if (identity.startsWith('0x')) { + return ethAddress.test(identity); + } + + if (identity[0] !== '1' || identity.length < 25 || identity.length > 34) { + return false; + } + + if (!base58Chars.test(identity)) { + return false; + } + + try { + // version byte + hash160. A bad checksum throws, and would fail signature + // verification just as surely as the wrong shape. + return bs58check.decode(identity).length === 21; + } catch { + return false; + } +} + /** * Verifies signature of application owner on bitcoin or ethereum networks * @@ -41,5 +82,6 @@ function verifySignature(message, address, signature) { } module.exports = { + isValidSigningIdentity, verifySignature, }; diff --git a/ZelBack/src/services/syncthingService.js b/ZelBack/src/services/syncthingService.js index 3a699a4643..65b43715b1 100644 --- a/ZelBack/src/services/syncthingService.js +++ b/ZelBack/src/services/syncthingService.js @@ -14,6 +14,7 @@ const log = require('../lib/log'); const messageHelper = require('./messageHelper'); const serviceHelper = require('./serviceHelper'); const verificationHelper = require('./verificationHelper'); +const { Privilege, authOf } = require('./utils/privileges'); const syncthingURL = `http://${config.syncthing.ip}:${config.syncthing.port}`; @@ -220,31 +221,120 @@ async function performRequest(method = 'get', urlpath = '', data, config) { return successResponse; } catch (error) { const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); + // The axios code is a category - ERR_BAD_REQUEST spans every 4xx - so the + // HTTP status rides along as itself: a caller telling "no such folder" + // (404) from a denial (403) needs the number, and the message's wording + // belongs to axios, not to us. Null when no HTTP answer arrived at all. + errorResponse.data.httpStatus = error.response?.status ?? null; return errorResponse; } } + +/** + * The failure of a syncthing request, as an exception rather than an envelope. + * + * Carries the HTTP status because absence and refusal are different answers and + * only the number separates them: a folder syncthing does not know answers 404, + * a stale api key answers 403, and axios reports both as ERR_BAD_REQUEST. A + * caller that acts on absence reads `httpStatus`; one that does not ignores it. + */ +class SyncthingError extends Error { + constructor(message, { name, code, httpStatus } = {}) { + super(message || 'syncthing request failed'); + // axios's own name and code, so the envelope the Api half rebuilds is the + // one these endpoints have always answered + this.name = name || 'SyncthingError'; + this.code = code; + this.httpStatus = httpStatus ?? null; + } +} + +/** + * A syncthing request: its data, or a throw. + * + * performRequest answers in band because that is the shape the wire needs, and + * a route serialises it unchanged. Nothing above this line wants the envelope + * or its vocabulary, so this is the seam - internal callers speak data and + * exceptions, and the Api half puts the envelope back on. + * @param {string} method HTTP method. + * @param {string} urlpath Syncthing REST path. + * @param {object} [data] Request body. + * @param {object} [config] Axios config. + * @returns {Promise<*>} The response data. + */ +async function request(method, urlpath, data, config) { + const response = await performRequest(method, urlpath, data, config); + if (response.status === 'success') return response.data; + const details = response.data || {}; + throw new SyncthingError(details.message, { + name: details.name, + code: details.code, + httpStatus: details.httpStatus, + }); +} + +/** + * The error envelope for a handler whose work threw. + * + * A SyncthingError carries the status its request failed with, which has always + * been on the wire for these endpoints; a validation error raised before any + * request went out has no status and never carried the key. + * @param {Error} error The thrown error. + * @returns {object} Message + */ +function errorEnvelope(error) { + const response = messageHelper.createErrorMessage(error.message, error.name, error.code); + if (error instanceof SyncthingError) response.data.httpStatus = error.httpStatus; + return response; +} /** * To get meta * @param {object} req Request. * @param {object} res Response. * @returns {object} Message. */ -async function getMeta(req, res) { - // does not require authentication - const response = await performRequest('get', '/meta.js'); +async function getMeta() { // "var metadata = {\"deviceID\":\"K6VOO4G-5RLTF3B-JTUFMHH-JWITKGM-63DTTMT-I6BMON6-7E3LVFW-V5WAIAO\"};\n" - return res ? res.json(response) : response; + return request('get', '/meta.js'); +} + +/** + * To get meta + * @param {object} req Request. + * @param {object} res Response. + * @returns {object} Message. + */ +async function getMetaApi(req, res) { + // does not require authentication + try { + res.json(messageHelper.createDataMessage(await getMeta())); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); + } +} + +/** + * Syncthing's own health check. The one syncthing endpoint that needs no api key. + * @returns {Promise} System health, {"status": "OK"}. + */ +async function getHealth() { + return request('get', '/rest/noauth/health'); } /** - * To get Syhcthing health + * To get Syncthing health * @param {object} req Request. * @param {object} res Response. - * @returns {object} System health, {"status": "OK"}. + * @returns {object} Message */ -async function getHealth(req, res) { - const response = await performRequest('get', '/rest/noauth/health'); - return res ? res.json(response) : response; +async function getHealthApi(req, res) { + try { + res.json(messageHelper.createDataMessage(await getHealth())); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); + } } // === STATISTICS ENDPOINTS === @@ -286,14 +376,14 @@ async function systemBrowse(req, res) { if (current) { apiPath += `?current=${current}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('get', apiPath); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -330,14 +420,14 @@ async function systemDebug(req, res) { } else if (disable) { apiPath += `?disable=${disable}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -360,14 +450,14 @@ async function systemDiscovery(req, res) { method = 'post'; apiPath += `?device=${device}&addr=${addr}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -377,14 +467,14 @@ async function systemDiscovery(req, res) { * @returns {object} Message */ async function systemErrorClear(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('post', '/rest/system/error/clear'); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -401,14 +491,14 @@ async function systemError(req, res) { if (message) { method = 'post'; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath, message); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -425,18 +515,18 @@ async function postSystemError(req, res) { req.on('end', async () => { const message = serviceHelper.ensureObject(body); try { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('post', '/rest/system/error', message); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -454,14 +544,14 @@ async function systemLog(req, res) { if (since) { apiPath += `?since=${since}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('get', apiPath); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -477,14 +567,14 @@ async function systemLogTxt(req, res) { if (since) { apiPath += `?since=${since}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('get', apiPath); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -494,37 +584,48 @@ async function systemLogTxt(req, res) { * @returns {object} Message */ async function systemPaths(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('get', '/rest/system/paths'); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** - * To pause the given device or all devices. Takes the optional parameter {device} (device ID). When omitted, pauses all devices. - * @param {object} req Request. - * @param {object} res Response. - * @returns {object} Message + * Pause a device, or every device when none is named. A paused device holds no + * connection, so every folder shared with it stops moving data until it resumes. + * @param {string} [device] Device ID. + * @returns {Promise<*>} Syncthing's answer. */ -async function systemPause(req, res) { - let { device } = req.params; - device = device || req.query.device; +async function systemPause(device) { let apiPath = '/rest/system/pause'; if (device) { apiPath += `?device=${device}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; - let response = null; - if (authorized === true) { - response = await performRequest('post', apiPath); - } else { - response = messageHelper.errUnauthorizedMessage(); + return request('post', apiPath); +} + +/** + * To pause the given device or all devices. Takes the optional parameter {device} (device ID). When omitted, pauses all devices. + * @param {object} req Request. + * @param {object} res Response. + * @returns {object} Message + */ +async function systemPauseApi(req, res) { + try { + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); + if (authorized !== true) { + res.json(messageHelper.errUnauthorizedMessage()); + return; + } + res.json(messageHelper.createDataMessage(await systemPause(req.params.device || req.query.device))); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); } - return res ? res.json(response) : response; } /** @@ -533,9 +634,23 @@ async function systemPause(req, res) { * @param {object} res Response. * @returns {object} Message */ -async function systemPing(req, res) { - const response = await performRequest('get', '/rest/system/ping'); // can also be 'post', same - return res ? res.json(response) : response; +async function systemPing() { + return request('get', '/rest/system/ping'); // can also be 'post', same +} + +/** + * Returns a {"ping": "pong"} object. + * @param {object} req Request. + * @param {object} res Response. + * @returns {object} Message + */ +async function systemPingApi(req, res) { + try { + res.json(messageHelper.createDataMessage(await systemPing())); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); + } } /** @@ -552,14 +667,14 @@ async function systemReset(req, res) { if (folder) { apiPath += `?folder=${folder}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('post', apiPath); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -579,45 +694,69 @@ async function systemResetFolderId(folderId) { } /** - * To immediately restart Syncthing - * @param {object} req Request. - * @param {object} res Response. - * @returns {object} Message + * Restart the syncthing process. Every folder's transfers stop and start again, + * which is why the folder-level nudge exists for the cases that only need one + * folder's index re-exchanged. + * @returns {Promise<*>} Syncthing's answer. */ -async function systemRestart(req, res) { +async function systemRestart() { log.info('Restarting Syncthing...'); - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; - let response = null; - if (authorized === true) { - response = await performRequest('post', '/rest/system/restart'); - } else { - response = messageHelper.errUnauthorizedMessage(); - } + const data = await request('post', '/rest/system/restart'); log.info('Syncthing restarted'); - return res ? res.json(response) : response; + return data; } /** - * To resume the given device or all devices. Takes the optional parameter {device} (device ID). When omitted, resumes all devices + * To immediately restart Syncthing * @param {object} req Request. * @param {object} res Response. * @returns {object} Message */ -async function systemResume(req, res) { - let { device } = req.params; - device = device || req.query.device; +async function systemRestartApi(req, res) { + try { + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); + if (authorized !== true) { + res.json(messageHelper.errUnauthorizedMessage()); + return; + } + res.json(messageHelper.createDataMessage(await systemRestart())); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); + } +} + +/** + * Resume a device, or every device when none is named. + * @param {string} [device] Device ID. + * @returns {Promise<*>} Syncthing's answer. + */ +async function systemResume(device) { let apiPath = '/rest/system/resume'; if (device) { apiPath += `?device=${device}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; - let response = null; - if (authorized === true) { - response = await performRequest('post', apiPath); - } else { - response = messageHelper.errUnauthorizedMessage(); + return request('post', apiPath); +} + +/** + * To resume the given device or all devices. Takes the optional parameter {device} (device ID). When omitted, resumes all devices + * @param {object} req Request. + * @param {object} res Response. + * @returns {object} Message + */ +async function systemResumeApi(req, res) { + try { + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); + if (authorized !== true) { + res.json(messageHelper.errUnauthorizedMessage()); + return; + } + res.json(messageHelper.createDataMessage(await systemResume(req.params.device || req.query.device))); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); } - return res ? res.json(response) : response; } /** @@ -627,14 +766,14 @@ async function systemResume(req, res) { * @returns {object} Message */ async function systemShutdown(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('post', '/rest/system/shutdown'); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -655,14 +794,14 @@ async function systemStatus(req, res) { * @returns {object} Message */ async function systemUpgrade(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('get', '/rest/system/upgrade'); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -672,14 +811,22 @@ async function systemUpgrade(req, res) { * @returns {object} Message */ async function postSystemUpgrade(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('post', '/rest/system/upgrade'); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); +} + +/** + * The running syncthing's version. + * @returns {Promise} Version information. + */ +async function systemVersion() { + return request('get', '/rest/system/version'); } /** @@ -688,28 +835,43 @@ async function postSystemUpgrade(req, res) { * @param {object} res Response. * @returns {object} Message */ -async function systemVersion(req, res) { - const response = await performRequest('get', '/rest/system/version'); - return res ? res.json(response) : response; +async function systemVersionApi(req, res) { + try { + res.json(messageHelper.createDataMessage(await systemVersion())); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); + } } // === CONFIG ENDPOINTS === +/** + * The entire syncthing configuration - every folder and every device. + * @returns {Promise} The configuration. + */ +async function getConfig() { + return request('get', '/rest/config'); +} + /** * Returns the entire config. * @param {object} req Request. * @param {object} res Response. * @returns {object} Message */ -async function getConfig(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; - let response = null; - if (authorized === true) { - response = await performRequest('get', '/rest/config'); - } else { - response = messageHelper.errUnauthorizedMessage(); +async function getConfigApi(req, res) { + try { + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); + if (authorized !== true) { + res.json(messageHelper.errUnauthorizedMessage()); + return; + } + res.json(messageHelper.createDataMessage(await getConfig())); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); } - return res ? res.json(response) : response; } /** @@ -727,18 +889,18 @@ async function postConfig(req, res) { try { const processedBody = serviceHelper.ensureObject(body); const newConfig = processedBody.config; - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('put', '/rest/config', newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -755,59 +917,65 @@ async function getConfigRestartRequired(req, res) { } /** - * Returns the folder for the given ID. - * @param {object} req Request. - * @param {object} res Response. - * @returns {object} Message + * The configured folders, or the one folder with the given id. + * @param {string} [id] Folder ID. Omitted, every folder. + * @returns {Promise} The folder configuration. */ -async function getConfigFolders(req, res) { - if (!req) { - // eslint-disable-next-line no-param-reassign - req = { - params: {}, - query: {}, - }; - } - let { id } = req.params; - id = id || req.query.id; +async function getConfigFolders(id) { let apiPath = '/rest/config/folders'; if (id) { if (!goodSyncthingChars.test(id)) { - const response = messageHelper.createErrorMessage('Invalid ID supplied'); - return res ? res.json(response) : response; + throw new Error('Invalid ID supplied'); } apiPath += `/${id}`; } - const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return request('get', apiPath); } /** - * Returns the device for the given ID. + * Returns the folder for the given ID. * @param {object} req Request. * @param {object} res Response. * @returns {object} Message */ -async function getConfigDevices(req, res) { - if (!req) { - // eslint-disable-next-line no-param-reassign - req = { - params: {}, - query: {}, - }; +async function getConfigFoldersApi(req, res) { + try { + res.json(messageHelper.createDataMessage(await getConfigFolders(req.params.id || req.query.id))); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); } - let { id } = req.params; - id = id || req.query.id; +} + +/** + * The configured devices, or the one device with the given id. + * @param {string} [id] Device ID. Omitted, every device. + * @returns {Promise} The device configuration. + */ +async function getConfigDevices(id) { let apiPath = '/rest/config/devices'; if (id) { if (!goodSyncthingChars.test(id)) { - const response = messageHelper.createErrorMessage('Invalid ID supplied'); - return res ? res.json(response) : response; + throw new Error('Invalid ID supplied'); } apiPath += `/${id}`; } - const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return request('get', apiPath); +} + +/** + * Returns the device for the given ID. + * @param {object} req Request. + * @param {object} res Response. + * @returns {object} Message + */ +async function getConfigDevicesApi(req, res) { + try { + res.json(messageHelper.createDataMessage(await getConfigDevices(req.params.id || req.query.id))); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); + } } /** @@ -847,18 +1015,18 @@ async function postConfigFolders(req, res) { const newConfig = processedBody.config; const { id } = processedBody; const method = (processedBody.method || 'post').toLowerCase(); - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await adjustConfigFolders(method, newConfig, id); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -900,18 +1068,18 @@ async function postConfigDevices(req, res) { const newConfig = processedBody.config; const { id } = processedBody; const method = (processedBody.method || 'post').toLowerCase(); - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await adjustConfigDevices(method, newConfig, id); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -922,9 +1090,23 @@ async function postConfigDevices(req, res) { * @param {object} res Response. * @returns {object} Message */ -async function getConfigDefaultsFolder(req, res) { - const response = await performRequest('get', '/rest/config/defaults/folder'); - return res ? res.json(response) : response; +async function getConfigDefaultsFolder() { + return request('get', '/rest/config/defaults/folder'); +} + +/** + * Returns the default folder config. + * @param {object} req Request. + * @param {object} res Response. + * @returns {object} Message + */ +async function getConfigDefaultsFolderApi(req, res) { + try { + res.json(messageHelper.createDataMessage(await getConfigDefaultsFolder())); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); + } } /** @@ -967,18 +1149,18 @@ async function postConfigDefaultsFolder(req, res) { const processedBody = serviceHelper.ensureObject(body); const newConfig = processedBody.config; const method = (processedBody.method || 'put').toLowerCase(); - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await adjustConfigDefaultsFolder(method, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -999,18 +1181,18 @@ async function postConfigDefaultsDevice(req, res) { const processedBody = serviceHelper.ensureObject(body); const newConfig = processedBody.config; const method = (processedBody.method || 'put').toLowerCase(); - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, '/rest/config/defaults/device', newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1042,18 +1224,18 @@ async function postConfigDefaultsIgnores(req, res) { const processedBody = serviceHelper.ensureObject(body); const newConfig = processedBody.config; const method = 'put'; - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, '/rest/config/defaults/ignores', newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1064,27 +1246,51 @@ async function postConfigDefaultsIgnores(req, res) { * @param {object} res Response. * @returns {object} Message */ -async function getConfigOptions(req, res) { - const response = await performRequest('get', '/rest/config/options'); - - return res ? res.json(response) : response; +async function getConfigOptions() { + return request('get', '/rest/config/options'); } /** - * Returns the gui object + * Returns the options. * @param {object} req Request. * @param {object} res Response. * @returns {object} Message */ -async function getConfigGui(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; - let response = null; - if (authorized === true) { - response = await performRequest('get', '/rest/config/gui'); - } else { - response = messageHelper.errUnauthorizedMessage(); +async function getConfigOptionsApi(req, res) { + try { + res.json(messageHelper.createDataMessage(await getConfigOptions())); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); + } +} + +/** + * The syncthing GUI's own configuration. + * @returns {Promise} The gui configuration. + */ +async function getConfigGui() { + return request('get', '/rest/config/gui'); +} + +/** + * To show the GUI configuration. Flux team only. + * @param {object} req Request. + * @param {object} res Response. + * @returns {Promise} + */ +async function getConfigGuiApi(req, res) { + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); + if (authorized !== true) { + res.json(messageHelper.errUnauthorizedMessage()); + return; + } + try { + res.json(messageHelper.createDataMessage(await getConfigGui())); + } catch (error) { + log.error(error); + res.json(errorEnvelope(error)); } - return res ? res.json(response) : response; } /** @@ -1127,18 +1333,18 @@ async function postConfigOptions(req, res) { const processedBody = serviceHelper.ensureObject(body); const newConfig = processedBody.config; const method = (processedBody.method || 'put').toLowerCase(); - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await adjustConfigOptions(method, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1159,18 +1365,18 @@ async function postConfigGui(req, res) { const processedBody = serviceHelper.ensureObject(body); const newConfig = processedBody.config; const method = (processedBody.method || 'put').toLowerCase(); - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, '/rest/config/gui', newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1191,18 +1397,18 @@ async function postConfigLdap(req, res) { const processedBody = serviceHelper.ensureObject(body); const newConfig = processedBody.config; const method = (processedBody.method || 'put').toLowerCase(); - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, '/rest/config/ldap', newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1241,18 +1447,18 @@ async function postClusterPendigDevices(req, res) { if (device) { apiPath += `?device=${device}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1289,18 +1495,18 @@ async function postClusterPendigFolders(req, res) { if (folder) { apiPath += `?folder=${folder}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1336,11 +1542,11 @@ async function getFolderErrors(req, res) { throw new Error('folder parameter is mandatory'); } const response = await getFolderIdErrors(folder); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -1361,11 +1567,11 @@ async function getFolderVersions(req, res) { throw new Error('folder parameter is mandatory'); } const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -1390,18 +1596,18 @@ async function postFolderVersions(req, res) { if (folder) { apiPath += `?folder=${folder}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1434,42 +1640,48 @@ async function getDbBrowse(req, res) { const qqStr = qs.stringify(qq); apiPath += `?${qqStr}`; const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } +/** + * How complete a folder is, optionally as one device sees it. + * + * `remoteState` is only set when a device is named, and it is the connectivity + * discriminator a caller needs: completion is computed from the last known + * index, so an offline peer still reports 100. + * @param {object} [selector] Selector. + * @param {string} [selector.folder] Folder ID. + * @param {string} [selector.device] Device ID. + * @returns {Promise} Completion percentage and byte/item counts. + */ +async function getDbCompletion({ folder, device } = {}) { + let apiPath = '/rest/db/completion'; + const query = qs.stringify({ folder, device }); + if (query) apiPath += `?${query}`; + return request('get', apiPath); +} + /** * Returns the completion percentage (0 to 100) and byte / item counts. Takes optional {device} and {folder} parameters. * @param {object} req Request. * @param {object} res Response. * @returns {object} Message */ -async function getDbCompletion(req, res) { +async function getDbCompletionApi(req, res) { try { - // tolerate being called internally with only a query (no params): callers like - // checkIfPeersAreSynced pass { query: {...} }, so req.params is undefined and a - // bare `const { folder } = req.params` would throw. Default the containers. - const { params = {}, query = {} } = req || {}; - const folder = params.folder || query.folder; - const device = params.device || query.device; - let apiPath = '/rest/db/completion'; - if (folder || device) apiPath += '?'; - const qq = { - folder, - device, - }; - const qqStr = qs.stringify(qq); - apiPath += `${qqStr}`; - const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + const data = await getDbCompletion({ + folder: req.params.folder || req.query.folder, + device: req.params.device || req.query.device, + }); + res.json(messageHelper.createDataMessage(data)); } catch (error) { log.error(error); - const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + res.json(errorEnvelope(error)); } } @@ -1494,11 +1706,11 @@ async function getDbFile(req, res) { const qqStr = qs.stringify(qq); apiPath += `${qqStr}`; const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -1515,14 +1727,39 @@ async function getDbIgnores(req, res) { let apiPath = '/rest/db/ignores'; if (folder) apiPath += `?folder=${folder}`; const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } +/** + * Read a folder's ignore patterns, for internal callers. Returns the standard + * message shape - { status, data: { ignore, expanded } } on success - and never + * throws, so the caller checks status rather than catching. + * @param {string} folderId syncthing folder id + * @returns {Promise} message + */ +async function getFolderIgnores(folderId) { + return performRequest('get', `/rest/db/ignores?folder=${encodeURIComponent(folderId)}`); +} + +/** + * Set a folder's ignore patterns, for internal callers. Syncthing owns and + * writes .stignore itself (atomically, and it never replicates it), so this is + * how FluxOS sets the ignores rather than writing the file. REPLACES the whole + * set - pass the complete desired list. Returns the standard message shape and + * never throws. + * @param {string} folderId syncthing folder id + * @param {Array} lines the full ignore pattern list + * @returns {Promise} message + */ +async function setFolderIgnores(folderId, lines) { + return performRequest('post', `/rest/db/ignores?folder=${encodeURIComponent(folderId)}`, { ignore: lines }); +} + /** * Returns the list of files which were changed locally in a receive-only folder. Takes one mandatory parameter, {folder} * @param {object} req Request. @@ -1540,11 +1777,11 @@ async function getDbLocalchanged(req, res) { throw new Error('folder parameter is mandatory'); } const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -1565,11 +1802,11 @@ async function getDbNeed(req, res) { throw new Error('folder parameter is mandatory'); } const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -1597,12 +1834,27 @@ async function getDbRemoteNeed(req, res) { throw new Error('device parameter is mandatory'); } const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); + } +} + +/** + * A folder's current status. + * + * Throws with `httpStatus` 404 when syncthing holds no such folder, which is an + * answer rather than a failure - the caller that acts on absence reads it. + * @param {string} folder Folder ID. + * @returns {Promise} The folder status. + */ +async function getDbStatus(folder) { + if (!folder) { + throw new Error('folder parameter is mandatory'); } + return request('get', `/rest/db/status?folder=${folder}`); } /** @@ -1611,21 +1863,13 @@ async function getDbRemoteNeed(req, res) { * @param {object} res Response. * @returns {object} Message */ -async function getDbStatus(req, res) { +async function getDbStatusApi(req, res) { try { - const folder = req?.params?.folder || req?.query?.folder; - let apiPath = '/rest/db/status'; - if (folder) { - apiPath += `?folder=${folder}`; - } else { - throw new Error('folder parameter is mandatory'); - } - const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + const data = await getDbStatus(req.params.folder || req.query.folder); + res.json(messageHelper.createDataMessage(data)); } catch (error) { log.error(error); - const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + res.json(errorEnvelope(error)); } } @@ -1650,18 +1894,24 @@ async function postDbIgnores(req, res) { if (folder) { apiPath += `?folder=${folder}`; } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + // fluxteam, not adminandfluxteam like its siblings. .stignore decides what + // LEAVES this node for an app the node operator does not own, and a pattern + // dropped here replicates that app's backup and operation staging to every + // other node running it - so the blast radius of this one call is the fleet, + // not the box. Reading the volume is the operator's already; choosing what + // the network carries is not. + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1689,18 +1939,18 @@ async function postDbOverride(req, res) { } else { throw new Error('folder parameter is mandatory'); } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1734,18 +1984,18 @@ async function postDbPrio(req, res) { } else { throw new Error('file parameter is mandatory'); } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1773,18 +2023,18 @@ async function postDbRevert(req, res) { } else { throw new Error('folder parameter is mandatory'); } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1831,18 +2081,18 @@ async function postDbScan(req, res) { }; const qqStr = qs.stringify(qq); apiPath += `${qqStr}`; - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest(method, apiPath, newConfig); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } }); } @@ -1856,14 +2106,14 @@ async function postDbScan(req, res) { * @returns {object} Message */ async function debugPeerCompletion(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('get', '/rest/debug/peerCompletion'); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -1873,14 +2123,14 @@ async function debugPeerCompletion(req, res) { * @returns {object} Message */ async function debugHttpmetrics(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('get', '/rest/debug/httpmetrics'); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -1890,7 +2140,7 @@ async function debugHttpmetrics(req, res) { * @returns {object} Message */ async function debugCpuprof(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { try { @@ -1905,12 +2155,12 @@ async function debugCpuprof(req, res) { } return response.data.pipe(res); } catch (error) { - return res ? res.json(error) : JSON.stringify(error); + return res.json(error); } } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -1920,7 +2170,7 @@ async function debugCpuprof(req, res) { * @returns {object} Message */ async function debugHeapprof(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { try { @@ -1935,12 +2185,12 @@ async function debugHeapprof(req, res) { } return response.data.pipe(res); } catch (error) { - return res ? res.json(error) : JSON.stringify(error); + return res.json(error); } } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -1950,14 +2200,14 @@ async function debugHeapprof(req, res) { * @returns {object} Message */ async function debugSupport(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { response = await performRequest('get', '/rest/debug/support', undefined, 60000); } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } /** @@ -1983,7 +2233,7 @@ async function debugFile(req, res) { } else { throw new Error('file parameter is mandatory'); } - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { try { @@ -1998,69 +2248,76 @@ async function debugFile(req, res) { } return response.data.pipe(res); } catch (error) { - return res ? res.json(error) : JSON.stringify(error); + return res.json(error); } } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } // === EVENT ENDPOINTS === +/** + * Syncthing's event stream, from `since` onwards. + * + * The endpoint long-polls: with a `timeout` hold requested, syncthing keeps the + * request open up to that many seconds before answering "nothing new", so the + * client-side abort must come strictly after the server-side hold rather than + * at the shared instance's 5s default. + * @param {object} [options] Options. + * @param {string} [options.events] Comma separated event types to subscribe to. + * @param {number} [options.since] Last event id already seen. + * @param {number} [options.limit] Maximum events to return. + * @param {number} [options.timeout] Seconds syncthing may hold the request open. + * @param {AbortSignal} [options.signal] Interrupts the long poll on shutdown. + * @returns {Promise} The events. + */ +async function getEvents({ + events, since, limit, timeout, signal, +} = {}) { + let apiPath = '/rest/events'; + const query = qs.stringify({ + events, since, limit, timeout, + }); + if (query) apiPath += `?${query}`; + const holdS = Number(timeout); + const config = {}; + if (Number.isFinite(holdS) && holdS > 0) config.timeout = (holdS + 10) * 1000; + // axios honours config.signal + if (signal) config.signal = signal; + // 3rd arg is the request body (none for GET); 4th is the axios config + return request('get', apiPath, undefined, config); +} + /** * To receive Syncthing events. takes {events}, {since}, {limit} and {timeout} parameters to filter the result. * @param {object} req Request. * @param {object} res Response. * @returns {object} Message */ -async function getEvents(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; - if (authorized !== true) { - const response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; - } +async function getEventsApi(req, res) { try { - let { events } = req.params; - events = events || req.query.events; - let { since } = req.params; - since = since || req.query.since; - let { limit } = req.params; - limit = limit || req.query.limit; - let { timeout } = req.params; - timeout = timeout || req.query.timeout; - let apiPath = '/rest/events'; - if (events || since || limit || timeout) apiPath += '?'; - const qq = { - events, - since, - limit, - timeout, - }; - const qqStr = qs.stringify(qq); - apiPath += `${qqStr}`; - // the events endpoint long-polls: with a `timeout` hold requested, syncthing - // keeps the request open up to that many seconds before answering "nothing - // new" - the client-side abort must come strictly after the server-side hold, - // not at the shared instance's 5s default - const holdS = Number(timeout); - const requestConfig = {}; - if (Number.isFinite(holdS) && holdS > 0) requestConfig.timeout = (holdS + 10) * 1000; - // a caller (e.g. the events consumer) may pass an AbortSignal to interrupt the - // long-poll on shutdown; axios honours config.signal - if (req.signal) requestConfig.signal = req.signal; - // 3rd arg is the request body (none for GET); 4th is the axios config - const response = await performRequest('get', apiPath, undefined, requestConfig); - return res ? res.json(response) : response; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); + if (authorized !== true) { + res.json(messageHelper.errUnauthorizedMessage()); + return; + } + const data = await getEvents({ + events: req.params.events || req.query.events, + since: req.params.since || req.query.since, + limit: req.params.limit || req.query.limit, + timeout: req.params.timeout || req.query.timeout, + }); + res.json(messageHelper.createDataMessage(data)); } catch (error) { log.error(error); - const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + res.json(errorEnvelope(error)); } } @@ -2071,10 +2328,10 @@ async function getEvents(req, res) { * @returns {object} Message */ async function getEventsDisk(req, res) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } try { let { since } = req.params; @@ -2093,11 +2350,11 @@ async function getEventsDisk(req, res) { const qqStr = qs.stringify(qq); apiPath += `${qqStr}`; const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -2120,11 +2377,11 @@ async function getSvcDeviceID(req, res) { throw new Error('id parameter is mandatory'); } const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -2142,20 +2399,20 @@ async function getSvcRandomString(req, res) { if (length) { const parsedLength = Number(length); if (!Number.isFinite(parsedLength) || parsedLength < 0 || parsedLength > 10000) { - const authorized = res ? await verificationHelper.verifyPrivilege('adminandfluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized !== true) { const response = messageHelper.errUnauthorizedMessage(); - return res ? res.json(response) : response; + return res.json(response); } } apiPath += `?length=${length}`; } const response = await performRequest('get', apiPath); - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -2186,9 +2443,9 @@ async function getDeviceId() { // not sure why this is necessary. If we only want one at a time, should implement a cache too. await asyncLock.enable(); - let meta = {}; - let healthy = {}; - let pingResponse = {}; + let meta = null; + let healthy = null; + let pingResponse = null; try { // if aborted, axios will reject immediately, without any network activity @@ -2204,9 +2461,9 @@ async function getDeviceId() { if (stc.aborted) return null; - if (meta.status === 'success' && pingResponse.data?.ping === 'pong' && healthy.data?.status === 'OK') { + if (meta && pingResponse?.ping === 'pong' && healthy?.status === 'OK') { syncthingStatusOk = true; - const adjustedString = meta.data.slice(15).slice(0, -2); + const adjustedString = meta.slice(15).slice(0, -2); const deviceObject = JSON.parse(adjustedString); const { deviceID } = deviceObject; return deviceID; @@ -2296,8 +2553,10 @@ async function adjustSyncthing() { log.info('Adjusting syncthing.'); try { - const currentConfigOptions = await getConfigOptions(); - const currentDefaultsFolderOptions = await getConfigDefaultsFolder(); + // best effort, as before: a read that fails leaves that block's settings + // alone and the next pass retries, rather than abandoning the whole adjust + const currentConfigOptions = await getConfigOptions().catch(() => null); + const currentDefaultsFolderOptions = await getConfigDefaultsFolder().catch(() => null); // use env so can run this module as standalone for testing const apiPort = process.env.FLUX_APIPORT || userconfig?.initial.apiport || config.server?.apiport; const myPort = +apiPort + 2; // end with 9 eg 16139 @@ -2315,31 +2574,30 @@ async function adjustSyncthing() { sendXattrs: true, maxConflicts: 0, }; - if (currentConfigOptions.status === 'success') { - if (currentConfigOptions.data.globalAnnounceEnabled !== newConfig.globalAnnounceEnabled - || currentConfigOptions.data.localAnnounceEnabled !== newConfig.localAnnounceEnabled - || currentConfigOptions.data.natEnabled !== newConfig.natEnabled - || serviceHelper.ensureString(currentConfigOptions.data.listenAddresses) !== serviceHelper.ensureString(newConfig.listenAddresses)) { + if (currentConfigOptions) { + if (currentConfigOptions.globalAnnounceEnabled !== newConfig.globalAnnounceEnabled + || currentConfigOptions.localAnnounceEnabled !== newConfig.localAnnounceEnabled + || currentConfigOptions.natEnabled !== newConfig.natEnabled + || serviceHelper.ensureString(currentConfigOptions.listenAddresses) !== serviceHelper.ensureString(newConfig.listenAddresses)) { // patch our config await adjustConfigOptions('patch', newConfig); } } - if (currentDefaultsFolderOptions.status === 'success') { - if (currentDefaultsFolderOptions.data.syncOwnership !== newConfigDefaultFolders.syncOwnership - || currentDefaultsFolderOptions.data.sendOwnership !== newConfigDefaultFolders.sendOwnership - || currentDefaultsFolderOptions.data.syncXattrs !== newConfigDefaultFolders.syncXattrs - || currentDefaultsFolderOptions.data.sendXattrs !== newConfigDefaultFolders.sendXattrs) { + if (currentDefaultsFolderOptions) { + if (currentDefaultsFolderOptions.syncOwnership !== newConfigDefaultFolders.syncOwnership + || currentDefaultsFolderOptions.sendOwnership !== newConfigDefaultFolders.sendOwnership + || currentDefaultsFolderOptions.syncXattrs !== newConfigDefaultFolders.syncXattrs + || currentDefaultsFolderOptions.sendXattrs !== newConfigDefaultFolders.sendXattrs) { // patch our defaults folder config await adjustConfigDefaultsFolder('patch', newConfigDefaultFolders); } } // remove default folder - const allFolders = await getConfigFolders(); - if (allFolders.status === 'success') { - const defaultFolderExists = allFolders.data.find((syncthingFolder) => syncthingFolder.id === 'default'); - if (defaultFolderExists) { - await adjustConfigFolders('delete', undefined, 'default'); - } + // best effort: a configuration that cannot be read leaves the default folder + // in place, exactly as an unsuccessful read did before + const allFolders = await getConfigFolders().catch(() => []); + if (allFolders.find((syncthingFolder) => syncthingFolder.id === 'default')) { + await adjustConfigFolders('delete', undefined, 'default'); } // enable gui debugging for development nodes only if (config.development) { @@ -2355,10 +2613,6 @@ async function adjustSyncthing() { } } } - const restartRequired = await getConfigRestartRequired(); - if (restartRequired.status === 'success' && restartRequired.data.requiresRestart === true) { - await systemRestart(); - } } catch (error) { log.error(error); } @@ -2802,6 +3056,19 @@ async function collectSyncthingMetrics() { errors, }; metrics.overall.issues.push(`Folder ${folder.label || folderId} has ${errors + pullErrors} error(s)`); + // The counts alone cannot be diagnosed from a log dump - + // surface the file-level causes, bounded so a sick folder + // cannot flood the log. + // eslint-disable-next-line no-await-in-loop + const folderErrorsResponse = await getFolderIdErrors(folderId); + const fileErrors = folderErrorsResponse.status === 'success' ? (folderErrorsResponse.data?.errors ?? []) : []; + const shown = fileErrors.slice(0, 5); + shown.forEach((fileError) => { + log.error(`Syncthing folder ${folder.label || folderId}: ${fileError.path}: ${fileError.error}`); + }); + if (fileErrors.length > shown.length) { + log.error(`Syncthing folder ${folder.label || folderId}: ${fileErrors.length - shown.length} further file error(s) not shown`); + } } } } catch (error) { @@ -2831,14 +3098,22 @@ async function collectSyncthingMetrics() { metrics.overall.issues.push(`Failed to collect folder metrics: ${error.message}`); } - // Collect system errors + // Drain syncthing's system error buffer. The buffer is cumulative for + // the daemon's lifetime and the daemon outlives FluxOS restarts, so each + // entry is an occurrence, not a state: log its content, clear the + // buffer, and report unhealthy only for the pass the errors arrived in. try { const errorsResponse = await performRequest('get', '/rest/system/error'); - if (errorsResponse.status === 'success' && errorsResponse.data?.errors) { + if (errorsResponse.status === 'success' && errorsResponse.data?.errors?.length) { metrics.errors.system = errorsResponse.data.errors; - if (metrics.errors.system.length > 0) { - metrics.overall.healthy = false; - metrics.overall.issues.push(`${metrics.errors.system.length}`); + metrics.overall.healthy = false; + metrics.overall.issues.push(`${metrics.errors.system.length} syncthing system error(s) this pass`); + metrics.errors.system.forEach((systemError) => { + log.error(`Syncthing system error at ${systemError.when}: ${systemError.message}`); + }); + const clearResponse = await performRequest('post', '/rest/system/error/clear'); + if (clearResponse.status !== 'success') { + log.warn(`Failed to clear syncthing system errors, they will re-log next pass: ${clearResponse.data?.message}`); } } } catch (error) { @@ -2877,7 +3152,7 @@ function saveMetricsSnapshot(metrics) { */ async function getSyncthingMetrics(req, res) { try { - const authorized = res ? await verificationHelper.verifyPrivilege('fluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { const metrics = await collectSyncthingMetrics(); @@ -2885,11 +3160,11 @@ async function getSyncthingMetrics(req, res) { } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -2901,7 +3176,7 @@ async function getSyncthingMetrics(req, res) { */ async function getSyncthingHealthSummary(req, res) { try { - const authorized = res ? await verificationHelper.verifyPrivilege('fluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { const metrics = await collectSyncthingMetrics(); @@ -2937,11 +3212,11 @@ async function getSyncthingHealthSummary(req, res) { } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -2953,7 +3228,7 @@ async function getSyncthingHealthSummary(req, res) { */ async function getSyncthingMetricsHistory(req, res) { try { - const authorized = res ? await verificationHelper.verifyPrivilege('fluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { let { limit } = req.params; @@ -2969,11 +3244,11 @@ async function getSyncthingMetricsHistory(req, res) { } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -3311,7 +3586,7 @@ async function getPeerSyncDiagnostics() { */ async function getPeerSyncDiagnosticsApi(req, res) { try { - const authorized = res ? await verificationHelper.verifyPrivilege('fluxteam', req) : true; + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); let response = null; if (authorized === true) { const diagnostics = await getPeerSyncDiagnostics(); @@ -3319,11 +3594,11 @@ async function getPeerSyncDiagnosticsApi(req, res) { } else { response = messageHelper.errUnauthorizedMessage(); } - return res ? res.json(response) : response; + return res.json(response); } catch (error) { log.error(error); const errorResponse = messageHelper.createErrorMessage(error.message, error.name, error.code); - return res ? res.json(errorResponse) : errorResponse; + return res.json(errorResponse); } } @@ -3333,7 +3608,9 @@ module.exports = { getDeviceId, getDeviceIdApi, getMeta, + getMetaApi, getHealth, + getHealthApi, statsDevice, statsFolder, systemBrowse, @@ -3347,33 +3624,44 @@ module.exports = { systemLogTxt, systemPaths, systemPause, + systemPauseApi, systemReset, systemResetFolderId, systemRestart, + systemRestartApi, systemResume, + systemResumeApi, systemShutdown, systemStatus, systemUpgrade, postSystemUpgrade, systemVersion, + systemVersionApi, systemPing, + systemPingApi, syncthingController, // CONFIG getConfig, + getConfigApi, postConfig, getConfigRestartRequired, getConfigFolders, + getConfigFoldersApi, getConfigDevices, + getConfigDevicesApi, postConfigFolders, postConfigDevices, getConfigDefaultsFolder, + getConfigDefaultsFolderApi, getConfigDefaultsDevice, postConfigDefaultsFolder, postConfigDefaultsDevice, getConfigDefaultsIgnores, postConfigDefaultsIgnores, getConfigOptions, + getConfigOptionsApi, getConfigGui, + getConfigGuiApi, getConfigLdap, postConfigOptions, postConfigGui, @@ -3391,12 +3679,16 @@ module.exports = { // DATABASE ENDPOINTS getDbBrowse, getDbCompletion, + getDbCompletionApi, getDbFile, getDbIgnores, + getFolderIgnores, + setFolderIgnores, getDbLocalchanged, getDbNeed, getDbRemoteNeed, getDbStatus, + getDbStatusApi, postDbIgnores, postDbOverride, postDbPrio, @@ -3405,6 +3697,7 @@ module.exports = { postDbScan, // EVENTS getEvents, + getEventsApi, getEventsDisk, // MISC getSvcDeviceID, diff --git a/ZelBack/src/services/systemService.js b/ZelBack/src/services/systemService.js index 08095efd53..bdd21587e4 100644 --- a/ZelBack/src/services/systemService.js +++ b/ZelBack/src/services/systemService.js @@ -14,6 +14,7 @@ const log = require('../lib/log'); const serviceHelper = require('./serviceHelper'); const syncthingService = require('./syncthingService'); const fifoQueue = require('./utils/fifoQueue'); +const fluxEventBus = require('./utils/fluxEventBus'); const daemonServiceUtils = require('./daemonService/daemonServiceUtils'); const isArcane = Boolean(process.env.FLUXOS_PATH); @@ -24,16 +25,38 @@ const isArcane = Boolean(process.env.FLUXOS_PATH); let syncthingTimer = null; /** - * A FIFO queue used to store and run apt commands - * @type {fifoQueue.FifoQueue} + * A FIFO queue used to store and run apt commands. + * @type {fifoQueue.FifoQueue|null} */ -const aptQueue = new fifoQueue.FifoQueue(); +let aptQueue = null; /** - * For testing + * The apt queue, built on first use. + * + * Built COMPLETE, and built LATE. Complete because a queue that arrives without + * its worker has to be finished off later by whoever happens to get there first, + * and until they do it silently accepts work it cannot run - so its behaviour + * depends on call order, and a test that touches it inherits whatever the last + * one left behind. Late because a module that is merely imported should not have + * started anything: an Arcane node never queues apt work at all, and a script or + * a test reaching in here for one unrelated function should not acquire a worker + * it did not ask for. Constructing the whole thing on first use is what satisfies + * both - there is no window in which the queue exists without its worker. + * * @returns {fifoQueue.FifoQueue} */ function getQueue() { + if (aptQueue) return aptQueue; + + aptQueue = new fifoQueue.FifoQueue({ worker: aptRunner }); + aptQueue.on('failed', monitorAptCache); + // The queue has stopped handing this one back. Worth saying out loud: the caller + // took its error long ago, so without this the work is simply never done again and + // nothing ever mentions it. + aptQueue.on('abandoned', ({ options, error, cycles }) => { + log.error(`Giving up on apt-get ${options.command} after ${cycles} attempts: ${error.message}`); + }); + return aptQueue; } @@ -61,6 +84,7 @@ async function aptRunner(options = {}) { // any apt after 1.9.11 has the DPkg::Lock::Timeout option. const params = [ '-y', // Auto-answer yes to prompts + '--no-install-recommends', // Only what the package needs, not what it suggests '-o', `DPkg::Lock::Timeout=${timeout}`, // How long to wait for a lock '-o', 'Dpkg::Options::=--force-confdef', // Use default for new config files '-o', 'Dpkg::Options::=--force-confold', // Keep old config files on conflict @@ -74,6 +98,14 @@ async function aptRunner(options = {}) { params: envParams, }); + // A fact, not a cadence: a legacy boot runs a handful of apt commands, and + // WHICH of them completed - and in what order - is the only direct evidence + // that the queue carried on past one that failed. Published at the single point + // every apt command passes through, on both exits, so a reader sees the failure + // and the work behind it as two events rather than inferring the second from + // packages appearing on disk some time later. + fluxEventBus.publish('system:apt-command', { command: options.command, ok: !error }); + // this is so this command can be retried by the worker runner if (error) throw error; @@ -111,11 +143,16 @@ async function cacheUpdateTime() { /** * @param {string} command The command to run - * @param {{params?: Array, timeout?: number, retries?: number, wait?: Boolean}} options + * @param {{params?: Array, timeout?: number, retries?: number, retryDelay?: number, + * retainErrors?: boolean, wait?: Boolean}} options * * params: params to pass to command - * timeout: how many seconds to wait (60 default) - * retries: how many times to retry (3 default) + * timeout: how many seconds to wait (180 default) + * retries: how many times to retry (queue default, 5) + * retryDelay: how long to wait between attempts in ms (queue default, 60000) + * retainErrors: keep a failed task and try it again later rather than dropping + * it (queue default, true). Every one of these is forwarded, so an option left + * unset takes the queue's default rather than this function's. * wait: should the queue item be awaited * * @returns {Promise} @@ -130,8 +167,16 @@ async function queueAptGetCommand(command, options = {}) { const wait = options.wait || false; const commandOptions = { command, params, timeout: options.timeout }; - const workerOptions = { retries: options.retries }; - return aptQueue.push({ commandOptions, workerOptions }, wait); + // Every worker option the caller set, not just retries. updateAptCache asks for + // retainErrors: false so a failed update is dropped rather than left at the head + // of the queue; dropping that request here left the queue holding a task it was + // told to bin, ready to run again the moment anything resumed it. + const workerOptions = { + retries: options.retries, + retryDelay: options.retryDelay, + retainErrors: options.retainErrors, + }; + return getQueue().push({ commandOptions, workerOptions }, wait); } /** @@ -321,12 +366,12 @@ async function addSyncthingRepository() { const packageName = 'syncthing'; // syncthing does this weird const dist = 'syncthing'; - const sourceUrl = 'https://apt.syncthing.net/'; + const sourceUrl = config.syncthing.aptSourceUrl; const components = ['stable-v2']; const sourceOptions = ['signed-by=/usr/share/keyrings/syncthing-archive-keyring.gpg']; // keyring vars - const keyUrl = 'https://syncthing.net/release-key.gpg'; + const keyUrl = config.syncthing.releaseKeyUrl; const keyringName = 'syncthing-archive-keyring.gpg'; // this will log errors @@ -477,8 +522,13 @@ async function ensurePackageVersion(systemPackage, requiredVersion, currentVersi if (!actualVersion) { log.info(`Package ${systemPackage} not found on system`); - await upgradePackage(systemPackage); - return true; // Package was installed/upgraded + // upgradePackage answers whether it FAILED. This branch used to discard that + // and report the install regardless, so an apt-get that could not find the + // package read back the same as one that installed it. The upgrade branch + // below has always read it. + const installError = await upgradePackage(systemPackage); + if (installError) log.error(`Package ${systemPackage} could not be installed`); + return !installError; } log.info(`Package ${systemPackage} version ${actualVersion} found`); @@ -507,13 +557,11 @@ async function monitorSyncthingPackage() { try { if (syncthingTimer) return; - await addSyncthingRepository(); - const versionChecker = async () => { const { data: { data }, } = await axios - .get('https://stats.runonflux.io/getmodulesminimumversions', { + .get(`${config.stats.baseUrl}/getmodulesminimumversions`, { timeout: 10000, }) .catch((error) => { @@ -532,7 +580,14 @@ async function monitorSyncthingPackage() { minSyncthingVersion, ); - if (upToDate) return; + if (upToDate) return false; + + // After the up-to-date return - a current node is not handed a keyring + // and a source it will never read - and BEFORE the sources rewrite: the + // rewrite fails on a node with no source file at all, and its failure + // must not stand between that node and the file being created, or the + // node warns once a day forever and never upgrades. + await addSyncthingRepository(); // The sources changed at version 2.0.0 from stable, to stable-v2 const hasNewSources = serviceHelper.minVersionSatisfy( @@ -544,11 +599,16 @@ async function monitorSyncthingPackage() { const updated = await updateSyncthingRepository(); if (!updated) { log.warn('Failed to update syncthing repository sources, skipping syncthing upgrade'); - return; + return false; } } } + // The uninstalled case: nothing on the system, so there is no version to + // judge and nothing to rewrite - the source is simply ensured before the + // install. + await addSyncthingRepository(); + const upgraded = await ensurePackageVersion( 'syncthing', minSyncthingVersion, @@ -558,15 +618,20 @@ async function monitorSyncthingPackage() { // we only restart if the package was installed (and running) in the first place if (currentSyncthingVersion && upgraded) { log.info('Syncthing upgraded, restarting to load new binary...'); - await syncthingService.systemRestart(null, null).catch(() => { }); + await syncthingService.systemRestart().catch(() => { }); } + + return upgraded; }; - await versionChecker(); + const upgraded = await versionChecker(); syncthingTimer = setInterval(versionChecker, 1000 * 60 * 60 * 24); // 24 hours + + return upgraded; } catch (error) { log.error(error); + return false; } } @@ -589,7 +654,7 @@ async function monitorAptCache(event) { // than apt-get install) if (options.command === 'update') { // we don't need to log here, as the error gets logged automatically by runCommand - aptQueue.resume(); + getQueue().resume(); return; } @@ -626,7 +691,7 @@ async function monitorAptCache(event) { // eslint-disable-next-line no-await-in-loop const { error: lockCheckError } = await serviceHelper.runCommand('apt-get', { runAsRoot: true, params: ['check'] }); if (!lockCheckError) { - aptQueue.resume(); + getQueue().resume(); return; } @@ -658,12 +723,12 @@ async function monitorAptCache(event) { const { error: checkError } = await serviceHelper.runCommand('apt-get', { runAsRoot: true, params: ['check'] }); if (!checkError) { - aptQueue.resume(); + getQueue().resume(); return; } log.error('Unable to run apt-get command(s), clearing the queue and resetting state.'); - aptQueue.clear(); + getQueue().clear(); } /** @@ -672,24 +737,49 @@ async function monitorAptCache(event) { async function monitorSystem() { if (isArcane) return; + let installed = []; try { - aptQueue.addWorker(aptRunner); - aptQueue.on('failed', monitorAptCache); - - // don't await these, let the queue deal with it - - // ubuntu 18.04 -> 24.04 all share this package - setImmediate(() => ensurePackageVersion('ca-certificates', '20230311')); - // 18.04 == 1.187 - // 20.04 == 1.206 - // 22.04 == 1.218 - // Debian 12 = 1.219 - setImmediate(() => ensurePackageVersion('netcat-openbsd', '1.187')); - setImmediate(() => monitorSyncthingPackage()); - // eslint-disable-next-line no-use-before-define - setImmediate(() => ensureChronyd()); + // ensureChronyd answers whether chrony ended up configured, which is true + // both when it installed it and when it was already there. What it installed + // is a different question, and only the package can answer it. + const chronyWasPresent = Boolean(await getPackageVersion('chrony')); + + // Started together and reported on together. The queue still serialises the + // apt work behind them, so running them concurrently costs nothing; the + // caller does not await monitorSystem, so the boot does not wait either. + const checks = { + // ubuntu 18.04 -> 24.04 all share this package + 'ca-certificates': ensurePackageVersion('ca-certificates', '20230311'), + // 18.04 == 1.187 + // 20.04 == 1.206 + // 22.04 == 1.218 + // Debian 12 = 1.219 + 'netcat-openbsd': ensurePackageVersion('netcat-openbsd', '1.187'), + syncthing: monitorSyncthingPackage(), + // eslint-disable-next-line no-use-before-define + chrony: ensureChronyd().then( + async () => !chronyWasPresent && Boolean(await getPackageVersion('chrony')), + ), + }; + + const names = Object.keys(checks); + const settled = await Promise.allSettled(Object.values(checks)); + // Named individually rather than as one failure, so a log says which check + // could not answer instead of only that something could not. + settled.forEach((outcome, i) => { + if (outcome.status === 'rejected') log.error(`monitorSystem - the ${names[i]} check failed: ${outcome.reason?.message ?? outcome.reason}`); + }); + installed = names.filter((_, i) => settled[i].status === 'fulfilled' && settled[i].value); } catch (error) { log.error(error); + } finally { + // Published on every path out of here, including the ones that failed. + // "Checked and had nothing to do" is a different fact from "has not run + // yet", and a check that threw is a third - tried and could not. All three + // are silence to a waiter, which then waits out its whole timeout for an + // answer that is never coming, and a node already provisioned looks the + // same as one whose checks never started. + fluxEventBus.publish('system:packages-checked', { installed }); } } @@ -766,10 +856,10 @@ async function mongodGpgKeyVeryfity() { const versionMatch = stdout.match(/MongoDB (\d+\.\d+) Release Signing Key/); if (expiredMatch) { if (versionMatch) { - const keyUrl = `https://pgp.mongodb.com/server-${versionMatch[1]}.asc`; + const keyUrl = `${config.mongodb.signingKeyBaseUrl}/server-${versionMatch[1]}.asc`; const filePath = '/usr/share/keyrings/mongodb-archive-keyring.gpg'; log.info(`MongoDB version: ${versionMatch[1]}`); - log.info(`GPG URL: https://pgp.mongodb.com/server-${versionMatch[1]}.asc`); + log.info(`GPG URL: ${keyUrl}`); log.info(`The key has expired on ${expiredMatch[1]}`); const command = `curl -fsSL ${keyUrl} | sudo gpg --batch --yes -o ${filePath} --dearmor`; // eslint-disable-next-line no-shadow diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 6ddfab91fe..ba9f3a36fe 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -8,13 +8,14 @@ const nodecmd = require('node-cmd'); const util = require('util'); const log = require('../lib/log'); +const { Privilege, authOf } = require('./utils/privileges'); const client = new natUpnp.Client(); if (config.upnp.gatewayUrl) { // eslint-disable-next-line global-require const { Device } = require('@runonflux/nat-upnp/build/src/nat-upnp/device'); - const gatewayUrl = config.upnp.gatewayUrl; + const { gatewayUrl } = config.upnp; const nodeIp = config.upnp.nodeIp || '127.0.0.1'; client.getGateway = async () => ({ gateway: new Device(gatewayUrl), @@ -310,7 +311,7 @@ async function removeMapUpnpPort(port) { */ async function mapPortApi(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized) { let { port } = req.params; port = port || req.query.port; @@ -357,7 +358,7 @@ async function mapPortApi(req, res) { */ async function removeMapPortApi(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized) { let { port } = req.params; port = port || req.query.port; @@ -397,7 +398,7 @@ async function removeMapPortApi(req, res) { */ async function getMapApi(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized) { const map = await client.getMappings(); const message = messageHelper.createDataMessage(map); @@ -424,7 +425,7 @@ async function getMapApi(req, res) { */ async function getIpApi(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized) { const ip = await client.getPublicIp(); const message = messageHelper.createDataMessage(ip); @@ -451,7 +452,7 @@ async function getIpApi(req, res) { */ async function getGatewayApi(req, res) { try { - const authorized = await verificationHelper.verifyPrivilege('adminandfluxteam', req); + const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized) { const gateway = await client.getGateway(); const message = messageHelper.createDataMessage(gateway); diff --git a/ZelBack/src/services/utils/FluxPeerManager.js b/ZelBack/src/services/utils/FluxPeerManager.js index 4865083710..c313c962e9 100644 --- a/ZelBack/src/services/utils/FluxPeerManager.js +++ b/ZelBack/src/services/utils/FluxPeerManager.js @@ -28,6 +28,10 @@ const CLOSE_CODE_NAMES = Object.freeze( Object.fromEntries(Object.entries(CLOSE_CODES).map(([name, code]) => [code, name])), ); +// Only for peers whose build cannot refuse a sync request. Deleted with the +// last of them, along with the branch in getEligibleSyncPeers. +const LEGACY_MIN_PEER_UPTIME_SECONDS = config.fluxapps.appSyncMinPeerUptime ?? 7500; + class FluxPeerManager extends EventEmitter { static CONNECTION_BACKOFF_MS = config.fluxapps.connectionBackoffMs ?? [2 * 60000, 5 * 60000, 10 * 60000, 15 * 60000]; @@ -68,7 +72,20 @@ class FluxPeerManager extends EventEmitter { /** @type {Set} peers removed since last peerUpdate broadcast */ #pendingRemoves = new Set(); - #syncRequestedPeers = new Set(); + /** + * Whether an arriving app-state sync response is still wanted, asked of + * whoever owns the outstanding requests. + * + * A function rather than a set of keys kept here. The orchestrator issues + * the requests and holds their deadlines, so it is the only thing that knows + * whether one is still open; a copy of that here would be a second record of + * the same fact, maintained by different code and cleared by a different + * rule. Registered by serviceManager once the orchestrator exists. + * @type {((peerSocket: FluxPeerSocket) => boolean)|null} + */ + syncResponseWanted = null; + + #ownSocketAddress = null; /** @type {ReturnType|null} debounce timer */ #peerUpdateTimer = null; /** @type {Array} Circular buffer of peer lifecycle events */ @@ -137,11 +154,15 @@ class FluxPeerManager extends EventEmitter { if (existing) { log.warn(`Replacing existing ${existing.direction} peer ${key}`); // Detach old handlers so its onclose doesn't remove the new peer - existing.ws.onclose = null; - existing.ws.onerror = null; - existing.ws.onmessage = null; + existing.detachHandlers(); try { existing.ws.close(CLOSE_CODES.DUPLICATE_PEER, 'replaced'); } catch (_e) { /* noop */ } this.#removeTracking(existing); + // A CONNECTION ENDED HERE, and it used to end in silence. The address + // stays in the map and the count does not move, so nothing that watches + // membership can see it - but a request written into that socket is as + // dead as one whose peer went away, and the only thing that told anyone + // was a sweep looking for connection ids that had changed underneath it. + this.emit('peerDisconnected', existing.key, existing.connectionId); } const peer = new FluxPeerSocket(ws, ip, String(port), this); peer.source = options.source || PEER_SOURCE.INBOUND; @@ -198,6 +219,18 @@ class FluxPeerManager extends EventEmitter { this.emit('peerThresholdReached', this.#peers.size); fluxEventBus.publish('peers:thresholdReached', { count: this.#peers.size, threshold: this.#syncPeerThreshold }); } + // Every connection, not just the one that crosses the threshold. The + // threshold is a latched edge and is cleared only below the DEGRADED level, + // so once it has fired it says nothing further about a pool that has since + // lost a member. A listener that has to top a pool of peers back up needs + // to hear about the peer that could fill it. + // + // Named for what it is. This fires for every connection established, + // including one replacing a dead socket at an address already held - where + // no peer was added and the count did not move. Its counterpart is + // peerDisconnected, and both name the connection rather than the address, + // because a request lives in a connection. + this.emit('peerConnected', peer.key, peer.connectionId); return peer; } @@ -211,7 +244,6 @@ class FluxPeerManager extends EventEmitter { const peer = this.#peers.get(key); if (!peer) return null; - this.#syncRequestedPeers.delete(key); this.#removeTracking(peer); // Clean up peer exchange topology and notify others @@ -257,6 +289,10 @@ class FluxPeerManager extends EventEmitter { this.emit('peersBelowThreshold', this.#peers.size); fluxEventBus.publish('peers:belowThreshold', { count: this.#peers.size, threshold: this.#syncDegradedThreshold }); } + // The counterpart of peerConnected. A listener waiting on this peer for an + // answer now knows the answer is never coming, which is a fact rather than + // something to be inferred from a deadline passing. + this.emit('peerDisconnected', key, peer.connectionId); return peer; } @@ -282,11 +318,41 @@ class FluxPeerManager extends EventEmitter { else this.#uniqueIps.set(ipKey, ipCount); } + /** + * Retire a peer this node has decided to drop. + * + * Membership of the peer map is this node's own account of who it peers with, so a + * decision to drop a peer takes effect here rather than when the remote gets round to + * answering. The close frame still goes out first, because the code it carries is what + * tells the remote whether to reconnect; the socket then finishes closing, or not, on + * its own time with nothing depending on it. + * + * The alternative - waiting for onclose - makes the count that drives the degraded + * threshold a function of the remote's cooperation. A peer that never answers stays + * counted until ws destroys the socket 30 seconds later, and neither route out of the + * map can reach it in the meantime: onclose has not fired, and ping() skips a socket + * that is not OPEN, so the missed-pong path cannot fire either. + * + * @param {string} key + * @param {number} [closeCode] + * @param {string} [reason] + * @returns {FluxPeerSocket|null} the peer removed, or null if it was not held + */ + evict(key, closeCode, reason) { + const peer = this.#peers.get(key); + if (!peer) return null; + try { peer.close(closeCode, reason); } catch (_e) { /* noop */ } + // After this the socket can neither deliver a frame nor call remove() a second time. + peer.detachHandlers(); + return this.remove(key, closeCode); + } + disconnectAll() { this.acceptingConnections = false; const count = this.#peers.size; - for (const peer of this.#peers.values()) { - try { peer.close(CLOSE_CODES.NODE_UNCONFIRMED, 'node unconfirmed'); } catch (_e) { /* noop */ } + // Snapshot the keys: evict() deletes from the map being walked. + for (const key of [...this.#peers.keys()]) { + this.evict(key, CLOSE_CODES.NODE_UNCONFIRMED, 'node unconfirmed'); } log.info(`Disconnected all ${count} peers, no longer accepting connections`); } @@ -443,19 +509,77 @@ class FluxPeerManager extends EventEmitter { return this.#peers.size; } + /** + * This node's own socket address, so it can be told apart from a peer. + * + * Set rather than derived here: this class is deliberately free of the + * network helpers, and the address is known by the time peering starts. + */ + setOwnSocketAddress(socketAddress) { + this.#ownSocketAddress = socketAddress; + } + + /** + * This node's own socket address, as last learned. + * + * Answered from here rather than fetched, because this is where the fact + * already lives: fluxNetworkHelper pushes every refresh in through + * setOwnSocketAddress, from the one place the node learns what it is. The + * alternative - asking benchmark - is an RPC with no cache behind it, and the + * dial path needs this on every attempt. + * + * @returns {string|null} ip:port, or null while the node has not been told + */ + getOwnSocketAddress() { + return this.#ownSocketAddress; + } + getPeerFluxUptime(key) { const peer = this.#peers.get(key); if (!peer || peer.remoteFluxUptime === null) return null; return peer.remoteFluxUptime + (Date.now() - peer.connectedAt) / 1000; } - getEligibleSyncPeers(minUptimeSeconds, count) { + /** + * Peers worth asking for app state. + * + * A peer that can refuse is asked whatever its uptime, because it answers the + * question the uptime was standing in for. The bar was 7500 seconds and the + * block fallback is 125 minutes, which is the same 7500 seconds, so it only + * ever admitted peers that had already become authoritative by the slow road + * - and it was self-reported, read from a header the peer sets on its own + * upgrade response, so it excluded honest young nodes and no dishonest one. + * + * A peer that CANNOT refuse still has the bar, and this is the whole reason + * the capability exists rather than the bar simply being deleted. An older + * build answers a request it cannot serve with an empty batch, which is + * indistinguishable from a complete survey of an empty network - so dropping + * the bar for those would put the original defect back during exactly the + * window that makes it likely, a rolling upgrade with many young nodes. + * + * The branch retires with the last build that cannot refuse, and the bar and + * getPeerFluxUptime go with it. + * @param {number} [count] Cap on how many to return. + * @returns {Array} peers, shuffled. + */ + getEligibleSyncPeers(count) { const eligible = []; + const ownKey = this.#ownSocketAddress; for (const peer of this.#peers.values()) { + // Never ourselves. A node that syncs from itself learns nothing it does + // not already hold, and this asks for a fixed small number of peers - so + // drawing self spends one of very few attempts on a guaranteed + // non-answer, and on a small fleet that is the difference between the + // spawner starting and never starting at all. Observed doing exactly + // that: a node asked its own address, timed out at zero completions, and + // never published SPAWNER_READY. + if (ownKey && peer.key === ownKey) continue; if (peer.missedPongs !== 0) continue; if (!peer.remoteCapabilities.has('appStateSync')) continue; - const uptime = this.getPeerFluxUptime(peer.key); - if (uptime === null || uptime < minUptimeSeconds) continue; + if (!peer.remoteCapabilities.has('appStateSyncRefusal')) { + const uptime = this.getPeerFluxUptime(peer.key); + if (uptime === null || uptime < LEGACY_MIN_PEER_UPTIME_SECONDS) continue; + } eligible.push(peer); } for (let i = eligible.length - 1; i > 0; i -= 1) { @@ -465,13 +589,23 @@ class FluxPeerManager extends EventEmitter { return count ? eligible.slice(0, count) : eligible; } - markSyncRequested(key) { this.#syncRequestedPeers.add(key); } - - isSyncRequested(key) { return this.#syncRequestedPeers.has(key); } - - completeSyncRequest(key) { this.#syncRequestedPeers.delete(key); } - - clearSyncRequested() { this.#syncRequestedPeers.clear(); } + /** + * Whether a sync response arriving on this connection is still wanted. + * + * Asked with the socket rather than the address because the two are not the + * same thing: a peer that reconnects keeps its `ip:port` while becoming a + * different connection, and nothing arriving on the new one is an answer to + * a request written into the old one. + * + * Closed by default. An unsolicited sync response is dropped, which is what + * happens before anything registers an answer here. + * @param {FluxPeerSocket} peerSocket + * @returns {boolean} + */ + isSyncResponseWanted(peerSocket) { + if (!this.syncResponseWanted) return false; + return this.syncResponseWanted(peerSocket); + } // --- Liveness --- @@ -582,6 +716,13 @@ class FluxPeerManager extends EventEmitter { if (closeCode === CLOSE_CODES.DEAD_CONNECTION) return true; // Max connections: remote is full, try again next cycle if (closeCode === CLOSE_CODES.MAX_CONNECTIONS) return true; + // Node unconfirmed: the remote is still booting and has not opened its + // application gate yet - a statement about when, not about us. It is the + // one refusal that is certain to stop applying, and on the fleet the same + // node dialled back 150ms later. A build that refuses the upgrade never + // gets this far, but the network is mixed and older nodes still close with + // it after the handshake. + if (closeCode === CLOSE_CODES.NODE_UNCONFIRMED) return true; // Everything else: policy violation, auth failure, admin close, duplicate — don't retry return false; } @@ -680,9 +821,17 @@ class FluxPeerManager extends EventEmitter { * @private */ async #broadcastToGroup(data, direction, exclude, delayMs) { - const iter = direction === DIRECTION.INBOUND ? this.inboundValues() : this.outboundValues(); - for (const peer of iter) { - if (exclude && peer.key === exclude) continue; + // The keys are taken once, and each is looked up again at the moment it is + // sent to. This loop awaits between sends, so the peer map is free to change + // under it - a peer dropped by the monitor, a peer this loop evicts itself - + // and a live iterator would make the result depend on when that happened. + // Looking the key up again keeps the one behaviour that matters: a peer that + // has gone while we were delaying is skipped rather than sent to. + const keys = direction === DIRECTION.INBOUND ? [...this.#inboundKeys] : [...this.#outboundKeys]; + for (const key of keys) { + if (exclude && key === exclude) continue; + const peer = this.#peers.get(key); + if (!peer) continue; try { await serviceHelper.delay(delayMs); if (!peer.send(data)) { @@ -691,7 +840,14 @@ class FluxPeerManager extends EventEmitter { } catch (e) { try { const code = direction === DIRECTION.OUTBOUND ? CLOSE_CODES.CLOSED_OUTBOUND : CLOSE_CODES.CLOSED_INBOUND; - peer.close(code, 'send failure'); + // Evicted, not closed. send() returns false only when the socket is + // already not open, so this path is reached exactly when close() can + // achieve nothing: no frame goes out, onclose has been and gone or + // will never come, and ping() skips a non-open socket so the missed + // pong that would eventually terminate it is never counted. The peer + // would sit in the map holding a place no reconnect is dialled for + // and offering itself as a sync source, until ws times the close out. + this.evict(peer.key, code, 'send failure'); } catch (err) { log.error(err); } diff --git a/ZelBack/src/services/utils/FluxPeerSocket.js b/ZelBack/src/services/utils/FluxPeerSocket.js index 6a37e6abd3..bb4ba94b15 100644 --- a/ZelBack/src/services/utils/FluxPeerSocket.js +++ b/ZelBack/src/services/utils/FluxPeerSocket.js @@ -1,4 +1,5 @@ const config = require('config'); +const { performance } = require('perf_hooks'); const WebSocket = require('ws'); const log = require('../../lib/log'); const serviceHelper = require('../serviceHelper'); @@ -12,6 +13,21 @@ function getFluxNetworkHelper() { return _fluxNetworkHelper; } +/** + * Milliseconds from a clock that only moves forward. + * + * Every elapsed measurement below uses this rather than Date.now(). Wall clock is adjusted by NTP + * and by hand, and a backward step makes an elapsed time negative while a forward one makes a peer + * look silent for however far the clock jumped — either can terminate a healthy connection. + * Timestamps that are REPORTED stay on the wall clock, because a caller reading connectedAt or + * lastPongTime wants a date, not milliseconds since this process started. + * + * @returns {number} + */ +function monotonicMs() { + return performance.now(); +} + const CLOSE_CODES = Object.freeze({ // Inbound validation (FluxPeerManager.validateAndAddInbound) MAX_CONNECTIONS: 4000, @@ -67,6 +83,16 @@ const DIRECTION = Object.freeze({ }); class FluxPeerSocket { + /** + * Distinguishes one connection to an address from the next one to it. + * + * `ip:port` names a NODE, and a node that reconnects is the same key over a + * different socket. Anything recorded against a connection - a request whose + * answer is still expected, and what that peer said about it - needs an + * identity the address alone does not give it. + */ + static #nextConnectionId = 1; + /** * @param {WebSocket} ws - raw WebSocket * @param {string} ip @@ -78,13 +104,37 @@ class FluxPeerSocket { this.ip = ip; this.port = String(port); this.key = `${ip}:${this.port}`; + this.connectionId = FluxPeerSocket.#nextConnectionId; + FluxPeerSocket.#nextConnectionId += 1; this.manager = manager; this.latency = null; + // Reported, so wall clock. The elapsed maths uses the monotonic pair below. this.lastPingTime = null; this.lastPongTime = null; + this.lastPingMono = null; this.missedPongs = 0; this.maxMissedPongs = config.peers.wsMaxMissedPongs ?? 3; + /** + * When anything was last heard from this peer, on the monotonic clock. + * + * A peer mid-conversation is the one case the heartbeat should never fire on, and crediting + * only pongs makes it fire there first: a pong is an ordinary frame that queues behind + * whatever else that peer sent, so the busiest connections look the most silent. + * + * Null until something actually arrives. Seeding it with the connection time would credit a + * peer for existing and give every new connection a free window in which no missed pong can + * terminate it. + */ + this.lastMessageMono = null; + /** + * How long a peer may go completely quiet before a missed-pong count can terminate it. + * + * Derived, not a new tunable: it is exactly the window the pong count already allows, so a + * genuinely silent peer is dropped on the same schedule as before and only a talking one is + * treated differently. + */ + this.livenessWindowMs = (config.peers.wsPingIntervalMs ?? 15000) * this.maxMissedPongs; this.connectedAt = Date.now(); this.nakCount = 0; this.nakWindowStart = Date.now(); @@ -124,8 +174,15 @@ class FluxPeerSocket { }; } + /** Anything received from this peer within the window the pong count allows. */ + get heardFromRecently() { + return this.lastMessageMono !== null + && monotonicMs() - this.lastMessageMono < this.livenessWindowMs; + } + get isAlive() { - return this.missedPongs < this.maxMissedPongs && this.ws.readyState === WebSocket.OPEN; + return this.ws.readyState === WebSocket.OPEN + && (this.missedPongs < this.maxMissedPongs || this.heardFromRecently); } get reconnects() { @@ -134,9 +191,13 @@ class FluxPeerSocket { onPingSent() { this.lastPingTime = Date.now(); + this.lastPingMono = monotonicMs(); this.missedPongs += 1; - if (this.missedPongs >= this.maxMissedPongs) { - log.info(`Peer ${this.key} missed ${this.missedPongs} pongs, terminating`); + // Unanswered pings only terminate a peer we have heard NOTHING else from. Three unanswered + // pings from a peer that is streaming messages at us means our own reader has not reached the + // pong yet, not that the peer is gone. + if (this.missedPongs >= this.maxMissedPongs && !this.heardFromRecently) { + log.info(`Peer ${this.key} missed ${this.missedPongs} pongs and sent nothing else, terminating`); this.terminate(); } } @@ -144,8 +205,9 @@ class FluxPeerSocket { onPongReceived() { this.missedPongs = 0; this.lastPongTime = Date.now(); - if (this.lastPingTime) { - this.latency = Math.ceil((this.lastPongTime - this.lastPingTime) / 2); + this.lastMessageMono = monotonicMs(); + if (this.lastPingMono !== null) { + this.latency = Math.ceil((monotonicMs() - this.lastPingMono) / 2); } } @@ -233,6 +295,20 @@ class FluxPeerSocket { this.ws.terminate(); } + /** + * Drop every handler on the underlying socket. + * + * Used when the manager has already accounted for this peer's departure. onclose is the + * only caller of FluxPeerManager.remove(), so a socket that completes its handshake + * afterwards would remove an entry that has since been replaced or already retired, and + * a frame arriving in the meantime would be processed for a peer no longer held. + */ + detachHandlers() { + this.ws.onclose = null; + this.ws.onerror = null; + this.ws.onmessage = null; + } + /** * Send a NAK (negative acknowledgement) back to sender. * @param {string} messageHash - 40-char hex hash @@ -320,6 +396,10 @@ class FluxPeerSocket { ws.onmessage = (evt) => { if (!evt) return; + // Before the rate limit: a frame we decline to process still proves the peer is there, and + // liveness is a question about the peer rather than about how much work we accept from it. + this.lastMessageMono = monotonicMs(); + const rateOK = rateLimit.lruRateLimit(`${this.ip}:${this.port}`, 120); if (!rateOK) return; @@ -384,7 +464,7 @@ class FluxPeerSocket { || syncType === 'fluxapprunningsync' || syncType === 'fluxappinstallingsync' || syncType === 'fluxappinstallingerrorssync') { - if (manager.syncResponseDispatcher && manager.isSyncRequested(this.key)) { + if (manager.syncResponseDispatcher && manager.isSyncResponseWanted(this)) { setImmediate(() => manager.syncResponseDispatcher(msgObj, this)); return; } @@ -403,6 +483,11 @@ const FLUX_CAPABILITIES = Object.freeze([ 'peerExchange', 'binaryMessages', 'appStateSync', + // This build answers a state-sync request it cannot usefully serve with a + // refusal, instead of an empty batch that reads as a completed survey. A peer + // that does not claim it cannot tell us it knows nothing, so it is still held + // to the uptime proxy - see getEligibleSyncPeers. + 'appStateSyncRefusal', ]); module.exports = { FluxPeerSocket, CLOSE_CODES, PEER_SOURCE, DIRECTION, FLUX_VERSION, FLUX_CAPABILITIES }; diff --git a/ZelBack/src/services/utils/appConstants.js b/ZelBack/src/services/utils/appConstants.js index 9d903c2cb4..30221b558f 100644 --- a/ZelBack/src/services/utils/appConstants.js +++ b/ZelBack/src/services/utils/appConstants.js @@ -36,6 +36,24 @@ const globalAppsInstallingErrorsBroadcasts = config.database.appsglobal.collecti const APP_NAME_REGEX = /^[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?$/; const APP_NAME_REGEX_LEGACY = /^[a-zA-Z0-9]+$/; +// Mount options for an app's FLUXFSVOL. An app volume holds data its owner +// writes, so nothing stored there should be able to confer privilege on +// whatever reads it back. +// +// `nosuid` makes any setuid/setgid bit on the volume inert. It does NOT prevent +// execution - an app that downloads and runs a binary from its own volume is +// unaffected - it only stops that binary switching to another user. Legitimate +// setuid binaries live in the container image, not in appdata. +// +// `nodev` stops a device node on the volume being honoured; an app that needs +// device access gets it from docker's device mapping. +// +// This belongs at the mount rather than in any one caller because there is more +// than one way for such a file to arrive: unpacking a user-supplied archive is +// the obvious one, but a copy preserves the bits too. A bind mount inherits its +// source's options, so a volume handed to a container carries them as well. +const APP_VOLUME_MOUNT_OPTIONS = 'loop,nosuid,nodev'; + // Supported architectures const supportedArchitectures = ['amd64', 'arm64']; @@ -58,34 +76,62 @@ const defaultNodeSpecs = { ssdStorage: 0, }; -// Apps monitored structure template -const appsMonitoredTemplate = { - // component1_appname2: { // >= 4 or name for <= 3 - // oneMinuteInterval: null, // interval - // fifteenMinInterval: null, // interval - // oneMinuteStatsStore: [ // stores last hour of stats of app measured every minute - // { // object of timestamp, data - // timestamp: 0, - // data: { }, - // }, - // ], - // fifteenMinStatsStore: [ // stores last 24 hours of stats of app measured every 15 minutes - // { // object of timestamp, data - // timestamp: 0, - // data: { }, - // }, - // ], - // }, -}; - -// Expiry / TTL constants (milliseconds) +// Expiry / TTL constants (milliseconds). +// +// The three stamped onto records live in config as seconds, so the harness can +// compress them the way it compresses every other cadence; the literal after +// `??` is the production default and is what a node runs when the key is absent. +// They lived here as bare literals from the day expiry moved per-document: the +// config keys were wired to the collection-level TTL indexes that scheme +// replaced, so when those indexes were dropped the keys were left reading +// nothing, and a later unused-variable sweep removed the last binding to them. +// Reading config here cannot reintroduce the cycle those literals were moved to +// break - that was messageVerifier -> registryManager -> messageStore -> +// messageVerifier, entirely between services, and `config` is a leaf this file +// already requires for the collection names above. const GOSSIP_VALIDITY_MS = 5 * 60 * 1000; -const RUNNING_EXPIRY_MS = 125 * 60 * 1000; -const INSTALLING_EXPIRY_MS = 15 * 60 * 1000; -const INSTALLING_ERRORS_EXPIRY_MS = 24 * 60 * 60 * 1000; -const SIGTERM_EXPIRY_MS = 420 * 1000; +const RUNNING_EXPIRY_MS = (config.fluxapps.locationTtlS ?? 7500) * 1000; +const INSTALLING_EXPIRY_MS = (config.fluxapps.installingTtlS ?? 900) * 1000; +const INSTALLING_ERRORS_EXPIRY_MS = (config.fluxapps.installErrorTtlS ?? 86400) * 1000; +// The grace a node gets after announcing its own shutdown, before peers treat +// its locations as gone. Config-driven like the three above, and for the same +// reason they are: a harness that compresses RUNNING_EXPIRY_MS and cannot +// compress this one inverts the pair. The `||` in appStartupManager's +// locationsExpired then fires on the running expiry first and this window +// becomes unreachable - a clean shutdown gets no grace at all, which is the +// opposite of what it is for. +// +// NOT compressible by the same ratio as RUNNING_EXPIRY_MS, though. What that +// one is coupled to is the announce interval, which is a compressed clock; what +// THIS one is measured across is a node boot, and a boot is real work the +// harness does not compress - see installingTtlS above for the same argument. +// Bound it by what it must outlive, not by a factor. +const SIGTERM_EXPIRY_MS = (config.fluxapps.sigtermExpiryS ?? 420) * 1000; const EVICTED_EXPIRY_MS = RUNNING_EXPIRY_MS; +/** + * How often a node announces the apps it is running. + * + * Derived, not configured. A node writes its OWN location row when it + * announces, and that row expires RUNNING_EXPIRY_MS after the announcement that + * carried it - so the interval and the expiry are one decision and the pair + * cannot be allowed to drift. Two numbers here were a pairing an edit could set + * wrong in either file, with a node's presence on the network as the thing that + * silently degraded. + * + * TWO announcements inside one lifetime, because a node must be able to miss + * one and still be refreshed before the row lapses. The 4% is the slack that + * miss needs: without it the refresh lands exactly as the row expires. + * + * Production 7500s -> 3600s, and the harness's 63s -> 30s: the values both + * configurations were carrying by hand. + */ +const ANNOUNCES_PER_EXPIRY = 2; +const ANNOUNCE_SLACK = 0.04; +const ANNOUNCE_INTERVAL_MS = Math.floor( + (RUNNING_EXPIRY_MS * (1 - ANNOUNCE_SLACK)) / ANNOUNCES_PER_EXPIRY / 1000, +) * 1000; + // Hash sync constants (blocks, at 30s per block) const HASH_EXPIRY_BLOCKS = 1051200; // ~1 year — permanently flag unresolvable hashes const HASH_RETRY_BACKOFF = [0, 100, 500, 2500, 12500, 50000, 100000]; // ~0, 50min, 4h, 21h, 4d, 17d, 35d @@ -115,17 +161,20 @@ module.exports = { APP_NAME_REGEX, APP_NAME_REGEX_LEGACY, + // Volumes + APP_VOLUME_MOUNT_OPTIONS, + // Configuration supportedArchitectures, enterpriseRequiredArchitectures, isArcane, appsThatMightBeUsingOldGatewayIpAssignment, defaultNodeSpecs, - appsMonitoredTemplate, // Expiry / TTL GOSSIP_VALIDITY_MS, RUNNING_EXPIRY_MS, + ANNOUNCE_INTERVAL_MS, INSTALLING_EXPIRY_MS, INSTALLING_ERRORS_EXPIRY_MS, SIGTERM_EXPIRY_MS, diff --git a/ZelBack/src/services/utils/appSpecHelpers.js b/ZelBack/src/services/utils/appSpecHelpers.js index e43e0f788e..ea21ef1655 100644 --- a/ZelBack/src/services/utils/appSpecHelpers.js +++ b/ZelBack/src/services/utils/appSpecHelpers.js @@ -352,7 +352,7 @@ async function getAppFiatAndFluxPrice(req, res) { if (myLongCache.has('appPrices')) { appPrices.push(myLongCache.get('appPrices')); } else { - let response = await axios.get('https://stats.runonflux.io/apps/getappspecsusdprice', axiosConfig).catch((error) => log.error(error)); + let response = await axios.get(`${config.stats.baseUrl}/apps/getappspecsusdprice`, axiosConfig).catch((error) => log.error(error)); if (response && response.data && response.data.status === 'success') { myLongCache.set('appPrices', response.data.data); appPrices.push(response.data.data); @@ -441,7 +441,7 @@ async function getAppFiatAndFluxPrice(req, res) { if (gSyncthgApp) { actualPriceToPay *= 0.8; } - const marketplaceResponse = await axios.get('https://stats.runonflux.io/marketplace/listapps').catch((error) => log.error(error)); + const marketplaceResponse = await axios.get(`${config.stats.baseUrl}/marketplace/listapps`).catch((error) => log.error(error)); let marketPlaceApps = []; if (marketplaceResponse && marketplaceResponse.data && marketplaceResponse.data.status === 'success') { marketPlaceApps = marketplaceResponse.data.data; @@ -490,7 +490,7 @@ async function getAppFiatAndFluxPrice(req, res) { if (myShortCache.has('fluxRates')) { fluxUSDRate = myShortCache.get('fluxRates'); } else { - fiatRates = await axios.get('https://viprates.runonflux.io/rates', axiosConfig).catch((error) => log.error(error)); + fiatRates = await axios.get(`${config.pricing.fluxRatesBaseUrl}/rates`, axiosConfig).catch((error) => log.error(error)); if (fiatRates && fiatRates.data) { const rateObj = fiatRates.data[0].find((rate) => rate.code === 'USD'); if (!rateObj) { @@ -503,7 +503,7 @@ async function getAppFiatAndFluxPrice(req, res) { fluxUSDRate = rateObj.rate * btcRateforFlux; myShortCache.set('fluxRates', fluxUSDRate); } else { - fiatRates = await axios.get('https://api.coingecko.com/api/v3/simple/price?vs_currencies=usd&ids=zelcash', axiosConfig); + fiatRates = await axios.get(`${config.pricing.coingeckoBaseUrl}/api/v3/simple/price?vs_currencies=usd&ids=zelcash`, axiosConfig); if (fiatRates && fiatRates.data && fiatRates.data.zelcash && fiatRates.data.zelcash.usd) { fluxUSDRate = fiatRates.data.zelcash.usd; myShortCache.set('fluxRates', fluxUSDRate); diff --git a/ZelBack/src/services/utils/appSyncEvents.js b/ZelBack/src/services/utils/appSyncEvents.js index 71ae6cab31..aec590dda0 100644 --- a/ZelBack/src/services/utils/appSyncEvents.js +++ b/ZelBack/src/services/utils/appSyncEvents.js @@ -4,6 +4,9 @@ const appSyncEvents = new EventEmitter(); const EVENTS = Object.freeze({ EPHEMERAL_SYNC_COMPLETE: 'ephemeralSyncComplete', + EPHEMERAL_SYNC_REFUSED: 'ephemeralSyncRefused', + EPHEMERAL_SYNC_UNVERIFIED: 'ephemeralSyncUnverified', + EPHEMERAL_SYNC_PROGRESS: 'ephemeralSyncProgress', SPAWNER_READY: 'spawnerReady', READINESS_LOST: 'readinessLost', HASH_RESPONSE_RECEIVED: 'hashResponseReceived', diff --git a/ZelBack/src/services/utils/appUtilities.js b/ZelBack/src/services/utils/appUtilities.js index 100f4b7b68..318ad2a933 100644 --- a/ZelBack/src/services/utils/appUtilities.js +++ b/ZelBack/src/services/utils/appUtilities.js @@ -1,6 +1,4 @@ -const path = require('path'); -const util = require('util'); -const nodecmd = require('node-cmd'); +const fs = require('fs/promises'); const config = require('config'); const log = require('../../lib/log'); const serviceHelper = require('../serviceHelper'); @@ -9,11 +7,11 @@ const dockerService = require('../dockerService'); const geolocationService = require('../geolocationService'); const { getChainParamsPriceUpdates } = require('./chainUtilities'); const mountParser = require('./mountParser'); +const appConstants = require('./appConstants'); +const fluxCaching = require('./cacheManager'); const globalAppsLocations = config.database.appsglobal.collections.appsLocations; -const cmdAsync = util.promisify(nodecmd.run); -const fluxDirPath = process.env.FLUXOS_PATH || path.join(process.env.HOME, 'zelflux'); /** * Calculate app price per month @@ -154,22 +152,82 @@ async function nodeFullGeolocation() { } /** - * Get app folder size - * @param {string} appName - Application name - * @returns {Promise} Folder size in bytes + * Bytes used on the filesystem a mount source sits on, when that filesystem belongs to + * the app. Each app volume is its own mounted image, so the filesystem's own accounting + * answers this without walking the tree. An unmounted volume falls through to the node's + * filesystem, where the same reading would report the whole node as the app's usage, so + * identify the volume by device and leave anything sharing the apps folder's device to + * the caller. + * @param {string} source - Mount source path + * @param {number} sharedDevice - Device of the filesystem the node shares, resolved by the caller + * on a mount's first need; undefined for a source outside the apps folder, which returns before reading it + * @returns {Promise<{device: number, used: number}|null>} Usage, or null when not a dedicated volume */ -async function getAppFolderSize(appName) { - try { - const appsDirPath = process.env.FLUX_APPS_FOLDER || path.join(fluxDirPath, 'ZelApps'); - const directoryPath = path.join(appsDirPath, appName); - const exec = `sudo du -s --block-size=1 ${directoryPath}`; - const cmdres = await cmdAsync(exec); - const size = serviceHelper.ensureString(cmdres).split('\t')[0] || 0; - return size; - } catch (error) { - log.error(`Error getting app folder size: ${error.message}`); - return 0; +async function dedicatedVolumeUsage(source, sharedDevice) { + // The whole-filesystem shortcut is only sound when the filesystem belongs to + // THIS APP, and a differing device number does not establish that on its own. + // It holds for an app's FLUXFSVOL image, which is mounted under the apps + // folder and is the case this exists for. It does not hold for a mount docker + // manages: an image that declares VOLUME on a path the spec does not bind gets + // an anonymous volume under docker's data root, and where that root is a + // separate filesystem from the apps folder, this would charge the app the + // whole of it - every image and every other app's container included. + // + // Verified reachable: an image declaring two VOLUMEs with only one bound + // reports the other as Type: 'volume' with a Source under + // /var/lib/docker/volumes//_data. mongo declares /data/db AND + // /data/configdb, so a spec that maps the data dir and not the config dir is + // enough. On the node layouts checked (arcane with docker root on /dat, + // legacy with everything on one disk) the two share a device and this returns + // null anyway - but that is the layout being kind, not the test being right. + // + // So identify the app's own volume by WHERE IT IS, and leave everything else + // to be walked, which is what the base did for every mount and was correct on + // every layout. + if (!source.startsWith(appConstants.appsFolder)) return null; + + const sourceStat = await fs.stat(source); + + if (sourceStat.dev === sharedDevice) return null; + + const { blocks, bfree, bsize } = await fs.statfs(source); + return { device: sourceStat.dev, used: (blocks - bfree) * bsize }; +} + +/** + * Bytes used beneath a path, counted by walking it. + * @param {string} source - Mount source path + * @returns {Promise} Size in bytes + */ +async function walkedUsage(source) { + // argv, never a shell string: the source is a path docker reports, and + // interpolating it into `sudo du -sb ${source}` makes any metacharacter in it + // part of the command. + const { error, stdout } = await serviceHelper.runCommand('du', { + runAsRoot: true, + logError: false, + params: ['-sb', source], + }); + + // du exits 1 at the first entry it cannot read - on a busy app that is an + // ordinary temp file vanishing mid-walk, not an exceptional state - and still + // prints the total for everything it did walk. Measured on Linux: over a + // directory with one unreadable child it prints the running total AND exits 1. + // + // So a non-zero exit is not the same as no answer, and runCommand keeps stdout + // on a failed exit precisely so the number survives. Treating the exit code + // alone as failure would throw away a figure we already have, once a minute, + // on exactly the apps that write the most. + const reported = stdout ? serviceHelper.ensureNumber(stdout.split('\t')[0]) : NaN; + if (Number.isNaN(reported)) { + // Nothing usable came back. This is the genuine failure, and the caller has + // to know the total it builds is short rather than serve it as complete. + throw error || new Error(`du reported no size for ${source}`); } + if (error) { + log.warn(`Partial size for ${source}: du could not read every entry (${error.message.split('\n')[0]})`); + } + return reported; } /** @@ -178,12 +236,31 @@ async function getAppFolderSize(appName) { * @returns {Promise} Storage usage information */ async function getContainerStorage(appName) { + const cache = fluxCaching.default.containerStorageCache; + const cached = cache.get(appName); + if (cached) return cached; + try { const containerInfo = await dockerService.dockerContainerInspect(appName, { size: true }); let bindMountsSize = 0; let volumeMountsSize = 0; + // Mount sources that could not be measured at all. Empty is the ordinary case + // and the only one that may be called a success. + const unmeasured = []; const containerRootFsSize = serviceHelper.ensureNumber(containerInfo.SizeRootFs) || 0; if (containerInfo?.Mounts?.length) { + // Which filesystem the node shares is one fact about this call rather than + // a step in the loop, so it is resolved once and remembered - but only + // when a mount under the apps folder actually asks. Only those mounts are + // classified against it, so only they can be failed by it: a container + // whose mounts all live elsewhere never depends on this fact at all. + let sharedDevicePromise = null; + const sharedDevice = () => { + sharedDevicePromise = sharedDevicePromise + ?? fs.stat(appConstants.appsFolderPath).then((folder) => folder.dev); + return sharedDevicePromise; + }; + // Collect all mount sources and filter out nested mounts to avoid double-counting const allMounts = containerInfo.Mounts.filter((m) => m?.Source); const mountsToCount = []; @@ -204,48 +281,73 @@ async function getContainerStorage(appName) { } } - await Promise.all(mountsToCount.map(async (mount) => { + // Sibling mounts of the same app share one volume, so a whole-filesystem reading + // counts for all of them together. + const countedDevices = new Set(); + + // eslint-disable-next-line no-restricted-syntax + for (const mount of mountsToCount) { const source = mount.Source; const mountType = mount.Type; - if (mountType === 'bind') { - const exec = `sudo du -sb ${source}`; - try { - const mountInfo = await cmdAsync(exec); - if (mountInfo) { - const sizeNum = serviceHelper.ensureNumber(mountInfo.split('\t')[0]) || 0; - bindMountsSize += sizeNum; - } else { - log.warn(`No mount info returned for source: ${source}`); - } - } catch (error) { - log.warn(`Failed to get size for bind mount ${source}: ${error.message}`); - } - } else if (mountType === 'volume') { - const exec = `sudo du -sb ${source}`; - try { - const mountInfo = await cmdAsync(exec); - if (mountInfo) { - const sizeNum = serviceHelper.ensureNumber(mountInfo.split('\t')[0]) || 0; - volumeMountsSize += sizeNum; - } else { - log.warn(`No mount info returned for source: ${source}`); + if (mountType !== 'bind' && mountType !== 'volume') { + log.warn(`Unsupported mount type or source: Type: ${mountType}, Source: ${source}`); + // eslint-disable-next-line no-continue + continue; + } + // Resolved OUTSIDE the try, and only on need: a mount under the apps + // folder that cannot be classified must fail the whole reading - + // sizing it at zero would report a working node as using almost no + // disk, and nothing under the apps folder can do better - while a + // mount living anywhere else never asks and is simply walked. + // eslint-disable-next-line no-await-in-loop + const shared = source.startsWith(appConstants.appsFolder) ? await sharedDevice() : undefined; + let size = 0; + try { + // eslint-disable-next-line no-await-in-loop + const volume = await dedicatedVolumeUsage(source, shared); + if (volume) { + if (countedDevices.has(volume.device)) { + // eslint-disable-next-line no-continue + continue; } - } catch (error) { - log.warn(`Failed to get size for volume mount ${source}: ${error.message}`); + countedDevices.add(volume.device); + size = volume.used; + } else { + // eslint-disable-next-line no-await-in-loop + size = await walkedUsage(source); } + } catch (error) { + // The mount contributes nothing, so the total below is SHORT. Recorded + // rather than swallowed: the reading is still worth serving - a disk bar + // showing most of the truth beats one showing none of it - but it must + // not go out labelled as a complete measurement, and it must not be + // cached, or one transient failure is served as fact for the whole + // window. + log.warn(`Failed to get size for ${mountType} mount ${source}: ${error.message}`); + unmeasured.push(source); + // eslint-disable-next-line no-continue + continue; + } + if (mountType === 'bind') { + bindMountsSize += size; } else { - log.warn(`Unsupported mount type or source: Type: ${mountType}, Source: ${source}`); + volumeMountsSize += size; } - })); + } } const usedSize = bindMountsSize + volumeMountsSize + containerRootFsSize; - return { + const storage = { bind: bindMountsSize, volume: volumeMountsSize, rootfs: containerRootFsSize, used: usedSize, - status: 'success', + status: unmeasured.length ? 'partial' : 'success', }; + if (unmeasured.length) storage.unmeasured = unmeasured; + // Only a complete reading is cached. A short one is recomputed next tick, + // which is where it gets the chance to come good. + if (!unmeasured.length) cache.set(appName, storage); + return storage; } catch (error) { log.error(`Error fetching container storage: ${error.message}`); return { @@ -260,27 +362,33 @@ async function getContainerStorage(appName) { } /** - * Get app ports from specifications + * The host ports an application specification declares, across every version. + * + * The one place this is derived. A second extraction living somewhere else is + * two answers to one question that nothing keeps in agreement, and whichever is + * fixed the other stays wrong. + * + * A field the shape does not have yields nothing rather than throwing or a NaN. + * A missing `ports` used to throw and a version-1 spec with no `port` used to + * produce `[NaN]` - and NaN is worse than an exception here, because it is a + * number that compares unequal to everything and fails a long way from home. + * * @param {object} appSpecs - Application specifications * @returns {Array} Array of port numbers */ function getAppPorts(appSpecs) { - const appPorts = []; - // eslint-disable-next-line no-restricted-syntax + if (!appSpecs) return []; + if (appSpecs.version === 1) { - appPorts.push(+appSpecs.port); - } else if (appSpecs.version <= 3) { - appSpecs.ports.forEach((port) => { - appPorts.push(+port); - }); - } else { - appSpecs.compose.forEach((component) => { - component.ports.forEach((port) => { - appPorts.push(+port); - }); - }); + return appSpecs.port ? [Number(appSpecs.port)] : []; + } + + if (appSpecs.version <= 3) { + return (appSpecs.ports || []).map(Number); } - return appPorts; + + return (appSpecs.compose || []) + .flatMap((component) => (component.ports || []).map(Number)); } /** @@ -1116,7 +1224,6 @@ module.exports = { appPricePerMonth, appUsesGSyncthingMode, findCommonArchitectures, - getAppFolderSize, getAppPorts, getContainerStorage, getNonGComponentIdentifiers, diff --git a/ZelBack/src/services/utils/asyncLock.js b/ZelBack/src/services/utils/asyncLock.js index a142c585b7..29d4eb9438 100644 --- a/ZelBack/src/services/utils/asyncLock.js +++ b/ZelBack/src/services/utils/asyncLock.js @@ -17,6 +17,18 @@ class AsyncLock { return Boolean(this.#lockUsers.length); } + /** + * How many slots are currently taken. + * + * `locked` answers this only for a maxConcurrent of 1. A caller that refuses + * rather than waits - because queueing would hold a request open behind + * someone else's long-running operation - needs the count to compare against + * the limit before calling register(). + */ + get activeCount() { + return this.#lockUsers.length; + } + get #userPromises() { return this.#lockUsers.map((user) => user[1]); } diff --git a/ZelBack/src/services/utils/cacheManager.js b/ZelBack/src/services/utils/cacheManager.js index f7b0ef9100..5595b912c2 100644 --- a/ZelBack/src/services/utils/cacheManager.js +++ b/ZelBack/src/services/utils/cacheManager.js @@ -111,6 +111,14 @@ class FluxCacheManager { max: 60, ttl: 3 * FluxCacheManager.oneHour, }, + // One answer per node - this asks what ports THIS node holds, so there is + // nothing to key on and exactly one entry. It changes only when this node + // installs or removes an app, and it is read by every sibling asking before + // it installs. + portsInUseCache: { + max: 1, + ttl: 30 * FluxCacheManager.oneSecond, + }, appPriceBlockedRepoCache: { max: 50, ttl: 3 * FluxCacheManager.oneHour, @@ -183,6 +191,12 @@ class FluxCacheManager { max: 200, ttl: 30 * FluxCacheManager.oneMinute, }, + // appUtilities - disk usage moves slowly, while the monitoring UI polls app stats + // every few seconds. Sample it once a minute however many callers ask. + containerStorageCache: { + max: 100, + ttl: FluxCacheManager.oneMinute, + }, }; constructor() { diff --git a/ZelBack/src/services/utils/cidrUtils.js b/ZelBack/src/services/utils/cidrUtils.js new file mode 100644 index 0000000000..58f8d9b31b --- /dev/null +++ b/ZelBack/src/services/utils/cidrUtils.js @@ -0,0 +1,124 @@ +// IP address arithmetic for placement fault domains. +// +// All values are carried as BigInt (IPv4 fits, IPv6 needs it) so ranges and +// prefixes compare uniformly across both versions. Inputs are bare IP strings - +// callers holding an ip:port socket address must extractIp() first; a string +// containing a port does not parse here. + +const net = require('node:net'); + +const IPV4_BITS = 32; +const IPV6_BITS = 128; +// ::ffff:0:0/96 - IPv4 addresses embedded in IPv6 notation +const V4_MAPPED_PREFIX = 0xffffn << 32n; +const V4_MAPPED_MASK = ~0xffffffffn; + +/** + * Parse a bare IPv4 or IPv6 address into integer form. + * IPv4-mapped IPv6 addresses (::ffff:a.b.c.d) normalize to version 4. + * @param {string} ip Bare IP address, no port + * @returns {{version: 4|6, value: BigInt} | null} null when not a valid bare IP + */ +function parseIp(ip) { + if (typeof ip !== 'string') return null; + if (net.isIPv4(ip)) { + const [a, b, c, d] = ip.split('.').map(Number); + return { version: 4, value: BigInt((((a * 256 + b) * 256 + c) * 256) + d) }; + } + if (net.isIPv6(ip)) { + let groupsPart = ip; + let value = 0n; + let tailBits = 0; + const v4TailMatch = ip.match(/:(\d+\.\d+\.\d+\.\d+)$/); + if (v4TailMatch) { + const tail = parseIp(v4TailMatch[1]); + if (!tail) return null; + ({ value } = tail); + tailBits = 32; + groupsPart = ip.slice(0, -v4TailMatch[1].length); + } + const halves = groupsPart.split('::'); + const headGroups = halves[0] ? halves[0].split(':').filter(Boolean) : []; + const tailGroups = halves.length > 1 && halves[1] ? halves[1].split(':').filter(Boolean) : []; + const presentGroups = headGroups.length + tailGroups.length + tailBits / 16; + const missing = halves.length > 1 ? 8 - presentGroups : 0; + const allGroups = [...headGroups, ...Array(missing).fill('0'), ...tailGroups]; + let acc = 0n; + allGroups.forEach((group) => { acc = (acc << 16n) + BigInt(parseInt(group, 16)); }); + value += acc << BigInt(tailBits); + if ((value & V4_MAPPED_MASK) === V4_MAPPED_PREFIX) { + return { version: 4, value: value & 0xffffffffn }; + } + return { version: 6, value }; + } + return null; +} + +/** + * Render an integer-form address back to its canonical string. + * @param {BigInt} value Integer form + * @param {4|6} version IP version + * @returns {string} + */ +function formatIp(value, version) { + if (version === 4) { + const v = Number(value); + // eslint-disable-next-line no-bitwise + return `${(v >>> 24) & 255}.${(v >>> 16) & 255}.${(v >>> 8) & 255}.${v & 255}`; + } + const groups = []; + for (let i = 7; i >= 0; i -= 1) { + groups.push(Number((value >> BigInt(i * 16)) & 0xffffn).toString(16)); + } + // RFC 5952: compress the longest run of zero groups (leftmost on ties) + let bestStart = -1; + let bestLen = 0; + for (let i = 0; i < groups.length; i += 1) { + if (groups[i] !== '0') continue; + let len = 0; + while (i + len < groups.length && groups[i + len] === '0') len += 1; + if (len > bestLen) { bestStart = i; bestLen = len; } + } + if (bestLen < 2) return groups.join(':'); + const head = groups.slice(0, bestStart).join(':'); + const tail = groups.slice(bestStart + bestLen).join(':'); + return `${head}::${tail}`; +} + +/** + * Canonical network prefix for an address, e.g. ("80.95.213.209", 16) -> "80.95.0.0/16". + * @param {string} ip Bare IP address + * @param {number} bits Prefix length, 0..32 for IPv4 and 0..128 for IPv6 + * @returns {string | null} null when the address or prefix length is invalid + */ +function prefixKey(ip, bits) { + const parsed = parseIp(ip); + if (!parsed) return null; + const width = parsed.version === 4 ? IPV4_BITS : IPV6_BITS; + if (!Number.isInteger(bits) || bits < 0 || bits > width) return null; + const shift = BigInt(width - bits); + // eslint-disable-next-line no-bitwise + const base = (parsed.value >> shift) << shift; + return `${formatIp(base, parsed.version)}/${bits}`; +} + +/** + * Whether two addresses fall inside the same prefix. Addresses of different + * IP versions never share a prefix. + * @param {string} ipA Bare IP address + * @param {string} ipB Bare IP address + * @param {number} bits Prefix length + * @returns {boolean} + */ +function sameSubnet(ipA, ipB, bits) { + const keyA = prefixKey(ipA, bits); + const keyB = prefixKey(ipB, bits); + return keyA !== null && keyA === keyB; +} + +module.exports = { + parseIp, + formatIp, + prefixKey, + sameSubnet, +}; diff --git a/ZelBack/src/services/utils/configManager.js b/ZelBack/src/services/utils/configManager.js index 0c472acc06..4874d5ede7 100644 --- a/ZelBack/src/services/utils/configManager.js +++ b/ZelBack/src/services/utils/configManager.js @@ -44,6 +44,14 @@ class ConfigManager extends EventEmitter { // eslint-disable-next-line global-require const userconfig = require('../../../../config/userconfig'); + // The writers rewrite this file whole, so a read can land between the truncate + // and the write. require() resolves an empty or truncated file to an object + // rather than throwing, which makes a missing `initial` the only evidence that + // it did. + if (!userconfig || typeof userconfig.initial !== 'object' || userconfig.initial === null) { + throw new Error('userconfig.js carries no initial section'); + } + // Set on globalThis for global access globalThis.userconfig = userconfig; @@ -55,8 +63,16 @@ class ConfigManager extends EventEmitter { if (isReload) { this.emit('configReloaded', userconfig); } + return true; } catch (error) { console.error('Error loading userconfig:', error); + // A node that already holds a config keeps it. The defaults below carry no zelid + // and no keypair, so publishing them over a good config would have the node act + // under an identity that is not its own — and the read that failed is most often + // a write in progress, which the next change event resolves. + if (globalThis.userconfig && globalThis.userconfig.initial) { + return false; + } // Initialize with defaults if load fails globalThis.userconfig = { initial: { @@ -73,6 +89,7 @@ class ConfigManager extends EventEmitter { blockedRepositories: [], }, }; + return false; } } @@ -85,8 +102,13 @@ class ConfigManager extends EventEmitter { if (hashCurrent === this.initialHash) { return; } - this.initialHash = hashCurrent; - this.loadConfig(true); + // Recorded only once the file has actually been read. Stamping it first marks a + // half-written file as the version this node holds, when what it holds is still + // the previous config - so the write that completes a moment later has to be + // noticed all over again to be picked up. + if (this.loadConfig(true)) { + this.initialHash = hashCurrent; + } } /** diff --git a/ZelBack/src/services/utils/enterpriseConfig.js b/ZelBack/src/services/utils/enterpriseConfig.js index 41d9241efd..40066ef4ab 100644 --- a/ZelBack/src/services/utils/enterpriseConfig.js +++ b/ZelBack/src/services/utils/enterpriseConfig.js @@ -7,7 +7,7 @@ const serviceHelper = require('../serviceHelper'); // helpers/ lives at the repo root, four levels up from this file. const HELPERS_DIR = path.join(__dirname, '..', '..', '..', '..', 'helpers'); const FILE = 'enterprisenodes.json'; -const URL = `${config.github.rawBaseUrl}/helpers/${FILE}`; +const URL = `${config.policy.baseUrl}/${FILE}`; const SYNC_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours const FETCH_TIMEOUT_MS = 10 * 1000; // bound the github fetch so boot is never stuck on it diff --git a/ZelBack/src/services/utils/enterpriseNetwork.js b/ZelBack/src/services/utils/enterpriseNetwork.js index fe81664b94..0d631084ae 100644 --- a/ZelBack/src/services/utils/enterpriseNetwork.js +++ b/ZelBack/src/services/utils/enterpriseNetwork.js @@ -43,7 +43,9 @@ function isEnterpriseAppOwner(owner) { async function isEnterpriseNode() { if (cachedNodePubKey === null) { const pubKey = await fluxNetworkHelper.getFluxNodePublicKey(); - // getFluxNodePublicKey swallows errors and returns the Error object on failure. + // Kept even though the accessor now answers null rather than the Error it + // used to. What arrives here has to be a key to be usable, and checking + // that is this function's business whatever its source promises. if (!pubKey || typeof pubKey !== 'string') { throw new Error('enterpriseNetwork: unable to resolve fluxnode public key (daemon/benchmark unavailable)'); } diff --git a/ZelBack/src/services/utils/fifoQueue.js b/ZelBack/src/services/utils/fifoQueue.js index f6e1152135..157927e7dd 100644 --- a/ZelBack/src/services/utils/fifoQueue.js +++ b/ZelBack/src/services/utils/fifoQueue.js @@ -16,6 +16,8 @@ class FifoQueue extends EventEmitter { static get defaultRetainErrors() { return true; } + static get defaultMaxRetainCycles() { return 3; } + /** * The main queue */ @@ -49,6 +51,11 @@ class FifoQueue extends EventEmitter { this.retryDelay = options.retryDelay ?? FifoQueue.defaultRetryDelay; this.maxSize = options.maxSize ?? FifoQueue.defaultMaxSize; // 0 infinite this.retainErrors = options.retainErrors ?? FifoQueue.defaultRetainErrors; + // How many times a retained task is handed back to the worker before the queue + // gives up on it. Retaining is what lets a task outlive a bad moment; without a + // ceiling it also lets one that can never succeed be retried for the life of + // the process. + this.maxRetainCycles = options.maxRetainCycles ?? FifoQueue.defaultMaxRetainCycles; } /** @@ -75,11 +82,17 @@ class FifoQueue extends EventEmitter { } /** - * Setter for worker + * Setter for worker. A queue has one worker for its lifetime; prefer passing it + * to the constructor so the queue is never in a state where it accepts work it + * cannot run. * @param {() => Promise} worker + * @throws {Error} If a worker is already set */ addWorker(worker) { - if (this.worker) return; + // Refused rather than ignored. Silently keeping the first worker leaves the + // caller believing its own was installed, and the queue then runs somebody + // else's - which is a difference nothing downstream can see. + if (this.worker) throw new Error('FifoQueue already has a worker'); this.worker = worker; if (this.workAvailable) this.finished = this.work(); @@ -97,6 +110,13 @@ class FifoQueue extends EventEmitter { */ resume() { this.halted = false; + // A 'failed' listener resumes from INSIDE the work loop, where work() returns + // at once because working is already true. Assigning that already-resolved + // promise to finished would tell clear() the queue is idle while a worker is + // still in flight, and clear() then wipes the list out from under it - a path + // systemService reaches on exactly the failures this listener handles. The + // running loop reads the flag itself, so there is nothing to start. + if (this.working) return; this.finished = this.work(); } @@ -195,16 +215,51 @@ class FifoQueue extends EventEmitter { if (!retriesRemaining) { // the emit callback runs before the resolve (resolve is awaited) resolve({ error }); - this.emit('failed', { options, error }); + // Halted BEFORE the emit, never after. An emit is a synchronous yield: + // the listener runs right here, and it is allowed to resume the queue - + // monitorAptCache does exactly that, synchronously, for a failed + // apt-get update. A halt written after the emit silently undoes that + // resume, the loop breaks out with work still queued, and nothing ever + // starts it again: push() only calls work() when it is not already + // working, and working goes false on the way out. Settle our own state, + // then notify. this.halted = true; + // commandOptions, not the raw payload: a listener asks what failed, and + // for a payload of the {commandOptions, workerOptions} shape the command + // is a level down. Emitting the payload put it out of reach, so every + // listener test against it silently matched nothing. + this.emit('failed', { options: commandOptions, error }); } // Can get halted externally too. - // we put this task back at the start of the queue and bail. if (this.halted) { - if (retainErrors) this.#list.unshift(props); + // To the BACK of the queue. At the front it is handed straight back to + // the worker on the next resume, ahead of everything else - so a task + // that cannot succeed is retried forever and nothing queued behind it + // ever runs. One package apt could not find is enough to stop a node + // installing any of the others. + // + // And only so many times. Each resume grants a fresh ladder of retries, + // so without a ceiling the pair of retain-and-resume is unbounded: the + // ladder ends, whatever is listening resumes the queue, and it begins + // again. Given up on rather than dropped silently - the caller already + // has its error, but nothing else would ever learn the work was + // abandoned. + const cycles = (props[2] ?? 0) + 1; + if (retainErrors && cycles <= this.maxRetainCycles) { + props[2] = cycles; + this.#list.push(props); + } else if (retainErrors) { + this.emit('abandoned', { options: commandOptions, error, cycles }); + } break; } + // Not halted, so a listener resumed from inside the emit above and the + // queue carries on - but this task's ladder is still over. Falling + // through would sleep out the retry delay with nothing left to retry, + // stalling every task behind it for the full interval. + if (!retriesRemaining) break; + // wait default 10 seconds between retries // eslint-disable-next-line no-await-in-loop await new Promise((r) => { setTimeout(r, retryDelay); }); diff --git a/ZelBack/src/services/utils/fileTransfer.js b/ZelBack/src/services/utils/fileTransfer.js new file mode 100644 index 0000000000..df7a63cde7 --- /dev/null +++ b/ZelBack/src/services/utils/fileTransfer.js @@ -0,0 +1,80 @@ +const log = require('../../lib/log'); +const { openNoFollow } = require('./pathSecurity'); + +/** + * Send a file to a client, from a handle rather than from a name. + * + * The path was checked before this is called, but the application owns the + * volume and keeps running: it can replace what the name refers to at any + * moment, including between the check and the send. So the name is used exactly + * once, and everything afterwards is decided from the descriptor that opening + * it produced. + * + * O_NOFOLLOW refuses to open a symlink at the final component, which is the + * swap that turns a checked path into somebody else's file. What remains + * expressible - swapping a parent directory - the earlier path check still + * covers, and this removes the half of it that a check cannot. Opening also + * refuses anything that is not a regular file, and refuses to WAIT for one: a + * named pipe at this path would otherwise hold the request open for as long as + * the app owner left it there. + * + * The length is measured from the same descriptor and the send is capped at it. + * An application writing to its own file during a download would otherwise make + * the body longer than the Content-Length already announced, which a client + * reads as a corrupt response. + * + * NOTE: this does not serve range requests. `res.download` did, by way of + * express, and nothing known asks for them - the dashboard fetches whole files. + * Accept-Ranges says so rather than leaving a client to discover it. The read + * below is already a byte range over the handle, so serving one means clamping + * what the client asked for into that start and end, answering 206 with a + * Content-Range, and advertising `bytes` here instead. + * + * @param {object} res - express response + * @param {string} filepath - already checked for containment + * @param {string} filename - what the client is told to call it + * @returns {Promise} + */ +async function sendFile(res, filepath, filename) { + // Both from the one open: the handle to send from, and the length as it was + // at the moment that handle was made. Measuring it again here would be a + // later answer, and the app is writing to the file the whole time. + const { handle, stats } = await openNoFollow(filepath); + + res.attachment(filename); + res.setHeader('Content-Length', String(stats.size)); + res.setHeader('Accept-Ranges', 'none'); + + if (stats.size === 0) { + await handle.close().catch(() => {}); + res.end(); + return; + } + + // Closed here rather than by the stream, so it is closed on every path - + // including a client that disconnects part way, which destroys the response + // and leaves the read stream to be cleaned up rather than ended. + const stream = handle.createReadStream({ start: 0, end: stats.size - 1, autoClose: false }); + const close = () => handle.close().catch(() => {}); + + stream.on('close', close); + stream.on('error', (error) => { + log.error(error); + close(); + // The status line and length are already sent, so there is no way to say + // what went wrong. Destroying the response is what tells the client the + // body it received is not the whole file. + res.destroy(); + }); + + // pipe() forwards data and ends the destination; it does not destroy the source + // when the destination dies. A client that disconnects part way therefore + // destroys the response and leaves this stream neither ended nor destroyed, so + // its 'close' never fires and the descriptor above is never released - one + // leaked per aborted download, on a route a caller can abort at will. + res.on('close', () => stream.destroy()); + + stream.pipe(res); +} + +module.exports = { sendFile }; diff --git a/ZelBack/src/services/utils/fluxBroadcastHelper.js b/ZelBack/src/services/utils/fluxBroadcastHelper.js index 771b41eab9..0d903217b2 100644 --- a/ZelBack/src/services/utils/fluxBroadcastHelper.js +++ b/ZelBack/src/services/utils/fluxBroadcastHelper.js @@ -1,28 +1,47 @@ -const fluxNetworkHelper = require('../fluxNetworkHelper'); -const verificationHelper = require('../verificationHelper'); const serviceHelper = require('../serviceHelper'); +const { nodeSigner } = require('./nodeSigner'); +/** + * A signature over a message, as this node - or null when it cannot sign. + * + * @param {string} message + * @param {string} [privatekey] - an explicit key, otherwise the daemon config's + * @returns {Promise} + */ async function getFluxMessageSignature(message, privatekey) { - const privKey = await fluxNetworkHelper.getFluxNodePrivateKey(privatekey); - const signature = await verificationHelper.signMessage(message, privKey); - return signature; + const signer = await nodeSigner(privatekey); + return signer ? signer.sign(message) : null; } +/** + * A broadcast, serialised and signed as this node - or null when it cannot sign. + * + * Null rather than a message carrying null where the key and signature go. + * Every peer refuses such a message without a word, so sending it costs each + * of them a signature check for nothing and tells this node nothing the + * signer's own warning did not. A caller handed null sends nothing. + * + * @param {object|string} dataToBroadcast + * @param {string} [privatekey] - an explicit key, otherwise the daemon config's + * @returns {Promise} the message to put on the wire + */ async function serialiseAndSignFluxBroadcast(dataToBroadcast, privatekey) { + const signer = await nodeSigner(privatekey); + if (!signer) return null; + const version = 1; const timestamp = Date.now(); - const pubKey = await fluxNetworkHelper.getFluxNodePublicKey(privatekey); const message = serviceHelper.ensureString(dataToBroadcast); - const messageToSign = version + message + timestamp; - const signature = await getFluxMessageSignature(messageToSign, privatekey); - const dataObj = { + const signature = signer.sign(version + message + timestamp); + if (!signature) return null; + + return JSON.stringify({ version, timestamp, - pubKey, + pubKey: signer.pubKey, signature, data: dataToBroadcast, - }; - return JSON.stringify(dataObj); + }); } module.exports = { diff --git a/ZelBack/src/services/utils/fluxController.js b/ZelBack/src/services/utils/fluxController.js index 065aa12173..ab8d8a59b4 100644 --- a/ZelBack/src/services/utils/fluxController.js +++ b/ZelBack/src/services/utils/fluxController.js @@ -1,5 +1,20 @@ const { AsyncLock } = require('./asyncLock'); +/** + * What the controller is for, as opposed to what its signal is doing. + * + * A signal fires once and has to be replaced to run again, so it cannot also + * carry whether the loop is meant to be running: replacing it makes "stopped" + * stop being true while the thing is still stopping, and whatever reads it next + * schedules another iteration. The state below is the durable answer, and the + * signal goes back to being what it is - a cancellation for the work in flight. + */ +const ControllerState = Object.freeze({ + IDLE: 'idle', + RUNNING: 'running', + STOPPING: 'stopping', +}); + class FluxController { /** * Used for functions to stop work @@ -27,9 +42,17 @@ class FluxController { #timeouts = new Map(); /** - * If the main runner loop is active + * Whether the loop is meant to be running. The only thing that decides + * whether an iteration schedules the next one. + */ + #state = ControllerState.IDLE; + + /** + * Which run a loop belongs to. An iteration that was already in flight when + * the controller was stopped belongs to the run before this one, and must not + * arm a timer beside the loop a later start has since begun. */ - #running = false; + #generation = 0; /** * async locks for functions to be able to tell the controller @@ -37,6 +60,12 @@ class FluxController { */ #locks = new Map([['default', new AsyncLock()]]); + /** + * Which locks a stop waits for. The default one always; a named one only if + * it was added as a barrier. + */ + #abortBlockers = new Set(['default']); + get ['lock']() { return this.#locks.get('default'); } @@ -49,8 +78,39 @@ class FluxController { return this.lock.locked; } + /** + * Whether anything is here to start beside. Not idle, rather than running: a + * controller part way through a stop is not one to start a second loop + * against, and a caller that read a stop in progress as stopped would reset + * its own state for a loop startLoop then refuses to begin. + * + * Guard a start on this. Condition a loop on `active`. + */ get ['running']() { - return this.#running; + return this.#state !== ControllerState.IDLE; + } + + /** + * Whether the run this work belongs to is still wanted. + * + * The condition for a runner that loops on its own rather than returning a + * delay. `aborted` cannot answer it: the signal is reissued when the abort + * finishes, so a loop that yields and re-reads it after the stop has + * completed sees false and carries on for the life of the process. This says + * no from the first line of `abort()` until the next `startLoop()`, whenever + * it is asked. + * + * Only from INSIDE a run. It is false when no loop is running, so a function + * that is called both as the runner and directly by something else must ask + * `aborted` instead - "is a cancellation in flight" - or it does nothing at + * all when nobody has started a loop. + */ + get ['active']() { + return this.#state === ControllerState.RUNNING; + } + + get ['state']() { + return this.#state; } get ['loopCount']() { @@ -79,18 +139,36 @@ class FluxController { } /** - * Loops user provided runner function + * Loops user provided runner function. + * + * Whether to run again is decided by this controller's own state and by the + * generation this iteration belongs to - never by the abort signal, which is + * reissued when the abort finishes and so goes false again while an iteration + * may still be unwinding. A runner is free to hold any lock or none, to watch + * the signal or ignore it: after a stop it is not run again. + * + * A runner that throws is deliberately NOT caught. Every runner this serves + * guards its own expected failures, so a throw arriving here is a fault nobody + * predicted - and what the node does with one of those is already decided at + * the process level: apiServer's uncaughtException handler logs it and exits, + * and systemd brings the node back thirty seconds later with every subsystem + * running again. Catching it here would keep the node up and leave this loop + * stopped for the life of the process instead, with nothing reading `running` + * to notice and nothing that would start it again. + * * @param {async function():number} runner function to be run + * @param {number} generation the run this iteration belongs to * @returns {Promise} */ - async loop(runner) { + async loop(runner, generation = this.#generation) { const ms = await runner(); this.#loopCount += 1; - if (this.aborted) return; + if (generation !== this.#generation) return; + if (this.#state !== ControllerState.RUNNING) return; - this.#loopTimeout = setTimeout(() => this.loop(runner), ms); + this.#loopTimeout = setTimeout(() => this.loop(runner, generation), ms); } /** @@ -117,19 +195,29 @@ class FluxController { * @returns {Boolean} If the runner was started */ startLoop(runner) { - if (this.#running) return false; + // Refused while stopping as well as while running: a start that overlaps a + // stop is a second loop beside the one being torn down. + if (this.#state !== ControllerState.IDLE) return false; - this.#running = true; - this.loop(runner); + this.#state = ControllerState.RUNNING; + this.#generation += 1; + this.loop(runner, this.#generation); return true; } /** - * Sets AbortController signal, Interrupts any sleeps that are running, - * awaits the lock and creates a new AbortController + * Stop the loop, cancel the work in flight, and return once it has let go. + * + * The state is taken first, so an iteration that finishes at any point from + * here on finds the loop no longer wanted - which is what makes the stop + * final, rather than the signal, which is reissued at the end of this so the + * controller can be used again. + * * @returns {Promise} */ async abort() { + this.#state = ControllerState.STOPPING; + this.#generation += 1; this.stopLoop(); this.#abortController.abort(); // eslint-disable-next-line no-restricted-syntax @@ -139,20 +227,43 @@ class FluxController { } this.#timeouts.clear(); this.#timeoutId = 0; - await this.lock.waitReady(); + // The default lock and any named one added as a barrier - not every lock: a + // named lock is usually a caller's own coordination, and waiting on those + // deadlocks a stop against the work it is stopping. + await Promise.all( + [...this.#abortBlockers] + .map((name) => this.#locks.get(name)) + .filter(Boolean) + .map((lock) => lock.waitReady()), + ); + // Reissued here rather than at the next start, because a controller's + // signal is also handed to work that has no loop: a client rebuilt straight + // after a stop would otherwise be born already cancelled. this.#abortController = new AbortController(); - this.#running = false; + this.#state = ControllerState.IDLE; } /** + * Add a named lock. + * + * A named lock is the caller's own by default: `abort()` does not wait for it, + * because what a caller uses one for is usually its own coordination - + * networkStateManager's `fetcher` is how its readers wait for a fetch, and a + * stop that waited on that would hang against the very thing it is stopping. + * + * `blocksAbort` says this one is different: work held under it is work a stop + * waits for, like the default lock. Use it for work that must finish before + * the controller is idle, not for work others merely watch. * * @param {string} name Name of the lock + * @param {{blocksAbort?: boolean}} options * @returns {boolean} If the lock was added */ - addLock(name) { + addLock(name, options = {}) { if (this.#locks.has(name)) return false; this.#locks.set(name, new AsyncLock()); + if (options.blocksAbort) this.#abortBlockers.add(name); return true; } @@ -177,9 +288,10 @@ class FluxController { if (name === 'default') return false; this.#locks.delete(name); + this.#abortBlockers.delete(name); return true; } } -module.exports = { FluxController }; +module.exports = { FluxController, ControllerState }; diff --git a/ZelBack/src/services/utils/fluxEventBus.js b/ZelBack/src/services/utils/fluxEventBus.js index 3ca73c85a5..be6fe57be3 100644 --- a/ZelBack/src/services/utils/fluxEventBus.js +++ b/ZelBack/src/services/utils/fluxEventBus.js @@ -1,3 +1,25 @@ +// The harness-only telemetry surface: an event stream and a set of counters, +// both dead in production (`testEventStream` is false there, and every entry +// point below returns before doing any work). +// +// THE RULE, and it is what keeps the ring a sensible size: FACTS AS EVENTS, +// CADENCE AS A COUNTER. +// +// An event marks something that HAPPENED - a block processed, a container +// actuated, a spec stored. Every one of the publishers in this codebase is of +// that kind, and because things happening are rare, a 1024-entry ring is +// generous. A heartbeat - "the loop ran again and did nothing" - is not a fact +// about the system, it is a fact about the clock, and putting one in here +// spends a SHARED budget that every other consumer draws from. The chatty +// publisher does not pay that cost; whichever other event a test needed pays +// it, silently. +// +// So when a test needs to know a loop has run N times, or which branch it took +// on each pass, that is a TALLY, not a stream: increment a counter with +// `count()` and let the reader ask for the number over /flux/testcounters, +// rather than broadcasting twenty messages a minute so it can count them. +// Publish an event only for the thing that actually happened. + const { EventEmitter } = require('node:events'); const config = require('config'); const log = require('../../lib/log'); @@ -19,6 +41,7 @@ class FluxEventBus extends EventEmitter { #writeCount; #nextId; #enabled; + #counters; constructor(enabled) { super(); @@ -39,6 +62,7 @@ class FluxEventBus extends EventEmitter { // is fine - no consumer survives one. this.#nextId = Number(process.hrtime.bigint() / 1000n); this.#enabled = enabled ?? (config.has('testEventStream') && config.get('testEventStream') === true); + this.#counters = new Map(); } get enabled() { return this.#enabled; } @@ -76,6 +100,69 @@ class FluxEventBus extends EventEmitter { return result; } + // The id of the oldest entry the ring still holds, or 0 while it has never + // wrapped (nothing has been dropped, so every id is still reachable). + // + // Ids are minted with a single ++, so they are contiguous: anything between a + // consumer's last-seen id and this one was published and then overwritten. + // Without this, since() cannot tell "nothing happened since you last looked" + // apart from "plenty happened and I threw it away", and a consumer that + // reconnects into a gap waits out its whole budget for an event that already + // came and went - failing at the deadline, as a product bug, rather than at + // the cause. + oldestRetainedId() { + if (this.#writeCount === 0 || this.#writeCount <= RING_BUFFER_SIZE) return 0; + const oldest = this.#buffer[this.#writeIndex]; + return oldest ? oldest.id : 0; + } + + // Cadence, not facts - see the rule at the top of this file. A no-op when + // disabled, exactly like publish(). + // + // count('masterSlave:cycles') -> counters['masterSlave:cycles'] + // count('masterSlave:decision', id, 'heldOnPeer') -> counters['masterSlave:decision'][id].heldOnPeer + count(name, ...path) { + if (!this.#enabled) return; + let node = this.#counters.get(name); + if (!node) { + node = path.length ? new Map() : 0; + this.#counters.set(name, node); + } + if (!path.length) { + this.#counters.set(name, (typeof node === 'number' ? node : 0) + 1); + return; + } + let cursor = node; + for (let i = 0; i < path.length - 1; i += 1) { + let next = cursor.get(path[i]); + if (!(next instanceof Map)) { + next = new Map(); + cursor.set(path[i], next); + } + cursor = next; + } + const leaf = path[path.length - 1]; + cursor.set(leaf, (cursor.get(leaf) || 0) + 1); + } + + counters() { + const plain = (value) => { + if (!(value instanceof Map)) return value; + const out = {}; + for (const [k, v] of value) out[k] = plain(v); + return out; + }; + return plain(this.#counters); + } + + countersHandler(req, res) { + if (!this.#enabled) { + res.status(404).json({ status: 'error', data: { message: 'Test counters not enabled' } }); + return; + } + res.json({ status: 'success', data: this.counters() }); + } + sseHandler(req, res) { if (!this.#enabled) { res.status(404).json({ status: 'error', data: { message: 'Event stream not enabled' } }); @@ -91,6 +178,13 @@ class FluxEventBus extends EventEmitter { res.flushHeaders(); const lastId = parseInt(req.headers['last-event-id'], 10) || 0; + // A resuming consumer that fell behind the ring is told so, and told how + // much, before it is handed the survivors. Silence here is what turns a + // dropped event into a timeout somewhere else entirely. + const oldestRetained = this.oldestRetainedId(); + if (lastId > 0 && oldestRetained > lastId + 1) { + sseWrite(res, `event: stream:gap\ndata: ${JSON.stringify({ afterId: lastId, oldestRetainedId: oldestRetained, dropped: oldestRetained - lastId - 1 })}\nid: ${lastId}\n\n`); + } const missed = this.since(lastId); for (const entry of missed) { sseWrite(res, `event: ${entry.event}\ndata: ${JSON.stringify(entry.data)}\nid: ${entry.id}\n\n`); diff --git a/ZelBack/src/services/utils/fluxHttpTestServer.js b/ZelBack/src/services/utils/fluxHttpTestServer.js index bd17458b1c..5ee8fc2306 100644 --- a/ZelBack/src/services/utils/fluxHttpTestServer.js +++ b/ZelBack/src/services/utils/fluxHttpTestServer.js @@ -1,5 +1,29 @@ const http = require('node:http'); +/** + * The response header the secret travels in. + * + * At the FRONT of the answer, and in an answer this file writes in full, both on + * purpose. A peer asked to read a port relays a BOUNDED PREFIX of what it found - + * bounded because the port may be forwarded to a neighbour at the same public + * address, so those can be a stranger's bytes - and the requester's only evidence + * is finding its secret inside that prefix. + * + * So proof that sits at the END of the stream is proof the bound can cut. It did + * sit there: the token rode in the body and finished 48 bytes short of the cap, + * and almost none of what preceded it was ours. Node emits Date, Connection and + * the transfer framing itself, and takes Connection from what the READING peer + * sent - so the margin was set by a request string in another service and moved + * between 20 and 80 bytes with it. Losing the token refuses an install while + * reporting that a neighbour holds the port, identically on every peer, which is + * the one shape the two-witness rule corroborates rather than catches. + * + * Hence every header below, including the three Node would otherwise append on + * its own: the reply is the same 187 bytes and the token ends at byte 67 whatever + * the peer asks for and whatever Node would have chosen. + */ +const TOKEN_HEADER = 'X-Flux-Port-Test'; + class FluxHttpTestServer extends http.Server { /** * The reason this class is necessary is because we allow old nodeJS versions. @@ -12,8 +36,49 @@ class FluxHttpTestServer extends http.Server { #currentConnectionId = 0; - constructor() { - super(() => { }); + /** + * The secret this server answers with, for this port test only. + * + * A peer asked to test a port reports whether something answered at our + * public address. Where several Flux nodes share that address the router + * forwards each port to exactly one of them, so what answered can be a + * neighbour's application - and the peer cannot tell, because from outside + * there is nothing to tell. + * + * Answering with a secret the requester never handed out makes it tellable: + * only the thing the requester started can produce it. The neighbour's + * application has never seen it. + */ + #token = null; + + constructor(token = null) { + super((req, res) => { + // Nothing about this answer is left to Node: no Date, an explicit length + // so there is no chunked framing, and our own Connection rather than the + // one it would mirror back from the request. What this file says is what + // goes on the wire. + res.sendDate = false; + + if (!this.#token) { + res.writeHead(204, { Connection: 'close' }); + res.end(); + return; + } + + // The token is not in here. It is a header, and this says only what the + // port is, for whoever reaches it with a browser. + const body = JSON.stringify({ status: 'success', data: { portTest: true } }); + + res.writeHead(200, { + [TOKEN_HEADER]: this.#token, + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body), + Connection: 'close', + }); + res.end(body); + }); + + this.#token = token; this.addListener('connection', (socket) => this.#handleConnection(socket)); } @@ -38,4 +103,4 @@ class FluxHttpTestServer extends http.Server { } } -module.exports = { FluxHttpTestServer }; +module.exports = { FluxHttpTestServer, TOKEN_HEADER }; diff --git a/ZelBack/src/services/utils/globalState.js b/ZelBack/src/services/utils/globalState.js index 3c8e2031fe..47fe5d2309 100644 --- a/ZelBack/src/services/utils/globalState.js +++ b/ZelBack/src/services/utils/globalState.js @@ -12,6 +12,7 @@ let masterSlaveAppsRunning = false; const daemonReadyGate = new AsyncGate(); const bootContainerStateSettledGate = new AsyncGate(); const dbReadyGate = new AsyncGate(); +let appStateAuthoritative = false; let updateSyncthingRunning = false; let syncthingAppsFirstRun = true; const backupInProgress = []; @@ -42,6 +43,44 @@ const runningAppsCache = new Set(); // Containers intentionally stopped by FluxOS — crash recovery skips die events for these const stoppingContainers = new Set(); +// Containers FluxOS removed and has not created again — who removed the container, +// which is the only thing the tampering decision turns on. Docker names, keyed as +// stoppingContainers is. +// +// An absent container is the strongest local evidence of host-side interference the +// node has, and the reconciler records it as `container_vanished`, the +// heaviest-weighted tampering event there is. That reading holds only for a +// container FluxOS did not remove: a teardown that fails part way leaves an absence +// FluxOS caused with the app's row intact, and the app keeps being reconciled, so +// membership here is what stops a node scoring its own removal against the app it +// is hosting. +// +// Written by dockerService's removal funnels, dropped by its creation funnel, and +// dropped for a whole app when the app's local row goes (nothing reconciles it +// after that, so there is no absence left to attribute). FluxOS removed it -> +// present; FluxOS created it -> absent; anything missing without an entry here is +// what the tampering event is for. +// +// In-memory deliberately: across a restart the node genuinely cannot tell its own +// removal from anyone else's, and an entry that survived would suppress a real +// signal. +const fluxRemovedContainers = new Set(); + +// Syncthing folders this node holds writable (sendreceive), refreshed by the +// syncthing monitor each pass and served to peers that ask before promoting a +// folder of their own. Kept here rather than read from syncthing per request: +// the route is unauthenticated, and an on-demand read would be an amplifier into +// syncthing on a node any peer can reach. +// +// null until the monitor's first validated read, and a Set from then on. "I hold +// nothing writable" and "I have not looked yet" are the same empty set but +// opposite answers to a peer deciding whether to promote, so they must not be the +// same value: a node that IS holding a folder would otherwise read as free, and +// the peer would promote alongside it. On a booting node that pass is not +// immediate, and a fleet-wide restart puts every holder of an app in the state at +// once. Same null-is-no-opinion convention appReconciler's controllerDesired uses. +let promotedFolderIds = null; + // Cache references - these will be initialized from cacheManager let spawnErrorsLongerAppCache = null; @@ -52,7 +91,7 @@ function initializeCaches(cacheManager) { if (cacheManager && cacheManager.appSpawnErrorCache && cacheManager.appSpawnCache) { spawnErrorsLongerAppCache = cacheManager.appSpawnErrorCache; trySpawningGlobalAppCache = cacheManager.appSpawnCache; - pendingAppUpdatesCache = cacheManager.pendingAppUpdatesCache; + ({ pendingAppUpdatesCache } = cacheManager); } } @@ -73,6 +112,27 @@ module.exports = { get reinstallationOfOldAppsInProgress() { return reinstallationOfOldAppsInProgress; }, set reinstallationOfOldAppsInProgress(value) { reinstallationOfOldAppsInProgress = value; }, + // The operation holding this node right now, named, or null. `except` is the + // caller's OWN flag: a guard excludes the operation it belongs to and no + // others, because a redeploy that asked without excluding itself would refuse + // its own reinstall. Order is the order the guards asked in. + // + // Every entry point that can START work asks this. The five flags used to be + // read as hand-picked subsets - forty-six guards, exactly one of which read + // reinstallationOfOldAppsInProgress - so the periodic reinstall pass announced + // itself and the spawner walked straight past it, took the node during the + // pass's own wait, and left an app torn down that could not be rebuilt. + operationHolding(except = null) { + const held = [ + ['removal', removalInProgress], + ['installation', installationInProgress], + ['soft redeploy', softRedeployInProgress], + ['hard redeploy', hardRedeployInProgress], + ['reinstallation', reinstallationOfOldAppsInProgress], + ].find(([name, on]) => on && name !== except); + return held ? held[0] : null; + }, + isOperationInProgress() { return removalInProgress || installationInProgress || softRedeployInProgress || hardRedeployInProgress || reinstallationOfOldAppsInProgress; }, @@ -92,14 +152,57 @@ module.exports = { set dbReady(value) { if (value) dbReadyGate.open(); else dbReadyGate.close(); }, waitForDbReady() { return dbReadyGate.wait(); }, + // Whether this node's ephemeral app-state store is worth another node's + // survey: its own state sync completed, or it has spent the block timer + // taking live broadcasts. NOT dbReady, which is about globalAppsInformation + // and a different set of collections entirely. + // + // It lives here rather than being read off the orchestrator because the only + // caller is the sync responder, and fluxCommunicationMessagesSender reaching + // back into appSyncOrchestrator is a cycle. The orchestrator owns the value + // and mirrors it; nothing else writes it. + get appStateAuthoritative() { return appStateAuthoritative; }, + set appStateAuthoritative(value) { appStateAuthoritative = Boolean(value); }, + get updateSyncthingRunning() { return updateSyncthingRunning; }, set updateSyncthingRunning(value) { updateSyncthingRunning = value; }, get syncthingAppsFirstRun() { return syncthingAppsFirstRun; }, set syncthingAppsFirstRun(value) { syncthingAppsFirstRun = value; }, - get backupInProgress() { return backupInProgress; }, - get restoreInProgress() { return restoreInProgress; }, + // A frozen snapshot, not the live array: readers (the monitor, the election, + // the reconciler) only ever test membership, and handing out the backing + // array let any of them push or splice it and bypass the atomic claim below. + // Frozen rather than merely copied so that a stray write throws here instead + // of silently mutating a copy nobody reads. The claim and release are the + // only writers, and they hold the real arrays. + get backupInProgress() { return Object.freeze([...backupInProgress]); }, + get restoreInProgress() { return Object.freeze([...restoreInProgress]); }, + + // Claiming an app for a backup or a restore is a test-and-set, not a read + // then a later write: these run to completion before the event loop hands the + // next request in, so two overlapping requests for one app cannot both find it + // free. The lists stay the observable "this app is busy" signal the monitor, + // the election and the reconciler read; only the claim on them is made + // indivisible here so a caller cannot split the test from the set. + tryStartBackup(appname) { + if (backupInProgress.includes(appname)) return false; + backupInProgress.push(appname); + return true; + }, + finishBackup(appname) { + const index = backupInProgress.indexOf(appname); + if (index !== -1) backupInProgress.splice(index, 1); + }, + tryStartRestore(appname) { + if (restoreInProgress.includes(appname)) return false; + restoreInProgress.push(appname); + return true; + }, + finishRestore(appname) { + const index = restoreInProgress.indexOf(appname); + if (index !== -1) restoreInProgress.splice(index, 1); + }, get appsMonitored() { return appsMonitored; }, set appsMonitored(value) { appsMonitored = value; }, @@ -120,10 +223,13 @@ module.exports = { get appsToBeCheckedLater() { return appsToBeCheckedLater; }, get appsSyncthingToBeCheckedLater() { return appsSyncthingToBeCheckedLater; }, get receiveOnlySyncthingAppsCache() { return receiveOnlySyncthingAppsCache; }, + get promotedFolderIds() { return promotedFolderIds; }, + set promotedFolderIds(ids) { promotedFolderIds = ids; }, get syncthingDevicesIDCache() { return syncthingDevicesIDCache; }, get folderHealthCache() { return folderHealthCache; }, get runningAppsCache() { return runningAppsCache; }, get stoppingContainers() { return stoppingContainers; }, + get fluxRemovedContainers() { return fluxRemovedContainers; }, get spawnErrorsLongerAppCache() { return spawnErrorsLongerAppCache; }, set spawnErrorsLongerAppCache(value) { spawnErrorsLongerAppCache = value; }, diff --git a/ZelBack/src/services/utils/imageVerifier.js b/ZelBack/src/services/utils/imageVerifier.js index e8e7ca4c44..1902bef4d3 100644 --- a/ZelBack/src/services/utils/imageVerifier.js +++ b/ZelBack/src/services/utils/imageVerifier.js @@ -1,8 +1,5 @@ -const config = require('config'); const serviceHelper = require('../serviceHelper'); -const { AsyncLock } = require('./asyncLock'); - /** * Docker Architecture * @typedef {"amd64" | "arm64"} Architecture @@ -11,7 +8,7 @@ const { AsyncLock } = require('./asyncLock'); class ImageVerifier { static defaultDockerRegistry = 'registry-1.docker.io'; - static imagePattern = /^(?:(?(?:(?:[\w-]+(?:\.[\w-]+)+)(?::\d+)?)|[\w]+:\d+)\/)?\/?(?(?:(?:[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)\/){0,2})(?[a-z0-9-_.]+\/{0,1}[a-z0-9-_.]+)[:]?(?[\w][\w.-]{0,127})?/; + static imagePattern = /^(?:(?(?:(?:[\w-]+(?:\.[\w-]+)+)(?::\d+)?)|[\w]+:\d+)\/)?\/?(?(?:(?:[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)\/){0,2})(?[a-z0-9-_.]+\/{0,1}[a-z0-9-_.]+)[:]?(?[\w][\w.-]{0,127})?/; static wwwAuthHeaderPattern = /(?Bearer|Basic)\s+realm="(?[^"]+)"(?:,\s*service="(?[^"]+)")?(?:,\s*scope="(?[^"]+)")?/; @@ -22,15 +19,6 @@ class ImageVerifier { 'application/vnd.docker.distribution.manifest.list.v2+json', ]; - static whitelistedImages = []; - - static lastWhitelistFetchTime = 0; - - static resetWhitelist() { - ImageVerifier.whitelistedImages = []; - ImageVerifier.lastWhitelistFetchTime = 0; - } - /** * Parse www-authenticate header * @param {string} authHeader # www-auth header @@ -44,8 +32,6 @@ class ImageVerifier { return { ...match.groups }; } - static fetchLock = new AsyncLock(); - #abortController = new AbortController(); #axiosInstance = null; @@ -168,49 +154,6 @@ class ImageVerifier { }); } - async #fetchWhitelist() { - // ToDo: use etag - if ( - this.error - && !this.#lookupErrorDetail.match('Unable to fetch whitelisted repositories') - ) { - return; - } - - const now = Number(process.hrtime.bigint() / BigInt(1_000_000_000)); - - await ImageVerifier.fetchLock.enable(); - - try { - if ( - ImageVerifier.whitelistedImages.length - && ImageVerifier.lastWhitelistFetchTime + 600 > now - ) return; - - const { data } = await serviceHelper - .axiosGet( - `${config.github.rawBaseUrl}/helpers/repositories.json`, - { timeout: 20_000 }, - ) - .catch((err) => { - this.#lookupErrorDetail = 'Unable to fetch whitelisted repositories. Try again later.'; - this.#lookupErrorMeta = { - httpStatus: err?.response?.status || null, - errorCode: err?.code || null, - errorType: 'whitelist_fetch_error', - }; - return { data: [] }; - }); - - ImageVerifier.lastWhitelistFetchTime = now; - - // this could throw if data not array - if (data.length) ImageVerifier.whitelistedImages = data; - } finally { - ImageVerifier.fetchLock.disable(); - } - } - #parseDockerTag() { if (this.error) return; @@ -638,49 +581,6 @@ class ImageVerifier { this.#abortController.abort(); } - async isWhitelisted() { - await this.#fetchWhitelist(); - - if (this.error) return false; - - if (!this.useable) { - this.#evaluationErrorDetail = `Image Tag: ${this.rawImageTag} is not in valid format [HOST[:PORT_NUMBER]/][NAMESPACE/]REPOSITORY:TAG`; - this.#lookupErrorMeta = { - httpStatus: null, - errorCode: null, - errorType: 'invalid_format', - }; - return false; - } - - const separators = ['/', ':']; - - const whitelisted = ImageVerifier.whitelistedImages.find( - // doesn't matter if rawImageTag is shorter than img - (otherTag) => { - const len = otherTag.length; - const thisTag = this.rawImageTag; - - return ( - thisTag === otherTag - || (thisTag.slice(0, len) === otherTag - && separators.includes(thisTag.slice(len, len + 1))) - ); - }, - ); - - if (!whitelisted) { - this.#evaluationErrorDetail = 'Repository is not whitelisted. Please contact Flux Team.'; - this.#lookupErrorMeta = { - httpStatus: null, - errorCode: null, - errorType: 'not_whitelisted', - }; - } - - return Boolean(whitelisted); - } - /** * Checks that the image is available for the provided architecture set, and that the image's size * is less that the configured maximum image size, for the provided architecture set. diff --git a/ZelBack/src/services/utils/installOutcome.js b/ZelBack/src/services/utils/installOutcome.js new file mode 100644 index 0000000000..0267186d9f --- /dev/null +++ b/ZelBack/src/services/utils/installOutcome.js @@ -0,0 +1,33 @@ +/** + * What an install attempt did, for a caller that has to decide what to clean up. + * + * A boolean cannot carry this. `false` meant both "another operation holds the + * node, I touched nothing" and "I got part way, it failed, and I have already + * torn the app down" - and a redeploy reading the first as the second answered a + * five-second scheduling collision by force-uninstalling a running application + * and broadcasting its removal to the network. + * + * The distinction is the whole point: REFUSED means the app is exactly as it was, + * FAILED means it is gone. + * + * Internal only - nothing answers these to a client, so unlike Privilege the + * values are ours as well as the names. They are strings rather than booleans so + * a call site reads as the question it is asking. + * + * All three are truthy, so a caller left on `if (!outcome)` reads a refusal as a + * success. That is why every call site was changed with the return type rather + * than left to be found later. + */ +const InstallOutcome = Object.freeze({ + // The app is installed and running. + INSTALLED: 'installed', + // Nothing was touched. Another operation holds the node, or the app is already + // installed. Whatever was running before is still running, and the caller has + // nothing to undo. + REFUSED: 'refused', + // The install got part way and cleaned up after itself, so the app is no longer + // on this node. The only outcome that justifies a caller acting on the loss. + FAILED: 'failed', +}); + +module.exports = { InstallOutcome }; diff --git a/ZelBack/src/services/utils/instanceOrdering.js b/ZelBack/src/services/utils/instanceOrdering.js new file mode 100644 index 0000000000..8f8607ae13 --- /dev/null +++ b/ZelBack/src/services/utils/instanceOrdering.js @@ -0,0 +1,101 @@ +// Deterministic ordering shared by every code path that ranks an app's +// installing claims or running instances to decide which node keeps the app +// and which stands aside. Each node sorts values carried inside the broadcast +// messages, so the fleet agrees on the outcome only if the order is total: +// a comparator that returns 0 on equal timestamps ranks tied entries by local +// arrival order - different on every node - and two nodes can each compute +// the winning rank for themselves. On any tie the lower socket address +// survives; the higher address is the junior entry and stands aside. + +/** + * Epoch milliseconds for a timestamp, which reaches these comparators as the + * Date the database returns, or as a number or ISO string from a message that + * has not been through storage. Comparing the raw values cannot order them: + * two Dates of the same instant are never `!==` equal, so an equality test + * always sees a difference and never reaches the tie-break, and the relational + * operators coerce a Date to a number but an ISO string to NaN, so a mixed pair + * compares false in both directions. Both leave ties to the array's own order, + * which is arrival order and differs on every node. + * @param {Date|number|string|null|undefined} value Timestamp in any of the shapes above. + * @returns {number|null} Epoch milliseconds, or null when there is no usable timestamp. + */ +function epochMs(value) { + if (value === null || value === undefined) { + return null; + } + const ms = value instanceof Date ? value.getTime() : new Date(value).getTime(); + return Number.isNaN(ms) ? null : ms; +} + +/** + * Orders installing claims for the collision resolver: earliest broadcastedAt + * first, a claim without a timestamp last (it cannot assert seniority), equal + * timestamps broken by socket address ascending - the lower address wins the + * slot. + * @param {{ip: string, broadcastedAt?: Date|number|string}} a Installing claim. + * @param {{ip: string, broadcastedAt?: Date|number|string}} b Installing claim. + * @returns {number} Comparator result for Array.prototype.sort. + */ +function compareInstallingClaims(a, b) { + const aTime = epochMs(a.broadcastedAt) ?? Number.MAX_SAFE_INTEGER; + const bTime = epochMs(b.broadcastedAt) ?? Number.MAX_SAFE_INTEGER; + if (aTime !== bTime) { + return aTime - bTime; + } + if (a.ip < b.ip) { + return -1; + } + if (a.ip > b.ip) { + return 1; + } + return 0; +} + +/** + * Orders running instances by seniority: longest-running first, an instance + * that has not yet reported runningSince ahead of all that have (an instance + * still settling is never the surplus one), equal runningSince broken by + * socket address ascending. Surplus-instance checks rank the junior end of + * this order; primary selection ranks the senior end. + * @param {{ip: string, runningSince?: Date|string|number}} a Running instance. + * @param {{ip: string, runningSince?: Date|string|number}} b Running instance. + * @returns {number} Comparator result for Array.prototype.sort. + */ +function compareInstanceSeniority(a, b) { + const aTime = epochMs(a.runningSince); + const bTime = epochMs(b.runningSince); + if (aTime === null && bTime !== null) { + return -1; + } + if (aTime !== null && bTime === null) { + return 1; + } + if (aTime !== bTime) { + return aTime - bTime; + } + if (a.ip < b.ip) { + return -1; + } + if (a.ip > b.ip) { + return 1; + } + return 0; +} + +/** + * Renders a ranked list for the resolver's decision logs - each entry's + * address with the timestamp it was ranked by - so a disputed outcome can be + * diagnosed from any single node's log. + * @param {object[]} list Entries in their ranked order. + * @param {string} timestampField Field the ranking was keyed by. + * @returns {string} One-line rendering of the ranked entries. + */ +function describeRanking(list, timestampField) { + return list.map((entry) => `${entry.ip}@${entry[timestampField] ?? 'unreported'}`).join(', '); +} + +module.exports = { + compareInstallingClaims, + compareInstanceSeniority, + describeRanking, +}; diff --git a/ZelBack/src/services/utils/jobRegistry.js b/ZelBack/src/services/utils/jobRegistry.js new file mode 100644 index 0000000000..17dc1d3f61 --- /dev/null +++ b/ZelBack/src/services/utils/jobRegistry.js @@ -0,0 +1,348 @@ +const crypto = require('crypto'); +const config = require('config'); +const log = require('../../lib/log'); + +// One registry for every long-running operation a node accepts, so a client +// polls one URL family, reads one status field and gets one error shape no +// matter which endpoint started the work. Before this, each feature invented +// its own: one said `settled: true`, another `state: 'done'`, neither sent a +// Retry-After, and a client had to know which endpoint it had called to know +// how to read the answer. +// +// Deliberately in-memory and node-local. The work these track is node-local +// too, so a job lost to a restart costs a re-ask and nothing else - the same +// call the client makes when a poll 404s. + +const NS_PER_MS = 1_000_000n; + +const JobStatus = Object.freeze({ + RUNNING: 'Running', + SUCCEEDED: 'Succeeded', + FAILED: 'Failed', + CANCELED: 'Canceled', + // The node took the work away. Distinct from both neighbours on purpose: a + // cancel says the caller asked for this, and a failure says their input was at + // fault. Neither is true here, and reporting it as either misattributes the + // node's decision to the person it was made against. + EVICTED: 'Evicted', +}); + +const TERMINAL = Object.freeze([ + JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELED, JobStatus.EVICTED, +]); + +const jobs = new Map(); + +/** + * An operation id, mintable before the operation is registered. + * + * The format lives here rather than at each caller so `op_` stays one fact. + * @returns {string} + */ +function mintJobId() { + return `op_${crypto.randomUUID()}`; +} + +function retentionMs() { + return config.fluxapps.operationRetentionMs ?? 60 * 60 * 1000; +} + +/** How long a client should wait before polling again, while a job is running. */ +function retryAfterSeconds() { + return config.fluxapps.operationRetryAfterSeconds ?? 2; +} + +function isTerminal(status) { + return TERMINAL.includes(status); +} + +function pruneExpired() { + const now = process.hrtime.bigint(); + for (const [id, job] of jobs) { + if (job.expiresAtNs !== null && now >= job.expiresAtNs) jobs.delete(id); + } +} + +function scheduleExpiry(job) { + job.expiresAtNs = process.hrtime.bigint() + BigInt(retentionMs()) * NS_PER_MS; +} + +/** + * Normalize a failure to RFC 9457 problem+json. Accepts an Error or an already + * shaped problem, so a caller can hand over whatever it has. + * + * Credentials are scrubbed rather than trusted not to appear: a registry auth + * failure can carry a repoauth string in its message, and this ends up in a + * response body. + */ +function toProblem(failure, jobId) { + const problem = failure instanceof Error + ? { + title: failure.name || 'Error', + detail: failure.message, + status: 500, + // An errno-style code a caller acts on - EEXIST for a taken name, + // EDESTRUCTIVE for a change refused to avoid deleting unnamed data - + // rides on the Error itself. Without carrying it here the spread below + // can never fire for an Error, and every code an operation attaches is + // dropped on the way to the client. + ...(failure.code ? { code: failure.code } : {}), + } + : { title: 'Error', status: 500, ...failure }; + + return { + type: problem.type ?? 'about:blank', + title: problem.title, + status: problem.status, + detail: scrubCredentials(problem.detail ?? ''), + instance: `/apps/operations/${jobId}`, + ...(problem.code ? { code: problem.code } : {}), + ...(problem.retryAfterMs ? { retryAfterMs: problem.retryAfterMs } : {}), + }; +} + +// Registry credentials reach error messages as "user:password" or as a +// provider:// config string. Neither belongs in a status response. +function scrubCredentials(detail) { + if (typeof detail !== 'string' || !detail) return ''; + return detail + .replace(/\b[\w.-]+:[^\s@/]{4,}@/g, '@') + .replace(/\b(?:aws|azure|gcp|gar|acr|ecr):\/\/\S+/gi, ''); +} + +/** + * Register a new operation. + * + * @param {object} params + * @param {string} params.kind what the operation is, e.g. 'imagepreflight' + * @param {string|null} [params.owner] the FluxID allowed to read it; null means + * the jobId alone is the capability + * @param {() => object} [params.detail] called at read time for the operation's + * own payload, so a service keeps its domain state where it already lives + * instead of copying it in here on every transition + * @param {() => void} [params.onCancel] called when a cancel is REQUESTED, so + * work that is waiting on something can be woken and see the flag. Without it + * a cancel is only observed the next time the work happens to look. + * @param {string} [params.jobId] a caller-minted id, from mintJobId(). For work + * whose identity is needed BEFORE it is certain to run: the playground names + * its containers and network after its session, and has to do so while + * deciding whether to accept it - long before there is a job to poll. The + * registration still happens last, so a refusal is still an answer on the + * request rather than a job someone has to poll to discover. + * @returns {{jobId: string, statusUrl: string}} + */ +function start(params) { + pruneExpired(); + + const { + kind, owner = null, detail = null, onCancel = null, + } = params; + const jobId = params.jobId ?? mintJobId(); + const now = Date.now(); + + jobs.set(jobId, { + jobId, + kind, + owner, + detail, + onCancel, + status: JobStatus.RUNNING, + createdAt: now, + lastUpdatedAt: now, + progress: [], + error: null, + canceled: false, + expiresAtNs: null, + }); + + return { jobId, statusUrl: statusUrlFor(jobId) }; +} + +/** + * The operation currently running for an app, if there is one. + * + * A caller refused for want of a slot is refused BECAUSE of this job, so it is + * what the refusal should name: something to watch or cancel rather than a + * suggestion to try again and find out. Scanned rather than indexed - a node + * runs a handful of these at a time, and an index would be a second thing to + * keep true. + * + * @param {string} app + * @returns {{jobId: string, kind: string, statusUrl: string, + * startedAt: number, detail: object}|null} + */ +function runningForApp(app) { + for (const job of jobs.values()) { + if (job.status !== JobStatus.RUNNING) continue; + const detail = typeof job.detail === 'function' ? job.detail() : job.detail; + if (!detail || detail.app !== app) continue; + return { + jobId: job.jobId, + kind: job.kind, + statusUrl: statusUrlFor(job.jobId), + startedAt: job.createdAt, + detail, + }; + } + return null; +} + +function statusUrlFor(jobId) { + return `/apps/operations/${jobId}`; +} + +function touch(jobId) { + const job = jobs.get(jobId); + if (job) job.lastUpdatedAt = Date.now(); +} + +/** + * Append one human-readable step. Progress is append-only and polls return the + * whole array, so a client that missed a poll loses nothing and can diff by + * index rather than parsing a stream. + * + * A step repeated is not a step. The executor reports liveness on a timer, + * with a status line that is fixed for the whole operation - so recording each + * one would add an identical entry every couple of seconds, and because every + * poll re-sends the whole array, a long operation costs more to report on than + * to perform. A repeat still means the job is alive, which is what + * lastUpdatedAt carries. + */ +function progress(jobId, message) { + const job = jobs.get(jobId); + if (!job || isTerminal(job.status)) return; + job.lastUpdatedAt = Date.now(); + const last = job.progress[job.progress.length - 1]; + if (last && last.message === message) return; + job.progress.push({ at: job.lastUpdatedAt, message }); +} + +function succeed(jobId) { + const job = jobs.get(jobId); + if (!job || isTerminal(job.status)) return; + job.status = JobStatus.SUCCEEDED; + job.lastUpdatedAt = Date.now(); + scheduleExpiry(job); +} + +function fail(jobId, failure) { + const job = jobs.get(jobId); + if (!job || isTerminal(job.status)) return; + job.status = JobStatus.FAILED; + job.error = toProblem(failure, jobId); + job.lastUpdatedAt = Date.now(); + scheduleExpiry(job); + log.warn(`Operation ${jobId} (${job.kind}) failed: ${job.error.detail}`); +} + +/** + * Best-effort cancel: the flag is raised here and the worker is expected to + * notice at its next checkpoint, so a job is only Canceled once it has actually + * stopped. + */ +function requestCancel(jobId) { + const job = jobs.get(jobId); + if (!job || isTerminal(job.status)) return false; + job.canceled = true; + job.lastUpdatedAt = Date.now(); + // A cancel is a thing that HAPPENED, so the work is told rather than left to + // notice. An operation that waits on events would otherwise not see the flag + // until whatever it is waiting for arrives - which for a quiet playground + // session is its full fifteen-minute deadline. + if (job.onCancel) { + try { + job.onCancel(); + } catch (err) { + log.error(`Operation ${jobId} cancel handler failed: ${err.message}`); + } + } + return true; +} + +function isCanceled(jobId) { + const job = jobs.get(jobId); + return Boolean(job && job.canceled); +} + +function cancelled(jobId) { + const job = jobs.get(jobId); + if (!job || isTerminal(job.status)) return; + job.status = JobStatus.CANCELED; + job.lastUpdatedAt = Date.now(); + scheduleExpiry(job); +} + +/** + * The node reclaimed what this operation was using. + * + * Carries a reason, because this is the one terminal state the caller had no + * part in and cannot infer: a status alone would leave them looking for what + * they did wrong. + * + * @param {string} jobId + * @param {string} reason - shown to the caller as-is + */ +function evicted(jobId, reason) { + const job = jobs.get(jobId); + if (!job || isTerminal(job.status)) return; + job.status = JobStatus.EVICTED; + job.error = { title: 'Ended by the node', detail: reason, instance: statusUrlFor(jobId) }; + job.lastUpdatedAt = Date.now(); + scheduleExpiry(job); + log.info(`Operation ${jobId} (${job.kind}) evicted: ${reason}`); +} + +/** + * The public view of an operation, or null when it is unknown, has aged out, or + * belongs to someone else. Unknown and not-yours are the same answer on + * purpose: a jobId must not be a probe for whether other people have jobs. + * + * @param {string} jobId + * @param {string|null} [owner] the authenticated caller, when the job has one + * @returns {object|null} + */ +function get(jobId, owner = null, readOptions = {}) { + pruneExpired(); + + const job = jobs.get(jobId); + if (!job) return null; + if (job.owner !== null && job.owner !== owner) return null; + + return { + jobId: job.jobId, + kind: job.kind, + status: job.status, + createdAt: job.createdAt, + lastUpdatedAt: job.lastUpdatedAt, + progress: job.progress, + error: job.error, + // Built at read time, and given the reader's options: an operation whose + // detail is a growing log needs to know where the caller got to. + detail: job.detail ? job.detail(readOptions) : null, + }; +} + +/** Test seam: drop every operation. */ +function reset() { + jobs.clear(); +} + +module.exports = { + JobStatus, + isTerminal, + retryAfterSeconds, + statusUrlFor, + runningForApp, + mintJobId, + start, + touch, + progress, + succeed, + fail, + requestCancel, + isCanceled, + cancelled, + evicted, + get, + reset, +}; diff --git a/ZelBack/src/services/utils/logCursor.js b/ZelBack/src/services/utils/logCursor.js new file mode 100644 index 0000000000..1d0ce6b9ef --- /dev/null +++ b/ZelBack/src/services/utils/logCursor.js @@ -0,0 +1,50 @@ +/** + * The position a log reader has reached, as an opaque token. + * + * Docker's `since` is inclusive and resolves to milliseconds, so a reader cannot + * ask for "everything after this line" - it can only ask for "everything from + * this millisecond", and gets back the lines it already holds along with the new + * ones. Advancing past them is what loses data: a line at .9265 is gone forever + * to a reader that asked from .927. + * + * So the position is a pair - the millisecond reached, and how many lines were + * already delivered from it. The re-read is deliberate, and the count is what + * makes it exact: docker returns the same lines in the same order, so the first + * `count` of them are the ones already held. Identical repeated lines within one + * millisecond are handled by this and are not handled by comparing text. + * + * Opaque on the wire because clients and nodes upgrade independently: a reader + * hands back what it was given without reading it, so this shape can change + * without every client changing with it. + */ + +/** + * @param {{ms: number, count: number}} position + * @returns {string} the token a reader hands back + */ +function encode(position) { + return Buffer.from(JSON.stringify({ v: 1, ms: position.ms, count: position.count })).toString('base64url'); +} + +/** + * A token that is absent, truncated, from a future version, or simply not one of + * ours is not an error - it is a reader with no position, which is answered with + * the most recent lines the same as a first request. + * + * @param {string} token + * @returns {{ms: number, count: number}|null} + */ +function decode(token) { + if (!token || typeof token !== 'string') return null; + try { + const parsed = JSON.parse(Buffer.from(token, 'base64url').toString('utf8')); + if (parsed.v !== 1) return null; + if (!Number.isFinite(parsed.ms) || !Number.isInteger(parsed.count)) return null; + if (parsed.ms < 0 || parsed.count < 0) return null; + return { ms: parsed.ms, count: parsed.count }; + } catch (error) { + return null; + } +} + +module.exports = { encode, decode }; diff --git a/ZelBack/src/services/utils/logFrameDecoder.js b/ZelBack/src/services/utils/logFrameDecoder.js new file mode 100644 index 0000000000..66ef2e7660 --- /dev/null +++ b/ZelBack/src/services/utils/logFrameDecoder.js @@ -0,0 +1,231 @@ +/** + * Docker's log framing, decoded as it arrives rather than all at once. + * + * A poll gets one complete payload and can walk it in a loop. A follow stream + * does not: docker writes when the container writes, and a chunk boundary falls + * wherever TCP put it - through a header, through a body, between the two. So + * the frame walk has to survive being interrupted, which means holding what is + * left over until the rest of it arrives. + * + * Two different partials, and both are real. A frame can arrive in pieces, so + * the byte buffer keeps what is not yet a whole frame. A LINE can also arrive in + * pieces - docker splits a message longer than 16KB across several frames - so + * the text buffer keeps what is not yet a whole line. Splitting on '\n' per + * frame instead would cut those messages into fragments and call each one a log + * line. + * + * Every app container is created with Tty false (appDockerCreate), so every + * write is framed with an 8-byte header carrying the stream id and length. A + * Tty container writes raw and would need a different reader; none of ours do. + */ +/** + * The most of one line a bounded decoder will hold and hand over. + * + * A line is held until its newline arrives and nothing obliges a container to + * send one, so an unbounded decoder lets a process writing a large blob to + * stdout decide how much of the node's memory a viewer costs - and then how much + * of it crosses the socket, because a line that finally completes is delivered + * whole. A megabyte is far past anything a log pane renders and far past what + * any reader of this is looking for; docker's own framing splits a message at + * 16KB, so this is sixty-four of those. + */ +const MAX_LINE_LENGTH = 1024 * 1024; + +class LogFrameDecoder { + /** + * @param {{maxLineLength?: number, timestamped?: boolean}} options + * `maxLineLength` truncates a line longer than it and discards the rest of + * that line. Unbounded by default: a read that returns one payload is + * already bounded by the payload, and it is the FOLLOW stream - which lives + * for as long as a viewer watches - that has nothing else to bound it. + * + * `timestamped` says the caller asked docker for timestamps, which is what + * makes every frame body begin with one. Declared rather than detected: what + * a body holds is the caller's request to docker, not something to infer + * from the bytes. + */ + constructor(options = {}) { + /** Bytes that are not yet a complete frame */ + this.bytes = Buffer.alloc(0); + /** Text that is not yet a complete line */ + this.partial = ''; + /** Characters discarded from lines that have ENDED, since last taken */ + this.truncated = 0; + this.maxLineLength = options.maxLineLength ?? Infinity; + this.timestamped = options.timestamped ?? false; + // The tail of a line already handed over truncated. Everything up to its + // newline is counted and dropped, so one absurd line costs the cap once + // rather than arriving as a run of invented lines. + this.discarding = false; + } + + // What has been cut from the line still arriving. Held rather than reported, + // because how much was cut from a line is not known until the line ends: a + // reader told what had been discarded so far would be told again on every + // batch until the newline came, dozens of times over, for one cut line. + #discarded = 0; + + // The stamp docker put on the frame that began the line being assembled. + // + // A message longer than 16KB is split into 16KB chunks and EVERY chunk is + // given the message's stamp - measured on a live daemon: three chunks of + // `len=16415`, which is 16384 and a 31-character stamp, all three carrying the + // stamp of the first. Joined as they arrive, a line over 16KB comes back with + // docker's timestamps spliced through its body every 16KB, which is a line the + // container never wrote and a reader cannot tell from one it did. + #lineStamp = null; + + /** + * The line is over, so what was cut from it is now the whole of what was cut. + * + * @returns {void} + */ + #settle() { + this.truncated += this.#discarded; + this.#discarded = 0; + } + + /** + * How much has been discarded since this was last asked, and zero afterwards. + * + * Drained rather than read, so a caller reports each discard once however + * often it asks. + * + * @returns {number} characters + */ + takeTruncated() { + const held = this.truncated; + this.truncated = 0; + return held; + } + + /** + * The complete lines this chunk finished, in docker's order. + * + * @param {Buffer} chunk + * @returns {string[]} + */ + push(chunk) { + this.bytes = this.bytes.length ? Buffer.concat([this.bytes, chunk]) : chunk; + + let offset = 0; + let text = ''; + // Whether what has been read so far ends mid-line, which is what makes the + // NEXT frame a continuation of one rather than the start of another. Taken + // from the state this push begins in: a held partial, or a line whose tail + // is being discarded. + let continuing = this.partial !== '' || this.discarding; + while (offset + 8 <= this.bytes.length) { + const length = this.bytes.readUInt32BE(offset + 4); + // The body has not all arrived: leave the header with it, so the next + // chunk resumes at a frame boundary rather than mid-body. + if (offset + 8 + length > this.bytes.length) break; + let body = this.bytes.toString('utf8', offset + 8, offset + 8 + length); + if (this.timestamped) body = this.#ownStamp(body, continuing); + text += body; + continuing = !body.endsWith('\n'); + offset += 8 + length; + } + this.bytes = offset ? this.bytes.subarray(offset) : this.bytes; + + if (!text) return []; + + if (this.discarding) { + const ends = text.indexOf('\n'); + if (ends === -1) { + this.#discarded += text.length; + return []; + } + this.#discarded += ends; + this.discarding = false; + this.#settle(); + text = text.slice(ends + 1); + if (!text) return []; + } + + const lines = (this.partial + text).split('\n'); + // The last element is whatever followed the final newline - empty when the + // text ended on one, and the start of the next line when it did not. + this.partial = lines.pop(); + + // Every line handed over is cut, not only the tail below: a line whose + // newline arrived inside this chunk never passed through `partial`, so + // bounding the tail alone would leave what a viewer is sent at the mercy of + // where the stream happened to chunk. + const cut = []; + for (let i = 0; i < lines.length; i += 1) { + cut.push(this.#cut(lines[i])); + // Its newline has arrived, so nothing more can be cut from this one. + this.#settle(); + } + + // The tail has no newline yet and may never get one, so it is handed over + // now and the REST of that line dropped - rather than held for a newline + // that is not coming, or split into a run of lines the container never + // wrote. It goes last because it is the tail of this chunk. + if (this.partial.length > this.maxLineLength) { + cut.push(this.#cut(this.partial)); + this.partial = ''; + this.discarding = true; + } + + return cut.filter((line) => line.trim()); + } + + /** + * The body with only the stamp that belongs to it. + * + * The one that begins a line is the line's own and is kept. The one on a + * continuation is docker's copy of it, and is dropped - but only when it is + * byte-identical to the stamp this line began with. Compared rather than + * matched by shape: content that merely looks like a timestamp is not this + * line's, and if docker ever stops repeating the stamp the comparison fails + * and the body is passed through exactly as it arrived. + * + * @param {string} body + * @param {boolean} continuing whether this frame continues the line before it + * @returns {string} + */ + #ownStamp(body, continuing) { + if (!continuing) { + const ends = body.indexOf(' '); + this.#lineStamp = ends === -1 ? null : body.slice(0, ends); + return body; + } + if (this.#lineStamp && body.startsWith(`${this.#lineStamp} `)) { + return body.slice(this.#lineStamp.length + 1); + } + return body; + } + + /** + * A line no longer than this decoder holds, and what it cost counted. + * + * @param {string} line + * @returns {string} + */ + #cut(line) { + if (line.length <= this.maxLineLength) return line; + this.#discarded += line.length - this.maxLineLength; + return line.slice(0, this.maxLineLength); + } + + /** + * The line held back because no newline ever followed it, released because the + * stream ended and none ever will. + * + * @returns {string[]} + */ + flush() { + const held = this.partial; + this.partial = ''; + this.discarding = false; + // The stream ended mid-discard, so the newline that would have settled what + // was cut is not coming. + this.#settle(); + return held.trim() ? [held] : []; + } +} + +module.exports = LogFrameDecoder; +module.exports.MAX_LINE_LENGTH = MAX_LINE_LENGTH; diff --git a/ZelBack/src/services/utils/mountParser.js b/ZelBack/src/services/utils/mountParser.js index 6770e26362..246653b768 100644 --- a/ZelBack/src/services/utils/mountParser.js +++ b/ZelBack/src/services/utils/mountParser.js @@ -17,6 +17,7 @@ */ const log = require('../../lib/log'); +const { isReservedName } = require('../appSystem/volumeReservedNames'); /** * Mount type enumeration @@ -105,6 +106,17 @@ function validateSubdirOrFilename(name) { throw new Error(`Subdirectory/filename cannot be a reserved name: ${reserved.join(', ')}`); } + // The volume root also holds entries that are not the owner's, and a mount is + // the one way an app reaches them: `.stignore` decides what leaves the node, + // `.stfolder` is how syncthing knows the folder is mounted at all, and an + // operation's staging is deleted underneath whoever holds it by the boot + // sweep. Refused through the same predicate the file browser refuses them by, + // so the two doors into that root cannot drift apart - which also means a name + // that merely resembles a staging directory stays the owner's to use. + if (isReservedName(name)) { + throw new Error(`Subdirectory/filename cannot be a reserved name: ${name}`); + } + // Check for special characters that might cause issues const invalidChars = /[<>:"|?*\0]/; if (invalidChars.test(name)) { diff --git a/ZelBack/src/services/utils/networkClassifier.js b/ZelBack/src/services/utils/networkClassifier.js new file mode 100644 index 0000000000..bb296a5915 --- /dev/null +++ b/ZelBack/src/services/utils/networkClassifier.js @@ -0,0 +1,243 @@ +// Is this node's address on an access (consumer) network, or in a data centre? +// +// The question `isDataCenter()` answers today is "did anything tell me this was +// hosting", so its false branch means "nothing told me", and absence of evidence +// reads as residential. Enforcing against that is what this module exists to +// avoid: here a node is RESIDENTIAL only when something positively says so and +// nothing contradicts it, and every other outcome is a state enforcement must +// leave alone. +// +// The four outcomes: +// RESIDENTIAL at least one positive signal, no contradiction +// DATACENTER at least one contradiction, no positive signal +// CONFLICTED both - the signals disagree, so we do not know +// UNKNOWN neither - nothing to go on +// +// CONFLICTED is not a rounding error on the way to a binary. Run over the 1,569 +// fleet hosts ip-api positively asserts are hosting, across all 29 hosting ASNs +// the fleet uses, this rule puts 1,567 in DATACENTER and 2 in CONFLICTED - +// 213.44.137.57 in Bouygues' consumer space and 212.83.170.245 on Online/ +// Scaleway, each carrying an access-network PTR over a hosting flag. Collapsing +// that bucket either way is what would lose them. +// +// It calls none of those 1,569 residential, and that is NOT a measured accuracy: +// `hosting` is itself a contradiction, so on this population RESIDENTIAL is +// unreachable by construction, whatever the other signals say. Quoting it as a +// false-positive rate - as this header did - measures the arithmetic rather than +// the rule. +// +// The rate that means anything is the one at the point of enforcement, and this +// module is not the authority there. It supplies evidence; +// geolocationService.getNetworkClassification lets the published table decide +// and uses evidenceAgainst only to DECLINE a published RESIDENTIAL. Of those +// same 1,569, exactly one carries a published residential verdict - +// 213.44.137.57 - and the veto covers it, so none is enforced against. One +// ledger-level error in 1,569, zero enforced. Measured against the published +// artifact read through this branch's own ipLocationStore, over every fleet host +// carrying an ip-api record. +// +// Signals are limited to what a node can determine about itself: the PTR record +// for its own address, the ip-api response geolocationService already fetches, +// and bench figures it already holds. Registration (RDAP) data separates these +// populations better still, but six thousand nodes cannot each query the RIRs - +// that signal belongs in the published location artifact, read as a table. + +// Access-network vocabulary. Generic across ISPs worldwide, which is the +// property that makes it worth more than a list of provider names: it fires on +// Optus, Charter, Vodafone and Slovak Telekom without any of them being listed. +const PTR_RESIDENTIAL = [ + 'dsl', 'ppp', 'dial', 'dyn', 'pool', 'dhcp', 'cpe', 'cust', 'client', + 'subscriber', 'subs.', 'user', 'home', 'broadband', 'bband', 'cable', + 'docsis', 'hsd', 'fios', 'lightspeed', 'bras', 'gpon', 'ftth', 'fibre', + 'abo.', 'wanadoo', 'hispeed', 'optusnet', 'res.', 'resnet', 'retail', + 'access', 'mobile', 'lte', 'wireless', 'wifi', 'ipoe', 'rev.', 'fixed.', +]; + +// Hosting vocabulary. Only ever read as a contradiction, never as evidence of +// anything by itself. +const PTR_DATACENTER = [ + 'vps', 'vmi', 'srv', 'server', 'dedi', 'cloud', 'hosted', 'hosting', + 'colo', 'datacenter', 'datacentre', 'instance', 'compute', 'baremetal', + 'your-server', 'contabo', 'ovh', 'hetzner', 'linode', 'vultr', + 'digitalocean', 'amazonaws', 'azure', 'leaseweb', 'infomaniak', 'static.tds', +]; + +// Operators known to sell hosting. Matched against ip-api's `isp` and `as` - +// the network operator - and deliberately NOT against `org`, which is the block +// registrant and is frequently a reseller or downstream customer. The two +// disagree on 67% of fleet hosts: 46.250.240.89 carries isp "Contabo Asia +// Private Limited" and org "Yorkshire Tech Limited", and reading org is why +// Contabo appears on this list today and is still classified residential. +const HOSTING_OPERATORS = [ + 'hetzner', 'ovh', 'netcup', 'hostnodes', 'contabo', 'hostslim', 'zayo', + 'cogent', 'lumen', 'digitalocean', 'linode', 'vultr', 'leaseweb', 'scaleway', + 'infomaniak', 'oracle', 'amazon', 'google', 'microsoft', 'azure', 'alibaba', + 'ionos', 'aruba', 'hostinger', 'namecheap', 'godaddy', 'upcloud', +]; + +const CLASSIFICATION = Object.freeze({ + RESIDENTIAL: 'RESIDENTIAL', + DATACENTER: 'DATACENTER', + CONFLICTED: 'CONFLICTED', + UNKNOWN: 'UNKNOWN', +}); + +// An access link is typically far faster down than up. A symmetric link proves +// nothing either way - FTTH in France and Sweden is ordinary consumer service - +// so this is only ever corroboration, never the sole reason for a verdict. +const ASYMMETRY_RATIO = 0.5; + +/** + * Which vocabularies a PTR record carries. + * @param {string} hostname Reverse DNS name, or empty when there is none. + * @returns {('residential'|'datacenter'|'both'|'neither'|'none')} + */ +function classifyPtr(hostname) { + if (!hostname || typeof hostname !== 'string') return 'none'; + const lower = hostname.toLowerCase(); + const residential = PTR_RESIDENTIAL.some((token) => lower.includes(token)); + const datacenter = PTR_DATACENTER.some((token) => lower.includes(token)); + if (residential && datacenter) return 'both'; + if (residential) return 'residential'; + if (datacenter) return 'datacenter'; + return 'neither'; +} + +/** + * True when the operator string names a company that sells hosting. + * @param {string} isp ip-api `isp`. + * @param {string} asn ip-api `as`, e.g. "AS24940 Hetzner Online GmbH". + * @returns {boolean} + */ +function isHostingOperator(isp, asn) { + const haystack = `${isp || ''} ${asn || ''}`.toLowerCase(); + return HOSTING_OPERATORS.some((operator) => haystack.includes(operator)); +} + +/** + * Classify a node's own network from the facts it holds about itself. + * + * Every input is optional: a node that could not resolve its PTR, or whose + * bench figures are missing, still gets an answer from whatever remains - it + * just gets a less decided one. Passing nothing yields UNKNOWN, which is the + * correct answer to "I know nothing about this address". + * + * @param {object} facts + * @param {string} [facts.ptr] Reverse DNS for the node's own address. + * @param {boolean} [facts.hosting] ip-api `hosting`. + * @param {boolean} [facts.proxy] ip-api `proxy`. + * @param {boolean} [facts.mobile] ip-api `mobile`. + * @param {string} [facts.isp] ip-api `isp` - the operator. + * @param {string} [facts.asn] ip-api `as` - the operator's AS. + * @param {number} [facts.uploadSpeed] Bench upload, Mbps. + * @param {number} [facts.downloadSpeed] Bench download, Mbps. + * @returns {{classification: string, evidenceFor: string[], evidenceAgainst: string[]}} + */ +function classifyNetwork(facts = {}) { + const { + ptr, hosting, proxy, mobile, isp, asn, uploadSpeed, downloadSpeed, + } = facts; + + const evidenceFor = []; + const evidenceAgainst = []; + + // Whether the signals that can CONTRADICT a residential reading were gathered + // at all. RESIDENTIAL means "something says residential and nothing says + // otherwise", and the second half is only worth anything if the question was + // asked. It is not always asked: when ip-api answers 200 with an unusable + // body, geolocationService falls back to stats.runonflux.io, which carries + // none of these - it never requests hosting, proxy, mobile or `as` from + // ip-api in the first place, and its /fluxlocation endpoint projects away + // everything but location and `org`. On that path all five arrive undefined, + // an empty evidenceAgainst means nobody looked rather than nothing was found, + // and a datacentre host reads as enforceably RESIDENTIAL. + // + // Inferred from the inputs rather than passed as a flag: a flag can be + // forgotten by a new caller and would default to whichever answer is + // convenient, and this survives a geolocation restored from the database, + // which persists these fields. + const contradictionSignalsGathered = hosting !== undefined || proxy !== undefined + || mobile !== undefined || isp !== undefined || asn !== undefined; + + // Evidence that this IS hosting - a narrower claim than "not a home line". + // Everything in evidenceAgainst contradicts a residential reading; only these + // say anything positive about a data centre, and DATACENTER is reached from + // this list rather than from the absence of residential evidence. + // + // `proxy` is the signal that belongs in one and not the other. It is a VPN + // artefact - this branch argues exactly that where static IP is concerned, + // and deleted a rule that granted static on it - so an address behind a VPN + // exit says nothing about the machine. Granting DATACENTER on it would be the + // same category error one function away, and it points the wrong way: an + // owner paying for `datacenter: true` is buying "not someone's house", and a + // home machine on a VPN carries this signature and nothing else. Measured on + // the fleet of 2026-08-18: 8 of 2,432 hosts reached DATACENTER on proxy alone. + const hostingEvidence = []; + + const ptrClass = classifyPtr(ptr); + if (ptrClass === 'residential') evidenceFor.push(`ptr access-network: ${ptr}`); + // 'both' is a contradiction on its own: a name carrying hosting vocabulary is + // not cleared by also carrying access vocabulary. + if (ptrClass === 'datacenter' || ptrClass === 'both') { + evidenceAgainst.push(`ptr hosting: ${ptr}`); + hostingEvidence.push('ptr'); + } + + if (mobile === true) evidenceFor.push('ip-api mobile'); + if (hosting === true) { + evidenceAgainst.push('ip-api hosting'); + hostingEvidence.push('ip-api hosting'); + } + if (proxy === true) evidenceAgainst.push('ip-api proxy'); + + if (isHostingOperator(isp, asn)) { + evidenceAgainst.push(`operator sells hosting: ${isp || asn}`); + hostingEvidence.push('operator'); + } + + // Corroboration only, and deliberately not counted as evidence. A bench figure + // is a speed test's result, not a property of the link: on its own it would + // call any node with a lopsided measurement residential, and enforcement would + // act on an instrument reading. It can support a verdict the real signals + // already reached; it can never reach one. + const corroborating = []; + if (uploadSpeed > 0 && downloadSpeed > 0 && uploadSpeed / downloadSpeed < ASYMMETRY_RATIO) { + corroborating.push(`asymmetric link ${Math.round(uploadSpeed)}/${Math.round(downloadSpeed)}`); + } + + let classification = CLASSIFICATION.UNKNOWN; + if (evidenceFor.length && !evidenceAgainst.length) { + // Only reachable when the contradicting signals were actually consulted. + // Without them this is UNKNOWN, which never enforces - the same answer the + // module already gives to every other question it cannot settle. + classification = contradictionSignalsGathered + ? CLASSIFICATION.RESIDENTIAL + : CLASSIFICATION.UNKNOWN; + if (contradictionSignalsGathered) evidenceFor.push(...corroborating); + } else if (evidenceAgainst.length && !evidenceFor.length) { + // DATACENTER stands either way where it stands at all: it rests on + // something found, not on something absent, and it enforces nothing. But it + // rests on POSITIVE hosting evidence specifically - a contradiction that + // only rules out a home line leaves this UNKNOWN, which confers nothing and + // enforces nothing, rather than promoting "not residential" to "hosting". + classification = hostingEvidence.length + ? CLASSIFICATION.DATACENTER + : CLASSIFICATION.UNKNOWN; + } else if (evidenceFor.length && evidenceAgainst.length) { + classification = CLASSIFICATION.CONFLICTED; + } + + return { + classification, evidenceFor, evidenceAgainst, contradictionSignalsGathered, + }; +} + +module.exports = { + CLASSIFICATION, + classifyNetwork, + classifyPtr, + isHostingOperator, + PTR_RESIDENTIAL, + PTR_DATACENTER, + HOSTING_OPERATORS, +}; diff --git a/ZelBack/src/services/utils/networkStateManager.js b/ZelBack/src/services/utils/networkStateManager.js index dcaf71e3b7..594f893100 100644 --- a/ZelBack/src/services/utils/networkStateManager.js +++ b/ZelBack/src/services/utils/networkStateManager.js @@ -1,6 +1,6 @@ const { EventEmitter } = require('node:events'); const { FluxController } = require('./fluxController'); -const { normalizeSocketAddress } = require('./socketAddressUtils'); +const { normalizeSocketAddress, ipsMatch } = require('./socketAddressUtils'); const log = require('../../lib/log'); @@ -47,7 +47,7 @@ class NetworkStateManager extends EventEmitter { #lastFetchTime = BigInt(0); /** - * @type {() => Promise | null} + * @type {(() => Promise) | null} */ #onStartComplete = null; @@ -66,6 +66,28 @@ class NetworkStateManager extends EventEmitter { }; }); + /** + * Whether a fetch has completed, so that what the indexes hold is what this + * node knows about the fleet - as distinct from having populated them, which + * an empty fleet never does. + */ + #answerable = false; + + /** + * @type {(() => void) | null} + */ + #onAnswerable = null; + + /** + * @type {Promise} + */ + #answerableWait = new Promise((resolve) => { + this.#onAnswerable = () => { + resolve(); + this.#onAnswerable = () => {}; + }; + }); + /** * @type { "polling" | "subscription" } */ @@ -175,6 +197,48 @@ class NetworkStateManager extends EventEmitter { return this.#controller.lock.waitReady(); } + /** + * Resolves once a lookup can be answered truthfully. + * + * A node list has three conditions, not two. Never fetched: the node cannot + * answer, and the indexing lock is free at that point, so waiting on that + * alone reads an empty index and reports every node absent - the answer a + * node gives about a peer it has simply not heard of yet. Fetched and empty: + * it can answer, and "absent" is the truth. Fetched and populated: it answers + * from the index. Only the first of those waits. + * + * The indexing wait then still earns its place: mid-rebuild it costs ~10ms + * and returns the newer state. + * @returns {Promise} + */ + async #waitAnswerable() { + if (!this.#answerable) await this.#answerableWait; + + await this.waitIndexesReady; + } + + /** + * Marks the node able to answer questions about the fleet: a fetch has come + * back and the indexes hold what it returned. + * @returns {void} + */ + #markAnswerable() { + this.#answerable = true; + + if (this.#onAnswerable) this.#onAnswerable(); + } + + /** + * Releases anything waiting to be able to answer. Called when the manager + * stops, where no fetch is coming and a waiter would never return. + * @returns {void} + */ + #releaseWaiters() { + if (this.#onStartComplete) this.#onStartComplete(); + + this.#markAnswerable(); + } + get nodeCount() { return this.#state.length; } @@ -264,45 +328,96 @@ class NetworkStateManager extends EventEmitter { } /** - * Gets a random node from the network state. Ensures that the connection is - * not to this node. When we build the indexes, we could also store the node - * keys in an array, however, that is another array we have to keep in memory. - * It may pay to do that though, as this is O(n), vs O(1) for array index. CPU - * tradeoff for memory is probably good though. - * @param {string} localSocketAddress The ip:port of this node + * Walks the socketAddress index from a random offset and answers with the + * first node the rule accepts, or null when none does. + * + * One walk, because the two questions callers ask differ only in what they + * exclude: "another node", and "a node that can see me from outside". When we + * build the indexes we could also store the keys in an array, but that is + * another array to keep in memory - O(n) here against O(1) for an array index + * is probably the right trade. + * + * It answers null rather than reaching for a neighbouring entry when nothing + * qualifies. The previous form took "the one before, or else the next" without + * checking either existed, so a single-node fleet threw where it should have + * said absent - and absent is a case every caller already handles, because an + * empty index has always returned it. + * + * @param {(socketAddress: string) => boolean} acceptable * @returns {Promise} A random socketAddress from the map */ - async getRandomSocketAddress(localSocketAddress) { - await this.waitIndexesReady; + async #randomSocketAddressWhere(acceptable) { + await this.#waitAnswerable(); const indexSize = this.#socketAddressIndex.size; if (!indexSize) return null; - let stepsRemaining = Math.floor(Math.random() * indexSize); - const iterator = this.#socketAddressIndex.values(); + const offset = Math.floor(Math.random() * indexSize); - let previous = null; + let position = 0; + let firstAcceptable = null; // eslint-disable-next-line no-restricted-syntax - for (const node of iterator) { + for (const node of this.#socketAddressIndex.values()) { const { ip: socketAddress } = node; - if (!stepsRemaining) { - const match = localSocketAddress === socketAddress; - // if we've been unlucky (or lucky however you look at it) enough to hit - // this node, we just take the value before, or if it's the initial index, - // the next value from the iterator - if (match) return previous || iterator.next().value.ip; - return socketAddress; + if (acceptable(socketAddress)) { + if (position >= offset) return socketAddress; + if (firstAcceptable === null) firstAcceptable = socketAddress; } - previous = socketAddress; - stepsRemaining -= 1; + position += 1; } - // this should never happen, should probably log it - return this.socketAddressIndex.values().next().value.ip; + // Everything acceptable sat before the offset, so wrap to the first of them. + return firstAcceptable; + } + + /** + * A random node that is not this one. + * + * For talking to a peer or fetching from one, where a Flux node behind our own + * router is a perfectly good answer. + * + * @param {string} localSocketAddress The ip:port of this node + * @returns {Promise} A random socketAddress from the map + */ + async getRandomSocketAddress(localSocketAddress) { + return this.#randomSocketAddressWhere( + (socketAddress) => socketAddress !== localSocketAddress, + ); + } + + /** + * A random node that can observe this one from OUTSIDE its address. + * + * For asking a peer what our address looks like from where it stands. A node + * sharing our public address is not outside it - its packets never leave the + * router - so whatever it can or cannot reach says nothing about what the + * internet can reach, which is the only question being asked. That holds + * however the router behaves; it is not a claim about hairpinning. + * Excluded here rather than at each caller, because the callers that need it + * are not the only ones drawing a peer and one of them already had to write + * the check by hand. + * + * Answers null when every other node shares our address, which is the honest + * answer: there is nobody who could tell us. The caller decides what that + * means; for the port test it means nothing was learned. + * + * `exclude` is how a caller asks for ANOTHER observer rather than another + * draw: a redraw that can return the peer just asked is not a second opinion, + * and a caller counting distinct witnesses would never reach two. + * + * @param {string} localSocketAddress The ip:port of this node + * @param {{exclude?: Array}} [options] Addresses already asked + * @returns {Promise} A random socketAddress from the map + */ + async getRandomExternalObserver(localSocketAddress, { exclude = [] } = {}) { + return this.#randomSocketAddressWhere( + (socketAddress) => !ipsMatch(socketAddress, localSocketAddress) + && !exclude.some((asked) => ipsMatch(socketAddress, asked)), + ); } /** @@ -331,6 +446,19 @@ class NetworkStateManager extends EventEmitter { this.#pubkeyIndex = new Map(); this.#socketAddressIndex = new Map(); this.#state = []; + // Back to un-started, which is the whole point of this method: the indexes + // above are empty again, so the manager must not go on saying it can answer + // from them. stop() releases anyone already waiting before it gets here, so + // rewinding cannot strand them - it only means the next start() has to + // fetch before anyone is answered, exactly as a freshly built one does. + this.#answerable = false; + this.#answerableWait = new Promise((resolve) => { + this.#onAnswerable = () => { + resolve(); + this.#onAnswerable = () => {}; + }; + }); + this.#started = false; } /** @@ -376,8 +504,14 @@ class NetworkStateManager extends EventEmitter { const blockMsg = blockHeight ? `. Block height: ${blockHeight}` : ''; log.info(elapsedMsg + blockMsg); - // eslint-disable-next-line no-await-in-loop - if (!state.length) await this.#controller.sleep(15_000); + if (!state.length) { + // An empty list is an answer, so lookups stop waiting here even though + // the loop keeps asking - it retries for a fleet, not for the ability + // to say there isn't one. + this.#markAnswerable(); + // eslint-disable-next-line no-await-in-loop + await this.#controller.sleep(15_000); + } } while (!populated && !state.length); if (state.length) { @@ -403,6 +537,10 @@ class NetworkStateManager extends EventEmitter { `pubkeyIndexSize: ${pubkeySize}, socketAddressSize: ${socketAddressSize}`, ); + // after the build, never before it: between the fetch returning and the + // indexes being swapped in, the index a waiter would read is still empty + this.#markAnswerable(); + if (!populated) { this.emit('populated'); if (this.#onStartComplete) this.#onStartComplete(); @@ -469,6 +607,14 @@ class NetworkStateManager extends EventEmitter { await this.fetchNetworkState(); await this.waitStarted; + // Only a manager that got its list runs a loop to keep it fresh. A stop + // landing during that first fetch breaks the loop without populating and + // then releases everything waiting - this included - so without this the + // updater is armed on a manager that has just been torn down. The abort + // flag cannot be read for it: abort() installs a fresh AbortController on + // its way out, so by here it may already say it was never aborted. + if (!this.#started) return; + const updater = this.#stateEmitter && this.stateEvent ? this.#startEventEmitter : this.#startPolling; @@ -479,6 +625,8 @@ class NetworkStateManager extends EventEmitter { async stop() { await this.#controller.abort(); + this.#releaseWaiters(); + if (this.#stateEmitter && this.#boundEventHandler) { this.#stateEmitter.removeListener(this.stateEvent, this.#boundEventHandler); if (this.progressEvent) { @@ -504,9 +652,7 @@ class NetworkStateManager extends EventEmitter { if (!Object.keys(this.#indexes).includes(type)) return null; - // if we are mid stroke indexing, may as well wait the ~10ms and get the - // latest block - await this.waitIndexesReady; + await this.#waitAnswerable(); const key = type === 'socketAddress' ? normalizeSocketAddress(filter) : filter; const cached = this.#indexes[type].get(key); @@ -526,9 +672,7 @@ class NetworkStateManager extends EventEmitter { if (!filter) return false; if (!Object.keys(this.#indexes).includes(type)) return false; - // if we are mid stroke indexing, may as well wait the 10ms (max) and get the - // latest block - await this.waitIndexesReady; + await this.#waitAnswerable(); const key = type === 'socketAddress' ? normalizeSocketAddress(filter) : filter; const found = this.#indexes[type].has(key); diff --git a/ZelBack/src/services/utils/nodeSigner.js b/ZelBack/src/services/utils/nodeSigner.js new file mode 100644 index 0000000000..499166e4d4 --- /dev/null +++ b/ZelBack/src/services/utils/nodeSigner.js @@ -0,0 +1,45 @@ +const fluxNetworkHelper = require('../fluxNetworkHelper'); +const verificationHelper = require('../verificationHelper'); +const log = require('../../lib/log'); + +/** + * This node's identity, ready to sign - or nothing, when it has none to hand. + * + * The one place that asks whether this node can speak as itself. It used to be + * asked at every call site or, more often, not asked at all: the key accessors + * answered a failure with a value, so `pubKey` could be an Error object that is + * truthy, is not a string, and becomes `{}` the moment it is stringified. Nine + * callers took the public key and two of them checked it. A message carrying + * `"pubKey":{}` is not refused by this node - it is refused by every node that + * receives it, which is a long way from where the key went missing. + * + * Answers rather than throws, because the callers are spread across paths with + * very different tolerances - one of them must never throw at all - and a + * primitive is in no position to know which it is being used from. + * + * The signature can still fail on its own after this succeeds, so `sign` + * answers null and its callers say so; a key that exists is not the same as a + * signing operation that worked. + * + * @param {string} [privatekey] - an explicit key, otherwise the daemon config's + * @returns {Promise<{pubKey: string, sign: (message: string) => string|null}|null>} + */ +async function nodeSigner(privatekey) { + const pubKey = await fluxNetworkHelper.getFluxNodePublicKey(privatekey); + const privKey = await fluxNetworkHelper.getFluxNodePrivateKey(privatekey); + + if (!pubKey || typeof pubKey !== 'string' || !privKey || typeof privKey !== 'string') { + log.warn('nodeSigner - this node cannot sign as itself; its key is unavailable'); + return null; + } + + return { + pubKey, + sign: (message) => { + const signature = verificationHelper.signMessage(message, privKey); + return typeof signature === 'string' && signature ? signature : null; + }, + }; +} + +module.exports = { nodeSigner }; diff --git a/ZelBack/src/services/utils/pathSecurity.js b/ZelBack/src/services/utils/pathSecurity.js index 564490cf0a..4bd6a52566 100644 --- a/ZelBack/src/services/utils/pathSecurity.js +++ b/ZelBack/src/services/utils/pathSecurity.js @@ -29,15 +29,37 @@ function rejectBackslashes(inputPath) { } /** - * Validate a single path component (directory or filename) against allowlist. - * This is the STRICT mode validation - only allows known-safe characters. + * Characters that cannot appear in a path component this code will handle. * - * Allowed characters: - * - Alphanumeric (a-z, A-Z, 0-9) - * - Dash (-), underscore (_) - * - Dot (.) - but not as sole character or double dots - * - Space ( ) - common in filenames - * - Additional safe chars: @, #, +, =, (), [], {} + * A path separator, because components are what a path splits INTO; a backslash, + * which on Linux is a legal filename character but only ever appears here by + * mistake or by attempt; and the control characters. + * + * Control characters are the interesting one. They are legal in a Linux + * filename, but a newline in particular corrupts anything line-oriented that + * later handles the name - /proc/self/mountinfo escapes them for exactly this + * reason, and a name reaches a log line, a mount table and a container's own + * output before anyone reads it. Rejecting them keeps a filename from being + * able to forge a record about itself. + */ +// eslint-disable-next-line no-control-regex +const UNSAFE_PATH_COMPONENT = /[\u0000-\u001F\u007F-\u009F\\/]/; + +/** + * Validate a single path component (directory or filename). + * + * This rejects what cannot be handled rather than permitting a known-safe list. + * It previously allowed only `[a-zA-Z0-9_\-. @#+=()[\]{}]`, which excludes the + * comma, the apostrophe, the ampersand and every non-ASCII character - so + * `café.jpg`, `Mary's photo.png` and `report,final.pdf` could be UPLOADED (the + * upload path applies no character rule) and then never renamed, moved, + * downloaded or deleted. The system accepted names it could not address. + * + * Widening is safe because nothing in the containment argument rests on the + * character set: traversal is caught by the `..` component check and by + * resolving against the base, symlink escape by verifyRealPath, and anything + * that slips past both lands inside a container with only that volume mounted. + * An allowlist of punctuation was never what made a path safe. * * @param {string} component - Single path component (no slashes) * @returns {boolean} True if component is safe @@ -57,16 +79,9 @@ function isValidPathComponent(component) { return false; } - // Allowlist pattern: alphanumeric, dash, underscore, dot, space, and common safe chars - // Note: We allow consecutive dots in FILENAMES (e.g., "file..backup.txt") since they're not traversal - // The traversal check is done by checking if component === '..' - const safePattern = /^[a-zA-Z0-9_\-. @#+=()[\]{}]+$/; - - if (!safePattern.test(component)) { - return false; - } - - return true; + // Consecutive dots WITHIN a name ("file..backup.txt") are not traversal; only + // the component being exactly '..' is, and that is checked above. + return !UNSAFE_PATH_COMPONENT.test(component); } /** @@ -290,8 +305,12 @@ async function verifyRealPath(targetPath, basePath) { async function verifyRealPathOfExistingPath(targetPath, basePath) { const normalizedBase = path.resolve(basePath); let currentPath = path.resolve(targetPath); + let ancestorStats = null; - // Walk up until we find an existing ancestor (or reach the base) + // Walk up until we find an existing ancestor (or reach the base). Bounded by + // the walk itself: every iteration either breaks or moves one level towards + // the base, and reaching the base breaks. + // eslint-disable-next-line no-constant-condition while (true) { const relativePath = path.relative(normalizedBase, currentPath); if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) { @@ -299,7 +318,7 @@ async function verifyRealPathOfExistingPath(targetPath, basePath) { } try { - await fs.promises.lstat(currentPath); + ancestorStats = await fs.promises.lstat(currentPath); break; } catch (error) { if (error.code !== 'ENOENT') { @@ -318,6 +337,21 @@ async function verifyRealPathOfExistingPath(targetPath, basePath) { } } + // A symlink in the path whose target does not resolve on the host is + // unverifiable, and unverifiable is not safe. The checks run in the host + // namespace, but the operation runs in the container where /work IS this + // volume - so a link the host sees as dangling (ln -s /work ..., or a + // relative climb to it) can resolve to the volume root in the container and + // reach a reserved name the guard would otherwise refuse. lstat succeeds on a + // dangling link and realpath then fails with ENOENT, which the ENOENT branch + // below would pass through as "cannot resolve, therefore safe". Refuse it. + if (ancestorStats && ancestorStats.isSymbolicLink()) { + const resolvedTarget = await fs.promises.realpath(currentPath).catch(() => null); + if (resolvedTarget === null) { + throw new Error('Invalid path: a symlink in the path does not resolve on the host'); + } + } + return verifyRealPath(currentPath, basePath); } @@ -375,21 +409,27 @@ function validateFilename(name) { throw new Error('Filename must be a non-empty string'); } - // Check for null bytes + // Named separately from the control-character rule that would also catch it: + // a null byte in a filename is a specific, well-known attempt to truncate a + // path in a downstream C string, and saying so is more use than a general + // message. if (name.includes('\0')) { throw new Error('Invalid filename: null bytes not allowed'); } - // Check for path separators (both / and \) - if (name.includes('/') || name.includes('\\')) { - throw new Error('Invalid filename: path separators not allowed'); - } - // Check for reserved traversal names if (name === '.' || name === '..') { throw new Error('Invalid filename: reserved name'); } + // The same rule the rest of the module applies to a path component. Upload + // used to have its own, looser one - no character check at all - which is how + // a name could be accepted here and then be unaddressable by every other + // endpoint. One rule, so what can be created can be managed. + if (!isValidPathComponent(name)) { + throw new Error('Invalid filename: path separators and control characters are not allowed'); + } + return name; } @@ -412,7 +452,80 @@ async function sanitizeAndVerifyPath(userPath, basePath, options = {}) { return verifyRealPath(sanitizedPath, basePath); } +/** + * Open a regular file without following a link at its final component, and + * without waiting for one. + * + * The one way to read a path an application owns. A checked name is only ever a + * claim about the moment it was checked - the owner keeps running and can + * replace what it refers to, and this process is root, so following a link + * there reads a file the owner could not open themselves. Everything after this + * is decided from the descriptor rather than from the name. + * + * O_NONBLOCK because opening a FIFO for reading WAITS for a writer, and the + * application owns the directory. A named pipe left where a file is expected + * blocks the open for as long as the pipe exists - and the boot sweep awaits + * its read, so one pipe planted in one app's own volume stops everything after + * it in startup: the app network reclaim, syncthing, the PGP identity. It + * survives the reboot, because the pipe is still there. The flag does nothing + * to a regular file. + * + * The type is then checked from the DESCRIPTOR, which is the same reason + * everything else here is. With the flag above a pipe or a device opens rather + * than hanging, so a caller would otherwise go on to read one as if it were a + * file. Callers get a regular file or an error, and never have to ask. + * + * The stats are handed back with the handle because they were taken to make + * that check, and every caller needs the size. Asking again would be a second + * answer to a settled question - and a later one: the application is writing to + * this file throughout, so a size measured a turn afterwards can already be + * larger than the one a download has announced. + * + * The caller closes the handle. Swapping a PARENT directory stays expressible + * and is what the containment checks in this module cover; this closes the half + * of it that a check cannot. + * @param {string} filePath - already checked for containment + * @returns {Promise<{handle: import('node:fs/promises').FileHandle, + * stats: import('node:fs').Stats}>} An open handle and what it is. + * @throws {Error} if the path is not a regular file + */ +async function openNoFollow(filePath) { + let handle; + try { + handle = await fs.promises.open( + filePath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK, + ); + } catch (error) { + // A symlink at the final component fails with ELOOP because O_NOFOLLOW + // refuses to follow it - the download reads on the host, not in the + // container, so following a link could hand back a file outside the volume. + // Answer the app owner with what to do instead of an opaque errno: a link + // an app legitimately keeps (latest.log -> dated.log) is served by naming + // its target, or by compressing the folder - which stores the link AND the + // real file - and downloading that archive. + if (error.code === 'ELOOP') { + throw new Error('A symbolic link cannot be downloaded directly; download the file it points to, or compress the folder and download the archive'); + } + throw error; + } + + try { + const stats = await handle.stat(); + if (!stats.isFile()) { + // No path in the message: this reaches an API caller, and the host path + // is not theirs to learn. + throw new Error('Only a regular file can be read'); + } + return { handle, stats }; + } catch (error) { + await handle.close().catch(() => {}); + throw error; + } +} + module.exports = { + openNoFollow, sanitizePath, validateFilename, validatePathAllowlist, diff --git a/ZelBack/src/services/utils/peerCodec.js b/ZelBack/src/services/utils/peerCodec.js index 3ad214be36..8ffaae6f61 100644 --- a/ZelBack/src/services/utils/peerCodec.js +++ b/ZelBack/src/services/utils/peerCodec.js @@ -200,6 +200,12 @@ function decodePeerUpdate(buf) { // Signed format: [type:1][sinceTs:8][requestTs:8][pubkeyLen:1][pubkey:var][sigLen:1][signature:var] function encodeSignedSyncRequest(type, sinceTimestamp, requestTimestamp, pubkey, signature) { + // The decoder refuses an empty key or signature, so an unsigned request is + // refused by every node it reaches. A caller holding neither skipped a + // check, so this throws rather than answering null. + if (!pubkey || typeof pubkey !== 'string') throw new Error('peerCodec - a signed sync request needs a public key'); + if (!signature || typeof signature !== 'string') throw new Error('peerCodec - a signed sync request needs a signature'); + const pubkeyBuf = Buffer.from(pubkey, 'hex'); const sigBuf = Buffer.from(signature, 'base64'); const buf = Buffer.allocUnsafe(1 + 8 + 8 + 1 + pubkeyBuf.length + 1 + sigBuf.length); diff --git a/ZelBack/src/services/utils/privileges.js b/ZelBack/src/services/utils/privileges.js new file mode 100644 index 0000000000..c515004cfe --- /dev/null +++ b/ZelBack/src/services/utils/privileges.js @@ -0,0 +1,55 @@ +/** + * The privileges a route may require. + * + * Each names a set of identities and asks whether the caller is one of them. + * None requires two identities at once, which is why every compound reads OR. + * + * The values are the wire vocabulary and are not ours to choose: /id/checkprivilege + * answers three of them to clients, and the frontend branches on those strings in + * a separately deployed repo. The names are ours, and they are what every call + * site reads. + * + * This module holds values and requires nothing, so it can be imported at the top + * of any file - including those that reach verificationHelper through a dynamic + * require to break a cycle. + */ +const Privilege = Object.freeze({ + // Any FluxID with a valid signature and a live session. + USER: 'user', + // The operator of THIS node, from config `initial.zelid`. They administer + // hardware; nothing about a customer's application follows from that. + NODE_OPERATOR: 'admin', + // Two identities: the flux team and flux support. + FLUX_TEAM: 'fluxteam', + NODE_OPERATOR_OR_FLUX_TEAM: 'adminandfluxteam', + // The FluxID that registered the application. + APP_OWNER: 'appowner', + APP_OWNER_OR_FLUX_TEAM: 'appownerorfluxteam', +}); + +/** + * The privileges whose question is about an application rather than an identity + * alone. They are the only ones that read an app name, and the only ones that + * may be given one. + * + * An array rather than a Set, because Object.freeze does not freeze a Set: its + * contents are not own properties, so a frozen Set still accepts an add(). + */ +const APP_SCOPED = Object.freeze([Privilege.APP_OWNER, Privilege.APP_OWNER_OR_FLUX_TEAM]); + +/** + * The auth a request carries, or null if it carries none. + * + * A request with no zelidauth header is ordinary - it is every unauthenticated + * call there has ever been - and answers false at the check like any other + * refusal. One absent value rather than three, so a caller never has to tell + * undefined from null from empty. + * + * Lives here so a route reads `verifyPrivilege(Privilege.X, authOf(req))`: the + * privilege it requires, and where the identity comes from, in one line. + * @param {object} req - express request + * @returns {string|null} the zelidauth header value + */ +const authOf = (req) => req?.headers?.zelidauth || null; + +module.exports = { Privilege, APP_SCOPED, authOf }; diff --git a/ZelBack/src/services/utils/routeGuards.js b/ZelBack/src/services/utils/routeGuards.js new file mode 100644 index 0000000000..7c76db3150 --- /dev/null +++ b/ZelBack/src/services/utils/routeGuards.js @@ -0,0 +1,163 @@ +// Guards that answer a request before it reaches a handler. + +const apicache = require('apicache'); + +const messageHelper = require('../messageHelper'); +const globalState = require('./globalState'); + +// What a caller turned away during boot is told to wait. The boot chain is +// bounded by its own daemon and sync timeouts rather than by this number, so it +// is a pacing hint and not a deadline: short enough that a dashboard retry feels +// responsive, long enough that a client polling through a slow boot is not +// making a request a second. +const BOOT_RETRY_AFTER_SECONDS = 15; + +/** + * Refuse a call that would create or destroy a container before boot + * reconciliation has decided which applications this node is keeping. + * + * That decision runs behind daemon readiness, node confirmation and DB sync, so + * it lands well after the API starts answering. Until it has, a container this + * node did not know about when it booted is one reconciliation may still remove: + * it keeps an app whose location record says this node is running it, and a + * container created moments ago has no such record, because a record is written + * from the running broadcast an app only makes once it has run. + * + * The internal actors already wait for the same gate - the reconciler queues + * into bootPending rather than actuating, and crash recovery holds off. This is + * the same rule at the front door. + * + * A refusal rather than a wait: the boot chain can run to its daemon timeout, and + * holding a request open that long serves the caller worse than telling them when + * to come back. + * @param {object} req Request + * @param {object} res Response + * @param {Function} next Next handler + * @returns {*} next(), or a 503 carrying a Retry-After + */ +function requireBootSettled(req, res, next) { + if (globalState.bootContainerStateSettled) return next(); + res.setHeader('Retry-After', String(BOOT_RETRY_AFTER_SECONDS)); + const errMessage = messageHelper.createErrorMessage( + 'Node is still reconciling its applications after boot', + 'ServiceUnavailable', + 503, + ); + return res.status(503).json(errMessage); +} + +/** + * Refuse a query string on an endpoint that reads none. + * + * The cache keys on the full request URL, so anything a caller puts in the query + * string becomes part of the key whether or not the handler reads it. On an + * endpoint that reads none, that turns the cache from a bound on the work into a + * way of multiplying it: every novel parameter is a guaranteed miss that + * recomputes the answer and retains another copy of it for the cache window. + * + * This runs BEFORE the cache middleware, so a refused request never reaches the + * store. + * + * Decided on the RAW url, not on req.query, because the raw url is what the + * cache keys on and parsing does not preserve it: express resolves '?=1' to an + * empty query object, so a guard reading req.query waves it through while the + * cache still files it under a key of its own. Anything that can vary the key + * has to be answered here, whether or not it survives parsing. + * + * A trailing '?' with nothing after it carries no parameter and is allowed. + * + * Rejecting rather than ignoring is the point. A caller sending parameters an + * endpoint does not accept has made a mistake, and quietly serving them an + * answer hides it - while still costing a cache entry apiece. + * @param {object} req Request + * @param {object} res Response + * @param {Function} next Next handler + * @returns {*} next(), or a 400 + */ +function rejectQueryParameters(req, res, next) { + const url = req.originalUrl ?? req.url ?? ''; + const queryStart = url.indexOf('?'); + if (queryStart === -1 || queryStart === url.length - 1) return next(); + const errMessage = messageHelper.createErrorMessage( + 'This endpoint takes no query parameters', + 'BadRequest', + 400, + ); + return res.status(400).json(errMessage); +} + +/** + * Hand a route handler's promise to express. + * + * Registering `(req, res) => handler(req, res)` drops it, so a rejection is + * unhandled: node raises it to the uncaughtException handler in apiServer, + * which exits the process. The caller gets no response at all, and the node + * restarts - once per request. Routed through here a rejection reaches + * express's error handler and answers 500, which is what a caller can act on + * and what leaves the node serving everyone else. + * @param {Function} handler Route handler taking (req, res) + * @returns {Function} express handler + */ +function asyncRoute(handler) { + return (req, res, next) => Promise.resolve(handler(req, res)).catch(next); +} + +// A cached response is served to everyone who asks next, so the store must hold +// only answers this node would give again. Express's own failures are kept out +// by status: an allowlist rather than a blocklist, so a status nobody predicted +// is excluded rather than stored until someone notices, and a route that ever +// wants a 201 or a redirect cached names it here deliberately. +apicache.options({ statusCodes: { include: [200], exclude: [] } }); + +/** + * Whether the answer just given is one worth remembering. + * + * apicache evaluates this when the response ends - and on a hit, before the + * handler runs, where nothing has been recorded and the stored answer is served. + * @param {object} _req Request, unused + * @param {object} res Response + * @returns {boolean} false if the handler reported a failure + */ +function answeredWithoutFailure(_req, res) { + return res.locals.payloadStatus !== 'error'; +} + +/** + * apicache's middleware, refusing to remember an answer that reported a failure. + * + * The status rule above cannot see these. A handler in this tree reports an + * error by answering 200 and putting `status: 'error'` in the body, so as far as + * express and apicache are concerned it succeeded. /benchmark/getstoredbenchmark + * is what that costs: it answers "No stored benchmark data available" for as + * long as a booting node has not benchmarked yet, and it caches for an hour, so + * the node goes on saying it for an hour after the benchmark has landed. + * + * The payload is recorded as the handler answers it, because that is the only + * point at which it exists - by the time apicache asks, the body is a buffer it + * is accumulating. + * + * Refusing failures rather than admitting successes, which is the opposite of + * the status rule and deliberate: a cached route may answer a payload this node + * did not build - the syncthing routes proxy syncthing's own - and those carry + * no status field at all, so an allowlist would stop caching them. + * @param {string} duration apicache duration, e.g. '30 seconds' + * @returns {Function} express middleware + */ +function cache(duration) { + const remember = apicache.middleware(duration, answeredWithoutFailure); + + // Named for what the route table shows: routeWiring.test.js finds the store in + // a chain by this name when it checks that a guard runs ahead of it. + return function cache(req, res, next) { + const { json } = res; + res.json = function recordThenAnswer(payload) { + res.locals.payloadStatus = (payload && typeof payload === 'object') ? payload.status : undefined; + return json.call(this, payload); + }; + return remember(req, res, next); + }; +} + +module.exports = { + asyncRoute, cache, rejectQueryParameters, requireBootSettled, +}; diff --git a/ZelBack/src/services/utils/socketAddressUtils.js b/ZelBack/src/services/utils/socketAddressUtils.js index b7cddfe3a1..eb99857795 100644 --- a/ZelBack/src/services/utils/socketAddressUtils.js +++ b/ZelBack/src/services/utils/socketAddressUtils.js @@ -17,6 +17,26 @@ function extractIp(address) { return address.split(':')[0]; } +// Bare IP from a node-list or app-location address. IPv6 literals pass through +// whole; anything else is treated as ip[:port]. +// +// An IPv4-mapped address is the one literal that is both: it is IPv6 in form and +// IPv4 in substance, and it carries dots - so "contains a dot" cannot on its own +// mean "ip[:port]". Split on the colon and ::ffff:1.2.3.4 yields the empty +// string, which reads downstream as an address that would not parse rather than +// as the ordinary IPv4 address it is. It resolves to that address instead. +const IPV4_MAPPED = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/i; + +function bareIp(address) { + if (typeof address !== 'string' || !address) return null; + const mapped = IPV4_MAPPED.exec(address); + if (mapped) return mapped[1]; + // More than one colon is an IPv6 literal, whatever else it holds. + if (address.indexOf(':') !== address.lastIndexOf(':')) return address; + if (address.includes('.') || !address.includes(':')) return extractIp(address); + return address; +} + function extractPort(address) { if (!address) return DEFAULT_API_PORT; const parts = address.split(':'); @@ -63,6 +83,7 @@ module.exports = { DEFAULT_API_PORT, normalizeSocketAddress, extractIp, + bareIp, extractPort, parseSocketAddress, socketAddressesMatch, diff --git a/ZelBack/src/services/utils/treeSize.js b/ZelBack/src/services/utils/treeSize.js new file mode 100644 index 0000000000..1495e4179c --- /dev/null +++ b/ZelBack/src/services/utils/treeSize.js @@ -0,0 +1,107 @@ +const path = require('path'); + +/** + * How many paths are stat'ed at once. + * + * Measured, not chosen: on a fleet-spec box, 20,000 files take 1169ms one at a + * time and 179ms at 32, with nothing further to gain past ~128. The same + * six-fold is what the published work on parallelising du's stat loop reports, + * so it is a property of the problem rather than of that box. Bounded, because + * the unbounded Promise.all fan-out this replaced opened a handle per entry in + * the tree. + */ +const DEFAULT_CONCURRENCY = 32; + +/** + * The unit lstat reports allocated blocks in. Fixed, and unrelated to the + * filesystem's own block size. + */ +const BLOCK_UNIT = 512; + +/** + * Bytes under a path, following nothing. + * + * `lstat`, never `stat`. The difference is the whole point of this module: an + * app owner can write a symlink into their own volume, and a walk that follows + * one leaves the volume entirely - `escape -> /` measures the host, and + * `loop -> ..` never finishes at all. A symlink therefore contributes zero, + * which is also the honest answer for the operations this feeds: `cp -a` and + * the archivers copy the link, not what it points at. + * + * No time limit, deliberately. Every tool that reports a total scans the whole + * tree first and takes as long as that takes - rsync builds its file list, + * Explorer shows "Calculating...". A limit here was only ever guarding against + * the cycle that lstat has already made impossible, and a scan that gives up + * would leave callers to invent a meaning for a missing figure. + * + * Iterative, so a deep tree cannot overflow the stack, and batched, so a wide + * one cannot open a handle per entry. + * + * Two different questions can be asked of a tree, and which one a caller wants + * is not guessable - so it has to say. `occupied` reports what the tree costs + * the filesystem: a file occupies whole blocks, so a hundred one-byte files are + * 100 bytes by their own account and 409,600 on disk. Anything compared against + * free space needs that one, because free space is itself a count of blocks. + * The default reports what the files say, which is what a listing shows a user + * and what every file browser means by the size of a folder. + * + * Directories are counted only for `occupied`, where they hold blocks like + * anything else. Their apparent size is an implementation detail of the + * filesystem rather than an amount of anyone's data. + * + * @param {string} root - absolute path to measure + * @param {object} fsPromises - fs.promises, or a stand-in with lstat/readdir + * @param {{concurrency?: number, occupied?: boolean}} [options] + * @returns {Promise} bytes + */ +async function measureTree(root, fsPromises, options = {}) { + const { concurrency = DEFAULT_CONCURRENCY, occupied = false } = options; + + let bytes = 0; + const pending = [root]; + + while (pending.length) { + const batch = pending.splice(0, concurrency); + // A path that cannot be read is skipped rather than fatal: a tree being + // written while it is measured loses entries between the readdir and the + // lstat, and that is not a reason to refuse the whole measurement. + // eslint-disable-next-line no-await-in-loop + const entries = await Promise.all(batch.map( + (p) => fsPromises.lstat(p).then((stats) => [p, stats]).catch(() => [p, null]), + )); + + const directories = []; + for (const [entryPath, stats] of entries) { + if (!stats) continue; + if (stats.isDirectory()) directories.push(entryPath); + // A symlink is neither, and contributes nothing either way: the + // operations this feeds copy the link rather than what it points at. + if (occupied) { + if (stats.isDirectory() || stats.isFile()) bytes += stats.blocks * BLOCK_UNIT; + } else if (stats.isFile()) { + bytes += stats.size; + } + } + + if (directories.length) { + // eslint-disable-next-line no-await-in-loop + const listings = await Promise.all(directories.map( + (d) => fsPromises.readdir(d).then((names) => [d, names]).catch(() => [d, []]), + )); + for (const [directory, names] of listings) { + // One push per name: spreading a readdir's names as arguments is + // bounded by V8's argument limit, and one wide directory - a mail + // spool, a cache - is enough to reach it. + for (const name of names) pending.push(path.join(directory, name)); + } + } + } + + return bytes; +} + +module.exports = { + measureTree, + DEFAULT_CONCURRENCY, + BLOCK_UNIT, +}; diff --git a/ZelBack/src/services/utils/verifyPool.js b/ZelBack/src/services/utils/verifyPool.js index 7523b5fda0..053e8a675b 100644 --- a/ZelBack/src/services/utils/verifyPool.js +++ b/ZelBack/src/services/utils/verifyPool.js @@ -3,76 +3,269 @@ const path = require('path'); const os = require('os'); const log = require('../../lib/log'); -const WORKER_PATH = path.join(__dirname, 'verifyWorker.js'); +const DEFAULT_WORKER_PATH = path.join(__dirname, 'verifyWorker.js'); + +// A single worker's share of a batch. Oversized posts are what pin memory: the +// serialised copy of a multi-megabyte batch is not returned to the OS once the +// batch is collected, so the pool buys a bounded footprint with a few extra +// round trips. +const CHUNK_SIZE = 256; + +// Signature verification is bursty - a sync response is thousands of items, then +// nothing for hours. One worker is kept between bursts and the rest are raised +// against demand and released. The pool is created on first use rather than at +// boot, so a node that never verifies anything never pays for a worker. +const RESIDENT_WORKERS = 1; + +const IDLE_REAP_MS = 60000; + +// A chunk that keeps killing its worker is abandoned rather than retried for +// ever, and a worker that never answers must not hold its slot for ever. Both +// failures resolve the batch as unverified: a signature we could not check is +// one we do not trust, so the messages are dropped rather than accepted. +const MAX_JOB_ATTEMPTS = 3; +const JOB_TIMEOUT_MS = 60000; let slots = []; +let queue = []; +let reapTimer = null; +let workerPath = DEFAULT_WORKER_PATH; + +function maxWorkers() { + return Math.max(RESIDENT_WORKERS, os.cpus().length - 1); +} + +function busyCount() { + return slots.filter((slot) => slot.job).length; +} + +function abandonJob(job, reason) { + log.error(`Verify job abandoned after ${job.attempts} attempt(s) - ${reason}; ` + + `${job.chunk.length} signature(s) treated as unverified`); + job.resolve(new Array(job.chunk.length).fill(false)); +} + +/** + * Put a job back at the head of the queue, unless it has already failed enough + * times to look like the batch itself is the problem. + */ +function requeue(job, reason) { + if (job.attempts >= MAX_JOB_ATTEMPTS) { + abandonJob(job, reason); + return; + } + queue.unshift(job); +} + +function clearJob(slot) { + if (slot.timer) { + clearTimeout(slot.timer); + slot.timer = null; + } + const { job } = slot; + slot.job = null; + return job; +} + +/** + * Hand queued chunks to whichever workers are free. + */ +function dispatch() { + // A failed handover puts its job back and leaves the slot free, so a pass can + // end with work queued and a worker idle - and every other way back into here + // is a worker event, which cannot arrive while no worker holds anything. The + // job would sit until the next verify() call happened to re-enter. Looping is + // what settles it, and it terminates: the attempt is counted before the + // handover, so a chunk that cannot be posted at all is abandoned after + // MAX_JOB_ATTEMPTS rather than retried for ever. + let handoverFailed = false; + do { + handoverFailed = false; + for (const slot of slots) { + if (!queue.length) return; + if (slot.job || slot.retired) continue; + + const job = queue.shift(); + job.attempts += 1; + + try { + slot.worker.postMessage(job.chunk); + } catch (error) { + // The handover never happened, so the slot is still free - claiming it + // before posting would retire the slot for the life of the process. + log.error(`Verify worker could not be given a batch: ${error.message}`); + requeue(job, `could not be handed to a worker: ${error.message}`); + handoverFailed = true; + continue; + } + + slot.job = job; + slot.timer = setTimeout(() => { + const stalled = clearJob(slot); + log.error('Verify worker did not answer in time, replacing it'); + slot.retired = true; + const idx = slots.indexOf(slot); + if (idx !== -1) slots.splice(idx, 1); + slot.worker.terminate(); + if (stalled) requeue(stalled, 'worker did not answer in time'); + // eslint-disable-next-line no-use-before-define + ensureWorkers(); + dispatch(); + }, JOB_TIMEOUT_MS); + if (slot.timer.unref) slot.timer.unref(); + } + } while (handoverFailed && queue.length); +} + +/** + * Release every idle worker above the resident count. A worker holding a chunk + * is never taken - its batch would have to be verified again. + */ +function reapIdle() { + reapTimer = null; + if (queue.length) return; + + for (let i = slots.length - 1; i >= 0 && slots.length > RESIDENT_WORKERS; i--) { + const slot = slots[i]; + if (slot.job) continue; + slot.retired = true; + slots.splice(i, 1); + slot.worker.terminate(); + } +} + +// Armed from the reply path alone, which reads like an omission and is not: the +// only workers that can still be resident when a burst ends are the ones that +// replied. A worker that dies takes its own slot out of the pool in its exit +// handler, and one that is abandoned as unpostable never held a slot - so a burst +// ending in an exit or an abandon has nothing raised left behind to release. +function scheduleReap() { + if (reapTimer) clearTimeout(reapTimer); + reapTimer = setTimeout(reapIdle, IDLE_REAP_MS); + if (reapTimer.unref) reapTimer.unref(); +} + +/** + * Raise the pool to match outstanding work, never past one worker per spare core. + */ +function ensureWorkers() { + const outstanding = queue.length + busyCount(); + const desired = Math.min(maxWorkers(), Math.max(RESIDENT_WORKERS, outstanding)); + + for (let i = slots.length; i < desired; i++) { + // eslint-disable-next-line no-use-before-define + slots.push(createSlot()); + } +} function createSlot() { - const worker = new Worker(WORKER_PATH); - const pending = []; + const worker = new Worker(workerPath); + const slot = { + worker, job: null, timer: null, retired: false, + }; + worker.on('error', (err) => log.error(`Verify worker error: ${err.message}`)); + worker.on('message', (results) => { - const entry = pending.shift(); - if (entry) entry.resolve(results); + const job = clearJob(slot); + if (job) job.resolve(results); + dispatch(); + scheduleReap(); }); - worker.on('exit', (code) => { - const idx = slots.findIndex((s) => s.worker === worker); - if (idx !== -1) { - slots[idx] = createSlot(); - if (code !== 0) { - log.error(`Verify worker exited with code ${code}, respawning and resubmitting ${pending.length} batches`); - const replacement = slots[idx]; - while (pending.length) { - const entry = pending.shift(); - replacement.pending.push(entry); - replacement.worker.postMessage(entry.batch); - } - } + + worker.on('exit', () => { + const idx = slots.indexOf(slot); + if (idx !== -1) slots.splice(idx, 1); + + const job = clearJob(slot); + if (job) { + log.error(`Verify worker exited holding ${job.chunk.length} items, requeueing`); + requeue(job, 'worker exited while holding the batch'); } + + if (!slot.retired) ensureWorkers(); + dispatch(); }); - return { worker, pending }; + + return slot; } -function start(poolSize) { - const size = poolSize ?? Math.max(1, os.cpus().length - 1); +/** + * @param {number} [poolSize] Workers to create up front. + * @param {object} [options] Overrides. + * @param {string} [options.workerPath] Worker script the pool runs. + */ +function start(poolSize, options = {}) { + const { workerPath: overridePath } = options; + if (overridePath) workerPath = overridePath; if (slots.length) return; + + const size = Math.min(maxWorkers(), Math.max(RESIDENT_WORKERS, poolSize ?? RESIDENT_WORKERS)); for (let i = 0; i < size; i++) { slots.push(createSlot()); } - log.info(`Verify worker pool started: ${slots.length} workers`); + log.info(`Verify worker pool started: ${slots.length} resident, scales to ${maxWorkers()}`); } function stop() { - for (const { worker } of slots) worker.terminate(); + if (reapTimer) { + clearTimeout(reapTimer); + reapTimer = null; + } + for (const slot of slots) { + slot.retired = true; + const job = clearJob(slot); + if (job) abandonJob(job, 'pool stopped'); + slot.worker.terminate(); + } slots = []; + for (const job of queue) abandonJob(job, 'pool stopped'); + queue = []; + workerPath = DEFAULT_WORKER_PATH; } -function sendToWorker(slot, batch) { - return new Promise((resolve) => { - slot.pending.push({ batch, resolve }); - slot.worker.postMessage(batch); - }); +/** + * Pool occupancy, for visibility into how hard the crypto path is being worked. + * @returns {{workers: number, busy: number, queued: number, maxWorkers: number}} + */ +function stats() { + return { + workers: slots.length, + busy: busyCount(), + queued: queue.length, + maxWorkers: maxWorkers(), + }; } +/** + * Verify a batch of signatures off the main thread. + * @param {Array<{messageToVerify: string, pubKey: string, signature: string}>} items Items to verify. + * @returns {Promise>} One result per item, in the order given. + */ async function verify(items) { - if (!slots.length) start(); - - const n = slots.length; - const chunkSize = Math.ceil(items.length / n); - const promises = []; - for (let i = 0; i < n; i++) { - const slice = items.slice(i * chunkSize, (i + 1) * chunkSize); - if (slice.length > 0) { - promises.push(sendToWorker(slots[i], slice)); - } + if (!items.length) return []; + + const jobs = []; + for (let offset = 0; offset < items.length; offset += CHUNK_SIZE) { + const chunk = items.slice(offset, offset + CHUNK_SIZE); + const job = { chunk, attempts: 0, resolve: null }; + job.promise = new Promise((resolve) => { job.resolve = resolve; }); + jobs.push(job); } - const chunks = await Promise.all(promises); + queue.push(...jobs); + ensureWorkers(); + dispatch(); + + const chunks = await Promise.all(jobs.map((job) => job.promise)); + const results = []; for (const chunk of chunks) { - for (const r of chunk) results.push(r); + for (const result of chunk) results.push(result); } return results; } -module.exports = { start, stop, verify }; +module.exports = { + start, stop, verify, stats, +}; diff --git a/ZelBack/src/services/utils/volumeConstructor.js b/ZelBack/src/services/utils/volumeConstructor.js index 544c804ed7..b04b6a4867 100644 --- a/ZelBack/src/services/utils/volumeConstructor.js +++ b/ZelBack/src/services/utils/volumeConstructor.js @@ -8,7 +8,6 @@ const log = require('../../lib/log'); const { MountType } = require('./mountParser'); const { appsFolder } = require('./appConstants'); -const config = require('../../../config/default'); /** * Get app identifier with proper flux prefix diff --git a/ZelBack/src/services/utils/volumeService.js b/ZelBack/src/services/utils/volumeService.js index 9ee34b4e1f..73b8def48a 100644 --- a/ZelBack/src/services/utils/volumeService.js +++ b/ZelBack/src/services/utils/volumeService.js @@ -1,14 +1,71 @@ const fs = require('fs').promises; const path = require('node:path'); -const util = require('node:util'); -const df = require('node-df'); const dockerService = require('../dockerService'); +const deviceHelper = require('../deviceHelper'); const serviceHelper = require('../serviceHelper'); const mountParser = require('./mountParser'); const log = require('../../lib/log'); -const { appsFolder, appVolumesPath, legacyAppVolumesPath } = require('./appConstants'); +const { + appsFolder, appVolumesPath, legacyAppVolumesPath, APP_VOLUME_MOUNT_OPTIONS, +} = require('./appConstants'); -const dfAsync = util.promisify(df); +/** + * The unit node capacity is counted in, which is the unit it is spent in: + * `fallocate -l G` takes 1024^3 bytes per unit, so this is what an app's + * `hdd` actually costs the filesystem. + */ +const BYTES_PER_GIB = 1024 ** 3; + +/** + * The host filesystems eligible to hold an app's FLUXFSVOL image. + * + * Block-backed, and neither the root nor a boot filesystem. Loop devices are + * excluded because a loop mount IS an app volume - treating one as a candidate + * host would place an app's image inside another app's volume. + * + * Throws when the mount table cannot be read; callers narrow their search to + * the appvolumes directories rather than treating that as "no disks". + * + * @returns {Promise>} mount rows from deviceHelper + */ +async function eligibleHostMounts() { + const filesystems = await deviceHelper.listMountedFilesystems(); + return filesystems.filter((entry) => entry.source.includes('/dev/') + && !entry.source.includes('loop') + && !entry.target.includes('boot') + && entry.target !== '/'); +} + +/** + * The host volumes that count towards this node's advertised capacity, sized in + * whole GiB. + * + * A wider set than eligibleHostMounts: a loop-mounted ROOT is included, because + * on some images that is the host disk rather than an app volume. Callers that + * place a FLUXFSVOL want the narrower set; callers that total up node capacity + * want this one. + * + * GiB, because that is the unit an app's `hdd` is spent in: `createAppVolume` + * allocates with `fallocate -l G`, and util-linux reads a bare `G` as + * 1024^3. nodeSpecs.ssdStorage is GiB for the same reason - fluxbench reports + * the disk that way - so every side of a capacity check speaks one unit. + * + * @returns {Promise>} + */ +async function capacityVolumesInGib() { + const mounts = await deviceHelper.listMountedFilesystems(); + return mounts + .filter((volume) => (volume.source.includes('/dev/') && !volume.source.includes('loop') && !volume.target.includes('boot')) + || (volume.source.includes('loop') && volume.target === '/')) + .map((volume) => ({ + filesystem: volume.source, + mount: volume.target, + size: Math.round(volume.sizeBytes / BYTES_PER_GIB), + used: Math.round(volume.usedBytes / BYTES_PER_GIB), + available: Math.round(volume.availableBytes / BYTES_PER_GIB), + })); +} /** * Whether a path currently has a filesystem mounted on it. Reads @@ -49,16 +106,12 @@ async function getVolumeFilePath(appId) { const candidates = []; try { - const dfres = await dfAsync({}); - dfres.forEach((volume) => { - const eligible = volume.filesystem.includes('/dev/') && !volume.filesystem.includes('loop') - && !volume.mount.includes('boot') && volume.mount !== '/'; - if (eligible) { - candidates.push(path.join(volume.mount, volumeFileName)); - } + const mounts = await eligibleHostMounts(); + mounts.forEach((mount) => { + candidates.push(path.join(mount.target, volumeFileName)); }); } catch (error) { - log.warn(`getVolumeFilePath - df failed (${error.message}), falling back to appvolumes locations only`); + log.warn(`getVolumeFilePath - findmnt failed (${error.message}), falling back to appvolumes locations only`); } candidates.push(path.join(appVolumesPath, volumeFileName)); @@ -89,16 +142,10 @@ async function getComponentAppIdsFromVolumeFiles(appName) { const searchDirs = new Set([appVolumesPath, legacyAppVolumesPath]); try { - const dfres = await dfAsync({}); - dfres.forEach((volume) => { - const eligible = volume.filesystem.includes('/dev/') && !volume.filesystem.includes('loop') - && !volume.mount.includes('boot') && volume.mount !== '/'; - if (eligible) { - searchDirs.add(volume.mount); - } - }); + const mounts = await eligibleHostMounts(); + mounts.forEach((mount) => searchDirs.add(mount.target)); } catch (error) { - log.warn(`getComponentAppIdsFromVolumeFiles - df failed (${error.message}), searching appvolumes locations only`); + log.warn(`getComponentAppIdsFromVolumeFiles - findmnt failed (${error.message}), searching appvolumes locations only`); } const escapedName = appName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); @@ -169,7 +216,7 @@ async function ensureAppVolumeMounted(identifier) { } const mountRes = await serviceHelper.runCommand('mount', { - runAsRoot: true, params: ['-o', 'loop', volumeFile, mountPoint], logError: false, + runAsRoot: true, params: ['-o', APP_VOLUME_MOUNT_OPTIONS, volumeFile, mountPoint], logError: false, }); if (mountRes.error) { // another actor (e.g. a legacy @reboot job on its last boot) may have @@ -341,11 +388,76 @@ async function ensureMountPathsExist(appSpecifications, appName, isComponent, fu } } +/** + * Delete everything an app holds in its volume, leaving the volume itself mounted + * @param {string} identifier - Component identifier + * @returns {Promise} + */ +async function clearAppVolumeData(identifier) { + const appId = dockerService.getAppIdentifier(identifier); + const appDataPath = path.join(appsFolder, appId, 'appdata'); + + // Enumerated AND deleted as root, in one command. + // + // Listing the directory host-side runs as the FluxOS user while the rm runs + // under sudo, and that asymmetry is fatal for exactly the apps g: mode exists + // to serve: a hardening image chmods its data dir (postgres does `chmod 700 + // $PGDATA`, and for a component mounting /var/lib/postgresql/data that dir IS + // this appdata), so readdir fails EACCES. The caller treats that as a failed + // wipe - correctly - and holds dataDesired at 'clear' with a paced retry, so + // the component would never start again. Refusing to wipe is the right answer + // to a wipe that failed; it is the wrong answer to one that could have + // succeeded as root. + // + // find, not a shell glob: `rm -rf /*` was the old shape and hits E2BIG on + // a large directory, misses dotfiles, and needs a shell. -mindepth 1 empties + // the directory without removing it - the mount structure has to stay - and + // -exec ... + batches, so this is one process rather than the concurrent, + // uncapped rm-per-entry it replaces. + const wipe = await serviceHelper.runCommand('find', { + runAsRoot: true, + params: [appDataPath, '-mindepth', '1', '-maxdepth', '1', '-exec', 'rm', '-rf', '{}', '+'], + }); + + if (wipe.error) { + // Nothing to clear is not a failed clear: an app whose volume was never + // populated must not hold the reconciler on a retry forever. + // + // Classified by exit code, never by find's message: that text is strerror + // output, rendered in the node's locale (sudo keeps LANG/LC_* through + // env_keep), so matching the English words works only on English nodes - + // anywhere else a missing directory reads as a failed wipe and the + // reconciler retries it every 5s forever. `test -d` answers with its exit + // status alone. As root, like the wipe: an unprivileged check paired with + // a root action fails on a data dir the image chmods to 700. + // + // And classified AFTER the wipe rather than checked before it: check-first + // races toward "falsely clean" when the directory appears inside the + // window, where this order races toward a throw - and the next pass wipes + // whatever arrived. + const probe = await serviceHelper.runCommand('test', { + runAsRoot: true, + logError: false, + params: ['-d', appDataPath], + }); + if (probe.error) { + log.info(`No data to delete for app ${appId}`); + return; + } + throw new Error(`Failed to delete data for app ${appId}: ${wipe.stderr || wipe.error.message || wipe.error}`); + } + + log.info(`Deleted data for app ${appId}`); +} + + module.exports = { verifyAppVolumeMount, ensureMountPathsExist, + capacityVolumesInGib, isPathMounted, getVolumeFilePath, getComponentAppIdsFromVolumeFiles, ensureAppVolumeMounted, + clearAppVolumeData, }; diff --git a/ZelBack/src/services/utils/workerRunner.js b/ZelBack/src/services/utils/workerRunner.js new file mode 100644 index 0000000000..b6c44342fe --- /dev/null +++ b/ZelBack/src/services/utils/workerRunner.js @@ -0,0 +1,82 @@ +/** + * workerRunner - runs one job in a throwaway worker + * + * Some dependencies are large and rarely needed: the three cloud registry SDKs + * total ~38MB across 427 modules, and openpgp holds ~19MB. Loading one into the main isolate holds that memory for the life of the process, because module caches are never + * released and freed pages are not returned to the OS. + * + * Each therefore lives at the top of its own worker script. A worker is + * spawned for a single exchange and terminated straight after, which is the one + * way this memory is genuinely reclaimed. + */ + +const path = require('path'); +const { Worker } = require('worker_threads'); + +const WORKER_DIR = path.join(__dirname, '..', 'workers'); + +// // Jobs are short: a token exchange or a crypto operation. Well past this, +// whatever needed the answer has failed anyway. +const DEFAULT_TIMEOUT_MS = 30000; + +/** + * Run a single exchange in a dedicated worker and tear it down. + * + * @param {string} workerName Base name of the worker script in the workers directory. + * @param {object} payload Values the worker needs to build its client and make the call. + * @param {object} [options] Overrides. + * @param {number} [options.timeoutMs] How long to wait before abandoning the exchange. + * @param {string} [options.workerDir] Directory holding the worker scripts. + * @returns {Promise<*>} Whatever the worker resolved for this exchange. + */ +function runInWorker(workerName, payload, options = {}) { + const { timeoutMs = DEFAULT_TIMEOUT_MS, workerDir = WORKER_DIR } = options; + + return new Promise((resolve, reject) => { + const worker = new Worker(path.join(workerDir, `${workerName}.js`)); + + let settled = false; + let timer = null; + + const settle = (action, value) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + worker.terminate(); + action(value); + }; + + timer = setTimeout( + () => settle(reject, new Error(`${workerName} timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + if (timer.unref) timer.unref(); + + worker.on('message', (message) => { + // Failure is signalled by an explicit flag, never by whether an error + // string happens to be truthy - a throw carrying no message would + // otherwise be indistinguishable from a successful empty result. + if (!message || message.ok !== true) { + settle(reject, new Error((message && message.error) || `${workerName} failed without a reason`)); + return; + } + settle(resolve, message.result); + }); + + worker.on('error', (error) => settle(reject, error)); + + worker.on('exit', (code) => { + settle(reject, new Error(`${workerName} exited without answering (code ${code})`)); + }); + + try { + worker.postMessage(payload); + } catch (error) { + // Without this the promise rejects but the worker lives on holding its + // dependency until the timeout fires. + settle(reject, error); + } + }); +} + +module.exports = { runInWorker }; diff --git a/ZelBack/src/services/verificationHelper.js b/ZelBack/src/services/verificationHelper.js index 642d02516c..22de82d2c6 100644 --- a/ZelBack/src/services/verificationHelper.js +++ b/ZelBack/src/services/verificationHelper.js @@ -5,42 +5,57 @@ const { randomBytes } = require('crypto'); const log = require('../lib/log'); const verificationHelperUtils = require('./verificationHelperUtils'); +const { Privilege, APP_SCOPED } = require('./utils/privileges'); /** - * Verifies a specific privilege based on request headers. - * @param {string} privilege - 'admin, 'fluxteam', 'adminandfluxteam', 'appownerabove', 'appowner', 'user' - * @param {object} req - * @param {string} appName + * Which verifier answers each privilege. * + * Every member of Privilege appears here and nothing else does, so a privilege + * that no longer resolves is a failing test rather than a silent refusal at + * runtime. The wrappers defer the lookup to call time, so a stubbed + * verificationHelperUtils is still the one that answers. + */ +const DISPATCH = Object.freeze({ + [Privilege.USER]: (auth) => verificationHelperUtils.verifyUserSession(auth), + [Privilege.NODE_OPERATOR]: (auth) => verificationHelperUtils.verifyNodeOperatorSession(auth), + [Privilege.FLUX_TEAM]: (auth) => verificationHelperUtils.verifyFluxTeamSession(auth), + [Privilege.NODE_OPERATOR_OR_FLUX_TEAM]: (auth) => verificationHelperUtils.verifyNodeOperatorOrFluxTeamSession(auth), + [Privilege.APP_OWNER]: (auth, appName) => verificationHelperUtils.verifyAppOwnerSession(auth, appName), + [Privilege.APP_OWNER_OR_FLUX_TEAM]: (auth, appName) => verificationHelperUtils.verifyAppOwnerOrFluxTeamSession(auth, appName), +}); + +/** + * Whether a caller holds a privilege. + * + * Takes the zelidauth header's value, not the request it arrived in. The check + * reads one field, and a function that accepts the whole request can reach + * anything else on it - including the parts the caller controls. + * + * @param {string} privilege - a Privilege member + * @param {string} zelidauth - the value of the zelidauth header + * @param {{appName?: string}} [options] - carried by, and only by, an app-scoped privilege * @returns {Promise} authorized */ -async function verifyPrivilege(privilege, req, appName) { +async function verifyPrivilege(privilege, zelidauth, options = {}) { + // Ahead of the try, because the catch below answers false. That is the right + // answer to a check that failed and the wrong answer to a call site that is + // wired wrongly, and the two must not reach a caller wearing the same face. + if (!(privilege in DISPATCH)) { + throw new TypeError(`verifyPrivilege: ${JSON.stringify(privilege)} is not a Privilege`); + } + // A header value is always a string, so anything else came from our own code: + // a request, a fabricated headers object, an already-parsed auth. Absent is + // not in this class - it is every unauthenticated request there has ever been. + if (zelidauth != null && typeof zelidauth !== 'string') { + throw new TypeError('verifyPrivilege: takes the zelidauth header value, not the request'); + } + const scoped = APP_SCOPED.includes(privilege); + if (!scoped && 'appName' in options) { + throw new TypeError(`verifyPrivilege: ${privilege} resolves an identity and reads no app name`); + } + try { - let authorized = false; - switch (privilege) { - case 'admin': - authorized = await verificationHelperUtils.verifyAdminSession(req.headers); - break; - case 'fluxteam': - authorized = await verificationHelperUtils.verifyFluxTeamSession(req.headers); - break; - case 'adminandfluxteam': - authorized = await verificationHelperUtils.verifyAdminAndFluxTeamSession(req.headers); - break; - case 'appownerabove': - authorized = await verificationHelperUtils.verifyAppOwnerOrHigherSession(req.headers, appName); - break; - case 'appowner': - authorized = await verificationHelperUtils.verifyAppOwnerSession(req.headers, appName); - break; - case 'user': - authorized = await verificationHelperUtils.verifyUserSession(req.headers); - break; - default: - authorized = false; - break; - } - return authorized; + return await DISPATCH[privilege](zelidauth, options.appName); } catch (error) { log.error(error); return false; @@ -128,7 +143,10 @@ function signMessage(message, pk) { // => different (but valid) signature each time } catch (e) { log.error(e); - signature = e; + // Null, not the Error, for the reason getFluxNodePublicKey gives: an Error + // returned as if it were the value is truthy and survives every guard that + // is not looking for it specifically. + signature = null; } return signature; } diff --git a/ZelBack/src/services/verificationHelperUtils.js b/ZelBack/src/services/verificationHelperUtils.js index 416d6aac28..69ff5bbc0a 100644 --- a/ZelBack/src/services/verificationHelperUtils.js +++ b/ZelBack/src/services/verificationHelperUtils.js @@ -8,20 +8,73 @@ const config = require('config'); const signatureVerifier = require('./signatureVerifier'); const serviceHelper = require('./serviceHelper'); const dbHelper = require('./dbHelper'); +const configManager = require('./utils/configManager'); // Removed registryManager to avoid circular dependency - will use dynamic require where needed +/** + * The Flux ID this node's operator administers it with, or null when the node + * has not read its own configuration yet. + * + * Read through configManager rather than off globalThis. The manager loads the + * file in its own constructor, so requiring it is what guarantees the config has + * been read at all - a module that only reads the global is relying on some other + * module having imported the manager first, and a privilege check that runs before + * that import sees nothing and throws. + * + * Null still has to be handled, because a load that fails installs defaults + * carrying no zelid rather than a config. It is the only safe answer there: a + * comparison against an identity we do not hold must fail rather than pass, and + * must never quietly resolve the caller to a lesser privilege as though the + * question had been answered. Callers that grant privileges on the result + * therefore refuse outright while it is null. + * + * @returns {string|null} + */ +function nodeOperatorZelid() { + return configManager.getConfigValue('initial.zelid') ?? null; +} + +/** + * The Flux IDs the support team holds, as a list. + * + * The config value is a list so support can be granted to more than one identity + * without a code change, but it is read defensively: a node whose own config still + * carries the single string this used to be must keep working, and reading that + * string as though it were an array would match nothing and lock support out of + * the node entirely rather than fail visibly. + * + * Falsy entries are dropped. An empty or missing value yields an empty list, which + * is the safe answer - it grants no one. + * + * @returns {string[]} + */ +function fluxSupportTeamZelids() { + const configured = config.fluxSupportTeamFluxID; + if (Array.isArray(configured)) return configured.filter(Boolean); + return configured ? [configured] : []; +} + +/** + * Whether a Flux ID belongs to the support team. + * + * @param {string} zelid + * @returns {boolean} + */ +function isFluxSupportTeamZelid(zelid) { + return Boolean(zelid) && fluxSupportTeamZelids().includes(zelid); +} + /** * Verifies admin session - * @param {object} headers + * @param {string|object} zelidauth - the value of the zelidauth header * * @returns {Promise} */ -async function verifyAdminSession(headers) { - if (!headers || !headers.zelidauth) return false; - const auth = serviceHelper.ensureObject(headers.zelidauth); +async function verifyNodeOperatorSession(zelidauth) { + if (!zelidauth) return false; + const auth = serviceHelper.ensureObject(zelidauth); if (!auth.zelid || !auth.signature || !auth.loginPhrase) return false; - const userconfig = globalThis.userconfig; - if (auth.zelid !== userconfig.initial.zelid) return false; + if (auth.zelid !== nodeOperatorZelid()) return false; const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); @@ -47,13 +100,13 @@ async function verifyAdminSession(headers) { /** * Verifies user session - * @param {object} headers + * @param {string|object} zelidauth - the value of the zelidauth header * * @returns {Promise} */ -async function verifyUserSession(headers) { - if (!headers || !headers.zelidauth) return false; - const auth = serviceHelper.ensureObject(headers.zelidauth); +async function verifyUserSession(zelidauth) { + if (!zelidauth) return false; + const auth = serviceHelper.ensureObject(zelidauth); if (!auth.zelid || !auth.signature || !auth.loginPhrase) return false; const db = dbHelper.databaseConnection(); @@ -89,15 +142,15 @@ async function verifyUserSession(headers) { /** * Verifies flux team session - * @param {object} headers + * @param {string|object} zelidauth - the value of the zelidauth header * * @returns {Promise} */ -async function verifyFluxTeamSession(headers) { - if (!headers || !headers.zelidauth) return false; - const auth = serviceHelper.ensureObject(headers.zelidauth); +async function verifyFluxTeamSession(zelidauth) { + if (!zelidauth) return false; + const auth = serviceHelper.ensureObject(zelidauth); if (!auth.zelid || !auth.signature || !auth.loginPhrase) return false; - if (auth.zelid !== config.fluxTeamFluxID && auth.zelid !== config.fluxSupportTeamFluxID) return false; + if (auth.zelid !== config.fluxTeamFluxID && !isFluxSupportTeamZelid(auth.zelid)) return false; const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); @@ -123,16 +176,15 @@ async function verifyFluxTeamSession(headers) { /** * Verifies admin or flux team session - * @param {object} headers + * @param {string|object} zelidauth - the value of the zelidauth header * * @returns {Promise} */ -async function verifyAdminAndFluxTeamSession(headers) { - if (!headers || !headers.zelidauth) return false; - const auth = serviceHelper.ensureObject(headers.zelidauth); +async function verifyNodeOperatorOrFluxTeamSession(zelidauth) { + if (!zelidauth) return false; + const auth = serviceHelper.ensureObject(zelidauth); if (!auth.zelid || !auth.signature || !auth.loginPhrase) return false; - const userconfig = globalThis.userconfig; - if (auth.zelid !== config.fluxTeamFluxID && auth.zelid !== userconfig.initial.zelid && auth.zelid !== config.fluxSupportTeamFluxID) return false; // admin is considered as fluxTeam + if (auth.zelid !== config.fluxTeamFluxID && auth.zelid !== nodeOperatorZelid() && !isFluxSupportTeamZelid(auth.zelid)) return false; // admin is considered as fluxTeam const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); @@ -157,13 +209,13 @@ async function verifyAdminAndFluxTeamSession(headers) { /** * Verifies app owner session - * @param {object} headers + * @param {string|object} zelidauth - the value of the zelidauth header * * @returns {Promise} */ -async function verifyAppOwnerSession(headers, appName) { - if (!headers || !headers.zelidauth || !appName) return false; - const auth = serviceHelper.ensureObject(headers.zelidauth); +async function verifyAppOwnerSession(zelidauth, appName) { + if (!zelidauth || !appName) return false; + const auth = serviceHelper.ensureObject(zelidauth); if (!auth.zelid || !auth.signature || !auth.loginPhrase) return false; // Use dynamic require to avoid circular dependency // eslint-disable-next-line global-require @@ -201,21 +253,48 @@ async function verifyAppOwnerSession(headers, appName) { } /** - * Verifies app owner (or higher privilege) session - * @param {object} headers + * Verifies an app-owner or flux-team session: the app's owner and the flux team, + * but NOT the node operator. * - * @returns {Promise} + * This is the gate for every app-scoped endpoint: the verbs that decide whether + * someone else's app runs or keeps its data - start, stop, restart, kill, + * redeploy, remove, the volume operations and backup/restore - and everything + * that discloses what is inside it - logs, inspect, stats, the process list, the + * file listings and downloads, and a decrypted enterprise spec. + * + * The node operator is the node's own admin, so verifyNodeOperatorSession admits them + * and this must not. + * + * Hosting an app is not owning it, and the two halves of that have the same + * answer. On run state: an app cannot exceed what was bought - dockerService + * sets NanoCPUs and Memory/MemorySwap on the container from the spec - so an app + * inside its allocation is spending cycles the operator sold, and an app outside + * one is a containment defect to fix in the limits rather than to paper over on + * a single node with a button. The operator is paid whether the container runs + * or not, so a per-app stop withholds the service and keeps the payment; + * stopping FluxOS forfeits the payment along with the obligation, which is what + * makes it the honest lever. + * + * On disclosure: hosting is a reason to know what an app COSTS you, which + * /apps/appsresources answers unauthenticated and in aggregate. It is not a + * reason to read the customer's environment variables, files or logs. That the + * operator may also have local access to the disk is not an argument for + * serving the same data over an authenticated API - remote, scriptable across a + * fleet, and exposed with the operator's zelid rather than with their machine. + * + * @param {string|object} zelidauth - the value of the zelidauth header + * @param {string} appName + * @returns {Promise} authorized */ -async function verifyAppOwnerOrHigherSession(headers, appName) { - if (!headers || !headers.zelidauth || !appName) return false; - const auth = serviceHelper.ensureObject(headers.zelidauth); +async function verifyAppOwnerOrFluxTeamSession(zelidauth, appName) { + if (!zelidauth || !appName) return false; + const auth = serviceHelper.ensureObject(zelidauth); if (!auth.zelid || !auth.signature || !auth.loginPhrase) return false; // Use dynamic require to avoid circular dependency // eslint-disable-next-line global-require const registryManager = require('./appDatabase/registryManager'); const ownerFluxID = await registryManager.getApplicationOwner(appName); - const userconfig = globalThis.userconfig; - if (auth.zelid !== ownerFluxID && auth.zelid !== config.fluxTeamFluxID && auth.zelid !== userconfig.initial.zelid && auth.zelid !== config.fluxSupportTeamFluxID) return false; + if (auth.zelid !== ownerFluxID && auth.zelid !== config.fluxTeamFluxID && !isFluxSupportTeamZelid(auth.zelid)) return false; const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); @@ -248,9 +327,12 @@ async function verifyAppOwnerOrHigherSession(headers, appName) { } module.exports = { - verifyAdminAndFluxTeamSession, - verifyAdminSession, - verifyAppOwnerOrHigherSession, + fluxSupportTeamZelids, + isFluxSupportTeamZelid, + nodeOperatorZelid, + verifyNodeOperatorOrFluxTeamSession, + verifyNodeOperatorSession, + verifyAppOwnerOrFluxTeamSession, verifyAppOwnerSession, verifyFluxTeamSession, verifyUserSession, diff --git a/ZelBack/src/services/workers/awsEcrAuthWorker.js b/ZelBack/src/services/workers/awsEcrAuthWorker.js new file mode 100644 index 0000000000..65ac9f0713 --- /dev/null +++ b/ZelBack/src/services/workers/awsEcrAuthWorker.js @@ -0,0 +1,37 @@ +/** + * AWS ECR auth worker - runs one ECR call for the provider. + * + * @aws-sdk/client-ecr is required here, at the top of a worker spawned per call + * and terminated after it, so the main isolate never carries it. Only the fields + * the provider consumes are returned; expiresAt survives as a Date because + * structured clone preserves them. + */ + +const { parentPort } = require('worker_threads'); +// eslint-disable-next-line import/no-unresolved +const { ECRClient, GetAuthorizationTokenCommand, DescribeRepositoriesCommand } = require('@aws-sdk/client-ecr'); + +const COMMANDS = { + getAuthorizationToken: GetAuthorizationTokenCommand, + describeRepositories: DescribeRepositoriesCommand, +}; + +parentPort.on('message', async (payload) => { + try { + const { operation, clientConfig, params } = payload; + + const Command = COMMANDS[operation]; + if (!Command) throw new Error(`Unsupported ECR operation: ${operation}`); + + const client = new ECRClient(clientConfig); + const response = await client.send(new Command(params || {})); + + const result = operation === 'getAuthorizationToken' + ? { authorizationData: response.authorizationData } + : { ok: true }; + + parentPort.postMessage({ ok: true, result }); + } catch (error) { + parentPort.postMessage({ ok: false, error: error.message || String(error) }); + } +}); diff --git a/ZelBack/src/services/workers/azureAcrAuthWorker.js b/ZelBack/src/services/workers/azureAcrAuthWorker.js new file mode 100644 index 0000000000..9449a9aa5b --- /dev/null +++ b/ZelBack/src/services/workers/azureAcrAuthWorker.js @@ -0,0 +1,33 @@ +/** + * Azure ACR auth worker - obtains an Azure AD access token for a service principal. + * + * This is the only step of the ACR flow that needs @azure/identity; the refresh + * and access token exchanges that follow it are plain HTTPS and stay in the + * provider. The SDK is required here, at the top of a worker that is spawned per + * exchange and terminated after it, so the main isolate never carries it. + */ + +const { parentPort } = require('worker_threads'); +// eslint-disable-next-line import/no-unresolved +const { ClientSecretCredential } = require('@azure/identity'); + +parentPort.on('message', async (payload) => { + try { + const { + tenantId, clientId, clientSecret, scopes, + } = payload; + + const credential = new ClientSecretCredential(tenantId, clientId, clientSecret); + const tokenResponse = await credential.getToken(scopes); + + // Only the fields the provider consumes cross the boundary - the credential + // object itself holds live handles that mean nothing outside this worker. + const result = tokenResponse + ? { token: tokenResponse.token, expiresOnTimestamp: tokenResponse.expiresOnTimestamp } + : null; + + parentPort.postMessage({ ok: true, result }); + } catch (error) { + parentPort.postMessage({ ok: false, error: error.message || String(error) }); + } +}); diff --git a/ZelBack/src/services/workers/googleGarAuthWorker.js b/ZelBack/src/services/workers/googleGarAuthWorker.js new file mode 100644 index 0000000000..d0c9dd57ff --- /dev/null +++ b/ZelBack/src/services/workers/googleGarAuthWorker.js @@ -0,0 +1,31 @@ +/** + * Google GAR auth worker - mints an OAuth access token for a service account. + * + * google-auth-library is required here, at the top of a worker spawned for a + * single exchange and terminated after it, so the main isolate never carries + * it. The expiry the provider needs lives on the client after the call, so it + * is read here and returned alongside the token. + */ + +const { parentPort } = require('worker_threads'); +// eslint-disable-next-line import/no-unresolved +const { JWT } = require('google-auth-library'); + +parentPort.on('message', async (payload) => { + try { + const { clientEmail, privateKey, scopes } = payload; + + const jwtClient = new JWT({ email: clientEmail, key: privateKey, scopes }); + const tokens = await jwtClient.getAccessToken(); + + parentPort.postMessage({ + ok: true, + result: { + token: tokens ? tokens.token : null, + expiryDate: jwtClient.credentials ? jwtClient.credentials.expiry_date : null, + }, + }); + } catch (error) { + parentPort.postMessage({ ok: false, error: error.message || String(error) }); + } +}); diff --git a/ZelBack/src/services/workers/pgpWorker.js b/ZelBack/src/services/workers/pgpWorker.js new file mode 100644 index 0000000000..5ca8ab9f5a --- /dev/null +++ b/ZelBack/src/services/workers/pgpWorker.js @@ -0,0 +1,64 @@ +/** + * PGP worker - runs one openpgp operation for pgpService. + * + * openpgp holds ~19MB (largely WASM linear memory) from the moment it is + * required, and a node needs it for one identity check at boot plus the secrets + * of the handful of v7 apps that still carry PGP-encrypted fields - v8 and + * later keep theirs inside the enterprise blob, which uses node's own crypto. + * Requiring it here, in a worker spawned per operation and terminated after it, + * keeps that memory out of the main isolate. + */ + +const { parentPort } = require('worker_threads'); +const openpgp = require('openpgp'); + +const operations = { + async derivePublicKey({ armoredPrivateKey }) { + const privateKey = await openpgp.readPrivateKey({ armoredKey: armoredPrivateKey }); + return privateKey.toPublic().armor(); + }, + + async generateKey({ name, email }) { + const keypair = await openpgp.generateKey({ + type: 'ecc', + curve: 'curve25519', + userIDs: [{ name, email }], + passphrase: '', + format: 'armored', + }); + return { privateKey: keypair.privateKey, publicKey: keypair.publicKey }; + }, + + async encrypt({ message, encryptionKeys }) { + const publicKeys = await Promise.all( + encryptionKeys.map((armoredKey) => openpgp.readKey({ armoredKey })), + ); + return openpgp.encrypt({ + message: await openpgp.createMessage({ text: message }), + encryptionKeys: publicKeys, + }); + }, + + async decrypt({ encryptedMessage, decryptionKey }) { + const messageEncrypted = await openpgp.readMessage({ armoredMessage: encryptedMessage }); + const privateKey = await openpgp.readPrivateKey({ armoredKey: decryptionKey }); + const decryptedMessage = await openpgp.decrypt({ + message: messageEncrypted, + decryptionKeys: privateKey, + }); + return decryptedMessage.data; + }, +}; + +parentPort.on('message', async (payload) => { + try { + const { operation, params } = payload; + + const run = operations[operation]; + if (!run) throw new Error(`Unsupported PGP operation: ${operation}`); + + parentPort.postMessage({ ok: true, result: await run(params || {}) }); + } catch (error) { + parentPort.postMessage({ ok: false, error: error.message || String(error) }); + } +}); diff --git a/apiServer.js b/apiServer.js index 91b38c9bec..80feb3b141 100644 --- a/apiServer.js +++ b/apiServer.js @@ -1,3 +1,8 @@ +// First, and above every other require: the environment this process answers from. This +// file runs as an entry point of its own under `require.main === module`, so it settles +// the environment rather than relying on whoever required it having done so. +require('./ZelBack/pinEnvironment'); + const configManager = require('./ZelBack/src/services/utils/configManager'); if (typeof AbortController === 'undefined') { @@ -7,8 +12,6 @@ if (typeof AbortController === 'undefined') { globalThis.AbortController = abortControler.AbortController; } -process.env.NODE_CONFIG_DIR = `${__dirname}/ZelBack/config/`; - const fs = require('node:fs'); const http = require('node:http'); const https = require('node:https'); diff --git a/app.js b/app.js index 61124ff62d..889ab75d50 100644 --- a/app.js +++ b/app.js @@ -1,4 +1,5 @@ -process.env.NODE_CONFIG_DIR = `${__dirname}/ZelBack/config/`; +// First, and above every other require: the environment this process answers from. +require('./ZelBack/pinEnvironment'); const log = require('./ZelBack/src/lib/log'); const path = require('path'); @@ -46,4 +47,8 @@ async function initiate() { }); } -initiate(); +// Guarded so the entry point can be loaded without starting a node: the four +// environment lines above decide what this process discloses, and a test can only +// read them off the real file. Both launchers - `node app.js` and `nodemon app.js` +// - enter here as main. +if (require.main === module) initiate(); diff --git a/babel.config.js b/babel.config.js deleted file mode 100644 index 757ff9b16a..0000000000 --- a/babel.config.js +++ /dev/null @@ -1,5 +0,0 @@ -module.exports = { - presets: [ - '@vue/cli-plugin-babel/preset', - ], -}; diff --git a/helpers/hashes.json b/helpers/hashes.json index 309e638c8a..41e185252d 100644 --- a/helpers/hashes.json +++ b/helpers/hashes.json @@ -8,5 +8,6 @@ "92b0d12de6cd7e4b06586543af649bc1", "f835d4ab10e48dd75110b27110799b1f", "b388b481e82febcb5427eb2503a1fe58", - "8ad927518ce5f37406aed39700134082" + "8ad927518ce5f37406aed39700134082", + "f91c1a1ec2604642ddd69edc121d18b6" ] diff --git a/homeServer.js b/homeServer.js deleted file mode 100644 index 34e9537dc2..0000000000 --- a/homeServer.js +++ /dev/null @@ -1,70 +0,0 @@ -process.env.NODE_CONFIG_DIR = `${__dirname}/ZelBack/config/`; -// Flux Home configuration -const config = require('config'); -const compression = require('compression'); -const path = require('path'); -const express = require('express'); -const log = require('./ZelBack/src/lib/log'); -const upnpService = require('./ZelBack/src/services/upnpService'); - -const userconfig = require('./config/userconfig'); - -// Cloud UI static files directory -const cloudUI = path.join(__dirname, './CloudUI'); - -const homeApp = express(); -homeApp.use(compression()); - -const apiPort = userconfig.initial.apiport || config.server.apiport; -const homePort = apiPort - 1; - -// Health check endpoint -homeApp.get('/health', (req, res) => { - res.type('text/plain'); - res.send('OK'); -}); - -// Serve static files from CloudUI -homeApp.use(express.static(cloudUI)); - -// SPA fallback - serve index.html for unmatched routes. -// File-like URLs (with an extension) return 404 so missing static assets such as -// /llms.txt or /ads.txt don't produce a "false 200" with the SPA shell. -homeApp.get('*', (req, res) => { - if (path.extname(req.path)) { - res.status(404).type('text/plain').send('Not Found'); - - return; - } - res.sendFile(path.join(cloudUI, 'index.html')); -}); - -async function initiate() { - if (!config.server.allowedPorts.includes(+apiPort)) { - log.error(`Flux port ${apiPort} is not supported. Shutting down.`); - process.exit(); - } - let verifyUpnp = false; - let setupUpnp = false; - if (userconfig.initial.apiport) { - verifyUpnp = await upnpService.verifyUPNPsupport(apiPort); - if (verifyUpnp) { - setupUpnp = await upnpService.setupUPNP(apiPort); - } - } - if ((userconfig.initial.apiport && userconfig.initial.apiport !== config.server.apiport) || userconfig.initial.routerIP) { - if (verifyUpnp !== true) { - log.error(`Flux port ${userconfig.initial.apiport} specified but UPnP failed to verify support. Shutting down.`); - process.exit(); - } - if (setupUpnp !== true) { - log.error(`Flux port ${userconfig.initial.apiport} specified but UPnP failed to map to api or home port. Shutting down.`); - process.exit(); - } - } - homeApp.listen(homePort, () => { - log.info(`Flux Home running on port ${homePort}!`); - }); -} - -initiate(); diff --git a/package.json b/package.json index 37dee61bd4..b36edc0baf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flux", - "version": "8.17.1", + "version": "8.18.0", "description": "Flux, Your Gateway to a Decentralized World", "repository": { "type": "git", @@ -67,8 +67,6 @@ "flux": "nodemon app.js", "start": "npm install --omit=dev --legacy-peer-deps && node init.js && npm run flux", "dev": "node init.js && npm run fluxdev", - "enterdevelopment": "git checkout development", - "entermaster": "git checkout master", "softupdate": "git checkout .; git pull", "softupdateinstall": "git checkout .; git pull; npm install --omit=dev --legacy-peer-deps", "updateflux": "git pull; git reset --hard; rm -rf .git/index.lock; git reset --hard; git pull", @@ -105,7 +103,6 @@ "nano-ethereum-signer": "~0.1.2", "node-abort-controller": "^3.1.1", "node-cmd": "~5.0.0", - "node-df": "~0.1.4", "nodemon": "~3.1.10", "object-hash": "~3.0.0", "openpgp": "~6.2.0", @@ -126,6 +123,7 @@ "eslint": "~8.57.1", "eslint-import-resolver-alias": "~1.1.2", "eslint-plugin-import": "~2.32.0", + "espree": "^9.6.1", "jsdoc": "~4.0.3", "mocha": "~11.7.1", "nyc": "~17.1.0", diff --git a/scripts/update-cloudui.sh b/scripts/update-cloudui.sh index 0ce03df2d6..ef949f3343 100755 --- a/scripts/update-cloudui.sh +++ b/scripts/update-cloudui.sh @@ -4,8 +4,11 @@ # Downloads the latest dist.tar.gz from fluxos-frontend GitHub releases, # verifies the SHA256 checksum, and updates the CloudUI folder. # -# Usage: npm run update:cloudui -# or: bash scripts/update-cloudui.sh +# The API base URL is required - see the note on API_BASE_URL below for why it is +# an argument rather than a default or an environment variable. +# +# Usage: npm run update:cloudui -- +# or: bash scripts/update-cloudui.sh # # Output: # - CloudUI/ folder with the latest frontend build @@ -24,7 +27,19 @@ PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" TEMP_DIR=$(mktemp -d) REPO="RunOnFlux/fluxos-frontend" CLOUDUI_DIR="$PROJECT_ROOT/CloudUI" -RELEASE_API="https://api.github.com/repos/$REPO/releases/latest" + +# The API host is supplied by the caller, which reads it from ZelBack's config. It is +# deliberately NOT defaulted and NOT read from the environment: config is the single source +# of truth for every endpoint a node reaches, and an environment variable would hand that +# choice back to whoever starts the process. A caller that forgets the argument gets a +# failure, never a silent fetch from the public internet. +API_BASE_URL="${1:-}" +if [ -z "$API_BASE_URL" ]; then + echo "Error: no API base URL given." + echo "Usage: $0 (e.g. https://api.github.com)" + exit 1 +fi +RELEASE_API="$API_BASE_URL/repos/$REPO/releases/latest" echo "==========================================" echo " FluxOS CloudUI Update Script" diff --git a/test-infra/Dockerfile.fluxos b/test-infra/Dockerfile.fluxos index 8e42474225..1cd29e87ca 100644 --- a/test-infra/Dockerfile.fluxos +++ b/test-infra/Dockerfile.fluxos @@ -1,4 +1,4 @@ -FROM ubuntu:26.04 +FROM ubuntu:26.04 AS base ENV DEBIAN_FRONTEND=noninteractive @@ -28,6 +28,83 @@ RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ && apt-get install -y --no-install-recommends nodejs \ && rm -rf /var/lib/apt/lists/* +# The packages FluxOS installs on a legacy node, and the repository it installs +# them from. +# +# monitorSystem() returns immediately when FLUXOS_PATH is set, so this path only +# runs on legacy nodes - and there it really installs, over the internet, inside +# every boot. Resolving the closure at build time and serving it to the fleet +# closes that without taking the install path out of the test: a seeded node +# still runs monitorSystem() for real and finds its prerequisites satisfied, and +# a suite that wants the install itself purges them first. +# +# The stage builds on `base`, not on ubuntu directly, because the closure is +# whatever apt resolves against THIS image's package state. Computed against a +# barer image it would name packages the node already has and miss ones it does +# not. +# +# Recommends are deliberately not suppressed. aptRunner passes no +# --no-install-recommends, so a node's own install pulls them - that is where +# shared-mime-info comes from - and a repository built without them would be +# missing packages the node goes on to ask for. +FROM base AS debcache + +COPY test-infra/syncthing-release-key.asc /tmp/syncthing-release-key.asc +COPY test-infra/build-apt-repo.sh /usr/local/bin/build-apt-repo.sh + +# Syncthing's repository carries only its two most recent releases, so there is +# no older version to pin to and no build arg that could hold one. The version +# is whatever it serves at build time, recorded in the repository so that +# whoever has to agree with it can read it rather than restate it. That is also +# how a production node gets syncthing: from the image it was built with. +# +# The key is vendored rather than fetched so the repository it authenticates is +# pinned in git and reviewed, not trusted on first use at build time. +RUN gpg --dearmor -o /usr/share/keyrings/syncthing-archive-keyring.gpg /tmp/syncthing-release-key.asc \ + && echo "deb [signed-by=/usr/share/keyrings/syncthing-archive-keyring.gpg] https://apt.syncthing.net/ syncthing stable-v2" \ + > /etc/apt/sources.list.d/syncthing.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends dpkg-dev apt-utils \ + && rm -rf /var/cache/apt/archives/*.deb \ + && apt-get install -y --download-only chrony syncthing netcat-openbsd \ + && chmod +x /usr/local/bin/build-apt-repo.sh \ + && /usr/local/bin/build-apt-repo.sh /opt/flux-apt-repo + +FROM base + +# The repository travels in the image because the stub that serves it to the +# fleet copies it from here: what a node installs at runtime is then the same +# file it was seeded with, not a second download that can differ. +COPY --from=debcache /opt/flux-apt-repo /opt/flux-apt-repo + +# Every source that reaches the internet goes, including the two this Dockerfile +# added for its own build - docker's and nodesource's outlive the layer that +# needed them, so any later apt-get update refreshes them, which is egress that +# exists in no product source file. Removing the directory wholesale rather than +# naming files means a source added later cannot be missed here. +# +# The base packages are served from the image over file:// because FluxOS never +# writes this source and serving it over HTTP would cover nothing. The one +# source FluxOS does write - syncthing's - is pointed at the stub over HTTP by +# config, so the keyring fetch, the source write and apt's HTTP transport are +# all exercised as they are on a node. +# +# Seeded here, so a normal fleet boots into the steady state. +RUN rm -f /etc/apt/sources.list.d/* /etc/apt/sources.list \ + && cp /opt/flux-apt-repo/keyring.gpg /usr/share/keyrings/flux-e2e-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/flux-e2e-archive-keyring.gpg] file:///opt/flux-apt-repo ubuntu main" \ + > /etc/apt/sources.list.d/flux-e2e.list \ + && apt-get update \ + && apt-get install -y chrony syncthing netcat-openbsd \ + && rm -rf /var/cache/apt/archives/*.deb \ + && syncthing --version + +# The package index stays. A node has one, and upgradePackage refreshes the cache +# without forcing - cacheUpdateTime falls back to the mtime of /var/lib/apt/lists, +# which on a freshly built image is minutes old, so the refresh is skipped as +# recent. Emptied, the index is then both absent and apparently current, and every +# install fails with `Unable to locate package` before reaching the network at all. + WORKDIR /flux COPY package.json ./ diff --git a/test-infra/build-apt-repo.sh b/test-infra/build-apt-repo.sh new file mode 100644 index 0000000000..ec3b4711dc --- /dev/null +++ b/test-infra/build-apt-repo.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Turns the .debs apt has already downloaded into a signed apt repository. +# +# Run inside the image build, after `apt-get install --download-only` has +# populated the archive cache. What lands in the cache IS the dependency +# closure apt resolved against this image's exact package state, so the +# repository is complete by construction rather than by a maintained list - +# nothing here names a package or a version. +# +# Two distributions over one pool. FluxOS writes the syncthing source itself +# (systemService addSyncthingRepository) and its distribution and component are +# fixed at `syncthing`/`stable-v2`, so the repository has to answer to that +# name; the base packages have no such constraint and use `ubuntu`/`main`. +# Both index the same pool, so a package needed by either is present in both. +set -eu + +REPO="${1:?usage: build-apt-repo.sh }" +ARCH="$(dpkg --print-architecture)" + +mkdir -p "$REPO/pool" +cp /var/cache/apt/archives/*.deb "$REPO/pool/" + +# The signing key is generated here, not vendored. It signs nothing that exists +# outside this image, and a private key committed to git invites reuse somewhere +# it would matter. The public half ships in the repository as keyring.gpg. +export GNUPGHOME=/tmp/flux-apt-repo-gnupg +mkdir -p "$GNUPGHOME" +chmod 700 "$GNUPGHOME" +gpg --batch --quiet --passphrase '' \ + --quick-generate-key 'Flux E2E Apt Repository' rsa3072 sign never + +cd "$REPO" +for dist in ubuntu syncthing; do + if [ "$dist" = 'syncthing' ]; then component='stable-v2'; else component='main'; fi + target="dists/$dist/$component/binary-$ARCH" + + mkdir -p "$target" + dpkg-scanpackages --arch "$ARCH" pool /dev/null > "$target/Packages" + gzip -9cn "$target/Packages" > "$target/Packages.gz" + + apt-ftparchive \ + -o "APT::FTPArchive::Release::Origin=flux-e2e" \ + -o "APT::FTPArchive::Release::Suite=$dist" \ + -o "APT::FTPArchive::Release::Codename=$dist" \ + -o "APT::FTPArchive::Release::Components=$component" \ + -o "APT::FTPArchive::Release::Architectures=$ARCH" \ + release "dists/$dist" > "dists/$dist/Release" + + # InRelease (inline signature) rather than a detached Release.gpg: apt prefers + # it, and it is one file to serve instead of two that can disagree. + gpg --batch --yes --clearsign -o "dists/$dist/InRelease" "dists/$dist/Release" +done + +gpg --batch --yes --export > "$REPO/keyring.gpg" +rm -rf "$GNUPGHOME" + +# The installed syncthing version is whatever the upstream repository served at +# build time, so nothing downstream may hardcode it. Recorded here for the stub +# that answers the minimum-version check, which has to agree with the image or +# the check compares against a version no node has. +dpkg-deb -f "$(ls "$REPO"/pool/syncthing_*.deb | head -1)" Version > "$REPO/syncthing.version" diff --git a/test-infra/build-images.sh b/test-infra/build-images.sh new file mode 100755 index 0000000000..3b5a7e3694 --- /dev/null +++ b/test-infra/build-images.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Build every image the harness runs, under one tag. +# +# One box hosts more than one branch's harness work at a time and the image +# names are fixed, so an untagged rebuild silently replaces whatever another +# branch built - and a run then mixes this branch's tree with that branch's +# images, which fails in ways that look like product bugs. Tag per branch and +# both can sit side by side: +# +# FLUX_E2E_TAG=placement ./test-infra/build-images.sh +# FLUX_E2E_TAG=placement npx mocha "tests/76-*.js" --timeout 400000 +# +# The default tag is `latest`, which is what an untagged `docker build` +# produces, so single-branch use is unchanged. +# +# Pass image names to build a subset: +# FLUX_E2E_TAG=placement ./test-infra/build-images.sh fluxos-01 external-http-stub +# +# Every image is stamped with `flux.e2e.src`, the digest of the sources it was +# built from (test-infra/image-digest.sh). verify-images.sh recomputes that from +# the working tree before a run and refuses to start when they differ, so an +# image left behind by another branch cannot quietly decide a gate. +set -euo pipefail + +TAG="${FLUX_E2E_TAG:-latest}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +# The stub set is DERIVED, never listed: it differs by lineage (v9 carries a +# fluxdrive-stub the development lineage neither builds nor references), and a +# hardcoded list silently skips whichever image the other lineage added. +mapfile -t STUBS < <( + for dir in test-infra/*/; do + [ -f "${dir}Dockerfile" ] && basename "$dir" + done +) + +build_fluxos() { + # The app binary the fixtures need; gitignored, so it survives branch switches + # and is easy to forget after a clean. Not guarded on the script existing: it + # is present on every lineage, and a guard there would let this produce a + # complete image set with no binary in it - which surfaces as eight suites + # failing twenty minutes later, looking like product bugs. Built BEFORE the + # digest is taken - it sits in the build context, so it is part of what the + # image is. + echo "==> test-app binary" + bash test-infra/test-app/build.sh + echo "==> flux-e2e-fluxos-01:${TAG}" + docker build -f test-infra/Dockerfile.fluxos \ + --label "flux.e2e.src=$(test-infra/image-digest.sh fluxos-01)" \ + -t "flux-e2e-fluxos-01:${TAG}" . +} + +build_stub() { + echo "==> flux-e2e-$1:${TAG}" + # The tag reaches every stub so one can build FROM the node image under the same + # tag; a stub whose Dockerfile declares no such ARG ignores it. external-http-stub + # does, which is why fluxos-01 is built first - and why its digest folds in the + # node image's, so a stale base makes the stub read stale too. + docker build --build-arg "FLUX_E2E_TAG=${TAG}" \ + --label "flux.e2e.src=$(test-infra/image-digest.sh "$1")" \ + -t "flux-e2e-$1:${TAG}" "test-infra/$1" +} + +targets=("$@") +if [ ${#targets[@]} -eq 0 ]; then + targets=(fluxos-01 "${STUBS[@]}") +fi + +for target in "${targets[@]}"; do + if [ "$target" = "fluxos-01" ]; then + build_fluxos + else + build_stub "$target" + fi +done + +echo +echo "built under tag '${TAG}':" +docker images --format ' {{.Repository}}:{{.Tag}} {{.CreatedSince}}' \ + | grep -E "flux-e2e-.*:${TAG}\b" || true +echo +echo "run with: FLUX_E2E_TAG=${TAG} npx mocha \"tests/NN-*.js\" --timeout 400000" diff --git a/test-infra/config/HARNESS_TUNABLES.md b/test-infra/config/HARNESS_TUNABLES.md new file mode 100644 index 0000000000..c819e53485 --- /dev/null +++ b/test-infra/config/HARNESS_TUNABLES.md @@ -0,0 +1,170 @@ +# Harness config tunables, as ratios of production + +Generated from `ZelBack/config/default.js` and `test-infra/config/shared.js`. +Regenerate rather than hand-edit; every suite header quotes these numbers and a +checker verifies them. + +A **factor** is production divided by harness: 10x means the harness reaches the +same behaviour ten times sooner. Two knobs the code relates by an inequality must +carry the SAME factor, or the property between them is deleted or inverted. + +## Pairs checked at fleet boot + +The rule above was written here and broken anyway, because the two halves of a +pair need not live in the same layer: `explorerPollIntervalMs` is in `shared.js` +and `residentialQueueStepMs` was a literal in one suite's overrides. Nothing +related them, so moving the poll 250ms -> 833ms moved the pass 4s -> 16s and +left the step at 15s - below the pass, where the property inverts. Two holders +of one app matured on the same pass and both handed it back. + +`test-infra/runner/framework/coupled-knobs.js` now derives these and +`test-env.js` asserts them on the EFFECTIVE config of every node of every fleet +before boot. Over production's ratio passes - an uncompressed knob is slow, not +wrong. Under throws. + +| property | harness pair | production ratio | how the harness gets it | +|---|---|---:|---| +| two holders of one app cannot mature on the same give-up pass | `residentialQueueStepMs` : `removeFluxAppsPeriod` x 4 x `explorerPollIntervalMs` | 1.82 | `derivedQueueStepMs(fluxapps)` - never a literal | +| a departure restarts the other holders' queue tickets | `residentialEvacuationIntervalMs` : `residentialQueueStepMs` x `TICKET_GAP_STEPS` | 4.5 | `derivedEvacuationIntervalMs(fluxapps)` - never a literal | +| a suite's wait covers a whole departure | `driveUntil` timeout : interval + base + position x step | - | `departureCycleMs(fluxapps, instances)` - never a literal | + +The second pair is bounded by what it must OUTLIVE rather than by production's +ratio, the same way the sigterm window is bounded by a node boot. A node inside +its departure interval records nothing against its queue tickets, so the block is +what restarts them - and it only reads as a restart if it is longer than the gap +a ticket tolerates. Production holds 6h against an 80min tolerance; the harness +needs only that tolerance plus one pass. Below it, tickets carry straight across +the block, every app is instantly ready the moment the interval clears, and the +suite goes green on a queue that stopped separating anything after the first +departure - the same defect a too-short step causes, through the other door. + +**The tolerance is TWO steps, and that factor is the whole lesson.** One step is +1.82 passes, so at one step a single LATE pass restarts a ticket. Production +hardly notices - 40 minutes of lateness on a 22-minute pass is an incident in its +own right - but the harness compresses the same ratio to about 30 seconds, where +six fleets booting at once make a late pass ordinary. On chud the countdown ran +2m, 1m, 0m and jumped back to 2m, three times over, never matured, and two suites +timed out on nodes that were behaving correctly. **Absolute jitter does not +compress with the clocks**, so a ratio that is comfortable at production scale is +not automatically comfortable here. `TICKET_GAP_STEPS` carries the factor, and a +unit test pins it to the product constant it models. + +The third pair is the same mistake one layer out. Suite 55's waits were four +minutes, written when the interval was four seconds; a departure is now an +interval PLUS a full ticket served again from scratch, and the typed number +quietly stopped covering one. A wait is as coupled to the pacing as the step is +to the pass. + +A block costs more than its poll; `BLOCK_COST_OVERHEAD` carries the difference +and is calibrated against a measurement, not chosen (model 15994ms against 15900ms +observed over nine consecutive passes on cindy, 2026-08-20). Modelling a block as +exactly one poll derives a step that is too SHORT, which is the direction that +loses the property, so the factor is pinned by a unit test. + +## Layer 2 - `shared.js`, applied to every suite + +| key | production | harness | factor | +|---|---:|---:|---:| +| `confirmation.daemonExpiredMs` | 19200000 | 600000 | 32.0x | +| `confirmation.daemonStaleMs` | 7500000 | 300000 | 25.0x | +| `confirmation.pollIntervalMs` | 30000 | 5000 | 6.0x | +| `fluxapps.appSyncDegradedThreshold` | 4 | 1 | 4.0x | +| `fluxapps.appSyncMinCompletions` | 3 | 1 | 3.0x | +| `fluxapps.appSyncMinPeerUptime` | 7500 | 0 | n/a | +| `fluxapps.appSyncPeerThreshold` | 12 | 2 | 6.0x | +| `fluxapps.bootDelayMultiplier` | 1 | 0.01 | 100.0x | +| `fluxapps.cpuCheckIntervalMs` | 900000 | 30000 | 30.0x | +| `fluxapps.daemonInfoIntervalMs` | 30000 | 5000 | 6.0x | +| `fluxapps.defaultSwap` | 2 | 0 | n/a | +| `fluxapps.discoveryConnectionDelayMs` | 500 | 100 | 5.0x | +| `fluxapps.discoveryFailRetryMs` | 120000 | 5000 | 24.0x | +| `fluxapps.discoveryRetryMs` | 60000 | 5000 | 12.0x | +| `fluxapps.explorerDeepRestoreBlocks` | 100 | 0 | n/a | +| `fluxapps.explorerPollIntervalMs` | 5000 | 833 | 6.0x | +| `fluxapps.explorerSyncRetryMs` | 120000 | 5000 | 24.0x | +| `fluxapps.forceRemovalIntervalMs` | 7200000 | 120000 | 60.0x | +| `fluxapps.globalCmdDelayMs` | 500 | 100 | 5.0x | +| `fluxapps.hashSyncEphemeralPeers` | 5 | 3 | 1.7x | +| `fluxapps.hashSyncFallbackRecheckBlocks` | 100 | 10 | 10.0x | +| `fluxapps.hashSyncMaxRetries` | 3 | 2 | 1.5x | +| `fluxapps.hashSyncRetryMs` | 300000 | 10000 | 30.0x | +| `fluxapps.hashSyncSettleMs` | 4000 | 2000 | 2.0x | +| `fluxapps.hddFileSystemMinimum` | 10 | 2 | 5.0x | +| `fluxapps.imageComplianceIntervalMs` | 3600000 | 60000 | 60.0x | +| `fluxapps.imageUpdateCheckIntervalMs` | 21600000 | 5000 | 4320.0x | +| `fluxapps.imageUpdateDelayAfterRedeployMs` | 120000 | 1000 | 120.0x | +| `fluxapps.imageUpdateDelayBetweenAppsMs` | 5000 | 100 | 50.0x | +| `fluxapps.imageUpdateDelayBetweenComponentsMs` | 1000 | 100 | 10.0x | +| `fluxapps.imageUpdateInitialDelayMaxMs` | 1800000 | 2000 | 900.0x | +| `fluxapps.imageUpdateInitialDelayMinMs` | 600000 | 1000 | 600.0x | +| `fluxapps.installation.delay` | 120 | 5 | 24.0x | +| `fluxapps.installCollisionWaitMs` | 90000 | 5000 | 18.0x | +| `fluxapps.locationTtlS` | 7500 | 63 | 119.0x | +| `fluxapps.masterSlaveIntervalMs` | 30000 | 3000 | 10.0x | +| `fluxapps.minHashSyncPeers` | 12 | 1 | 12.0x | +| `fluxapps.minIncoming` | 4 | 2 | 2.0x | +| `fluxapps.minOutgoing` | 8 | 4 | 2.0x | +| `fluxapps.minUniqueIpsIncoming` | 3 | 2 | 1.5x | +| `fluxapps.minUniqueIpsOutgoing` | 7 | 3 | 2.3x | +| `fluxapps.minUpTime` | 1800 | 10 | 180.0x | +| `fluxapps.nodeMonitorCheckTimeoutMs` | 10000 | 5000 | 2.0x | +| `fluxapps.nodeMonitorConfirmationLossDelayMs` | 1200000 | 10000 | 120.0x | +| `fluxapps.nodeMonitorDosRecoveryDelayMs` | 600000 | 10000 | 60.0x | +| `fluxapps.nodeMonitorErrorRecoveryDelayMs` | 120000 | 5000 | 24.0x | +| `fluxapps.nodeMonitorIntervalMs` | 1200000 | 10000 | 120.0x | +| `fluxapps.nodeMonitorRemovalDelayMs` | 60000 | 1000 | 60.0x | +| `fluxapps.nonEnterpriseSpawnDelayMs` | 120000 | 500 | 240.0x | +| `fluxapps.portRestoreIntervalMs` | 600000 | 30000 | 20.0x | +| `fluxapps.portTestBindDelayMs` | 5000 | 100 | 50.0x | +| `fluxapps.portTestMaxAttempts` | 5 | 2 | 2.5x | +| `fluxapps.portTestPeerTimeoutMs` | 30000 | 3000 | 10.0x | +| `fluxapps.portTestPropagationDelayMs` | 10000 | 100 | 100.0x | +| `fluxapps.redeploy.composedDelay` | 5 | 1 | 5.0x | +| `fluxapps.redeploy.delay` | 30 | 1 | 30.0x | +| `fluxapps.removal.delay` | 300 | 5 | 60.0x | +| `fluxapps.spawnDeferrals.capacityGap.largeMs.enterprise` | 1800000 | 350 | 5142.9x | +| `fluxapps.spawnDeferrals.capacityGap.largeMs.standard` | 7020000 | 700 | 10028.6x | +| `fluxapps.spawnDeferrals.capacityGap.mediumMs.enterprise` | 1260000 | 400 | 3150.0x | +| `fluxapps.spawnDeferrals.capacityGap.mediumMs.standard` | 5220000 | 800 | 6525.0x | +| `fluxapps.spawnDeferrals.capacityGap.smallMs.enterprise` | 720000 | 450 | 1600.0x | +| `fluxapps.spawnDeferrals.capacityGap.smallMs.standard` | 3420000 | 900 | 3800.0x | +| `fluxapps.spawnDeferrals.datacenterMs.enterprise` | 1620000 | 250 | 6480.0x | +| `fluxapps.spawnDeferrals.datacenterMs.standard` | 3420000 | 500 | 6840.0x | +| `fluxapps.spawnDeferrals.staticIpMs.enterprise` | 1620000 | 200 | 8100.0x | +| `fluxapps.spawnDeferrals.staticIpMs.standard` | 3420000 | 400 | 8550.0x | +| `fluxapps.spawnDeferrals.targetedNodesMs.enterprise` | 1800000 | 150 | 12000.0x | +| `fluxapps.spawnDeferrals.targetedNodesMs.standard` | 3420000 | 300 | 11400.0x | +| `fluxapps.spawnDelayMultiplier` | 1 | 0.002 | 500.0x | +| `fluxapps.spawnReconfirmDelayMs` | 7500000 | 30000 | 250.0x | +| `fluxapps.syncResponseThrottleMs` | 300000 | 10000 | 30.0x | +| `fluxapps.syncTimeoutMs` | 120000 | 30000 | 4.0x | +| `fluxapps.tempMsgTtlS` | 3600 | 300 | 12.0x | +| `fluxapps.wsHandshakeTimeoutMs` | 10000 | 5000 | 2.0x | +| `peers.wsMaxMissedPongs` | 3 | 2 | 1.5x | +| `peers.wsPingIntervalMs` | 15000 | 2000 | 7.5x | +| `syncthing.monitorIntervalMs` | 30000 | 3000 | 10.0x | +| `syncthing.stallNudgeAfterMs` | 180000 | 6000 | 30.0x | +| `syncthing.stallNudgeMaxIntervalMs` | 900000 | 12000 | 75.0x | +| `syncthing.stallRemoveMinNudges` | 3 | 2 | 1.5x | +| `syncthing.stallRemoveMinWindowMs` | 1200000 | 30000 | 40.0x | +| `system.bootDaemonTimeoutMs` | 300000 | 30000 | 10.0x | +| `system.bootSyncTimeoutMs` | 300000 | 30000 | 10.0x | +| `system.heartbeatIntervalMs` | 30000 | 10000 | 3.0x | + +## Knobs with no reader - the harness value changes nothing + +These are set in `shared.js` and look like compression. The code never reads them; +it uses a hardcoded constant instead, at the production value, in every suite. + +| key | production | harness | apparent factor | what the code actually uses | +|---|---:|---:|---:|---| +| `fluxapps.hashSyncIntervalMs` | 1800000 | 30000 | 60.0x | literal `30 * 60 * 1000` (serviceManager.js:679) | +| `fluxapps.removalSpacingMs` | 60000 | 1000 | 60.0x | nothing - no equivalent | +| `fluxapps.spawnDelayMs` | 0 | 10000 | n/a | nothing - no equivalent | + +## Values production and the harness share + +Deliberately uncompressed - a suite depending on one of these is depending on the +production number. + +`syncthing.port` = 8384 · `fluxapps.maxAppsPerNode` = 200 · `fluxapps.blocksLasting` = 22000 · `fluxapps.newMinBlocksAllowance` = 100 · `fluxapps.daemonPONFork` = 2020000 · `fluxapps.hashSyncResponseTimePerHashMs` = 150 · `fluxapps.hashSyncBufferMs` = 5000 · `fluxapps.hashSyncMaxRounds` = 4 · `fluxapps.hashSyncPeersPerRound` = 3 · `fluxapps.installingTtlS` = 900 · `fluxapps.installErrorTtlS` = 86400 · `fluxapps.installation.probability` = 100 · `fluxapps.removal.probability` = 25 · `fluxapps.redeploy.probability` = 2 diff --git a/test-infra/config/generate-configs.js b/test-infra/config/generate-configs.js index c147675789..3f55cbeee4 100644 --- a/test-infra/config/generate-configs.js +++ b/test-infra/config/generate-configs.js @@ -21,6 +21,9 @@ function databaseConfig(prefix) { benchmark: 'benchmark', appTamperingEvents: 'apptamperingevents', nodeStartupTracker: 'nodestartuptracker', + policyDocuments: 'policydocuments', + ipRanges: 'ipranges', + nodeLocations: 'nodelocations', }, }, daemon: { diff --git a/test-infra/config/node-01/default.js b/test-infra/config/node-01/default.js index 4c7a825cbb..39f896f915 100644 --- a/test-infra/config/node-01/default.js +++ b/test-infra/config/node-01/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-02/default.js b/test-infra/config/node-02/default.js index 31a185947a..cae9a1948d 100644 --- a/test-infra/config/node-02/default.js +++ b/test-infra/config/node-02/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-03/default.js b/test-infra/config/node-03/default.js index 9766c4e6de..f0037c5f22 100644 --- a/test-infra/config/node-03/default.js +++ b/test-infra/config/node-03/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-04/default.js b/test-infra/config/node-04/default.js index f31d998ac2..c2479c3224 100644 --- a/test-infra/config/node-04/default.js +++ b/test-infra/config/node-04/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-05/default.js b/test-infra/config/node-05/default.js index d47ac717ce..3ad219ad5c 100644 --- a/test-infra/config/node-05/default.js +++ b/test-infra/config/node-05/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-06/default.js b/test-infra/config/node-06/default.js index 2dacd65112..a789f25c4f 100644 --- a/test-infra/config/node-06/default.js +++ b/test-infra/config/node-06/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-07/default.js b/test-infra/config/node-07/default.js index 07bef67661..18493ac6d2 100644 --- a/test-infra/config/node-07/default.js +++ b/test-infra/config/node-07/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-08/default.js b/test-infra/config/node-08/default.js index bb048e2b25..ccb8149bf5 100644 --- a/test-infra/config/node-08/default.js +++ b/test-infra/config/node-08/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-09/default.js b/test-infra/config/node-09/default.js index 9686714990..340c8c27de 100644 --- a/test-infra/config/node-09/default.js +++ b/test-infra/config/node-09/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-10/default.js b/test-infra/config/node-10/default.js index b5a34ce2dc..5ced39261b 100644 --- a/test-infra/config/node-10/default.js +++ b/test-infra/config/node-10/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-11/default.js b/test-infra/config/node-11/default.js index 241044ae41..18808983ee 100644 --- a/test-infra/config/node-11/default.js +++ b/test-infra/config/node-11/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-12/default.js b/test-infra/config/node-12/default.js index df9e210bbc..7302c9a8e4 100644 --- a/test-infra/config/node-12/default.js +++ b/test-infra/config/node-12/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-13/default.js b/test-infra/config/node-13/default.js index 56b118896e..3031d2b280 100644 --- a/test-infra/config/node-13/default.js +++ b/test-infra/config/node-13/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-14/default.js b/test-infra/config/node-14/default.js index 748c951a33..2e9fe95f8a 100644 --- a/test-infra/config/node-14/default.js +++ b/test-infra/config/node-14/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-15/default.js b/test-infra/config/node-15/default.js index e1c7e89e4b..592c3ad237 100644 --- a/test-infra/config/node-15/default.js +++ b/test-infra/config/node-15/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/node-16/default.js b/test-infra/config/node-16/default.js index 08390d7ad3..8bb85b5aa2 100644 --- a/test-infra/config/node-16/default.js +++ b/test-infra/config/node-16/default.js @@ -16,7 +16,10 @@ module.exports = { "geolocation": "geolocation", "benchmark": "benchmark", "appTamperingEvents": "apptamperingevents", - "nodeStartupTracker": "nodestartuptracker" + "nodeStartupTracker": "nodestartuptracker", + "policyDocuments": "policydocuments", + "ipRanges": "ipranges", + "nodeLocations": "nodelocations" } }, "daemon": { diff --git a/test-infra/config/shared.js b/test-infra/config/shared.js index 46a133c862..a7390d8e22 100644 --- a/test-infra/config/shared.js +++ b/test-infra/config/shared.js @@ -4,7 +4,15 @@ module.exports = { fluxTeamFluxID: '19J4Ef396goaQhrqgNLTFvtCXYqjFAx2Js', daemon: { host: '198.18.0.3' }, benchmark: { host: '198.18.0.3' }, - upnp: { gatewayUrl: '', nodeIp: '' }, + // upnpService builds its client at module load, and with no gateway URL the client + // discovers one by SSDP - every node multicasting to 239.255.255.250:1900 for the life of + // the run. Naming a gateway replaces discovery with a fixed device, which is the point of + // the hook. The stub serves a device description with no WAN connection service, so + // support verification fails exactly as it does today and no node believes it has UPnP - + // the behaviour is unchanged, only the searching stops. + // nodeIp stays empty as in production: it is the node's own address for a port mapping, + // it cannot be a shared constant, and no mapping is ever made because verification fails. + upnp: { gatewayUrl: 'http://198.18.0.6:3000/upnp/device.xml', nodeIp: '' }, // Empty disables analytics. The app default is the live cloudaudit endpoint and // the fleet network has egress, so without this a run reports suite activity - // generated app names, fixture identities, 198.18.x addresses - as real traffic. @@ -21,6 +29,10 @@ module.exports = { stallNudgeMaxIntervalMs: 12000, stallRemoveMinWindowMs: 30000, stallRemoveMinNudges: 2, + // The repository a legacy node installs syncthing from, served by the external + // stub out of the node image rather than by apt.syncthing.net. + aptSourceUrl: 'http://198.18.0.6:3000/apt/', + releaseKeyUrl: 'http://198.18.0.6:3000/apt/keyring.gpg', }, system: { bootIdPath: '/tmp/flux-boot-config/boot-id', @@ -47,7 +59,21 @@ module.exports = { }, geolocation: { ipApiBaseUrl: 'http://198.18.0.6:3000', - statsApiBaseUrl: 'http://198.18.0.6:3000', + }, + stats: { baseUrl: 'http://198.18.0.6:3000' }, + pricing: { + fluxRatesBaseUrl: 'http://198.18.0.6:3000', + coingeckoBaseUrl: 'http://198.18.0.6:3000', + }, + mongodb: { signingKeyBaseUrl: 'http://198.18.0.6:3000' }, + // the stub serves iplocation.bin.gz, so harness nodes exercise the real + // table reader rather than skipping it. Its default artifact puts the whole + // harness range in ONE organisation, which is the single-fault-domain + // posture the tableless fallback produced - suites written against that keep + // their meaning. POST /iplocation {domains:n} to the stub's control port + // splits the fleet n ways. Nothing here ever calls out to github. + policy: { + baseUrl: 'http://198.18.0.6:3000', }, fluxapps: { minOutgoing: 4, @@ -64,9 +90,24 @@ module.exports = { defaultSwap: 0, appSyncPeerThreshold: 2, appSyncDegradedThreshold: 1, - appSyncMinPeerUptime: 0, appSyncMinCompletions: 1, + appSyncMinPeerUptime: 0, syncTimeoutMs: 30000, + // 10 blocks at the stub's 5s tick, so 50s, against production's 250 blocks. + // A fleet where every node boots at once has nobody who can answer a state + // sync yet, so the fallback is the road every node takes and 250 blocks is + // 21 minutes of it. + // + // Not lower. Suites that stop the ticker drive a handful of blocks in their + // own setup and several of them require a node to STAY in SYNCING - at 2 + // blocks suite 13's peer-drop case cleared its premise by a single block, + // which is a window rather than a margin. 10 leaves room for a setup to + // drive blocks without deciding the test. + // + // A suite that wants a peer able to answer from the start asks for + // syncedNodes; a suite measuring this budget itself declares the production + // value, as suite 19 does. + appSyncFallbackMinutes: 5, hashSyncMaxRetries: 2, hashSyncRetryMs: 10000, hashSyncSettleMs: 2000, @@ -87,13 +128,53 @@ module.exports = { bootDelayMultiplier: 0.01, spawnDelayMs: 10000, removalSpacingMs: 1000, - locationTtlS: 300, - installingTtlS: 60, - installErrorTtlS: 300, + // Per-document expiry for the ephemeral app collections, in seconds. These + // three also serve as GOSSIP ACCEPTANCE WINDOWS - messageStore drops an + // incoming broadcast whose broadcastedAt is older than the window - so a + // value below what the fleet takes to produce and deliver a message does + // not make a suite faster, it makes peers refuse each other. + // + // They read as 300/60/300 from the day the harness was first stood up until + // 2026-08-20 and none of them ever took effect: the config keys were wired + // to collection-level TTL indexes that were dropped when expiry moved + // per-document, so every suite ran on the production durations while this + // file claimed otherwise. The numbers below are derived; the old ones were + // round guesses that nothing could contradict. + // + // A running-app location record, and with it the announce interval: + // appConstants derives that from this number, so compressing this one + // compresses both and the ratio between them - how many announcements a + // node may miss before its apps look gone - is structural rather than a + // pairing this file has to hold. 63s gives a 30s announce, the value this + // file used to carry by hand. + locationTtlS: 63, + // NOT the announce ratio, deliberately. What locationTtlS is coupled to is + // a compressed clock; what this is measured across is a NODE BOOT, and the + // harness does not compress boots. Measured on cindy under a MAXN=6 gate: a + // fixture pinning 300s of downtime was read by the node as 316s, so a boot + // costs 16s of drift. At production's 120x this key would be 3.5s - smaller + // than the drift - and the within-the-window test could never pass. + // + // So it is bounded by what it must outlive, like installingTtlS below: + // comfortably above the 16s drift, comfortably below locationTtlS's 63s so + // the ordering holds and a clean shutdown still gets a grace the running + // expiry does not pre-empt. coupled-knobs.js asserts both ends. + sigtermExpiryS: 30, + // NOT compressed, and not compressible by a ratio. What this must outlive is + // an install, and the harness does not compress installs - they are real + // image pulls and real container starts. The suites' own budgets say so: + // waitForAppInstalled is given 120s routinely and 300s at the top end. At + // the old 60s the marker expired mid-install and peers rejected any + // installing claim older than a minute; suite 78 reads exactly that claim. + installingTtlS: 900, + // NOT compressed, for the same reason: the errors it accumulates come from + // real failed installs, and suite 27 waits for five of them to reach the + // network-wide threshold. No knob paces that, so there is no ratio to hold. + installErrorTtlS: 86400, tempMsgTtlS: 300, hashSyncIntervalMs: 30000, - peerNotifyIntervalMs: 30000, cpuCheckIntervalMs: 30000, + statsSampleIntervalMs: 2000, portRestoreIntervalMs: 30000, imageComplianceIntervalMs: 60000, forceRemovalIntervalMs: 120000, @@ -122,6 +203,29 @@ module.exports = { }, spawnDelayMultiplier: 0.002, daemonInfoIntervalMs: 5000, + // The poll is NOT how often the chain is asked - pollForNewBlocks reads a + // height cached by daemonServiceMiscRpcs and refreshed on its own + // daemonInfoIntervalMs timer. It is the rate at which the node works + // through blocks once it knows it is behind, and its share of the + // tip-refresh window is what decides whether a block is still the tip when + // it is processed - which is what gates every maintenance pass hung off + // block processing. + // + // Both clocks are already 1:1 with block time at either scale - 30s/30s in + // production, 5s/5s here - so one block arrives per window either way. What + // has to hold is the poll's share of that window: + // + // production 5000 / 30000 = 16.7% + // here 833 / 5000 = 16.7% + // + // 250ms was three times MORE forgiving than production, which is the wrong + // direction: a node too slow to clear a window's block, so blocks bunch and + // maintenance is skipped, is a production failure the harness would never + // show. Suite 55 hit that failure twice and it was read as a harness + // artefact; making the harness faster until it stops happening is the same + // move one layer down. Measured at 4.8s a block at the old hardcoded 5000, + // which was poll-dominated - a block sat unnoticed for a whole window. + explorerPollIntervalMs: 833, explorerSyncRetryMs: 5000, explorerDeepRestoreBlocks: 0, imageUpdateCheckIntervalMs: 5000, @@ -133,6 +237,11 @@ module.exports = { masterSlaveIntervalMs: 3000, // compressed g: FDM election cycle (prod 30s) installation: { probability: 100, delay: 5 }, removal: { probability: 25, delay: 5 }, - redeploy: { probability: 2, delay: 1, composedDelay: 1 }, + // 1 = every pass. `Math.floor(Math.random() * probability) === 0` gates the + // reinstall of an app whose on-chain spec has changed; production spreads + // that over the fleet at 50% so a spec change does not restart every + // instance at once. A suite has one app and a bounded wait, and nothing here + // has an obsolete spec unless a suite deliberately made one. + redeploy: { probability: 1, delay: 1, composedDelay: 1 }, }, }; diff --git a/test-infra/daemon-stub/index.js b/test-infra/daemon-stub/index.js index b7a4aa7b0c..01f8bca832 100644 --- a/test-infra/daemon-stub/index.js +++ b/test-infra/daemon-stub/index.js @@ -24,13 +24,26 @@ const DAEMON_SUBVERSION = '/Flux:6.1.0/'; const BENCH_VERSION = '6.3.1'; const FLUX_VERSION = '8.0.0'; -let currentHeight = Number(process.env.INITIAL_HEIGHT) || 2100000; +// No fallback on purpose. This was a third copy of the chain start and it was +// the stale one, so any path that started the stub without the variable put the +// whole chain below the v8 fork and ran every suite on the pre-fork branch of +// every rule keyed on it - silently, and looking exactly like a product bug. +// The chain start has one home (runner/framework/chain-start.cjs) and test-env +// always passes it; a stub started without it is misconfigured, not defaulted. +if (!process.env.INITIAL_HEIGHT) { + throw new Error('daemon-stub: INITIAL_HEIGHT is required - the chain start comes from runner/framework/chain-start.cjs, never from a default here'); +} +let currentHeight = Number(process.env.INITIAL_HEIGHT); let deterministicNodeList = []; let originalNodeList = []; let pendingBlocks = []; const nodeStatusOverrides = new Map(); const rpcFailures = new Map(); +// ip -> false. Absent means ArcaneOS, which is what 85% of the real fleet runs +// and what every suite that does not care about this should see: a node reading +// systemsecure=true is never a residential-DOS target. +const nonArcaneNodes = new Map(); const requestJournal = []; const MAX_JOURNAL_SIZE = 10000; @@ -58,7 +71,101 @@ try { function nodeBySourceIp(sourceIp) { const clean = sourceIp.replace('::ffff:', ''); - return deterministicNodeList.find((n) => n.ip.split(':')[0] === clean) || null; + const listed = deterministicNodeList.find((n) => n.ip.split(':')[0] === clean); + if (listed) return listed; + // A node that has really moved arrives from its NEW address while its list entry + // still says where it was - which is exactly the state an address change puts it + // in, before the chain catches up. It is the same node, and failing to recognise + // it here costs it its own identity: every answer about it falls back to the + // not-found defaults, so it reads its own address as 127.0.0.1, decides it is not + // in the confirmed list, and skips the availability check that would have told it + // what happened. + for (const [realIp, override] of reportedAddresses) { + if (String(override.reported).split(':')[0] === clean) { + return deterministicNodeList.find((n) => n.ip.split(':')[0] === realIp) || null; + } + } + return null; +} + +// Where a node really is, versus where the network is told it is. +// +// A node's list entry carries ONE address and the stub uses it for two different +// jobs: matching the address a request ARRIVES from, to work out whose request it +// is, and answering every chain-facing question about where that node lives. +// Moving the second by editing the entry breaks the first - the node stops being +// findable by its own requests - so the override sits beside the list, keyed by +// where the node really is, and only the ANSWERS move. The container keeps its +// address and nothing is renumbered underneath it. +// +// Every answer that names a node's address reads through here: getbenchmarks +// (which is what a node's own getLocalSocketAddress reads, and so what it believes +// about itself), getpublicip, its status, and the deterministic list the network is +// served. One dial rather than one per RPC - a suite that moves an address means +// all of them, and a per-RPC dial is how a suite comes to move an answer the +// product never reads. +const reportedAddresses = new Map(); // real ip -> { reported, scope } + +// Two facts, and a real address change separates them for as long as it takes the +// chain to catch up: where the network says the node lives, and what benchmark's +// public-IP probe reads right now. `all` moves both. `publicip` moves only the +// probe - benchmark still reports the old address as the node's own, the node is +// still listed there, and only getpublicip has noticed. That is the state a node +// is actually in when its address moves, and it is what lets it detect the move at +// all: the two answers disagreeing IS the detection. +function overrideFor(node) { + if (!node) return null; + return reportedAddresses.get(node.ip.split(':')[0]) || null; +} + +// The address the network believes this node has. Unmoved under `publicip`. +function reportedAddressOf(node) { + if (!node) return null; + const o = overrideFor(node); + return o && o.scope === 'all' ? o.reported : node.ip; +} + +// What benchmark's public-IP probe answers, which moves under either scope. +function publicIpAnswerFor(node) { + if (!node) return null; + const o = overrideFor(node); + return (o ? o.reported : node.ip).split(':')[0]; +} + +// Nodes hidden from their PEERS' view of the network, but never from their own. +// +// This is what makes a node unreachable to the fleet deterministically. Asked +// whether it can reach a node, a peer consults its node list FIRST and answers +// "not available" outright when the address is not in it - no probe, no timeout, +// no dependence on how fast anything answers. Blocking packets instead only makes +// the probe slow, and a slow probe is a different answer from an absent one: the +// asker times out on the PEER and never learns it is unreachable. +// +// The node still sees ITSELF, because it has its own confirmed-list gate to pass +// before it will run the availability check at all. Hiding it from everyone, +// itself included, stops the check ever running. +const hiddenFromPeers = new Set(); // real ip -> hidden from other nodes' lists + +// The list as a given requester sees it: every entry at its reported address, +// minus any node hidden from that requester. A run with nothing hidden and nothing +// moved serves the very same objects it always did. +function publishedNodeList(sourceIp) { + const asker = sourceIp ? sourceIp.replace('::ffff:', '') : null; + const visible = hiddenFromPeers.size + ? deterministicNodeList.filter((n) => { + const ip = n.ip.split(':')[0]; + return !hiddenFromPeers.has(ip) || ip === asker; + }) + : deterministicNodeList; + return publishReported(visible); +} + +function publishReported(list) { + if (!reportedAddresses.size) return list; + return list.map((n) => { + const o = reportedAddresses.get(n.ip.split(':')[0]); + return o && o.scope === 'all' ? { ...n, ip: o.reported } : n; + }); } const rpcHandlers = { @@ -159,8 +266,8 @@ const rpcHandlers = { getmempoolinfo: () => ({ size: 0, bytes: 0, usage: 0 }), getrawmempool: () => [], - viewdeterministiczelnodelist: () => deterministicNodeList, - viewdeterministicfluxnodelist: () => deterministicNodeList, + viewdeterministiczelnodelist: (params, sourceIp) => publishedNodeList(sourceIp), + viewdeterministicfluxnodelist: (params, sourceIp) => publishedNodeList(sourceIp), getzelnodestatus: (params, sourceIp) => { const node = nodeBySourceIp(sourceIp); @@ -171,7 +278,7 @@ const rpcHandlers = { collateral: node ? node.collateral : 'COutPoint(0000000000000000000000000000000000000000000000000000000000000000, 0)', txhash: node ? node.txhash : '0000000000000000000000000000000000000000000000000000000000000000', outidx: node ? node.outidx : '0', - ip: node ? node.ip : '127.0.0.1', + ip: node ? reportedAddressOf(node) : '127.0.0.1', network: '', added_height: node ? node.added_height : currentHeight - 1000, confirmed_height: node ? node.confirmed_height : currentHeight - 500, @@ -204,8 +311,8 @@ const rpcHandlers = { getdoslist: () => [], getstartlist: () => [], - listfluxnodes: () => deterministicNodeList, - listzelnodes: () => deterministicNodeList, + listfluxnodes: (params, sourceIp) => publishedNodeList(sourceIp), + listzelnodes: (params, sourceIp) => publishedNodeList(sourceIp), getrawtransaction: (params) => { const txid = params[0]; @@ -282,7 +389,7 @@ const benchHandlers = { }; const s = specs[tier] || specs.cumulus; return { - ipaddress: node ? node.ip : '127.0.0.1', + ipaddress: node ? reportedAddressOf(node) : '127.0.0.1', cores: s.cores, ram: s.ram, ssd: s.ssd, @@ -297,6 +404,9 @@ const benchHandlers = { bench_version: BENCH_VERSION, flux_version: FLUX_VERSION, architecture: 'amd64', + // The ArcaneOS attestation residentialNodeDosService reads. A node absent + // from the override map is attested. + systemsecure: !nonArcaneNodes.has(node ? node.ip.split(':')[0] : ''), thunder: false, real_cores: s.cores, speed: 3000, @@ -311,7 +421,7 @@ const benchHandlers = { getpublicip: (params, sourceIp) => { const node = nodeBySourceIp(sourceIp); - return node ? node.ip.split(':')[0] : '127.0.0.1'; + return node ? publicIpAnswerFor(node) : '127.0.0.1'; }, getpublickey: (params, sourceIp) => { @@ -444,6 +554,7 @@ control.get('/state', (req, res) => { tickerRunning: tickerHandle !== null, statusOverrides: nodeStatusOverrides.size, rpcFailures: rpcFailures.size, + nonArcaneNodes: [...nonArcaneNodes.keys()], }); }); @@ -495,7 +606,12 @@ control.post('/advance-block', (req, res) => { if (block) { block.height = block.height || currentHeight; block.hash = block.hash || `000000000000stub${currentHeight}`; - block.confirmations = 1; + // Computed live, the way a synthesized block computes it at the top of this + // file. Stamping 1 at push time made a block carrying transactions read as + // the chain tip no matter how far behind the node was, while an empty block + // did not - so the one class of block whose isSynced skip matters most, an + // app registration, was the one class the stub had made unskippable. + block.confirmations = currentHeight - block.height + 1; block.tx = [...(block.tx || []), ...txs]; pendingBlocks.push(block); } else if (txs.length > 0) { @@ -530,6 +646,56 @@ control.post('/set-node-list', (req, res) => { res.json({ nodeCount: deterministicNodeList.length }); }); +// Move where a node is said to be. `:ip` is where it really is - the address its +// requests arrive from, which never changes - and `reported` is the address that +// then answers for it. Sending no `reported` puts it back to the truth. +// +// `scope: 'all'` (default) moves every chain-facing answer together: benchmark's +// reply about the node itself, getpublicip, its status, and its list entry. That is +// an address change already settled everywhere. +// +// `scope: 'publicip'` moves only benchmark's public-IP probe. Everything else still +// says the node is where it was. That is the state a node is in the moment its +// address moves and before the chain has caught up - and it is the only state in +// which a node can DETECT the move, because detection is precisely those two +// answers disagreeing. +// Hide a node from its PEERS' view of the network, or put it back. The node keeps +// seeing itself, which is what lets it still run the availability check that then +// comes back "unreachable" - deterministically, because a peer answers from its +// list rather than from a probe. +control.post('/node-visibility/:ip', (req, res) => { + const key = String(req.params.ip).split(':')[0]; + const node = deterministicNodeList.find((n) => n.ip.split(':')[0] === key); + if (!node) return res.status(404).json({ error: `no node in the list at ${key}` }); + const hidden = (req.body || {}).hidden !== false; + if (hidden) hiddenFromPeers.add(key); + else hiddenFromPeers.delete(key); + return res.json({ node: key, hiddenFromPeers: hidden }); +}); + +control.post('/node-address/:ip', (req, res) => { + const key = String(req.params.ip).split(':')[0]; + const node = deterministicNodeList.find((n) => n.ip.split(':')[0] === key); + if (!node) return res.status(404).json({ error: `no node in the list at ${key}` }); + + const { reported, scope = 'all' } = req.body || {}; + if (!reported) { + reportedAddresses.delete(key); + return res.json({ node: key, reported: node.ip, scope: null }); + } + if (scope !== 'all' && scope !== 'publicip') { + return res.status(400).json({ error: `scope must be 'all' or 'publicip', got '${scope}'` }); + } + + // A bare address keeps the node's own port. The api port is what its peers reach + // it on, and moving that is a different change from moving the address. + const value = String(reported); + const realPort = node.ip.split(':')[1]; + const full = value.includes(':') || !realPort ? value : `${value}:${realPort}`; + reportedAddresses.set(key, { reported: full, scope }); + return res.json({ node: key, reported: full, scope }); +}); + control.post('/queue-app-tx', (req, res) => { const { appHash } = req.body; if (!appHash) return res.status(400).json({ error: 'appHash required' }); @@ -617,6 +783,23 @@ control.post('/node-tier/:ip', (req, res) => { return res.json({ ip, tier, collateral: amounts[tier] }); }); +// -- ArcaneOS attestation control -- + +control.post('/system-secure/:ip', (req, res) => { + const { secure } = req.body; + if (typeof secure !== 'boolean') { + return res.status(400).json({ error: 'secure must be a boolean' }); + } + if (secure) nonArcaneNodes.delete(req.params.ip); + else nonArcaneNodes.set(req.params.ip, false); + return res.json({ ip: req.params.ip, systemsecure: secure }); +}); + +control.delete('/system-secure', (req, res) => { + nonArcaneNodes.clear(); + res.json({ cleared: true }); +}); + // -- RPC failure simulation -- control.post('/rpc-fail/:ip', (req, res) => { @@ -675,6 +858,8 @@ control.delete('/seed-data', (req, res) => { control.post('/reset', (req, res) => { nodeStatusOverrides.clear(); + reportedAddresses.clear(); + hiddenFromPeers.clear(); rpcFailures.clear(); deterministicNodeList = [...originalNodeList]; pendingBlocks = []; diff --git a/test-infra/docker-compose.single.yml b/test-infra/docker-compose.single.yml index b9a09f8f5f..ece8ec81ce 100644 --- a/test-infra/docker-compose.single.yml +++ b/test-infra/docker-compose.single.yml @@ -42,6 +42,11 @@ services: FLUXD_PORT: "16124" BENCHD_PORT: "16224" CONTROL_PORT: "18232" + # Required, no default: the chain start has one home + # (runner/framework/chain-start.cjs) and a defaulted copy here goes stale + # and silently runs every suite pre-fork. Launch with: + # INITIAL_HEIGHT=$(node -p "require('./runner/framework/chain-start.cjs').DEFAULT_INITIAL_HEIGHT") docker compose up + INITIAL_HEIGHT: "${INITIAL_HEIGHT:?set from runner/framework/chain-start.cjs - see the comment above this line}" volumes: - ./fixtures:/fixtures healthcheck: diff --git a/test-infra/docker-compose.yml b/test-infra/docker-compose.yml index 13a86e1adf..0c09652973 100644 --- a/test-infra/docker-compose.yml +++ b/test-infra/docker-compose.yml @@ -73,6 +73,11 @@ services: BENCHD_PORT: "16224" CONTROL_PORT: "18232" TICKER_AUTOSTART: "false" + # Required, no default: the chain start has one home + # (runner/framework/chain-start.cjs) and a defaulted copy here goes stale + # and silently runs every suite pre-fork. Launch with: + # INITIAL_HEIGHT=$(node -p "require('./runner/framework/chain-start.cjs').DEFAULT_INITIAL_HEIGHT") docker compose up + INITIAL_HEIGHT: "${INITIAL_HEIGHT:?set from runner/framework/chain-start.cjs - see the comment above this line}" volumes: - ./fixtures:/fixtures healthcheck: diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index bdedbcd6e8..ad4d2656f4 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -3,6 +3,32 @@ set -e ip addr add 169.254.43.43/32 dev lo 2>/dev/null || true +# A default route, or deliberately none - declared by the suite, never inherited +# from the topology. +# +# The harness network is created Internal, so docker gives the container no +# default route at all. FluxOS decides whether this node holds a fixed public +# address by looking for one (fluxNetworkHelper.hasPublicIpOnInterface reads +# /proc/net/route), so left to the wiring EVERY node reads DYNAMIC - which is how +# suite 21's static_ip deferrals silently stopped firing. +# +# This restores the FACT, not connectivity: an internal network's gateway +# forwards nothing outward, so the fleet stays exactly as isolated as Internal +# makes it. Installed here because a node reads its address once during boot - +# anything applied after the fleet is up is never seen. +# +# NOT swallowed. A `|| true` here would make the one failure that matters - +# the route not being installable - look exactly like a node that was never +# asked for one, and the suite would then fail somewhere far away on a +# classification it could not explain. +if [ -n "$FLUX_E2E_DEFAULT_ROUTE" ]; then + if ! ip route replace default via "$FLUX_E2E_DEFAULT_ROUTE"; then + echo "ERROR: could not install default route via $FLUX_E2E_DEFAULT_ROUTE;" \ + "this node would read DYNAMIC and any static-IP assertion would fail" >&2 + exit 1 + fi +fi + # App installs mount each app's FLUXFSVOL via `mount -o loop`. Loop devices are a # shared host-kernel resource (not namespaced); the kernel default pool (max_loop, # typically 8) is small and on-demand creation races under concurrent installs, so a @@ -20,7 +46,12 @@ mkdir -p /dat/var/lib/fluxd \ /dat/usr/lib/fluxwatchdog \ /mnt/appdata/flux-apps -cp /flux/test-infra/fixtures/syncthing-config.xml /dat/usr/lib/syncthing/config.xml 2>/dev/null || true +# In stub mode FluxOS only needs somewhere to read an API key from; the calls +# themselves go to the shared stub. In binary mode the config is syncthing's own +# and this fixture must not be in the way of it. +if [ "$FLUX_SYNCTHING_MODE" != "binary" ]; then + cp /flux/test-infra/fixtures/syncthing-config.xml /dat/usr/lib/syncthing/config.xml 2>/dev/null || true +fi # Overlay test config into ZelBack/config/ so app.js loads it naturally. # app.js hardcodes NODE_CONFIG_DIR to ZelBack/config/ (cannot be overridden @@ -30,14 +61,82 @@ if [ -n "$NODE_CONFIG_DIR" ] && [ -d "$NODE_CONFIG_DIR" ]; then cp "$(dirname "$NODE_CONFIG_DIR")/shared.js" /flux/ZelBack/ 2>/dev/null || true fi -if [ "$FLUX_DISCOVERY_AUTOSTART" = "true" ]; then - sed -i 's/discoveryAutostart: false/discoveryAutostart: true/' /flux/ZelBack/shared.js +# The runner's own overrides arrive as JSON and are merged OVER the per-node file +# copied above, which is where the per-node database names come from - replacing +# that file rather than merging would take them with it. +# +# They used to arrive as NODE_CONFIG. The config package merges that variable over +# every file whatever directory is pinned, so it could redirect any endpoint +# without touching the directory fluxbenchd hashes - the one change tamper +# detection cannot see. The entry points delete it now, and this carries the same +# content to the same place through a file instead. +if [ -n "$FLUX_TEST_CONFIG" ]; then + node -e ' + const fs = require("fs"); + const target = "/flux/ZelBack/config/local.js"; + const base = fs.existsSync(target) ? require(target) : {}; + const isPlain = (v) => v && typeof v === "object" && !Array.isArray(v); + const merge = (a, b) => { + const out = { ...a }; + for (const [k, v] of Object.entries(b)) out[k] = isPlain(v) && isPlain(a[k]) ? merge(a[k], v) : v; + return out; + }; + const merged = merge(base, JSON.parse(process.env.FLUX_TEST_CONFIG)); + fs.writeFileSync(target, `module.exports = ${JSON.stringify(merged, null, 2)};\n`); + ' fi -# Syncthing listens on apiport+2 in production. The availability checker -# tests this port. Forward it to the syncthing stub's API port. +# The image ships these installed, which is the state a node is in on every boot +# after its first. A suite that wants to exercise the install asks for a node +# without them, and gets one here - before FluxOS starts, so monitorSystem() +# meets the same absence a real first boot does. +# +# Purge, not remove: a removed package leaves its configuration behind and +# dpkg-query reports `deinstall ok config-files`, which is neither installed nor +# absent. getPackageVersion returns '' for that as well as for absent, so the +# node would behave plausibly while sitting in a state no real node is ever in. +if [ "$FLUX_APT_SEEDED" = "false" ]; then + DEBIAN_FRONTEND=noninteractive apt-get purge -y chrony syncthing netcat-openbsd >/dev/null 2>&1 || true +fi + +# A source apt cannot reach, ALONGSIDE the good one rather than instead of it. +# apt-get update then exits non-zero exactly as it does on a real node behind an +# unreachable mirror, an expired key or a DNS blip - while the packages queued +# behind that failure stay installable from the repository the image built, so a +# node that survives the failure still finishes its checks. Replacing the good +# source instead would fail the installs too, and prove only that a broken node +# stays broken. +if [ "$FLUX_APT_BAD_SOURCE" = "true" ]; then + echo "deb [trusted=yes] file:///opt/flux-apt-repo-does-not-exist ubuntu main" \ + > /etc/apt/sources.list.d/flux-e2e-unreachable.list +fi + +# Syncthing listens on apiport+2 in production. The availability checker tests +# that port. SYNCTHING_LISTEN_PORT=$((${FLUX_API_PORT:-16127} + 2)) -if [ -n "$FLUX_SYNCTHING_HOST" ]; then +if [ "$FLUX_SYNCTHING_MODE" = "binary" ]; then + # A real daemon, one per node. Nothing here writes syncthing's config: it + # generates its own identity on first run, which is what gives each node a + # distinct device id, and FluxOS then sets discovery off, NAT off and + # listenAddresses to apiport+2 through the API exactly as it does on a node. + # The endpoint is decided by the runner through the local.js written above, + # which node-config loads last. No socat either way: whoever + # starts the daemon, it binds apiport+2 itself. + # + # WHO starts it depends on the node type, and SYNCTHING_PATH is the same + # signal FluxOS reads to decide. Set, FluxOS takes the node for ArcaneOS and + # leaves supervision to the OS - so the harness stands in for the OS here. + # Unset, it is a legacy node and FluxOS supervises the daemon itself, so this + # must keep its hands off or there would be two. + if [ -n "$SYNCTHING_PATH" ]; then + # the flags a real Arcane node is supervised with, read off a live one + mkdir -p /dat/var/log + nohup syncthing --no-browser --allow-newer-config --home "$SYNCTHING_PATH" \ + --logfile /dat/var/log/syncthing.log --logflags=3 \ + --log-max-old-files=2 --log-max-size=26214400 \ + >/dev/null 2>&1 |'. An app may name its region the way ip-api does rather than by +// code, and this is what lets a node answer such an entry at region granularity +// instead of falling back to the whole country. Index-aligned with the two +// tables above: GEO_REGION_NAMES[k] names GEO_REGIONS[k] in GEO_COUNTRIES[k]. +const GEO_REGION_NAMES = ['Hesse', 'Ile-de-France', 'North Holland', 'Uusimaa', 'Capital']; + +let fillerBytesCache = null; + +/** + * The filler section's row bytes, identical for every artifact: encoded once, + * reused on every regeneration. Filler org indices are 0 and 1 - the two + * filler organisations lead the combined orgs table, so these bytes never + * depend on the fleet split being served. + * @returns {Buffer} + */ +function fillerBytes() { + if (fillerBytesCache) return fillerBytesCache; + const bytes = []; + for (let i = 0; i < GEO_FILLER_ROWS; i += 1) { + writeVarint(bytes, i === 0 ? GEO_FILLER_START : 0); // gap (prevEnd starts at -1) + writeVarint(bytes, 0); // single-address row + writeVarint(bytes, (i % 2) + 1); // filler org index + 1 + writeVarint(bytes, 1); // countries[0] + writeVarint(bytes, 0); // no region + } + fillerBytesCache = Buffer.from(bytes); + return fillerBytesCache; +} + +let lastGenerated = 0; + +/** + * A distinct ISO timestamp for every regeneration. Nodes key their cached + * per-node locations on the table's `generated` and invalidate them when it + * changes, so two artifacts published in the same millisecond must still + * differ or the second split is never seen. + * @returns {string} ISO timestamp + */ +function nextGenerated() { + lastGenerated = Math.max(Date.now(), lastGenerated + 1); + return new Date(lastGenerated).toISOString(); +} + +/** + * Which region each organisation's addresses carry. + * + * Organisation k takes GEO_REGIONS[k % GEO_REGIONS.length] - the region of the + * country the split already gives it - EXCEPT the last organisation, which + * carries none. A regioned fleet therefore holds both nodes whose region the + * table proves and nodes whose region it does not carry, which is what the + * region-pin semantics need on both sides: a pin is satisfied only by a proven + * region, and a region deny catches only a proven region. + * + * Without regions every organisation carries none, which is the artifact every + * other suite sees. + * @param {number} domains How many organisations the fleet is split across + * @param {boolean} withRegions Whether to assign regions at all + * @returns {{table: string[], assigned: object, unassigned: string[]}} + */ +function regionAssignment(domains, withRegions) { + const orgCount = Math.max(domains, 1); + const assigned = {}; + for (let org = 0; org < orgCount; org += 1) { + assigned[org] = withRegions && org !== orgCount - 1 + ? GEO_REGIONS[org % GEO_REGIONS.length] + : null; + } + const taken = new Set(Object.values(assigned).filter((region) => region !== null)); + return { + table: [...GEO_REGIONS], + assigned, + // regions the vocabulary publishes that no address in this artifact claims + unassigned: GEO_REGIONS.filter((region) => !taken.has(region)), + // the name each published region also answers to, so a suite can pin with + // the vocabulary an app actually carries rather than the code + names: Object.fromEntries(GEO_REGIONS.map((code, k) => [code, GEO_REGION_NAMES[k]])), + }; +} + +/** + * An iplocation artifact for the harness range. The same rows feed both served + * representations: this object is what /iplocation.json serves, and + * encodeGeoTable turns it into the format-2 /iplocation.bin.gz. + * + * `domains: 1` (the default) puts the whole fleet in one organisation, which + * is the single-fault-domain posture the tableless fallback produced - suites + * written against that keep their meaning while now exercising the real table + * reader rather than skipping it. + * + * `domains: n` with a `subnet` (`198.18.5`) assigns that /24's addresses to n + * organisations ROUND-ROBIN, one range per address. The harness gives its + * nodes consecutive addresses from .10, so anything coarser than per-address + * puts the whole fleet in one bucket; interleaving is what actually splits it. + * Everything outside that /24 stays in one organisation, so the artifact is a + * few hundred ranges rather than a hundred thousand. + * + * `withRegions` additionally gives every row the region its organisation + * carries (see regionAssignment), as the optional fifth row element. Without + * it the rows stay four elements long and no row claims a region, so both + * representations are byte-identical to what a caller that never asks for + * regions has always been served. + * + * `networkClasses` says which of the organisations run access networks and + * which sell hosting - `{ 0: 'residential', 1: 'hosting' }` keyed by + * organisation INDEX, since that is what a caller controls. Omitted, the + * artifact carries an EMPTY orgClasses section - which is what a real build + * carrying no verdicts publishes, and what every suite that does not care about + * classification should see: an organisation with no verdict is one nothing + * enforces against. + * @param {number} domains How many organisations to split across + * @param {string} [subnet] Dotted /24 prefix to split, e.g. '198.18.5' + * @param {boolean} [withRegions] Whether rows carry a region + * @param {object} [networkClasses] Organisation index -> 'residential'|'hosting' + * @returns {object} artifact in format 1 + */ +function buildIpLocationArtifact(domains, subnet, withRegions = false, networkClasses = null) { + const orgs = Array.from({ length: Math.max(domains, 1) }, (unused, i) => `harness:org-${i}`); + const countries = GEO_COUNTRIES; + const { assigned } = regionAssignment(domains, withRegions); + // a row is [start, end, orgIdx, ccIdx] and, once regions are asked for, + // [start, end, orgIdx, ccIdx, regionIdx] with null for "no region" + const row = (start, end, org, cc) => (withRegions + ? [start, end, org, cc, assigned[org] === null ? null : GEO_REGIONS.indexOf(assigned[org])] + : [start, end, org, cc]); + const v4 = []; + if (domains <= 1 || !subnet) { + v4.push(row(HARNESS_NET_START, HARNESS_NET_END, 0, 0)); + } else { + const [a, b, c] = subnet.split('.').map(Number); + const base = (a * 2 ** 24) + (b * 2 ** 16) + (c * 2 ** 8); + if (base > HARNESS_NET_START) v4.push(row(HARNESS_NET_START, base - 1, 0, 0)); + for (let octet = 0; octet < 256; octet += 1) { + const org = octet % domains; + v4.push(row(base + octet, base + octet, org, org % countries.length)); + } + if (base + 255 < HARNESS_NET_END) v4.push(row(base + 256, HARNESS_NET_END, 0, 0)); + } + return { + format: 1, + generated: nextGenerated(), + sources: { harness: 'stub' }, + countries, + continents: { + DE: 'EU', FR: 'EU', NL: 'EU', FI: 'EU', BH: 'AS', + }, + orgs, + // the vocabulary a real build publishes; which of them any row claims is + // regionAssignment's business + regions: GEO_REGIONS, + regionNames: Object.fromEntries( + GEO_REGIONS.map((code, k) => [`${GEO_COUNTRIES[k]}|${GEO_REGION_NAMES[k]}`, code]), + ), + // Keyed by organisation TOKEN, which is what the header carries and what a + // reader looks up - the caller names an index because that is what it + // controls, and the two are joined here rather than in the suite. + ...(networkClasses ? { + orgClasses: Object.fromEntries( + Object.entries(networkClasses) + .filter(([index]) => orgs[Number(index)]) + .map(([index, klass]) => [orgs[Number(index)], NETWORK_CLASS_CODES[klass]]), + ), + } : {}), + v4, + v6: [], + }; +} + +/** + * Append one unsigned LEB128 varint. Plain arithmetic rather than shifts: + * range bounds run past 2^31 (198.18.0.0 is 3,323,068,416), which the signed + * 32-bit shift operators cannot carry. + * @param {number[]} bytes Output byte list, appended in place + * @param {number} value Non-negative integer + */ +function writeVarint(bytes, value) { + let remaining = value; + while (remaining >= 0x80) { + bytes.push((remaining % 0x80) + 0x80); + remaining = Math.floor(remaining / 0x80); + } + bytes.push(remaining); +} + +/** + * Encode a format-1 artifact as the format-2 wire artifact iplocation.bin.gz. + * + * Layout, little-endian, gzipped whole: magic FLXGEO, version byte 2, u32 + * header length, the UTF-8 JSON header, u32 row count, then five unsigned + * LEB128 varints per row - gap (start - previousEnd - 1, previousEnd starting + * at -1), len (end - start), then org, country and region as their table index + * PLUS ONE, with 0 meaning "none". + * + * Throws on anything a strict reader rejects, so the stub cannot publish bytes + * it presents as well-formed and are not - with one deliberate exception, + * `pad: false`. The padded artifact is the one every suite wants: it meets the + * reader's truncation floor (>= 1,500,000 rows), which is a fleet-integrity + * invariant rather than a knob. `pad: false` drops the filler section and + * publishes the fleet rows alone - a structurally valid artifact whose row + * count is a few hundred, i.e. FLOOR BAIT BY DESIGN, and the only way a + * harness fleet can exercise the floor at all. Everything else about the two + * encodings is identical, header included: the filler organisations still lead + * the orgs table, so fleet row indices do not move and the padded encoding is + * byte-identical to what a caller that never passes `pad` has always been served. + * @param {object} artifact Format-1 artifact + * @param {{pad?: boolean}} [options] pad: false omits the filler rows + * @returns {Buffer} gzipped format-2 bytes + */ +function encodeGeoTable(artifact, { pad = true } = {}) { + const { countries, orgs, v4 } = artifact; + const regions = artifact.regions ?? []; + const header = Buffer.from(JSON.stringify({ + generated: artifact.generated, + sources: artifact.sources, + countries, + continents: artifact.continents, + // the two filler organisations lead, so fleet org indices shift by two + // in the wire artifact - and by nothing anywhere else + orgs: [...GEO_FILLER_ORGS, ...orgs], + regions, + // omitted when the artifact carries no regions, so a suite can publish the + // pre-vocabulary artifact a node held before the section existed + ...(regions.length ? { regionNames: artifact.regionNames ?? {} } : {}), + // omitted when nothing is classified, for the same reason: a build carrying + // no verdicts is a state the reader already handles, and it must stay + // distinguishable from one that carries them + // ALWAYS emitted, including as {}, because that is what the real builder + // does. Omitting it when empty meant the stub covered a shape production + // never produces, and did not cover the one production produces when the + // ledger classifies nothing - the shape every node saw the day the section + // shipped. + orgClasses: artifact.orgClasses ?? {}, + }), 'utf8'); + const preamble = Buffer.alloc(GEO_MAGIC.length + 1 + 4); + preamble.write(GEO_MAGIC, 0, 'ascii'); + preamble.writeUInt8(GEO_FORMAT, GEO_MAGIC.length); + preamble.writeUInt32LE(header.length, GEO_MAGIC.length + 1); + const rowCount = Buffer.alloc(4); + rowCount.writeUInt32LE((pad ? GEO_FILLER_ROWS : 0) + v4.length, 0); + const rows = []; + let previousEnd = pad ? GEO_FILLER_END : -1; + v4.forEach(([start, end, org, cc, region], i) => { + if (!Number.isInteger(start) || !Number.isInteger(end) || end < start || start <= previousEnd) { + throw new Error(`row ${i}: bounds unsorted, overlapping, below the rows already written or not integers`); + } + const indexes = [org === null || org === undefined ? 0 : org + 1 + GEO_FILLER_ORGS.length, (cc ?? -1) + 1, (region ?? -1) + 1]; + const limits = [orgs.length + GEO_FILLER_ORGS.length, countries.length, regions.length]; + indexes.forEach((index, column) => { + if (!Number.isInteger(index) || index < 0 || index > limits[column]) { + throw new Error(`row ${i}: index out of table bounds`); + } + }); + writeVarint(rows, start - previousEnd - 1); + writeVarint(rows, end - start); + indexes.forEach((index) => writeVarint(rows, index)); + previousEnd = end; + }); + const sections = [preamble, header, rowCount]; + if (pad) sections.push(fillerBytes()); + sections.push(Buffer.from(rows)); + return zlib.gzipSync(Buffer.concat(sections)); +} + +/** + * The wire artifact for whatever is being served, and the row count its header + * claims. A caller-supplied malformed artifact (the reject-and-keep suites) has + * no valid format-2 encoding, so the binary route serves its gzipped JSON: + * bytes that fetch cleanly and fail the reader exactly like their JSON + * counterpart - and no row count, because those bytes carry none. + * @param {object|null} artifact Format-1 artifact, or null for no artifact + * @param {{pad?: boolean}} [options] pad: false omits the filler rows + * @returns {{bytes: Buffer|null, rowCount: number|null}} + */ +function encodeIpLocationBinary(artifact, { pad = true } = {}) { + if (!artifact) return { bytes: null, rowCount: null }; + try { + return { + bytes: encodeGeoTable(artifact, { pad }), + rowCount: (pad ? GEO_FILLER_ROWS : 0) + artifact.v4.length, + }; + } catch { + return { bytes: zlib.gzipSync(Buffer.from(JSON.stringify(artifact), 'utf8')), rowCount: null }; + } +} + +/** + * Fetch counters for one artifact route, from zero. Both representations are + * counted separately: the two node lineages sharing this stub fetch different + * ones, and a suite asserting "this node did not download the artifact again" + * must not have its answer moved by the other route. + * @returns {{total: number, ok: number, notModified: number, missing: number}} + */ +function newRouteCounters() { + return { total: 0, ok: 0, notModified: 0, missing: 0 }; +} + +const IPLOCATION_JSON_ROUTE = '/iplocation.json'; +const IPLOCATION_BINARY_ROUTE = '/iplocation.bin.gz'; + +// The apt repository copied out of the node image at build time, served to the fleet +// so a legacy node installs its packages from here instead of from the internet. It is +// the same tree the image seeded itself from, so a node that purges and reinstalls gets +// the file it started with. +const APT_REPO_DIR = '/repo'; + +/** + * The syncthing version the node image ships, recorded by the repository build. + * Absent only if the stub image was built against a node image without one, which is + * a build-ordering fault worth failing loudly on rather than papering over with a + * default that would quietly make the minimum-version check meaningless. + */ +function imageSyncthingVersion() { + const recorded = fs.readFileSync(`${APT_REPO_DIR}/syncthing.version`, 'utf8').trim(); + if (!recorded) throw new Error(`${APT_REPO_DIR}/syncthing.version is empty`); + return recorded; +} + const state = { blockedRepositories: [], vettedRepositories: [], @@ -10,8 +360,75 @@ const state = { tamperingBlocklist: [], latestRelease: { tag_name: 'v0.0.0', name: 'stub-release' }, geolocation: {}, + // The syncthing the node image ships, read from the repository the image was built + // with rather than restated here. Serving the version the fleet already has is what + // makes the boot-time check a no-op; a suite that wants the upgrade path raises this + // instead of reaching syncthing's own service. A restated version goes stale silently: + // it moves whenever the image is rebuilt, and a minimum every node exceeds asserts + // nothing at all. + moduleMinimumVersions: { syncthing: imageSyncthingVersion(), docker: '26.1.2' }, + marketplaceApps: [], + appSpecsUsdPrice: [], + // Fixed rates, so a price assertion is arithmetic rather than a bet on the market. + // usdPerBtc * btcPerFlux is what the caller multiplies out, and it must equal usdPerFlux + // so the coingecko fallback cannot change an answer. + usdPerBtc: 100000, + btcPerFlux: 0.000002, + usdPerFlux: 0.2, + // published below; null in either representation serves a 404, which leaves + // nodes tableless on the /16 arithmetic + ipLocation: null, + ipLocationBinary: null, + ipLocationVersion: 0, + // the row count the served binary's header claims; null when the served bytes + // are not a format-2 artifact at all + ipLocationRowCount: null, + // per-route fetch counters SINCE THE CURRENT ARTIFACT WAS PUBLISHED. A + // publication is the only thing that resets them, so a suite reads them as + // "what the fleet did about THIS artifact": which nodes downloaded it (ok), + // which found their copy current (notModified) and which found none at all + // (missing, a 404). The lifecycle suites assert against these rather than + // inferring a refetch from a node's own logs. + ipLocationFetches: { + [IPLOCATION_JSON_ROUTE]: newRouteCounters(), + [IPLOCATION_BINARY_ROUTE]: newRouteCounters(), + }, }; +/** + * Count one artifact fetch. + * @param {string} route Which representation was fetched + * @param {'ok'|'notModified'|'missing'} outcome What it was answered with + */ +function countIpLocationFetch(route, outcome) { + const counters = state.ipLocationFetches[route]; + counters.total += 1; + counters[outcome] += 1; +} + +/** + * Publish one artifact in both representations. Both bodies and the version + * their etags carry move in a single synchronous step, so no fetch can catch + * the stub serving a JSON artifact and a binary from different splits. + * @param {object|null} artifact Format-1 artifact, or null to serve 404s + * @param {{pad?: boolean}} [options] pad: false publishes the binary without + * the filler rows - below the reader's truncation floor by design + */ +function serveIpLocation(artifact, { pad = true } = {}) { + const { bytes, rowCount } = encodeIpLocationBinary(artifact, { pad }); + state.ipLocation = artifact; + state.ipLocationBinary = bytes; + state.ipLocationRowCount = rowCount; + state.ipLocationVersion += 1; + // a new artifact is a new question for the fleet: count the answers to it + state.ipLocationFetches = { + [IPLOCATION_JSON_ROUTE]: newRouteCounters(), + [IPLOCATION_BINARY_ROUTE]: newRouteCounters(), + }; +} + +serveIpLocation(buildIpLocationArtifact(1)); + function defaultGeoResponse(ip) { return { status: 'success', @@ -26,8 +443,10 @@ function defaultGeoResponse(ip) { query: ip, org: 'Hetzner Online GmbH', isp: 'Hetzner Online GmbH', + as: 'AS24940 Hetzner Online GmbH', proxy: false, hosting: true, + mobile: false, }; } @@ -36,12 +455,14 @@ function defaultGeoResponse(ip) { const app = express(); app.use(express.json()); -// GitHub raw content endpoints -app.get('/helpers/blockedrepositories.json', (req, res) => { +// Policy documents. Served at the repo root (the fluxos-network-policy layout, +// config.policy.baseUrl) and at the retired /helpers/ paths (the RunOnFlux/flux +// layout, config.github.rawBaseUrl) so one stub covers nodes from either era. +app.get(['/blockedrepositories.json', '/helpers/blockedrepositories.json'], (req, res) => { res.json(state.blockedRepositories); }); -app.get('/helpers/vettedrepositories.json', (req, res) => { +app.get(['/vettedrepositories.json', '/helpers/vettedrepositories.json'], (req, res) => { res.json(state.vettedRepositories); }); @@ -49,10 +470,52 @@ app.get('/helpers/repositories.json', (req, res) => { res.json(state.whitelistedRepositories); }); -app.get('/helpers/tamperingblockednodes.json', (req, res) => { +app.get(['/tamperingblockednodes.json', '/helpers/tamperingblockednodes.json'], (req, res) => { res.json(state.tamperingBlocklist); }); +// The IP location artifact. Served with a strong etag so the conditional +// refresh path (If-None-Match -> 304) is exercised, not just the first fetch. +app.get(IPLOCATION_JSON_ROUTE, (req, res) => { + if (!state.ipLocation) { + countIpLocationFetch(IPLOCATION_JSON_ROUTE, 'missing'); + res.status(404).json({ error: 'no artifact configured' }); + return; + } + const body = JSON.stringify(state.ipLocation); + const etag = `"iplocation-${state.ipLocationVersion}"`; + res.set('ETag', etag); + if (req.headers['if-none-match'] === etag) { + countIpLocationFetch(IPLOCATION_JSON_ROUTE, 'notModified'); + res.status(304).end(); + return; + } + countIpLocationFetch(IPLOCATION_JSON_ROUTE, 'ok'); + res.type('application/json').send(body); +}); + +// The same artifact in the format-2 wire encoding. Both routes stay served: +// the two node lineages sharing this stub fetch different ones. Content-Encoding +// is deliberately not set - the gzip is the artifact's own framing rather than a +// transfer encoding, and a client that transparently inflated it would hand the +// reader the wrong bytes. +app.get(IPLOCATION_BINARY_ROUTE, (req, res) => { + if (!state.ipLocationBinary) { + countIpLocationFetch(IPLOCATION_BINARY_ROUTE, 'missing'); + res.status(404).json({ error: 'no artifact configured' }); + return; + } + const etag = `"iplocationbin-${state.ipLocationVersion}"`; + res.set('ETag', etag); + if (req.headers['if-none-match'] === etag) { + countIpLocationFetch(IPLOCATION_BINARY_ROUTE, 'notModified'); + res.status(304).end(); + return; + } + countIpLocationFetch(IPLOCATION_BINARY_ROUTE, 'ok'); + res.type('application/octet-stream').send(state.ipLocationBinary); +}); + // GitHub API endpoints app.get('/repos/:owner/:repo/releases/latest', (req, res) => { res.json(state.latestRelease); @@ -62,6 +525,61 @@ app.get('/repos/:owner/:repo', (req, res) => { res.json({ full_name: `${req.params.owner}/${req.params.repo}` }); }); +// UPnP: a device description with no WANIPConnection service. upnpService is pointed here +// so its client stops searching for a gateway by SSDP multicast; support verification then +// fails on the missing service, which is the same verdict a node reaches today, so no node +// changes its mind about having UPnP. +app.get('/upnp/device.xml', (req, res) => { + res.type('text/xml').send( + '' + + '' + + 'urn:schemas-upnp-org:device:InternetGatewayDevice:1' + + 'flux-e2e-stub-gateway' + + '', + ); +}); + +// The apt repository a legacy node installs syncthing from. FluxOS writes this source +// itself (systemService addSyncthingRepository) from config.syncthing.aptSourceUrl, and +// fetches the keyring from config.syncthing.releaseKeyUrl, so both are pointed here and +// the keyring fetch, the source write, apt's HTTP transport and signature verification +// all run exactly as they do on a node - against a repository that never leaves the +// fleet network. +app.use('/apt', express.static(APT_REPO_DIR, { fallthrough: false })); + +// Stats: the minimum module versions a node checks its own syncthing against at boot. +// The harness names the version its image ships, so the check is satisfied and no upgrade +// is attempted; a suite exercising the upgrade path raises it through the control port. +app.get('/getmodulesminimumversions', (req, res) => { + res.json({ status: 'success', data: state.moduleMinimumVersions }); +}); + +// Stats: marketplace listings. Empty by default - a suite that needs a listed app puts one +// in through the control port rather than depending on what the live marketplace holds. +app.get('/marketplace/listapps', (req, res) => { + res.json({ status: 'success', data: state.marketplaceApps }); +}); + +app.get('/marketplace/listdevapps', (req, res) => { + res.json({ status: 'success', data: state.marketplaceApps }); +}); + +// Stats: per-spec USD pricing. +app.get('/apps/getappspecsusdprice', (req, res) => { + res.json({ status: 'success', data: state.appSpecsUsdPrice }); +}); + +// Pricing: viprates.runonflux.io/rates. The real service answers a two-element array - +// [fiatRates, coinRates] - and the caller reads USD from the first and FLUX from the second. +app.get('/rates', (req, res) => { + res.json([[{ code: 'USD', rate: state.usdPerBtc }], { FLUX: state.btcPerFlux }]); +}); + +// Pricing: the coingecko fallback, reached only when /rates above is unavailable. +app.get('/api/v3/simple/price', (req, res) => { + res.json({ zelcash: { usd: state.usdPerFlux } }); +}); + // Geolocation: ip-api.com format (primary) app.get('/json/:ip', (req, res) => { const custom = state.geolocation[req.params.ip]; @@ -69,8 +587,18 @@ app.get('/json/:ip', (req, res) => { }); // Geolocation: stats.runonflux.io format (fallback) +// +// LOCATION ONLY, and deliberately so. The real service builds this collection +// by batch-querying ip-api itself for every IP on the deterministic node list, +// asking for `status,continent,continentCode,country,countryCode,region, +// regionName,lat,lon,query,org,isp` - no hosting, no proxy, no mobile, no `as` +// - and its /fluxlocation/:ip handler then projects to exactly the ten fields +// below. It has never carried `static` or `dataCenter`; this stub used to +// synthesise both from the ip-api fixture, which made the fallback look richer +// here than it is on a real node and put the one path where the classifier +// loses its contradiction signals beyond anything a suite could observe. app.get('/fluxlocation/:ip', (req, res) => { - const ip = req.params.ip; + const { ip } = req.params; const custom = state.geolocation[ip]; const geo = { ...defaultGeoResponse(ip), ...custom }; res.json({ @@ -86,19 +614,54 @@ app.get('/fluxlocation/:ip', (req, res) => { lat: geo.lat, lon: geo.lon, org: geo.org, - static: !geo.proxy && geo.hosting, - dataCenter: geo.hosting, }, }); }); +// Arbitrary bytes a node can fetch over real HTTP. The restore suites need an +// archive that actually arrives down the wire from inside the subnet, because +// the whole remote path - the download, the content-length comparison, the file +// landing in backup/remote - has no other way to be exercised. +const artifacts = new Map(); + +// HEAD is answered separately because the size a downloader is PROMISED and the +// bytes it actually receives have to be able to disagree - that disagreement is +// the whole subject of the short-download check, and FluxOS learns the promise +// from a HEAD (IOUtils.getRemoteFileSize). +app.head('/artifact/:name', (req, res) => { + const artifact = artifacts.get(req.params.name); + if (!artifact) return res.status(404).end(); + res.setHeader('content-type', 'application/gzip'); + res.setHeader('content-length', String(artifact.declaredLength ?? artifact.body.length)); + return res.end(); +}); + +app.get('/artifact/:name', (req, res) => { + const artifact = artifacts.get(req.params.name); + if (!artifact) return res.status(404).json({ error: 'no such artifact' }); + res.setHeader('content-type', 'application/gzip'); + if (artifact.declaredLength == null) { + res.setHeader('content-length', String(artifact.body.length)); + return res.end(artifact.body); + } + // With a declared length the body is sent chunked and the connection closes + // cleanly: the transfer SUCCEEDS and the file on disk is simply shorter than + // HEAD promised, which is the case the received-vs-expected comparison exists + // for. Sending a content-length that contradicts the body instead leaves the + // client waiting for bytes that never come - that is a timeout, not a short + // download, and it takes the suite's whole budget to find out. + return res.end(artifact.body); +}); + // --- Control API --- const control = express(); control.use(express.json()); control.get('/state', (req, res) => { - res.json(state); + // the wire artifact is opaque bytes; its size, its claimed row count and the + // per-route fetch counters (ipLocationFetches) are the readable parts + res.json({ ...state, ipLocationBinary: undefined, ipLocationBinaryBytes: state.ipLocationBinary?.length ?? 0 }); }); control.post('/blocked-repos', (req, res) => { @@ -136,6 +699,58 @@ control.delete('/geolocation/:ip', (req, res) => { res.json({ ok: true }); }); +control.post('/iplocation', (req, res) => { + // { domains: n } serves a generated artifact splitting each /24 n ways; + // adding { regions: true } gives each split address the region of its + // organisation, the last organisation carrying none (see regionAssignment) - + // omit it and the artifact carries no region at all, exactly as before. + // { artifact: {...} } serves a caller-supplied one (malformed included, to + // exercise reject-and-keep); { artifact: null } serves a 404 (tableless). + // Adding { pad: false } publishes the binary WITHOUT the filler rows, i.e. + // below the reader's truncation floor - floor bait, and the only artifact + // here a healthy node is expected to refuse. + // Whichever it is, both /iplocation.json and /iplocation.bin.gz follow it, + // and the fetch counters start again from zero. + const pad = req.body.pad !== false; + let regions = null; // a caller-supplied artifact has no assignment to report + if (Object.prototype.hasOwnProperty.call(req.body, 'artifact')) { + serveIpLocation(req.body.artifact, { pad }); + } else { + const domains = req.body.domains ?? 1; + const withRegions = req.body.regions === true; + // { classes: { 0: 'residential' } } publishes a verdict for organisation 0. + // Absent, the artifact carries an empty orgClasses section, so every node falls + // back to deciding its own address - which is what the reader does with a + // build that classified nothing. + serveIpLocation( + buildIpLocationArtifact(domains, req.body.subnet, withRegions, req.body.classes ?? null), + { pad }, + ); + regions = regionAssignment(domains, withRegions); + } + res.json({ + ok: true, + ranges: state.ipLocation?.v4?.length ?? 0, + // what the served binary's header claims, filler included - null when the + // bytes are not a format-2 artifact + rowCount: state.ipLocationRowCount, + padded: pad, + bytes: state.ipLocationBinary?.length ?? 0, + regions, + orgClasses: state.ipLocation?.orgClasses ?? null, + }); +}); + +control.post('/artifact', (req, res) => { + const { name, base64, declaredLength = null } = req.body || {}; + if (!name || typeof base64 !== 'string') { + return res.status(400).json({ error: 'name and base64 are required' }); + } + const body = Buffer.from(base64, 'base64'); + artifacts.set(name, { body, declaredLength }); + return res.json({ ok: true, name, bytes: body.length, declaredLength }); +}); + control.post('/reset', (req, res) => { state.blockedRepositories = []; state.vettedRepositories = []; @@ -143,6 +758,8 @@ control.post('/reset', (req, res) => { state.tamperingBlocklist = []; state.latestRelease = { tag_name: 'v0.0.0', name: 'stub-release' }; state.geolocation = {}; + artifacts.clear(); + serveIpLocation(buildIpLocationArtifact(1)); res.json({ ok: true }); }); @@ -150,6 +767,119 @@ control.get('/health', (req, res) => { res.json({ status: 'ok' }); }); +// Every name a node asked for that nothing on the fleet could answer, with the +// node that asked. A suite asserts this is empty; when it is not, the failure +// names the host and the node instead of describing itself as slow. +control.get('/dns-attempts', (req, res) => { + res.json({ attempts: dnsAttempts }); +}); + +control.post('/dns-attempts/reset', (req, res) => { + dnsAttempts.length = 0; + res.json({ ok: true }); +}); + +// The fleet's resolver. +// +// Blocking a network is not the same as failing loudly on it: a blocked packet +// surfaces as a timeout, and a timeout reads as slowness rather than as a node +// reaching somewhere it should not. So the nodes resolve here instead, and a name +// the fleet cannot answer comes back NXDOMAIN at once, recorded against the node +// that asked for it. +// +// Fleet names still resolve, because this relays to its own embedded Docker +// resolver at 127.0.0.11 - the same one that knows every container alias on this +// network. Relaying rather than answering means aliases need no list here and +// cannot drift from the ones the runner actually creates. +const dnsAttempts = []; + +function questionName(query) { + // QNAME begins after the 12-byte header, as length-prefixed labels ending in 0. + let offset = 12; + const labels = []; + while (offset < query.length) { + const len = query[offset]; + if (len === 0 || len > 63) break; + labels.push(query.subarray(offset + 1, offset + 1 + len).toString('ascii')); + offset += len + 1; + } + return labels.join('.'); +} + +function startResolver() { + const server = dgram.createSocket('udp4'); + + server.on('message', (query, rinfo) => { + const name = questionName(query); + const upstream = dgram.createSocket('udp4'); + let settled = false; + + const answer = (response, resolved) => { + if (settled) return; + settled = true; + clearTimeout(timer); + // answer() also runs as the upstream's own error handler, and close() + // throws on a socket that never bound - from inside an error handler + // nothing catches, so one unlucky query would take the resolver down for + // the whole run. An uncloseable socket is left to the garbage collector. + try { + upstream.close(); + } catch { + // nothing to close + } + if (!resolved) dnsAttempts.push({ name, node: rinfo.address, at: new Date().toISOString() }); + server.send(response, rinfo.port, rinfo.address); + }; + + // NXDOMAIN built from the query: same id and question, QR and RCODE 3 set. + const refuse = () => { + const response = Buffer.from(query); + response[2] |= 0x80; + response[3] = (response[3] & 0xf0) | 0x03; + answer(response, false); + }; + + // Short, because this is the whole point: an unanswerable name must fail now + // rather than at whatever deadline the caller happens to carry. + const timer = setTimeout(refuse, 300); + + upstream.on('error', refuse); + upstream.on('message', (response) => { + const rcode = response[3] & 0x0f; + if (rcode === 0) answer(response, true); + else refuse(); + }); + + upstream.send(query, 53, '127.0.0.11'); + }); + + // A dgram socket with no 'error' listener turns any failure into an uncaught + // exception. The failure that actually happens is the bind - port 53 already + // held, usually by the previous run's container on its way out - and without a + // listener the stub dies on a stack trace that mentions neither DNS nor the + // port, while every node in the fleet silently fails to resolve anything. That + // reads as a fleet-wide product fault and is nothing of the kind. + // + // Fatal on purpose: a resolver that never bound is not a resolver, and the run + // should say so at startup rather than eighty suites later. Errors after the + // bind cost one query and are logged, which this handler covers for free. + let bound = false; + server.on('error', (error) => { + if (!bound) { + console.error(`External HTTP stub resolver could not bind to 53: ${error.message}`); + process.exit(1); + } + console.error(`External HTTP stub resolver socket error: ${error.message}`); + }); + + server.bind(53, () => { + bound = true; + console.log('External HTTP stub resolver on port 53'); + }); +} + +startResolver(); + app.listen(PORT, () => { console.log(`External HTTP stub listening on port ${PORT}`); }); diff --git a/test-infra/fdm-stub/index.js b/test-infra/fdm-stub/index.js index d2042ddc9e..ba1e405286 100644 --- a/test-infra/fdm-stub/index.js +++ b/test-infra/fdm-stub/index.js @@ -13,6 +13,19 @@ const CONTROL_PORT = parseInt(process.env.CONTROL_PORT || '16131', 10); // which mirrors the real FDM returning an empty ips array (the node waits). const elected = new Map(); +// Whether FDM is answering at all. Electing and clearing are both FDM giving a +// verdict, so neither reaches the node's third state — "FDM did not answer" — +// which is the one the election stands down on. That state needs the service to +// stop producing verdicts: +// 'refuse' the listening socket is closed, so the node gets ECONNREFUSED. +// This is the production outage signature: the error carries no +// response at all. +// 'unavailable' 503, FDM reachable but declining to answer because it reports +// itself as still starting up. +// null => answering normally. +let outageMode = null; +let server = null; + // --- FDM API (what the FluxOS node polls) --- const app = express(); @@ -22,16 +35,31 @@ app.use(express.json()); // then data.ips[0] (passed through extractIp, which splits on ':' — bare IP is fine). // An empty ips array is the "no primary set" path: the node keeps waiting. app.get('/appips/:app', (req, res) => { + if (outageMode === 'unavailable') { + res.status(503).json({ status: 'error', data: 'FDM starting up' }); + return; + } const ip = elected.get(req.params.app); res.json({ status: 'success', data: { ips: ip ? [ip] : [] } }); }); app.all('*', (req, res) => { console.log(`Unhandled FDM request: ${req.method} ${req.path}`); + if (outageMode === 'unavailable') { + res.status(503).json({ status: 'error', data: 'FDM starting up' }); + return; + } res.json({ status: 'success', data: { ips: [] } }); }); -app.listen(PORT, () => console.log(`FDM stub listening on port ${PORT}`)); +function listen(done) { + server = app.listen(PORT, () => { + console.log(`FDM stub listening on port ${PORT}`); + if (done) done(); + }); +} + +listen(); // --- Test harness control API --- @@ -43,7 +71,7 @@ control.get('/health', (req, res) => { }); control.get('/state', (req, res) => { - res.json({ elected: Object.fromEntries(elected) }); + res.json({ elected: Object.fromEntries(elected), outage: outageMode }); }); // elect (or fail over) the primary for an app @@ -60,9 +88,51 @@ control.post('/clear/:app', (req, res) => { res.json({ ok: true }); }); +// Stop answering. The control API is a second server on its own port, so it +// stays reachable to end the outage again. +function beginOutage(mode, done) { + outageMode = mode; + if (mode !== 'refuse' || !server) { + done(); + return; + } + // close() only stops new connections being accepted; a keep-alive socket the + // node already holds would go on being answered, so the poll has to lose the + // connection it has rather than read a stale success off it. + if (server.closeAllConnections) server.closeAllConnections(); + server.close(() => { + server = null; + done(); + }); +} + +function endOutage(done) { + const wasRefusing = outageMode === 'refuse'; + outageMode = null; + if (!wasRefusing || server) { + done(); + return; + } + listen(done); +} + +control.post('/outage', (req, res) => { + const mode = (req.body && req.body.mode) || 'refuse'; + if (mode !== 'refuse' && mode !== 'unavailable') { + return res.status(400).json({ error: "mode must be 'refuse' or 'unavailable'" }); + } + return beginOutage(mode, () => res.json({ ok: true, outage: mode })); +}); + +control.post('/recover', (req, res) => { + endOutage(() => res.json({ ok: true, outage: null })); +}); + +// Suites reset in both setup and teardown, so this has to put every piece of +// stub state back - an outage left behind would answer for the next suite. control.post('/reset', (req, res) => { elected.clear(); - res.json({ ok: true }); + endOutage(() => res.json({ ok: true })); }); control.listen(CONTROL_PORT, () => console.log(`FDM stub control API on port ${CONTROL_PORT}`)); diff --git a/test-infra/fixtures/mongo-init.js b/test-infra/fixtures/mongo-init.js index 4c1476ce2c..37e0528076 100644 --- a/test-infra/fixtures/mongo-init.js +++ b/test-infra/fixtures/mongo-init.js @@ -1,8 +1,13 @@ // Pre-seed explorer scanned height for all 16 nodes so the explorer // starts near the daemon tip instead of scanning from block 694000. // Mounted into /docker-entrypoint-initdb.d/ — runs once on first boot. - -const INITIAL_HEIGHT = 2100000; +// +// A FIRST-BOOT DEFAULT ONLY. seedMongo upserts the run's own height over this on +// every run, including a suite that asked for a different one, so this value only +// covers the window before the first seed. Kept in step with +// runner/framework/chain-start.cjs, which is where the number is decided; this +// file runs under mongosh and cannot require it. +const INITIAL_HEIGHT = 2952000; const NODE_COUNT = 16; for (let i = 1; i <= NODE_COUNT; i++) { diff --git a/test-infra/generate-compose.js b/test-infra/generate-compose.js index 7c126f9733..123f9953f8 100644 --- a/test-infra/generate-compose.js +++ b/test-infra/generate-compose.js @@ -60,10 +60,20 @@ w(' networks:'); w(' flux-test-net:'); w(` ipv4_address: ${DAEMON_IP}`); w(' environment:'); +w(' FLUX_TEST_HARNESS: "true"'); w(' FLUXD_PORT: "16124"'); w(' BENCHD_PORT: "16224"'); w(' CONTROL_PORT: "18232"'); w(' TICKER_AUTOSTART: "false"'); +// Required at launch, never defaulted here: the chain start has one home +// (runner/framework/chain-start.cjs), and a number frozen into the generated +// file goes stale between generation and use - the stub refuses to boot on +// exactly that kind of copy. +w(' # Required, no default: the chain start has one home'); +w(' # (runner/framework/chain-start.cjs) and a defaulted copy here goes stale'); +w(' # and silently runs every suite pre-fork. Launch with:'); +w(' # INITIAL_HEIGHT=$(node -p "require(\'./runner/framework/chain-start.cjs\').DEFAULT_INITIAL_HEIGHT") docker compose up'); +w(' INITIAL_HEIGHT: "${INITIAL_HEIGHT:?set from runner/framework/chain-start.cjs - see the comment above this line}"'); w(' volumes:'); w(' - ./fixtures:/fixtures'); w(' healthcheck:'); diff --git a/test-infra/image-digest.sh b/test-infra/image-digest.sh new file mode 100755 index 0000000000..2113cc112e --- /dev/null +++ b/test-infra/image-digest.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Prints the digest of the sources an image is built from. +# +# An image name is fixed but its contents are not, and nothing else ties a built +# image to the tree it came from. A stub rebuilt on another branch, or simply not +# rebuilt after its source moved, runs against this branch's suites and fails +# looking like a product bug - suites 87/88/90 died in a `before each` calling a +# stub endpoint the image did not carry, which reads as a restore defect and is +# not one. Stamping this digest at build time and comparing it before a run turns +# that into a refusal with a name on it. +# +# The digest covers the BUILD CONTEXT, not a marker someone remembered to add: +# a marker proves one line, and picking one per change is the same memory test +# that fails in the first place. +# +# usage: image-digest.sh fluxos-01 | image-digest.sh +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +if command -v sha256sum >/dev/null 2>&1; then HASH=sha256sum; else HASH="shasum -a 256"; fi + +# Hash every file under the given roots, path included, order fixed. LC_ALL=C so +# the sort is byte order on any host, and -print0/-0 so a path with a space in it +# cannot split a filename into two. An empty result is a failure, not a digest: +# this refuses runs, so it must never hand back something that merely looks like +# an answer. +digest_roots() { + local out + out="$(find "$@" -type f -not -path 'test-infra/runner/*' -print0 2>/dev/null \ + | LC_ALL=C sort -z \ + | xargs -0 -r $HASH \ + | $HASH | cut -d' ' -f1)" + [ -n "$out" ] || { echo "image-digest: hashed nothing under: $*" >&2; return 3; } + printf '%s\n' "$out" +} + +# fluxos-01 is built with `COPY . .` from the repo root, so its context is every +# top-level entry .dockerignore does not exclude. Derived from the file rather +# than restated here: a new exclusion would otherwise change what docker bakes +# without changing what this measures. +# +# test-infra/runner is the one exception, and it is a principled one rather than +# a list: the runner DRIVES the fleet from the host and never executes inside a +# node container, so a suite edit changes bytes the image carries but nothing it +# does. Counting it would force a full node rebuild for every suite tweak - the +# kind of friction that gets a check bypassed, which costs more than it saves. +# Excluded for the stubs too, where it can never match anything. +fluxos_roots() { + local ignore=() line entry skip i + while IFS= read -r line || [ -n "$line" ]; do + line="${line%%#*}" + line="$(printf '%s' "$line" | tr -d '[:space:]')" + [ -n "$line" ] && ignore+=("$line") + done < .dockerignore + + for entry in * .[!.]*; do + [ -e "$entry" ] || continue + skip=0 + for i in "${ignore[@]:-}"; do [ "$entry" = "$i" ] && skip=1; done + [ "$skip" -eq 0 ] && printf '%s\n' "$entry" + done +} + +image="${1:?usage: image-digest.sh }" + +if [ "$image" = "fluxos-01" ]; then + roots=() + while IFS= read -r entry; do roots+=("$entry"); done < <(fluxos_roots) + [ "${#roots[@]}" -gt 0 ] || { echo "image-digest: empty build context" >&2; exit 3; } + digest_roots "${roots[@]}" + exit 0 +fi + +dir="test-infra/$image" +[ -d "$dir" ] || { echo "image-digest: no such image source: $dir" >&2; exit 2; } + +# A stub that builds FROM the node image inherits its contents, so a stale +# fluxos-01 makes the stub stale too even when its own directory has not moved. +# external-http-stub is the one that does this today; the ARG is what says so. +if grep -q 'FLUX_E2E_TAG' "$dir/Dockerfile" 2>/dev/null; then + base="$("$0" fluxos-01)" || exit 3 + own="$(digest_roots "$dir")" || exit 3 + printf '%s\n%s\n' "$own" "$base" | $HASH | cut -d' ' -f1 +else + digest_roots "$dir" +fi diff --git a/test-infra/peer-stub/index.js b/test-infra/peer-stub/index.js index b3ee5257e1..1f220aad02 100644 --- a/test-infra/peer-stub/index.js +++ b/test-infra/peer-stub/index.js @@ -1,4 +1,5 @@ const http = require('http'); +const net = require('net'); const { WebSocketServer } = require('ws'); const { signAsync } = require('@noble/secp256k1'); const { sha256 } = require('@noble/hashes/sha2'); @@ -10,22 +11,86 @@ if (process.env.FLUX_TEST_HARNESS !== 'true') { const WS_PORT = Number(process.env.WS_PORT) || 16127; const CONTROL_PORT = Number(process.env.CONTROL_PORT) || 16128; -const PRIVATE_KEY = process.env.PRIVATE_KEY; -const PUBLIC_KEY = process.env.PUBLIC_KEY; -const NODE_IP = process.env.NODE_IP; +const SILENT_APP_STATE_SYNC = process.env.SILENT_APP_STATE_SYNC === 'true'; +// A PEER THAT ANSWERS WITH SOMETHING NOBODY CAN STAND BEHIND. The other way an +// answer fails: not silence, not a refusal, but bytes that arrive and do not +// verify. Every real responder signs correctly with its own key, so without +// this the fleet has no way to reach the path a node takes when a peer's +// envelope cannot be attributed to it. +const UNVERIFIABLE_APP_STATE_SYNC = process.env.UNVERIFIABLE_APP_STATE_SYNC === 'true'; +const DIAL_TARGETS = (process.env.DIAL_TARGETS || '').split(',').filter(Boolean); +const { PRIVATE_KEY, PUBLIC_KEY, NODE_IP } = process.env; if (!PRIVATE_KEY || !PUBLIC_KEY) { console.error('PRIVATE_KEY and PUBLIC_KEY env vars are required'); process.exit(1); } +// The first byte of a binary frame is its type. Only the four sync requests +// matter here, and each is answered with the stream it asks for. +const SYNC_REQUEST_RESPONSES = Object.freeze({ + 0x20: 'fluxapptempsync', + 0x21: 'fluxapprunningsync', + 0x22: 'fluxappinstallingsync', + 0x23: 'fluxappinstallingerrorssync', +}); + const messages = new Map(); let connectionsReceived = 0; let requestsReceived = 0; let messagesServed = 0; +let unverifiableResponsesSent = 0; const requestLog = []; +// What this peer answers when a node asks what it is holding, and when it was +// asked. The arrival times are the point: a node decides promotion for every +// folder in one monitor pass, so two arrivals milliseconds apart mean it asked +// once per folder rather than once for the pass. +let promotedFolders = { ready: true, folders: [] }; +const promotedFolderRequests = []; + +// The status this peer answers that question with. 200 is a peer that can answer +// it; anything else is a peer that cannot, and the one that matters is 404 - +// /apps/promotedfolders is new, so every node in the fleet answers 404 until it +// is upgraded. The fleet runs one image and can never be version-mixed, so +// without this a rollout is unreachable here and the asking node's handling of +// it can only ever be guessed at. +let promotedFoldersStatus = 200; + +// Whether this peer answers that question AT ALL. A status - even 404 - is an +// answer, and the asking node reads any answer as proof the peer is alive. A +// peer whose FluxOS is down does not answer: the connection is refused. That is +// a different fact and the node treats it differently, so a suite that needs a +// holder which is alive but unanswerable has to be able to produce it. +let promotedFoldersRefuse = false; + +// Whether this peer passes a port test WITHOUT connecting to the asker. +// +// This is what a shared public address looks like from the asker's side. Several +// Flux nodes commonly sit behind one router, which forwards each port to exactly +// one of them, so a peer probing the shared address can reach a sibling node's +// application while the asker's own test server sits unreached behind the same +// NAT. The peer is not lying and cannot tell: something genuinely answered. +// +// The stub cannot reproduce the NAT, but it can reproduce what the asker +// receives - a pass for a port the asker was never reached on - which is the +// only part of it the asker can act on. +let portProbeAnswersBlind = false; + +// Answers the port test with a reading that is NOT the asker's - what the asker +// receives when the router forwarded that port to a neighbour and a different +// application replied. The stub cannot reproduce the NAT; it reproduces what +// comes back through it, which is the only part the asker can act on. +let portProbeAnswersForeign = false; + +// The nodes currently connected to this peer. Held so the stub can SAY things +// rather than only answer them: a suite that needs a rival claim, a stale +// broadcast or a message a real node would never send gets a real peer sending +// it, signed and over the wire, instead of a row written behind the node's back. +const connectedNodes = new Set(); +let broadcastsSent = 0; + function hash256(data) { return sha256(sha256(data)); } @@ -64,7 +129,48 @@ async function serialiseAndSignBroadcast(data) { return JSON.stringify({ version, timestamp, pubKey: PUBLIC_KEY, signature, data }); } +/** + * A response that is correctly shaped and cannot be attributed to this peer. + * + * Signed over a payload one millisecond off the one it carries, so the + * signature is a real signature of the right length made with the right key - + * it simply is not a signature of this message. That is the shape a forgery + * has on the wire, and it is what an envelope check has to catch. A random + * string would be caught by the parse instead, which tests the parser. + * @param {string} type The sync response type being answered. + * @returns {Promise} the wire frame. + */ +async function serialiseUnverifiableSyncResponse(type) { + const version = 1; + const timestamp = Date.now(); + const data = { type, messages: [], done: true }; + const message = JSON.stringify(data); + const signature = await signBtcMessage(`${version}${message}${timestamp + 1}`, PRIVATE_KEY); + return JSON.stringify({ version, timestamp, pubKey: PUBLIC_KEY, signature, data }); +} + +async function handleSyncRequest(ws, rawData) { + const responseType = SYNC_REQUEST_RESPONSES[rawData[0]]; + if (!responseType) return; + // Nothing holds this promise - the socket handler calls and returns - so a + // throw here would take the stub down with an unhandled rejection. + try { + const frame = await serialiseUnverifiableSyncResponse(responseType); + ws.send(frame); + unverifiableResponsesSent++; + } catch (e) { + console.error('Error answering sync request:', e.message); + } +} + async function handleMessage(ws, rawData) { + // Binary frames are the peer protocol's own encoding - hash traffic and the + // app-state sync requests. This stub speaks the second only when asked to, + // and a JSON parse error per incoming request is noise rather than a finding. + if (rawData[0] !== 0x7b) { + if (UNVERIFIABLE_APP_STATE_SYNC) await handleSyncRequest(ws, rawData); + return; + } try { const msg = JSON.parse(rawData); const { data } = msg; @@ -75,7 +181,7 @@ async function handleMessage(ws, rawData) { let hashes = []; if (data.version === 2 && Array.isArray(data.hashes)) { - hashes = data.hashes; + ({ hashes } = data); } else if (data.version === 1 && typeof data.hash === 'string') { hashes = [data.hash]; } @@ -100,25 +206,195 @@ async function handleMessage(ws, rawData) { const wss = new WebSocketServer({ noServer: true }); wss.on('headers', (headers) => { - headers.push('X-Flux-Capabilities: peerExchange,appStateSync'); + // peerExchange only. This stub does NOT implement the app-state sync + // endpoints - apprunning, appinstalling and apperrors are all absent - and a + // real node picks its sync peers by exactly this capability + // (FluxPeerManager.getEligibleSyncPeers). Claiming it made stubs eligible, + // so a node would ask one, get nothing, time out, and never publish + // SPAWNER_READY - which never starts the spawn loop. A fleet with several + // stubs then had a real chance of drawing only stubs, and every test needing + // an app to be spawned waited out its whole budget for a spawner that was + // never running. Suite 98 lost a gate to it. + // DELIBERATELY SILENT, when asked for. Claiming appStateSync without + // implementing it is what cost suite 98 a gate: a node asked a stub, got + // nothing, timed out and never started its spawner. That is no longer a trap + // but a case with a name - a peer whose socket is perfectly healthy and which + // never answers - and the asker is now expected to give up on it and ask + // someone else. So a suite can ask for exactly that, and gets it only when it + // does. + const answersAppStateSync = SILENT_APP_STATE_SYNC || UNVERIFIABLE_APP_STATE_SYNC; + const capabilities = answersAppStateSync ? 'peerExchange,appStateSync' : 'peerExchange'; + headers.push(`X-Flux-Capabilities: ${capabilities}`); headers.push('X-Flux-Version: 8.0.0'); headers.push('X-Flux-Uptime: 1000'); }); wss.on('connection', (ws) => { connectionsReceived++; + connectedNodes.add(ws); + ws.on('close', () => connectedNodes.delete(ws)); ws.on('message', (data) => handleMessage(ws, data)); ws.on('error', () => {}); }); -const wsServer = http.createServer((req, res) => { - if (req.url === '/flux/version') { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ status: 'success', data: '8.0.0' })); - return; +// Whether a TCP connection to the asker's port completes. A timeout answers for +// a port that is filtered rather than refused - to the node asking, both mean +// the same thing. +// Reads what a port replies, capped, the way a node on current code does. +function portRead(ip, port, timeoutMs = 5000) { + return new Promise((resolve) => { + const socket = net.connect({ host: ip, port }); + let received = ''; + let settled = false; + const done = (answer) => { + if (settled) return; + settled = true; + socket.removeAllListeners(); + socket.destroy(); + resolve(answer); + }; + socket.setTimeout(timeoutMs); + socket.once('connect', () => { + socket.write(`GET / HTTP/1.1\r\nHost: ${ip}:${port}\r\nConnection: close\r\n\r\n`); + }); + socket.on('data', (chunk) => { + received += chunk.toString('utf8'); + if (received.length >= 256) done(received.slice(0, 256)); + }); + socket.once('end', () => done(received || null)); + socket.once('timeout', () => done(received || null)); + socket.once('error', () => done(null)); + }); +} + +function portAnswers(ip, port, timeoutMs = 5000) { + return new Promise((resolve) => { + const socket = net.connect({ host: ip, port }); + const done = (reachable) => { + socket.removeAllListeners(); + socket.destroy(); + resolve(reachable); + }; + socket.setTimeout(timeoutMs); + socket.once('connect', () => done(true)); + socket.once('timeout', () => done(false)); + socket.once('error', () => done(false)); + }); +} + +const wsServer = http.createServer(async (req, res) => { + // The handler awaits, so a request that fails while being read rejects rather + // than throwing, and an unhandled rejection takes the process with it. Every + // path answers. + try { + if (req.method === 'POST' && req.url === '/flux/checkappavailability') { + // Before installing, a node opens its ports and asks a RANDOM peer to + // confirm they answer from outside; it aborts the install if no peer + // confirms within its attempts. A stub that 404s here is a peer that can + // never confirm, so every node that draws one burns an attempt, and a fleet + // carrying several of them fails installs with nothing wrong. + // + // Really connected rather than answered blind: the asker opens those ports + // for this check alone, and a stub that always said yes would mask the exact + // failure the check exists to find. + const body = await readBody(req); + let asked; + try { + asked = JSON.parse(body); + } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'error', data: { message: 'Unparseable request' } })); + return; + } + // Probed together rather than in turn: the asker gives this whole exchange + // one timeout, and a serial walk of several ports spends that budget before + // it can answer. + const ports = Array.isArray(asked.ports) ? asked.ports : []; + if (portProbeAnswersBlind) { + // Passed without connecting: see portProbeAnswersBlind. + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'success', data: { message: 'Ports are available' } })); + return; + } + // A requester asking for proof gets what each port actually said. One that + // is not - an older node - gets the reachability answer it always got, and + // no `answered` field, which is how the asker knows this peer cannot prove + // anything either way. + if (asked.echo && !portProbeAnswersBlind) { + const answered = {}; + for (const port of ports) { + // eslint-disable-next-line no-await-in-loop + const reply = portProbeAnswersForeign + ? 'HTTP/1.1 200 OK\r\n\r\n{"status":"success","data":{"token":"a-neighbours-application"}}' + : await portRead(asked.ip, port); + if (reply === null) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'error', data: { message: `Failed port: ${port}` } })); + return; + } + answered[port] = reply; + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'success', + data: { message: 'Ports are available', answered }, + })); + return; + } + const reachable = await Promise.all(ports.map((port) => portAnswers(asked.ip, port))); + const failedAt = reachable.indexOf(false); + if (failedAt !== -1) { + // Named, because the asker reads the number back out of this message to + // decide which port to retest. + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'error', data: { message: `Failed port: ${ports[failedAt]}` } })); + return; + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'success', data: { message: 'Ports are available' } })); + return; + } + if (req.url === '/flux/version') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'success', data: '8.0.0' })); + return; + } + if (req.url === '/syncthing/deviceid') { + // Every real node answers this, however old - the endpoint long predates + // /apps/promotedfolders, which is the only version distinction this stub + // models. Nodes cache the answer to name this peer in queries against + // their own syncthing, so a stub that 404s here starves that cache and + // silently disables every check built on it. + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'success', data: 'PEERSTUB-DEVICE-0000001' })); + return; + } + if (req.url === '/apps/promotedfolders') { + // Recorded before the status is applied: a peer that answers 404 was still + // asked, and a suite proving the asker kept asking needs to see that. + promotedFolderRequests.push(Date.now()); + if (promotedFoldersRefuse) { + // Destroyed rather than answered, so the asker sees the transport fail + // exactly as it does against a node whose FluxOS is not listening. + req.socket.destroy(); + return; + } + if (promotedFoldersStatus !== 200) { + res.writeHead(promotedFoldersStatus); + res.end(); + return; + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'success', data: promotedFolders })); + return; + } + res.writeHead(404); + res.end(); + } catch (e) { + console.error(`peer stub: ${req.method} ${req.url} failed: ${e.message}`); + if (!res.headersSent) res.writeHead(500, { 'Content-Type': 'application/json' }); + if (!res.writableEnded) res.end(JSON.stringify({ status: 'error', data: { message: e.message } })); } - res.writeHead(404); - res.end(); }); wsServer.on('upgrade', (req, socket, head) => { @@ -133,8 +409,37 @@ wsServer.on('upgrade', (req, socket, head) => { wsServer.listen(WS_PORT, () => { console.log(`Peer stub ${NODE_IP} WS server listening on port ${WS_PORT}`); + if (DIAL_TARGETS.length) askToBeDialled(); }); +// ASK THE NODES TO DIAL US, because nothing else will. +// +// The mesh forms by reciprocity: a node asks a peer to add it as an outgoing +// peer, and the PEER dials back. A stub is a server with no client, so it can +// never dial back, which is why discovery reaches a stub and gives up - the +// node's request fails and it records a failed connection. Doing what a real +// node does over plain HTTP is the whole of what was missing. +// +// Only when a suite asked for it, so every stub that came before behaves +// exactly as it did: acquiring connections a suite did not ask for would change +// peer counts under fleets that were written against the old behaviour. +function askToBeDialled() { + const ask = (ip) => { + const req = http.get( + `http://${ip}:${WS_PORT}/flux/addoutgoingpeer/${NODE_IP}:${WS_PORT}`, + { timeout: 5000 }, + (res) => { res.resume(); }, + ); + req.on('timeout', () => req.destroy()); + // A node that is not up yet refuses; the next pass finds it. + req.on('error', () => {}); + }; + const round = () => DIAL_TARGETS.forEach(ask); + round(); + const timer = setInterval(round, 5000); + timer.unref(); +} + function readBody(req) { return new Promise((resolve, reject) => { let body = ''; @@ -158,12 +463,89 @@ const controlServer = http.createServer(async (req, res) => { connectionsReceived, requestsReceived, messagesServed, + unverifiableResponsesSent, messagesLoaded: messages.size, requestLog, + promotedFolderRequests, + broadcastsSent, + connectedNodes: connectedNodes.size, })); return; } + // Say something to every node connected right now, signed with this peer's + // own key and framed exactly as a real broadcast - so the receiving node + // validates it, stores it and acts on it through the path it uses for any + // other peer. The caller supplies the whole message, because what makes a + // message interesting to a suite is usually the field a real peer would + // never get wrong. + if (req.method === 'POST' && req.url === '/broadcast') { + const body = await readBody(req); + const data = JSON.parse(body); + const wire = await serialiseAndSignBroadcast(data); + let sent = 0; + for (const ws of connectedNodes) { + if (ws.readyState === 1) { ws.send(wire); sent++; } + } + broadcastsSent += sent; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'ok', sent, connected: connectedNodes.size })); + return; + } + + if (req.method === 'POST' && req.url === '/promoted-folders') { + const body = await readBody(req); + const wanted = JSON.parse(body); + promotedFolders = { + ready: wanted.ready !== false, + folders: Array.isArray(wanted.folders) ? wanted.folders : [], + }; + // Reset the arrival log HERE, in the same request that states the claim, + // because /clear does both and a caller that wants a fresh log without + // dropping the claim has to make two calls. Between them this peer claims + // to hold nothing, and a node polling in that gap sees the folder free, + // promotes, and stops asking - which is the measurement suite 79 exists + // to take, ruined by the act of starting it. + if (wanted.resetRequests) promotedFolderRequests.length = 0; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'ok', promotedFolders })); + return; + } + + if (req.method === 'POST' && req.url === '/promoted-folders-status') { + const body = await readBody(req); + const wanted = JSON.parse(body); + promotedFoldersStatus = Number(wanted.status) || 200; + promotedFoldersRefuse = false; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'ok', promotedFoldersStatus })); + return; + } + + if (req.method === 'POST' && req.url === '/promoted-folders-refuse') { + const body = await readBody(req); + promotedFoldersRefuse = JSON.parse(body).refuse !== false; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'ok', promotedFoldersRefuse })); + return; + } + + if (req.method === 'POST' && req.url === '/port-probe-foreign') { + const body = await readBody(req); + portProbeAnswersForeign = JSON.parse(body).foreign !== false; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'ok', portProbeAnswersForeign })); + return; + } + + if (req.method === 'POST' && req.url === '/port-probe-blind') { + const body = await readBody(req); + portProbeAnswersBlind = JSON.parse(body).blind !== false; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'ok', portProbeAnswersBlind })); + return; + } + if (req.method === 'POST' && req.url === '/load-message') { const body = await readBody(req); const msg = JSON.parse(body); @@ -187,6 +569,13 @@ const controlServer = http.createServer(async (req, res) => { connectionsReceived = 0; requestsReceived = 0; messagesServed = 0; + promotedFolderRequests.length = 0; + promotedFolders = { ready: true, folders: [] }; + promotedFoldersStatus = 200; + promotedFoldersRefuse = false; + portProbeAnswersBlind = false; + portProbeAnswersForeign = false; + broadcastsSent = 0; res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ status: 'ok' })); return; diff --git a/test-infra/runner/framework/app-helper.js b/test-infra/runner/framework/app-helper.js index 2a207ec024..53f876c934 100644 --- a/test-infra/runner/framework/app-helper.js +++ b/test-infra/runner/framework/app-helper.js @@ -5,6 +5,7 @@ import { authenticate, signBtcMessage } from '../auth.js'; import { appOwnerKey } from './keys.js'; import { buildEnterpriseBlob } from './enterprise-helper.js'; import { REGISTRY_REPO_HOST } from './subnet-config.js'; +import { assignPorts } from './port-allocator.js'; import * as daemon from './daemon-control.js'; import { waitFor } from './wait.js'; @@ -25,7 +26,7 @@ const defaultSpec = { // suites 07/08/09 in the 2026-07-02 gate). The env registry is seeded // with this image at bootstrap (test-env.js). repotag: `${REGISTRY_REPO_HOST}/e2e-pause:v1`, - ports: [31111], + ports: [], domains: [''], environmentParameters: [], commands: [], @@ -59,14 +60,31 @@ export function assertHermeticRepotags(spec, allowExternalRepotag) { } } -export function buildAppSpec({ enterprise = false, allowExternalRepotag = false, ...overrides } = {}) { +export function buildAppSpec({ + enterprise = false, allowExternalRepotag = false, allowPortReuse = false, ...overrides +} = {}) { const ownerKey = appOwnerKey(); const spec = { ...defaultSpec, owner: ownerKey.zelid, ...overrides }; assertHermeticRepotags(spec, allowExternalRepotag); - if (overrides.compose) { - spec.compose = overrides.compose; - } + // Copied, never referenced. `{ ...defaultSpec }` is shallow, so spec.compose IS + // defaultSpec.compose - and assignPorts writes the allocated port into the + // component it is given. Without this the first build in a process stamps a port + // into the module-level default, and the second reads it back as a hand-picked + // port inside the allocator's own range and throws. A caller's array is copied + // for the same reason: nothing reads a port back out of what it passed in, and a + // builder that writes into its caller's input is the same defect waiting for the + // first suite that reuses one. + spec.compose = (overrides.compose ?? defaultSpec.compose).map((c) => ({ ...c })); + + // Registered rather than seeded, but a port is a port: an app here and a + // seeded app in the same suite would otherwise be drawing from two spaces + // that nothing keeps apart. Before the blob, because an enterprise spec's + // compose is emptied into it - and before registerApp signs, which is the + // next thing that happens to this object. The allocation is stable per app + // name, so a suite that builds the same app twice to update it does not + // hand itself a port change it never asked for. + if (spec.compose?.length) assignPorts(spec.compose, spec.name, { allowPortReuse }); if (enterprise) { spec.enterprise = buildEnterpriseBlob(spec.compose, spec.contacts); @@ -95,11 +113,20 @@ export async function signAppSpec(spec, type = 'fluxappregister') { return { type, version, appSpecification: spec, timestamp, signature }; } +// The endpoint follows the message type, because they are two different +// handlers: appregister refuses a name that already exists, appupdate refuses +// one that does not. Sending an update to appregister is answered with "Flux App +// already registered", which reads like a harness fault rather than the wrong +// door. +function endpointForType(type) { + return type === 'fluxappupdate' || type === 'zelappupdate' ? 'appupdate' : 'appregister'; +} + export async function registerApp(nodeUrl, adminKeypair, spec, type = 'fluxappregister') { const auth = await authenticate(nodeUrl, adminKeypair); const signed = await signAppSpec(spec, type); - const res = await fetch(`${nodeUrl}/apps/appregister`, { + const res = await fetch(`${nodeUrl}/apps/${endpointForType(type)}`, { method: 'POST', headers: { 'Content-Type': 'text/plain', zelidauth: auth.zelidauth }, body: JSON.stringify(signed), @@ -151,6 +178,18 @@ export async function registerAndConfirm(nodeUrl, adminKeypair, spec, nodes, { }; } +// A spec change, confirmed on chain the same way a registration is. The node +// compares the hash it holds against the one the chain now carries, which is +// what the periodic reinstall pass acts on - so this is the only way to reach +// every path that answers an owner changing a running app. +// +// The spec must differ from the installed one somewhere, or the hash matches and +// nothing happens: the update is accepted and the node correctly does nothing +// with it. +export async function updateAndConfirm(nodeUrl, adminKeypair, spec, nodes, options = {}) { + return registerAndConfirm(nodeUrl, adminKeypair, spec, nodes, { ...options, type: 'fluxappupdate' }); +} + export async function checkPermanentSpec(nodes, appName) { let count = 0; for (const node of nodes) { diff --git a/test-infra/runner/framework/boot-lock.js b/test-infra/runner/framework/boot-lock.js new file mode 100644 index 0000000000..aba77a4891 --- /dev/null +++ b/test-infra/runner/framework/boot-lock.js @@ -0,0 +1,169 @@ +import { readdirSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +// ---- host-wide boot semaphore ---- +// Fleet boot is the heaviest phase a suite has: every node in the fleet starts its +// own dockerd and runs FluxOS DB prep at once. Overlapping boots contend, and a +// healthy node can blow its event-wait budget while merely slow (observed in the +// 42-suite gate: suite 22's second fleet booted at load ~15 on 16 cores and mongo +// collection prep crawled at 7-17s a step). Running fleets are cheap, so bound the +// boot phase host-wide and let everything else overlap. +// +// BOOT_LOCK_WIDTH is how many fleets may boot at once. Two ten-node fleets booting +// together each take ~37s where one alone takes ~25s, so the pair completes in the +// time 1.5 boots would cost serially: they do contend, but the overlap wins. The +// gate is ~89% boot-lock-held wall clock, so that ratio is most of its duration. +// +// The queue is ORDERED BY ARRIVAL, and that is the whole point. A protocol that +// has every waiter race the same atomic create when the lock frees serves whoever +// wakes first, not whoever waited longest, so a suite can lose that race without +// bound while its siblings cycle through boots around it: observed in the +// 2026-08-06 gate, where suite 13 sat 30 minutes through repeated fleet boots by +// other suites and was killed by the runner's wall-clock backstop with 18 of its +// 19 tests already passed. Arrival order makes starvation structural rather than +// statistical - the queue drains in the order it formed. +// +// Each waiter owns one ticket file named `-`. The holder is the +// lowest live ticket. The pid is IN THE NAME so a ticket is created by a single +// atomic operation and can never be read half-written; a ticket whose process is +// gone is removed by whichever waiter notices, which is what reclaims the queue +// after a suite is killed mid-boot. +export const BOOT_LOCK_DIR = process.env.E2E_BOOT_LOCK_DIR ?? join(tmpdir(), 'e2e-boot-lock'); +const BOOT_LOCK_POLL_MS = Number(process.env.E2E_BOOT_LOCK_POLL_MS ?? 250); +// How many fleets may boot at once. Held below MAXN so a gate still spends most of +// itself running suites rather than booting them; 1 restores strict serialisation. +export const BOOT_LOCK_WIDTH = Math.max(1, Number(process.env.E2E_BOOT_LOCK_WIDTH ?? 2)); +// Generous against a FIFO queue: the worst honest wait is the suites ahead of you +// divided by the width, times one boot - ~5 boots at MAXN=6 and width 1, fewer as +// the width rises. Reaching this means the queue is wedged, not busy, and it stays +// well inside run-all.sh's 1800s per-suite backstop so the failure is reported by +// the lock rather than by a SIGKILL that explains nothing. +export const BOOT_LOCK_MAX_WAIT_MS = Number(process.env.E2E_BOOT_LOCK_MAX_WAIT_MS ?? 600000); + +const sleep = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }); + +// Arrival stamps are wall-clock because they are compared ACROSS processes, and a +// monotonic clock has a per-process origin so its values are not comparable +// between them. Elapsed time is measured monotonically below, which is what a +// deadline actually needs. +const ticketOrder = (name) => { + const [ms, pid] = name.split('-'); + return [Number(ms), Number(pid)]; +}; + +// The live queue in service order. Tickets whose process is gone are removed on +// sight, which is what lets the queue drain past a suite killed mid-boot. +export function bootQueue() { + let names; + try { + names = readdirSync(BOOT_LOCK_DIR); + } catch { + return []; + } + const live = []; + for (const name of names.filter((n) => /^\d+-\d+$/.test(n))) { + const [, pid] = ticketOrder(name); + if (pid === process.pid) { live.push(name); continue; } + try { + process.kill(pid, 0); + live.push(name); + } catch { + try { rmSync(join(BOOT_LOCK_DIR, name), { force: true }); } catch { /* raced */ } + } + } + return live.sort((a, b) => { + const [ams, apid] = ticketOrder(a); + const [bms, bpid] = ticketOrder(b); + return ams - bms || apid - bpid; + }); +} + +let heldTicket = null; +let heldSince = null; +let heldFleet = ''; + +// Queued-vs-held is the number the lock's WIDTH turns on, and nothing recorded it: +// a suite's wall time is boot plus however long it sat behind five siblings, and the +// two are indistinguishable from the outside. Emitted as TAP comments because +// run-all.sh pipes mocha's `2>&1` straight into the .tap it later tallies with +// `grep -c '^ok '` - a leading `#` is a comment to any TAP reader and can never be +// counted as a result. +const report = (fields) => { console.log(`# boot-lock ${fields}`); }; + +// The fleet's shape is what explains held_ms, and a duration without it is two +// populations stirred together: nodes boot in parallel, so fleet size moves the cost +// by a few times rather than in proportion, and `syncthing: 'binary'` gives every node +// its own daemon instead of one shared stub. Both axes are needed to tell whether the +// width can vary BY fleet - two small boots overlapping is a different question from +// two ten-node ones. Carried on both lines so each is self-describing: a process takes +// the lock once per fleet it builds, so a pid does not pair an acquire with its release. +const fleetShape = ({ + nodes = 0, deferred = 0, legacy = 0, syncthing = 'stub', +} = {}) => `nodes=${nodes} deferred=${deferred} legacy=${legacy} syncthing=${syncthing}`; + +export async function acquireBootLock(fleet) { + mkdirSync(BOOT_LOCK_DIR, { recursive: true }); + const ticket = `${Date.now()}-${process.pid}`; + writeFileSync(join(BOOT_LOCK_DIR, ticket), ''); + heldTicket = ticket; + heldFleet = fleetShape(fleet); + const startedAt = process.hrtime.bigint(); + let aheadOnArrival; // undefined until the first queue read; null = position unknown + for (;;) { + const queue = bootQueue(); + if (aheadOnArrival === undefined) { + // Same -1 the wait error below refuses to conflate. A ticket that is not in + // the queue has no position, and reporting one as 0 reads as "arrived to an + // empty queue" - the most misleading value available, because bootQueue() + // returns [] exactly when the directory cannot be read, which is the + // defeated-semaphore case this lock exists to make visible. + const arrivalPosition = queue.indexOf(ticket); + aheadOnArrival = arrivalPosition < 0 ? null : arrivalPosition; + } + const waitedMs = Number((process.hrtime.bigint() - startedAt) / 1000000n); + // Arrival order still decides service; the width only changes how many of the + // front of the queue are being served at once. A ticket that is NOT in the + // queue indexes to -1, which is inside any width - so the position has to be + // real before it can be compared. bootQueue() returns [] whenever the + // directory cannot be read, and run-parallel.sh removes that directory, so + // without this every waiter would read "I hold the lock" at the same moment + // and the semaphore would be silently defeated - exactly the contention it + // exists to prevent. Holding instead means the wait ends at the explicit + // wedged error, which is the honest signal. + const position = queue.indexOf(ticket); + if (position >= 0 && position < BOOT_LOCK_WIDTH) { + heldSince = process.hrtime.bigint(); + report(`acquired waited_ms=${waitedMs} ahead_on_arrival=${aheadOnArrival ?? 'unknown'} width=${BOOT_LOCK_WIDTH} ${heldFleet} pid=${process.pid}`); + return; + } + if (waitedMs > BOOT_LOCK_MAX_WAIT_MS) { + const ahead = queue.indexOf(ticket); + const holders = queue.slice(0, BOOT_LOCK_WIDTH).map((t) => ticketOrder(t)[1]); + releaseBootLock(); + throw new Error( + `boot lock: waited ${Math.round(waitedMs / 1000)}s for ${BOOT_LOCK_DIR}, ` + + `still ${ahead < 0 ? 'unknown' : ahead} ahead in the queue ` + + `(width ${BOOT_LOCK_WIDTH}, holder pids ${holders.length ? holders.join(',') : 'none'}). ` + + 'The queue is wedged, not merely busy.', + ); + } + await sleep(BOOT_LOCK_POLL_MS); + } +} + +export function releaseBootLock() { + if (!heldTicket) return; + // Null when the wait timed out rather than succeeded, which is a queue that never + // held the lock and must not report a boot duration. + if (heldSince !== null) { + report(`released held_ms=${Number((process.hrtime.bigint() - heldSince) / 1000000n)} ${heldFleet} pid=${process.pid}`); + heldSince = null; + } + try { + rmSync(join(BOOT_LOCK_DIR, heldTicket), { force: true }); + } catch { + // already released or reclaimed + } + heldTicket = null; +} diff --git a/test-infra/runner/framework/chain-start.cjs b/test-infra/runner/framework/chain-start.cjs new file mode 100644 index 0000000000..043aab4b73 --- /dev/null +++ b/test-infra/runner/framework/chain-start.cjs @@ -0,0 +1,24 @@ +// The block height the harness chain starts at, and the single place it is written. +// +// It has to sit ABOVE every block-height gate in ZelBack/config/default.js, or the +// suites silently run on the wrong side of a fork: the validator takes a height as +// an argument rather than reading the tip, so a chain that starts below a fork +// exercises the pre-fork branch of every rule keyed on it. +// +// That is exactly how the minimumInstancesV8Block override came to exist. The start +// was set just above daemonPONFork (2020000) when that was the highest gate, a +// higher one landed later at 2176519, and the harness compensated by lowering the +// FORK for every suite - so the rule under test stopped being the production rule. +// +// CommonJS, and required rather than duplicated, because the guard test in +// tests/unit reads this same file. Two copies of this number is the trap being +// removed, not a detail. +// +// Kept in step with test-infra/fixtures/mongo-init.js, which carries the same +// number as a first-boot default because it runs under mongosh and cannot +// require this file. +// +// Pinned by tests/unit/harnessChainStart.test.js, which fails if any gate in the +// production config rises above it. A suite that WANTS to be before a fork asks +// for it: createTestEnv({ initialHeight }). +module.exports = { DEFAULT_INITIAL_HEIGHT: 2952000 }; diff --git a/test-infra/runner/framework/container.js b/test-infra/runner/framework/container.js index f74af3354f..7d250c151b 100644 --- a/test-infra/runner/framework/container.js +++ b/test-infra/runner/framework/container.js @@ -1,14 +1,103 @@ +import { throwIfInfraDead, sleepUnlessInfraDead } from './infra-death.js'; + export async function execInContainer(container, command) { const args = Array.isArray(command) ? command : ['sh', '-c', command]; const result = await container.exec(args); return { stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, output: result.output }; } +// Move a node to a different address on the fleet network: the new one goes on, +// the old one comes off. +// +// This is what an address change IS, and doing it for real is what makes the rest +// of the fixture honest. Its peers then find it unreachable at the old address +// because it genuinely is not there - no packet filter simulating it, and nothing +// hidden from the node list, so they still recognise it as the sender of the +// fluxipchanged broadcast that follows. +// +// The timing matters and is measured. A probe to an address that is gone fails +// with EHOSTUNREACH at ~3.1s (ARP gives up), NOT a hang - so a peer's own probe +// budget of 5s sees a failure rather than a timeout, and it answers the asking +// node inside that node's 7s budget. Those margins are why this works where +// dropping packets did not: a dropped probe burns the full 5s, and the answer +// then arrives after the asker has already given up, which reads as "I could not +// ask" rather than "you are unreachable" - a different branch entirely, and one +// that never consults benchmark. +// +// @param {object} container The node's container. +// @param {string} to Bare address to move to, inside the fleet's own /24. +// @param {string} from Bare address to give up. +export async function moveNodeAddress(container, to, from, { prefix = 24, iface = 'eth0' } = {}) { + const add = await execInContainer(container, `ip addr add ${to}/${prefix} dev ${iface}`); + if (add.exitCode !== 0 && !/File exists/i.test(add.output || '')) { + throw new Error(`moveNodeAddress: could not add ${to}/${prefix} to ${iface}: ${add.output}`); + } + const del = await execInContainer(container, `ip addr del ${from}/${prefix} dev ${iface}`); + if (del.exitCode !== 0 && !/Cannot assign|not exist/i.test(del.output || '')) { + throw new Error(`moveNodeAddress: could not remove ${from}/${prefix} from ${iface}: ${del.output}`); + } + return to; +} + +// Make a node unreachable to the named peers, without taking it off the network. +// +// The node keeps its address, its list entry and its outbound connections; what +// stops is inbound traffic to its API port FROM those peers. That is what a node +// whose address has moved looks like from the outside - still listed where it was, +// no longer answering there - and it is the state that makes a peer's availability +// probe fail, which is what a node needs before it will ask benchmark whether its +// address changed. +// +// REJECT rather than DROP, and the difference decides whether this works at all. +// A peer asked whether it can reach this node probes it and answers within the +// asker's own timeout budget. Dropped packets blackhole, so that probe burns its +// full timeout and the peer answers too late - the asker times out on the PEER and +// reads "I could not ask" instead of "I am unreachable", which retries without ever +// consulting benchmark. Refusing fails the probe instantly, so the answer arrives +// in time and says what it is meant to say. +// +// Named peers rather than the subnet: the runner reaches the node from the docker +// gateway on that same /24, so a blanket rule would cut off the very client doing +// the asserting. +// +// @param {object} container The node's container. +// @param {string[]} peerIps Bare addresses whose traffic to drop. +// @param {number} apiPort The node's API port. +export async function blockPeerAccess(container, peerIps, apiPort) { + for (const peerIp of peerIps) { + // eslint-disable-next-line no-await-in-loop + const r = await execInContainer(container, `iptables -I INPUT -p tcp --dport ${apiPort} -s ${peerIp} -j REJECT --reject-with tcp-reset`); + if (r.exitCode !== 0) { + throw new Error(`blockPeerAccess: could not drop ${peerIp} -> :${apiPort}: ${r.output}`); + } + } + return peerIps; +} + +// Undo blockPeerAccess. Tolerates a rule that is already gone so teardown after a +// failed test cannot fail in its own right. +export async function unblockPeerAccess(container, peerIps, apiPort) { + for (const peerIp of peerIps) { + // eslint-disable-next-line no-await-in-loop + await execInContainer(container, `iptables -D INPUT -p tcp --dport ${apiPort} -s ${peerIp} -j REJECT --reject-with tcp-reset`); + } +} + +// THE READ CAN FAIL, AND SAYS SO. `2>/dev/null || echo ""` gave a broken docker +// exec the same answer as a node with no containers on it - an empty list - so +// every caller read "the app is not running" and every wait built on one spent +// its whole budget and ended reporting only that the condition never held. A +// failed read is not an observation, and the callers that poll are built to +// retry a throw; the ones that assert have no business ruling on a look they +// never took. export async function listAppContainers(container, { all = false } = {}) { const flag = all ? ' -a' : ''; - const { stdout } = await execInContainer(container, - `docker ps${flag} --format "{{.Names}}\t{{.Status}}\t{{.Image}}" 2>/dev/null || echo ""`, + const { stdout, stderr, exitCode } = await execInContainer(container, + `docker ps${flag} --format "{{.Names}}\t{{.Status}}\t{{.Image}}"`, ); + if (exitCode !== 0) { + throw new Error(`docker ps in the node container failed (exit ${exitCode}): ${(stderr || stdout || '').trim()}`); + } return stdout.trim().split('\n') .filter((line) => line && !line.includes('NAMES')) .map((line) => { @@ -50,6 +139,18 @@ export async function crashAppContainer(container, appName, componentName) { return execInContainer(container, `docker kill ${appContainerName(appName, componentName)}`); } +// The container's docker id, which is what distinguishes a container that was +// REPLACED from one that was merely restarted: a redeploy removes and recreates, +// so the id changes, while a restart keeps it. Status and image name are equal +// either way, so neither can tell the two apart. null if the container is absent. +export async function getAppContainerId(container, appName, componentName) { + const { stdout } = await execInContainer(container, + `docker inspect --format '{{.Id}}' ${appContainerName(appName, componentName)} 2>/dev/null || echo ""`, + ); + const id = stdout.trim(); + return id === '' ? null : id; +} + // the actual exit code the reconciler reads from Docker (null if container absent) export async function getAppContainerExitCode(container, appName, componentName) { const { stdout } = await execInContainer(container, @@ -71,13 +172,15 @@ export async function restartDockerd(container, { readyTimeoutMs = 40000, interv const start = Date.now(); let sawDown = false; while (Date.now() - start < readyTimeoutMs) { + // an infra death voids the run - don't spend the budget proving it + throwIfInfraDead(); // eslint-disable-next-line no-await-in-loop const r = await execInContainer(container, 'docker info > /dev/null 2>&1'); const up = r.exitCode === 0; if (!up) sawDown = true; if (sawDown && up) return; // eslint-disable-next-line no-await-in-loop - await new Promise((res) => setTimeout(res, interval)); + await sleepUnlessInfraDead(interval); } throw new Error(`restartDockerd: dockerd did not cycle down and back up within ${readyTimeoutMs}ms`); } @@ -99,13 +202,15 @@ export async function restartFluxos(container, { apiPort = 16127, readyTimeoutMs const start = Date.now(); let sawDown = false; while (Date.now() - start < readyTimeoutMs) { + // an infra death voids the run - don't spend the budget proving it + throwIfInfraDead(); // eslint-disable-next-line no-await-in-loop const r = await execInContainer(container, probe); const up = r.exitCode === 0; if (!up) sawDown = true; if (sawDown && up) return; // eslint-disable-next-line no-await-in-loop - await new Promise((res) => setTimeout(res, interval)); + await sleepUnlessInfraDead(interval); } throw new Error(`restartFluxos: FluxOS did not cycle down and back up within ${readyTimeoutMs}ms`); } diff --git a/test-infra/runner/framework/control-fetch.js b/test-infra/runner/framework/control-fetch.js new file mode 100644 index 0000000000..d748661e77 --- /dev/null +++ b/test-infra/runner/framework/control-fetch.js @@ -0,0 +1,58 @@ +// Talking to a stub's control API or a node's own API, with the failure legible. +// +// `fetch` reports every transport failure as the same three words - "TypeError: +// fetch failed" - and puts the part that identifies it (ECONNREFUSED, socket +// hang up, EAI_AGAIN) in `cause`, which nothing prints. A suite that loses a +// control call therefore fails with no endpoint, no errno and no stack into the +// harness, and the only way to find out which of a dozen control APIs went +// quiet is to run it again with a guess bolted on. +// +// That cost this suite three ten-minute runs. So every control call goes through +// here, and a failure names the method, the URL and the cause. +// +// The node clients go through it too. They were left on bare `fetch` when this +// was written, and a gate found the gap the same way: a suite that passes twice +// on an idle box loses a wait to one unattributable `fetch failed` under load, +// and the report names neither which node nor why. + +/** + * fetch, with the failure identifying itself. + * @param {string} url Absolute control-API URL. + * @param {object} [init] fetch init. + * @returns {Promise} + */ +export async function controlFetch(url, init) { + try { + return await fetch(url, init); + } catch (error) { + const method = init?.method ?? 'GET'; + const cause = error?.cause; + const detail = cause + ? `${cause.code ?? cause.name ?? 'unknown'}${cause.message ? `: ${cause.message}` : ''}` + : 'no cause reported'; + const wrapped = new Error(`${method} ${url} failed - ${detail}`); + wrapped.cause = error; + throw wrapped; + } +} + +/** + * The same, parsed as JSON. A control API that answers with a body the caller + * cannot read is its own failure, and it reads identically to a transport one + * unless it says so. + * @param {string} url Absolute control-API URL. + * @param {object} [init] fetch init. + * @returns {Promise} + */ +export async function controlJson(url, init) { + const res = await controlFetch(url, init); + const text = await res.text(); + try { + return JSON.parse(text); + } catch { + throw new Error( + `${init?.method ?? 'GET'} ${url} answered ${res.status} with a body that is not JSON: ` + + `${text.slice(0, 200)}`, + ); + } +} diff --git a/test-infra/runner/framework/coupled-knobs.js b/test-infra/runner/framework/coupled-knobs.js new file mode 100644 index 0000000000..da20007a2f --- /dev/null +++ b/test-infra/runner/framework/coupled-knobs.js @@ -0,0 +1,278 @@ +// Harness knobs that only mean anything RELATIVE to another knob. +// +// A compressed harness is a set of ratios, not a set of numbers. Compress two +// coupled knobs by different factors and the property between them does not get +// faster - it inverts, and the suite that was written to prove it goes green +// while proving the opposite. +// +// That is not hypothetical here. residentialQueueStepMs was set to 15s against +// a pass the comment beside it called "about 4s", giving an apparent 3.75x +// margin. The pass is a function of explorerPollIntervalMs - a block costs one +// poll - and when that moved 250ms -> 833ms the pass moved with it to ~16s. +// Nothing re-derived the step, so the harness ended up at 0.94x, BELOW one, +// while production sits at 1.8x. Two nodes then matured on the same pass and +// both handed the same app back - the exact defect production's own 15-minute +// step had, which was a merge blocker on this branch. +// +// So the numbers below are derived from production's ratios and checked at +// fleet boot, for every suite, against whatever that suite overrode. + +import fs from 'node:fs'; +import path from 'node:path'; +import vm from 'node:vm'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SHARED_PATH = path.join(HERE, '../../config/shared.js'); + +/** + * config/shared.js is CJS text inside a package declaring "type": "module", so + * it can be neither imported nor required - it is evaluated. + * @returns {object} The shared harness config. + */ +export function loadSharedConfig() { + const sandbox = { module: { exports: {} }, exports: null }; + sandbox.exports = sandbox.module.exports; + vm.runInNewContext(fs.readFileSync(SHARED_PATH, 'utf8'), sandbox); + return sandbox.module.exports; +} + +// Production's side of every ratio here. Held as constants rather than read +// from ZelBack/config/default.js because that file requires the gitignored +// config/userconfig.js; tests/unit/coupledKnobs.test.js asserts they still +// match the fleet's, so drift fails a test rather than silently weakening the +// check. +export const PRODUCTION = Object.freeze({ + blockMs: 30000, // post-PON; stated in-repo at fluxService.js:1774 + removeFluxAppsPeriod: 11, + residentialQueueBaseMs: 30 * 60 * 1000, + residentialQueueStepMs: 40 * 60 * 1000, + locationTtlS: 7500, + sigtermExpiryS: 420, +}); + +// What one node boot costs, measured: a suite-19 fixture pinning 300s of +// downtime was read by the node as 316s, on cindy under a MAXN=6 gate - so this +// is a loaded figure, not an idle-box one. Any window a fixture has to be +// measured INSIDE must clear it with room, because the boot lands in the middle +// of the measurement and no ratio shrinks it. +export const BOOT_DRIFT_MS = 16000; + +// explorerService.js:610. Applies to both sides, so it cancels out of the +// ratio - named anyway, because the pass interval is not readable without it. +export const PON_SPEED_MULTIPLIER = 4; + +// A block costs at least one poll, and in practice more: processing, the +// database write and the maintenance hung off it all land between polls. +// Measured on cindy 2026-08-20 at explorerPollIntervalMs 833 - modelled pass +// 13.3s, observed 15.9s over nine consecutive give-up passes. Applied so the +// model is not optimistic, because optimism here derives a step that is too +// SHORT, which is the direction that loses the property. +export const BLOCK_COST_OVERHEAD = 1.2; + +/** + * How long between two runs of the give-up pass. + * + * The pass runs every removeFluxAppsPeriod * PON_SPEED_MULTIPLIER blocks. What a + * block COSTS is the only part that differs between production and the harness: + * production waits out a real 30s block, the harness drives its own and pays one + * explorer poll for each. + * @param {{removeFluxAppsPeriod: number}} fluxapps Effective app config. + * @param {number} blockCostMs What one block costs to reach and process. + * @returns {number} Milliseconds between passes. + */ +export function giveUpPassMs(fluxapps, blockCostMs) { + return fluxapps.removeFluxAppsPeriod * PON_SPEED_MULTIPLIER * blockCostMs; +} + +/** What one block costs the harness: a poll, plus what processing adds. */ +export function harnessBlockCostMs(fluxapps) { + return fluxapps.explorerPollIntervalMs * BLOCK_COST_OVERHEAD; +} + +/** + * The ratio the queue step has to hold against the pass, taken from production. + * + * Above 1 is the property; production's specific margin is what the fleet has + * been reasoned about, so the harness matches it rather than merely clearing 1. + * @returns {number} + */ +export function productionQueueRatio() { + const pass = giveUpPassMs(PRODUCTION, PRODUCTION.blockMs); + return PRODUCTION.residentialQueueStepMs / pass; +} + +/** + * The queue step a suite should use, derived rather than chosen. + * + * Call this instead of writing a number: it moves when explorerPollIntervalMs + * or removeFluxAppsPeriod moves, which is the whole failure this file exists + * for. + * @param {{removeFluxAppsPeriod: number, explorerPollIntervalMs: number}} fluxapps + * @returns {number} residentialQueueStepMs, in milliseconds. + */ +// How many queue steps a ticket tolerates going unobserved before it starts +// again. Mirrors MAX_TICKET_GAP_MS in residentialNodeDosService: a gap has to +// mean a pass was MISSED, and one step is only 1.82 passes - so at one step a +// single late pass restarts the ticket. Production hardly notices; the harness +// compresses the same ratio to about 30 seconds, where six fleets booting at +// once make a late pass ordinary, and the ticket then never matures at all. +// Absolute jitter does not compress with the clocks. +export const TICKET_GAP_STEPS = 2; + +export function derivedQueueStepMs(fluxapps) { + const pass = giveUpPassMs(fluxapps, harnessBlockCostMs(fluxapps)); + return Math.ceil((pass * productionQueueRatio()) / 1000) * 1000; +} + +/** + * The departure interval a suite should use, derived rather than chosen. + * + * A node inside its departure interval records nothing against its queue + * tickets, so the block has to read afterwards as a gap the ticket will not + * carry across - otherwise a departure stops restarting the queue and position + * separates the first departure and nothing after it. Bounded by what it must + * OUTLIVE rather than by production's ratio, the same way the boot-drift rule + * below is: the ticket's tolerance is MAX_TICKET_GAP_MS, which IS the queue + * step. One extra pass on top, so the restart cannot land ambiguously on the + * pass grid. + * @param {{removeFluxAppsPeriod: number, explorerPollIntervalMs: number, residentialQueueStepMs: number}} fluxapps + * @returns {number} residentialEvacuationIntervalMs, in milliseconds. + */ +export function derivedEvacuationIntervalMs(fluxapps) { + const pass = giveUpPassMs(fluxapps, harnessBlockCostMs(fluxapps)); + const step = fluxapps.residentialQueueStepMs ?? derivedQueueStepMs(fluxapps); + return Math.ceil(((step * TICKET_GAP_STEPS) + pass) / 1000) * 1000; +} + +/** + * How long ONE departure takes end to end, for a suite that has to wait for it. + * + * A departure is not just the removal. The node serves its departure interval, + * and then serves its queue ticket AGAIN from scratch - the interval reads as a + * gap and restarts it, which is the point of the interval - and the ticket is + * base plus position times step, the worst position being one short of the + * instance count. + * + * Derived because it MOVED. Suite 55's waits were four minutes against a + * four-second interval; the interval is now tens of seconds, and a hand-typed + * four minutes quietly stopped covering a single departure. A wait is as coupled + * to the pacing as the step is to the pass, and belongs here for the same reason. + * @param {object} fluxapps Effective fluxapps config for the fleet. + * @param {number} instances How many instances the app under test carries. + * @returns {number} Milliseconds one departure can take, at the worst position. + */ +export function departureCycleMs(fluxapps, instances) { + const step = fluxapps.residentialQueueStepMs ?? derivedQueueStepMs(fluxapps); + const base = fluxapps.residentialQueueBaseMs ?? PRODUCTION.residentialQueueBaseMs; + const interval = fluxapps.residentialEvacuationIntervalMs ?? derivedEvacuationIntervalMs(fluxapps); + return interval + base + (Math.max(instances - 1, 0) * step); +} + +/** + * Refuse to boot a fleet whose coupled knobs do not hold production's ratios. + * + * Runs on the EFFECTIVE config - shared.js plus whatever the suite overrode - + * because the override is where this went wrong, not the shared file. Throws + * rather than warning: a fleet configured this way produces a green suite that + * has stopped testing its property, which is worse than no run. + * + * Over production's ratio is fine and is not flagged. A suite may deliberately + * leave a knob uncompressed, and a step longer than it needs only costs time. + * UNDER is the failure, because that is where the property inverts. + * @param {object} fluxapps Effective fluxapps config for the fleet. + * @throws {Error} When a ratio is below production's. + */ +export function assertSigtermOrdering(fluxapps) { + const sigtermMs = (fluxapps.sigtermExpiryS ?? PRODUCTION.sigtermExpiryS) * 1000; + const runningMs = (fluxapps.locationTtlS ?? PRODUCTION.locationTtlS) * 1000; + + // appStartupManager: (cleanShutdown && downtime > sigterm) || downtime > + // running. Above the running expiry this window is unreachable and a clean + // shutdown gets no grace, which is the opposite of what it is for. Production + // holds 420s under 7500s. + if (sigtermMs >= runningMs) { + throw new Error( + 'coupled-knobs: sigtermExpiryS is not below locationTtlS.\n' + + ` sigterm ${sigtermMs}ms, running ${runningMs}ms\n` + + ' appStartupManager expires on (cleanShutdown && downtime > sigterm) || downtime >\n' + + ' running, so at this ordering the running expiry fires first and the clean-shutdown\n' + + ' grace can never be reached.', + ); + } + + // A fixture asserting "within the window" is measured across a node boot, and + // the boot lands inside the measurement. A window at or under the drift can + // never be tested from the inside, whatever the fixture pins. + if (sigtermMs <= BOOT_DRIFT_MS) { + throw new Error( + 'coupled-knobs: sigtermExpiryS is at or below one node boot.\n' + + ` sigterm ${sigtermMs}ms, measured boot drift ${BOOT_DRIFT_MS}ms\n` + + ' A fixture pinning any downtime is read by the node as that downtime PLUS a boot,\n' + + ' so nothing can land inside this window. It is bounded by what it must outlive,\n' + + ' not by production\'s ratio - a boot is not a compressed clock.', + ); + } +} + +/** + * Refuse to boot a fleet whose departure interval is shorter than the gap its + * queue tickets tolerate. + * + * mayEvacuateApp records nothing while the interval gate is refusing, so the + * block is meant to read afterwards as one long gap and restart every ticket. + * That is what keeps position binding on the SECOND departure and every one + * after it. Compress the interval below the step and the block stops looking + * like a gap: tickets carry straight across it, every app is instantly ready + * the moment the block clears, and two holders whose blocks expire in the same + * pass hand back the same app together - the same defect a too-short step + * causes, through the other door, and just as invisible to a green suite. + * + * Production holds 6h against a 40min step, ~9x. The bound here is 1x plus a + * pass, because this pair is fixed by what it must outlive rather than by a + * ratio anyone reasoned about. + * @param {object} fluxapps Effective fluxapps config for the fleet. + * @throws {Error} When the interval does not outlive the ticket gap. + */ +export function assertDepartureOutlivesTicket(fluxapps) { + const interval = fluxapps.residentialEvacuationIntervalMs; + if (!interval) return; + const required = derivedEvacuationIntervalMs(fluxapps); + if (interval >= required) return; + throw new Error( + 'coupled-knobs: residentialEvacuationIntervalMs does not outlive the queue ticket.\n' + + ` interval ${interval}ms\n` + + ` step ${fluxapps.residentialQueueStepMs}ms x ${TICKET_GAP_STEPS} = the ticket's gap tolerance\n` + + ` needed ${required}ms -> that tolerance plus one give-up pass\n` + + ' Below this a departure no longer restarts the other holders\' tickets, so position\n' + + ' separates the first departure and nothing after it. Use\n' + + ' derivedEvacuationIntervalMs(fluxapps) rather than a literal.', + ); +} + +/** + * Every coupled-knob rule this harness enforces, in one call. + * @param {object} fluxapps Effective fluxapps config for the fleet. + * @throws {Error} When any relationship does not hold. + */ +export function assertCoupledRatios(fluxapps) { + if (!fluxapps) return; + assertSigtermOrdering(fluxapps); + if (!fluxapps.residentialQueueStepMs) return; + assertDepartureOutlivesTicket(fluxapps); + const blockCost = harnessBlockCostMs(fluxapps); + const pass = giveUpPassMs(fluxapps, blockCost); + const ratio = fluxapps.residentialQueueStepMs / pass; + const required = productionQueueRatio(); + if (ratio >= required) return; + throw new Error( + 'coupled-knobs: residentialQueueStepMs is too short for this fleet\'s give-up pass.\n' + + ` pass ${Math.round(pass)}ms (removeFluxAppsPeriod ${fluxapps.removeFluxAppsPeriod}` + + ` x ${PON_SPEED_MULTIPLIER} blocks, each costing ${Math.round(blockCost)}ms` + + ` at explorerPollIntervalMs ${fluxapps.explorerPollIntervalMs})\n` + + ` step ${fluxapps.residentialQueueStepMs}ms -> ratio ${ratio.toFixed(2)}\n` + + ` needed ${Math.round(pass * required)}ms -> production's ratio ${required.toFixed(2)}\n` + + ' Two holders of one app mature on the same pass at this ratio and both hand it\n' + + ' back. Use derivedQueueStepMs(fluxapps) rather than a literal.', + ); +} diff --git a/test-infra/runner/framework/daemon-control.js b/test-infra/runner/framework/daemon-control.js index 3666682e81..6332383b13 100644 --- a/test-infra/runner/framework/daemon-control.js +++ b/test-infra/runner/framework/daemon-control.js @@ -1,4 +1,5 @@ import { getSubnetConfig } from './subnet-config.js'; +import { loadSharedConfig } from './coupled-knobs.js'; const CONTROL = process.env.DAEMON_CONTROL || `http://${getSubnetConfig().daemon}:18232`; @@ -48,6 +49,102 @@ export async function advanceBlocks(count) { } } +/** + * Drive the chain until a condition holds, at a stated RATE, up to a stated + * deadline. + * + * Two reasons to drive rather than let the ticker free-run: + * + * The ticker produces blocks on the same period the explorer polls on, so the + * node learns about them in bursts and processes a burst back to back - and only + * the last block of a burst is still the chain tip. FluxOS runs its app + * maintenance, the give-up pass included, only for a block that was the tip and + * only on every Nth block, so whether the pass runs at all comes down to which + * parity the race settles on. Suite 96 asserted that nothing had happened after + * a 24-block burst and was right for the wrong reason: the pass had run once + * across five nodes, so there was nothing for the assertion to have caught. + * + * And these waits are WALL-CLOCK. A node's queue ticket is real time, so a + * budget counted in blocks means nothing: the deadline is therefore in + * milliseconds. The chain's actual rate is not this function's to set - a block + * is not processed until the node's next explorer poll, so + * `explorerPollIntervalMs` is the floor, and that floor is what fixes how long a + * give-up pass takes in wall-clock, which is what the queue step has to outlast. + * Changing one without the other is what quietly deletes the ordering those + * tests exist to check. + * + * THE RATE IS THE NODE'S POLL, not something faster. Driving four blocks per + * poll produces four times the work for the same pass cadence: a height only + * counts when it lands as the tip, tips arrive one per poll, and the pass comes + * every `removeFluxAppsPeriod x N` polls whatever rate this drives at. The extra + * blocks are pure load. At 200ms one suite drove about a thousand blocks per + * wait, held a runner slot for the full 1800s wall clock and starved the box: + * two unrelated suites ran 3-4x slower in the same gate and hit that wall + * themselves. + * + * @param {object} node The node client to pace against - the one whose + * `block:processed` decides when the next block may be sent. A CLIENT, not an + * index: the two suites that grew this independently disagreed about whether + * the index was 0- or 1-based, which is not a mistake worth leaving available. + * BUDGETED IN THE UNIT THE CONDITION IS COUNTED IN. A pass that fires on + * `blockHeight % N === 0` is waiting for BLOCKS, and a wall-clock budget buys a + * number of them that depends on how busy the box is: 420s buys about 504 + * blocks against an idle daemon and about 84 when the node is slow to process + * them, so the pass gets six times fewer chances with nothing in the suite + * saying so. `blocks` is the same count on any box - it takes longer to spend, + * which is the point. + * + * `timeoutMs` is for a wait whose cadence is a real interval - a departure + * cycle, an election cycle - because seconds are the unit it is counted in. + * Exactly one of the two, because a block budget with a wall clock beside it is + * bounded by whichever runs out first, and under load that is always the clock. + * + * A block budget carries no outer clock. A node that has stopped processing is + * caught by the per-block wait below, which gives every block 60s of its own. + * + * @param {Function} condition Async predicate; driving stops when it holds. + * @param {object} opts Exactly one of `blocks` or `timeoutMs`. + * @param {number} [opts.blocks] Blocks to drive before giving up. + * @param {number} [opts.timeoutMs] How long to keep driving before giving up. + * @param {number} [opts.blockIntervalMs] Minimum wall-clock between blocks. + * @param {string} [opts.label] What the caller was waiting for, for the error. + * @returns {Promise} Blocks driven. + */ +export async function driveUntil(node, condition, { + blocks: blockBudget, timeoutMs, blockIntervalMs, label, +} = {}) { + const budgetedInBlocks = Number.isFinite(blockBudget); + if (budgetedInBlocks === Number.isFinite(timeoutMs)) { + // No default for either. Every caller's budget is derived from something of + // its own, so a shared default would be one suite's number silently applied + // to another's wait - and naming both leaves the wait bounded by whichever + // expires first rather than by the one the caller reasoned about. + throw new Error('driveUntil: exactly one of `blocks` or `timeoutMs` is required'); + } + const interval = blockIntervalMs ?? loadSharedConfig().fluxapps.explorerPollIntervalMs; + const deadline = budgetedInBlocks ? Infinity : Date.now() + timeoutMs; + let blocks = 0; + while (budgetedInBlocks ? blocks < blockBudget : Date.now() < deadline) { + // eslint-disable-next-line no-await-in-loop + if (await condition()) return blocks; + const startedAt = Date.now(); + const afterId = node.getLastEventId(); + // eslint-disable-next-line no-await-in-loop + await advanceBlock(); + // eslint-disable-next-line no-await-in-loop + await node.waitForEvent('block:processed', () => true, 60000, { afterId }); + blocks += 1; + const spent = Date.now() - startedAt; + // eslint-disable-next-line no-await-in-loop + if (spent < interval) await new Promise((resolve) => { setTimeout(resolve, interval - spent); }); + } + if (await condition()) return blocks; + const budgetSpent = budgetedInBlocks + ? `${blocks} blocks driven of the ${blockBudget} budgeted` + : `${timeoutMs}ms (${blocks} blocks driven)`; + throw new Error(`${label ? `${label}: ` : ''}condition not reached in ${budgetSpent}`); +} + export async function setHeight(height) { return post('/set-height', { height }); } @@ -56,6 +153,57 @@ export async function queueAppTx(appHash) { return post('/queue-app-tx', { appHash }); } +// -- Where the network believes a node is -- + +/** + * Move a node's address as the whole network sees it: what benchmark tells the + * node about itself (which is where it learns its address changed), what + * getpublicip answers, its node status, and its entry in the deterministic list. + * + * The container is untouched. `node` is where it really is - the address its + * requests arrive from - and that never moves, so the fleet stays reachable and + * every other node keeps talking to it exactly as before. + * + * A bare address keeps the node's own api port. + * + * `scope: 'all'` (default) moves every answer at once - an address change already + * settled. `scope: 'publicip'` moves only benchmark's public-IP probe, leaving the + * node listed and reporting itself where it was: the state a node is in the moment + * its address moves, and the only one in which it can notice. + * + * @param {string} node Where the node really is. + * @param {string} reported The address the network should now carry for it. + */ +export async function setNodeAddress(node, reported, { scope = 'all' } = {}) { + return post(`/node-address/${String(node).split(':')[0]}`, { reported, scope }); +} + +/** + * Hide a node from its peers' view of the network, so they answer "not available" + * for it without probing anything. + * + * A peer asked whether it can reach a node consults its node list first and answers + * outright when the address is not in it. That is an answer, not a timeout, so it + * arrives inside the asker's budget every time - which is what makes an unreachable + * node a deterministic fixture rather than a race. + * + * The node keeps seeing itself: it has its own confirmed-list gate to pass before it + * will run the availability check at all. + */ +export async function hideNodeFromPeers(node) { + return post(`/node-visibility/${String(node).split(':')[0]}`, { hidden: true }); +} + +/** Put a node back into its peers' view. */ +export async function revealNodeToPeers(node) { + return post(`/node-visibility/${String(node).split(':')[0]}`, { hidden: false }); +} + +/** Put a node's address back to where it really is. */ +export async function clearNodeAddress(node) { + return post(`/node-address/${String(node).split(':')[0]}`, {}); +} + // -- Per-node status -- export async function setNodeStatus(ip, status) { @@ -106,6 +254,20 @@ export async function setNodeTier(ip, tier) { return post(`/node-tier/${ip}`, { tier }); } +// -- ArcaneOS attestation -- + +/** + * Set whether a node's benchmark reports it as attested (ArcaneOS). Nodes are + * attested by default, so only a suite that cares has to say anything. + */ +export async function setSystemSecure(ip, secure) { + return post(`/system-secure/${ip}`, { secure }); +} + +export async function clearSystemSecure() { + return del('/system-secure'); +} + // -- RPC failure -- export async function enableRpcFailure(ip) { diff --git a/test-infra/runner/framework/db-client.js b/test-infra/runner/framework/db-client.js index 5fb6d7df1c..9364a3e35d 100644 --- a/test-infra/runner/framework/db-client.js +++ b/test-infra/runner/framework/db-client.js @@ -128,6 +128,31 @@ export function dbClient(nodeNum) { return cpDb.collection('chainmessages').find({}).toArray(); }, + // residentialNodeDosService's settling window. Persisted so that restarting + // FluxOS cannot restart the clock; a suite reads it to tell "held" from + // "evacuating", and writes it to put the window in the past. + async residentialMarker() { + const localDb = await db('local'); + return localDb.collection('nodestartuptracker').findOne({ _id: 'residentialDos' }); + }, + + // Serve the settling window without waiting it out. + // + // The gate counts time the node OBSERVED the verdict, not time that passed, + // so backdating residentialSince alone no longer serves it - that field is + // kept for the operator reading the record. observedMs is what the node + // compares against residentialSettleMs, and lastConfirmedAt is set to now so + // the node's next tick credits nothing on top and the total stays put. + async serveSettleWindow(observedMs = 48 * 60 * 60 * 1000) { + const localDb = await db('local'); + const now = Date.now(); + await localDb.collection('nodestartuptracker').updateOne( + { _id: 'residentialDos' }, + { $set: { residentialSince: now - observedMs, lastConfirmedAt: now, observedMs } }, + { upsert: true }, + ); + }, + async geolocation() { const localDb = await db('local'); return localDb.collection('geolocation').findOne({ _id: 'nodeGeolocation' }); @@ -170,36 +195,6 @@ export function dbClient(nodeNum) { ); }, - async seedGeolocation(ip) { - const localDb = await db('local'); - await localDb.collection('geolocation').updateOne( - { _id: 'nodeGeolocation' }, - { - $set: { - geolocation: { - ip, - continent: 'Europe', - continentCode: 'EU', - country: 'Germany', - countryCode: 'DE', - region: 'HE', - regionName: 'Hesse', - lat: 50.1109, - lon: 8.6821, - org: 'Test Network', - static: true, - dataCenter: true, - }, - staticIp: true, - dataCenter: true, - lastIpChangeDate: null, - updatedAt: Date.now(), - }, - }, - { upsert: true }, - ); - }, - async seedAppHash(hash, height, resolved = false) { const explorerDb = await db('explorer'); await explorerDb.collection('zelappshashes').insertOne({ @@ -248,14 +243,28 @@ export function dbClient(nodeNum) { ); }, + // A COPY, because insertOne stamps _id onto the object it is handed and the + // caller's app fixture is reused - seeded on several nodes, and read again to + // build the app's next specification. An _id carried into that lands in the + // hashed payload and in a later replaceOne, where mongo refuses it outright. async seedGlobalAppSpec(spec) { const globalDb = await db('appsGlobal'); - await globalDb.collection('zelappsinformation').insertOne(spec); + await globalDb.collection('zelappsinformation').insertOne({ ...spec }); + }, + + // zelappsinformation holds one row per app - the CURRENT specification. An + // update replaces it, the way hash sync does when the chain carries a newer + // message; inserting a second row leaves the node reading whichever it finds + // first. + async replaceGlobalAppSpec(spec) { + const globalDb = await db('appsGlobal'); + const { _id: _ignored, ...replacement } = spec; + await globalDb.collection('zelappsinformation').replaceOne({ name: spec.name }, replacement, { upsert: true }); }, async seedPermanentMessage(msg) { const globalDb = await db('appsGlobal'); - await globalDb.collection('zelappsmessages').insertOne(msg); + await globalDb.collection('zelappsmessages').insertOne({ ...msg }); }, async seedAppLocation({ name, ip, hash, broadcastedAt, runningSince }) { @@ -276,6 +285,23 @@ export function dbClient(nodeNum) { await globalDb.collection('appstateevents').insertOne(event); }, + /** + * The same, for a whole set, in ONE round trip. + * + * These events carry `broadcastedAt`, and the window that accepts them is + * real - messageStore refuses a broadcast older than locationTtlS, which the + * harness compresses to 63s. Seeding a few hundred of them an insertOne at a + * time costs more wall-clock than the window itself, so the events expire + * during their own seeding and the suite reads an empty location list rather + * than a rejected message. Ordered, because a suite that seeds two broadcasts + * from one node is usually proving which of them wins. + */ + async seedAppStateEvents(events) { + if (!events.length) return; + const globalDb = await db('appsGlobal'); + await globalDb.collection('appstateevents').insertMany(events, { ordered: true }); + }, + async seedLocalApp(spec) { const localDb = await db('appsLocal'); await localDb.collection('zelappsinformation').insertOne(spec); @@ -306,15 +332,6 @@ export function dbClient(nodeNum) { }); }, - async dropAndReseed(ip, height) { - const client = await getClient(); - for (const name of Object.values(dbNames)) { - await client.db(name).dropDatabase(); - } - await this.seedScannedHeight(height); - await this.seedGeolocation(ip); - }, - async failpointFind(collection, { times = 1, errorCode = 50 } = {}) { const client = await getClient(); const namespace = `${dbNames.explorer}.${collection}`; diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js new file mode 100644 index 0000000000..af1661ca90 --- /dev/null +++ b/test-infra/runner/framework/external-http-control.js @@ -0,0 +1,85 @@ +// Drives the external HTTP stub's artifact store: arbitrary bytes a node can +// fetch over real HTTP from inside the subnet. The restore suites need this +// because the remote path - the download, the content-length comparison, the +// file landing in backup/remote - cannot be reached with a local archive. +import { getSubnetConfig } from './subnet-config.js'; + +const HOST = getSubnetConfig().externalStub; +const CONTROL = process.env.EXTERNAL_HTTP_CONTROL || `http://${HOST}:3001`; +const SERVE_PORT = 3000; + +async function post(path, body) { + const res = await fetch(`${CONTROL}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: body != null ? JSON.stringify(body) : undefined, + }); + return res.json(); +} + +/** + * Stage bytes to be served at artifactUrl(name). + * + * `declaredLength` overrides the content-length header without changing what is + * sent, which is how a download that stops short of what was promised is + * reproduced - a dropped connection, or an error page served as 200. + * + * @param {string} name - artifact name, used in the URL + * @param {string} base64 - the bytes, base64 encoded + * @param {{declaredLength?: number}} [opts] + */ +export async function stageArtifact(name, base64, { declaredLength = null } = {}) { + return post('/artifact', { name, base64, declaredLength }); +} + +/** + * The URL a node should be given to fetch a staged artifact. Built from the + * subnet config rather than a literal, so it follows a re-based fleet. + * @param {string} name - artifact name + * @returns {string} URL reachable from inside the fleet + */ +export function artifactUrl(name) { + return `http://${HOST}:${SERVE_PORT}/artifact/${name}`; +} + +/** + * Every name a node asked for that nothing on the fleet could answer. + * + * The fleet network has no route off it, so a hardcoded address cannot reach + * anywhere - but blocking alone only turns it into a timeout, and a timeout reads + * as a slow test rather than as a node reaching somewhere it should not. The + * resolver records instead, so this list names the host and the node that wanted + * it. + * + * @returns {Promise>} + */ +export async function dnsAttempts() { + const res = await fetch(`${CONTROL}/dns-attempts`); + const { attempts } = await res.json(); + return attempts; +} + +/** + * Forget every recorded attempt. Called before the window a suite intends to + * assert over, so it measures its own fleet rather than whatever ran before it. + */ +export async function resetDnsAttempts() { + return post('/dns-attempts/reset'); +} + +/** + * Fail naming the host and the node, rather than leaving a caller to compare + * lists. `allowed` is for a suite that means to reach something - it should be + * rare enough that writing the name down is the easy part. + * + * @param {{allowed?: string[]}} [opts] + */ +export async function expectNoUnexpectedDns({ allowed = [] } = {}) { + const unexpected = (await dnsAttempts()).filter((a) => !allowed.includes(a.name)); + if (!unexpected.length) return; + + const detail = unexpected + .map((a) => `${a.name} (node ${a.node})`) + .join(', '); + throw new Error(`nodes reached for names the fleet does not serve: ${detail}`); +} diff --git a/test-infra/runner/framework/fdm-control.js b/test-infra/runner/framework/fdm-control.js index f0144c174e..dfb798d5ac 100644 --- a/test-infra/runner/framework/fdm-control.js +++ b/test-infra/runner/framework/fdm-control.js @@ -1,11 +1,12 @@ // Drives the FDM stub (test-infra/fdm-stub) that masterSlaveApps polls for the // elected g: primary. Default host matches test-env's FDM_IP/control port. import { getSubnetConfig } from './subnet-config.js'; +import { controlFetch } from './control-fetch.js'; const CONTROL = process.env.FDM_CONTROL || `http://${getSubnetConfig().fdm}:16131`; async function post(path, body) { - const res = await fetch(`${CONTROL}${path}`, { + const res = await controlFetch(`${CONTROL}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: body != null ? JSON.stringify(body) : undefined, @@ -14,7 +15,7 @@ async function post(path, body) { } async function get(path) { - const res = await fetch(`${CONTROL}${path}`); + const res = await controlFetch(`${CONTROL}${path}`); return res.json(); } @@ -36,3 +37,18 @@ export async function resetFdm() { export async function getFdmState() { return get('/state'); } + +// Stop FDM answering, which is the only way to reach the node's third state: +// not "no primary yet" (clearMaster, above - that is FDM answering) but "FDM +// gave no verdict at all", which the election stands down on rather than acting +// on evidence it does not have. +// 'refuse' close the socket - the node's poll gets ECONNREFUSED, the +// production outage signature +// 'unavailable' 503 - reachable, but reporting itself as still starting up +export async function startFdmOutage(mode = 'refuse') { + return post('/outage', { mode }); +} + +export async function endFdmOutage() { + return post('/recover'); +} diff --git a/test-infra/runner/framework/g-app-placement.js b/test-infra/runner/framework/g-app-placement.js new file mode 100644 index 0000000000..9722d3d7d1 --- /dev/null +++ b/test-infra/runner/framework/g-app-placement.js @@ -0,0 +1,89 @@ +// Where a `g:` app's writer ends up, and how a suite arranges for it to end up +// somewhere in particular. +// +// TWO ORDERINGS decide what a g: app looks like on a fleet, and different things +// set them: +// +// the WRITER the holder that gets the writable folder at a cold start and +// runs the component. It is the LOWEST IP among the holders - the +// syncthing state machine's designated leader - and the election +// does not choose it. +// the ORDER runningSince ascending, which records the order the holders +// were PLACED. masterSlaveApps ranks its senior end; the surplus +// rule and the evacuation queue rank its junior end. +// +// ELECTING A MASTER DOES NOT MOVE THE WRITER. FDM reports which node is primary; +// it does not start containers, and the election refuses to start a second +// writer while a peer is running one - the split-brain guard. A suite that wants +// the writer in a particular position must PLACE the holders so it lands there, +// and suite 96 spent three minutes of its first run waiting on a container that +// was never going to start because it tried to elect one instead. +// +// Owned here because three suites depend on this and each used to restate it in +// its own words, hand-deriving a placement order from it. A fact written down in +// three places is a fact that drifts in two of them. +// +// Pure, and deliberately dependent on nothing but the subnet layout, so the +// orders it produces are unit-testable without a fleet. + +import { getSubnetConfig } from './subnet-config.js'; + +const defaultIpOf = (index) => getSubnetConfig().nodeIp(index + 1); + +/** + * Numeric IP ordering. A lexical compare puts `.10` before `.9`, which is the + * kind of thing that is right for every fixture anyone happens to write and + * wrong for the first one that spans the boundary. + * @param {string} a Dotted-quad address. + * @param {string} b Dotted-quad address. + * @returns {number} Comparator result. + */ +function compareIps(a, b) { + const left = a.split('.').map(Number); + const right = b.split('.').map(Number); + for (let i = 0; i < Math.max(left.length, right.length); i += 1) { + const diff = (left[i] ?? 0) - (right[i] ?? 0); + if (diff !== 0) return diff; + } + return 0; +} + +/** + * Which holder will seed the folder, and therefore run the g: component. + * @param {number[]} holders Node indices holding the app. + * @param {Function} [ipOf] Index to address, injectable for tests. + * @returns {number} The seeding holder's node index. + */ +export function syncthingSeedIndex(holders, ipOf = defaultIpOf) { + if (!Array.isArray(holders) || !holders.length) { + throw new Error('syncthingSeedIndex: holders must be a non-empty array of node indices'); + } + return [...holders].sort((a, b) => compareIps(ipOf(a), ipOf(b)))[0]; +} + +/** + * A placement order that lands the seed at a chosen position in the instance + * order - which is the same thing as its election index, because both are + * runningSince ascending and placement is what sets runningSince. + * + * Position 0 is the SENIOR end (first placed, lowest election index); the last + * position is the newest copy, which is the one the surplus rule and the + * evacuation queue pick first. + * @param {number[]} holders Node indices holding the app. + * @param {number} seedPosition Where the seed should land, 0-based. + * @param {Function} [ipOf] Index to address, injectable for tests. + * @returns {number[]} Placement order for placeGAppInOrder. + */ +export function placementOrderWithSeedAt(holders, seedPosition, ipOf = defaultIpOf) { + const seed = syncthingSeedIndex(holders, ipOf); + if (!Number.isInteger(seedPosition) || seedPosition < 0 || seedPosition >= holders.length) { + // Thrown rather than clamped: a position off the end means the caller has a + // different fleet in mind than the one it is describing, and clamping would + // hand it a plausible order for the wrong shape. + throw new Error( + `placementOrderWithSeedAt: seedPosition ${seedPosition} is outside 0..${holders.length - 1}`, + ); + } + const others = holders.filter((index) => index !== seed); + return [...others.slice(0, seedPosition), seed, ...others.slice(seedPosition)]; +} diff --git a/test-infra/runner/framework/http-wait-strategy.js b/test-infra/runner/framework/http-wait-strategy.js index f46db59647..86356b1613 100644 --- a/test-infra/runner/framework/http-wait-strategy.js +++ b/test-infra/runner/framework/http-wait-strategy.js @@ -1,3 +1,5 @@ +import { throwIfInfraDead, sleepUnlessInfraDead } from './infra-death.js'; + // A testcontainers WaitStrategy that polls an HTTP URL until it responds OK, // bypassing Docker's health state machine entirely. // @@ -27,11 +29,16 @@ export class HttpPollWaitStrategy { #startupTimeoutSet = false; #pollIntervalMs; #probeTimeoutMs; + #validate; - constructor(url, { pollIntervalMs = 500, probeTimeoutMs = 2000 } = {}) { + // `validate` inspects the response beyond res.ok — needed when the target + // returns 200 with an error body while a dependency (e.g. mongo) is still + // coming up. Defaults to res.ok when omitted. + constructor(url, { pollIntervalMs = 500, probeTimeoutMs = 2000, validate = null } = {}) { this.#url = url; this.#pollIntervalMs = pollIntervalMs; this.#probeTimeoutMs = probeTimeoutMs; + this.#validate = validate; } withStartupTimeout(startupTimeoutMs) { @@ -51,13 +58,17 @@ export class HttpPollWaitStrategy { async waitUntilReady() { const deadline = Date.now() + this.#startupTimeoutMs; while (Date.now() < deadline) { + // An infra container died while this one was coming up: nothing that + // depends on it can answer, so surface the death instead of spending the + // full startup budget proving mongo is gone. + throwIfInfraDead(); try { const res = await fetch(this.#url, { signal: AbortSignal.timeout(this.#probeTimeoutMs) }); - if (res.ok) return; + if (this.#validate ? await this.#validate(res) : res.ok) return; } catch { // not serving yet — keep polling until the deadline } - await new Promise((r) => setTimeout(r, this.#pollIntervalMs)); + await sleepUnlessInfraDead(this.#pollIntervalMs); } throw new Error(`HttpPollWaitStrategy: ${this.#url} not ready after ${this.#startupTimeoutMs}ms`); } diff --git a/test-infra/runner/framework/infra-death.js b/test-infra/runner/framework/infra-death.js new file mode 100644 index 0000000000..b32252e09c --- /dev/null +++ b/test-infra/runner/framework/infra-death.js @@ -0,0 +1,80 @@ +// The infra containers - mongo, the daemon/syncthing/external/fdm stubs, the +// registry - are supposed to outlive every suite that boots them. When one dies +// mid-run nothing notices at the time: the waits already in flight simply stop +// being satisfiable and expire 30-60s later as "Timeout after 60000ms waiting for +// event: app:installed" or EHOSTUNREACH against an address that no longer +// answers. That is indistinguishable from a product bug, and it is how the +// 2026-07-30 parallel gate lost three suites to a mongo:8 SIGSEGV. +// +// This module is the single kill-switch the wait machinery consults. The death +// watcher in test-env.js trips it; waitForEvent (node-client.js), waitFor and +// assertNoEvent (wait.js), and the two poll wait strategies fail out of it +// immediately with a message that names the container, its exit code and the +// time. That message leads with the literal string INFRA-DEAD so gate tooling can +// grep a .tap for a void run without parsing anything. +// +// It is a module of its own rather than part of test-env.js so the wait machinery +// can import it without an import cycle (test-env.js already imports node-client.js). + +const handlers = new Set(); +let death = null; + +export function infraDeathError() { + return death; +} + +export function throwIfInfraDead() { + if (death) throw death; +} + +// Called once per unexpected infra death. The FIRST death is the one worth +// reporting: a mongo that dies takes the whole fleet's database with it and +// anything that dies afterwards is a consequence of that, so later deaths are +// logged but never replace the recorded cause. +export function reportInfraDeath({ name, exitCode, at }) { + const message = `INFRA-DEAD: ${name} exited code=${exitCode} at ${at}; run is void`; + console.error(message); + if (death) return; + death = new Error(message); + death.infraDead = true; + // Fire once: a waiter that arrives later reads infraDeathError() instead. + const parked = [...handlers]; + handlers.clear(); + for (const handler of parked) handler(death); +} + +// Arm a fresh environment. A previous env's death must not fail the next env's +// waits - createTestEnv calls this before it starts watching. +export function clearInfraDeath() { + death = null; + handlers.clear(); +} + +// The one way the framework sleeps inside a poll loop. Rejects immediately if a +// death is already recorded, and rejects AT a death that lands mid-sleep - so +// consulting the kill switch is a property of the primitive rather than a +// convention each loop re-implements (the convention already failed once: a +// hand-rolled stability window slept through a death and passed its suite over +// a dead env). What this cannot do is stop a conclusion being drawn from an +// observation made while the env died - nodes answer from memory - so a loop +// still calls throwIfInfraDead() between observing and concluding. +export function sleepUnlessInfraDead(ms) { + if (death) return Promise.reject(death); + return new Promise((resolve, reject) => { + const onDeath = (error) => { clearTimeout(timer); reject(error); }; + const timer = setTimeout(() => { offInfraDeath(onDeath); resolve(); }, ms); + onInfraDeath(onDeath); + }); +} + +// Waits parked on a listener or a timer register here so they can be rejected AT +// the death rather than at their own deadline. Callers must check +// infraDeathError() first: the switch fires once, so a handler registered after +// it tripped is never called. +export function onInfraDeath(handler) { + handlers.add(handler); +} + +export function offInfraDeath(handler) { + handlers.delete(handler); +} diff --git a/test-infra/runner/framework/log-on-failure.js b/test-infra/runner/framework/log-on-failure.js index 563d9319ed..7d98da2dba 100644 --- a/test-infra/runner/framework/log-on-failure.js +++ b/test-infra/runner/framework/log-on-failure.js @@ -9,14 +9,19 @@ function sanitize(label) { return (label || 'unknown').replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 120); } -// Dump each node's logs and SSE events to its OWN file under test-logs/