🐛 fix(ci): make pipefail dead-fallback handlers actually reachable

.github/tests/ci-verify-find-pipefail-guard.test.ts

@@ -0,0 +1,132 @@
+import { fileURLToPath } from 'node:url';
+
+import { loadWorkflow as loadWorkflowFrom } from './workflow-test-utils';
+
+// GitHub runs every `run:` step without an explicit `shell:` under
+// `bash -eo pipefail`. `find <dir> ...` exits 0 when `<dir>` exists but is
+// empty, but exits 1 when `<dir>` itself is MISSING (verified on this
+// machine). ci-verify.yml's load-test steps create their artifact directory
+// only on a successful run (scripts/run-load-test.sh), so a failed load test
+// leaves the directory absent. Piping that failing `find` straight into
+// `sort | head | cut` inside `report="$(...)"` aborts the step under
+// pipefail before the deliberate `if [ -z "${report}" ]` empty-report
+// handling in scripts/summarize-load-test-report.sh and
+// scripts/check-load-test-correctness.sh ever runs -- the same "pipefail
+// dead-fallback" class as the i18n-crowdin.yml grep bug covered by
+// crowdin-base-branch-resolver.test.ts. The fix wraps the `find` in
+// `{ find ... || true; }` so a missing directory yields an empty result
+// instead of aborting the step.
+//
+// Actually executing these steps needs docker + real Artillery artifacts, so
+// this is a static assertion over the workflow's `run:` text rather than a
+// behavioral test. That's a deliberate tradeoff: it can't prove the guarded
+// form behaves correctly end to end, but it can prove -- and keep proving --
+// that no `find` piped into a variable assignment ships unguarded.
+//
+// The sites are discovered dynamically by walking every job/step and
+// regex-matching the find-into-assignment shape, rather than pinned to the
+// current 8 line numbers or step names. That's the point: it also catches a
+// brand new unguarded `find` added later, not just a revert of today's fix.
+
+const workflowPath = fileURLToPath(new URL('../workflows/ci-verify.yml', import.meta.url));
+const loadWorkflow = loadWorkflowFrom.bind(undefined, workflowPath);
+
+// Matches a whole-line shell assignment that captures a command
+// substitution's output, e.g.:
+//   report="$({ find ... || true; } | sort -rn | head -n1 | cut -d' ' -f2-)"
+//   current_report="$(find ... | sort -rn | head -n1 | cut -d' ' -f2-)"
+const FIND_ASSIGNMENT_LINE = /^\s*\w+="\$\((.*)\)"\s*$/;
+
+// A guarded `find` invocation is wrapped so a non-zero exit (missing
+// directory) is swallowed before it can hit the pipe: `{ find ... || true; }`.
+const GUARDED_FIND_PREFIX = /^\{\s*find\b[\s\S]*?\|\|\s*true;\s*\}/;
+
+interface FindAssignmentSite {
+  jobId: string;
+  stepName: string;
+  inner: string;
+  line: string;
+}
+
+// Reconstructs logical shell lines from a `run:` block by folding backslash
+// continuations, so a site reformatted across `\` line breaks is still
+// discovered by the single-line FIND_ASSIGNMENT_LINE regex below. A line
+// ending in a trailing `\` (optionally followed by trailing whitespace)
+// continues onto the next physical line.
+function joinContinuations(run: string): string[] {
+  const logicalLines: string[] = [];
+  let pending = '';
+
+  for (const rawLine of run.split('\n')) {
+    const continued = /\\\s*$/.test(rawLine);
+    const chunk = continued ? rawLine.replace(/\\\s*$/, '') : rawLine;
+    pending += (pending ? ' ' : '') + chunk.trim();
+
+    if (!continued) {
+      logicalLines.push(pending);
+      pending = '';
+    }
+  }
+
+  if (pending) logicalLines.push(pending);
+
+  return logicalLines;
+}
+
+function findFindAssignmentSites(): FindAssignmentSite[] {
+  const workflow = loadWorkflow();
+  const sites: FindAssignmentSite[] = [];
+
+  for (const [jobId, job] of Object.entries(workflow.jobs ?? {})) {
+    for (const step of job.steps ?? []) {
+      if (!step.run) continue;
+
+      for (const line of joinContinuations(step.run)) {
+        const match = line.match(FIND_ASSIGNMENT_LINE);
+        if (!match) continue;
+
+        const inner = match[1].trim();
+        if (!/\bfind\b/.test(inner)) continue;
+
+        sites.push({
+          jobId,
+          stepName: step.name ?? '(unnamed step)',
+          inner,
+          line: line.trim(),
+        });
+      }
+    }
+  }
+
+  return sites;
+}
+
+// Pinned to the current count of guarded find-into-assignment sites in
+// ci-verify.yml. This must be bumped deliberately, in either direction, when
+// a site is added or removed -- a plain `> 0` sanity check would let the
+// sweep silently drop sites (e.g. a regex that stops matching a reformatted
+// site) without failing.
+const EXPECTED_FIND_ASSIGNMENT_SITE_COUNT = 8;
+
+test('ci-verify.yml has find-into-variable assignments to guard against pipefail', () => {
+  // Sanity check on the sweep itself: pinned to the exact expected count
+  // rather than `> 0` so a drop in EITHER direction fails loudly -- including
+  // a site silently falling out of the sweep (e.g. reformatted across a
+  // backslash continuation that the regex no longer matches).
+  expect(findFindAssignmentSites().length).toBe(EXPECTED_FIND_ASSIGNMENT_SITE_COUNT);
+});
+
+test('every find-into-variable pipeline in ci-verify.yml is guarded against a missing-directory exit', () => {
+  const unguarded = findFindAssignmentSites().filter(
+    (site) =>
+      !GUARDED_FIND_PREFIX.test(site.inner) ||
+      // GUARDED_FIND_PREFIX only anchors the *first* stage, so a pipeline like
+      // `{ find a || true; } | find b ...` would otherwise pass with its second
+      // find still able to abort under pipefail. Rather than try to parse each
+      // stage, require exactly one find per assignment: every site here has one,
+      // and a second one should fail this test so someone has to look at it.
+      (site.inner.match(/\bfind\b/g) ?? []).length !== 1,
+  );
+
+  expect(unguarded).toStrictEqual([]);
+});

.github/tests/crowdin-base-branch-resolver.test.ts

@@ -0,0 +1,137 @@
+import { execFileSync } from 'node:child_process';
+import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { loadWorkflow } from './workflow-test-utils';
+
+// This test extracts and EXECUTES the real `run:` block from the workflow
+// (rather than a copy pasted into the test) so it can't drift from what
+// actually ships, and so it fails outright if someone reverts the step to the
+// broken pre-2ad6c5c9 form: a no-match `grep` under `pipefail` used to exit 1
+// and abort the assignment before the `if [ -z "${base}" ]` fallback could
+// ever run.
+const workflowPath = fileURLToPath(new URL('../workflows/i18n-crowdin.yml', import.meta.url));
+
+const stubGitScript = `#!/usr/bin/env bash
+# Stub 'git' for the resolver script under test. Only implements the one
+# subcommand the resolver calls: 'ls-remote --heads origin <pattern>'.
+if [ "\${1:-}" = "ls-remote" ]; then
+  if [ -n "\${FAKE_GIT_LS_REMOTE_EXIT:-}" ] && [ "\${FAKE_GIT_LS_REMOTE_EXIT}" != "0" ]; then
+    echo "fatal: unable to access 'origin': stubbed network failure" >&2
+    exit "\${FAKE_GIT_LS_REMOTE_EXIT}"
+  fi
+  printf '%s' "\${FAKE_GIT_LS_REMOTE_OUTPUT:-}"
+  exit 0
+fi
+echo "stub git: unsupported invocation: $*" >&2
+exit 127
+`;
+
+function loadResolverRunBlock(): string {
+  const workflow = loadWorkflow(workflowPath);
+  const step = workflow.jobs?.sync?.steps?.find((candidate) => candidate.id === 'base');
+
+  if (!step?.run) {
+    throw new Error(
+      "Expected i18n-crowdin.yml's sync job to include a step with id 'base' and a run block",
+    );
+  }
+
+  return step.run;
+}
+
+interface ResolverResult {
+  status: number;
+  stdout: string;
+  stderr: string;
+  output: Record<string, string>;
+}
+
+function runResolver(options: {
+  lsRemoteOutput?: string;
+  lsRemoteExit?: number;
+  defaultBranch?: string;
+}): ResolverResult {
+  const workdir = mkdtempSync(join(tmpdir(), 'crowdin-base-resolver-'));
+  try {
+    const stubDir = join(workdir, 'bin');
+    mkdirSync(stubDir);
+    writeFileSync(join(stubDir, 'git'), stubGitScript, { mode: 0o755 });
+    chmodSync(join(stubDir, 'git'), 0o755);
+
+    const scriptPath = join(workdir, 'resolve-base.sh');
+    writeFileSync(scriptPath, loadResolverRunBlock());
+
+    const outputPath = join(workdir, 'github_output');
+    writeFileSync(outputPath, '');
+
+    const env: NodeJS.ProcessEnv = {
+      PATH: `${stubDir}:${process.env.PATH ?? ''}`,
+      DEFAULT_BRANCH: options.defaultBranch ?? 'main',
+      GITHUB_OUTPUT: outputPath,
+      FAKE_GIT_LS_REMOTE_OUTPUT: options.lsRemoteOutput ?? '',
+    };
+    if (options.lsRemoteExit !== undefined) {
+      env.FAKE_GIT_LS_REMOTE_EXIT = String(options.lsRemoteExit);
+    }
+
+    let status = 0;
+    let stdout = '';
+    let stderr = '';
+    try {
+      stdout = execFileSync('bash', [scriptPath], { env, encoding: 'utf8' });
+    } catch (error) {
+      const execError = error as { status?: number; stdout?: string; stderr?: string };
+      status = execError.status ?? 1;
+      stdout = execError.stdout ?? '';
+      stderr = execError.stderr ?? '';
+    }
+
+    const output: Record<string, string> = {};
+    for (const line of readFileSync(outputPath, 'utf8').split('\n')) {
+      const eq = line.indexOf('=');
+      if (eq === -1) continue;
+      output[line.slice(0, eq)] = line.slice(eq + 1);
+    }
+
+    return { status, stdout, stderr, output };
+  } finally {
+    rmSync(workdir, { recursive: true, force: true });
+  }
+}
+
+test('falls back to the default branch when origin has no dev/vX.Y branches', () => {
+  const result = runResolver({
+    lsRemoteOutput: 'deadbeef\trefs/heads/main\ncafef00d\trefs/heads/feature/something\n',
+    defaultBranch: 'main',
+  });
+
+  expect(result.status).toBe(0);
+  expect(result.output.name).toBe('main');
+});
+
+test('picks the highest dev/vX.Y branch numerically, not lexically', () => {
+  const result = runResolver({
+    lsRemoteOutput: [
+      'aaaaaaa\trefs/heads/dev/v1.6',
+      'bbbbbbb\trefs/heads/dev/v1.9',
+      'ccccccc\trefs/heads/dev/v1.10',
+      'ddddddd\trefs/heads/not-a-dev-branch',
+    ].join('\n'),
+  });
+
+  expect(result.status).toBe(0);
+  expect(result.output.name).toBe('dev/v1.10');
+});
+
+test('a genuinely failing ls-remote propagates and does not silently fall back to main', () => {
+  const result = runResolver({ lsRemoteExit: 128, defaultBranch: 'main' });
+
+  expect(result.status).not.toBe(0);
+  // The step must die before it ever reaches the fallback assignment — no
+  // output should have been written at all.
+  expect(result.output.name).toBeUndefined();
+  expect(result.stdout).not.toContain('main');
+});

.github/workflows/ci-verify.yml

@@ -1249,7 +1249,7 @@ jobs:
       - name: Summarize load test metrics (ci)
         if: always()
         run: |
-          report="$(find artifacts/load-test/ci -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -n1 | cut -d' ' -f2-)"
+          report="$({ find artifacts/load-test/ci -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null || true; } | sort -rn | head -n1 | cut -d' ' -f2-)"
           ./scripts/summarize-load-test-report.sh "$report" "Load Test (CI)" "test/load-test-baselines/ci.json"
 
       - name: Correctness check (ci, enforced)
@@ -1261,7 +1261,7 @@ jobs:
           DD_LOAD_TEST_MIN_429: '0'
           DD_LOAD_TEST_MAX_429: '0'
         run: |
-          report="$(find artifacts/load-test/ci -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -n1 | cut -d' ' -f2-)"
+          report="$({ find artifacts/load-test/ci -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null || true; } | sort -rn | head -n1 | cut -d' ' -f2-)"
           ./scripts/check-load-test-correctness.sh "$report" "Load Test Correctness (CI)"
 
       - name: Resolve committed load test baseline (ci)
@@ -1299,7 +1299,7 @@ jobs:
             exit 1
           fi
 
-          current_report="$(find artifacts/load-test/ci -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -n1 | cut -d' ' -f2-)"
+          current_report="$({ find artifacts/load-test/ci -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null || true; } | sort -rn | head -n1 | cut -d' ' -f2-)"
           if [ -z "${current_report}" ]; then
             echo "::error::Current CI report not found; cannot run regression gate."
             exit 1
@@ -1372,7 +1372,7 @@ jobs:
       - name: Summarize load test metrics (behavior)
         if: always()
         run: |
-          report="$(find artifacts/load-test/behavior -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -n1 | cut -d' ' -f2-)"
+          report="$({ find artifacts/load-test/behavior -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null || true; } | sort -rn | head -n1 | cut -d' ' -f2-)"
           ./scripts/summarize-load-test-report.sh "$report" "Load Test (Behavior)" "test/load-test-baselines/behavior.json"
 
       - name: Correctness check (behavior, advisory)
@@ -1384,7 +1384,7 @@ jobs:
           DD_LOAD_TEST_MIN_429: '0'
           DD_LOAD_TEST_MAX_429: '0'
         run: |
-          report="$(find artifacts/load-test/behavior -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -n1 | cut -d' ' -f2-)"
+          report="$({ find artifacts/load-test/behavior -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null || true; } | sort -rn | head -n1 | cut -d' ' -f2-)"
           ./scripts/check-load-test-correctness.sh "$report" "Load Test Correctness (Behavior)"
 
       - name: Resolve committed load test baseline (behavior)
@@ -1417,7 +1417,7 @@ jobs:
         run: |
           set -euo pipefail
 
-          current_report="$(find artifacts/load-test/behavior -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -n1 | cut -d' ' -f2-)"
+          current_report="$({ find artifacts/load-test/behavior -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null || true; } | sort -rn | head -n1 | cut -d' ' -f2-)"
           ./scripts/check-load-test-regression.sh "${current_report}" "${BASELINE_REPORT}"
 
       - name: Upload load test artifact (behavior)
@@ -1442,7 +1442,7 @@ jobs:
       - name: Summarize load test metrics (stress)
         if: always()
         run: |
-          report="$(find artifacts/load-test/stress -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -n1 | cut -d' ' -f2-)"
+          report="$({ find artifacts/load-test/stress -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null || true; } | sort -rn | head -n1 | cut -d' ' -f2-)"
           ./scripts/summarize-load-test-report.sh "$report" "Load Test (Stress)"
 
       - name: Correctness check (stress, advisory)
@@ -1454,7 +1454,7 @@ jobs:
           DD_LOAD_TEST_MIN_429: '0'
           DD_LOAD_TEST_MAX_429: '0'
         run: |
-          report="$(find artifacts/load-test/stress -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -n1 | cut -d' ' -f2-)"
+          report="$({ find artifacts/load-test/stress -maxdepth 1 -name '*.json' -printf '%T@ %p\n' 2>/dev/null || true; } | sort -rn | head -n1 | cut -d' ' -f2-)"
           ./scripts/check-load-test-correctness.sh "$report" "Load Test Correctness (Stress)"
 
       - name: Upload load test artifact (stress)

.github/workflows/i18n-crowdin.yml

@@ -48,9 +48,15 @@ jobs:
           # dev/v1.9. Falls back to the default branch during the window between
           # a GA and the next dev branch being cut, so translations are never
           # stranded with nowhere to land.
+          #
+          # The grep is guarded: with `pipefail` a no-match grep exits 1 and
+          # aborts the assignment, which killed the fallback below before it
+          # could ever run (2026-07-24, when the sync PR merge briefly deleted
+          # dev/v1.6). A failing ls-remote still propagates, so a network or
+          # auth error fails loudly instead of silently retargeting main.
           base="$(git ls-remote --heads origin 'refs/heads/dev/v*' \
             | sed 's#.*refs/heads/##' \
-            | grep -E '^dev/v[0-9]+\.[0-9]+$' \
+            | { grep -E '^dev/v[0-9]+\.[0-9]+$' || true; } \
             | sort -t/ -k2 -V \
             | tail -1)"
           if [ -z "${base}" ]; then

test/cpu-bench.sh

@@ -68,7 +68,7 @@ printf "%-20s %10s %10s %10s %10s\n" "Container" "Avg CPU%" "Min CPU%" "Max CPU%
 printf "%-20s %10s %10s %10s %10s\n" "--------------------" "----------" "----------" "----------" "----------"
 
 for c in $CONTAINERS; do
-	RESULT=$(grep "^$c " "$TMPFILE" | awk '
+	RESULT=$({ grep "^$c " "$TMPFILE" || true; } | awk '
     BEGIN { sum=0; count=0; min=9999; max=0 }
     {
       sum += $2; count++