From dfbb6b73fe83d4153a8858c3e23fc7e44eac61e6 Mon Sep 17 00:00:00 2001 From: Ariel Rolfo Date: Wed, 24 Jun 2026 18:52:05 -0300 Subject: [PATCH] feat(ci): update-app-env workflow + Secrets Manager IAM grant Adds a workflow_dispatch job that mutates the AWS Secrets Manager secret feeding the ctdl-xtra-app-env k8s Secret (via External Secrets Operator), then rolls the api and worker Deployments so they pick up the new values. Sandbox/prod are gated through GitHub Environments, matching promote-* workflows. Why - ESO already syncs ctdl-xtra/{env}/app from Secrets Manager into the ctdl-xtra-app-env k8s Secret (5m refresh). The api/worker Deployments consume it via envFrom. Today there is no audited path to add/change/ remove keys other than the AWS console; result is no run log, no Slack notice, and no automatic pod refresh. - The workflow closes that gap with a controlled, scriptable surface that never echoes values to logs and produces a key-level diff in the run summary and Slack. How - Inputs: environment, changes (JSON {"KEY":"value" | null}), restart, dry_run. null deletes the key; non-string values are rejected; keys must match [A-Za-z_][A-Za-z0-9_]*. - All values (both incoming and existing) are passed through ::add-mask:: before any further step runs. - jq merge yields the next secret JSON, written to a tmp file; diff is emitted as added/changed/removed key lists only (never values). - aws secretsmanager put-secret-value creates a new version; previous versions remain available for one-step revert via the AWS console. - ESO is force-refreshed with a force-sync annotation on the ExternalSecret; we poll the target Secret's resourceVersion (up to 60s) to confirm it actually picked up the new version before any rollout. Avoids the silent "values still old, pods bounced for nothing" failure mode. - Rollout: kubectl rollout restart deploy/ctdl-xtra-api and deploy/ctdl-xtra-worker, then rollout status with a 5m timeout each. - Slack always notifies (success or failure) with the key-level diff, new VersionId, and outcome (applied / applied+restarted / dry-run / no-op). IAM - New aws_iam_policy.github_actions_secretsmanager grants Describe/Get/Put/ListSecretVersionIds scoped to ctdl-xtra/{test,sandbox,prod}/app-* only. Attached to the existing ctdl-xtra-github-actions-ci role used by every other workflow. - Must be applied with `terraform apply` in infra/terraform/github-ci-oidc/ before the workflow can succeed. Test plan - Dispatch on test with dry_run=true and changes={"LOG_LEVEL":"debug"} -> step summary shows changed: ["LOG_LEVEL"], no put-secret-value. - Dispatch on test for real with the same input -> new VersionId, ctdl-xtra-app-env resourceVersion bumps, api/worker rollout restart + rollout status OK, pods come up with new LOG_LEVEL. - Dispatch on test with changes={"LOG_LEVEL":null} -> key removed. - Dispatch on test with changes={"LOG_LEVEL":"info"} and restart=false -> secret updated, pods not bounced. --- .github/workflows/update-app-env.yml | 287 +++++++++++++++++++++++++ infra/terraform/github-ci-oidc/main.tf | 36 ++++ 2 files changed, 323 insertions(+) create mode 100644 .github/workflows/update-app-env.yml diff --git a/.github/workflows/update-app-env.yml b/.github/workflows/update-app-env.yml new file mode 100644 index 0000000..3945eb2 --- /dev/null +++ b/.github/workflows/update-app-env.yml @@ -0,0 +1,287 @@ +name: Update app env vars + +on: + workflow_dispatch: + inputs: + environment: + description: "Target environment / cluster" + type: choice + required: true + default: test + options: + - test + - sandbox + - prod + changes: + description: 'JSON map of keys to set or remove. Example: {"LOG_LEVEL":"debug","OLD_FLAG":null}' + type: string + required: true + restart: + description: "Restart api and worker Deployments after applying" + type: boolean + required: true + default: true + dry_run: + description: "Compute and print the diff without writing" + type: boolean + required: true + default: false + +permissions: + id-token: write + contents: read + +env: + AWS_REGION: us-east-1 + APP_NAMESPACE: ctdl-xtra + ESO_EXTERNALSECRET: ctdl-xtra-app + TARGET_SECRET: ctdl-xtra-app-env + ROLLOUT_TIMEOUT: 5m + +jobs: + update: + if: ${{ github.repository_owner == 'CredentialEngine' }} + runs-on: ubuntu-latest + environment: ${{ inputs.environment == 'prod' && 'PRODUCTION' || (inputs.environment == 'sandbox' && 'SANDBOX' || 'TEST') }} + steps: + - uses: actions/checkout@v4 + + - name: Resolve target cluster + AWS secret name + id: target + env: + ENVIRONMENT: ${{ inputs.environment }} + run: | + set -euo pipefail + case "${ENVIRONMENT}" in + test) + echo "cluster=ctdl-xtra-test" >> "$GITHUB_OUTPUT" + echo "secret_id=ctdl-xtra/test/app" >> "$GITHUB_OUTPUT" + ;; + sandbox) + echo "cluster=ctdl-xtra-sandbox" >> "$GITHUB_OUTPUT" + echo "secret_id=ctdl-xtra/sandbox/app" >> "$GITHUB_OUTPUT" + ;; + prod) + echo "cluster=ctdl-xtra-prod" >> "$GITHUB_OUTPUT" + echo "secret_id=ctdl-xtra/prod/app" >> "$GITHUB_OUTPUT" + ;; + *) + echo "Unknown environment: ${ENVIRONMENT}" >&2 + exit 1 + ;; + esac + + - name: Validate changes JSON + env: + CHANGES: ${{ inputs.changes }} + run: | + set -euo pipefail + if ! echo "$CHANGES" | jq -e 'type=="object"' >/dev/null; then + echo "Input 'changes' must be a JSON object (got: $(echo "$CHANGES" | jq -r 'type'))" >&2 + exit 1 + fi + # Reject nested values and non-string non-null leaves + if echo "$CHANGES" | jq -e 'to_entries[] | select(.value | type | . != "string" and . != "null")' >/dev/null; then + echo "Each value must be a string (to set) or null (to remove)" >&2 + echo "$CHANGES" | jq . + exit 1 + fi + # Reject empty / illegal keys + if echo "$CHANGES" | jq -e 'keys[] | select(test("^[A-Za-z_][A-Za-z0-9_]*$") | not)' >/dev/null; then + echo "Keys must match [A-Za-z_][A-Za-z0-9_]*" >&2 + exit 1 + fi + + - name: Mask change values + env: + CHANGES: ${{ inputs.changes }} + run: | + # Mask every non-null value in the log so they never appear in diffs or errors. + echo "$CHANGES" | jq -r 'to_entries[] | select(.value != null) | .value' \ + | while IFS= read -r v; do [ -n "$v" ] && echo "::add-mask::$v"; done + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - uses: azure/setup-kubectl@v4 + with: + version: v1.29.6 + + - name: Update kubeconfig + run: aws eks update-kubeconfig --name "${{ steps.target.outputs.cluster }}" --region "$AWS_REGION" + + - name: Fetch current secret, compute next, diff + id: diff + env: + SECRET_ID: ${{ steps.target.outputs.secret_id }} + CHANGES: ${{ inputs.changes }} + run: | + set -euo pipefail + CURRENT=$(aws secretsmanager get-secret-value --secret-id "$SECRET_ID" --query SecretString --output text) + if ! echo "$CURRENT" | jq -e 'type=="object"' >/dev/null; then + echo "Existing secret value is not a JSON object; refusing to mutate" >&2 + exit 1 + fi + # Mask current values too so we never echo them later. + echo "$CURRENT" | jq -r 'to_entries[] | .value | select(. != null and . != "")' \ + | while IFS= read -r v; do echo "::add-mask::$v"; done + + # Merge changes; null in CHANGES means delete. Filter null values from the union. + NEXT=$(jq -nc --argjson c "$CURRENT" --argjson p "$CHANGES" \ + '($c * $p) | with_entries(select(.value != null))') + + # Key-level diff (never values) + ADDED=$(jq -nc --argjson c "$CURRENT" --argjson n "$NEXT" '($n|keys) - ($c|keys)') + REMOVED=$(jq -nc --argjson c "$CURRENT" --argjson n "$NEXT" '($c|keys) - ($n|keys)') + CHANGED=$(jq -nc --argjson c "$CURRENT" --argjson n "$NEXT" \ + '[ ($n|keys[]) as $k | select(($c[$k]//null) != null and ($n[$k]//null) != null and $c[$k] != $n[$k]) | $k ]') + + { + echo "## Pending env change for \`${SECRET_ID}\`" + echo + echo "- **Added keys:** \`$ADDED\`" + echo "- **Removed keys:** \`$REMOVED\`" + echo "- **Changed keys:** \`$CHANGED\`" + } >> "$GITHUB_STEP_SUMMARY" + + echo "added=$ADDED" >> "$GITHUB_OUTPUT" + echo "removed=$REMOVED" >> "$GITHUB_OUTPUT" + echo "changed=$CHANGED" >> "$GITHUB_OUTPUT" + + # Stash next value for the apply step (file, not env, to avoid log exposure). + printf '%s' "$NEXT" > /tmp/next-secret.json + + NOOP=$(jq -nc --argjson a "$ADDED" --argjson r "$REMOVED" --argjson ch "$CHANGED" \ + '($a|length)+($r|length)+($ch|length) == 0') + echo "noop=$NOOP" >> "$GITHUB_OUTPUT" + + - name: Exit if no-op + if: ${{ steps.diff.outputs.noop == 'true' }} + run: | + echo "No keys added, changed, or removed — nothing to do." + echo "::notice::No-op: ${{ steps.target.outputs.secret_id }} already matches desired state." + + - name: Exit if dry run + if: ${{ inputs.dry_run && steps.diff.outputs.noop != 'true' }} + run: | + echo "::notice::Dry run requested — not writing to Secrets Manager." + + - name: Put new secret version + id: put + if: ${{ !inputs.dry_run && steps.diff.outputs.noop != 'true' }} + env: + SECRET_ID: ${{ steps.target.outputs.secret_id }} + run: | + set -euo pipefail + VERSION=$(aws secretsmanager put-secret-value \ + --secret-id "$SECRET_ID" \ + --secret-string file:///tmp/next-secret.json \ + --query VersionId --output text) + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "::notice::Wrote new version $VERSION to $SECRET_ID" + + - name: Force ExternalSecret sync and wait for k8s Secret refresh + if: ${{ !inputs.dry_run && steps.diff.outputs.noop != 'true' }} + env: + NS: ${{ env.APP_NAMESPACE }} + ES: ${{ env.ESO_EXTERNALSECRET }} + TARGET: ${{ env.TARGET_SECRET }} + run: | + set -euo pipefail + BEFORE=$(kubectl -n "$NS" get secret "$TARGET" -o jsonpath='{.metadata.resourceVersion}') + echo "k8s Secret resourceVersion before: $BEFORE" + kubectl -n "$NS" annotate externalsecret "$ES" \ + "force-sync=$(date +%s)" --overwrite >/dev/null + # Wait up to 60s for ESO to bump the target Secret. + for i in $(seq 1 30); do + NOW=$(kubectl -n "$NS" get secret "$TARGET" -o jsonpath='{.metadata.resourceVersion}') + if [ "$NOW" != "$BEFORE" ]; then + echo "k8s Secret refreshed; resourceVersion now: $NOW" + exit 0 + fi + sleep 2 + done + echo "Timed out waiting for ESO to refresh $NS/$TARGET" >&2 + kubectl -n "$NS" describe externalsecret "$ES" || true + exit 1 + + - name: Restart api and worker + id: rollout + if: ${{ !inputs.dry_run && steps.diff.outputs.noop != 'true' && inputs.restart }} + env: + NS: ${{ env.APP_NAMESPACE }} + run: | + set -euo pipefail + kubectl -n "$NS" rollout restart deploy/ctdl-xtra-api deploy/ctdl-xtra-worker + kubectl -n "$NS" rollout status deploy/ctdl-xtra-api --timeout="${ROLLOUT_TIMEOUT}" + kubectl -n "$NS" rollout status deploy/ctdl-xtra-worker --timeout="${ROLLOUT_TIMEOUT}" + + - name: Notify Slack + if: ${{ always() }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + REPO: ${{ github.repository }} + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + ACTOR: ${{ github.actor }} + ENVIRONMENT: ${{ inputs.environment }} + SECRET_ID: ${{ steps.target.outputs.secret_id }} + VERSION: ${{ steps.put.outputs.version }} + ADDED: ${{ steps.diff.outputs.added }} + REMOVED: ${{ steps.diff.outputs.removed }} + CHANGED: ${{ steps.diff.outputs.changed }} + NOOP: ${{ steps.diff.outputs.noop }} + DRY_RUN: ${{ inputs.dry_run }} + RESTART: ${{ inputs.restart }} + STATUS: ${{ job.status }} + run: | + if [ -z "${SLACK_WEBHOOK_URL}" ]; then + echo "SLACK_WEBHOOK_URL not set; skipping notification" + exit 0 + fi + if [ "$DRY_RUN" = "true" ]; then SUMMARY="dry-run" + elif [ "$NOOP" = "true" ]; then SUMMARY="no-op" + elif [ "$RESTART" = "true" ]; then SUMMARY="applied + restarted" + else SUMMARY="applied (no restart)" + fi + payload=$(jq -nc \ + --arg repo "$REPO" \ + --arg run "$RUN_URL" \ + --arg actor "$ACTOR" \ + --arg env "$ENVIRONMENT" \ + --arg secret_id "$SECRET_ID" \ + --arg version "${VERSION:-n/a}" \ + --arg added "${ADDED:-[]}" \ + --arg removed "${REMOVED:-[]}" \ + --arg changed "${CHANGED:-[]}" \ + --arg status "$STATUS" \ + --arg summary "$SUMMARY" \ + ' + { + text: "App env update (\($env)): \($summary)", + blocks: [ + { "type": "header", "text": { "type": "plain_text", "text": "App env update (\($env))" } }, + { "type": "section", "fields": [ + {"type":"mrkdwn", "text": "*Status:*\n\($status)"}, + {"type":"mrkdwn", "text": "*Outcome:*\n\($summary)"}, + {"type":"mrkdwn", "text": "*Requested by:*\n\($actor)"}, + {"type":"mrkdwn", "text": "*Secret:*\n\($secret_id)"}, + {"type":"mrkdwn", "text": "*New version:*\n\($version)"} + ] + }, + { "type": "section", "fields": [ + {"type":"mrkdwn", "text": "*Added:*\n```\($added)```"}, + {"type":"mrkdwn", "text": "*Changed:*\n```\($changed)```"}, + {"type":"mrkdwn", "text": "*Removed:*\n```\($removed)```"} + ] + }, + { "type": "context", "elements": [ + {"type":"mrkdwn", "text": "<\($run)|View run>"}, + {"type":"mrkdwn", "text": $repo} + ] + } + ] + } + ') + curl -sS -X POST -H 'Content-type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" || true diff --git a/infra/terraform/github-ci-oidc/main.tf b/infra/terraform/github-ci-oidc/main.tf index 3c65bd4..d7d0ae6 100644 --- a/infra/terraform/github-ci-oidc/main.tf +++ b/infra/terraform/github-ci-oidc/main.tf @@ -157,3 +157,39 @@ resource "aws_iam_role_policy_attachment" "github_actions_eks" { role = aws_iam_role.github_actions_ci.name policy_arn = aws_iam_policy.github_actions_eks.arn } + +# --------------------------------------------------------------- +# Secrets Manager — read/write the app env secrets that ESO syncs +# into the ctdl-xtra-app-env k8s Secret. Used by the +# update-app-env workflow to push key changes from CI. +# --------------------------------------------------------------- + +resource "aws_iam_policy" "github_actions_secretsmanager" { + name = "ctdl-xtra-github-actions-secretsmanager" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AppEnvReadWrite" + Effect = "Allow" + Action = [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue", + "secretsmanager:PutSecretValue", + "secretsmanager:ListSecretVersionIds", + ] + Resource = [ + "arn:aws:secretsmanager:us-east-1:${local.aws_account_id}:secret:ctdl-xtra/test/app-*", + "arn:aws:secretsmanager:us-east-1:${local.aws_account_id}:secret:ctdl-xtra/sandbox/app-*", + "arn:aws:secretsmanager:us-east-1:${local.aws_account_id}:secret:ctdl-xtra/prod/app-*", + ] + }, + ] + }) +} + +resource "aws_iam_role_policy_attachment" "github_actions_secretsmanager" { + role = aws_iam_role.github_actions_ci.name + policy_arn = aws_iam_policy.github_actions_secretsmanager.arn +}