feat(sandbox-env): optional KEDA autoscaling for tenant warm pools

.github/workflows/helm-test.yml

@@ -104,6 +104,41 @@ jobs:
             --set warmPool.size=2 \
             > /dev/null
 
+      - name: Render (tenant pool + KEDA autoscaling)
+        run: |
+          helm template sandbox-env deploy/helm/sandbox-env \
+            --namespace agent-sandbox-system \
+            -f deploy/helm/sandbox-env/examples/values-tenant-pool-keda.yaml \
+            > /dev/null
+
+      # The chart ships no default trigger on purpose (the operator's claim
+      # metric can't tell two tenant pools apart). A ScaledObject with no
+      # triggers would sit at minReplicaCount forever, so that must fail the
+      # render rather than deploy dormant.
+      - name: Render (tenant pool autoscaling without triggers must fail)
+        run: |
+          set +e
+          helm template sandbox-env deploy/helm/sandbox-env \
+            --namespace agent-sandbox-system \
+            --set envName=ci \
+            --set tenantPools[0].name=tenant-acme-ci \
+            --set tenantPools[0].size=1 \
+            --set tenantPools[0].autoscaling.enabled=true \
+            --set tenantPools[0].autoscaling.minReplicaCount=1 \
+            --set tenantPools[0].autoscaling.maxReplicaCount=4 \
+            > /tmp/render.out 2>&1
+          rc=$?
+          set -e
+          if [ "${rc}" -eq 0 ]; then
+            echo "::error::tenant pool autoscaling rendered with no triggers — validation is missing."
+            exit 1
+          fi
+          if ! grep -q "autoscaling.triggers" /tmp/render.out; then
+            echo "::error::failed for the wrong reason — expected the triggers-required error."
+            cat /tmp/render.out
+            exit 1
+          fi
+
       - name: Render (envName missing must fail)
         run: |
           set +e

deploy/helm/sandbox-env/Chart.yaml

@@ -9,6 +9,13 @@ description: |
   releases coexist in the shared `agent-sandbox-system` namespace.
   Requires the sandbox-operator chart to already be installed.
 type: application
+# 0.17.0: tenantPools entries take an optional `autoscaling` block rendering a
+# KEDA ScaledObject on the pool's scale subresource. Off by default and no
+# default trigger ships: the operator's claim metric is labelled by
+# sandbox_template, which every tenant pool shares, so the scaling signal has
+# to be Studio's per-pool `studio_sandbox_pool_pods{pool,state}` gauge. Turning
+# it on also needs the pool covered by the Argo app's ignoreDifferences on
+# SandboxWarmPool /spec/replicas.
 # 0.16.2: renewals now land at idleTtlSeconds + `housekeeper.renewSlackSeconds`
 # (default 60) instead of exactly idleTtlSeconds, so the sweep's graceful reap
 # — HTTPRoute deleted before the pod drains — stays ahead of the operator's own
@@ -154,7 +161,7 @@ type: application
 # 0.8.0: org-fs sidecar (orgFs.*) — privileged rclone mounter + main-container
 # mounts. NOTE: the publish workflow skips existing versions, so this MUST
 # stay ahead of the latest published chart (0.7.5 at time of writing).
-version: 0.16.37
+version: 0.17.0
 # appVersion tracks the studio-sandbox image version (image.tag default).
 appVersion: "1.60.8"
 kubeVersion: ">=1.30.0-0"

deploy/helm/sandbox-env/README.md

@@ -14,6 +14,11 @@ Renders:
 - `Secret` `studio-sandbox-sentinel-<envName>` (initial daemon token)
 - `SandboxWarmPool` `studio-sandbox-<envName>` and `...-medium` (optional)
 - `HorizontalPodAutoscaler` for the warm pool (optional; requires explicit metrics)
+- `SandboxWarmPool` per entry in `tenantPools` — pods already running ONE org's
+  repo + dev server (optional)
+- KEDA `ScaledObject` per tenant pool that sets `autoscaling.enabled`
+  (optional; requires explicit `triggers` and KEDA in the cluster — see
+  `examples/values-tenant-pool-keda.yaml`)
 - `Deployment` `studio-sandbox-placeholder-<envName>` — node "balloon" (optional)
 - `Gateway` + `Certificate` `agent-sandbox-preview-<envName>` (optional;
   per-claim HTTPRoutes are minted by the Studio runner, not by this chart)

deploy/helm/sandbox-env/examples/values-tenant-pool-keda.yaml

@@ -0,0 +1,39 @@
+# Tenant warm pool with the optional KEDA ScaledObject turned on.
+#
+#   helm template sandbox-env deploy/helm/sandbox-env \
+#     -n agent-sandbox-system --set envName=ci \
+#     -f deploy/helm/sandbox-env/examples/values-tenant-pool-keda.yaml
+#
+# Needs KEDA in the cluster and a `vm-main` ClusterTriggerAuthentication
+# (infra_applications provisioning/keda/main). Also add the pool to the Argo
+# app's ignoreDifferences on SandboxWarmPool /spec/replicas, or every sync
+# reverts the scaler back to `size`.
+envName: ci
+
+tenantPools:
+  - name: tenant-acme-ci
+    # Seed / Argo-reapplied replica count. KEDA drives it after that.
+    size: 2
+    autoscaling:
+      enabled: true
+      # Never 0: a tenant pool at 0 guarantees a cold start (clone + install +
+      # dev boot, ~2.5min measured) for whoever claims next.
+      minReplicaCount: 1
+      maxReplicaCount: 8
+      pollingInterval: 30
+      cooldownPeriod: 300
+      triggers:
+        - type: prometheus
+          metadata:
+            serverAddress: https://vm-main.infra.deco.cx/prometheus
+            # state="bound" = pods actually handed to users, so this tracks
+            # concurrent demand. NOT state="ready", which counts "config
+            # posted" rather than "dev server up" and reads full while the
+            # pool is useless. `+ 1` keeps one spare warming ahead of the next
+            # claim: a pool holds a ready pod only while size > rate x warmTime.
+            query: sum(studio_sandbox_pool_pods{pool="tenant-acme-ci",state="bound"}) + 1
+            threshold: "1"
+            authModes: bearer
+          authenticationRef:
+            name: vm-main
+            kind: ClusterTriggerAuthentication

deploy/helm/sandbox-env/templates/_helpers.tpl

@@ -240,6 +240,14 @@ stays identical on both sides, and a collision is a template-time failure.
 {{- if lt (int .size) 1 }}
 {{- fail (printf "sandbox-env: tenantPools[%s].size must be >= 1 (got %v) — a pool with 0 replicas pre-warms nothing." .name .size) -}}
 {{- end }}
+{{- if and .autoscaling .autoscaling.enabled }}
+{{- if eq (len (default (list) .autoscaling.triggers)) 0 }}
+{{- fail (printf "sandbox-env: tenantPools[%s].autoscaling.enabled=true requires at least one entry in .autoscaling.triggers — this chart ships no default trigger (the operator's claim metric is labelled by sandbox_template, which every tenant pool shares). A ScaledObject with no triggers sits at minReplicaCount forever. See values.yaml for a studio_sandbox_pool_pods example." .name) -}}
+{{- end }}
+{{- if lt (int .autoscaling.maxReplicaCount) (int .autoscaling.minReplicaCount) }}
+{{- fail (printf "sandbox-env: tenantPools[%s].autoscaling.maxReplicaCount (%v) must be >= minReplicaCount (%v)." .name .autoscaling.maxReplicaCount .autoscaling.minReplicaCount) -}}
+{{- end }}
+{{- end }}
 {{- end }}
 {{- end }}
 

deploy/helm/sandbox-env/templates/sandbox-tenant-pool-keda.yaml

@@ -0,0 +1,48 @@
+{{- range .Values.tenantPools }}
+{{- if .autoscaling }}
+{{- if .autoscaling.enabled }}
+---
+# Optional KEDA ScaledObject for one tenant pool, driving the SandboxWarmPool's
+# native scale subresource (same subresource `kubectl scale` and the generic
+# pool's HPA use). Off unless the entry sets autoscaling.enabled — a tenant pool
+# stays a fixed `size` until someone opts in, and the chart works with KEDA
+# absent from the cluster.
+#
+# One ScaledObject per pool, declared next to the pool it scales: the pool name
+# is already the join key between this chart and Studio's
+# STUDIO_SANDBOX_TENANT_POOLS, and putting the scaler in a third place is a
+# third copy of that string to keep in sync.
+#
+# BYO trigger, no default: the operator's own claim metric is labelled by
+# `sandbox_template`, and every tenant pool shares the ONE template this chart
+# renders — so the obvious copy of the generic pool's query would scale every
+# tenant pool on the whole env's claim rate. Studio's per-pool gauge
+# `studio_sandbox_pool_pods{pool,state}` is the signal that actually
+# distinguishes them; scale on state="bound" (a pod handed to a user), not
+# state="ready", which counts "config posted" rather than "dev server up".
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: {{ .name }}
+  namespace: agent-sandbox-system
+  labels:
+    {{- include "sandbox-env.sandboxLabels" $ | nindent 4 }}
+    studio.decocms.com/tenant-pool: {{ .name }}
+spec:
+  scaleTargetRef:
+    apiVersion: extensions.agents.x-k8s.io/v1alpha1
+    kind: SandboxWarmPool
+    name: {{ .name }}
+  {{- with .autoscaling.pollingInterval }}
+  pollingInterval: {{ . }}
+  {{- end }}
+  {{- with .autoscaling.cooldownPeriod }}
+  cooldownPeriod: {{ . }}
+  {{- end }}
+  minReplicaCount: {{ .autoscaling.minReplicaCount }}
+  maxReplicaCount: {{ .autoscaling.maxReplicaCount }}
+  triggers:
+    {{- toYaml .autoscaling.triggers | nindent 4 }}
+{{- end }}
+{{- end }}
+{{- end }}

deploy/helm/sandbox-env/values.yaml

@@ -652,6 +652,46 @@ warmPool:
 #   tenantPools:
 #     - name: tenant-acme-site-prod  # DNS label ending in -<envName>
 #       size: 2                      # keep this many warm pods ready
+#
+# Each entry may also carry an optional `autoscaling` block that renders a KEDA
+# ScaledObject against the pool's scale subresource (see
+# templates/sandbox-tenant-pool-keda.yaml). Off unless `enabled: true`, so the
+# chart still installs on clusters without KEDA. `size` stays the seeded /
+# Argo-reapplied replica count — when you turn this on, add the pool to the
+# Argo app's ignoreDifferences on SandboxWarmPool /spec/replicas, or every sync
+# reverts the scaler.
+#
+# There is no default trigger, on purpose. The operator's claim metric
+# (agent_sandbox_claim_creation_total) is labelled by `sandbox_template`, and
+# every tenant pool binds the ONE template this chart renders — copying the
+# generic pool's query would scale each tenant pool on the whole environment's
+# claim rate. Studio's own per-pool gauge is the signal that distinguishes them:
+#
+#   tenantPools:
+#     - name: tenant-acme-site-prod
+#       size: 2
+#       autoscaling:
+#         enabled: true
+#         minReplicaCount: 1
+#         maxReplicaCount: 8
+#         pollingInterval: 30
+#         cooldownPeriod: 300
+#         # Raw KEDA `triggers` entries, passed through verbatim.
+#         # Scale on state="bound" (a pod actually handed to a user), NOT
+#         # state="ready": `ready` counts "config posted", not "dev server
+#         # up", so it reads full while the pool is useless. The `+ 1` keeps
+#         # one spare warming ahead of the next claim — a pool holds a ready
+#         # pod only while size > claim rate x warm time.
+#         triggers:
+#           - type: prometheus
+#             metadata:
+#               serverAddress: https://vm-main.infra.deco.cx/prometheus
+#               query: sum(studio_sandbox_pool_pods{pool="tenant-acme-site-prod",state="bound"}) + 1
+#               threshold: "1"
+#               authModes: bearer
+#             authenticationRef:
+#               name: vm-main
+#               kind: ClusterTriggerAuthentication
 tenantPools: []
 
 # ── node placeholder / "balloon" (optional) ────────────────────────────