From 0cb92ecdba71834661d4151a1d5b6c4a76275261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Fran=C3=A7a?= Date: Fri, 4 Sep 2026 12:14:02 -0300 Subject: [PATCH] feat(sandbox-env): optional KEDA autoscaling for tenant warm pools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tenant warm pools were fixed-size only, while the generic warmPool could be autoscaled. A tenant pool holds a ready pod only while `size > claim rate x warm time`, so the right size moves with the org's traffic and a static number is either idle spend or a guaranteed cold start. Each `tenantPools` entry now takes an optional `autoscaling` block rendering a KEDA ScaledObject against the pool's native scale subresource. Off by default, so the chart still installs on clusters without KEDA. No default trigger ships, deliberately: the operator's `agent_sandbox_claim_creation_total` is labelled by `sandbox_template`, and every tenant pool binds the ONE template this chart renders — copying the generic pool's query would scale each tenant pool on the whole environment's claim rate. Studio's per-pool `studio_sandbox_pool_pods{pool,state}` gauge is the signal that distinguishes them, and the docs point at `state="bound"` rather than `state="ready"` (which counts "config posted", not "dev server up"). `triggers` is required and validated at template time: a ScaledObject with none sits at minReplicaCount forever. `size` stays the seeded / Argo-reapplied replica count, so enabling this also needs the pool covered by the Argo app's ignoreDifferences on SandboxWarmPool /spec/replicas — noted in values.yaml, the README and the example. Chart 0.16.37 -> 0.17.0. CI renders the example and asserts the triggers-required validation fails the render. --- .github/workflows/helm-test.yml | 35 ++++++++++++++ deploy/helm/sandbox-env/Chart.yaml | 9 +++- deploy/helm/sandbox-env/README.md | 5 ++ .../examples/values-tenant-pool-keda.yaml | 39 +++++++++++++++ .../helm/sandbox-env/templates/_helpers.tpl | 8 ++++ .../templates/sandbox-tenant-pool-keda.yaml | 48 +++++++++++++++++++ deploy/helm/sandbox-env/values.yaml | 40 ++++++++++++++++ 7 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 deploy/helm/sandbox-env/examples/values-tenant-pool-keda.yaml create mode 100644 deploy/helm/sandbox-env/templates/sandbox-tenant-pool-keda.yaml diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml index fdca06d503..a8bf7f9fd7 100644 --- a/.github/workflows/helm-test.yml +++ b/.github/workflows/helm-test.yml @@ -104,6 +104,41 @@ jobs: --set warmPool.size=2 \ > /dev/null + - name: Render (tenant pool + KEDA autoscaling) + run: | + helm template sandbox-env deploy/helm/sandbox-env \ + --namespace agent-sandbox-system \ + -f deploy/helm/sandbox-env/examples/values-tenant-pool-keda.yaml \ + > /dev/null + + # The chart ships no default trigger on purpose (the operator's claim + # metric can't tell two tenant pools apart). A ScaledObject with no + # triggers would sit at minReplicaCount forever, so that must fail the + # render rather than deploy dormant. + - name: Render (tenant pool autoscaling without triggers must fail) + run: | + set +e + helm template sandbox-env deploy/helm/sandbox-env \ + --namespace agent-sandbox-system \ + --set envName=ci \ + --set tenantPools[0].name=tenant-acme-ci \ + --set tenantPools[0].size=1 \ + --set tenantPools[0].autoscaling.enabled=true \ + --set tenantPools[0].autoscaling.minReplicaCount=1 \ + --set tenantPools[0].autoscaling.maxReplicaCount=4 \ + > /tmp/render.out 2>&1 + rc=$? + set -e + if [ "${rc}" -eq 0 ]; then + echo "::error::tenant pool autoscaling rendered with no triggers — validation is missing." + exit 1 + fi + if ! grep -q "autoscaling.triggers" /tmp/render.out; then + echo "::error::failed for the wrong reason — expected the triggers-required error." + cat /tmp/render.out + exit 1 + fi + - name: Render (envName missing must fail) run: | set +e diff --git a/deploy/helm/sandbox-env/Chart.yaml b/deploy/helm/sandbox-env/Chart.yaml index dc79525e66..eb37d823dc 100644 --- a/deploy/helm/sandbox-env/Chart.yaml +++ b/deploy/helm/sandbox-env/Chart.yaml @@ -9,6 +9,13 @@ description: | releases coexist in the shared `agent-sandbox-system` namespace. Requires the sandbox-operator chart to already be installed. type: application +# 0.17.0: tenantPools entries take an optional `autoscaling` block rendering a +# KEDA ScaledObject on the pool's scale subresource. Off by default and no +# default trigger ships: the operator's claim metric is labelled by +# sandbox_template, which every tenant pool shares, so the scaling signal has +# to be Studio's per-pool `studio_sandbox_pool_pods{pool,state}` gauge. Turning +# it on also needs the pool covered by the Argo app's ignoreDifferences on +# SandboxWarmPool /spec/replicas. # 0.16.2: renewals now land at idleTtlSeconds + `housekeeper.renewSlackSeconds` # (default 60) instead of exactly idleTtlSeconds, so the sweep's graceful reap # — HTTPRoute deleted before the pod drains — stays ahead of the operator's own @@ -154,7 +161,7 @@ type: application # 0.8.0: org-fs sidecar (orgFs.*) — privileged rclone mounter + main-container # mounts. NOTE: the publish workflow skips existing versions, so this MUST # stay ahead of the latest published chart (0.7.5 at time of writing). -version: 0.16.37 +version: 0.17.0 # appVersion tracks the studio-sandbox image version (image.tag default). appVersion: "1.60.8" kubeVersion: ">=1.30.0-0" diff --git a/deploy/helm/sandbox-env/README.md b/deploy/helm/sandbox-env/README.md index d6b7c941c5..8939ee1e3b 100644 --- a/deploy/helm/sandbox-env/README.md +++ b/deploy/helm/sandbox-env/README.md @@ -14,6 +14,11 @@ Renders: - `Secret` `studio-sandbox-sentinel-` (initial daemon token) - `SandboxWarmPool` `studio-sandbox-` and `...-medium` (optional) - `HorizontalPodAutoscaler` for the warm pool (optional; requires explicit metrics) +- `SandboxWarmPool` per entry in `tenantPools` — pods already running ONE org's + repo + dev server (optional) +- KEDA `ScaledObject` per tenant pool that sets `autoscaling.enabled` + (optional; requires explicit `triggers` and KEDA in the cluster — see + `examples/values-tenant-pool-keda.yaml`) - `Deployment` `studio-sandbox-placeholder-` — node "balloon" (optional) - `Gateway` + `Certificate` `agent-sandbox-preview-` (optional; per-claim HTTPRoutes are minted by the Studio runner, not by this chart) diff --git a/deploy/helm/sandbox-env/examples/values-tenant-pool-keda.yaml b/deploy/helm/sandbox-env/examples/values-tenant-pool-keda.yaml new file mode 100644 index 0000000000..5a99781139 --- /dev/null +++ b/deploy/helm/sandbox-env/examples/values-tenant-pool-keda.yaml @@ -0,0 +1,39 @@ +# Tenant warm pool with the optional KEDA ScaledObject turned on. +# +# helm template sandbox-env deploy/helm/sandbox-env \ +# -n agent-sandbox-system --set envName=ci \ +# -f deploy/helm/sandbox-env/examples/values-tenant-pool-keda.yaml +# +# Needs KEDA in the cluster and a `vm-main` ClusterTriggerAuthentication +# (infra_applications provisioning/keda/main). Also add the pool to the Argo +# app's ignoreDifferences on SandboxWarmPool /spec/replicas, or every sync +# reverts the scaler back to `size`. +envName: ci + +tenantPools: + - name: tenant-acme-ci + # Seed / Argo-reapplied replica count. KEDA drives it after that. + size: 2 + autoscaling: + enabled: true + # Never 0: a tenant pool at 0 guarantees a cold start (clone + install + + # dev boot, ~2.5min measured) for whoever claims next. + minReplicaCount: 1 + maxReplicaCount: 8 + pollingInterval: 30 + cooldownPeriod: 300 + triggers: + - type: prometheus + metadata: + serverAddress: https://vm-main.infra.deco.cx/prometheus + # state="bound" = pods actually handed to users, so this tracks + # concurrent demand. NOT state="ready", which counts "config + # posted" rather than "dev server up" and reads full while the + # pool is useless. `+ 1` keeps one spare warming ahead of the next + # claim: a pool holds a ready pod only while size > rate x warmTime. + query: sum(studio_sandbox_pool_pods{pool="tenant-acme-ci",state="bound"}) + 1 + threshold: "1" + authModes: bearer + authenticationRef: + name: vm-main + kind: ClusterTriggerAuthentication diff --git a/deploy/helm/sandbox-env/templates/_helpers.tpl b/deploy/helm/sandbox-env/templates/_helpers.tpl index 051ad0a44a..c761473e8a 100644 --- a/deploy/helm/sandbox-env/templates/_helpers.tpl +++ b/deploy/helm/sandbox-env/templates/_helpers.tpl @@ -240,6 +240,14 @@ stays identical on both sides, and a collision is a template-time failure. {{- if lt (int .size) 1 }} {{- fail (printf "sandbox-env: tenantPools[%s].size must be >= 1 (got %v) — a pool with 0 replicas pre-warms nothing." .name .size) -}} {{- end }} +{{- if and .autoscaling .autoscaling.enabled }} +{{- if eq (len (default (list) .autoscaling.triggers)) 0 }} +{{- fail (printf "sandbox-env: tenantPools[%s].autoscaling.enabled=true requires at least one entry in .autoscaling.triggers — this chart ships no default trigger (the operator's claim metric is labelled by sandbox_template, which every tenant pool shares). A ScaledObject with no triggers sits at minReplicaCount forever. See values.yaml for a studio_sandbox_pool_pods example." .name) -}} +{{- end }} +{{- if lt (int .autoscaling.maxReplicaCount) (int .autoscaling.minReplicaCount) }} +{{- fail (printf "sandbox-env: tenantPools[%s].autoscaling.maxReplicaCount (%v) must be >= minReplicaCount (%v)." .name .autoscaling.maxReplicaCount .autoscaling.minReplicaCount) -}} +{{- end }} +{{- end }} {{- end }} {{- end }} diff --git a/deploy/helm/sandbox-env/templates/sandbox-tenant-pool-keda.yaml b/deploy/helm/sandbox-env/templates/sandbox-tenant-pool-keda.yaml new file mode 100644 index 0000000000..79f35ec66a --- /dev/null +++ b/deploy/helm/sandbox-env/templates/sandbox-tenant-pool-keda.yaml @@ -0,0 +1,48 @@ +{{- range .Values.tenantPools }} +{{- if .autoscaling }} +{{- if .autoscaling.enabled }} +--- +# Optional KEDA ScaledObject for one tenant pool, driving the SandboxWarmPool's +# native scale subresource (same subresource `kubectl scale` and the generic +# pool's HPA use). Off unless the entry sets autoscaling.enabled — a tenant pool +# stays a fixed `size` until someone opts in, and the chart works with KEDA +# absent from the cluster. +# +# One ScaledObject per pool, declared next to the pool it scales: the pool name +# is already the join key between this chart and Studio's +# STUDIO_SANDBOX_TENANT_POOLS, and putting the scaler in a third place is a +# third copy of that string to keep in sync. +# +# BYO trigger, no default: the operator's own claim metric is labelled by +# `sandbox_template`, and every tenant pool shares the ONE template this chart +# renders — so the obvious copy of the generic pool's query would scale every +# tenant pool on the whole env's claim rate. Studio's per-pool gauge +# `studio_sandbox_pool_pods{pool,state}` is the signal that actually +# distinguishes them; scale on state="bound" (a pod handed to a user), not +# state="ready", which counts "config posted" rather than "dev server up". +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ .name }} + namespace: agent-sandbox-system + labels: + {{- include "sandbox-env.sandboxLabels" $ | nindent 4 }} + studio.decocms.com/tenant-pool: {{ .name }} +spec: + scaleTargetRef: + apiVersion: extensions.agents.x-k8s.io/v1alpha1 + kind: SandboxWarmPool + name: {{ .name }} + {{- with .autoscaling.pollingInterval }} + pollingInterval: {{ . }} + {{- end }} + {{- with .autoscaling.cooldownPeriod }} + cooldownPeriod: {{ . }} + {{- end }} + minReplicaCount: {{ .autoscaling.minReplicaCount }} + maxReplicaCount: {{ .autoscaling.maxReplicaCount }} + triggers: + {{- toYaml .autoscaling.triggers | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/deploy/helm/sandbox-env/values.yaml b/deploy/helm/sandbox-env/values.yaml index 9d21e214c0..3534b14bf4 100644 --- a/deploy/helm/sandbox-env/values.yaml +++ b/deploy/helm/sandbox-env/values.yaml @@ -652,6 +652,46 @@ warmPool: # tenantPools: # - name: tenant-acme-site-prod # DNS label ending in - # size: 2 # keep this many warm pods ready +# +# Each entry may also carry an optional `autoscaling` block that renders a KEDA +# ScaledObject against the pool's scale subresource (see +# templates/sandbox-tenant-pool-keda.yaml). Off unless `enabled: true`, so the +# chart still installs on clusters without KEDA. `size` stays the seeded / +# Argo-reapplied replica count — when you turn this on, add the pool to the +# Argo app's ignoreDifferences on SandboxWarmPool /spec/replicas, or every sync +# reverts the scaler. +# +# There is no default trigger, on purpose. The operator's claim metric +# (agent_sandbox_claim_creation_total) is labelled by `sandbox_template`, and +# every tenant pool binds the ONE template this chart renders — copying the +# generic pool's query would scale each tenant pool on the whole environment's +# claim rate. Studio's own per-pool gauge is the signal that distinguishes them: +# +# tenantPools: +# - name: tenant-acme-site-prod +# size: 2 +# autoscaling: +# enabled: true +# minReplicaCount: 1 +# maxReplicaCount: 8 +# pollingInterval: 30 +# cooldownPeriod: 300 +# # Raw KEDA `triggers` entries, passed through verbatim. +# # Scale on state="bound" (a pod actually handed to a user), NOT +# # state="ready": `ready` counts "config posted", not "dev server +# # up", so it reads full while the pool is useless. The `+ 1` keeps +# # one spare warming ahead of the next claim — a pool holds a ready +# # pod only while size > claim rate x warm time. +# triggers: +# - type: prometheus +# metadata: +# serverAddress: https://vm-main.infra.deco.cx/prometheus +# query: sum(studio_sandbox_pool_pods{pool="tenant-acme-site-prod",state="bound"}) + 1 +# threshold: "1" +# authModes: bearer +# authenticationRef: +# name: vm-main +# kind: ClusterTriggerAuthentication tenantPools: [] # ── node placeholder / "balloon" (optional) ────────────────────────────