diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml
index c6b55a9..9e42053 100644
--- a/.github/workflows/quality.yml
+++ b/.github/workflows/quality.yml
@@ -90,6 +90,27 @@ jobs:
- name: Test Console workspace
run: just test console
+ - name: Apply migrations to the disposable CI database
+ run: cargo run -p grass-control-api -- --config "$RUNNER_TEMP/grass-migration.toml" migrate
+ env:
+ GWAPI_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
+
+ - name: Verify certificate database columns and constraints
+ run: >-
+ cargo test -p grass-control-api
+ infra::database::migration::m20260910_000032_managed_certificates::tests::managed_certificate_schema_matches_signed_lifecycle_and_ack_protocol
+ -- --ignored --exact
+ env:
+ GRASS_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
+
+ - name: Verify regional ingress migration upgrade and rollback
+ run: >-
+ cargo test -p grass-control-api
+ infra::database::migrate::tests::postgres_regional_ingress_schema_matches_domain_and_is_reversible
+ -- --ignored --exact
+ env:
+ GRASS_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
+
- name: Check project license metadata
run: just license-check
diff --git a/Cargo.lock b/Cargo.lock
index 2bb17b2..b661762 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -320,6 +320,73 @@ dependencies = [
"regex-syntax",
]
+[[package]]
+name = "asn1-rs"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048"
+dependencies = [
+ "asn1-rs-derive 0.5.1",
+ "asn1-rs-impl",
+ "displaydoc",
+ "nom 7.1.3",
+ "num-traits",
+ "rusticata-macros",
+ "thiserror 1.0.69",
+ "time",
+]
+
+[[package]]
+name = "asn1-rs"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8"
+dependencies = [
+ "asn1-rs-derive 0.6.0",
+ "asn1-rs-impl",
+ "displaydoc",
+ "nom 7.1.3",
+ "num-traits",
+ "rusticata-macros",
+ "thiserror 2.0.18",
+ "time",
+]
+
+[[package]]
+name = "asn1-rs-derive"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+ "synstructure",
+]
+
+[[package]]
+name = "asn1-rs-derive"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+ "synstructure",
+]
+
+[[package]]
+name = "asn1-rs-impl"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "async-channel"
version = "1.9.0"
@@ -543,6 +610,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
dependencies = [
"aws-lc-sys",
+ "untrusted 0.7.1",
"zeroize",
]
@@ -761,7 +829,7 @@ dependencies = [
"serde_json",
"serde_repr",
"serde_urlencoded",
- "thiserror",
+ "thiserror 2.0.18",
"tokio",
"tokio-util",
"tower-service",
@@ -1257,6 +1325,34 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "der-parser"
+version = "9.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553"
+dependencies = [
+ "asn1-rs 0.6.2",
+ "displaydoc",
+ "nom 7.1.3",
+ "num-bigint",
+ "num-traits",
+ "rusticata-macros",
+]
+
+[[package]]
+name = "der-parser"
+version = "10.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6"
+dependencies = [
+ "asn1-rs 0.7.2",
+ "displaydoc",
+ "nom 7.1.3",
+ "num-bigint",
+ "num-traits",
+ "rusticata-macros",
+]
+
[[package]]
name = "deranged"
version = "0.5.8"
@@ -1719,7 +1815,7 @@ version = "0.1.0"
dependencies = [
"config",
"serde",
- "thiserror",
+ "thiserror 2.0.18",
"toml 0.8.23",
]
@@ -1745,21 +1841,26 @@ dependencies = [
"grass-token",
"grass-validator",
"hex",
+ "hyper-util",
"image",
+ "instant-acme",
"jsonwebtoken",
"lettre",
"mime_guess",
"object_store",
"rand 0.8.6",
+ "rcgen",
"reqwest 0.12.28",
"ring",
+ "rustls",
+ "rustls-pemfile",
"sea-orm",
"sea-orm-migration",
"serde",
"serde_json",
"sha2",
"subtle",
- "thiserror",
+ "thiserror 2.0.18",
"time",
"tokio",
"tokio-tungstenite 0.26.2",
@@ -1774,6 +1875,7 @@ dependencies = [
"uuid",
"walkdir",
"webp",
+ "x509-parser 0.16.0",
]
[[package]]
@@ -1784,7 +1886,7 @@ dependencies = [
"base64 0.22.1",
"ring",
"serde",
- "thiserror",
+ "thiserror 2.0.18",
]
[[package]]
@@ -1792,7 +1894,7 @@ name = "grass-git-source"
version = "0.1.0"
dependencies = [
"serde",
- "thiserror",
+ "thiserror 2.0.18",
"url",
]
@@ -1820,7 +1922,8 @@ dependencies = [
"subtle",
"sysinfo",
"tar",
- "thiserror",
+ "tempfile",
+ "thiserror 2.0.18",
"tokio",
"tokio-tungstenite 0.26.2",
"tokio-util",
@@ -2110,6 +2213,8 @@ dependencies = [
"hyper",
"hyper-util",
"rustls",
+ "rustls-native-certs",
+ "rustls-platform-verifier",
"tokio",
"tokio-rustls",
"tower-service",
@@ -2348,6 +2453,32 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "instant-acme"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f05ad37c421b962354c358d347d4a6130151df9407978372d3ad7f0c8f71a64"
+dependencies = [
+ "async-trait",
+ "aws-lc-rs",
+ "base64 0.22.1",
+ "bytes",
+ "http",
+ "http-body",
+ "http-body-util",
+ "httpdate",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "rcgen",
+ "rustls",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.18",
+ "tokio",
+]
+
[[package]]
name = "ipnet"
version = "2.12.0"
@@ -2396,7 +2527,7 @@ dependencies = [
"jni-sys",
"log",
"simd_cesu8",
- "thiserror",
+ "thiserror 2.0.18",
"walkdir",
"windows-link 0.2.1",
]
@@ -2512,7 +2643,7 @@ dependencies = [
"httpdate",
"idna",
"mime",
- "nom",
+ "nom 8.0.0",
"percent-encoding",
"quoted_printable",
"rustls",
@@ -2737,6 +2868,12 @@ dependencies = [
"unicase",
]
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
[[package]]
name = "miniz_oxide"
version = "0.8.9"
@@ -2813,6 +2950,16 @@ dependencies = [
"libc",
]
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
[[package]]
name = "nom"
version = "8.0.0"
@@ -2962,7 +3109,7 @@ dependencies = [
"serde",
"serde_json",
"serde_urlencoded",
- "thiserror",
+ "thiserror 2.0.18",
"tokio",
"tracing",
"url",
@@ -2972,6 +3119,24 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "oid-registry"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9"
+dependencies = [
+ "asn1-rs 0.6.2",
+]
+
+[[package]]
+name = "oid-registry"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7"
+dependencies = [
+ "asn1-rs 0.7.2",
+]
+
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -3339,7 +3504,7 @@ dependencies = [
"rustc-hash",
"rustls",
"socket2 0.6.3",
- "thiserror",
+ "thiserror 2.0.18",
"tokio",
"tracing",
"web-time",
@@ -3362,7 +3527,7 @@ dependencies = [
"rustls",
"rustls-pki-types",
"slab",
- "thiserror",
+ "thiserror 2.0.18",
"tinyvec",
"tracing",
"web-time",
@@ -3500,6 +3665,21 @@ dependencies = [
"rand_core 0.10.1",
]
+[[package]]
+name = "rcgen"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e"
+dependencies = [
+ "aws-lc-rs",
+ "pem",
+ "ring",
+ "rustls-pki-types",
+ "time",
+ "x509-parser 0.18.1",
+ "yasna",
+]
+
[[package]]
name = "redis"
version = "0.31.0"
@@ -3688,7 +3868,7 @@ dependencies = [
"cfg-if",
"getrandom 0.2.17",
"libc",
- "untrusted",
+ "untrusted 0.9.0",
"windows-sys 0.52.0",
]
@@ -3807,6 +3987,15 @@ dependencies = [
"semver",
]
+[[package]]
+name = "rusticata-macros"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
+dependencies = [
+ "nom 7.1.3",
+]
+
[[package]]
name = "rustix"
version = "1.1.4"
@@ -3848,6 +4037,15 @@ dependencies = [
"security-framework",
]
+[[package]]
+name = "rustls-pemfile"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
+dependencies = [
+ "rustls-pki-types",
+]
+
[[package]]
name = "rustls-pki-types"
version = "1.14.1"
@@ -3894,7 +4092,7 @@ dependencies = [
"aws-lc-rs",
"ring",
"rustls-pki-types",
- "untrusted",
+ "untrusted 0.9.0",
]
[[package]]
@@ -3997,7 +4195,7 @@ dependencies = [
"serde_json",
"sqlx",
"strum",
- "thiserror",
+ "thiserror 2.0.18",
"time",
"tracing",
"url",
@@ -4012,7 +4210,7 @@ checksum = "5c2eee8405f16c1f337fe3a83389361caea83c928d14dbd666a480407072c365"
dependencies = [
"arrow",
"sea-query",
- "thiserror",
+ "thiserror 2.0.18",
]
[[package]]
@@ -4090,7 +4288,7 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
- "thiserror",
+ "thiserror 2.0.18",
]
[[package]]
@@ -4369,7 +4567,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
dependencies = [
"num-bigint",
"num-traits",
- "thiserror",
+ "thiserror 2.0.18",
"time",
]
@@ -4476,7 +4674,7 @@ dependencies = [
"serde_json",
"sha2",
"smallvec",
- "thiserror",
+ "thiserror 2.0.18",
"time",
"tokio",
"tokio-stream",
@@ -4563,7 +4761,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
- "thiserror",
+ "thiserror 2.0.18",
"time",
"tracing",
"uuid",
@@ -4604,7 +4802,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
- "thiserror",
+ "thiserror 2.0.18",
"time",
"tracing",
"uuid",
@@ -4631,7 +4829,7 @@ dependencies = [
"serde",
"serde_urlencoded",
"sqlx-core",
- "thiserror",
+ "thiserror 2.0.18",
"time",
"tracing",
"url",
@@ -4769,13 +4967,46 @@ dependencies = [
"xattr",
]
+[[package]]
+name = "tempfile"
+version = "3.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
+dependencies = [
+ "fastrand",
+ "getrandom 0.4.2",
+ "once_cell",
+ "rustix",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl 1.0.69",
+]
+
[[package]]
name = "thiserror"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [
- "thiserror-impl",
+ "thiserror-impl 2.0.18",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
]
[[package]]
@@ -5199,7 +5430,7 @@ dependencies = [
"rustls",
"rustls-pki-types",
"sha1",
- "thiserror",
+ "thiserror 2.0.18",
"utf-8",
]
@@ -5216,7 +5447,7 @@ dependencies = [
"log",
"rand 0.9.5",
"sha1",
- "thiserror",
+ "thiserror 2.0.18",
]
[[package]]
@@ -5264,6 +5495,12 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
[[package]]
name = "untrusted"
version = "0.9.0"
@@ -6037,6 +6274,42 @@ dependencies = [
"tap",
]
+[[package]]
+name = "x509-parser"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69"
+dependencies = [
+ "asn1-rs 0.6.2",
+ "data-encoding",
+ "der-parser 9.0.0",
+ "lazy_static",
+ "nom 7.1.3",
+ "oid-registry 0.7.1",
+ "rusticata-macros",
+ "thiserror 1.0.69",
+ "time",
+]
+
+[[package]]
+name = "x509-parser"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202"
+dependencies = [
+ "asn1-rs 0.7.2",
+ "aws-lc-rs",
+ "data-encoding",
+ "der-parser 10.0.0",
+ "lazy_static",
+ "nom 7.1.3",
+ "oid-registry 0.8.1",
+ "ring",
+ "rusticata-macros",
+ "thiserror 2.0.18",
+ "time",
+]
+
[[package]]
name = "xattr"
version = "1.6.1"
@@ -6053,6 +6326,15 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
+[[package]]
+name = "yasna"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd"
+dependencies = [
+ "time",
+]
+
[[package]]
name = "yoke"
version = "0.8.2"
@@ -6169,7 +6451,7 @@ dependencies = [
"flate2",
"indexmap 2.14.0",
"memchr",
- "thiserror",
+ "thiserror 2.0.18",
"zopfli",
]
diff --git a/Cargo.toml b/Cargo.toml
index 4e54e23..4545b35 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -68,6 +68,7 @@ bollard = "0.19"
tar = "0.4"
futures-util = "0.3"
object_store = { version = "0.14.1", default-features = false, features = ["fs", "aws"] }
+instant-acme = { version = "0.8.5", default-features = true, features = ["rcgen"] }
grass-archive = { path = "crates/archive" }
grass-assets = { path = "crates/assets" }
diff --git a/apps/console/src/components/certificate-import-dialog.tsx b/apps/console/src/components/certificate-import-dialog.tsx
new file mode 100644
index 0000000..4b69af5
--- /dev/null
+++ b/apps/console/src/components/certificate-import-dialog.tsx
@@ -0,0 +1,110 @@
+import { useMutation } from "@tanstack/react-query";
+import { useId, useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+ DialogTrigger,
+} from "@/components/ui/dialog";
+import { Field, FieldDescription, FieldGroup, FieldLabel } from "@/components/ui/field";
+import { Textarea } from "@/components/ui/textarea";
+
+export function CertificateImportDialog({
+ hostname,
+ onImport,
+ onImported,
+}: {
+ hostname: string;
+ onImport: (input: { certificate_pem: string; private_key_pem: string }) => Promise;
+ onImported: () => void;
+}) {
+ const id = useId();
+ const [open, setOpen] = useState(false);
+ const [certificate, setCertificate] = useState("");
+ const [privateKey, setPrivateKey] = useState("");
+ const clear = () => {
+ setCertificate("");
+ setPrivateKey("");
+ };
+ const mutation = useMutation({
+ mutationFn: () => onImport({ certificate_pem: certificate, private_key_pem: privateKey }),
+ onSuccess: () => {
+ clear();
+ setOpen(false);
+ onImported();
+ },
+ });
+ return (
+ {
+ setOpen(value);
+ if (!value) clear();
+ }}
+ >
+
+
+ Import certificate
+
+
+
+
+ Import certificate for {hostname}
+
+ The certificate must cover this hostname and match its private key. Imported
+ certificates are renewed manually.
+
+
+
+
+
+ );
+}
diff --git a/apps/console/src/features/admin/admin-route.test.tsx b/apps/console/src/features/admin/admin-route.test.tsx
index 8603cc6..bc378d1 100644
--- a/apps/console/src/features/admin/admin-route.test.tsx
+++ b/apps/console/src/features/admin/admin-route.test.tsx
@@ -77,6 +77,7 @@ it("lists nodes with health state for administrators", async () => {
{
id: "node-1",
name: "build-node-1",
+ region: "default",
status: "active",
healthy: true,
build_enabled: true,
diff --git a/apps/console/src/features/admin/admin-sections.ts b/apps/console/src/features/admin/admin-sections.ts
index 1b7c2f2..f02be0a 100644
--- a/apps/console/src/features/admin/admin-sections.ts
+++ b/apps/console/src/features/admin/admin-sections.ts
@@ -26,6 +26,7 @@ export const adminSections: AdminSection[] = [
{ to: "/admin/projects", label: "Projects", icon: FolderGitIcon },
{ to: "/admin/nodes", label: "Nodes", icon: ServerIcon },
{ to: "/admin/host-sources", label: "Host sources", icon: GlobeIcon },
+ { to: "/admin/regional-ingresses", label: "Regional ingresses", icon: GlobeIcon },
{ to: "/admin/quota-plans", label: "Quota plans", icon: GaugeIcon },
{ to: "/admin/team-groups", label: "Team groups", icon: LayersIcon },
{ to: "/admin/users", label: "Users", icon: UserIcon },
diff --git a/apps/console/src/features/admin/admin.api.ts b/apps/console/src/features/admin/admin.api.ts
index 0994b35..119b0f4 100644
--- a/apps/console/src/features/admin/admin.api.ts
+++ b/apps/console/src/features/admin/admin.api.ts
@@ -18,6 +18,7 @@ export type NodeStatus = "pending" | "active" | "draining" | "offline" | "disabl
export interface AdminNode {
id: string;
name: string;
+ region: string;
status: NodeStatus;
healthy: boolean;
build_enabled: boolean;
@@ -85,6 +86,7 @@ export interface AdminNodeConfigurationSync {
export interface NodeConfiguration {
node: {
id: string;
+ region: string;
control_api: string;
work_root: string;
capabilities: { build: boolean; serve: boolean };
@@ -180,6 +182,7 @@ export interface AdminHostSource {
kind: HostSourceKind;
label: string;
base_domain: string;
+ region: string;
enabled: boolean;
allows_auto_assign: boolean;
is_default: boolean;
@@ -188,6 +191,40 @@ export interface AdminHostSource {
created_at: string;
}
+export interface AdminRegionalIngress {
+ id: string;
+ region: string;
+ hostname: string;
+ enabled: boolean;
+ health_check_path: string;
+ health_check_interval_seconds: number;
+ origin_host_preservation: boolean;
+ tls_enabled: boolean;
+ certificate_issuer: "letsencrypt" | "zerossl" | "manual";
+ certificate_auto_renew: boolean;
+ certificate_status: "pending" | "issuing" | "active" | "expiring" | "failed" | "disabled";
+ certificate_expires_at: string | null;
+ certificate_issued_at?: string | null;
+ certificate_revision?: string | null;
+ certificate_retry_at?: string | null;
+ node_statuses?: Array<{
+ node_id: string;
+ tls_ready: boolean;
+ challenge_revision: string;
+ checked_at: string | null;
+ certificate_revision: string | null;
+ }>;
+ certificate_error: string | null;
+ dns_challenge_provider: string | null;
+ dns_challenge_config_keys: string[];
+ dns_challenge_status: "not_configured" | "pending" | "valid" | "failed";
+ dns_challenge_record_name: string | null;
+ dns_challenge_record_value: string | null;
+ healthy_nodes: Array<{ node_id: string; base_url: string; priority: number }>;
+ created_at: string;
+ updated_at: string;
+}
+
export interface AdminUser {
id: string;
email: string;
@@ -686,6 +723,7 @@ export const adminApi = {
label: string;
kind: HostSourceKind;
base_domain: string;
+ region?: string;
is_default?: boolean;
provider?: string;
config?: Record;
@@ -698,7 +736,7 @@ export const adminApi = {
updateHostSource: (
sourceId: string,
input: Partial<
- Pick
+ Pick
> & {
provider?: string;
/** Shallow-merged server side; a null value deletes the key. */
@@ -713,10 +751,69 @@ export const adminApi = {
removeHostSource: (sourceId: string) =>
request<{ ok: true }>(`/api/v1/admin/host-sources/${sourceId}`, { method: "DELETE" }),
+ listRegionalIngresses: () =>
+ request<{ regional_ingresses: AdminRegionalIngress[] }>("/api/v1/admin/regional-ingresses"),
+
+ createRegionalIngress: (input: {
+ region: string;
+ hostname: string;
+ health_check_path?: string;
+ health_check_interval_seconds?: number;
+ tls_enabled?: boolean;
+ certificate_issuer?: AdminRegionalIngress["certificate_issuer"];
+ certificate_auto_renew?: boolean;
+ dns_challenge_provider?: string;
+ dns_challenge_config?: Record;
+ }) =>
+ request<{ regional_ingress: AdminRegionalIngress }>("/api/v1/admin/regional-ingresses", {
+ method: "POST",
+ body: JSON.stringify(input),
+ }),
+
+ updateRegionalIngress: (
+ ingressId: string,
+ input: Partial<
+ Pick<
+ AdminRegionalIngress,
+ | "hostname"
+ | "enabled"
+ | "health_check_path"
+ | "health_check_interval_seconds"
+ | "origin_host_preservation"
+ | "tls_enabled"
+ | "certificate_issuer"
+ | "certificate_auto_renew"
+ | "dns_challenge_provider"
+ >
+ > & { dns_challenge_config?: Record },
+ ) =>
+ request<{ regional_ingress: AdminRegionalIngress }>(
+ `/api/v1/admin/regional-ingresses/${ingressId}`,
+ { method: "PATCH", body: JSON.stringify(input) },
+ ),
+
+ removeRegionalIngress: (ingressId: string) =>
+ request<{ ok: true }>(`/api/v1/admin/regional-ingresses/${ingressId}`, { method: "DELETE" }),
+
+ renewRegionalIngressCertificate: (ingressId: string) =>
+ request<{ regional_ingress: AdminRegionalIngress }>(
+ `/api/v1/admin/regional-ingresses/${ingressId}/certificate/renew`,
+ { method: "POST" },
+ ),
+
+ importRegionalIngressCertificate: (
+ ingressId: string,
+ input: { certificate_pem: string; private_key_pem: string },
+ ) =>
+ request<{ regional_ingress: AdminRegionalIngress }>(
+ `/api/v1/admin/regional-ingresses/${ingressId}/certificate/import`,
+ { method: "POST", body: JSON.stringify(input) },
+ ),
+
listNodes: () =>
request<{ nodes: AdminNode[]; local_process: AdminLocalProcessInfo }>("/api/v1/admin/nodes"),
- createNode: (input: { name: string; start_local?: boolean }) =>
+ createNode: (input: { name: string; region?: string; start_local?: boolean }) =>
request<{
node: AdminNode;
token: string;
diff --git a/apps/console/src/features/admin/components/host-sources-panel.test.tsx b/apps/console/src/features/admin/components/host-sources-panel.test.tsx
index c741c95..b4d88c1 100644
--- a/apps/console/src/features/admin/components/host-sources-panel.test.tsx
+++ b/apps/console/src/features/admin/components/host-sources-panel.test.tsx
@@ -17,6 +17,7 @@ const wildcardSource = {
kind: "wildcard",
label: "Platform apps",
base_domain: "apps.example.com",
+ region: "default",
enabled: true,
allows_auto_assign: true,
is_default: true,
@@ -31,6 +32,7 @@ const cloudflareSource = {
kind: "dns_provider",
label: "Cloudflare zone",
base_domain: "cf.example.com",
+ region: "default",
is_default: false,
provider: "cloudflare",
config_keys: ["api_token", "zone_id", "record_type", "record_value", "proxied"],
diff --git a/apps/console/src/features/admin/components/host-sources-panel.tsx b/apps/console/src/features/admin/components/host-sources-panel.tsx
index a3662a6..c3c2a4f 100644
--- a/apps/console/src/features/admin/components/host-sources-panel.tsx
+++ b/apps/console/src/features/admin/components/host-sources-panel.tsx
@@ -193,6 +193,7 @@ export function HostSourcesPanel() {
Label
Kind
Base domain
+ Region
Flags
Actions
@@ -210,6 +211,7 @@ export function HostSourcesPanel() {
{source.kind.replace("_", " ")}
{source.base_domain}
+ {source.region}
{source.is_default && Default }
{source.allows_auto_assign && Auto-assign }
@@ -254,6 +256,7 @@ function CreateHostSourceDialog({ onCreated }: { onCreated: () => void }) {
const [label, setLabel] = useState("");
const [kind, setKind] = useState("wildcard");
const [baseDomain, setBaseDomain] = useState("");
+ const [region, setRegion] = useState("default");
const [isDefault, setIsDefault] = useState(false);
const [cloudflare, setCloudflare] = useState(emptyCloudflareForm);
@@ -263,6 +266,7 @@ function CreateHostSourceDialog({ onCreated }: { onCreated: () => void }) {
label,
kind,
base_domain: baseDomain,
+ region,
is_default: isDefault,
...(kind === "dns_provider"
? {
@@ -275,6 +279,7 @@ function CreateHostSourceDialog({ onCreated }: { onCreated: () => void }) {
setOpen(false);
setLabel("");
setBaseDomain("");
+ setRegion("default");
setIsDefault(false);
setCloudflare(emptyCloudflareForm);
onCreated();
@@ -312,6 +317,16 @@ function CreateHostSourceDialog({ onCreated }: { onCreated: () => void }) {
required
/>
+
+ Region
+ setRegion(event.target.value)}
+ required
+ />
+
Kind
setKind(value as HostSourceKind)}>
@@ -371,6 +386,7 @@ function EditHostSourceDialog({
const [open, setOpen] = useState(false);
const [label, setLabel] = useState(source.label);
const [enabled, setEnabled] = useState(source.enabled);
+ const [region, setRegion] = useState(source.region);
const [allowsAutoAssign, setAllowsAutoAssign] = useState(source.allows_auto_assign);
const [cloudflare, setCloudflare] = useState(emptyCloudflareForm);
const [shapeTouched, setShapeTouched] = useState(false);
@@ -381,6 +397,7 @@ function EditHostSourceDialog({
const config = cloudflareConfig(cloudflare, { includeShape: shapeTouched });
return adminApi.updateHostSource(source.id, {
label,
+ region,
enabled,
allows_auto_assign: allowsAutoAssign,
...(isCloudflare && Object.keys(config).length > 0
@@ -403,6 +420,7 @@ function EditHostSourceDialog({
setOpen(next);
if (next) {
setLabel(source.label);
+ setRegion(source.region);
setEnabled(source.enabled);
setAllowsAutoAssign(source.allows_auto_assign);
setCloudflare(emptyCloudflareForm);
@@ -440,6 +458,15 @@ function EditHostSourceDialog({
required
/>
+
+ Region
+ setRegion(event.target.value)}
+ required
+ />
+
Name
+ Region
Health
Capabilities
Serve load
@@ -374,6 +375,9 @@ export function NodesPanel() {
)}
+
@@ -770,6 +774,16 @@ function EditConfigurationDialog({ node }: { node: AdminNode }) {
)
}
/>
+
+ setConfiguration((current) =>
+ current ? { ...current, node: { ...current.node, region } } : current,
+ )
+ }
+ />
adminApi.createNode({ name, start_local: startLocal }),
+ mutationFn: () => adminApi.createNode({ name, region, start_local: startLocal }),
onSuccess: ({ token, warnings }) => {
setOpen(false);
setName("");
+ setRegion("default");
onCreated(token, warnings ?? []);
},
});
@@ -1479,6 +1495,16 @@ function CreateNodeDialog({
required
/>
+
+ Region
+ setRegion(event.target.value)}
+ required
+ />
+
= [];
+ vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
+ calls.push({ url: String(input), init });
+ return new Response(
+ JSON.stringify({
+ code: 200,
+ message: "OK",
+ data: init?.method ? { regional_ingress: ingress } : { regional_ingresses: items },
+ }),
+ { status: 200, headers: { "Content-Type": "application/json" } },
+ );
+ });
+ const client = new QueryClient({
+ defaultOptions: { queries: { retry: false }, mutations: { retry: false } },
+ });
+ render(
+
+
+ ,
+ );
+ return calls;
+}
+
+afterEach(() => vi.restoreAllMocks());
+
+it("creates an ingress with usable DNS credentials and the reserved health path", async () => {
+ const user = userEvent.setup();
+ const calls = setup([]);
+ await user.click(screen.getByRole("button", { name: "Add regional ingress" }));
+ await user.type(screen.getByLabelText("Entry hostname"), "eu.edge.example.com");
+ await user.type(screen.getByLabelText("Cloudflare API token"), "test-token");
+ await user.type(screen.getByLabelText("Cloudflare zone ID"), "test-zone");
+ await user.click(screen.getByRole("button", { name: "Create ingress" }));
+ await waitFor(() => expect(calls.some((call) => call.init?.method === "POST")).toBe(true));
+ const body = JSON.parse(String(calls.find((call) => call.init?.method === "POST")!.init!.body));
+ expect(body).toMatchObject({
+ hostname: "eu.edge.example.com",
+ health_check_path: "/_grass/health",
+ dns_challenge_provider: "cloudflare",
+ dns_challenge_config: { api_token: "test-token", zone_id: "test-zone" },
+ });
+});
+
+it("edits an ingress without replacing stored credentials with empty fields", async () => {
+ const user = userEvent.setup();
+ const calls = setup();
+ await user.click(await screen.findByRole("button", { name: "Edit" }));
+ expect(screen.getByLabelText("Cloudflare API token")).toHaveValue("");
+ await user.clear(screen.getByLabelText("Entry hostname"));
+ await user.type(screen.getByLabelText("Entry hostname"), "new.edge.example.com");
+ await user.click(screen.getByRole("button", { name: "Save changes" }));
+ await waitFor(() => expect(calls.some((call) => call.init?.method === "PATCH")).toBe(true));
+ const body = JSON.parse(String(calls.find((call) => call.init?.method === "PATCH")!.init!.body));
+ expect(body.hostname).toBe("new.edge.example.com");
+ expect(body.dns_challenge_config).toEqual({});
+});
+
+it("shows installation and failure state and queues certificate renewal", async () => {
+ const user = userEvent.setup();
+ const calls = setup();
+ expect(await screen.findByText("DNS validation failed")).toBeInTheDocument();
+ expect(screen.getByText("1 using current certificate")).toBeInTheDocument();
+ await user.click(screen.getByRole("button", { name: "Renew / retry" }));
+ await waitFor(() =>
+ expect(
+ calls.some(
+ (call) => call.url.endsWith("/ingress-1/certificate/renew") && call.init?.method === "POST",
+ ),
+ ).toBe(true),
+ );
+});
+
+it("removes an optional stored credential only when explicitly selected", async () => {
+ const user = userEvent.setup();
+ const calls = setup([
+ { ...ingress, dns_challenge_config_keys: ["api_token", "zone_id", "contact_email"] },
+ ]);
+ await user.click(await screen.findByRole("button", { name: "Edit" }));
+ await user.click(screen.getByRole("button", { name: "Remove Certificate contact email" }));
+ await user.click(screen.getByRole("button", { name: "Save changes" }));
+ await waitFor(() => expect(calls.some((call) => call.init?.method === "PATCH")).toBe(true));
+ const body = JSON.parse(String(calls.find((call) => call.init?.method === "PATCH")!.init!.body));
+ expect(body.dns_challenge_config).toEqual({ contact_email: null });
+});
+
+it("imports a manual certificate and clears key material after the dialog closes", async () => {
+ const user = userEvent.setup();
+ const calls = setup();
+ await user.click(await screen.findByRole("button", { name: "Import certificate" }));
+ const dialog = screen.getByRole("dialog");
+ await user.type(within(dialog).getByLabelText("Certificate chain (PEM)"), "TEST CERTIFICATE");
+ await user.type(within(dialog).getByLabelText("Private key (PEM)"), "TEST PRIVATE KEY");
+ await user.click(within(dialog).getByRole("button", { name: "Import", exact: true }));
+ await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument());
+ const request = calls.find((call) => call.url.endsWith("/certificate/import"))!;
+ expect(JSON.parse(String(request.init!.body))).toEqual({
+ certificate_pem: "TEST CERTIFICATE",
+ private_key_pem: "TEST PRIVATE KEY",
+ });
+ await user.click(screen.getByRole("button", { name: "Import certificate" }));
+ expect(screen.getByLabelText("Private key (PEM)")).toHaveValue("");
+});
diff --git a/apps/console/src/features/admin/components/regional-ingresses-panel.tsx b/apps/console/src/features/admin/components/regional-ingresses-panel.tsx
new file mode 100644
index 0000000..19fc336
--- /dev/null
+++ b/apps/console/src/features/admin/components/regional-ingresses-panel.tsx
@@ -0,0 +1,551 @@
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { Globe2Icon, PlusIcon, RefreshCwIcon, Trash2Icon } from "lucide-react";
+import { useId, useState } from "react";
+
+import { CertificateImportDialog } from "@/components/certificate-import-dialog";
+import { Alert, AlertDescription } from "@/components/ui/alert";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { Checkbox } from "@/components/ui/checkbox";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+ DialogTrigger,
+} from "@/components/ui/dialog";
+import {
+ Empty,
+ EmptyDescription,
+ EmptyHeader,
+ EmptyMedia,
+ EmptyTitle,
+} from "@/components/ui/empty";
+import {
+ Field,
+ FieldDescription,
+ FieldGroup,
+ FieldLabel,
+ FieldLegend,
+ FieldSet,
+} from "@/components/ui/field";
+import { Input } from "@/components/ui/input";
+import {
+ Select,
+ SelectContent,
+ SelectGroup,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import { Skeleton } from "@/components/ui/skeleton";
+import {
+ Table,
+ TableBody,
+ TableCell,
+ TableHead,
+ TableHeader,
+ TableRow,
+} from "@/components/ui/table";
+
+import { adminApi, type AdminRegionalIngress } from "../admin.api";
+
+function statusVariant(
+ status: AdminRegionalIngress["certificate_status"],
+): "success" | "warning" | "destructive" | "secondary" {
+ if (status === "active") return "success";
+ if (status === "failed" || status === "expiring") return "destructive";
+ if (status === "pending" || status === "issuing") return "warning";
+ return "secondary";
+}
+
+function displayTime(value?: string | null) {
+ return value ? new Date(value).toLocaleString() : "—";
+}
+
+export function RegionalIngressesPanel() {
+ const queryClient = useQueryClient();
+ const query = useQuery({
+ queryKey: ["admin", "regional-ingresses"],
+ queryFn: adminApi.listRegionalIngresses,
+ refetchInterval: 10_000,
+ });
+ const invalidate = () => {
+ void queryClient.invalidateQueries({ queryKey: ["admin", "regional-ingresses"] });
+ };
+ const removeMutation = useMutation({
+ mutationFn: adminApi.removeRegionalIngress,
+ onSuccess: invalidate,
+ });
+ const toggleMutation = useMutation({
+ mutationFn: ({ id, enabled }: { id: string; enabled: boolean }) =>
+ adminApi.updateRegionalIngress(id, { enabled }),
+ onSuccess: invalidate,
+ });
+ const renewMutation = useMutation({
+ mutationFn: adminApi.renewRegionalIngressCertificate,
+ onSuccess: invalidate,
+ });
+
+ return (
+
+
+
+
Regional ingresses
+
+ Manage regional entry domains, certificates, and healthy entry nodes.
+
+
+
+
+ {query.isLoading &&
}
+ {query.isError && (
+
+ Regional ingresses could not be loaded.
+
+ )}
+ {query.data &&
+ (query.data.regional_ingresses.length === 0 ? (
+
+
+
+
+
+ No regional ingresses
+
+ Add an entry domain to enable regional CNAME guidance and automated certificates.
+
+
+
+ ) : (
+
+
+
+ Region / hostname
+ Certificate
+ DNS validation
+ Entry nodes
+ Actions
+
+
+
+ {query.data.regional_ingresses.map((item) => (
+
+
+ {item.region}
+ {item.hostname}
+ {!item.enabled && Disabled }
+
+
+
+ {item.tls_enabled ? item.certificate_status : "off"}
+
+
+ {item.certificate_issuer} ·{" "}
+ {item.certificate_auto_renew ? "Auto renewal" : "Manual renewal"}
+
+ Expires {displayTime(item.certificate_expires_at)}
+ {item.certificate_retry_at && (
+
+ Next attempt {displayTime(item.certificate_retry_at)}
+
+ )}
+ {item.certificate_error && (
+ {item.certificate_error}
+ )}
+
+
+
+ {item.dns_challenge_status}
+
+
+ {item.dns_challenge_provider ?? "Not configured"}
+
+
+
+ {item.healthy_nodes.length} healthy
+
+ {
+ (item.node_statuses ?? []).filter(
+ (node) =>
+ node.tls_ready &&
+ item.healthy_nodes.some(
+ (healthy) => healthy.node_id === node.node_id,
+ ) &&
+ !!item.certificate_revision &&
+ node.certificate_revision === item.certificate_revision,
+ ).length
+ }{" "}
+ using current certificate
+
+ {(item.node_statuses?.length ?? 0) > 0 && (
+
+ Node status
+
+ {item.node_statuses?.map((node) => (
+
+ {node.node_id} · {node.tls_ready ? "TLS ready" : "TLS unavailable"} ·{" "}
+ {displayTime(node.checked_at)}
+
+ ))}
+
+
+ )}
+
+
+
+
+ renewMutation.mutate(item.id)}
+ >
+
+ Renew / retry
+
+
+ adminApi.importRegionalIngressCertificate(item.id, input)
+ }
+ onImported={invalidate}
+ />
+
+ toggleMutation.mutate({ id: item.id, enabled: !item.enabled })
+ }
+ disabled={toggleMutation.isPending}
+ >
+ {item.enabled ? "Disable" : "Enable"}
+
+ removeMutation.mutate(item.id)}
+ disabled={removeMutation.isPending}
+ >
+
+
+
+
+
+ ))}
+
+
+ ))}
+
+ );
+}
+
+type Provider = "cloudflare" | "dnspod" | "route53";
+const providerFields: Record<
+ Provider,
+ Array<{ key: string; label: string; secret?: boolean; optional?: boolean }>
+> = {
+ cloudflare: [
+ { key: "api_token", label: "Cloudflare API token", secret: true },
+ { key: "zone_id", label: "Cloudflare zone ID" },
+ { key: "zone", label: "DNS zone", optional: true },
+ ],
+ dnspod: [
+ { key: "secret_id", label: "Tencent Cloud secret ID", secret: true },
+ { key: "secret_key", label: "Tencent Cloud secret key", secret: true },
+ { key: "domain", label: "DNSPod domain" },
+ ],
+ route53: [
+ { key: "access_key_id", label: "AWS access key ID", secret: true },
+ { key: "secret_access_key", label: "AWS secret access key", secret: true },
+ { key: "hosted_zone_id", label: "Route53 hosted zone ID" },
+ { key: "zone", label: "DNS zone", optional: true },
+ { key: "region", label: "AWS signing region", optional: true },
+ ],
+};
+
+function RegionalIngressDialog({
+ item,
+ onSaved,
+}: {
+ item?: AdminRegionalIngress;
+ onSaved: () => void;
+}) {
+ const [open, setOpen] = useState(false);
+ return (
+
+
+
+ {!item && }
+ {item ? "Edit" : "Add regional ingress"}
+
+
+
+
+ {item ? "Edit regional ingress" : "Add regional ingress"}
+
+ Configure the entry domain and certificate authority for this region.
+
+
+ {open && (
+ {
+ setOpen(false);
+ onSaved();
+ }}
+ />
+ )}
+
+
+ );
+}
+
+function IngressForm({ item, onSaved }: { item?: AdminRegionalIngress; onSaved: () => void }) {
+ const id = useId();
+ const [region, setRegion] = useState(item?.region ?? "default");
+ const [hostname, setHostname] = useState(item?.hostname ?? "");
+ const [healthPath, setHealthPath] = useState(item?.health_check_path ?? "/_grass/health");
+ const [healthInterval, setHealthInterval] = useState(
+ String(item?.health_check_interval_seconds ?? 30),
+ );
+ const [issuer, setIssuer] = useState(
+ item?.certificate_issuer ?? "letsencrypt",
+ );
+ const [provider, setProvider] = useState(
+ (item?.dns_challenge_provider as Provider) in providerFields
+ ? (item!.dns_challenge_provider as Provider)
+ : "cloudflare",
+ );
+ const [tlsEnabled, setTlsEnabled] = useState(item?.tls_enabled ?? true);
+ const [autoRenew, setAutoRenew] = useState(item?.certificate_auto_renew ?? true);
+ const [config, setConfig] = useState>({});
+ const configured =
+ item?.dns_challenge_provider === provider ? item.dns_challenge_config_keys : [];
+ const fields = [
+ ...providerFields[provider],
+ { key: "contact_email", label: "Certificate contact email", optional: true },
+ ...(issuer === "zerossl"
+ ? [
+ { key: "eab_kid", label: "ZeroSSL EAB key ID", secret: true },
+ { key: "eab_hmac_key", label: "ZeroSSL EAB HMAC key", secret: true },
+ ]
+ : []),
+ ];
+ const mutation = useMutation({
+ mutationFn: () => {
+ const dnsConfig = Object.fromEntries(
+ Object.entries(config)
+ .filter(([, value]) => value === null || value.trim())
+ .map(([key, value]) => [key, value?.trim() ?? null]),
+ );
+ const input = {
+ hostname: hostname.trim(),
+ health_check_path: healthPath.trim(),
+ health_check_interval_seconds: Number(healthInterval),
+ tls_enabled: tlsEnabled,
+ certificate_issuer: issuer,
+ certificate_auto_renew: issuer !== "manual" && autoRenew,
+ ...(tlsEnabled && issuer !== "manual"
+ ? { dns_challenge_provider: provider, dns_challenge_config: dnsConfig }
+ : {}),
+ };
+ return item
+ ? adminApi.updateRegionalIngress(item.id, input)
+ : adminApi.createRegionalIngress({ ...input, region: region.trim() });
+ },
+ onSuccess: onSaved,
+ });
+ return (
+ {
+ event.preventDefault();
+ mutation.mutate();
+ }}
+ >
+
+
+ Region
+ setRegion(event.target.value)}
+ disabled={!!item}
+ required
+ />
+
+
+ Entry hostname
+ setHostname(event.target.value)}
+ placeholder="eu.edge.example.com"
+ required
+ />
+
+
+ Health checks
+
+
+ Health check path
+ setHealthPath(event.target.value)}
+ required
+ pattern="/.*"
+ />
+
+
+ Interval (seconds)
+ setHealthInterval(event.target.value)}
+ required
+ />
+
+
+
+
+ HTTPS
+
+
+ setTlsEnabled(value === true)}
+ />
+ Enable HTTPS
+
+ {tlsEnabled && (
+ <>
+
+ Certificate authority
+ setIssuer(value as typeof issuer)}
+ >
+
+
+
+
+
+ Let's Encrypt
+ ZeroSSL
+ Manual certificate
+
+
+
+
+ {issuer !== "manual" && (
+ <>
+
+ setAutoRenew(value === true)}
+ />
+
+ Automatically renew certificates
+
+
+
+ DNS provider
+ {
+ setProvider(value as Provider);
+ setConfig({});
+ }}
+ >
+
+
+
+
+
+ Cloudflare
+ DNSPod
+ Route53
+
+
+
+
+ Allow the provider to create and remove validation TXT records in the entry
+ domain's zone.
+
+
+ {fields.map((field) => (
+
+ {field.label}
+
+ setConfig({ ...config, [field.key]: event.target.value })
+ }
+ required={!field.optional && !configured.includes(field.key)}
+ autoComplete={field.secret ? "new-password" : "off"}
+ placeholder={
+ configured.includes(field.key)
+ ? "Configured — leave blank to keep"
+ : undefined
+ }
+ />
+ {field.optional && configured.includes(field.key) && (
+
+ setConfig({
+ ...config,
+ [field.key]: config[field.key] === null ? "" : null,
+ })
+ }
+ >
+ {config[field.key] === null
+ ? `Keep ${field.label}`
+ : `Remove ${field.label}`}
+
+ )}
+
+ ))}
+ >
+ )}
+ >
+ )}
+
+
+
+
+
+ {mutation.isPending ? "Saving…" : item ? "Save changes" : "Create ingress"}
+
+
+
+ );
+}
diff --git a/apps/console/src/features/deployments/deployment-detail-route.test.tsx b/apps/console/src/features/deployments/deployment-detail-route.test.tsx
index 0e70ea5..064ee23 100644
--- a/apps/console/src/features/deployments/deployment-detail-route.test.tsx
+++ b/apps/console/src/features/deployments/deployment-detail-route.test.tsx
@@ -28,6 +28,7 @@ function detailFixture(overrides: Partial = {}):
id: "deployment-1",
project_id: "project-1",
team_id: "team-1",
+ region: "default",
build_node: { id: "build-node-1", name: "builder-1" },
serve_node: { id: "serve-node-1", name: "serve-node-1" },
environment: "production",
diff --git a/apps/console/src/features/deployments/deployments-tab.test.tsx b/apps/console/src/features/deployments/deployments-tab.test.tsx
index c688679..543b6cc 100644
--- a/apps/console/src/features/deployments/deployments-tab.test.tsx
+++ b/apps/console/src/features/deployments/deployments-tab.test.tsx
@@ -55,6 +55,7 @@ function deploymentFixture(overrides: Partial = {}): Deployment {
id: "deployment-1",
project_id: "project-1",
team_id: "team-1",
+ region: "default",
build_node: { id: "build-node-1", name: "builder-1" },
serve_node: { id: "serve-node-1", name: "serve-node-1" },
environment: "production",
@@ -149,6 +150,7 @@ it("creates with automatic placement by default and sends a selected serve node"
{
id: "node-1",
name: "serve-node-1",
+ region: "default",
healthy: true,
capacity: {
cpu_millicores: 1000,
@@ -166,6 +168,7 @@ it("creates with automatic placement by default and sends a selected serve node"
{
id: "node-2",
name: "serve-node-2",
+ region: "eu-west",
healthy: true,
capacity: {
cpu_millicores: 2000,
diff --git a/apps/console/src/features/deployments/deployments-tab.tsx b/apps/console/src/features/deployments/deployments-tab.tsx
index fe3078c..4884aa8 100644
--- a/apps/console/src/features/deployments/deployments-tab.tsx
+++ b/apps/console/src/features/deployments/deployments-tab.tsx
@@ -57,6 +57,7 @@ export function DeploymentsTab({
const [deploymentEnvironment, setDeploymentEnvironment] = useState(
null,
);
+ const [deploymentRegion, setDeploymentRegion] = useState("automatic");
const [serveNodeId, setServeNodeId] = useState("automatic");
const deploymentsQuery = useQuery({
@@ -90,12 +91,17 @@ export function DeploymentsTab({
});
const serveNodes = serveNodesQuery.data?.serve_nodes ?? [];
- const selectedNode = serveNodes.find((node) => node.id === serveNodeId);
+ const regions = [...new Set(serveNodes.map((node) => node.region))].sort();
+ const regionalServeNodes =
+ deploymentRegion === "automatic"
+ ? serveNodes
+ : serveNodes.filter((node) => node.region === deploymentRegion);
+ const selectedNode = regionalServeNodes.find((node) => node.id === serveNodeId);
const canSubmit =
!serveNodesQuery.isLoading &&
!serveNodesQuery.isError &&
(serveNodeId === "automatic"
- ? serveNodes.some((node) => node.schedulable)
+ ? regionalServeNodes.some((node) => node.schedulable)
: selectedNode?.schedulable === true);
useEffect(() => {
@@ -108,6 +114,7 @@ export function DeploymentsTab({
}, [projectId, serveNodes, serveNodesQuery.data]);
const openDeploymentDialog = (environment: DeploymentEnvironment) => {
+ setDeploymentRegion("automatic");
setServeNodeId("automatic");
setDeploymentEnvironment(environment);
};
@@ -173,10 +180,44 @@ export function DeploymentsTab({
if (!deploymentEnvironment || !canSubmit) return;
createMutation.mutate({
environment: deploymentEnvironment,
+ ...(deploymentRegion === "automatic" ? {} : { region: deploymentRegion }),
...(serveNodeId === "automatic" ? {} : { serve_node_id: serveNodeId }),
});
}}
>
+
+ Region
+ {
+ setDeploymentRegion(value);
+ setServeNodeId("automatic");
+ }}
+ >
+
+
+
+
+ node.schedulable)}
+ >
+ Automatic · any region
+
+ {regions.map((region) => (
+ node.region === region && node.schedulable)
+ }
+ >
+ {region}
+
+ ))}
+
+
+
Serve node
@@ -186,13 +227,13 @@ export function DeploymentsTab({
node.schedulable)}
+ disabled={!regionalServeNodes.some((node) => node.schedulable)}
>
Automatic · least loaded
- {serveNodes.map((node) => (
+ {regionalServeNodes.map((node) => (
- {node.name} · {formatNodeUsage(node)}
+ {node.name} · {node.region} · {formatNodeUsage(node)}
{node.overflow_only ? " · overflow" : ""}
))}
@@ -283,7 +324,8 @@ export function DeploymentsTab({
{deployment.overcommitted && Overflow }
- {deployment.serve_node?.name ?? "Unassigned"}
+ {deployment.serve_node?.name ?? "Unassigned"}
+ · {deployment.region}
{deployment.serve_resources.cpu_millicores}m ·{" "}
diff --git a/apps/console/src/features/deployments/deployments.api.ts b/apps/console/src/features/deployments/deployments.api.ts
index bc97577..66d52cd 100644
--- a/apps/console/src/features/deployments/deployments.api.ts
+++ b/apps/console/src/features/deployments/deployments.api.ts
@@ -36,6 +36,7 @@ export interface NodeUsage extends ServeResources {
export interface ServeNodeTarget {
id: string;
name: string;
+ region: string;
healthy: boolean;
capacity: NodeResources;
usage: NodeUsage;
@@ -50,6 +51,7 @@ export interface Deployment {
id: string;
project_id: string;
team_id: string;
+ region: string;
build_node: NodeRef | null;
serve_node: NodeRef | null;
environment: DeploymentEnvironment;
@@ -142,15 +144,24 @@ export const deploymentsApi = {
create: (
projectId: string,
- input: { environment: DeploymentEnvironment; branch?: string; serve_node_id?: string },
+ input: {
+ environment: DeploymentEnvironment;
+ branch?: string;
+ serve_node_id?: string;
+ region?: string;
+ },
) =>
request<{ deployment: Deployment }>(`/api/v1/projects/${projectId}/deployments`, {
method: "POST",
body: JSON.stringify(input),
}),
- serveNodes: (projectId: string) =>
- request<{ serve_nodes: ServeNodeTarget[] }>(`/api/v1/projects/${projectId}/serve-nodes`),
+ serveNodes: (projectId: string, region?: string) => {
+ const query = region ? `?region=${encodeURIComponent(region)}` : "";
+ return request<{ serve_nodes: ServeNodeTarget[] }>(
+ `/api/v1/projects/${projectId}/serve-nodes${query}`,
+ );
+ },
detail: (projectId: string, deploymentId: string) =>
request(`/api/v1/projects/${projectId}/deployments/${deploymentId}`),
diff --git a/apps/console/src/features/projects/domain-certificate-controls.tsx b/apps/console/src/features/projects/domain-certificate-controls.tsx
new file mode 100644
index 0000000..761ca80
--- /dev/null
+++ b/apps/console/src/features/projects/domain-certificate-controls.tsx
@@ -0,0 +1,168 @@
+import { useMutation } from "@tanstack/react-query";
+import { useId } from "react";
+
+import { CertificateImportDialog } from "@/components/certificate-import-dialog";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { Checkbox } from "@/components/ui/checkbox";
+import { Field, FieldDescription, FieldGroup, FieldLabel } from "@/components/ui/field";
+import {
+ Select,
+ SelectContent,
+ SelectGroup,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+
+import { projectsApi, type DomainCertificate, type ProjectHost } from "./projects.api";
+
+export function DomainCertificateControls({
+ host,
+ projectId,
+ canEdit,
+ onChange,
+}: {
+ host: ProjectHost;
+ projectId: string;
+ canEdit: boolean;
+ onChange: () => void;
+}) {
+ const id = useId();
+ const certificate = host.certificate;
+ const verified = host.ownership_status === "verified";
+ const renew = useMutation({
+ mutationFn: () => projectsApi.renewHostCertificate(projectId, host.id),
+ onSuccess: onChange,
+ });
+ const configure = useMutation({
+ mutationFn: (input: {
+ challenge_method?: "http01" | "dns01";
+ certificate_auto_renew?: boolean;
+ certificate_issuer?: DomainCertificate["issuer"];
+ }) => projectsApi.updateHostCertificate(projectId, host.id, input),
+ onSuccess: onChange,
+ });
+ if (host.kind !== "custom") return Platform domain ;
+ return (
+
+
+ {certificate?.status ?? "Awaiting verification"}
+
+ {certificate?.expires_at && (
+
Expires {new Date(certificate.expires_at).toLocaleString()}
+ )}
+ {certificate?.error &&
{certificate.error}
}
+ {certificate?.retry_at && (
+
+ Next attempt {new Date(certificate.retry_at).toLocaleString()}
+
+ )}
+ {certificate && (
+
+ Certificate settings
+
+
+ Certificate validation
+ {canEdit ? (
+
+ configure.mutate({ challenge_method: value as "http01" | "dns01" })
+ }
+ >
+
+
+
+
+
+ HTTP validation
+ DNS delegation
+
+
+
+ ) : (
+
+ {certificate.challenge_method === "dns01" ? "DNS delegation" : "HTTP validation"}
+
+ )}
+
+ {certificate.challenge_method === "http01"
+ ? "The domain must point at the regional entry and accept HTTP traffic on port 80."
+ : "Publish the validation CNAME below. The platform manages challenge TXT records in its own zone."}
+
+
+ {certificate.challenge_method === "dns01" &&
+ certificate.dns_delegation_name &&
+ certificate.dns_delegation_target && (
+
+ CNAME {certificate.dns_delegation_name} → {certificate.dns_delegation_target}
+
+ )}
+
+ Certificate authority: {certificate.issuer}
+
+ {canEdit && certificate.issuer !== "manual" && (
+
+
+ configure.mutate({ certificate_auto_renew: value === true })
+ }
+ />
+ Automatic renewal
+
+ )}
+
+
+ )}
+ {canEdit && verified && (
+
+ {certificate?.issuer === "manual" &&
+ certificate.regional_issuer &&
+ certificate.regional_issuer !== "manual" && (
+
+ configure.mutate({
+ certificate_issuer: certificate.regional_issuer,
+ certificate_auto_renew: true,
+ })
+ }
+ >
+ Use managed certificate
+
+ )}
+ {certificate?.issuer !== "manual" && (
+ renew.mutate()}
+ >
+ Renew certificate
+
+ )}
+ projectsApi.importHostCertificate(projectId, host.id, input)}
+ onImported={onChange}
+ />
+
+ )}
+
+ );
+}
diff --git a/apps/console/src/features/projects/project-domains-route.test.tsx b/apps/console/src/features/projects/project-domains-route.test.tsx
index 6c77f59..8b99fe4 100644
--- a/apps/console/src/features/projects/project-domains-route.test.tsx
+++ b/apps/console/src/features/projects/project-domains-route.test.tsx
@@ -1,5 +1,6 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
import { beforeEach, expect, it, vi } from "vite-plus/test";
import { ProjectDomainsRoute } from "./project-domains-route";
@@ -11,10 +12,72 @@ vi.mock("./projects.api", async (importOriginal) => {
const actual = await importOriginal();
return {
...actual,
- projectsApi: { ...actual.projectsApi, listHosts: vi.fn() },
+ projectsApi: {
+ ...actual.projectsApi,
+ listHosts: vi.fn(),
+ verifyHost: vi.fn(),
+ renewHostCertificate: vi.fn(),
+ updateHostCertificate: vi.fn(),
+ },
};
});
+it("restores managed renewal after a custom certificate import", async () => {
+ vi.mocked(useProject).mockReturnValue({
+ role: "owner",
+ project: { id: "project-1", name: "Website" },
+ } as ReturnType);
+ const listed = await projectsApi.listHosts("project-1");
+ const certificate = {
+ status: "active" as const,
+ issuer: "manual" as const,
+ regional_issuer: "letsencrypt" as const,
+ challenge_method: "http01" as const,
+ auto_renew: false,
+ issued_at: null,
+ expires_at: "2026-12-01T00:00:00Z",
+ error: null,
+ retry_at: null,
+ revision: "manual-1",
+ dns_delegation_name: null,
+ dns_delegation_target: null,
+ };
+ vi.mocked(projectsApi.listHosts).mockResolvedValue({
+ hosts: [{ ...listed.hosts[0], ownership_status: "verified", certificate }],
+ });
+ vi.mocked(projectsApi.updateHostCertificate).mockResolvedValue({ certificate });
+ const client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
+ const user = userEvent.setup();
+ render(
+
+
+ ,
+ );
+ await user.click(await screen.findByRole("button", { name: "Use managed certificate" }));
+ expect(projectsApi.updateHostCertificate).toHaveBeenCalledWith("project-1", "host-1", {
+ certificate_issuer: "letsencrypt",
+ certificate_auto_renew: true,
+ });
+});
+
+it("verifies custom-domain ownership through the domain action", async () => {
+ vi.mocked(useProject).mockReturnValue({
+ role: "owner",
+ project: { id: "project-1", name: "Website" },
+ } as ReturnType);
+ const listed = await projectsApi.listHosts("project-1");
+ vi.mocked(projectsApi.verifyHost).mockResolvedValue({ host: listed.hosts[0], verified: true });
+ const client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
+ const user = userEvent.setup();
+ render(
+
+
+ ,
+ );
+ await user.click(await screen.findByRole("button", { name: "Verify domain" }));
+ expect(projectsApi.verifyHost).toHaveBeenCalledWith("project-1", "host-1");
+});
+
beforeEach(() => {
vi.mocked(useProject).mockReturnValue({
role: "viewer",
@@ -26,6 +89,7 @@ beforeEach(() => {
id: "host-1",
project_id: "project-1",
host: "www.example.com",
+ region: "default",
kind: "custom",
environment: "production",
status: "failed",
diff --git a/apps/console/src/features/projects/project-domains-route.tsx b/apps/console/src/features/projects/project-domains-route.tsx
index 895bb76..5207f60 100644
--- a/apps/console/src/features/projects/project-domains-route.tsx
+++ b/apps/console/src/features/projects/project-domains-route.tsx
@@ -4,7 +4,7 @@ import { useState } from "react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
-import { Field, FieldLabel } from "@/components/ui/field";
+import { Field, FieldGroup, FieldLabel } from "@/components/ui/field";
import { Input } from "@/components/ui/input";
import { Skeleton } from "@/components/ui/skeleton";
import {
@@ -19,6 +19,7 @@ import { canContributeToProjects } from "@/features/teams/team-permissions";
import { projectsApi, type HostStatus } from "./projects.api";
import { useProject } from "./project-layout";
+import { DomainCertificateControls } from "./domain-certificate-controls";
export function hostStatusVariant(
status: HostStatus,
@@ -41,19 +42,22 @@ export function ProjectDomainsRoute() {
const canEdit = canContributeToProjects(role);
const queryClient = useQueryClient();
const [newHost, setNewHost] = useState("");
+ const [newRegion, setNewRegion] = useState("default");
const hostsQuery = useQuery({
queryKey: ["project-hosts", projectId],
queryFn: () => projectsApi.listHosts(projectId),
+ refetchInterval: 10_000,
});
const invalidate = () =>
queryClient.invalidateQueries({ queryKey: ["project-hosts", projectId] });
const addMutation = useMutation({
- mutationFn: () => projectsApi.createHost(projectId, { host: newHost }),
+ mutationFn: () => projectsApi.createHost(projectId, { host: newHost, region: newRegion }),
onSuccess: () => {
setNewHost("");
+ setNewRegion("default");
invalidate();
},
});
@@ -69,36 +73,50 @@ export function ProjectDomainsRoute() {
mutationFn: (hostId: string) => projectsApi.provisionHost(projectId, hostId),
onSuccess: invalidate,
});
+ const verifyMutation = useMutation({
+ mutationFn: (hostId: string) => projectsApi.verifyHost(projectId, hostId),
+ onSuccess: invalidate,
+ });
return (
-
+
Domains
- Platform domains are assigned automatically; custom domains must point at the platform
- nodes.
+ Point custom domains at their regional entry, verify ownership, and manage HTTPS.
{canEdit && (
{
event.preventDefault();
if (newHost.trim()) addMutation.mutate();
}}
>
-
- Add domain
- setNewHost(event.target.value)}
- />
-
+
+
+ Add domain
+ setNewHost(event.target.value)}
+ />
+
+
+ Region
+ setNewRegion(event.target.value)}
+ />
+
+
- Add
+ Add
)}
@@ -116,8 +134,10 @@ export function ProjectDomainsRoute() {
Domain
Kind
+ Region
Environment
Status
+ HTTPS
Serving
{canEdit && Actions }
@@ -135,12 +155,55 @@ export function ProjectDomainsRoute() {
{host.failure_reason && (
{host.failure_reason}
)}
+ {host.kind === "custom" && (
+
+
+ {host.ownership_status === "verified"
+ ? "Ownership verified"
+ : "Ownership verification required"}
+
+ {host.ownership_error && (
+
{host.ownership_error}
+ )}
+
+ )}
+ {host.ingress && (
+
+ DNS configuration
+
+
+ CNAME {host.ingress.cname.name} → {host.ingress.cname.target}
+
+
+ TXT {host.ingress.txt.name} → {host.ingress.txt.value}
+
+
+
+ TLS {host.ingress.certificate.status} · DNS-01{" "}
+ {host.ingress.dns_challenge.status} · {host.ingress.entrance_nodes.length}{" "}
+ healthy entrance node{host.ingress.entrance_nodes.length === 1 ? "" : "s"}
+
+
+ )}
{host.kind}
+
{host.region ?? "default"}
{host.environment}
{host.status}
+
+ {
+ void invalidate();
+ }}
+ />
+
{host.serving
@@ -151,37 +214,49 @@ export function ProjectDomainsRoute() {
{canEdit && (
-
- {(host.status === "pending" || host.status === "failed") &&
- host.host_source_id && (
+
+
+ {host.kind === "custom" && (
+ verifyMutation.mutate(host.id)}
+ >
+ Verify domain
+
+ )}
+ {(host.status === "pending" || host.status === "failed") &&
+ host.host_source_id && (
+ provisionMutation.mutate(host.id)}
+ disabled={provisionMutation.isPending}
+ >
+ Retry
+
+ )}
+ {!host.is_primary && (
provisionMutation.mutate(host.id)}
- disabled={provisionMutation.isPending}
+ onClick={() => primaryMutation.mutate(host.id)}
+ disabled={primaryMutation.isPending}
>
- Retry
+ Make primary
)}
- {!host.is_primary && (
primaryMutation.mutate(host.id)}
- disabled={primaryMutation.isPending}
+ variant="ghost"
+ aria-label={`Remove ${host.host}`}
+ onClick={() => removeMutation.mutate(host.id)}
+ disabled={removeMutation.isPending}
>
- Make primary
+
- )}
- removeMutation.mutate(host.id)}
- disabled={removeMutation.isPending}
- >
-
-
+
)}
diff --git a/apps/console/src/features/projects/projects.api.ts b/apps/console/src/features/projects/projects.api.ts
index 853fa8b..f97b336 100644
--- a/apps/console/src/features/projects/projects.api.ts
+++ b/apps/console/src/features/projects/projects.api.ts
@@ -43,6 +43,7 @@ export interface ProjectHost {
id: string;
project_id: string;
host: string;
+ region?: string;
kind: "platform" | "custom";
environment: HostEnvironment;
status: HostStatus;
@@ -52,6 +53,49 @@ export interface ProjectHost {
serving?: boolean;
created_at: string;
provision_events?: ProvisionEvent[];
+ ingress?: ProjectHostIngress | null;
+ ownership_status?: "pending" | "verified" | "failed" | "not_required";
+ ownership_checked_at?: string | null;
+ ownership_error?: string | null;
+ certificate?: DomainCertificate | null;
+}
+
+export interface DomainCertificate {
+ status: "pending" | "issuing" | "active" | "expiring" | "failed" | "disabled";
+ issuer: "letsencrypt" | "zerossl" | "manual";
+ regional_issuer: "letsencrypt" | "zerossl" | "manual";
+ challenge_method: "http01" | "dns01";
+ auto_renew: boolean;
+ issued_at: string | null;
+ expires_at: string | null;
+ error: string | null;
+ retry_at: string | null;
+ revision: string | null;
+ dns_delegation_name: string | null;
+ dns_delegation_target: string | null;
+}
+
+export interface ProjectHostIngress {
+ region: string;
+ cname: { record_type: "CNAME"; name: string; target: string };
+ txt: { record_type: "TXT"; name: string; value: string };
+ origin_host_preservation: boolean;
+ health_check: { path: string; interval_seconds: number };
+ entrance_nodes: Array<{ node_id: string; base_url: string; priority: number }>;
+ certificate: {
+ enabled: boolean;
+ issuer: "letsencrypt" | "zerossl" | "manual";
+ auto_renew: boolean;
+ status: "pending" | "issuing" | "active" | "expiring" | "failed" | "disabled";
+ expires_at: string | null;
+ error: string | null;
+ };
+ dns_challenge: {
+ provider: string | null;
+ status: "not_configured" | "pending" | "valid" | "failed";
+ record_name: string | null;
+ record_value: string | null;
+ };
}
export interface CreateProjectInput {
@@ -138,7 +182,10 @@ export const projectsApi = {
listHosts: (projectId: string) =>
request<{ hosts: ProjectHost[] }>(`/api/v1/projects/${projectId}/hosts`),
- createHost: (projectId: string, input: { host: string; environment?: HostEnvironment }) =>
+ createHost: (
+ projectId: string,
+ input: { host: string; region?: string; environment?: HostEnvironment },
+ ) =>
request<{ host: ProjectHost }>(`/api/v1/projects/${projectId}/hosts`, {
method: "POST",
body: JSON.stringify(input),
@@ -156,4 +203,40 @@ export const projectsApi = {
request<{ host: ProjectHost }>(`/api/v1/projects/${projectId}/hosts/${hostId}/provision`, {
method: "POST",
}),
+
+ verifyHost: (projectId: string, hostId: string) =>
+ request<{ host: ProjectHost; verified: boolean }>(
+ `/api/v1/projects/${projectId}/hosts/${hostId}/verify`,
+ { method: "POST" },
+ ),
+
+ updateHostCertificate: (
+ projectId: string,
+ hostId: string,
+ input: {
+ challenge_method?: "http01" | "dns01";
+ certificate_auto_renew?: boolean;
+ certificate_issuer?: DomainCertificate["issuer"];
+ },
+ ) =>
+ request<{ certificate: DomainCertificate }>(
+ `/api/v1/projects/${projectId}/hosts/${hostId}/certificate`,
+ { method: "PATCH", body: JSON.stringify(input) },
+ ),
+
+ renewHostCertificate: (projectId: string, hostId: string) =>
+ request<{ certificate: DomainCertificate }>(
+ `/api/v1/projects/${projectId}/hosts/${hostId}/certificate/renew`,
+ { method: "POST" },
+ ),
+
+ importHostCertificate: (
+ projectId: string,
+ hostId: string,
+ input: { certificate_pem: string; private_key_pem: string },
+ ) =>
+ request<{ certificate: DomainCertificate }>(
+ `/api/v1/projects/${projectId}/hosts/${hostId}/certificate/import`,
+ { method: "POST", body: JSON.stringify(input) },
+ ),
};
diff --git a/apps/console/src/router.tsx b/apps/console/src/router.tsx
index b61c612..9b76330 100644
--- a/apps/console/src/router.tsx
+++ b/apps/console/src/router.tsx
@@ -91,6 +91,11 @@ const HostSourcesPanel = lazy(() =>
default: HostSourcesPanel,
})),
);
+const RegionalIngressesPanel = lazy(() =>
+ import("@/features/admin/components/regional-ingresses-panel").then(
+ ({ RegionalIngressesPanel }) => ({ default: RegionalIngressesPanel }),
+ ),
+);
const QuotaPlansPanel = lazy(() =>
import("@/features/admin/components/quota-plans-panel").then(({ QuotaPlansPanel }) => ({
default: QuotaPlansPanel,
@@ -336,6 +341,7 @@ export function Router() {
} />
} />
} />
+ } />
} />
} />
} />
diff --git a/apps/control-api/Cargo.toml b/apps/control-api/Cargo.toml
index 5e749c3..8c48cc8 100644
--- a/apps/control-api/Cargo.toml
+++ b/apps/control-api/Cargo.toml
@@ -20,6 +20,7 @@ toml.workspace = true
grass-node-protocol.workspace = true
hex.workspace = true
image.workspace = true
+instant-acme = { version = "0.8.5", default-features = true, features = ["rcgen"] }
lettre.workspace = true
jsonwebtoken.workspace = true
sha2.workspace = true
@@ -36,6 +37,8 @@ mime_guess.workspace = true
rand.workspace = true
reqwest.workspace = true
ring.workspace = true
+rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
+rustls-pemfile = "2"
sea-orm.workspace = true
sea-orm-migration.workspace = true
serde.workspace = true
@@ -51,9 +54,12 @@ tracing.workspace = true
tracing-subscriber.workspace = true
url.workspace = true
uuid.workspace = true
+x509-parser = "0.16"
webp.workspace = true
walkdir.workspace = true
[dev-dependencies]
+hyper-util = { version = "0.1", features = ["client-legacy", "http1", "tokio"] }
+rcgen = { version = "0.14", features = ["x509-parser"] }
sea-orm = { workspace = true, features = ["mock"] }
tower = { version = "0.5", features = ["util"] }
diff --git a/apps/control-api/src/domain/acme.rs b/apps/control-api/src/domain/acme.rs
new file mode 100644
index 0000000..af21074
--- /dev/null
+++ b/apps/control-api/src/domain/acme.rs
@@ -0,0 +1,827 @@
+//! Automatic issuance with persistent retry state and challenge publication barriers.
+
+use anyhow::{Context, ensure};
+use base64::{
+ Engine,
+ engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD},
+};
+use instant_acme::{
+ Account, AccountCredentials, AuthorizationStatus, ChallengeType, ExternalAccountKey,
+ Identifier, LetsEncrypt, NewAccount, NewOrder, OrderStatus, RetryPolicy, ZeroSsl,
+};
+use sea_orm::{
+ ColumnTrait, Condition, DatabaseConnection, EntityTrait, QueryFilter, QuerySelect, Set,
+ TransactionTrait, sea_query::Expr,
+};
+use serde_json::{Value, json};
+use time::{Duration, OffsetDateTime};
+use uuid::Uuid;
+
+use super::certificates::{self, ACCOUNT_KEY, BUNDLE_KEY, PemBundle};
+use crate::infra::{
+ database::entity::{
+ managed_certificate as cert, node_ingress_status, project_host_binding, regional_ingress,
+ },
+ host_provision::DnsProviderHostProvisioner,
+};
+
+const LEASE_SECONDS: i64 = 600;
+const ATTEMPT_SECONDS: u64 = 480;
+const CHALLENGE_SECONDS: i64 = 600;
+const RENEW_DAYS: i64 = 30;
+
+fn retry_delay(failures: i32) -> Duration {
+ Duration::seconds((300_i64.saturating_mul(1_i64 << failures.clamp(0, 8))).min(86_400))
+}
+
+fn due(item: &cert::Model, now: OffsetDateTime) -> bool {
+ item.issuer != "manual"
+ && !item.lease_until.is_some_and(|until| until > now)
+ && !item.retry_at.is_some_and(|until| until > now)
+ && (matches!(item.status.as_str(), "pending" | "issuing" | "failed")
+ || item.bundle.is_none()
+ || (item.auto_renew
+ && item
+ .expires_at
+ .is_none_or(|until| until - now <= Duration::days(RENEW_DAYS))))
+}
+
+/// Checks that the current configuration still owns this issuance.
+async fn current(db: &DatabaseConnection, item: &cert::Model) -> anyhow::Result {
+ cert::Entity::find_by_id(item.id)
+ .filter(cert::Column::Generation.eq(item.generation))
+ .one(db)
+ .await?
+ .context("certificate configuration changed during issuance")
+}
+
+pub(super) async fn save_current(
+ db: &DatabaseConnection,
+ item: &cert::Model,
+ active: cert::ActiveModel,
+) -> anyhow::Result {
+ Ok(cert::Entity::update(active)
+ .validate()?
+ .filter(cert::Column::Generation.eq(item.generation))
+ .exec(db)
+ .await?)
+}
+
+async fn account(
+ db: &DatabaseConnection,
+ item: &cert::Model,
+ config: &Value,
+ secret: &str,
+) -> anyhow::Result {
+ let _ = rustls::crypto::ring::default_provider().install_default();
+ if let Some(value) = &item.acme_account {
+ let credentials: AccountCredentials =
+ serde_json::from_value(certificates::decrypt(secret, item.id, ACCOUNT_KEY, value)?)?;
+ return Ok(Account::builder()?.from_credentials(credentials).await?);
+ }
+ let directory = match item.issuer.as_str() {
+ "letsencrypt" => LetsEncrypt::Production.url(),
+ "zerossl" => ZeroSsl::Production.url(),
+ _ => anyhow::bail!("unsupported certificate issuer"),
+ };
+ // Staging CA endpoints are an operator runtime choice, never tenant-controlled API input.
+ let directory =
+ std::env::var("GRASS_ACME_DIRECTORY_URL").unwrap_or_else(|_| directory.to_owned());
+ let contact = config
+ .get("contact_email")
+ .and_then(Value::as_str)
+ .map(|v| format!("mailto:{v}"));
+ let contacts = contact.iter().map(String::as_str).collect::>();
+ let eab = external_account_key(config)?;
+ ensure!(
+ item.issuer != "zerossl" || eab.is_some(),
+ "ZeroSSL requires eab_kid and eab_hmac_key"
+ );
+ let (account, credentials) = Account::builder()?
+ .create(
+ &NewAccount {
+ contact: &contacts,
+ terms_of_service_agreed: true,
+ only_return_existing: false,
+ },
+ directory,
+ eab.as_ref(),
+ )
+ .await?;
+ let mut active: cert::ActiveModel = current(db, item).await?.into();
+ active.acme_account = Set(Some(certificates::encrypt(
+ secret,
+ item.id,
+ ACCOUNT_KEY,
+ &serde_json::to_value(credentials)?,
+ )?));
+ save_current(db, item, active).await?;
+ Ok(account)
+}
+
+fn external_account_key(config: &Value) -> anyhow::Result> {
+ let Some(kid) = config.get("eab_kid").and_then(Value::as_str) else {
+ return Ok(None);
+ };
+ let hmac = config
+ .get("eab_hmac_key")
+ .and_then(Value::as_str)
+ .context("eab_hmac_key is required")?;
+ let bytes = URL_SAFE_NO_PAD
+ .decode(hmac)
+ .or_else(|_| STANDARD.decode(hmac))
+ .map_err(|_| anyhow::anyhow!("eab_hmac_key must be base64 encoded"))?;
+ Ok(Some(ExternalAccountKey::new(kid.to_owned(), &bytes)))
+}
+
+async fn wait_http_ack(
+ db: &DatabaseConnection,
+ item: &cert::Model,
+ ingress: ®ional_ingress::Model,
+ secret: &str,
+) -> anyhow::Result<()> {
+ let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(90);
+ loop {
+ let current = current(db, item).await?;
+ ensure!(
+ certificates::eligible(db, ¤t, ingress).await?,
+ "domain is no longer eligible for certificate issuance"
+ );
+ let snapshot = certificates::snapshot(db, &ingress.region, secret).await?;
+ let ready =
+ regional_challenge_acknowledged(db, &ingress.region, &snapshot.challenge_revision)
+ .await?;
+ if ready {
+ return Ok(());
+ }
+ ensure!(
+ tokio::time::Instant::now() < deadline,
+ "regional entry nodes did not acknowledge the HTTP challenge within 90 seconds"
+ );
+ tokio::time::sleep(std::time::Duration::from_secs(2)).await;
+ }
+}
+
+async fn regional_challenge_acknowledged(
+ db: &DatabaseConnection,
+ region: &str,
+ revision: &str,
+) -> anyhow::Result {
+ let now = OffsetDateTime::now_utc();
+ let nodes = super::ingress::healthy_serve_nodes(db, region, now).await?;
+ if nodes.is_empty() {
+ return Ok(false);
+ }
+ for node in nodes {
+ let status = node_ingress_status::Entity::find_by_id(Uuid::parse_str(&node.node_id)?)
+ .one(db)
+ .await?;
+ if !status.is_some_and(|s| {
+ s.challenge_revision == revision && now - s.checked_at < Duration::seconds(30)
+ }) {
+ return Ok(false);
+ }
+ }
+ Ok(true)
+}
+
+async fn wait_dns(name: &str, value: &str) -> anyhow::Result<()> {
+ let client = reqwest::Client::builder()
+ .timeout(std::time::Duration::from_secs(10))
+ .build()?;
+ let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(120);
+ loop {
+ if super::ingress::verify_dns_record_at(
+ &client,
+ "https://cloudflare-dns.com/dns-query",
+ name,
+ "TXT",
+ value,
+ )
+ .await?
+ == super::ingress::DnsVerification::Verified
+ {
+ return Ok(());
+ }
+ ensure!(
+ tokio::time::Instant::now() < deadline,
+ "DNS challenge has not propagated within 120 seconds"
+ );
+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ }
+}
+
+async fn issue(
+ db: &DatabaseConnection,
+ item: &cert::Model,
+ ingress: ®ional_ingress::Model,
+ config: &Value,
+ secret: &str,
+) -> anyhow::Result {
+ let account = account(db, item, config, secret).await?;
+ issue_order(db, item, ingress, config, secret, &account).await
+}
+
+async fn issue_order(
+ db: &DatabaseConnection,
+ item: &cert::Model,
+ ingress: ®ional_ingress::Model,
+ config: &Value,
+ secret: &str,
+ account: &Account,
+) -> anyhow::Result {
+ let identifiers = [Identifier::Dns(item.hostname.clone())];
+ let mut order = account.new_order(&NewOrder::new(&identifiers)).await?;
+ {
+ let mut authorizations = order.authorizations();
+ while let Some(result) = authorizations.next().await {
+ let mut authorization = result?;
+ if authorization.status == AuthorizationStatus::Valid {
+ continue;
+ }
+ let challenge_type = if item.challenge_method == "http01" {
+ ChallengeType::Http01
+ } else {
+ ChallengeType::Dns01
+ };
+ let mut challenge = authorization
+ .challenge(challenge_type)
+ .context("ACME order does not offer the configured challenge method")?;
+ let mut active: cert::ActiveModel = current(db, item).await?.into();
+ if item.challenge_method == "http01" {
+ active.challenge_token = Set(Some(challenge.token.clone()));
+ active.challenge_value =
+ Set(Some(challenge.key_authorization().as_str().to_owned()));
+ active.challenge_expires_at = Set(Some(
+ OffsetDateTime::now_utc() + Duration::seconds(CHALLENGE_SECONDS),
+ ));
+ save_current(db, item, active).await?;
+ wait_http_ack(db, item, ingress, secret).await?;
+ } else {
+ let provider = ingress
+ .dns_challenge_provider
+ .as_deref()
+ .context("DNS challenge provider is not configured")?;
+ let zone = config
+ .get("zone")
+ .or_else(|| config.get("domain"))
+ .and_then(Value::as_str)
+ .unwrap_or(&ingress.hostname);
+ let source_name = format!("_acme-challenge.{}", item.hostname);
+ let name = if item.host_binding_id.is_some() {
+ let name = certificates::delegation_target(item, ingress);
+ let client = reqwest::Client::builder()
+ .timeout(std::time::Duration::from_secs(10))
+ .build()?;
+ ensure!(
+ super::ingress::verify_dns_record_at(
+ &client,
+ "https://cloudflare-dns.com/dns-query",
+ &source_name,
+ "CNAME",
+ &name
+ )
+ .await?
+ == super::ingress::DnsVerification::Verified,
+ "custom domain must delegate its ACME CNAME to the displayed regional challenge target"
+ );
+ name
+ } else {
+ source_name.clone()
+ };
+ let value = challenge.key_authorization().dns_value();
+ // Persist before creating DNS so crash recovery can clean this exact TXT value.
+ active.dns_record_name = Set(Some(name.clone()));
+ active.dns_record_value = Set(Some(value.clone()));
+ active.dns_cleanup = Set(Some(certificates::encrypt(
+ secret,
+ item.id,
+ "dns-cleanup-v1",
+ &json!({"provider":provider,"config":config,"zone":zone}),
+ )?));
+ save_current(db, item, active).await?;
+ DnsProviderHostProvisioner::new()
+ .ensure_txt_record(provider, config, zone, &name, &value)
+ .await
+ .map_err(|_| {
+ anyhow::anyhow!("DNS provider could not create the challenge record")
+ })?;
+ wait_dns(&source_name, &value).await?;
+ }
+ let fresh = current(db, item).await?;
+ ensure!(
+ certificates::eligible(db, &fresh, ingress).await?,
+ "domain is no longer eligible for issuance"
+ );
+ challenge.set_ready().await?;
+ }
+ }
+ let policy = RetryPolicy::new()
+ .initial_delay(std::time::Duration::from_secs(2))
+ .timeout(std::time::Duration::from_secs(120));
+ ensure!(
+ order.poll_ready(&policy).await? == OrderStatus::Ready,
+ "ACME order did not become ready"
+ );
+ let private_key_pem = order.finalize().await?;
+ let certificate_pem = order.poll_certificate(&policy).await?;
+ Ok(PemBundle {
+ certificate_pem,
+ private_key_pem,
+ })
+}
+
+async fn cleanup(db: &DatabaseConnection, item: &cert::Model, secret: &str) -> anyhow::Result<()> {
+ let fresh = current(db, item).await?;
+ let mut dns_clean = true;
+ if let (Some(name), Some(value), Some(cleanup)) = (
+ &fresh.dns_record_name,
+ &fresh.dns_record_value,
+ &fresh.dns_cleanup,
+ ) {
+ let cleanup = certificates::decrypt(secret, item.id, "dns-cleanup-v1", cleanup)?;
+ let provider = cleanup["provider"]
+ .as_str()
+ .context("stored cleanup provider missing")?;
+ let zone = cleanup["zone"]
+ .as_str()
+ .context("stored cleanup zone missing")?;
+ dns_clean = DnsProviderHostProvisioner::new()
+ .remove_txt_record(provider, &cleanup["config"], zone, name, value)
+ .await
+ .is_ok();
+ }
+ let mut active: cert::ActiveModel = fresh.into();
+ active.challenge_token = Set(None);
+ active.challenge_value = Set(None);
+ active.challenge_expires_at = Set(None);
+ if dns_clean {
+ active.dns_record_name = Set(None);
+ active.dns_record_value = Set(None);
+ active.dns_cleanup = Set(None);
+ }
+ save_current(db, item, active).await?;
+ ensure!(dns_clean, "DNS challenge cleanup will be retried");
+ Ok(())
+}
+
+async fn reconcile_record(
+ db: &DatabaseConnection,
+ item: &cert::Model,
+ ingress: ®ional_ingress::Model,
+ secret: &str,
+) -> anyhow::Result<()> {
+ let transaction = db.begin().await?;
+ let ingress = regional_ingress::Entity::find_by_id(ingress.id)
+ .lock_exclusive()
+ .one(&transaction)
+ .await?
+ .context("regional ingress no longer exists")?;
+ let now = OffsetDateTime::now_utc();
+ if !due(item, now) || !certificates::eligible(&transaction, item, &ingress).await? {
+ return Ok(());
+ }
+ let lock = cert::Entity::update_many()
+ .col_expr(
+ cert::Column::LeaseUntil,
+ Expr::value(now + Duration::seconds(LEASE_SECONDS)),
+ )
+ .col_expr(cert::Column::Status, Expr::value("issuing"))
+ .filter(cert::Column::Id.eq(item.id))
+ .filter(cert::Column::Generation.eq(item.generation))
+ .filter(
+ Condition::any()
+ .add(cert::Column::LeaseUntil.is_null())
+ .add(cert::Column::LeaseUntil.lte(now)),
+ )
+ .exec(&transaction)
+ .await?;
+ transaction.commit().await?;
+ if lock.rows_affected == 0 {
+ return Ok(());
+ }
+ let config = certificates::config(&ingress, secret)?;
+ let result = match tokio::time::timeout(
+ std::time::Duration::from_secs(ATTEMPT_SECONDS),
+ issue(db, item, &ingress, &config, secret),
+ )
+ .await
+ {
+ Ok(result) => result,
+ Err(_) => Err(anyhow::anyhow!("certificate issuance timed out")),
+ };
+ let cleanup_result = cleanup(db, item, secret).await;
+ let fresh = current(db, item).await?;
+ let result = result.and_then(|bundle| {
+ certificates::validate_pem(&item.hostname, &bundle, OffsetDateTime::now_utc())
+ .map(|validity| (bundle, validity))
+ });
+ let mut active: cert::ActiveModel = fresh.clone().into();
+ active.lease_until = Set(None);
+ active.updated_at = Set(OffsetDateTime::now_utc());
+ match result {
+ Ok((bundle, validity)) => {
+ ensure!(
+ certificates::eligible(db, &fresh, &ingress).await?,
+ "domain became ineligible during issuance"
+ );
+ active.bundle = Set(Some(certificates::encrypt(
+ secret,
+ item.id,
+ BUNDLE_KEY,
+ &serde_json::to_value(bundle)?,
+ )?));
+ active.revision = Set(validity.revision);
+ active.issued_at = Set(Some(validity.issued_at));
+ active.expires_at = Set(Some(validity.expires_at));
+ active.status = Set("active".to_owned());
+ active.error = Set(cleanup_result
+ .err()
+ .map(|_| "DNS cleanup is pending; it will be retried".to_owned()));
+ active.retry_at = Set(None);
+ active.failure_count = Set(0);
+ }
+ Err(_error) => {
+ // ACME/provider errors may contain challenge/account material; expose bounded diagnostics.
+ active.status = Set("failed".to_owned());
+ active.error=Set(Some("Certificate issuance failed; check DNS delegation, regional entry acknowledgements and ACME account configuration. Automatic retry is scheduled.".to_owned()));
+ active.retry_at = Set(Some(
+ OffsetDateTime::now_utc() + retry_delay(item.failure_count),
+ ));
+ active.failure_count = Set(item.failure_count.saturating_add(1));
+ }
+ }
+ let updated = save_current(db, item, active).await?;
+ certificates::sync_regional_status(db, &updated).await?;
+ Ok(())
+}
+
+pub async fn sweep(db: &DatabaseConnection, secret: &str) -> anyhow::Result<()> {
+ let ingresses = regional_ingress::Entity::find().all(db).await?;
+ for ingress in ingresses {
+ if sweep_ingress(db, ingress.id, secret).await.is_err() {
+ tracing::warn!(operation="control_api.acme.ingress_sweep_failed",ingress_id=%ingress.id,"regional certificate sweep failed; other regions will continue");
+ }
+ }
+ Ok(())
+}
+
+async fn sweep_ingress(
+ db: &DatabaseConnection,
+ ingress_id: Uuid,
+ secret: &str,
+) -> anyhow::Result<()> {
+ let Some(ingress) = certificates::prepare_regional_sweep(db, ingress_id, secret).await? else {
+ return Ok(());
+ };
+ if ingress.enabled && ingress.tls_enabled && ingress.deleted_at.is_none() {
+ let bindings = project_host_binding::Entity::find()
+ .filter(project_host_binding::Column::Region.eq(&ingress.region))
+ .filter(project_host_binding::Column::DeletedAt.is_null())
+ .all(db)
+ .await?;
+ for binding in bindings
+ .iter()
+ .filter(|b| certificates::binding_eligible(b))
+ {
+ certificates::ensure_record(db, &ingress, Some(binding)).await?;
+ }
+ }
+ let items = cert::Entity::find()
+ .filter(cert::Column::IngressId.eq(ingress.id))
+ .all(db)
+ .await?;
+ for item in items {
+ if !item
+ .lease_until
+ .is_some_and(|until| until > OffsetDateTime::now_utc())
+ {
+ // Recover challenges after process interruption or domain/ingress removal.
+ if (item.dns_record_name.is_some() || item.challenge_token.is_some())
+ && cleanup(db, &item, secret).await.is_err()
+ {
+ continue;
+ }
+ }
+ if let Err(_error) = reconcile_record(db, &item, &ingress, secret).await {
+ tracing::warn!(operation="control_api.acme.reconcile_failed",certificate_id=%item.id,"certificate reconciliation failed; state retained for retry");
+ }
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[tokio::test]
+ async fn malformed_legacy_bundle_does_not_block_other_regions() {
+ let mut first = certificates::tests::ingress_fixture();
+ first.certificate_issuer = "manual".to_owned();
+ first.certificate_auto_renew = false;
+ first.dns_challenge_config =
+ certificates::seal_config(first.id, &json!({}), "secret").unwrap();
+ first.certificate_bundle = Some(json!({"invalid_old_envelope":true}));
+ let mut second = certificates::tests::ingress_fixture();
+ second.region = "us".to_owned();
+ second.hostname = "us.example.org".to_owned();
+ second.certificate_issuer = "manual".to_owned();
+ second.certificate_auto_renew = false;
+ second.dns_challenge_config =
+ certificates::seal_config(second.id, &json!({}), "secret").unwrap();
+ let manual = |ingress: ®ional_ingress::Model| {
+ let mut item = certificates::tests::certificate_fixture(ingress);
+ item.issuer = "manual".to_owned();
+ item.auto_renew = false;
+ item
+ };
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![first.clone(), second.clone()], vec![first.clone()]])
+ .append_query_results([vec![manual(&first)]])
+ .append_query_results([Vec::::new()])
+ .append_query_results([Vec::::new()])
+ .append_query_results([vec![second.clone()]])
+ .append_query_results([vec![manual(&second)]])
+ .append_query_results([Vec::::new()])
+ .append_query_results([Vec::::new()])
+ .into_connection();
+ sweep(&db, "secret").await.unwrap();
+ let log = db.into_transaction_log();
+ assert_eq!(
+ log.iter()
+ .flat_map(|t| t.statements())
+ .filter(|s| s.sql.contains("FROM \"project_host_bindings\""))
+ .count(),
+ 2
+ );
+ assert!(
+ !log.iter()
+ .flat_map(|t| t.statements())
+ .any(|s| s.sql.starts_with("UPDATE "))
+ );
+ }
+
+ #[tokio::test]
+ async fn malformed_legacy_provider_config_is_isolated_to_its_region() {
+ let mut broken = certificates::tests::ingress_fixture();
+ broken.dns_challenge_config = json!(false);
+ let mut healthy = certificates::tests::ingress_fixture();
+ healthy.enabled = false;
+ healthy.dns_challenge_config =
+ certificates::seal_config(healthy.id, &json!({}), "secret").unwrap();
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([
+ vec![broken.clone(), healthy.clone()],
+ vec![broken],
+ vec![healthy.clone()],
+ ])
+ .append_query_results([Vec::::new()])
+ .into_connection();
+ sweep(&db, "secret").await.unwrap();
+ let healthy_id: sea_orm::Value = healthy.id.into();
+ assert!(
+ db.into_transaction_log()
+ .iter()
+ .flat_map(|t| t.statements())
+ .any(|s| s.sql.contains("FROM \"managed_certificates\"")
+ && s.values
+ .as_ref()
+ .is_some_and(|values| values.0.contains(&healthy_id)))
+ );
+ }
+ #[tokio::test]
+ async fn http_validation_waits_for_every_eligible_entry_revision() {
+ use crate::infra::database::entity::regional_ingress_health;
+ let ingress = certificates::tests::ingress_fixture();
+ let first = super::super::ingress::tests::node_fixture();
+ let second = super::super::ingress::tests::node_fixture();
+ let now = OffsetDateTime::now_utc();
+ let revision = "a".repeat(64);
+ for second_revision in ["b".repeat(64), revision.clone()] {
+ let nodes = vec![first.clone(), second.clone()];
+ let statuses = vec![
+ node_ingress_status::Model {
+ node_id: first.id,
+ certificates: json!([]),
+ challenge_revision: revision.clone(),
+ tls_ready: true,
+ checked_at: now,
+ },
+ node_ingress_status::Model {
+ node_id: second.id,
+ certificates: json!([]),
+ challenge_revision: second_revision.clone(),
+ tls_ready: true,
+ checked_at: now,
+ },
+ ];
+ let health = nodes
+ .iter()
+ .map(|n| regional_ingress_health::Model {
+ ingress_id: ingress.id,
+ node_id: n.id,
+ status: "healthy".to_owned(),
+ checked_at: Some(now),
+ latency_ms: Some(1),
+ error: None,
+ })
+ .collect::>();
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![ingress.clone()]])
+ .append_query_results([health])
+ .append_query_results([nodes])
+ .append_query_results([
+ statuses.clone(),
+ vec![statuses[0].clone()],
+ vec![statuses[1].clone()],
+ ])
+ .into_connection();
+ assert_eq!(
+ regional_challenge_acknowledged(&db, "eu", &revision)
+ .await
+ .unwrap(),
+ second_revision == revision
+ );
+ }
+ }
+ #[derive(Clone)]
+ struct MockCa {
+ origin: String,
+ certificate: std::sync::Arc>>,
+ accounts: std::sync::Arc,
+ issued: std::sync::Arc,
+ }
+
+ async fn mock_ca(
+ axum::extract::State(state): axum::extract::State,
+ uri: axum::http::Uri,
+ body: axum::body::Bytes,
+ ) -> axum::response::Response {
+ use axum::response::IntoResponse;
+ use std::sync::atomic::Ordering;
+ let payload = if body.is_empty() {
+ json!({})
+ } else {
+ let body: Value = serde_json::from_slice(&body).unwrap();
+ let encoded = body["payload"].as_str().unwrap();
+ if encoded.is_empty() {
+ json!({})
+ } else {
+ serde_json::from_slice(&URL_SAFE_NO_PAD.decode(encoded).unwrap()).unwrap()
+ }
+ };
+ let order = || json!({"status":if state.issued.load(Ordering::SeqCst){"valid"}else{"ready"},"authorizations":[format!("{}/authorization",state.origin)],"finalize":format!("{}/finalize",state.origin),"certificate":if state.issued.load(Ordering::SeqCst){Some(format!("{}/certificate",state.origin))}else{None}});
+ let mut response=match uri.path(){
+ "/directory"=>axum::Json(json!({"newNonce":format!("{}/nonce",state.origin),"newAccount":format!("{}/account",state.origin),"newOrder":format!("{}/new-order",state.origin)})).into_response(),
+ "/nonce"=>axum::http::StatusCode::OK.into_response(),
+ "/account"=>{state.accounts.fetch_add(1,Ordering::SeqCst);axum::Json(json!({"status":"valid"})).into_response()},
+ "/new-order"=>{assert_eq!(payload["identifiers"][0]["value"],"site.example.org");axum::Json(order()).into_response()},
+ "/authorization"=>axum::Json(json!({"status":"valid","identifier":{"type":"dns","value":"site.example.org"},"challenges":[]})).into_response(),
+ "/finalize"=>{
+ let der=URL_SAFE_NO_PAD.decode(payload["csr"].as_str().unwrap()).unwrap();
+ let mut csr=rcgen::CertificateSigningRequestParams::from_der(&der.into()).unwrap();
+ csr.params.not_before=OffsetDateTime::now_utc()-Duration::minutes(1);csr.params.not_after=OffsetDateTime::now_utc()+Duration::days(14);
+ let issuer=rcgen::Issuer::new(rcgen::CertificateParams::default(),rcgen::KeyPair::generate().unwrap());
+ *state.certificate.lock().unwrap()=Some(csr.signed_by(&issuer).unwrap().pem());state.issued.store(true,Ordering::SeqCst);axum::Json(order()).into_response()
+ },
+ "/order"=>axum::Json(order()).into_response(),
+ "/certificate"=>state.certificate.lock().unwrap().clone().unwrap().into_response(),
+ _=>axum::http::StatusCode::NOT_FOUND.into_response(),
+ };
+ response
+ .headers_mut()
+ .insert("replay-nonce", "bW9jay1ub25jZQ".parse().unwrap());
+ if uri.path() == "/account" {
+ response.headers_mut().insert(
+ "location",
+ format!("{}/account/1", state.origin).parse().unwrap(),
+ );
+ }
+ if uri.path() == "/new-order" {
+ response.headers_mut().insert(
+ "location",
+ format!("{}/order", state.origin).parse().unwrap(),
+ );
+ }
+ response
+ }
+
+ #[tokio::test]
+ async fn mock_acme_issues_a_key_matched_certificate_and_reuses_encrypted_account() {
+ let _ = rustls::crypto::ring::default_provider().install_default();
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let origin = format!("http://{}", listener.local_addr().unwrap());
+ let state = MockCa {
+ origin: origin.clone(),
+ certificate: Default::default(),
+ accounts: Default::default(),
+ issued: Default::default(),
+ };
+ let app = axum::Router::new()
+ .fallback(mock_ca)
+ .with_state(state.clone());
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ let client =
+ hyper_util::client::legacy::Client::builder(hyper_util::rt::TokioExecutor::new())
+ .build_http::>();
+ let (_, credentials) = Account::builder_with_http(Box::new(client.clone()))
+ .create(
+ &NewAccount {
+ contact: &[],
+ terms_of_service_agreed: true,
+ only_return_existing: false,
+ },
+ format!("{origin}/directory"),
+ None,
+ )
+ .await
+ .unwrap();
+ let ingress = certificates::tests::ingress_fixture();
+ let mut item = certificates::tests::certificate_fixture(&ingress);
+ item.hostname = "site.example.org".to_owned();
+ item.acme_account = Some(
+ certificates::encrypt(
+ "test-key",
+ item.id,
+ ACCOUNT_KEY,
+ &serde_json::to_value(credentials).unwrap(),
+ )
+ .unwrap(),
+ );
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres).into_connection();
+ let credentials = serde_json::from_value(
+ certificates::decrypt(
+ "test-key",
+ item.id,
+ ACCOUNT_KEY,
+ item.acme_account.as_ref().unwrap(),
+ )
+ .unwrap(),
+ )
+ .unwrap();
+ let account = Account::builder_with_http(Box::new(client))
+ .from_credentials(credentials)
+ .await
+ .unwrap();
+ let bundle = issue_order(&db, &item, &ingress, &json!({}), "test-key", &account)
+ .await
+ .unwrap();
+ let validity =
+ certificates::validate_pem(&item.hostname, &bundle, OffsetDateTime::now_utc()).unwrap();
+ assert!((validity.expires_at - OffsetDateTime::now_utc()).whole_days() <= 14);
+ assert_eq!(state.accounts.load(std::sync::atomic::Ordering::SeqCst), 1);
+ assert!(state.issued.load(std::sync::atomic::Ordering::SeqCst));
+ server.abort();
+ }
+ #[test]
+ fn renewal_respects_expiry_manual_mode_backoff_and_active_lease() {
+ let now = OffsetDateTime::now_utc();
+ let ingress = certificates::tests::ingress_fixture();
+ let mut item = certificates::tests::certificate_fixture(&ingress);
+ assert!(due(&item, now));
+ item.status = "active".to_owned();
+ item.bundle = Some(json!({}));
+ item.expires_at = Some(now + Duration::days(40));
+ assert!(!due(&item, now));
+ item.expires_at = Some(now + Duration::days(20));
+ assert!(due(&item, now));
+ item.auto_renew = false;
+ assert!(!due(&item, now));
+ item.status = "pending".to_owned();
+ assert!(due(&item, now));
+ item.retry_at = Some(now + Duration::seconds(1));
+ assert!(!due(&item, now));
+ item.retry_at = None;
+ item.lease_until = Some(now + Duration::seconds(1));
+ assert!(!due(&item, now));
+ item.lease_until = None;
+ item.issuer = "manual".to_owned();
+ assert!(!due(&item, now));
+ }
+ #[test]
+ fn interrupted_or_failed_forced_renewal_retries_with_auto_renew_disabled() {
+ let now = OffsetDateTime::now_utc();
+ let ingress = certificates::tests::ingress_fixture();
+ let mut item = certificates::tests::certificate_fixture(&ingress);
+ item.auto_renew = false;
+ item.bundle = Some(json!({}));
+ item.expires_at = Some(now + Duration::days(60));
+ item.status = "issuing".to_owned();
+ item.lease_until = Some(now - Duration::seconds(1));
+ assert!(due(&item, now));
+ item.status = "failed".to_owned();
+ item.retry_at = Some(now + Duration::seconds(1));
+ assert!(!due(&item, now));
+ assert!(due(&item, now + Duration::seconds(2)));
+ }
+ #[test]
+ fn retry_delay_is_bounded_and_zero_ssl_requires_complete_eab() {
+ assert_eq!(retry_delay(0), Duration::minutes(5));
+ assert_eq!(retry_delay(20), Duration::seconds(76_800));
+ assert!(external_account_key(&json!({"eab_kid":"id"})).is_err());
+ assert!(
+ external_account_key(&json!({"eab_kid":"id","eab_hmac_key":"c2VjcmV0"}))
+ .unwrap()
+ .is_some()
+ );
+ }
+}
diff --git a/apps/control-api/src/domain/certificates.rs b/apps/control-api/src/domain/certificates.rs
new file mode 100644
index 0000000..759340c
--- /dev/null
+++ b/apps/control-api/src/domain/certificates.rs
@@ -0,0 +1,933 @@
+//! Persistent certificate material, eligibility and authoritative regional snapshots.
+
+use anyhow::{Context, ensure};
+use grass_node_protocol::{CertificateBundle, CertificateBundlesResponse, HttpChallenge};
+use sea_orm::{
+ ActiveModelTrait, ColumnTrait, ConnectionTrait, DatabaseConnection, EntityTrait, QueryFilter,
+ QuerySelect, Set, TransactionTrait, sea_query::OnConflict,
+};
+use serde_json::{Value, json};
+use sha2::{Digest, Sha256};
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use super::authentication::authentication_key;
+use crate::infra::database::entity::{
+ HostBindingKind, HostBindingStatus, HostReviewStatus, managed_certificate as cert,
+ project_host_binding, regional_ingress,
+};
+
+pub const ACCOUNT_KEY: &str = "regional-ingress-acme-account-v1";
+pub const BUNDLE_KEY: &str = "regional-ingress-certificate-v1";
+const CONFIG_KEY: &str = "regional-ingress-provider-v1";
+
+pub fn encrypt(secret: &str, id: Uuid, key: &str, value: &Value) -> anyhow::Result {
+ Ok(serde_json::to_value(grass_crypto::encrypt_secret(
+ key,
+ &authentication_key(secret),
+ &serde_json::to_vec(value)?,
+ format!("grass-regional-ingress:{key}:{id}").as_bytes(),
+ )?)?)
+}
+
+pub fn decrypt(secret: &str, id: Uuid, key: &str, value: &Value) -> anyhow::Result {
+ let envelope: grass_crypto::AeadEnvelope = serde_json::from_value(value.clone())?;
+ ensure!(envelope.key_id == key, "unsupported stored secret key id");
+ let plaintext = grass_crypto::decrypt_secret(
+ &envelope,
+ &authentication_key(secret),
+ format!("grass-regional-ingress:{key}:{id}").as_bytes(),
+ )?;
+ Ok(serde_json::from_slice(&plaintext)?)
+}
+
+pub fn config(ingress: ®ional_ingress::Model, secret: &str) -> anyhow::Result {
+ if ingress.dns_challenge_config.get("key_id").is_some() {
+ decrypt(
+ secret,
+ ingress.id,
+ CONFIG_KEY,
+ &ingress.dns_challenge_config,
+ )
+ } else {
+ // Previous releases stored plaintext. The certificate sweep upgrades it in place.
+ Ok(ingress.dns_challenge_config.clone())
+ }
+}
+
+pub fn seal_config(id: Uuid, value: &Value, secret: &str) -> anyhow::Result {
+ ensure!(value.is_object(), "DNS configuration must be an object");
+ ensure!(
+ serde_json::to_vec(value)?.len() <= 16_384,
+ "DNS configuration exceeds 16 KiB"
+ );
+ encrypt(secret, id, CONFIG_KEY, value)
+}
+
+pub fn merge_config(mut previous: Value, patch: &Value) -> anyhow::Result {
+ let previous = previous
+ .as_object_mut()
+ .context("stored DNS configuration is invalid")?;
+ for (key, value) in patch
+ .as_object()
+ .context("DNS configuration must be an object")?
+ {
+ ensure!(key.len() <= 128, "DNS configuration key exceeds 128 bytes");
+ if value.is_null() {
+ previous.remove(key);
+ } else if value.as_str() != Some("") {
+ previous.insert(key.clone(), value.clone());
+ }
+ }
+ Ok(Value::Object(previous.clone()))
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+pub struct PemBundle {
+ pub certificate_pem: String,
+ pub private_key_pem: String,
+}
+
+pub struct CertificateValidity {
+ pub issued_at: OffsetDateTime,
+ pub expires_at: OffsetDateTime,
+ pub revision: String,
+}
+
+pub fn validate_pem(
+ hostname: &str,
+ bundle: &PemBundle,
+ now: OffsetDateTime,
+) -> anyhow::Result {
+ ensure!(
+ bundle.certificate_pem.len() <= 65_536 && bundle.private_key_pem.len() <= 16_384,
+ "certificate material exceeds size limit"
+ );
+ let chain = rustls_pemfile::certs(&mut bundle.certificate_pem.as_bytes())
+ .collect::, _>>()?;
+ let leaf = chain.first().context("certificate chain is empty")?;
+ let (_, certificate) = x509_parser::parse_x509_certificate(leaf.as_ref())
+ .map_err(|_| anyhow::anyhow!("invalid X.509 certificate"))?;
+ let mut issued_at =
+ OffsetDateTime::from_unix_timestamp(certificate.validity().not_before.timestamp())?;
+ let mut expires_at =
+ OffsetDateTime::from_unix_timestamp(certificate.validity().not_after.timestamp())?;
+ ensure!(
+ issued_at <= now && expires_at > now,
+ "certificate is not currently valid"
+ );
+ for entry in chain.iter().skip(1) {
+ let (_, certificate) = x509_parser::parse_x509_certificate(entry.as_ref())
+ .map_err(|_| anyhow::anyhow!("invalid certificate chain entry"))?;
+ let starts =
+ OffsetDateTime::from_unix_timestamp(certificate.validity().not_before.timestamp())?;
+ let ends =
+ OffsetDateTime::from_unix_timestamp(certificate.validity().not_after.timestamp())?;
+ ensure!(
+ starts <= now && ends > now,
+ "certificate chain is not currently valid"
+ );
+ issued_at = issued_at.max(starts);
+ expires_at = expires_at.min(ends);
+ }
+ let parsed = rustls::server::ParsedCertificate::try_from(leaf)?;
+ rustls::client::verify_server_name(
+ &parsed,
+ &rustls::pki_types::ServerName::try_from(hostname.to_owned())?,
+ )?;
+ let key = rustls_pemfile::private_key(&mut bundle.private_key_pem.as_bytes())?
+ .context("private key is missing")?;
+ rustls::sign::CertifiedKey::new(chain, rustls::crypto::ring::sign::any_supported_type(&key)?)
+ .keys_match()?;
+ Ok(CertificateValidity {
+ issued_at,
+ expires_at,
+ revision: hex::encode(Sha256::digest(bundle.certificate_pem.as_bytes())),
+ })
+}
+
+pub fn binding_eligible(binding: &project_host_binding::Model) -> bool {
+ binding.deleted_at.is_none()
+ && matches!(binding.kind, HostBindingKind::Custom)
+ && matches!(binding.status, HostBindingStatus::Active)
+ && binding.ownership_status == "verified"
+ && matches!(
+ binding.review_status,
+ HostReviewStatus::Approved | HostReviewStatus::NotRequired
+ )
+}
+
+pub async fn eligible(
+ db: &C,
+ item: &cert::Model,
+ ingress: ®ional_ingress::Model,
+) -> anyhow::Result {
+ let Some(ingress) = regional_ingress::Entity::find_by_id(ingress.id)
+ .one(db)
+ .await?
+ else {
+ return Ok(false);
+ };
+ if !ingress.enabled
+ || !ingress.tls_enabled
+ || ingress.deleted_at.is_some()
+ || item.ingress_id != ingress.id
+ {
+ return Ok(false);
+ }
+ if let Some(id) = item.host_binding_id {
+ Ok(project_host_binding::Entity::find_by_id(id)
+ .one(db)
+ .await?
+ .is_some_and(|binding| {
+ binding_eligible(&binding)
+ && binding.host == item.hostname
+ && binding.region == ingress.region
+ }))
+ } else {
+ Ok(item.hostname == ingress.hostname)
+ }
+}
+
+pub async fn ensure_record(
+ db: &DatabaseConnection,
+ ingress: ®ional_ingress::Model,
+ binding: Option<&project_host_binding::Model>,
+) -> anyhow::Result {
+ let transaction = db.begin().await?;
+ let ingress = regional_ingress::Entity::find_by_id(ingress.id)
+ .lock_exclusive()
+ .one(&transaction)
+ .await?
+ .context("regional ingress no longer exists")?;
+ let item = ensure_record_inner(&transaction, &ingress, binding).await?;
+ transaction.commit().await?;
+ Ok(item)
+}
+
+/// Upgrade local legacy state while holding the same ingress/certificate locks
+/// used by administrator edits. Only the ID comes from the sweep's earlier list.
+pub async fn prepare_regional_sweep(
+ db: &DatabaseConnection,
+ ingress_id: Uuid,
+ secret: &str,
+) -> anyhow::Result> {
+ let transaction = db.begin().await?;
+ let Some(mut ingress) = regional_ingress::Entity::find_by_id(ingress_id)
+ .lock_exclusive()
+ .one(&transaction)
+ .await?
+ else {
+ return Ok(None);
+ };
+ if ingress.dns_challenge_config.get("key_id").is_none() {
+ let sealed = seal_config(ingress.id, &ingress.dns_challenge_config, secret)?;
+ let mut active: regional_ingress::ActiveModel = ingress.into();
+ active.dns_challenge_config = Set(sealed);
+ ingress = active.update(&transaction).await?;
+ }
+ if ingress.enabled && ingress.tls_enabled && ingress.deleted_at.is_none() {
+ let regional = ensure_record_inner(&transaction, &ingress, None).await?;
+ // An in-flight order owns its status and material until it releases its lease.
+ if !regional
+ .lease_until
+ .is_some_and(|until| until > OffsetDateTime::now_utc())
+ {
+ let mut active: cert::ActiveModel = regional.clone().into();
+ let auto_renew = regional.issuer != "manual" && ingress.certificate_auto_renew;
+ let mut changed = regional.auto_renew != auto_renew;
+ if changed {
+ active.auto_renew = Set(auto_renew);
+ }
+ if regional.bundle.is_none()
+ && let Some(old) = &ingress.certificate_bundle
+ {
+ let restored = (|| -> anyhow::Result<_> {
+ let bundle: PemBundle =
+ serde_json::from_value(decrypt(secret, ingress.id, BUNDLE_KEY, old)?)?;
+ validate_pem(&ingress.hostname, &bundle, OffsetDateTime::now_utc())
+ })();
+ match restored {
+ Ok(validity) => {
+ active.bundle = Set(Some(old.clone()));
+ active.revision = Set(validity.revision);
+ active.issued_at = Set(Some(validity.issued_at));
+ active.expires_at = Set(Some(validity.expires_at));
+ active.status = Set("active".to_owned());
+ changed = true;
+ }
+ Err(_) => {
+ tracing::warn!(operation="control_api.certificate.invalid_legacy_bundle",ingress_id=%ingress.id,"legacy certificate could not be restored; current certificate state is retained");
+ }
+ }
+ }
+ if changed {
+ active.update(&transaction).await?;
+ }
+ }
+ }
+ transaction.commit().await?;
+ Ok(Some(ingress))
+}
+
+async fn ensure_record_inner(
+ db: &C,
+ ingress: ®ional_ingress::Model,
+ binding: Option<&project_host_binding::Model>,
+) -> anyhow::Result {
+ let id = binding.map_or(ingress.id, |b| b.id);
+ let hostname = binding.map_or(&ingress.hostname, |b| &b.host);
+ if let Some(item) = cert::Entity::find_by_id(id)
+ .lock_exclusive()
+ .one(db)
+ .await?
+ {
+ let changed = item.hostname != *hostname
+ || item.ingress_id != ingress.id
+ || (item.issuer != "manual" && item.issuer != ingress.certificate_issuer);
+ if !changed {
+ return Ok(item);
+ }
+ let mut active: cert::ActiveModel = item.clone().into();
+ active.hostname = Set(hostname.clone());
+ active.ingress_id = Set(ingress.id);
+ active.issuer = Set(ingress.certificate_issuer.clone());
+ active.acme_account = Set(None);
+ active.status = Set("pending".to_owned());
+ active.error = Set(None);
+ active.retry_at = Set(None);
+ active.failure_count = Set(0);
+ active.lease_until = Set(None);
+ active.generation = Set(Uuid::now_v7());
+ active.challenge_token = Set(None);
+ active.challenge_value = Set(None);
+ active.challenge_expires_at = Set(None);
+ if item.hostname != *hostname {
+ active.bundle = Set(None);
+ active.revision = Set(String::new());
+ active.expires_at = Set(None);
+ active.issued_at = Set(None);
+ }
+ return Ok(active.update(db).await?);
+ }
+ let active = cert::ActiveModel {
+ id: Set(id),
+ ingress_id: Set(ingress.id),
+ host_binding_id: Set(binding.map(|b| b.id)),
+ hostname: Set(hostname.clone()),
+ issuer: Set(ingress.certificate_issuer.clone()),
+ challenge_method: Set(if binding.is_some() { "http01" } else { "dns01" }.to_owned()),
+ auto_renew: Set(ingress.certificate_auto_renew),
+ generation: Set(Uuid::now_v7()),
+ ..Default::default()
+ };
+ cert::Entity::insert(active)
+ .on_conflict(OnConflict::column(cert::Column::Id).do_nothing().to_owned())
+ .try_insert()
+ .exec(db)
+ .await?;
+ cert::Entity::find_by_id(id)
+ .one(db)
+ .await?
+ .context("certificate record disappeared")
+}
+
+pub async fn queue(db: &DatabaseConnection, item: &cert::Model) -> anyhow::Result {
+ let transaction = db.begin().await?;
+ let item = cert::Entity::find_by_id(item.id)
+ .lock_exclusive()
+ .one(&transaction)
+ .await?
+ .context("certificate no longer exists")?;
+ ensure!(
+ item.issuer != "manual",
+ "manual certificates must be imported"
+ );
+ let now = OffsetDateTime::now_utc();
+ ensure!(
+ !item.lease_until.is_some_and(|until| until > now),
+ "certificate issuance is already in progress"
+ );
+ ensure!(
+ !item.retry_at.is_some_and(|until| until > now),
+ "certificate retry is temporarily delayed after failure"
+ );
+ let mut active: cert::ActiveModel = item.clone().into();
+ active.status = Set("pending".to_owned());
+ active.retry_at = Set(None);
+ active.error = Set(None);
+ active.generation = Set(Uuid::now_v7());
+ let item = active.update(&transaction).await?;
+ transaction.commit().await?;
+ Ok(item)
+}
+
+pub async fn import(
+ db: &DatabaseConnection,
+ item: &cert::Model,
+ bundle: PemBundle,
+ secret: &str,
+) -> anyhow::Result {
+ let transaction = db.begin().await?;
+ let item = cert::Entity::find_by_id(item.id)
+ .lock_exclusive()
+ .one(&transaction)
+ .await?
+ .context("certificate no longer exists")?;
+ ensure!(
+ !item
+ .lease_until
+ .is_some_and(|until| until > OffsetDateTime::now_utc()),
+ "wait for the in-progress certificate attempt before importing"
+ );
+ let validity = validate_pem(&item.hostname, &bundle, OffsetDateTime::now_utc())?;
+ let mut active: cert::ActiveModel = item.clone().into();
+ active.issuer = Set("manual".to_owned());
+ active.auto_renew = Set(false);
+ active.status = Set("active".to_owned());
+ active.error = Set(None);
+ active.bundle = Set(Some(encrypt(
+ secret,
+ item.id,
+ BUNDLE_KEY,
+ &serde_json::to_value(bundle)?,
+ )?));
+ active.issued_at = Set(Some(validity.issued_at));
+ active.expires_at = Set(Some(validity.expires_at));
+ active.revision = Set(validity.revision);
+ active.generation = Set(Uuid::now_v7());
+ active.lease_until = Set(None);
+ active.retry_at = Set(None);
+ active.failure_count = Set(0);
+ active.challenge_token = Set(None);
+ active.challenge_value = Set(None);
+ active.challenge_expires_at = Set(None);
+ let updated = active.update(&transaction).await?;
+ transaction.commit().await?;
+ sync_regional_status(db, &updated).await?;
+ Ok(updated)
+}
+
+pub async fn sync_regional_status(
+ db: &DatabaseConnection,
+ item: &cert::Model,
+) -> anyhow::Result<()> {
+ if item.host_binding_id.is_some() {
+ return Ok(());
+ }
+ if let Some(ingress) = regional_ingress::Entity::find_by_id(item.ingress_id)
+ .one(db)
+ .await?
+ {
+ if ingress.hostname != item.hostname {
+ return Ok(());
+ }
+ let mut active: regional_ingress::ActiveModel = ingress.into();
+ active.certificate_status = Set(item.status.clone());
+ active.certificate_error = Set(item.error.clone());
+ active.certificate_issued_at = Set(item.issued_at);
+ active.certificate_expires_at = Set(item.expires_at);
+ active.update(db).await?;
+ }
+ Ok(())
+}
+
+pub fn delegation_target(item: &cert::Model, ingress: ®ional_ingress::Model) -> String {
+ format!("_acme-{}.{}", item.id.simple(), ingress.hostname)
+}
+
+pub fn view(item: Option<&cert::Model>, ingress: ®ional_ingress::Model) -> Value {
+ let mut view = match item {
+ Some(item) => {
+ json!({ "enabled": ingress.enabled && ingress.tls_enabled, "status": item.status, "issuer": item.issuer, "challenge_method": item.challenge_method, "auto_renew": item.auto_renew, "issued_at": crate::infra::http::timestamps::ts(item.issued_at), "expires_at": crate::infra::http::timestamps::ts(item.expires_at), "error": item.error, "retry_at": crate::infra::http::timestamps::ts(item.retry_at), "revision": item.revision, "dns_delegation_name": format!("_acme-challenge.{}", item.hostname), "dns_delegation_target": delegation_target(item, ingress) })
+ }
+ None => {
+ json!({"enabled": ingress.enabled && ingress.tls_enabled, "status":"pending", "issuer":ingress.certificate_issuer, "challenge_method":"http01", "auto_renew":ingress.certificate_auto_renew, "issued_at":null, "expires_at":null, "error":null, "retry_at":null, "revision":"", "dns_delegation_name":null, "dns_delegation_target":null})
+ }
+ };
+ view["regional_issuer"] = json!(ingress.certificate_issuer);
+ if !ingress.enabled || !ingress.tls_enabled || ingress.deleted_at.is_some() {
+ view["status"] = json!("disabled");
+ }
+ view
+}
+
+pub fn challenge_revision(challenges: &[HttpChallenge]) -> String {
+ hex::encode(Sha256::digest(
+ serde_json::to_vec(challenges).expect("challenge snapshot serializes"),
+ ))
+}
+
+pub async fn snapshot(
+ db: &DatabaseConnection,
+ region: &str,
+ secret: &str,
+) -> anyhow::Result {
+ let mut bundles = Vec::new();
+ let mut challenges = Vec::new();
+ if let Some(ingress) = super::ingress::get_enabled_by_region(db, region).await? {
+ let items = cert::Entity::find()
+ .filter(cert::Column::IngressId.eq(ingress.id))
+ .all(db)
+ .await?;
+ let now = OffsetDateTime::now_utc();
+ for item in items {
+ if !eligible(db, &item, &ingress).await? {
+ continue;
+ }
+ if let (Some(token), Some(value), Some(expires)) = (
+ &item.challenge_token,
+ &item.challenge_value,
+ item.challenge_expires_at,
+ ) && expires > now
+ {
+ challenges.push(HttpChallenge {
+ hostname: item.hostname.clone(),
+ token: token.clone(),
+ key_authorization: value.clone(),
+ expires_at_unix: expires.unix_timestamp(),
+ });
+ }
+ if let Some(value) = &item.bundle
+ && item.expires_at.is_some_and(|expiry| expiry > now)
+ {
+ let parsed = (|| -> anyhow::Result<_> {
+ let pem: PemBundle =
+ serde_json::from_value(decrypt(secret, item.id, BUNDLE_KEY, value)?)?;
+ let validity = validate_pem(&item.hostname, &pem, now)?;
+ Ok((pem, validity))
+ })();
+ let Ok((pem, validity)) = parsed else {
+ tracing::warn!(operation="control_api.certificate.invalid_stored_bundle",certificate_id=%item.id,"invalid stored certificate omitted from regional snapshot");
+ continue;
+ };
+ bundles.push(CertificateBundle {
+ ingress_id: item.id,
+ hostname: item.hostname,
+ certificate_pem: pem.certificate_pem,
+ private_key_pem: pem.private_key_pem,
+ issued_at_unix: Some(validity.issued_at.unix_timestamp()),
+ revision: validity.revision,
+ expires_at_unix: Some(validity.expires_at.unix_timestamp()),
+ });
+ }
+ }
+ }
+ bundles.sort_by(|a, b| a.hostname.cmp(&b.hostname));
+ challenges.sort_by(|a, b| (&a.hostname, &a.token).cmp(&(&b.hostname, &b.token)));
+ let challenge_revision = challenge_revision(&challenges);
+ Ok(CertificateBundlesResponse {
+ bundles,
+ challenges,
+ challenge_revision,
+ })
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+ use super::*;
+ #[tokio::test]
+ async fn sweep_preparation_preserves_locked_rotated_credentials_and_manual_certificate() {
+ let stale = ingress_fixture();
+ let mut current = stale.clone();
+ current.dns_challenge_config =
+ seal_config(current.id, &json!({"api_token":"rotated-token"}), "secret").unwrap();
+ // The certificate import has committed; its admin endpoint has not yet
+ // changed the ingress's old automatic-renewal preference.
+ let mut imported = certificate_fixture(¤t);
+ imported.issuer = "manual".to_owned();
+ imported.auto_renew = false;
+ imported.bundle = Some(json!({"current_manual_material":true}));
+ imported.status = "active".to_owned();
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![current.clone()]])
+ .append_query_results([vec![imported]])
+ .into_connection();
+ let prepared = prepare_regional_sweep(&db, stale.id, "secret")
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ config(&prepared, "secret").unwrap(),
+ json!({"api_token":"rotated-token"})
+ );
+ let log = db.into_transaction_log();
+ assert_eq!(log.len(), 1);
+ assert_eq!(
+ log[0]
+ .statements()
+ .iter()
+ .filter(|s| s.sql.contains("FOR UPDATE"))
+ .count(),
+ 2
+ );
+ assert!(
+ !log[0]
+ .statements()
+ .iter()
+ .any(|s| s.sql.starts_with("UPDATE "))
+ );
+ }
+
+ #[tokio::test]
+ async fn sweep_preparation_syncs_only_the_locked_current_renewal_preference() {
+ let stale = ingress_fixture();
+ let mut current = stale.clone();
+ current.certificate_auto_renew = false;
+ current.dns_challenge_config = seal_config(current.id, &json!({}), "secret").unwrap();
+ let old = certificate_fixture(¤t);
+ let mut updated = old.clone();
+ updated.auto_renew = false;
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![current]])
+ .append_query_results([vec![old], vec![updated]])
+ .into_connection();
+ assert!(
+ !prepare_regional_sweep(&db, stale.id, "secret")
+ .await
+ .unwrap()
+ .unwrap()
+ .certificate_auto_renew
+ );
+ let log = db.into_transaction_log();
+ let updates = log
+ .iter()
+ .flat_map(|t| t.statements())
+ .filter(|s| s.sql.starts_with("UPDATE "))
+ .collect::>();
+ assert_eq!(updates.len(), 1);
+ assert!(updates[0].sql.contains("SET \"auto_renew\" ="));
+ assert_eq!(
+ updates[0].values.as_ref().unwrap().0[0],
+ sea_orm::Value::Bool(Some(false))
+ );
+ }
+
+ #[tokio::test]
+ async fn sweep_preparation_does_not_restore_legacy_material_over_an_in_flight_order() {
+ let mut ingress = ingress_fixture();
+ ingress.dns_challenge_config = seal_config(ingress.id, &json!({}), "secret").unwrap();
+ ingress.certificate_bundle = Some(json!({"old_envelope":"not-yet-read"}));
+ let mut current = certificate_fixture(&ingress);
+ current.status = "issuing".to_owned();
+ current.lease_until = Some(OffsetDateTime::now_utc() + time::Duration::minutes(1));
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![ingress.clone()]])
+ .append_query_results([vec![current]])
+ .into_connection();
+ prepare_regional_sweep(&db, ingress.id, "secret")
+ .await
+ .unwrap();
+ assert!(
+ !db.into_transaction_log()
+ .iter()
+ .flat_map(|t| t.statements())
+ .any(|s| s.sql.starts_with("UPDATE "))
+ );
+ }
+ #[tokio::test]
+ async fn corrupt_bundle_does_not_block_another_domains_withdrawal() {
+ let ingress = ingress_fixture();
+ let mut corrupt = certificate_fixture(&ingress);
+ corrupt.bundle = Some(json!({"invalid":true}));
+ corrupt.expires_at = Some(OffsetDateTime::now_utc() + time::Duration::days(30));
+ let mut binding = binding_fixture();
+ binding.deleted_at = Some(OffsetDateTime::now_utc());
+ let mut removed = certificate_fixture(&ingress);
+ removed.id = binding.id;
+ removed.host_binding_id = Some(binding.id);
+ removed.hostname = binding.host.clone();
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![ingress.clone()]])
+ .append_query_results([vec![corrupt, removed]])
+ .append_query_results([vec![ingress.clone()], vec![ingress]])
+ .append_query_results([vec![binding]])
+ .into_connection();
+ let snapshot = snapshot(&db, "eu", "secret").await.unwrap();
+ assert!(snapshot.bundles.is_empty());
+ assert!(snapshot.challenges.is_empty());
+ }
+
+ #[tokio::test]
+ async fn regional_hostname_change_preserves_custom_manual_certificate_override() {
+ let ingress = ingress_fixture();
+ let binding = binding_fixture();
+ let mut item = certificate_fixture(&ingress);
+ item.id = binding.id;
+ item.host_binding_id = Some(binding.id);
+ item.hostname = binding.host.clone();
+ item.issuer = "manual".to_owned();
+ item.status = "active".to_owned();
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![ingress.clone()]])
+ .append_query_results([vec![item.clone()]])
+ .into_connection();
+ let found = ensure_record(&db, &ingress, Some(&binding)).await.unwrap();
+ assert_eq!(found, item);
+ }
+
+ #[test]
+ fn valid_leaf_does_not_hide_expired_chain_certificate() {
+ let leaf = rcgen::generate_simple_self_signed(vec!["site.example.org".to_owned()]).unwrap();
+ let now = OffsetDateTime::now_utc();
+ let mut params =
+ rcgen::CertificateParams::new(vec!["issuer.example.org".to_owned()]).unwrap();
+ params.not_before = now - time::Duration::days(3);
+ params.not_after = now - time::Duration::days(1);
+ let intermediate = params
+ .self_signed(&rcgen::KeyPair::generate().unwrap())
+ .unwrap();
+ let bundle = PemBundle {
+ certificate_pem: format!("{}{}", leaf.cert.pem(), intermediate.pem()),
+ private_key_pem: leaf.signing_key.serialize_pem(),
+ };
+ assert!(validate_pem("site.example.org", &bundle, now).is_err());
+ }
+ pub(crate) fn ingress_fixture() -> regional_ingress::Model {
+ regional_ingress::Model {
+ id: Uuid::now_v7(),
+ region: "eu".to_owned(),
+ hostname: "entry.example.org".to_owned(),
+ enabled: true,
+ health_check_path: "/_grass/health".to_owned(),
+ health_check_interval_seconds: 30,
+ origin_host_preservation: true,
+ tls_enabled: true,
+ certificate_issuer: "letsencrypt".to_owned(),
+ certificate_auto_renew: true,
+ certificate_status: "pending".to_owned(),
+ certificate_expires_at: None,
+ certificate_error: None,
+ dns_challenge_provider: None,
+ dns_challenge_config: json!({}),
+ dns_challenge_status: "not_configured".to_owned(),
+ dns_challenge_record_name: None,
+ dns_challenge_record_value: None,
+ acme_account: None,
+ certificate_bundle: None,
+ certificate_issued_at: None,
+ deleted_at: None,
+ created_at: OffsetDateTime::now_utc(),
+ updated_at: OffsetDateTime::now_utc(),
+ }
+ }
+ pub(crate) fn certificate_fixture(ingress: ®ional_ingress::Model) -> cert::Model {
+ cert::Model {
+ id: ingress.id,
+ ingress_id: ingress.id,
+ host_binding_id: None,
+ hostname: ingress.hostname.clone(),
+ issuer: "letsencrypt".to_owned(),
+ challenge_method: "dns01".to_owned(),
+ auto_renew: true,
+ status: "pending".to_owned(),
+ error: None,
+ bundle: None,
+ acme_account: None,
+ revision: String::new(),
+ issued_at: None,
+ expires_at: None,
+ retry_at: None,
+ failure_count: 0,
+ lease_until: None,
+ generation: Uuid::now_v7(),
+ challenge_token: None,
+ challenge_value: None,
+ challenge_expires_at: None,
+ dns_record_name: None,
+ dns_record_value: None,
+ dns_cleanup: None,
+ updated_at: OffsetDateTime::now_utc(),
+ }
+ }
+ pub(crate) fn binding_fixture() -> project_host_binding::Model {
+ project_host_binding::Model {
+ id: Uuid::now_v7(),
+ project_id: Uuid::now_v7(),
+ team_id: Uuid::now_v7(),
+ host_source_id: None,
+ host: "site.example.org".to_owned(),
+ region: "eu".to_owned(),
+ kind: HostBindingKind::Custom,
+ environment: crate::infra::database::entity::HostBindingEnvironment::Production,
+ status: HostBindingStatus::Active,
+ failure_reason: None,
+ is_primary: false,
+ review_status: HostReviewStatus::Approved,
+ reviewed_by_user_id: None,
+ reviewed_at: None,
+ review_reason: None,
+ ownership_status: "verified".to_owned(),
+ ownership_checked_at: None,
+ ownership_error: None,
+ deleted_at: None,
+ created_at: OffsetDateTime::now_utc(),
+ updated_at: OffsetDateTime::now_utc(),
+ }
+ }
+
+ #[test]
+ fn ownership_review_and_lifecycle_are_independent_gates() {
+ let binding = binding_fixture();
+ assert!(binding_eligible(&binding));
+ let mut pending = binding.clone();
+ pending.ownership_status = "pending".to_owned();
+ assert!(!binding_eligible(&pending));
+ let mut rejected = binding.clone();
+ rejected.review_status = HostReviewStatus::Rejected;
+ assert!(!binding_eligible(&rejected));
+ let mut disabled = binding.clone();
+ disabled.status = HostBindingStatus::Disabled;
+ assert!(!binding_eligible(&disabled));
+ let mut deleted = binding;
+ deleted.deleted_at = Some(OffsetDateTime::now_utc());
+ assert!(!binding_eligible(&deleted));
+ }
+
+ #[tokio::test]
+ async fn failed_renewal_keeps_valid_certificate_and_expired_challenges_are_withdrawn() {
+ let ingress = ingress_fixture();
+ let mut item = certificate_fixture(&ingress);
+ let generated = rcgen::generate_simple_self_signed(vec![ingress.hostname.clone()]).unwrap();
+ let bundle = PemBundle {
+ certificate_pem: generated.cert.pem(),
+ private_key_pem: generated.signing_key.serialize_pem(),
+ };
+ let validity = validate_pem(&ingress.hostname, &bundle, OffsetDateTime::now_utc()).unwrap();
+ item.bundle = Some(
+ encrypt(
+ "secret",
+ item.id,
+ BUNDLE_KEY,
+ &serde_json::to_value(bundle).unwrap(),
+ )
+ .unwrap(),
+ );
+ item.expires_at = Some(validity.expires_at);
+ item.status = "failed".to_owned();
+ item.challenge_token = Some("expired".to_owned());
+ item.challenge_value = Some("expired.value".to_owned());
+ item.challenge_expires_at = Some(OffsetDateTime::now_utc() - time::Duration::seconds(1));
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![ingress.clone()]])
+ .append_query_results([vec![item]])
+ .append_query_results([vec![ingress]])
+ .into_connection();
+ let snapshot = snapshot(&db, "eu", "secret").await.unwrap();
+ assert_eq!(snapshot.bundles.len(), 1);
+ assert!(snapshot.challenges.is_empty());
+ assert_eq!(snapshot.challenge_revision, challenge_revision(&[]));
+ assert_eq!(snapshot.bundles[0].revision, validity.revision);
+ }
+
+ #[tokio::test]
+ async fn custom_domain_disablement_withdraws_certificate_and_http_challenge() {
+ let ingress = ingress_fixture();
+ let mut binding = binding_fixture();
+ binding.status = HostBindingStatus::Disabled;
+ let mut item = certificate_fixture(&ingress);
+ item.id = binding.id;
+ item.host_binding_id = Some(binding.id);
+ item.hostname = binding.host.clone();
+ item.challenge_token = Some("token".to_owned());
+ item.challenge_value = Some("token.thumbprint".to_owned());
+ item.challenge_expires_at = Some(OffsetDateTime::now_utc() + time::Duration::minutes(5));
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![ingress.clone()]])
+ .append_query_results([vec![item]])
+ .append_query_results([vec![ingress]])
+ .append_query_results([vec![binding]])
+ .into_connection();
+ let snapshot = snapshot(&db, "eu", "secret").await.unwrap();
+ assert!(snapshot.bundles.is_empty());
+ assert!(snapshot.challenges.is_empty());
+ }
+
+ #[tokio::test]
+ async fn custom_http_challenge_is_published_before_deployment_exists() {
+ let ingress = ingress_fixture();
+ let binding = binding_fixture();
+ let mut item = certificate_fixture(&ingress);
+ item.id = binding.id;
+ item.host_binding_id = Some(binding.id);
+ item.hostname = binding.host.clone();
+ item.challenge_token = Some("token".to_owned());
+ item.challenge_value = Some("token.thumbprint".to_owned());
+ item.challenge_expires_at = Some(OffsetDateTime::now_utc() + time::Duration::minutes(5));
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![ingress.clone()]])
+ .append_query_results([vec![item]])
+ .append_query_results([vec![ingress]])
+ .append_query_results([vec![binding.clone()]])
+ .into_connection();
+ let snapshot = snapshot(&db, "eu", "secret").await.unwrap();
+ assert!(snapshot.bundles.is_empty());
+ assert_eq!(snapshot.challenges.len(), 1);
+ assert_eq!(snapshot.challenges[0].hostname, binding.host);
+ assert!(
+ !db.into_transaction_log()
+ .iter()
+ .any(|entry| format!("{entry:?}").contains("deployments"))
+ );
+ }
+
+ #[tokio::test]
+ async fn stale_generation_cannot_restore_replaced_issuer_or_challenge() {
+ let ingress = ingress_fixture();
+ let mut old = certificate_fixture(&ingress);
+ old.issuer = "zerossl".to_owned();
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([Vec::::new()])
+ .into_connection();
+ assert!(
+ super::super::acme::save_current(&db, &old, old.clone().into())
+ .await
+ .is_err()
+ );
+ let log = format!("{:?}", db.into_transaction_log());
+ assert!(log.contains("generation"));
+ }
+ #[test]
+ fn provider_config_is_encrypted_bound_and_patchable() {
+ let id = Uuid::now_v7();
+ let value = json!({"api_token":"secret", "zone":"example.org"});
+ let sealed = seal_config(id, &value, "key").unwrap();
+ assert!(!sealed.to_string().contains("secret"));
+ assert_eq!(decrypt("key", id, CONFIG_KEY, &sealed).unwrap(), value);
+ assert!(decrypt("key", Uuid::now_v7(), CONFIG_KEY, &sealed).is_err());
+ assert_eq!(
+ merge_config(
+ value,
+ &json!({"api_token":"", "zone":null, "contact_email":"a@example.org"})
+ )
+ .unwrap(),
+ json!({"api_token":"secret","contact_email":"a@example.org"})
+ );
+ }
+ #[test]
+ fn certificate_name_key_and_signed_expiry_are_validated() {
+ let rcgen::CertifiedKey { cert, signing_key } =
+ rcgen::generate_simple_self_signed(vec!["site.example.org".to_owned()]).unwrap();
+ let bundle = PemBundle {
+ certificate_pem: cert.pem(),
+ private_key_pem: signing_key.serialize_pem(),
+ };
+ let validity =
+ validate_pem("site.example.org", &bundle, OffsetDateTime::now_utc()).unwrap();
+ assert!(validity.expires_at > OffsetDateTime::now_utc());
+ assert!(validate_pem("other.example.org", &bundle, OffsetDateTime::now_utc()).is_err());
+ assert!(validate_pem("site.example.org", &bundle, validity.expires_at).is_err());
+ let other = rcgen::KeyPair::generate().unwrap();
+ assert!(
+ validate_pem(
+ "site.example.org",
+ &PemBundle {
+ certificate_pem: bundle.certificate_pem,
+ private_key_pem: other.serialize_pem()
+ },
+ OffsetDateTime::now_utc()
+ )
+ .is_err()
+ );
+ }
+}
diff --git a/apps/control-api/src/domain/delivery.rs b/apps/control-api/src/domain/delivery.rs
index 0145c83..cebf780 100644
--- a/apps/control-api/src/domain/delivery.rs
+++ b/apps/control-api/src/domain/delivery.rs
@@ -386,7 +386,15 @@ pub async fn request_release(
let placement = if target.serve_node_id.is_none()
|| matches!(target.serve_status, DeploymentServeStatus::Retired)
{
- Some(scheduler::place_deployment(tx, serve_resources(&target)?, None).await?)
+ Some(
+ scheduler::place_deployment_in_region(
+ tx,
+ serve_resources(&target)?,
+ None,
+ Some(&target.region),
+ )
+ .await?,
+ )
} else {
None
};
@@ -591,6 +599,7 @@ mod tests {
},
Placement {
node_id,
+ region: "default".to_owned(),
overcommitted: false,
mode: PlacementMode::Automatic,
},
@@ -672,6 +681,7 @@ mod tests {
let node = node::ActiveModel {
id: Set(Uuid::now_v7()),
name: Set("serve-1".to_owned()),
+ region: Set("default".to_owned()),
token_hash: Set("test-token-hash".to_owned()),
status: Set(NodeStatus::Active),
build_enabled: Set(true),
diff --git a/apps/control-api/src/domain/deployments.rs b/apps/control-api/src/domain/deployments.rs
index 1d33504..27c818d 100644
--- a/apps/control-api/src/domain/deployments.rs
+++ b/apps/control-api/src/domain/deployments.rs
@@ -231,6 +231,7 @@ pub async fn create_deployment(
id: Set(Uuid::now_v7()),
project_id: Set(project.id),
team_id: Set(project.team_id),
+ region: Set(placement.region.clone()),
build_node_id: Set(None),
serve_node_id: Set(Some(placement.node_id)),
environment: Set(params.environment),
diff --git a/apps/control-api/src/domain/hosts.rs b/apps/control-api/src/domain/hosts.rs
index f5303a0..9622c26 100644
--- a/apps/control-api/src/domain/hosts.rs
+++ b/apps/control-api/src/domain/hosts.rs
@@ -20,6 +20,7 @@ pub struct CreateHostSourceParams {
pub kind: HostSourceKind,
pub label: String,
pub base_domain: String,
+ pub region: String,
pub enabled: bool,
pub allows_auto_assign: bool,
pub is_default: bool,
@@ -29,6 +30,7 @@ pub struct CreateHostSourceParams {
pub struct UpdateHostSourceParams {
pub label: Option,
+ pub region: Option,
pub enabled: Option,
pub allows_auto_assign: Option,
pub is_default: Option,
@@ -104,6 +106,7 @@ pub async fn create_source(
kind: Set(params.kind),
label: Set(params.label),
base_domain: Set(params.base_domain),
+ region: Set(params.region),
enabled: Set(params.enabled),
allows_auto_assign: Set(params.allows_auto_assign),
is_default: Set(params.is_default),
@@ -133,6 +136,9 @@ pub async fn update_source(
if let Some(label) = params.label {
active.label = Set(label);
}
+ if let Some(region) = params.region {
+ active.region = Set(region);
+ }
if let Some(enabled) = params.enabled {
active.enabled = Set(enabled);
}
@@ -263,6 +269,7 @@ pub struct CreateBindingParams {
pub team_id: Uuid,
pub host_source_id: Option,
pub host: String,
+ pub region: String,
pub kind: HostBindingKind,
pub environment: HostBindingEnvironment,
pub status: HostBindingStatus,
@@ -358,6 +365,7 @@ pub async fn create_binding(
team_id: Set(params.team_id),
host_source_id: Set(params.host_source_id),
host: Set(params.host),
+ region: Set(params.region),
kind: Set(params.kind),
environment: Set(params.environment),
status: Set(params.status),
@@ -367,6 +375,13 @@ pub async fn create_binding(
reviewed_by_user_id: Set(None),
reviewed_at: Set(None),
review_reason: Set(None),
+ ownership_status: Set(if params.host_source_id.is_some() {
+ "not_required".to_owned()
+ } else {
+ "pending".to_owned()
+ }),
+ ownership_checked_at: Set(None),
+ ownership_error: Set(None),
deleted_at: Set(None),
created_at: Set(now),
updated_at: Set(now),
@@ -546,6 +561,7 @@ mod tests {
kind: HostSourceKind::Wildcard,
label: label.to_owned(),
base_domain: "example.test".to_owned(),
+ region: "default".to_owned(),
enabled,
allows_auto_assign,
is_default,
diff --git a/apps/control-api/src/domain/ingress.rs b/apps/control-api/src/domain/ingress.rs
new file mode 100644
index 0000000..70e38ce
--- /dev/null
+++ b/apps/control-api/src/domain/ingress.rs
@@ -0,0 +1,580 @@
+use sea_orm::{
+ ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter, QueryOrder, Set, sea_query::OnConflict,
+};
+use subtle::ConstantTimeEq;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use crate::infra::database::entity::{
+ NodeStatus, node, node_ingress_status, regional_ingress, regional_ingress_health,
+};
+
+pub const HEARTBEAT_STALE_SECONDS: i64 = 90;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CnameGuidance {
+ pub record_type: &'static str,
+ pub name: String,
+ pub target: String,
+ pub verification_name: String,
+ pub verification_value: String,
+ pub region: String,
+ pub origin_host_preservation: bool,
+}
+
+pub struct CnameGuidanceInput<'a> {
+ pub host: &'a str,
+ pub region: &'a str,
+ pub ingress_hostname: &'a str,
+ pub verification_name: &'a str,
+ pub verification_value: &'a str,
+ pub origin_host_preservation: bool,
+}
+
+pub fn cname_guidance(input: CnameGuidanceInput<'_>) -> CnameGuidance {
+ CnameGuidance {
+ record_type: "CNAME",
+ name: input.host.to_owned(),
+ target: input.ingress_hostname.to_owned(),
+ verification_name: input.verification_name.to_owned(),
+ verification_value: input.verification_value.to_owned(),
+ region: input.region.to_owned(),
+ origin_host_preservation: input.origin_host_preservation,
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct IngressCandidateInput {
+ pub node_id: String,
+ pub region: String,
+ pub base_url: String,
+ pub healthy: bool,
+ pub priority: i32,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct IngressCandidate {
+ pub node_id: String,
+ pub base_url: String,
+ pub priority: i32,
+}
+
+pub fn healthy_candidates(
+ candidates: &[IngressCandidateInput],
+ region: &str,
+) -> Vec {
+ let mut selected = candidates
+ .iter()
+ .filter(|candidate| candidate.region == region && candidate.healthy)
+ .map(|candidate| IngressCandidate {
+ node_id: candidate.node_id.clone(),
+ base_url: candidate.base_url.clone(),
+ priority: candidate.priority,
+ })
+ .collect::>();
+ selected.sort_by(|left, right| {
+ left.priority
+ .cmp(&right.priority)
+ .then_with(|| left.node_id.cmp(&right.node_id))
+ });
+ selected
+}
+
+pub async fn list(db: &C) -> anyhow::Result> {
+ regional_ingress::Entity::find()
+ .filter(regional_ingress::Column::DeletedAt.is_null())
+ .order_by_asc(regional_ingress::Column::Region)
+ .all(db)
+ .await
+ .map_err(Into::into)
+}
+
+pub async fn get_by_id(
+ db: &C,
+ id: Uuid,
+) -> anyhow::Result> {
+ regional_ingress::Entity::find_by_id(id)
+ .filter(regional_ingress::Column::DeletedAt.is_null())
+ .one(db)
+ .await
+ .map_err(Into::into)
+}
+
+pub async fn get_enabled_by_region(
+ db: &C,
+ region: &str,
+) -> anyhow::Result> {
+ regional_ingress::Entity::find()
+ .filter(regional_ingress::Column::Region.eq(region))
+ .filter(regional_ingress::Column::Enabled.eq(true))
+ .filter(regional_ingress::Column::DeletedAt.is_null())
+ .one(db)
+ .await
+ .map_err(Into::into)
+}
+
+pub async fn healthy_serve_nodes(
+ db: &C,
+ region: &str,
+ now: OffsetDateTime,
+) -> anyhow::Result> {
+ let ingress = regional_ingress::Entity::find()
+ .filter(regional_ingress::Column::Region.eq(region))
+ .filter(regional_ingress::Column::Enabled.eq(true))
+ .filter(regional_ingress::Column::DeletedAt.is_null())
+ .one(db)
+ .await?;
+ let health_freshness = ingress
+ .as_ref()
+ .map(|item| health_check_freshness_seconds(item.health_check_interval_seconds))
+ .unwrap_or(120);
+ let tls_required = ingress.as_ref().is_some_and(|i| i.tls_enabled);
+ let health = if let Some(ingress) = ingress {
+ regional_ingress_health::Entity::find()
+ .filter(regional_ingress_health::Column::IngressId.eq(ingress.id))
+ .all(db)
+ .await?
+ .into_iter()
+ .map(|item| (item.node_id, item))
+ .collect::>()
+ } else {
+ std::collections::HashMap::new()
+ };
+ let nodes = node::Entity::find()
+ .filter(node::Column::Region.eq(region))
+ .filter(node::Column::ServeEnabled.eq(true))
+ .filter(node::Column::BaseUrl.is_not_null())
+ .filter(node::Column::DeletedAt.is_null())
+ .order_by_asc(node::Column::Id)
+ .all(db)
+ .await?;
+ let tls_status = node_ingress_status::Entity::find()
+ .filter(node_ingress_status::Column::NodeId.is_in(nodes.iter().map(|n| n.id)))
+ .all(db)
+ .await?
+ .into_iter()
+ .map(|s| (s.node_id, s))
+ .collect::>();
+ let candidates = nodes
+ .iter()
+ .filter_map(|node| {
+ let base_url = node.base_url.as_deref()?;
+ Some(IngressCandidateInput {
+ node_id: node.id.to_string(),
+ region: node.region.clone(),
+ base_url: base_url.to_owned(),
+ healthy: matches!(node.status, NodeStatus::Active)
+ && (!tls_required
+ || tls_status.get(&node.id).is_some_and(|s| {
+ s.tls_ready && (now - s.checked_at).whole_seconds() <= 30
+ }))
+ && node.last_heartbeat_at.is_some_and(|heartbeat| {
+ (now - heartbeat).whole_seconds() <= HEARTBEAT_STALE_SECONDS
+ })
+ && health.get(&node.id).is_some_and(|item| {
+ item.status == "healthy"
+ && item.checked_at.is_some_and(|checked| {
+ (now - checked).whole_seconds() <= i64::from(health_freshness)
+ })
+ }),
+ priority: 0,
+ })
+ })
+ .collect::>();
+ Ok(healthy_candidates(&candidates, region))
+}
+
+fn health_check_freshness_seconds(interval_seconds: i32) -> i32 {
+ interval_seconds.saturating_mul(3).max(60)
+}
+
+pub fn health_check_due(
+ last_checked: Option,
+ interval_seconds: i32,
+ now: OffsetDateTime,
+) -> bool {
+ last_checked
+ .is_none_or(|checked| (now - checked).whole_seconds() >= i64::from(interval_seconds.max(5)))
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ProbeResult {
+ pub status: &'static str,
+ pub latency_ms: Option,
+ pub error: Option,
+}
+
+pub async fn probe_node(
+ client: &reqwest::Client,
+ base_url: &str,
+ hostname: &str,
+ path: &str,
+) -> ProbeResult {
+ let started = std::time::Instant::now();
+ let url = match url::Url::parse(base_url).and_then(|base| base.join(path)) {
+ Ok(url) if matches!(url.scheme(), "http" | "https") && url.host_str().is_some() => url,
+ _ => {
+ return ProbeResult {
+ status: "unhealthy",
+ latency_ms: None,
+ error: Some("invalid node base URL".to_owned()),
+ };
+ }
+ };
+ let response = client
+ .get(url)
+ .header(reqwest::header::HOST, hostname)
+ .send()
+ .await;
+ let latency_ms = i32::try_from(started.elapsed().as_millis()).ok();
+ match response {
+ Ok(response) if response.status().is_success() => ProbeResult {
+ status: "healthy",
+ latency_ms,
+ error: None,
+ },
+ Ok(response) => ProbeResult {
+ status: "unhealthy",
+ latency_ms,
+ error: Some(format!("health endpoint returned {}", response.status())),
+ },
+ Err(error) => ProbeResult {
+ status: "unhealthy",
+ latency_ms,
+ error: Some(error.without_url().to_string()),
+ },
+ }
+}
+
+pub async fn probe_regional_ingress(
+ db: &sea_orm::DatabaseConnection,
+ ingress: ®ional_ingress::Model,
+ now: OffsetDateTime,
+) -> anyhow::Result<()> {
+ let last_checked = regional_ingress_health::Entity::find()
+ .filter(regional_ingress_health::Column::IngressId.eq(ingress.id))
+ .order_by_desc(regional_ingress_health::Column::CheckedAt)
+ .one(db)
+ .await?
+ .and_then(|item| item.checked_at);
+ if !health_check_due(last_checked, ingress.health_check_interval_seconds, now) {
+ return Ok(());
+ }
+ let client = reqwest::Client::builder()
+ .redirect(reqwest::redirect::Policy::none())
+ .connect_timeout(std::time::Duration::from_secs(3))
+ .timeout(std::time::Duration::from_secs(10))
+ .build()?;
+ let nodes = node::Entity::find()
+ .filter(node::Column::Region.eq(&ingress.region))
+ .filter(node::Column::ServeEnabled.eq(true))
+ .filter(node::Column::BaseUrl.is_not_null())
+ .filter(node::Column::DeletedAt.is_null())
+ .all(db)
+ .await?;
+ for node in nodes {
+ let result = probe_node(
+ &client,
+ node.base_url.as_deref().unwrap_or_default(),
+ &ingress.hostname,
+ &ingress.health_check_path,
+ )
+ .await;
+ let active = regional_ingress_health::ActiveModel {
+ ingress_id: Set(ingress.id),
+ node_id: Set(node.id),
+ status: Set(result.status.to_owned()),
+ checked_at: Set(Some(now)),
+ latency_ms: Set(result.latency_ms),
+ error: Set(result.error),
+ };
+ regional_ingress_health::Entity::insert(active)
+ .on_conflict(
+ OnConflict::columns([
+ regional_ingress_health::Column::IngressId,
+ regional_ingress_health::Column::NodeId,
+ ])
+ .update_columns([
+ regional_ingress_health::Column::Status,
+ regional_ingress_health::Column::CheckedAt,
+ regional_ingress_health::Column::LatencyMs,
+ regional_ingress_health::Column::Error,
+ ])
+ .to_owned(),
+ )
+ .exec(db)
+ .await?;
+ }
+ Ok(())
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DnsVerification {
+ Verified,
+ Missing,
+ Mismatch,
+}
+
+pub async fn verify_dns_txt(host: &str, expected: &str) -> anyhow::Result {
+ verify_dns_txt_at(
+ &reqwest::Client::builder()
+ .connect_timeout(std::time::Duration::from_secs(5))
+ .timeout(std::time::Duration::from_secs(10))
+ .build()?,
+ "https://cloudflare-dns.com/dns-query",
+ host,
+ expected,
+ )
+ .await
+}
+
+pub async fn verify_dns_txt_at(
+ client: &reqwest::Client,
+ endpoint: &str,
+ host: &str,
+ expected: &str,
+) -> anyhow::Result {
+ let name = format!("_grass.{}", host.trim_end_matches('.'));
+ verify_dns_record_at(client, endpoint, &name, "TXT", expected).await
+}
+
+pub async fn verify_dns_record_at(
+ client: &reqwest::Client,
+ endpoint: &str,
+ name: &str,
+ record_type: &str,
+ expected: &str,
+) -> anyhow::Result {
+ let response = client
+ .get(endpoint)
+ .query(&[("name", name), ("type", record_type)])
+ .header("accept", "application/dns-json")
+ .send()
+ .await?;
+ if !response.status().is_success() {
+ anyhow::bail!("DNS TXT query returned {}", response.status());
+ }
+ let body: serde_json::Value = response.json().await?;
+ if body
+ .get("Status")
+ .and_then(serde_json::Value::as_u64)
+ .is_some_and(|status| status != 0 && status != 3)
+ {
+ anyhow::bail!("DNS resolver returned an unsuccessful status");
+ }
+ let answers = body
+ .get("Answer")
+ .and_then(serde_json::Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ let mut found = false;
+ for answer in answers {
+ let wanted = if record_type == "CNAME" { 5 } else { 16 };
+ if answer.get("type").and_then(serde_json::Value::as_u64) != Some(wanted) {
+ continue;
+ }
+ let Some(data) = answer.get("data").and_then(serde_json::Value::as_str) else {
+ continue;
+ };
+ let text = if record_type == "CNAME" {
+ data.trim().trim_end_matches('.').to_ascii_lowercase()
+ } else {
+ data.trim().trim_matches('"').replace("\" \"", "")
+ };
+ let value = text.as_str();
+ if value.as_bytes().ct_eq(expected.as_bytes()).into() {
+ return Ok(DnsVerification::Verified);
+ }
+ found = true;
+ }
+ Ok(if found {
+ DnsVerification::Mismatch
+ } else {
+ DnsVerification::Missing
+ })
+}
+
+pub fn dns_verification_token(secret_key: &str, binding_id: Uuid, host: &str) -> String {
+ let key = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, secret_key.as_bytes());
+ let message = format!("grass-domain-v1:{binding_id}:{host}");
+ format!(
+ "grass-{}",
+ hex::encode(ring::hmac::sign(&key, message.as_bytes()).as_ref())
+ )
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+ use super::{CnameGuidanceInput, IngressCandidateInput, cname_guidance, healthy_candidates};
+ use axum::{Json, Router, routing::get};
+ pub(crate) fn node_fixture() -> super::node::Model {
+ let now = time::OffsetDateTime::now_utc();
+ super::node::Model {
+ id: uuid::Uuid::now_v7(),
+ name: "entry".to_owned(),
+ region: "eu".to_owned(),
+ token_hash: "hash".to_owned(),
+ status: super::NodeStatus::Active,
+ build_enabled: false,
+ serve_enabled: true,
+ build_concurrency: 1,
+ base_url: Some("http://entry.example.org".to_owned()),
+ work_root: None,
+ capacity_cpu_millicores: 2000,
+ capacity_memory_mb: 2048,
+ capacity_disk_mb: 10000,
+ max_deployments: 10,
+ metadata: serde_json::json!({}),
+ last_heartbeat_at: Some(now),
+ desired_config: None,
+ desired_config_revision: 0,
+ effective_config: None,
+ effective_config_revision: 0,
+ config_sync_status: crate::infra::database::entity::NodeConfigSyncStatus::Applied,
+ config_sync_error: None,
+ node_token_configured: true,
+ config_updated_at: None,
+ config_applied_at: None,
+ deleted_at: None,
+ created_at: now,
+ updated_at: now,
+ }
+ }
+
+ #[tokio::test]
+ async fn tls_ingress_excludes_http_only_and_stale_tls_nodes() {
+ use super::*;
+ let now = OffsetDateTime::now_utc();
+ let ingress = crate::domain::certificates::tests::ingress_fixture();
+ let ready = node_fixture();
+ let http_only = node_fixture();
+ let stale = node_fixture();
+ let nodes = vec![ready.clone(), http_only.clone(), stale.clone()];
+ let health = nodes
+ .iter()
+ .map(|node| regional_ingress_health::Model {
+ ingress_id: ingress.id,
+ node_id: node.id,
+ status: "healthy".to_owned(),
+ checked_at: Some(now),
+ latency_ms: Some(1),
+ error: None,
+ })
+ .collect::>();
+ let status = nodes
+ .iter()
+ .map(|node| node_ingress_status::Model {
+ node_id: node.id,
+ certificates: serde_json::json!([]),
+ challenge_revision: String::new(),
+ tls_ready: node.id != http_only.id,
+ checked_at: if node.id == stale.id {
+ now - time::Duration::seconds(60)
+ } else {
+ now
+ },
+ })
+ .collect::>();
+ let db = sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![ingress]])
+ .append_query_results([health])
+ .append_query_results([nodes])
+ .append_query_results([status])
+ .into_connection();
+ let selected = healthy_serve_nodes(&db, "eu", now).await.unwrap();
+ assert_eq!(selected.len(), 1);
+ assert_eq!(selected[0].node_id, ready.id.to_string());
+ }
+
+ #[tokio::test]
+ async fn dns_txt_verification_accepts_matching_answer_and_rejects_missing() {
+ let app = Router::new().route(
+ "/dns-query",
+ get(|| async {
+ Json(serde_json::json!({
+ "Answer": [{"type": 16, "data": "\"expected-token\""}]
+ }))
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let endpoint = format!("http://{}/dns-query", listener.local_addr().unwrap());
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ let client = reqwest::Client::new();
+ assert_eq!(
+ super::verify_dns_txt_at(&client, &endpoint, "app.example.com", "expected-token")
+ .await
+ .unwrap(),
+ super::DnsVerification::Verified
+ );
+ assert_eq!(
+ super::verify_dns_txt_at(&client, &endpoint, "app.example.com", "other-token")
+ .await
+ .unwrap(),
+ super::DnsVerification::Mismatch
+ );
+ server.abort();
+ }
+
+ #[test]
+ fn cname_guidance_includes_cname_and_txt_ownership_records() {
+ let guidance = cname_guidance(CnameGuidanceInput {
+ host: "app.example.com",
+ region: "eu-west",
+ ingress_hostname: "eu-west.edge.example.net",
+ verification_name: "_grass.app.example.com",
+ verification_value: "grass-token",
+ origin_host_preservation: true,
+ });
+
+ assert_eq!(guidance.record_type, "CNAME");
+ assert_eq!(guidance.name, "app.example.com");
+ assert_eq!(guidance.target, "eu-west.edge.example.net");
+ assert_eq!(guidance.verification_name, "_grass.app.example.com");
+ assert_eq!(guidance.verification_value, "grass-token");
+ assert!(guidance.origin_host_preservation);
+ }
+
+ #[test]
+ fn healthy_candidates_are_limited_to_region_and_sorted_by_priority() {
+ let candidates = vec![
+ IngressCandidateInput {
+ node_id: "node-2".to_owned(),
+ region: "eu-west".to_owned(),
+ base_url: "https://node-2.example.net".to_owned(),
+ healthy: true,
+ priority: 20,
+ },
+ IngressCandidateInput {
+ node_id: "node-1".to_owned(),
+ region: "eu-west".to_owned(),
+ base_url: "https://node-1.example.net".to_owned(),
+ healthy: true,
+ priority: 10,
+ },
+ IngressCandidateInput {
+ node_id: "node-3".to_owned(),
+ region: "us-east".to_owned(),
+ base_url: "https://node-3.example.net".to_owned(),
+ healthy: true,
+ priority: 1,
+ },
+ IngressCandidateInput {
+ node_id: "node-4".to_owned(),
+ region: "eu-west".to_owned(),
+ base_url: "https://node-4.example.net".to_owned(),
+ healthy: false,
+ priority: 0,
+ },
+ ];
+
+ let selected = healthy_candidates(&candidates, "eu-west");
+
+ assert_eq!(
+ selected
+ .iter()
+ .map(|candidate| candidate.node_id.as_str())
+ .collect::>(),
+ vec!["node-1", "node-2"]
+ );
+ }
+}
diff --git a/apps/control-api/src/domain/mod.rs b/apps/control-api/src/domain/mod.rs
index 9257f64..fc5ec02 100644
--- a/apps/control-api/src/domain/mod.rs
+++ b/apps/control-api/src/domain/mod.rs
@@ -1,10 +1,13 @@
+pub mod acme;
pub mod audits;
pub mod authentication;
+pub mod certificates;
pub mod cleanup;
pub mod codes;
pub mod delivery;
pub mod deployments;
pub mod hosts;
+pub mod ingress;
pub mod node_deletions;
pub mod nodes;
pub mod notifications;
diff --git a/apps/control-api/src/domain/node_deletions.rs b/apps/control-api/src/domain/node_deletions.rs
index d4c4077..7467ff1 100644
--- a/apps/control-api/src/domain/node_deletions.rs
+++ b/apps/control-api/src/domain/node_deletions.rs
@@ -839,6 +839,7 @@ mod tests {
node::ActiveModel {
id: Set(Uuid::now_v7()),
name: Set(name.to_owned()),
+ region: Set("default".to_owned()),
token_hash: Set(format!("token-{name}")),
status: Set(NodeStatus::Active),
build_enabled: Set(true),
@@ -891,6 +892,7 @@ mod tests {
},
Placement {
node_id: source_node_id,
+ region: "default".to_owned(),
overcommitted: false,
mode: PlacementMode::Automatic,
},
@@ -1339,6 +1341,7 @@ mod tests {
},
Placement {
node_id: fixture.source.id,
+ region: "default".to_owned(),
overcommitted: false,
mode: PlacementMode::Automatic,
},
diff --git a/apps/control-api/src/domain/nodes.rs b/apps/control-api/src/domain/nodes.rs
index b4898cb..30432a0 100644
--- a/apps/control-api/src/domain/nodes.rs
+++ b/apps/control-api/src/domain/nodes.rs
@@ -17,6 +17,7 @@ pub fn gateway_token(secret: &str) -> String {
pub struct CreateNodeParams {
pub name: String,
+ pub region: String,
pub token_hash: String,
pub storage_root: Option,
}
@@ -37,6 +38,7 @@ pub async fn create_node(
node::ActiveModel {
id: Set(Uuid::now_v7()),
name: Set(params.name.clone()),
+ region: Set(params.region),
token_hash: Set(params.token_hash),
status: Set(NodeStatus::Pending),
build_enabled: Set(true),
@@ -145,6 +147,7 @@ pub async fn list(db: &C) -> anyhow::Result
pub struct RegisterNodeParams {
pub name: String,
+ pub region: String,
pub version: String,
pub build_enabled: bool,
pub serve_enabled: bool,
@@ -282,6 +285,7 @@ pub async fn apply_registration(
let initialize_deployments = first_resource_report;
let mut active: node::ActiveModel = node.into();
active.name = Set(params.name);
+ active.region = Set(params.region);
active.build_enabled = Set(params.build_enabled);
active.serve_enabled = Set(params.serve_enabled);
active.build_concurrency = Set(if params.build_enabled {
@@ -498,6 +502,7 @@ mod tests {
node::Model {
id: Uuid::nil(),
name: "test".to_owned(),
+ region: "default".to_owned(),
token_hash: String::new(),
status,
build_enabled: true,
diff --git a/apps/control-api/src/domain/retention.rs b/apps/control-api/src/domain/retention.rs
index 7026b5e..159c5b9 100644
--- a/apps/control-api/src/domain/retention.rs
+++ b/apps/control-api/src/domain/retention.rs
@@ -270,6 +270,7 @@ mod tests {
id: Uuid::now_v7(),
project_id: Uuid::now_v7(),
team_id: Uuid::now_v7(),
+ region: "default".to_owned(),
build_node_id: None,
serve_node_id: None,
environment,
diff --git a/apps/control-api/src/domain/scheduler.rs b/apps/control-api/src/domain/scheduler.rs
index f024f55..0c54278 100644
--- a/apps/control-api/src/domain/scheduler.rs
+++ b/apps/control-api/src/domain/scheduler.rs
@@ -27,6 +27,7 @@ GROUP BY d.serve_node_id
const ELIGIBLE_CANDIDATES_SQL: &str = r#"
SELECT
n.id AS node_id,
+ n.region,
n.capacity_cpu_millicores,
n.capacity_memory_mb,
n.capacity_disk_mb,
@@ -83,6 +84,7 @@ pub struct NodeUsage {
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Candidate {
pub node_id: Uuid,
+ pub region: String,
pub capacity: NodeResources,
pub usage: NodeUsage,
}
@@ -93,9 +95,10 @@ pub enum PlacementMode {
Manual,
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Placement {
pub node_id: Uuid,
+ pub region: String,
pub overcommitted: bool,
pub mode: PlacementMode,
}
@@ -104,6 +107,8 @@ pub struct Placement {
pub enum ScheduleError {
#[error("no serve node has enough capacity")]
NoCapacity,
+ #[error("no healthy serve node is available in region '{0}'")]
+ NoCapacityInRegion(String),
#[error("selected serve node is unavailable")]
SelectedNodeUnavailable,
#[error("selected serve node has no remaining capacity")]
@@ -194,6 +199,29 @@ pub fn choose_candidate(
)
}
+pub fn choose_candidate_in_region(
+ candidates: &[Candidate],
+ requested: ServeResources,
+ region: Option<&str>,
+ selected_node_id: Option,
+) -> Result {
+ let filtered = region
+ .map(|region| {
+ candidates
+ .iter()
+ .filter(|candidate| candidate.region == region)
+ .cloned()
+ .collect::>()
+ })
+ .unwrap_or_else(|| candidates.to_vec());
+ if let Some(region) = region
+ && filtered.is_empty()
+ {
+ return Err(ScheduleError::NoCapacityInRegion(region.to_owned()));
+ }
+ choose_candidate(&filtered, requested, selected_node_id)
+}
+
pub fn choose_candidate_with_rng(
candidates: &[Candidate],
requested: ServeResources,
@@ -212,6 +240,7 @@ pub fn choose_candidate_with_rng(
}
return Ok(Placement {
node_id: candidate.node_id,
+ region: candidate.region.clone(),
overcommitted: !candidate.projected(requested).all_at_most_one(),
mode: PlacementMode::Manual,
});
@@ -229,6 +258,7 @@ pub fn choose_candidate_with_rng(
) {
return Ok(Placement {
node_id: candidate.node_id,
+ region: candidate.region.clone(),
overcommitted: false,
mode: PlacementMode::Automatic,
});
@@ -241,6 +271,7 @@ pub fn choose_candidate_with_rng(
)
.map(|candidate| Placement {
node_id: candidate.node_id,
+ region: candidate.region.clone(),
overcommitted: true,
mode: PlacementMode::Automatic,
})
@@ -273,14 +304,15 @@ where
tied.choose(rng).copied()
}
-pub async fn place_deployment(
+pub async fn place_deployment_in_region(
transaction: &DatabaseTransaction,
requested: ServeResources,
selected_node_id: Option,
+ region: Option<&str>,
) -> Result {
lock_placement(transaction).await?;
let candidates = eligible_candidates(transaction).await?;
- choose_candidate(&candidates, requested, selected_node_id)
+ choose_candidate_in_region(&candidates, requested, region, selected_node_id)
}
/// Serializes operations that can change whether a Serve Node has room for
@@ -328,6 +360,7 @@ pub async fn eligible_candidates(
#[derive(Debug, FromQueryResult)]
struct CandidateRow {
node_id: Uuid,
+ region: String,
capacity_cpu_millicores: i64,
capacity_memory_mb: i64,
capacity_disk_mb: i64,
@@ -378,6 +411,7 @@ impl TryFrom for Candidate {
fn try_from(row: CandidateRow) -> Result {
Ok(Self {
node_id: row.node_id,
+ region: row.region,
capacity: NodeResources {
cpu_millicores: row
.capacity_cpu_millicores
@@ -448,6 +482,7 @@ mod tests {
) -> Candidate {
Candidate {
node_id: Uuid::from_u128(id),
+ region: "default".to_owned(),
capacity: NodeResources {
cpu_millicores: capacity.0,
memory_mb: capacity.1,
@@ -483,6 +518,25 @@ mod tests {
assert_eq!(placement.mode, PlacementMode::Automatic);
}
+ #[test]
+ fn regional_candidates_are_limited_to_the_requested_region() {
+ let candidates = vec![
+ candidate_in_region(1, "us-east"),
+ candidate_in_region(2, "eu-west"),
+ ];
+
+ let placement =
+ choose_candidate_in_region(&candidates, request(), Some("eu-west"), None).unwrap();
+
+ assert_eq!(placement.node_id, Uuid::from_u128(2));
+ }
+
+ fn candidate_in_region(id: u128, region: &str) -> Candidate {
+ let mut candidate = candidate(id, (2_000, 2_000, 10_000, 10), (0, 0, 0, 0));
+ candidate.region = region.to_owned();
+ candidate
+ }
+
#[test]
fn selection_uses_projected_dominant_resource() {
let lower_current = candidate(1, (1_000, 1_000, 10_000, 10), (0, 0, 0, 0));
diff --git a/apps/control-api/src/domain/storage_migrations.rs b/apps/control-api/src/domain/storage_migrations.rs
index 5acc142..002323d 100644
--- a/apps/control-api/src/domain/storage_migrations.rs
+++ b/apps/control-api/src/domain/storage_migrations.rs
@@ -477,6 +477,7 @@ mod tests {
std::env::temp_dir().join(format!("grass-migration-node-{}.toml", Uuid::now_v7()));
let node_config = GenerateParams {
node_name: "local-node",
+ region: "default",
node_token: "node-token",
control_api_url: "http://127.0.0.1:7817".to_owned(),
storage_root: source_root.to_str().unwrap(),
diff --git a/apps/control-api/src/features/api/v1/admin.rs b/apps/control-api/src/features/api/v1/admin.rs
index 8fe8e4f..e8abcd6 100644
--- a/apps/control-api/src/features/api/v1/admin.rs
+++ b/apps/control-api/src/features/api/v1/admin.rs
@@ -10,6 +10,7 @@ pub mod identity_providers;
pub mod nodes;
pub mod projects;
pub mod quota_plans;
+pub mod regional_ingresses;
pub mod registration;
pub mod reviews;
pub mod settings;
@@ -56,6 +57,22 @@ pub fn router() -> Router {
"/host-sources/{source_id}",
patch(host_sources::update).delete(host_sources::remove),
)
+ .route(
+ "/regional-ingresses",
+ get(regional_ingresses::list).post(regional_ingresses::create),
+ )
+ .route(
+ "/regional-ingresses/{ingress_id}",
+ patch(regional_ingresses::update).delete(regional_ingresses::remove),
+ )
+ .route(
+ "/regional-ingresses/{ingress_id}/certificate/renew",
+ post(regional_ingresses::renew_certificate),
+ )
+ .route(
+ "/regional-ingresses/{ingress_id}/certificate/import",
+ post(regional_ingresses::import_certificate),
+ )
.route("/audit-events", get(audit_events::list))
.route(
"/cleanup/audit-events",
diff --git a/apps/control-api/src/features/api/v1/admin/domains.rs b/apps/control-api/src/features/api/v1/admin/domains.rs
index 270fd51..bec4359 100644
--- a/apps/control-api/src/features/api/v1/admin/domains.rs
+++ b/apps/control-api/src/features/api/v1/admin/domains.rs
@@ -154,7 +154,7 @@ async fn decide(
let reason = optional_reason(reason);
let DomainDecision::Apply {
review_status: new_review_status,
- binding_status: new_binding_status,
+ binding_status: mut new_binding_status,
} = domain_decision(&binding.review_status, approved)
else {
return Err(AppError::Conflict {
@@ -162,6 +162,12 @@ async fn decide(
message: "domain review has already been decided".to_owned(),
});
};
+ if approved && binding.ownership_status != "verified" {
+ new_binding_status = HostBindingStatus::Pending;
+ }
+ if matches!(binding.status, HostBindingStatus::Disabled) {
+ new_binding_status = HostBindingStatus::Disabled;
+ }
let before = json!({ "review_status": review_status(&binding.review_status), "status": binding_status(&binding.status) });
let mut active: project_host_binding::ActiveModel = binding.clone().into();
active.review_status = Set(new_review_status);
diff --git a/apps/control-api/src/features/api/v1/admin/host_sources.rs b/apps/control-api/src/features/api/v1/admin/host_sources.rs
index fb32914..3df8fd9 100644
--- a/apps/control-api/src/features/api/v1/admin/host_sources.rs
+++ b/apps/control-api/src/features/api/v1/admin/host_sources.rs
@@ -28,6 +28,7 @@ fn source_view(source: &host_source::Model) -> serde_json::Value {
},
"label": source.label,
"base_domain": source.base_domain,
+ "region": source.region,
"enabled": source.enabled,
"allows_auto_assign": source.allows_auto_assign,
"is_default": source.is_default,
@@ -144,6 +145,8 @@ pub struct CreateHostSourceRequest {
pub kind: String,
pub label: String,
pub base_domain: String,
+ #[serde(default = "default_region")]
+ pub region: String,
#[serde(default = "default_true")]
pub enabled: bool,
#[serde(default = "default_true")]
@@ -160,6 +163,10 @@ const fn default_true() -> bool {
true
}
+fn default_region() -> String {
+ "default".to_owned()
+}
+
/// POST /api/v1/admin/host-sources
pub async fn create(
State(state): State,
@@ -181,6 +188,11 @@ pub async fn create(
message: format!("base_domain: {error}"),
}
})?;
+ let region =
+ grass_validator::normalize_region(&body.region).map_err(|error| AppError::Validation {
+ op: OP,
+ message: format!("region: {error}"),
+ })?;
let provider = body
.provider
@@ -199,6 +211,7 @@ pub async fn create(
kind,
label: body.label.trim().to_owned(),
base_domain,
+ region,
enabled: body.enabled,
allows_auto_assign: body.allows_auto_assign,
is_default: body.is_default,
@@ -217,6 +230,8 @@ pub struct UpdateHostSourceRequest {
#[serde(default)]
pub label: Option,
#[serde(default)]
+ pub region: Option,
+ #[serde(default)]
pub enabled: Option,
#[serde(default)]
pub allows_auto_assign: Option,
@@ -248,6 +263,14 @@ pub async fn update(
let provider_patch = body.provider.map(|provider| {
Some(provider.trim().to_ascii_lowercase()).filter(|provider| !provider.is_empty())
});
+ let region = body
+ .region
+ .map(|value| grass_validator::normalize_region(&value))
+ .transpose()
+ .map_err(|error| AppError::Validation {
+ op: OP,
+ message: format!("region: {error}"),
+ })?;
let config_patch = body
.config
.map(|patch| merge_config(&source.config, patch, OP))
@@ -267,6 +290,7 @@ pub async fn update(
source,
UpdateHostSourceParams {
label: body.label.filter(|label| !label.trim().is_empty()),
+ region,
enabled: body.enabled,
allows_auto_assign: body.allows_auto_assign,
is_default: body.is_default,
diff --git a/apps/control-api/src/features/api/v1/admin/nodes.rs b/apps/control-api/src/features/api/v1/admin/nodes.rs
index 34ab006..c993384 100644
--- a/apps/control-api/src/features/api/v1/admin/nodes.rs
+++ b/apps/control-api/src/features/api/v1/admin/nodes.rs
@@ -76,6 +76,7 @@ fn node_view(
"build_enabled": node.build_enabled,
"serve_enabled": node.serve_enabled,
"build_concurrency": node.build_concurrency,
+ "region": node.region,
"base_url": node.base_url,
"work_root": node.work_root,
"version": node.metadata.get("version"),
@@ -328,6 +329,9 @@ fn validate_node_configuration(
if identity.id.trim().is_empty() || identity.id.chars().count() > 120 {
return Err("node id must contain between 1 and 120 characters".to_owned());
}
+ if grass_validator::normalize_region(&identity.region).is_err() {
+ return Err("node region is invalid".to_owned());
+ }
if !validate_http_url(identity.control_api.trim()) {
return Err("control API must be an absolute HTTP(S) URL without credentials".to_owned());
}
@@ -699,6 +703,8 @@ pub async fn health(
#[derive(Deserialize)]
pub struct CreateNodeRequest {
pub name: String,
+ #[serde(default)]
+ pub region: Option,
/// Generate the local node config and start the managed process.
#[serde(default)]
pub start_local: bool,
@@ -722,11 +728,19 @@ pub async fn create(
});
}
+ let region = body.region.as_deref().unwrap_or("default");
+ let region =
+ grass_validator::normalize_region(region).map_err(|error| AppError::Validation {
+ op: OP,
+ message: format!("region: {error}"),
+ })?;
+
let token = grass_token::generate_token();
let node = nodes::create_node(
db,
CreateNodeParams {
name: body.name.trim().to_owned(),
+ region: region.clone(),
token_hash: grass_token::hash_token(&token),
storage_root: None,
},
@@ -771,6 +785,7 @@ pub async fn create(
&config_path,
&config_file::GenerateParams {
node_name: &node.name,
+ region: ®ion,
node_token: &token,
control_api_url,
storage_root: &storage_root,
@@ -1002,6 +1017,7 @@ mod tests {
node::Model {
id: Uuid::nil(),
name: "serve-node-1".to_owned(),
+ region: "default".to_owned(),
token_hash: String::new(),
status: NodeStatus::Active,
build_enabled: false,
diff --git a/apps/control-api/src/features/api/v1/admin/projects.rs b/apps/control-api/src/features/api/v1/admin/projects.rs
index 449390f..c3fa500 100644
--- a/apps/control-api/src/features/api/v1/admin/projects.rs
+++ b/apps/control-api/src/features/api/v1/admin/projects.rs
@@ -473,6 +473,7 @@ fn admin_binding_view(binding: &project_host_binding::Model) -> serde_json::Valu
"id": binding.id,
"project_id": binding.project_id,
"host": binding.host,
+ "region": binding.region,
"kind": match binding.kind { HostBindingKind::Platform => "platform", HostBindingKind::Custom => "custom" },
"environment": match binding.environment { crate::infra::database::entity::HostBindingEnvironment::Production => "production", crate::infra::database::entity::HostBindingEnvironment::Preview => "preview", crate::infra::database::entity::HostBindingEnvironment::All => "all" },
"status": match binding.status { HostBindingStatus::Pending => "pending", HostBindingStatus::Active => "active", HostBindingStatus::Failed => "failed", HostBindingStatus::Disabled => "disabled" },
diff --git a/apps/control-api/src/features/api/v1/admin/regional_ingresses.rs b/apps/control-api/src/features/api/v1/admin/regional_ingresses.rs
new file mode 100644
index 0000000..a49e92a
--- /dev/null
+++ b/apps/control-api/src/features/api/v1/admin/regional_ingresses.rs
@@ -0,0 +1,725 @@
+use axum::{
+ Json,
+ extract::{Path, State},
+ response::IntoResponse,
+};
+use sea_orm::{
+ ActiveModelTrait, ActiveValue::Set, ColumnTrait, EntityTrait, QueryFilter, QuerySelect,
+ TransactionTrait,
+};
+use serde::Deserialize;
+use serde_json::json;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use crate::infra::http::timestamps::ts;
+use crate::{
+ domain::{certificates, ingress},
+ infra::{
+ database::{
+ self,
+ entity::{
+ managed_certificate, node, node_ingress_status, regional_ingress,
+ regional_ingress_health,
+ },
+ },
+ error::{AppError, ok_response},
+ },
+ state::ControlApiState,
+};
+
+fn ingress_view(item: ®ional_ingress::Model) -> serde_json::Value {
+ json!({
+ "id": item.id,
+ "region": item.region,
+ "hostname": item.hostname,
+ "enabled": item.enabled,
+ "health_check_path": item.health_check_path,
+ "health_check_interval_seconds": item.health_check_interval_seconds,
+ "origin_host_preservation": item.origin_host_preservation,
+ "tls_enabled": item.tls_enabled,
+ "certificate_issuer": item.certificate_issuer,
+ "certificate_auto_renew": item.certificate_auto_renew,
+ "certificate_status": item.certificate_status,
+ "certificate_expires_at": ts(item.certificate_expires_at),
+ "certificate_issued_at": ts(item.certificate_issued_at),
+ "certificate_error": item.certificate_error,
+ "dns_challenge_provider": item.dns_challenge_provider,
+ "dns_challenge_config_keys": [],
+ "dns_challenge_status": item.dns_challenge_status,
+ "dns_challenge_record_name": item.dns_challenge_record_name,
+ "dns_challenge_record_value": item.dns_challenge_record_value,
+ "healthy_nodes": [],
+ "deleted_at": ts(item.deleted_at),
+ "created_at": ts(item.created_at),
+ "updated_at": ts(item.updated_at),
+ })
+}
+
+async fn detailed_view(
+ db: &sea_orm::DatabaseConnection,
+ item: ®ional_ingress::Model,
+ secret: &str,
+) -> anyhow::Result {
+ let mut view = ingress_view(item);
+ view["dns_challenge_config_keys"] = json!(
+ certificates::config(item, secret)?
+ .as_object()
+ .map(|o| o.keys().cloned().collect::>())
+ .unwrap_or_default()
+ );
+ let cert = managed_certificate::Entity::find_by_id(item.id)
+ .one(db)
+ .await?;
+ view["certificate_revision"] = json!(
+ cert.as_ref()
+ .map(|c| c.revision.as_str())
+ .unwrap_or_default()
+ );
+ view["certificate_retry_at"] = ts(cert.as_ref().and_then(|c| c.retry_at));
+ if let Some(cert) = &cert {
+ view["certificate_status"] = json!(cert.status);
+ view["certificate_error"] = json!(cert.error);
+ view["certificate_expires_at"] = ts(cert.expires_at);
+ view["certificate_issued_at"] = ts(cert.issued_at);
+ }
+ let nodes = node::Entity::find()
+ .filter(node::Column::Region.eq(&item.region))
+ .filter(node::Column::ServeEnabled.eq(true))
+ .filter(node::Column::DeletedAt.is_null())
+ .all(db)
+ .await?;
+ let mut statuses = Vec::new();
+ for node in nodes {
+ let status = node_ingress_status::Entity::find_by_id(node.id)
+ .one(db)
+ .await?;
+ let health = regional_ingress_health::Entity::find_by_id((item.id, node.id))
+ .one(db)
+ .await?;
+ let installed = status
+ .as_ref()
+ .and_then(|s| s.certificates.as_array())
+ .and_then(|rows| {
+ rows.iter()
+ .find(|c| c["ingress_id"].as_str() == Some(&item.id.to_string()))
+ })
+ .and_then(|c| c["revision"].as_str());
+ statuses.push(json!({"node_id":node.id,"tls_ready":status.as_ref().is_some_and(|s|s.tls_ready),"challenge_revision":status.as_ref().map(|s|s.challenge_revision.as_str()).unwrap_or_default(),"checked_at":ts(status.as_ref().map(|s|s.checked_at)),"certificate_revision":installed,"health_status":health.as_ref().map(|h|h.status.as_str()).unwrap_or("unknown"),"health_checked_at":ts(health.as_ref().and_then(|h|h.checked_at)),"health_error":health.as_ref().and_then(|h|h.error.as_deref())}));
+ }
+ view["node_statuses"] = json!(statuses);
+ if !item.enabled || !item.tls_enabled {
+ view["certificate_status"] = json!("disabled");
+ }
+ Ok(view)
+}
+
+fn parse_issuer(value: &str, op: &'static str) -> Result {
+ let normalized = value.trim().to_ascii_lowercase();
+ if matches!(normalized.as_str(), "letsencrypt" | "zerossl" | "manual") {
+ Ok(normalized)
+ } else {
+ Err(AppError::Validation {
+ op,
+ message: "certificate_issuer must be letsencrypt, zerossl, or manual".to_owned(),
+ })
+ }
+}
+
+fn validate_challenge(
+ provider: Option<&str>,
+ config: &serde_json::Value,
+ op: &'static str,
+) -> Result, AppError> {
+ let provider = provider
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ .map(str::to_ascii_lowercase);
+ if provider
+ .as_ref()
+ .is_some_and(|p| !matches!(p.as_str(), "cloudflare" | "dnspod" | "route53"))
+ {
+ return Err(AppError::Validation {
+ op,
+ message: "unsupported DNS challenge provider".to_owned(),
+ });
+ }
+ if provider.is_some() && !config.is_object() {
+ return Err(AppError::Validation {
+ op,
+ message: "dns_challenge_config must be a JSON object".to_owned(),
+ });
+ }
+ let required: &[&str] = match provider.as_deref() {
+ Some("cloudflare") => &["api_token", "zone_id"],
+ Some("dnspod") => &["secret_id", "secret_key", "domain"],
+ Some("route53") => &["access_key_id", "secret_access_key", "hosted_zone_id"],
+ _ => &[],
+ };
+ for field in required {
+ if config
+ .get(*field)
+ .and_then(serde_json::Value::as_str)
+ .is_none_or(|s| s.trim().is_empty())
+ {
+ return Err(AppError::Validation {
+ op,
+ message: format!("dns_challenge_config.{field} is required"),
+ });
+ }
+ }
+ Ok(provider)
+}
+
+fn validate_health_check(path: &str, interval: i32, op: &'static str) -> Result {
+ let path = path.trim();
+ if !path.starts_with('/')
+ || path.starts_with("//")
+ || path.contains(['?', '#', '\\'])
+ || path.len() > 512
+ {
+ return Err(AppError::Validation {
+ op,
+ message: "health_check_path must start with / and be at most 512 bytes".to_owned(),
+ });
+ }
+ if !(5..=3600).contains(&interval) {
+ return Err(AppError::Validation {
+ op,
+ message: "health_check_interval_seconds must be between 5 and 3600".to_owned(),
+ });
+ }
+ Ok(path.to_owned())
+}
+
+#[derive(Deserialize)]
+pub struct CreateRegionalIngressRequest {
+ pub region: String,
+ pub hostname: String,
+ #[serde(default = "default_true")]
+ pub enabled: bool,
+ #[serde(default = "default_health_path")]
+ pub health_check_path: String,
+ #[serde(default = "default_health_interval")]
+ pub health_check_interval_seconds: i32,
+ #[serde(default = "default_true")]
+ pub origin_host_preservation: bool,
+ #[serde(default = "default_true")]
+ pub tls_enabled: bool,
+ #[serde(default = "default_issuer")]
+ pub certificate_issuer: String,
+ #[serde(default = "default_true")]
+ pub certificate_auto_renew: bool,
+ #[serde(default)]
+ pub dns_challenge_provider: Option,
+ #[serde(default)]
+ pub dns_challenge_config: serde_json::Value,
+}
+
+const fn default_true() -> bool {
+ true
+}
+
+fn default_health_path() -> String {
+ "/_grass/health".to_owned()
+}
+
+const fn default_health_interval() -> i32 {
+ 30
+}
+
+fn default_issuer() -> String {
+ "letsencrypt".to_owned()
+}
+
+/// GET /api/v1/admin/regional-ingresses
+pub async fn list(State(state): State) -> Result {
+ const OP: &str = "admin.regional_ingresses.list";
+ let db = super::database(&state, OP)?;
+ let items = ingress::list(db)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ let now = OffsetDateTime::now_utc();
+ let mut views = Vec::with_capacity(items.len());
+ for item in &items {
+ let candidates = ingress::healthy_serve_nodes(db, &item.region, now)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ let secret = state.config.read().unwrap().secrets.secret_key.clone();
+ let mut view = detailed_view(db, item, &secret)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ view["healthy_nodes"] = json!(
+ candidates
+ .iter()
+ .map(|candidate| json!({
+ "node_id": candidate.node_id,
+ "base_url": candidate.base_url,
+ "priority": candidate.priority,
+ }))
+ .collect::>()
+ );
+ views.push(view);
+ }
+ Ok(ok_response(json!({ "regional_ingresses": views })))
+}
+
+/// POST /api/v1/admin/regional-ingresses
+pub async fn create(
+ State(state): State,
+ Json(body): Json,
+) -> Result {
+ const OP: &str = "admin.regional_ingresses.create";
+ let db = super::database(&state, OP)?;
+ let region =
+ grass_validator::normalize_region(&body.region).map_err(|error| AppError::Validation {
+ op: OP,
+ message: format!("region: {error}"),
+ })?;
+ let hostname =
+ grass_validator::normalize_host(&body.hostname).map_err(|error| AppError::Validation {
+ op: OP,
+ message: format!("hostname: {error}"),
+ })?;
+ let health_check_path = validate_health_check(
+ &body.health_check_path,
+ body.health_check_interval_seconds,
+ OP,
+ )?;
+ let certificate_issuer = parse_issuer(&body.certificate_issuer, OP)?;
+ let dns_challenge_config = if body.dns_challenge_config.is_null() {
+ json!({})
+ } else {
+ body.dns_challenge_config
+ };
+ let dns_challenge_provider = validate_challenge(
+ body.dns_challenge_provider.as_deref(),
+ &dns_challenge_config,
+ OP,
+ )?;
+ let now = OffsetDateTime::now_utc();
+ let id = Uuid::now_v7();
+ let secret = state.config.read().unwrap().secrets.secret_key.clone();
+ let dns_challenge_config = certificates::seal_config(id, &dns_challenge_config, &secret)
+ .map_err(|source| AppError::Validation {
+ op: OP,
+ message: source.to_string(),
+ })?;
+ let item = regional_ingress::ActiveModel {
+ id: Set(id),
+ region: Set(region),
+ hostname: Set(hostname),
+ enabled: Set(body.enabled),
+ health_check_path: Set(health_check_path),
+ health_check_interval_seconds: Set(body.health_check_interval_seconds),
+ origin_host_preservation: Set(body.origin_host_preservation),
+ tls_enabled: Set(body.tls_enabled),
+ certificate_issuer: Set(certificate_issuer),
+ certificate_auto_renew: Set(body.certificate_auto_renew),
+ certificate_status: Set(if body.tls_enabled {
+ "pending"
+ } else {
+ "disabled"
+ }
+ .to_owned()),
+ certificate_expires_at: Set(None),
+ certificate_error: Set(None),
+ acme_account: Set(None),
+ certificate_bundle: Set(None),
+ certificate_issued_at: Set(None),
+ dns_challenge_provider: Set(dns_challenge_provider.clone()),
+ dns_challenge_config: Set(dns_challenge_config),
+ dns_challenge_status: Set(if dns_challenge_provider.is_some() {
+ "pending"
+ } else {
+ "not_configured"
+ }
+ .to_owned()),
+ dns_challenge_record_name: Set(None),
+ dns_challenge_record_value: Set(None),
+ deleted_at: Set(None),
+ created_at: Set(now),
+ updated_at: Set(now),
+ }
+ .insert(db)
+ .await
+ .map_err(|source| {
+ let source: anyhow::Error = source.into();
+ if database::is_unique_violation(&source) {
+ AppError::Conflict {
+ op: OP,
+ message: "an active regional ingress already uses this region or hostname"
+ .to_owned(),
+ }
+ } else {
+ AppError::Infrastructure { op: OP, source }
+ }
+ })?;
+ Ok(ok_response(
+ json!({ "regional_ingress": detailed_view(db,&item,&secret).await.map_err(|source| AppError::Infrastructure {op:OP,source})? }),
+ ))
+}
+
+#[derive(Deserialize)]
+pub struct UpdateRegionalIngressRequest {
+ #[serde(default)]
+ pub hostname: Option,
+ #[serde(default)]
+ pub enabled: Option,
+ #[serde(default)]
+ pub health_check_path: Option,
+ #[serde(default)]
+ pub health_check_interval_seconds: Option,
+ #[serde(default)]
+ pub origin_host_preservation: Option,
+ #[serde(default)]
+ pub tls_enabled: Option,
+ #[serde(default)]
+ pub certificate_issuer: Option,
+ #[serde(default)]
+ pub certificate_auto_renew: Option,
+ #[serde(default)]
+ pub dns_challenge_provider: Option,
+ #[serde(default)]
+ pub dns_challenge_config: Option,
+}
+
+/// PATCH /api/v1/admin/regional-ingresses/{ingress_id}
+pub async fn update(
+ State(state): State,
+ Path(ingress_id): Path,
+ Json(body): Json,
+) -> Result {
+ const OP: &str = "admin.regional_ingresses.update";
+ let db = super::database(&state, OP)?;
+ let transaction = db
+ .begin()
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ let item = regional_ingress::Entity::find_by_id(ingress_id)
+ .filter(regional_ingress::Column::DeletedAt.is_null())
+ .lock_exclusive()
+ .one(&transaction)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?
+ .ok_or_else(|| AppError::NotFound {
+ op: OP,
+ message: "regional ingress not found".to_owned(),
+ })?;
+ let mut active: regional_ingress::ActiveModel = item.clone().into();
+ let old_issuer = item.certificate_issuer.clone();
+ let configuration_changed =
+ body.dns_challenge_config.is_some() || body.dns_challenge_provider.is_some();
+ let secret = state.config.read().unwrap().secrets.secret_key.clone();
+ let records = managed_certificate::Entity::find()
+ .filter(managed_certificate::Column::IngressId.eq(item.id))
+ .lock_exclusive()
+ .all(&transaction)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ if (body.hostname.is_some()
+ || body.certificate_issuer.is_some()
+ || body.dns_challenge_config.is_some()
+ || body.dns_challenge_provider.is_some())
+ && records.iter().any(|record| {
+ record
+ .lease_until
+ .is_some_and(|until| until > OffsetDateTime::now_utc())
+ })
+ {
+ return Err(AppError::Conflict{op:OP,message:"wait for certificate issuance to finish before changing hostname, issuer or DNS credentials".to_owned()});
+ }
+ let mut reset_certificate = false;
+ if let Some(hostname) = body.hostname {
+ reset_certificate = hostname.trim() != item.hostname;
+ active.hostname = Set(grass_validator::normalize_host(&hostname).map_err(|error| {
+ AppError::Validation {
+ op: OP,
+ message: format!("hostname: {error}"),
+ }
+ })?);
+ }
+ let has_health_check_path = body.health_check_path.is_some();
+ let has_health_check_interval = body.health_check_interval_seconds.is_some();
+ let next_interval = body
+ .health_check_interval_seconds
+ .unwrap_or(item.health_check_interval_seconds);
+ if let Some(path) = body.health_check_path.as_deref() {
+ active.health_check_path = Set(validate_health_check(path, next_interval, OP)?);
+ } else if has_health_check_interval {
+ validate_health_check(&item.health_check_path, next_interval, OP)?;
+ active.health_check_interval_seconds = Set(next_interval);
+ }
+ if has_health_check_path && has_health_check_interval {
+ active.health_check_interval_seconds = Set(next_interval);
+ }
+ if let Some(value) = body.enabled {
+ active.enabled = Set(value);
+ active.certificate_status = Set(if value && item.tls_enabled {
+ "pending"
+ } else {
+ "disabled"
+ }
+ .to_owned());
+ }
+ if let Some(value) = body.origin_host_preservation {
+ active.origin_host_preservation = Set(value);
+ }
+ if let Some(value) = body.tls_enabled {
+ active.tls_enabled = Set(value);
+ active.certificate_status = Set(if value && item.enabled {
+ "pending"
+ } else {
+ "disabled"
+ }
+ .to_owned());
+ }
+ if let Some(value) = body.certificate_issuer {
+ let issuer = parse_issuer(&value, OP)?;
+ reset_certificate |= issuer != item.certificate_issuer;
+ active.certificate_issuer = Set(issuer);
+ if reset_certificate {
+ active.acme_account = Set(None);
+ }
+ }
+ if let Some(value) = body.certificate_auto_renew {
+ active.certificate_auto_renew = Set(value);
+ }
+ let provider = body.dns_challenge_provider.map(Some);
+ let config = body.dns_challenge_config;
+ if provider.is_some() || config.is_some() {
+ let provider_value = provider
+ .flatten()
+ .or_else(|| item.dns_challenge_provider.clone());
+ let previous = certificates::config(&item, &secret)
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ let config_value = match config {
+ Some(patch) => certificates::merge_config(previous, &patch).map_err(|source| {
+ AppError::Validation {
+ op: OP,
+ message: source.to_string(),
+ }
+ })?,
+ None => previous,
+ };
+ let provider_value = validate_challenge(provider_value.as_deref(), &config_value, OP)?;
+ active.dns_challenge_provider = Set(provider_value.clone());
+ active.dns_challenge_config = Set(certificates::seal_config(
+ item.id,
+ &config_value,
+ &secret,
+ )
+ .map_err(|source| AppError::Validation {
+ op: OP,
+ message: source.to_string(),
+ })?);
+ active.dns_challenge_status = Set(if provider_value.is_some() {
+ "pending"
+ } else {
+ "not_configured"
+ }
+ .to_owned());
+ }
+ active.updated_at = Set(OffsetDateTime::now_utc());
+ let item = active.update(&transaction).await.map_err(|source| {
+ let source: anyhow::Error = source.into();
+ if database::is_unique_violation(&source) {
+ AppError::Conflict {
+ op: OP,
+ message: "an active regional ingress already uses this hostname".to_owned(),
+ }
+ } else {
+ AppError::Infrastructure { op: OP, source }
+ }
+ })?;
+ if reset_certificate || configuration_changed {
+ for record in records {
+ let mut cert: managed_certificate::ActiveModel = record.clone().into();
+ cert.generation = Set(Uuid::now_v7());
+ let reissue = reset_certificate
+ && (record.host_binding_id.is_none()
+ || (record.issuer != "manual" && old_issuer != item.certificate_issuer));
+ if !reissue {
+ cert.update(&transaction)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ continue;
+ }
+ cert.lease_until = Set(None);
+ cert.status = Set("pending".to_owned());
+ cert.retry_at = Set(None);
+ cert.failure_count = Set(0);
+ cert.acme_account = Set(None);
+ cert.issuer = Set(item.certificate_issuer.clone());
+ cert.challenge_token = Set(None);
+ cert.challenge_value = Set(None);
+ cert.challenge_expires_at = Set(None);
+ if record.host_binding_id.is_none() && record.hostname != item.hostname {
+ cert.hostname = Set(item.hostname.clone());
+ cert.bundle = Set(None);
+ cert.revision = Set(String::new());
+ cert.issued_at = Set(None);
+ cert.expires_at = Set(None);
+ }
+ cert.update(&transaction)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ }
+ }
+ transaction
+ .commit()
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ Ok(ok_response(
+ json!({ "regional_ingress": detailed_view(db,&item,&secret).await.map_err(|source|AppError::Infrastructure{op:OP,source})? }),
+ ))
+}
+
+/// DELETE /api/v1/admin/regional-ingresses/{ingress_id}
+pub async fn remove(
+ State(state): State,
+ Path(ingress_id): Path,
+) -> Result {
+ const OP: &str = "admin.regional_ingresses.remove";
+ let db = super::database(&state, OP)?;
+ let item = ingress::get_by_id(db, ingress_id)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?
+ .ok_or_else(|| AppError::NotFound {
+ op: OP,
+ message: "regional ingress not found".to_owned(),
+ })?;
+ let mut active: regional_ingress::ActiveModel = item.into();
+ active.enabled = Set(false);
+ active.certificate_status = Set("disabled".to_owned());
+ active.deleted_at = Set(Some(OffsetDateTime::now_utc()));
+ active.updated_at = Set(OffsetDateTime::now_utc());
+ active
+ .update(db)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ Ok(ok_response(json!({ "ok": true })))
+}
+
+/// POST /api/v1/admin/regional-ingresses/{ingress_id}/certificate/renew
+pub async fn renew_certificate(
+ State(state): State,
+ Path(ingress_id): Path,
+) -> Result {
+ const OP: &str = "admin.regional_ingresses.renew_certificate";
+ let db = super::database(&state, OP)?;
+ let item = ingress::get_by_id(db, ingress_id)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?
+ .ok_or_else(|| AppError::NotFound {
+ op: OP,
+ message: "regional ingress not found".to_owned(),
+ })?;
+ if !item.enabled || !item.tls_enabled {
+ return Err(AppError::Conflict {
+ op: OP,
+ message: "enable this ingress and TLS before renewing".to_owned(),
+ });
+ }
+ let cert = certificates::ensure_record(db, &item, None)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ certificates::queue(db, &cert)
+ .await
+ .map_err(|source| AppError::Conflict {
+ op: OP,
+ message: source.to_string(),
+ })?;
+ let refreshed = ingress::get_by_id(db, ingress_id)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?
+ .ok_or_else(|| AppError::NotFound {
+ op: OP,
+ message: "regional ingress was removed during renewal".to_owned(),
+ })?;
+ let secret = state.config.read().unwrap().secrets.secret_key.clone();
+ Ok(ok_response(
+ json!({ "regional_ingress": detailed_view(db,&refreshed,&secret).await.map_err(|source|AppError::Infrastructure{op:OP,source})? }),
+ ))
+}
+
+pub async fn import_certificate(
+ State(state): State,
+ Path(id): Path,
+ Json(body): Json,
+) -> Result {
+ const OP: &str = "admin.regional_ingresses.import_certificate";
+ let db = super::database(&state, OP)?;
+ let item = ingress::get_by_id(db, id)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?
+ .ok_or_else(|| AppError::NotFound {
+ op: OP,
+ message: "regional ingress not found".to_owned(),
+ })?;
+ let secret = state.config.read().unwrap().secrets.secret_key.clone();
+ let cert = certificates::ensure_record(db, &item, None)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ certificates::import(db, &cert, body, &secret)
+ .await
+ .map_err(|source| AppError::Validation {
+ op: OP,
+ message: source.to_string(),
+ })?;
+ let mut active: regional_ingress::ActiveModel = item.into();
+ active.certificate_issuer = Set("manual".to_owned());
+ active.certificate_auto_renew = Set(false);
+ let item = active
+ .update(db)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ Ok(ok_response(
+ json!({"regional_ingress":detailed_view(db,&item,&secret).await.map_err(|source|AppError::Infrastructure{op:OP,source})?}),
+ ))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{parse_issuer, validate_health_check};
+
+ #[test]
+ fn regional_ingress_validation_accepts_supported_controls() {
+ assert_eq!(parse_issuer(" ZeroSSL ", "test").unwrap(), "zerossl");
+ assert_eq!(
+ validate_health_check("/ready", 30, "test").unwrap(),
+ "/ready"
+ );
+ }
+
+ #[test]
+ fn regional_ingress_validation_rejects_unsafe_controls() {
+ assert!(parse_issuer("acme", "test").is_err());
+ assert!(validate_health_check("ready", 30, "test").is_err());
+ assert!(validate_health_check("/ready", 4, "test").is_err());
+ }
+}
diff --git a/apps/control-api/src/features/api/v1/internal.rs b/apps/control-api/src/features/api/v1/internal.rs
index 223f3c3..f692503 100644
--- a/apps/control-api/src/features/api/v1/internal.rs
+++ b/apps/control-api/src/features/api/v1/internal.rs
@@ -60,6 +60,8 @@ pub fn router(state: ControlApiState) -> Router {
post(serve::release_ssr_lease),
)
.route("/serve/routes", get(serve::routes))
+ .route("/serve/certificates", get(serve::certificates))
+ .route("/serve/ingress-status", post(serve::ingress_status))
.route("/serve/resolve-host", get(serve::resolve_host))
.route("/serve/preview/authorize", post(super::preview_auth::start))
.route(
diff --git a/apps/control-api/src/features/api/v1/internal/deployments.rs b/apps/control-api/src/features/api/v1/internal/deployments.rs
index 0df7888..5a51f3d 100644
--- a/apps/control-api/src/features/api/v1/internal/deployments.rs
+++ b/apps/control-api/src/features/api/v1/internal/deployments.rs
@@ -1479,6 +1479,7 @@ mod tests {
id: Uuid::now_v7(),
project_id,
team_id,
+ region: "default".to_owned(),
build_node_id: None,
serve_node_id: None,
environment: DeploymentEnvironment::Preview,
@@ -1580,6 +1581,7 @@ mod tests {
let authenticated_node = node::Model {
id: node_id,
name: "build-1".to_owned(),
+ region: "default".to_owned(),
token_hash: "unused".to_owned(),
status: NodeStatus::Active,
build_enabled: true,
diff --git a/apps/control-api/src/features/api/v1/internal/nodes.rs b/apps/control-api/src/features/api/v1/internal/nodes.rs
index ee8e3d2..9a11312 100644
--- a/apps/control-api/src/features/api/v1/internal/nodes.rs
+++ b/apps/control-api/src/features/api/v1/internal/nodes.rs
@@ -44,6 +44,12 @@ pub async fn register(
node,
RegisterNodeParams {
name,
+ region: grass_validator::normalize_region(&body.region).map_err(|error| {
+ AppError::Validation {
+ op: OP,
+ message: format!("region: {error}"),
+ }
+ })?,
version: body.version,
build_enabled: body.capabilities.build,
serve_enabled: body.capabilities.serve,
@@ -88,6 +94,9 @@ pub async fn register(
}
fn validate_registration(body: &RegisterRequest) -> Result<(), &'static str> {
+ if grass_validator::normalize_region(&body.region).is_err() {
+ return Err("region is invalid");
+ }
if !body.capabilities.build && !body.capabilities.serve {
return Err("node must enable build or serve");
}
@@ -159,6 +168,7 @@ mod tests {
name: "node-a".to_owned(),
version: "0.1.0".to_owned(),
capabilities: NodeCapabilities { build, serve },
+ region: "default".to_owned(),
build_concurrency: u16::from(build),
serve_base_url: serve.then(|| "http://node-a:8080".to_owned()),
resources: serve.then_some(grass_node_protocol::NodeResources {
diff --git a/apps/control-api/src/features/api/v1/internal/serve.rs b/apps/control-api/src/features/api/v1/internal/serve.rs
index 175baf0..4f6ffcb 100644
--- a/apps/control-api/src/features/api/v1/internal/serve.rs
+++ b/apps/control-api/src/features/api/v1/internal/serve.rs
@@ -6,9 +6,10 @@ use axum::{
response::IntoResponse,
};
use grass_node_protocol::{
- ReportServeStatusRequest, ReportServeStatusResponse, ReportedServeStatus, ResolveHostResponse,
- RouteSnapshotResponse, ServeAccess, ServeArtifact, ServeAssignment, ServeAssignmentStatus,
- ServeAssignmentsResponse, ServeResources, ServeRoute, SsrLeaseResponse,
+ ReportIngressStatusRequest, ReportServeStatusRequest, ReportServeStatusResponse,
+ ReportedServeStatus, ResolveHostResponse, RouteSnapshotResponse, ServeAccess, ServeArtifact,
+ ServeAssignment, ServeAssignmentStatus, ServeAssignmentsResponse, ServeResources, ServeRoute,
+ SsrLeaseResponse,
};
use sea_orm::{
ActiveModelTrait, ColumnTrait, Condition, EntityTrait, QueryFilter, QueryOrder, QuerySelect,
@@ -24,8 +25,8 @@ use crate::{
database::entity::{
DeploymentArtifactKind, DeploymentBuildStatus, DeploymentEnvironment,
DeploymentReleaseStatus, DeploymentServeStatus, HostBindingEnvironment,
- HostBindingStatus, NodeDeploymentMigrationStatus, NodeStatus, deployment,
- deployment_artifact, node, node_deployment_migration, project_host_binding,
+ HostBindingKind, HostBindingStatus, NodeDeploymentMigrationStatus, NodeStatus,
+ deployment, deployment_artifact, node, node_deployment_migration, project_host_binding,
},
error::{AppError, ok_response},
http::middlewares::node_auth::AuthenticatedNode,
@@ -70,12 +71,14 @@ fn route_revision(routes: &[ServeRoute]) -> String {
canonical.sort_by(|left, right| {
(
&left.host,
+ &left.region,
left.deployment_id,
left.target_node_id,
&left.target_base_url,
)
.cmp(&(
&right.host,
+ &right.region,
right.deployment_id,
right.target_node_id,
&right.target_base_url,
@@ -549,6 +552,11 @@ pub async fn routes(
})?;
let mut hosts_by_project = HashMap::>::new();
for binding in bindings {
+ if matches!(binding.kind, HostBindingKind::Custom)
+ && !crate::domain::certificates::binding_eligible(&binding)
+ {
+ continue;
+ }
hosts_by_project
.entry(binding.project_id)
.or_default()
@@ -622,6 +630,7 @@ pub async fn routes(
.into_iter()
.map(|(host, access)| ServeRoute {
host,
+ region: deployment.region.clone(),
deployment_id: deployment.id,
target_node_id: node_id,
target_base_url: target_base_url.clone(),
@@ -635,6 +644,77 @@ pub async fn routes(
Ok(ok_response(RouteSnapshotResponse { revision, routes }))
}
+/// GET /api/v1/internal/serve/certificates
+pub async fn certificates(
+ State(state): State,
+ Extension(AuthenticatedNode(node)): Extension,
+) -> Result {
+ const OP: &str = "internal.serve.certificates";
+ ensure_serve_node(&node, OP)?;
+ let db = super::database(&state, OP)?;
+ let platform_secret = state.config.read().unwrap().secrets.secret_key.clone();
+ let snapshot = crate::domain::certificates::snapshot(db, &node.region, &platform_secret)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ Ok(ok_response(snapshot))
+}
+
+/// POST /api/v1/internal/serve/ingress-status
+pub async fn ingress_status(
+ State(state): State,
+ Extension(AuthenticatedNode(node)): Extension,
+ Json(body): Json,
+) -> Result {
+ use crate::infra::database::entity::node_ingress_status as status;
+ use sea_orm::{Set, sea_query::OnConflict};
+ const OP: &str = "internal.serve.ingress_status";
+ ensure_serve_node(&node, OP)?;
+ let revision_valid = |s: &str| {
+ s.len() == 64
+ && s.bytes()
+ .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
+ };
+ if body.certificates.len() > 4096
+ || (!body.challenge_revision.is_empty() && !revision_valid(&body.challenge_revision))
+ || body
+ .certificates
+ .iter()
+ .any(|c| !revision_valid(&c.revision))
+ {
+ return Err(AppError::Validation {
+ op: OP,
+ message: "ingress status requires bounded SHA256 revisions".to_owned(),
+ });
+ }
+ let db = super::database(&state, OP)?;
+ status::Entity::insert(status::ActiveModel {
+ node_id: Set(node.id),
+ certificates: Set(
+ serde_json::to_value(body.certificates).expect("certificate status serializes")
+ ),
+ challenge_revision: Set(body.challenge_revision),
+ tls_ready: Set(body.tls_ready),
+ checked_at: Set(time::OffsetDateTime::now_utc()),
+ })
+ .on_conflict(
+ OnConflict::column(status::Column::NodeId)
+ .update_columns([
+ status::Column::Certificates,
+ status::Column::ChallengeRevision,
+ status::Column::TlsReady,
+ status::Column::CheckedAt,
+ ])
+ .to_owned(),
+ )
+ .exec(db)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ Ok(ok_response(serde_json::json!({"ok":true})))
+}
+
async fn artifact_available(
db: &sea_orm::DatabaseConnection,
deployment_id: Uuid,
@@ -680,6 +760,14 @@ pub async fn resolve_host(
message: "host binding is not active".to_owned(),
});
}
+ if matches!(binding.kind, HostBindingKind::Custom)
+ && !crate::domain::certificates::binding_eligible(&binding)
+ {
+ return Err(AppError::NotFound {
+ op: OP,
+ message: "host ownership or review is not approved".to_owned(),
+ });
+ }
if matches!(binding.environment, HostBindingEnvironment::Preview) {
return Err(AppError::NotFound {
op: OP,
@@ -769,6 +857,62 @@ mod tests {
route_revision, validate_status_report,
};
+ #[tokio::test]
+ async fn ingress_status_http_route_accepts_authenticated_node_and_rejects_unbounded_revisions()
+ {
+ use crate::{
+ infra::{
+ config::ControlApiConfig, database::entity::node_ingress_status,
+ http::middlewares::node_auth::AuthenticatedNode,
+ },
+ state::ControlApiState,
+ };
+ use axum::{
+ Extension, Router,
+ body::Body,
+ http::{Request, StatusCode},
+ routing::post,
+ };
+ use tower::ServiceExt;
+ let node = crate::domain::ingress::tests::node_fixture();
+ let state = ControlApiState::new(ControlApiConfig::default(), "unused-test-config");
+ let revision = "a".repeat(64);
+ let status = node_ingress_status::Model {
+ node_id: node.id,
+ certificates: serde_json::json!([]),
+ challenge_revision: revision.clone(),
+ tls_ready: true,
+ checked_at: time::OffsetDateTime::now_utc(),
+ };
+ state
+ .database
+ .set(
+ sea_orm::MockDatabase::new(sea_orm::DbBackend::Postgres)
+ .append_query_results([vec![status]])
+ .into_connection(),
+ )
+ .unwrap();
+ let app = Router::new()
+ .route("/ingress-status", post(super::ingress_status))
+ .layer(Extension(AuthenticatedNode(node)))
+ .with_state(state);
+ let request = |revision: String| {
+ Request::builder().method("POST").uri("/ingress-status").header("content-type","application/json").body(Body::from(serde_json::json!({"certificates":[],"challenge_revision":revision,"tls_ready":true}).to_string())).unwrap()
+ };
+ assert_eq!(
+ app.clone()
+ .oneshot(request("invalid".to_owned()))
+ .await
+ .unwrap()
+ .status(),
+ StatusCode::BAD_REQUEST
+ );
+ assert_eq!(
+ app.oneshot(request(revision)).await.unwrap().status(),
+ StatusCode::OK
+ );
+ }
+
#[test]
fn ready_shadow_assignments_remain_authorized_until_atomic_cutover() {
for status in [
@@ -867,6 +1011,7 @@ mod tests {
};
let first = ServeRoute {
host: "a.example.com".to_owned(),
+ region: "default".to_owned(),
deployment_id: Uuid::now_v7(),
target_node_id: Uuid::now_v7(),
target_base_url: "http://node-a:8080".to_owned(),
@@ -875,6 +1020,7 @@ mod tests {
};
let second = ServeRoute {
host: "b.example.com".to_owned(),
+ region: "default".to_owned(),
deployment_id: Uuid::now_v7(),
target_node_id: Uuid::now_v7(),
target_base_url: "http://node-b:8080".to_owned(),
diff --git a/apps/control-api/src/features/api/v1/projects.rs b/apps/control-api/src/features/api/v1/projects.rs
index 9d06b8c..9dc4d63 100644
--- a/apps/control-api/src/features/api/v1/projects.rs
+++ b/apps/control-api/src/features/api/v1/projects.rs
@@ -1,6 +1,7 @@
pub mod create;
pub mod deployments;
pub mod detail;
+pub mod host_certificates;
pub mod hosts;
pub mod lifecycle;
pub mod list;
@@ -64,6 +65,22 @@ pub fn router() -> Router {
"/projects/{project_id}/hosts/{host_id}/primary",
post(hosts::set_primary),
)
+ .route(
+ "/projects/{project_id}/hosts/{host_id}/verify",
+ post(hosts::verify),
+ )
+ .route(
+ "/projects/{project_id}/hosts/{host_id}/certificate",
+ get(host_certificates::get).patch(host_certificates::update),
+ )
+ .route(
+ "/projects/{project_id}/hosts/{host_id}/certificate/renew",
+ post(host_certificates::renew),
+ )
+ .route(
+ "/projects/{project_id}/hosts/{host_id}/certificate/import",
+ post(host_certificates::import),
+ )
.route(
"/projects/{project_id}/hosts/{host_id}/provision",
post(hosts::provision),
diff --git a/apps/control-api/src/features/api/v1/projects/create.rs b/apps/control-api/src/features/api/v1/projects/create.rs
index 4bfbdb0..33cdb9c 100644
--- a/apps/control-api/src/features/api/v1/projects/create.rs
+++ b/apps/control-api/src/features/api/v1/projects/create.rs
@@ -262,6 +262,7 @@ async fn auto_assign_host(
team,
source: Some(&source),
host,
+ region: source.region.clone(),
kind: HostBindingKind::Platform,
environment: HostBindingEnvironment::Production,
is_primary: true,
diff --git a/apps/control-api/src/features/api/v1/projects/deployments.rs b/apps/control-api/src/features/api/v1/projects/deployments.rs
index edca756..e8a37eb 100644
--- a/apps/control-api/src/features/api/v1/projects/deployments.rs
+++ b/apps/control-api/src/features/api/v1/projects/deployments.rs
@@ -194,6 +194,7 @@ pub(crate) fn deployment_view(
"id": deployment.id,
"project_id": deployment.project_id,
"team_id": deployment.team_id,
+ "region": deployment.region,
"build_node": node_view(deployment.build_node_id),
"serve_node": node_view(deployment.serve_node_id),
"environment": deployments::environment_value(&deployment.environment),
@@ -382,6 +383,14 @@ pub struct CreateDeploymentRequest {
pub commit_message: Option,
#[serde(default)]
pub serve_node_id: Option,
+ #[serde(default)]
+ pub region: Option,
+}
+
+#[derive(Debug, Default, Deserialize)]
+pub struct ServeNodesQuery {
+ #[serde(default)]
+ pub region: Option,
}
fn default_environment() -> String {
@@ -430,6 +439,7 @@ async fn create_placed_deployment(
db: &sea_orm::DatabaseConnection,
params: CreateDeploymentParams,
selected_node_id: Option,
+ region: Option<&str>,
op: &'static str,
) -> Result {
let requested = deployments::runtime_serve_resources(¶ms.project.runtime);
@@ -440,15 +450,22 @@ async fn create_placed_deployment(
op,
source: source.into(),
})?;
- let placement =
- match scheduler::place_deployment(&transaction, requested, selected_node_id).await {
- Ok(placement) => placement,
- Err(error) => {
- let error = map_schedule_error(error, op);
- let _ = transaction.rollback().await;
- return Err(error);
- }
- };
+ let placement = match scheduler::place_deployment_in_region(
+ &transaction,
+ requested,
+ selected_node_id,
+ region,
+ )
+ .await
+ {
+ Ok(placement) => placement,
+ Err(error) => {
+ let error = map_schedule_error(error, op);
+ let _ = transaction.rollback().await;
+ return Err(error);
+ }
+ };
+ let placement_for_event = placement.clone();
let deployment = match deployments::create_deployment(&transaction, params, placement).await {
Ok(deployment) => deployment,
Err(source) => {
@@ -456,7 +473,7 @@ async fn create_placed_deployment(
return Err(AppError::Infrastructure { op, source });
}
};
- let mode = match placement.mode {
+ let mode = match placement_for_event.mode {
PlacementMode::Automatic => "automatic",
PlacementMode::Manual => "manual",
};
@@ -467,9 +484,9 @@ async fn create_placed_deployment(
"deployment assigned to serve node",
json!({
"mode": mode,
- "serve_node_id": placement.node_id,
+ "serve_node_id": placement_for_event.node_id,
"resources": requested,
- "overcommitted": placement.overcommitted,
+ "overcommitted": placement_for_event.overcommitted,
}),
)
.await
@@ -518,6 +535,15 @@ pub async fn create(
});
}
let environment = parse_environment(&body.environment, OP)?;
+ let requested_region = body
+ .region
+ .as_deref()
+ .map(grass_validator::normalize_region)
+ .transpose()
+ .map_err(|error| AppError::Validation {
+ op: OP,
+ message: format!("region: {error}"),
+ })?;
let quota = QuotaService::new(db, cache);
let reservation = quota
@@ -575,6 +601,7 @@ pub async fn create(
source_credential_version_id,
},
body.serve_node_id,
+ requested_region.as_deref(),
OP,
)
.await
@@ -790,11 +817,21 @@ pub async fn serve_nodes(
State(state): State,
session: Session,
Path(project_id): Path,
+ Query(query): Query,
) -> Result {
const OP: &str = "deployments.serve_nodes";
let access = super::project_access(&state, &session, project_id, false, OP).await?;
let db = super::database(&state, OP)?;
let requested = deployments::runtime_serve_resources(&access.project.runtime);
+ let requested_region = query
+ .region
+ .as_deref()
+ .map(grass_validator::normalize_region)
+ .transpose()
+ .map_err(|error| AppError::Validation {
+ op: OP,
+ message: format!("region: {error}"),
+ })?;
let candidates = scheduler::eligible_candidates(db)
.await
.map_err(|error| map_schedule_error(error, OP))?;
@@ -821,6 +858,12 @@ pub async fn serve_nodes(
let views = candidates
.iter()
.filter_map(|candidate| {
+ if requested_region
+ .as_deref()
+ .is_some_and(|region| candidate.region != region)
+ {
+ return None;
+ }
let node = nodes.get(&candidate.node_id)?;
let placement = scheduler::choose_candidate(
std::slice::from_ref(candidate),
@@ -833,10 +876,11 @@ pub async fn serve_nodes(
Some(json!({
"id": node.id,
"name": node.name,
+ "region": candidate.region,
"healthy": true,
"capacity": candidate.capacity,
"usage": candidate.usage,
- "normal_available": placement.is_some_and(|placement| !placement.overcommitted),
+ "normal_available": placement.as_ref().is_some_and(|placement| !placement.overcommitted),
"schedulable": placement.is_some(),
"overflow_only": placement.is_some_and(|placement| placement.overcommitted),
"disk_available_mb": candidate.capacity.disk_mb.saturating_sub(candidate.usage.disk_mb),
@@ -1332,6 +1376,7 @@ pub async fn retry(
source_credential_version_id: source_deployment.source_credential_version_id,
},
None,
+ Some(&source_deployment.region),
OP,
)
.await
diff --git a/apps/control-api/src/features/api/v1/projects/host_certificates.rs b/apps/control-api/src/features/api/v1/projects/host_certificates.rs
new file mode 100644
index 0000000..55d4413
--- /dev/null
+++ b/apps/control-api/src/features/api/v1/projects/host_certificates.rs
@@ -0,0 +1,225 @@
+use axum::{
+ Json,
+ extract::{Path, State},
+ response::IntoResponse,
+};
+use sea_orm::{ColumnTrait, Condition, EntityTrait, QueryFilter, Set};
+use serde::Deserialize;
+use serde_json::json;
+use uuid::Uuid;
+
+use crate::{
+ domain::{certificates, ingress},
+ infra::{
+ database::entity::{managed_certificate, project_host_binding, regional_ingress},
+ error::{AppError, ok_response},
+ http::extractors::Session,
+ },
+ state::ControlApiState,
+};
+
+async fn binding_and_ingress(
+ state: &ControlApiState,
+ session: &Session,
+ project_id: Uuid,
+ host_id: Uuid,
+ write: bool,
+ op: &'static str,
+) -> Result<(project_host_binding::Model, regional_ingress::Model), AppError> {
+ let access = super::project_access(state, session, project_id, false, op).await?;
+ if write {
+ access.require_member(op)?;
+ }
+ let db = super::database(state, op)?;
+ let binding = super::hosts::load_binding(db, &access, host_id, op).await?;
+ if !matches!(
+ binding.kind,
+ crate::infra::database::entity::HostBindingKind::Custom
+ ) {
+ return Err(AppError::Conflict {
+ op,
+ message: "this endpoint manages custom domain certificates".to_owned(),
+ });
+ }
+ let ingress = ingress::get_enabled_by_region(db, &binding.region)
+ .await
+ .map_err(|source| AppError::Infrastructure { op, source })?
+ .ok_or_else(|| AppError::Conflict {
+ op,
+ message: "no enabled regional ingress is configured".to_owned(),
+ })?;
+ Ok((binding, ingress))
+}
+
+fn require_eligible(
+ binding: &project_host_binding::Model,
+ ingress: ®ional_ingress::Model,
+ op: &'static str,
+) -> Result<(), AppError> {
+ if !certificates::binding_eligible(binding) || !ingress.tls_enabled {
+ return Err(AppError::Conflict{op,message:"verify TXT ownership, complete domain review and enable the binding and regional TLS first".to_owned()});
+ }
+ Ok(())
+}
+
+pub async fn get(
+ State(state): State,
+ session: Session,
+ Path((project_id, host_id)): Path<(Uuid, Uuid)>,
+) -> Result {
+ const OP: &str = "projects.hosts.certificate.get";
+ let (binding, ingress) =
+ binding_and_ingress(&state, &session, project_id, host_id, false, OP).await?;
+ let db = super::database(&state, OP)?;
+ let item = managed_certificate::Entity::find_by_id(host_id)
+ .one(db)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ let mut view = certificates::view(item.as_ref(), &ingress);
+ view["dns_delegation_name"] = json!(format!("_acme-challenge.{}", binding.host));
+ view["dns_delegation_target"] = json!(format!(
+ "_acme-{}.{}",
+ binding.id.simple(),
+ ingress.hostname
+ ));
+ Ok(ok_response(json!({"certificate":view})))
+}
+
+#[derive(Deserialize)]
+pub struct UpdateRequest {
+ pub challenge_method: Option,
+ pub certificate_auto_renew: Option,
+ pub certificate_issuer: Option,
+}
+
+pub async fn update(
+ State(state): State,
+ session: Session,
+ Path((project_id, host_id)): Path<(Uuid, Uuid)>,
+ Json(body): Json,
+) -> Result {
+ const OP: &str = "projects.hosts.certificate.update";
+ let (binding, ingress) =
+ binding_and_ingress(&state, &session, project_id, host_id, true, OP).await?;
+ let db = super::database(&state, OP)?;
+ let item = certificates::ensure_record(db, &ingress, Some(&binding))
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ let mut active: managed_certificate::ActiveModel = item.clone().into();
+ let mut restart = false;
+ if let Some(method) = body.challenge_method {
+ if !matches!(method.as_str(), "http01" | "dns01") {
+ return Err(AppError::Validation {
+ op: OP,
+ message: "challenge_method must be http01 or dns01".to_owned(),
+ });
+ }
+ restart |= method != item.challenge_method;
+ active.challenge_method = Set(method);
+ }
+ if let Some(value) = body.certificate_auto_renew {
+ active.auto_renew = Set(value);
+ }
+ if let Some(issuer) = body.certificate_issuer {
+ if issuer != "manual" && issuer != ingress.certificate_issuer {
+ return Err(AppError::Validation {
+ op: OP,
+ message: "automatic issuer must match the regional ingress account".to_owned(),
+ });
+ }
+ restart |= issuer != item.issuer;
+ active.issuer = Set(issuer);
+ active.acme_account = Set(None);
+ }
+ if restart {
+ if item
+ .lease_until
+ .is_some_and(|until| until > time::OffsetDateTime::now_utc())
+ {
+ return Err(AppError::Conflict {
+ op: OP,
+ message: "wait for the in-progress certificate attempt before changing its method"
+ .to_owned(),
+ });
+ }
+ active.generation = Set(Uuid::now_v7());
+ active.status = Set("pending".to_owned());
+ active.retry_at = Set(None);
+ active.failure_count = Set(0);
+ }
+ active.generation = Set(Uuid::now_v7());
+ let item = managed_certificate::Entity::update(active)
+ .validate()
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?
+ .filter(managed_certificate::Column::Generation.eq(item.generation))
+ .filter(
+ Condition::any()
+ .add(managed_certificate::Column::LeaseUntil.is_null())
+ .add(managed_certificate::Column::LeaseUntil.lte(time::OffsetDateTime::now_utc())),
+ )
+ .exec(db)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ Ok(ok_response(
+ json!({"certificate":certificates::view(Some(&item),&ingress)}),
+ ))
+}
+
+pub async fn renew(
+ State(state): State,
+ session: Session,
+ Path((project_id, host_id)): Path<(Uuid, Uuid)>,
+) -> Result {
+ const OP: &str = "projects.hosts.certificate.renew";
+ let (binding, ingress) =
+ binding_and_ingress(&state, &session, project_id, host_id, true, OP).await?;
+ require_eligible(&binding, &ingress, OP)?;
+ let db = super::database(&state, OP)?;
+ let item = certificates::ensure_record(db, &ingress, Some(&binding))
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ let item = certificates::queue(db, &item)
+ .await
+ .map_err(|source| AppError::Conflict {
+ op: OP,
+ message: source.to_string(),
+ })?;
+ Ok(ok_response(
+ json!({"certificate":certificates::view(Some(&item),&ingress)}),
+ ))
+}
+
+pub async fn import(
+ State(state): State,
+ session: Session,
+ Path((project_id, host_id)): Path<(Uuid, Uuid)>,
+ Json(body): Json,
+) -> Result {
+ const OP: &str = "projects.hosts.certificate.import";
+ let (binding, ingress) =
+ binding_and_ingress(&state, &session, project_id, host_id, true, OP).await?;
+ require_eligible(&binding, &ingress, OP)?;
+ let db = super::database(&state, OP)?;
+ let secret = state.config.read().unwrap().secrets.secret_key.clone();
+ let item = certificates::ensure_record(db, &ingress, Some(&binding))
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ let item = certificates::import(db, &item, body, &secret)
+ .await
+ .map_err(|source| AppError::Validation {
+ op: OP,
+ message: source.to_string(),
+ })?;
+ Ok(ok_response(
+ json!({"certificate":certificates::view(Some(&item),&ingress)}),
+ ))
+}
diff --git a/apps/control-api/src/features/api/v1/projects/hosts.rs b/apps/control-api/src/features/api/v1/projects/hosts.rs
index 0c3a6ab..b93d621 100644
--- a/apps/control-api/src/features/api/v1/projects/hosts.rs
+++ b/apps/control-api/src/features/api/v1/projects/hosts.rs
@@ -3,15 +3,16 @@ use axum::{
extract::{Path, State},
response::IntoResponse,
};
-use sea_orm::TransactionTrait;
+use sea_orm::{ActiveModelTrait, EntityTrait, TransactionTrait};
use serde::Deserialize;
use serde_json::json;
+use time::OffsetDateTime;
use uuid::Uuid;
use crate::infra::http::timestamps::ts;
use crate::{
- domain::deployments,
domain::hosts::{self, DomainReviewMode},
+ domain::{certificates, deployments, ingress},
infra::{
database::entity::{
DeploymentEnvironment, HostBindingEnvironment, HostBindingKind, HostBindingStatus,
@@ -29,6 +30,7 @@ fn binding_view(binding: &project_host_binding::Model) -> serde_json::Value {
"id": binding.id,
"project_id": binding.project_id,
"host": binding.host,
+ "region": binding.region,
"kind": match binding.kind {
HostBindingKind::Platform => "platform",
HostBindingKind::Custom => "custom",
@@ -46,10 +48,101 @@ fn binding_view(binding: &project_host_binding::Model) -> serde_json::Value {
"reviewed_by_user_id": binding.reviewed_by_user_id,
"reviewed_at": binding.reviewed_at.map(ts),
"review_reason": binding.review_reason,
+ "ownership_status": binding.ownership_status,
+ "ownership_checked_at": binding.ownership_checked_at.map(ts),
+ "ownership_error": binding.ownership_error,
"created_at": ts(binding.created_at),
+ "ingress": serde_json::Value::Null,
})
}
+async fn attach_ingress_guidance(
+ state: &ControlApiState,
+ db: &sea_orm::DatabaseConnection,
+ binding: &project_host_binding::Model,
+ mut view: serde_json::Value,
+ op: &'static str,
+) -> Result {
+ if !matches!(binding.kind, HostBindingKind::Custom) {
+ return Ok(view);
+ }
+ let Some(regional_ingress) = ingress::get_enabled_by_region(db, &binding.region)
+ .await
+ .map_err(|source| AppError::Infrastructure { op, source })?
+ else {
+ return Ok(view);
+ };
+ let candidates = ingress::healthy_serve_nodes(db, &binding.region, OffsetDateTime::now_utc())
+ .await
+ .map_err(|source| AppError::Infrastructure { op, source })?;
+ let secret_key = state.config.read().unwrap().secrets.secret_key.clone();
+ let verification_value =
+ ingress::dns_verification_token(&secret_key, binding.id, &binding.host);
+ let guidance = ingress::cname_guidance(ingress::CnameGuidanceInput {
+ host: &binding.host,
+ region: &binding.region,
+ ingress_hostname: ®ional_ingress.hostname,
+ verification_name: &format!("_grass.{}", binding.host),
+ verification_value: &verification_value,
+ origin_host_preservation: regional_ingress.origin_host_preservation,
+ });
+ view["ingress"] = json!({
+ "region": guidance.region,
+ "cname": {
+ "record_type": guidance.record_type,
+ "name": guidance.name,
+ "target": guidance.target,
+ },
+ "txt": {
+ "record_type": "TXT",
+ "name": guidance.verification_name,
+ "value": guidance.verification_value,
+ },
+ "origin_host_preservation": guidance.origin_host_preservation,
+ "health_check": {
+ "path": regional_ingress.health_check_path,
+ "interval_seconds": regional_ingress.health_check_interval_seconds,
+ },
+ "entrance_nodes": candidates.iter().map(|candidate| json!({
+ "node_id": candidate.node_id,
+ "base_url": candidate.base_url,
+ "priority": candidate.priority,
+ })).collect::>(),
+ "certificate": {
+ "enabled": regional_ingress.tls_enabled,
+ "issuer": regional_ingress.certificate_issuer,
+ "auto_renew": regional_ingress.certificate_auto_renew,
+ "status": regional_ingress.certificate_status,
+ "expires_at": ts(regional_ingress.certificate_expires_at),
+ "error": regional_ingress.certificate_error,
+ },
+ "dns_challenge": {
+ "provider": regional_ingress.dns_challenge_provider,
+ "status": regional_ingress.dns_challenge_status,
+ "record_name": regional_ingress.dns_challenge_record_name,
+ "record_value": regional_ingress.dns_challenge_record_value,
+ },
+ });
+ let record =
+ crate::infra::database::entity::managed_certificate::Entity::find_by_id(binding.id)
+ .one(db)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op,
+ source: source.into(),
+ })?;
+ let mut certificate = certificates::view(record.as_ref(), ®ional_ingress);
+ certificate["dns_delegation_name"] = json!(format!("_acme-challenge.{}", binding.host));
+ certificate["dns_delegation_target"] = json!(format!(
+ "_acme-{}.{}",
+ binding.id.simple(),
+ regional_ingress.hostname
+ ));
+ view["certificate"] = certificate.clone();
+ view["ingress"]["certificate"] = certificate;
+ Ok(view)
+}
+
fn review_status_value(status: &HostReviewStatus) -> &'static str {
match status {
HostReviewStatus::NotRequired => "not_required",
@@ -107,7 +200,8 @@ pub async fn list(
let events = hosts::list_provision_events_for_binding(db, binding.id)
.await
.map_err(|source| AppError::Infrastructure { op: OP, source })?;
- let mut view = binding_view(binding);
+ let view = binding_view(binding);
+ let mut view = attach_ingress_guidance(&state, db, binding, view, OP).await?;
view["serving"] = json!(
matches!(binding.status, HostBindingStatus::Active)
&& match binding.environment {
@@ -144,6 +238,8 @@ pub async fn list(
#[derive(Deserialize)]
pub struct CreateHostRequest {
pub host: String,
+ #[serde(default)]
+ pub region: Option,
#[serde(default = "default_environment")]
pub environment: String,
#[serde(default)]
@@ -173,7 +269,6 @@ pub async fn create(
message: error.to_string(),
})?;
let environment = parse_environment(&body.environment, OP)?;
-
let source = match body.host_source_id {
Some(source_id) => Some(
hosts::get_source_by_id(db, source_id)
@@ -186,6 +281,29 @@ pub async fn create(
),
None => None,
};
+ let region = body
+ .region
+ .as_deref()
+ .map(grass_validator::normalize_region)
+ .transpose()
+ .map_err(|error| AppError::Validation {
+ op: OP,
+ message: format!("region: {error}"),
+ })?
+ .or_else(|| source.as_ref().map(|source| source.region.clone()))
+ .unwrap_or_else(|| "default".to_owned());
+
+ if let Some(source) = source.as_ref()
+ && region != source.region
+ {
+ return Err(AppError::Validation {
+ op: OP,
+ message: format!(
+ "region must match the host source region ({})",
+ source.region
+ ),
+ });
+ }
// Custom hosts require the team group policy to allow them; hosts under
// a platform source must live under that source's base domain.
@@ -243,6 +361,7 @@ pub async fn create(
team: &access.team,
source: source.as_ref(),
host,
+ region,
kind: if source.is_some() {
HostBindingKind::Platform
} else {
@@ -256,7 +375,8 @@ pub async fn create(
)
.await?;
- Ok(ok_response(json!({ "host": binding_view(&binding) })))
+ let view = attach_ingress_guidance(&state, db, &binding, binding_view(&binding), OP).await?;
+ Ok(ok_response(json!({ "host": view })))
}
#[derive(Deserialize)]
@@ -267,6 +387,89 @@ pub struct UpdateHostRequest {
pub status: Option,
}
+/// POST /api/v1/projects/{project_id}/hosts/{host_id}/verify
+pub async fn verify(
+ State(state): State,
+ session: Session,
+ Path((project_id, host_id)): Path<(Uuid, Uuid)>,
+) -> Result {
+ const OP: &str = "projects.hosts.verify";
+ let access = super::project_access(&state, &session, project_id, false, OP).await?;
+ access.require_member(OP)?;
+ let db = super::database(&state, OP)?;
+ let binding = load_binding(db, &access, host_id, OP).await?;
+ if !matches!(binding.kind, HostBindingKind::Custom) {
+ return Err(AppError::Conflict {
+ op: OP,
+ message: "platform domains do not require ownership verification".to_owned(),
+ });
+ }
+ let secret_key = state.config.read().unwrap().secrets.secret_key.clone();
+ let expected = ingress::dns_verification_token(&secret_key, binding.id, &binding.host);
+ let result = ingress::verify_dns_txt(&binding.host, &expected)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?;
+ let transaction = db
+ .begin()
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ let binding = hosts::get_binding_by_id_for_update(&transaction, host_id)
+ .await
+ .map_err(|source| AppError::Infrastructure { op: OP, source })?
+ .filter(|binding| binding.project_id == project_id)
+ .ok_or_else(|| AppError::NotFound {
+ op: OP,
+ message: "host binding no longer exists".to_owned(),
+ })?;
+ let mut active: project_host_binding::ActiveModel = binding.clone().into();
+ active.ownership_checked_at = sea_orm::ActiveValue::Set(Some(OffsetDateTime::now_utc()));
+ match result {
+ ingress::DnsVerification::Verified => {
+ active.ownership_status = sea_orm::ActiveValue::Set("verified".to_owned());
+ active.ownership_error = sea_orm::ActiveValue::Set(None);
+ if matches!(binding.review_status, HostReviewStatus::Approved) {
+ active.status = sea_orm::ActiveValue::Set(HostBindingStatus::Active);
+ }
+ }
+ ingress::DnsVerification::Missing => {
+ active.ownership_status = sea_orm::ActiveValue::Set("failed".to_owned());
+ active.ownership_error =
+ sea_orm::ActiveValue::Set(Some("TXT ownership record was not found".to_owned()));
+ active.status = sea_orm::ActiveValue::Set(HostBindingStatus::Pending);
+ }
+ ingress::DnsVerification::Mismatch => {
+ active.ownership_status = sea_orm::ActiveValue::Set("failed".to_owned());
+ active.ownership_error =
+ sea_orm::ActiveValue::Set(Some("TXT ownership record did not match".to_owned()));
+ active.status = sea_orm::ActiveValue::Set(HostBindingStatus::Pending);
+ }
+ }
+ if matches!(binding.status, HostBindingStatus::Disabled) {
+ active.status = sea_orm::ActiveValue::Set(HostBindingStatus::Disabled);
+ }
+ let updated = active
+ .update(&transaction)
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ transaction
+ .commit()
+ .await
+ .map_err(|source| AppError::Infrastructure {
+ op: OP,
+ source: source.into(),
+ })?;
+ let view = attach_ingress_guidance(&state, db, &updated, binding_view(&updated), OP).await?;
+ Ok(ok_response(
+ json!({ "host": view, "verified": result == ingress::DnsVerification::Verified }),
+ ))
+}
+
/// PATCH /api/v1/projects/{project_id}/hosts/{host_id}
pub async fn update(
State(state): State,
@@ -308,7 +511,8 @@ pub async fn update(
source: source.into(),
})?;
- Ok(ok_response(json!({ "host": binding_view(&binding) })))
+ let view = attach_ingress_guidance(&state, db, &binding, binding_view(&binding), OP).await?;
+ Ok(ok_response(json!({ "host": view })))
}
/// DELETE /api/v1/projects/{project_id}/hosts/{host_id}
@@ -442,10 +646,11 @@ pub async fn provision(
let service = HostBindingService::new(db, cache);
let binding = service.provision(OP, binding, &source).await?;
- Ok(ok_response(json!({ "host": binding_view(&binding) })))
+ let view = attach_ingress_guidance(&state, db, &binding, binding_view(&binding), OP).await?;
+ Ok(ok_response(json!({ "host": view })))
}
-async fn load_binding(
+pub(super) async fn load_binding(
db: &sea_orm::DatabaseConnection,
access: &super::ProjectAccess,
host_id: Uuid,
@@ -482,6 +687,7 @@ mod tests {
team_id: Uuid::now_v7(),
host_source_id: None,
host: "manual.example.test".to_owned(),
+ region: "default".to_owned(),
kind: HostBindingKind::Custom,
environment: HostBindingEnvironment::Production,
status: HostBindingStatus::Pending,
@@ -491,6 +697,9 @@ mod tests {
reviewed_by_user_id: Some(reviewer_id),
reviewed_at: Some(OffsetDateTime::UNIX_EPOCH),
review_reason: Some("Ownership could not be verified".to_owned()),
+ ownership_status: "pending".to_owned(),
+ ownership_checked_at: None,
+ ownership_error: None,
deleted_at: None,
created_at: OffsetDateTime::UNIX_EPOCH,
updated_at: OffsetDateTime::UNIX_EPOCH,
diff --git a/apps/control-api/src/features/api/v1/projects/lifecycle.rs b/apps/control-api/src/features/api/v1/projects/lifecycle.rs
index 86580f2..a4ad7fc 100644
--- a/apps/control-api/src/features/api/v1/projects/lifecycle.rs
+++ b/apps/control-api/src/features/api/v1/projects/lifecycle.rs
@@ -344,6 +344,7 @@ mod tests {
team_id,
host_source_id: None,
host: "example.invalid".to_owned(),
+ region: "default".to_owned(),
kind: crate::infra::database::entity::HostBindingKind::Custom,
environment: crate::infra::database::entity::HostBindingEnvironment::Preview,
status: crate::infra::database::entity::HostBindingStatus::Active,
@@ -353,6 +354,9 @@ mod tests {
reviewed_by_user_id: None,
reviewed_at: None,
review_reason: None,
+ ownership_status: "pending".to_owned(),
+ ownership_checked_at: None,
+ ownership_error: None,
deleted_at: Some(now),
created_at: now,
updated_at: now,
@@ -440,6 +444,7 @@ mod tests {
team_id,
host_source_id: None,
host: "retry.example.invalid".to_owned(),
+ region: "default".to_owned(),
kind: crate::infra::database::entity::HostBindingKind::Custom,
environment: crate::infra::database::entity::HostBindingEnvironment::Preview,
status: crate::infra::database::entity::HostBindingStatus::Active,
@@ -449,6 +454,9 @@ mod tests {
reviewed_by_user_id: None,
reviewed_at: None,
review_reason: None,
+ ownership_status: "pending".to_owned(),
+ ownership_checked_at: None,
+ ownership_error: None,
deleted_at: Some(tombstone_at),
created_at: time::OffsetDateTime::UNIX_EPOCH,
updated_at: tombstone_at,
@@ -628,6 +636,7 @@ mod tests {
team_id,
host_source_id: None,
host: format!("{id}.example.invalid"),
+ region: "default".to_owned(),
kind: crate::infra::database::entity::HostBindingKind::Custom,
environment: crate::infra::database::entity::HostBindingEnvironment::Preview,
status: crate::infra::database::entity::HostBindingStatus::Active,
@@ -637,6 +646,9 @@ mod tests {
reviewed_by_user_id: None,
reviewed_at: None,
review_reason: None,
+ ownership_status: "pending".to_owned(),
+ ownership_checked_at: None,
+ ownership_error: None,
deleted_at: Some(deleted_at),
created_at: time::OffsetDateTime::UNIX_EPOCH,
updated_at: deleted_at,
diff --git a/apps/control-api/src/features/api/v1/setup/node.rs b/apps/control-api/src/features/api/v1/setup/node.rs
index 7a3070a..1ab3b9c 100644
--- a/apps/control-api/src/features/api/v1/setup/node.rs
+++ b/apps/control-api/src/features/api/v1/setup/node.rs
@@ -64,6 +64,7 @@ pub async fn handler(
db,
CreateNodeParams {
name: name.to_owned(),
+ region: "default".to_owned(),
token_hash,
storage_root: storage_root.clone(),
},
@@ -103,6 +104,7 @@ pub async fn handler(
&config_path,
&crate::infra::node_manager::config_file::GenerateParams {
node_name: &node.name,
+ region: "default",
node_token: &token,
control_api_url,
storage_root: &storage_root,
diff --git a/apps/control-api/src/infra/database/entity/deployment.rs b/apps/control-api/src/infra/database/entity/deployment.rs
index 4d435fa..c8250e2 100644
--- a/apps/control-api/src/infra/database/entity/deployment.rs
+++ b/apps/control-api/src/infra/database/entity/deployment.rs
@@ -13,6 +13,7 @@ pub struct Model {
pub id: Uuid,
pub project_id: Uuid,
pub team_id: Uuid,
+ pub region: String,
pub build_node_id: Option,
pub serve_node_id: Option,
pub environment: DeploymentEnvironment,
diff --git a/apps/control-api/src/infra/database/entity/host_source.rs b/apps/control-api/src/infra/database/entity/host_source.rs
index 9da1af5..8bbbfbe 100644
--- a/apps/control-api/src/infra/database/entity/host_source.rs
+++ b/apps/control-api/src/infra/database/entity/host_source.rs
@@ -11,6 +11,7 @@ pub struct Model {
pub kind: HostSourceKind,
pub label: String,
pub base_domain: String,
+ pub region: String,
pub enabled: bool,
pub allows_auto_assign: bool,
pub is_default: bool,
diff --git a/apps/control-api/src/infra/database/entity/managed_certificate.rs b/apps/control-api/src/infra/database/entity/managed_certificate.rs
new file mode 100644
index 0000000..5a80c50
--- /dev/null
+++ b/apps/control-api/src/infra/database/entity/managed_certificate.rs
@@ -0,0 +1,36 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel)]
+#[sea_orm(table_name = "managed_certificates")]
+pub struct Model {
+ #[sea_orm(primary_key, auto_increment = false)]
+ pub id: Uuid,
+ pub ingress_id: Uuid,
+ pub host_binding_id: Option,
+ pub hostname: String,
+ pub issuer: String,
+ pub challenge_method: String,
+ pub auto_renew: bool,
+ pub status: String,
+ pub error: Option,
+ pub bundle: Option,
+ pub acme_account: Option,
+ pub revision: String,
+ pub issued_at: Option,
+ pub expires_at: Option,
+ pub retry_at: Option,
+ pub failure_count: i32,
+ pub lease_until: Option,
+ pub generation: Uuid,
+ pub challenge_token: Option,
+ pub challenge_value: Option,
+ pub challenge_expires_at: Option,
+ pub dns_record_name: Option,
+ pub dns_record_value: Option,
+ pub dns_cleanup: Option,
+ pub updated_at: TimeDateTimeWithTimeZone,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/apps/control-api/src/infra/database/entity/mod.rs b/apps/control-api/src/infra/database/entity/mod.rs
index 6e8446a..5142b9f 100644
--- a/apps/control-api/src/infra/database/entity/mod.rs
+++ b/apps/control-api/src/infra/database/entity/mod.rs
@@ -11,9 +11,11 @@ pub mod enums;
pub mod host_policy;
pub mod host_provision_event;
pub mod host_source;
+pub mod managed_certificate;
pub mod node;
pub mod node_deletion_job;
pub mod node_deployment_migration;
+pub mod node_ingress_status;
pub mod project;
pub mod project_host_binding;
pub mod project_source_credential;
@@ -21,6 +23,8 @@ pub mod quota_event;
pub mod quota_limit;
pub mod quota_plan;
pub mod quota_usage_counter;
+pub mod regional_ingress;
+pub mod regional_ingress_health;
pub mod registration_email_allowlist;
pub mod release;
pub mod source_credential;
@@ -91,6 +95,8 @@ pub use quota_plan::Entity as QuotaPlan;
#[allow(unused_imports)]
pub use quota_usage_counter::Entity as QuotaUsageCounter;
#[allow(unused_imports)]
+pub use regional_ingress::Entity as RegionalIngress;
+#[allow(unused_imports)]
pub use registration_email_allowlist::Entity as RegistrationEmailAllowlist;
#[allow(unused_imports)]
pub use release::Entity as Release;
diff --git a/apps/control-api/src/infra/database/entity/node.rs b/apps/control-api/src/infra/database/entity/node.rs
index 127a28b..04ec5b9 100644
--- a/apps/control-api/src/infra/database/entity/node.rs
+++ b/apps/control-api/src/infra/database/entity/node.rs
@@ -9,6 +9,7 @@ pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: Uuid,
pub name: String,
+ pub region: String,
pub token_hash: String,
pub status: NodeStatus,
pub build_enabled: bool,
diff --git a/apps/control-api/src/infra/database/entity/node_ingress_status.rs b/apps/control-api/src/infra/database/entity/node_ingress_status.rs
new file mode 100644
index 0000000..ff09d64
--- /dev/null
+++ b/apps/control-api/src/infra/database/entity/node_ingress_status.rs
@@ -0,0 +1,16 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel)]
+#[sea_orm(table_name = "node_ingress_status")]
+pub struct Model {
+ #[sea_orm(primary_key, auto_increment = false)]
+ pub node_id: Uuid,
+ pub certificates: Json,
+ pub challenge_revision: String,
+ pub tls_ready: bool,
+ pub checked_at: TimeDateTimeWithTimeZone,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/apps/control-api/src/infra/database/entity/project_host_binding.rs b/apps/control-api/src/infra/database/entity/project_host_binding.rs
index 6cd696e..268241d 100644
--- a/apps/control-api/src/infra/database/entity/project_host_binding.rs
+++ b/apps/control-api/src/infra/database/entity/project_host_binding.rs
@@ -12,6 +12,7 @@ pub struct Model {
pub team_id: Uuid,
pub host_source_id: Option,
pub host: String,
+ pub region: String,
pub kind: HostBindingKind,
pub environment: HostBindingEnvironment,
pub status: HostBindingStatus,
@@ -21,6 +22,9 @@ pub struct Model {
pub reviewed_by_user_id: Option,
pub reviewed_at: Option,
pub review_reason: Option,
+ pub ownership_status: String,
+ pub ownership_checked_at: Option,
+ pub ownership_error: Option,
pub deleted_at: Option,
pub created_at: TimeDateTimeWithTimeZone,
pub updated_at: TimeDateTimeWithTimeZone,
diff --git a/apps/control-api/src/infra/database/entity/regional_ingress.rs b/apps/control-api/src/infra/database/entity/regional_ingress.rs
new file mode 100644
index 0000000..9c414b5
--- /dev/null
+++ b/apps/control-api/src/infra/database/entity/regional_ingress.rs
@@ -0,0 +1,38 @@
+use sea_orm::entity::prelude::*;
+
+#[allow(dead_code)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel)]
+#[sea_orm(table_name = "regional_ingresses")]
+pub struct Model {
+ #[sea_orm(primary_key, auto_increment = false)]
+ pub id: Uuid,
+ pub region: String,
+ pub hostname: String,
+ pub enabled: bool,
+ pub health_check_path: String,
+ pub health_check_interval_seconds: i32,
+ pub origin_host_preservation: bool,
+ pub tls_enabled: bool,
+ pub certificate_issuer: String,
+ pub certificate_auto_renew: bool,
+ pub certificate_status: String,
+ pub certificate_expires_at: Option,
+ pub certificate_error: Option,
+ pub dns_challenge_provider: Option,
+ pub dns_challenge_config: Json,
+ pub dns_challenge_status: String,
+ pub dns_challenge_record_name: Option,
+ pub dns_challenge_record_value: Option,
+ pub acme_account: Option,
+ pub certificate_bundle: Option,
+ pub certificate_issued_at: Option,
+ pub deleted_at: Option,
+ pub created_at: TimeDateTimeWithTimeZone,
+ pub updated_at: TimeDateTimeWithTimeZone,
+}
+
+#[allow(dead_code)]
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/apps/control-api/src/infra/database/entity/regional_ingress_health.rs b/apps/control-api/src/infra/database/entity/regional_ingress_health.rs
new file mode 100644
index 0000000..f0316d8
--- /dev/null
+++ b/apps/control-api/src/infra/database/entity/regional_ingress_health.rs
@@ -0,0 +1,50 @@
+use sea_orm::entity::prelude::*;
+
+#[allow(dead_code)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "regional_ingress_health")]
+pub struct Model {
+ #[sea_orm(primary_key, auto_increment = false)]
+ pub ingress_id: Uuid,
+ #[sea_orm(primary_key, auto_increment = false)]
+ pub node_id: Uuid,
+ pub status: String,
+ pub checked_at: Option,
+ pub latency_ms: Option,
+ pub error: Option,
+}
+
+#[allow(dead_code)]
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+ #[sea_orm(
+ belongs_to = "super::regional_ingress::Entity",
+ from = "Column::IngressId",
+ to = "super::regional_ingress::Column::Id",
+ on_update = "NoAction",
+ on_delete = "Cascade"
+ )]
+ Ingress,
+ #[sea_orm(
+ belongs_to = "super::node::Entity",
+ from = "Column::NodeId",
+ to = "super::node::Column::Id",
+ on_update = "NoAction",
+ on_delete = "Cascade"
+ )]
+ Node,
+}
+
+impl Related for Entity {
+ fn to() -> RelationDef {
+ Relation::Ingress.def()
+ }
+}
+
+impl Related for Entity {
+ fn to() -> RelationDef {
+ Relation::Node.def()
+ }
+}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/apps/control-api/src/infra/database/migrate.rs b/apps/control-api/src/infra/database/migrate.rs
index 1fb0028..1569c89 100644
--- a/apps/control-api/src/infra/database/migrate.rs
+++ b/apps/control-api/src/infra/database/migrate.rs
@@ -40,6 +40,10 @@ impl MigratorTrait for Migrator {
Box::new(migration::m20260807_000026_avatars::Migration),
Box::new(migration::m20260807_000027_deployment_screenshots::Migration),
Box::new(migration::m20260808_000028_object_storage::Migration),
+ Box::new(migration::m20260908_000029_regional_routing::Migration),
+ Box::new(migration::m20260908_000030_regional_ingress::Migration),
+ Box::new(migration::m20260909_000031_regional_ingress_lifecycle::Migration),
+ Box::new(migration::m20260910_000032_managed_certificates::Migration),
]
}
}
@@ -150,7 +154,7 @@ mod tests {
fn registers_audit_foundation_migration() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(11).expect("twelfth migration").name(),
"m20260729_000012_audit_foundation"
@@ -179,7 +183,7 @@ mod tests {
fn registers_team_group_review_policy_migration() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(12).expect("thirteenth migration").name(),
"m20260729_000013_team_group_review_policy"
@@ -190,7 +194,7 @@ mod tests {
fn registers_node_config_sync_migration() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(13).expect("fourteenth migration").name(),
"m20260729_000014_node_config_sync"
@@ -201,7 +205,7 @@ mod tests {
fn registers_node_deletion_queue_migration() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(14).expect("fifteenth migration").name(),
"m20260729_000015_node_deletion_queue"
@@ -212,7 +216,7 @@ mod tests {
fn registers_domain_review_policy_after_node_deletion_queue() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(14).expect("fifteenth migration").name(),
"m20260729_000015_node_deletion_queue"
@@ -227,7 +231,7 @@ mod tests {
fn registers_project_notifications_after_domain_review_policy() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(15).expect("sixteenth migration").name(),
"m20260730_000016_domain_review_policy"
@@ -250,7 +254,7 @@ mod tests {
fn registers_scoped_codes_after_authentication_migrations() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(23).expect("twenty-fourth migration").name(),
"m20260806_000024_scoped_codes"
@@ -261,7 +265,7 @@ mod tests {
fn registers_registration_allowlist_after_scoped_codes() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(24).expect("twenty-fifth migration").name(),
"m20260806_000025_registration_allowlist"
@@ -272,7 +276,7 @@ mod tests {
fn registers_avatar_versions_after_registration_allowlist() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(25).expect("twenty-sixth migration").name(),
"m20260807_000026_avatars"
@@ -283,15 +287,19 @@ mod tests {
fn registers_object_storage_after_deployment_screenshots() {
let migrations = Migrator::migrations();
- assert_eq!(migrations.len(), 28);
+ assert_eq!(migrations.len(), 32);
assert_eq!(
migrations.get(26).expect("twenty-seventh migration").name(),
"m20260807_000027_deployment_screenshots"
);
assert_eq!(
- migrations.last().expect("last migration").name(),
+ migrations.get(27).expect("twenty-eighth migration").name(),
"m20260808_000028_object_storage"
);
+ assert_eq!(
+ migrations.last().expect("last migration").name(),
+ "m20260910_000032_managed_certificates"
+ );
}
#[tokio::test]
@@ -303,29 +311,40 @@ mod tests {
let test_db = PostgresMigrationDatabase::start(&database_url).await?;
let verification = async {
- Migrator::up(&test_db.db, None).await?;
- assert_migration_tracking(&test_db.db, 28, 0).await?;
+ Migrator::up(&test_db.db, Some(31)).await?;
+ assert_migration_tracking(&test_db.db, 31, 0).await?;
assert_avatar_schema(&test_db.db).await?;
assert_screenshot_schema(&test_db.db).await?;
assert_object_storage_schema(&test_db.db).await?;
Migrator::down(&test_db.db, Some(1)).await?;
- assert_migration_tracking(&test_db.db, 27, 1).await?;
+ assert_migration_tracking(&test_db.db, 30, 1).await?;
+ assert_regional_ingress_lifecycle_absent(&test_db.db).await?;
assert_avatar_schema(&test_db.db).await?;
assert_screenshot_schema(&test_db.db).await?;
- assert_object_storage_schema_absent(&test_db.db).await?;
+ assert_object_storage_schema(&test_db.db).await?;
+
+ Migrator::down(&test_db.db, Some(1)).await?;
+ assert_migration_tracking(&test_db.db, 29, 2).await?;
+ assert_regional_ingress_schema_absent(&test_db.db).await?;
+ assert_avatar_schema(&test_db.db).await?;
+ assert_screenshot_schema(&test_db.db).await?;
+ assert_object_storage_schema(&test_db.db).await?;
Migrator::down(&test_db.db, Some(1)).await?;
- assert_migration_tracking(&test_db.db, 26, 2).await?;
- assert_screenshot_schema_absent(&test_db.db).await?;
+ assert_migration_tracking(&test_db.db, 28, 3).await?;
assert_avatar_schema(&test_db.db).await?;
+ assert_screenshot_schema(&test_db.db).await?;
+ assert_object_storage_schema(&test_db.db).await?;
Migrator::down(&test_db.db, Some(1)).await?;
- assert_migration_tracking(&test_db.db, 25, 3).await?;
- assert_avatar_schema_absent(&test_db.db).await?;
+ assert_migration_tracking(&test_db.db, 27, 4).await?;
+ assert_avatar_schema(&test_db.db).await?;
+ assert_screenshot_schema(&test_db.db).await?;
+ assert_object_storage_schema_absent(&test_db.db).await?;
- Migrator::up(&test_db.db, Some(3)).await?;
- assert_migration_tracking(&test_db.db, 28, 0).await?;
+ Migrator::up(&test_db.db, Some(4)).await?;
+ assert_migration_tracking(&test_db.db, 31, 0).await?;
assert_avatar_schema(&test_db.db).await?;
assert_screenshot_schema(&test_db.db).await?;
assert_object_storage_schema(&test_db.db).await
@@ -342,6 +361,44 @@ mod tests {
}
}
+ #[tokio::test]
+ #[ignore = "requires GRASS_TEST_DATABASE_URL"]
+ async fn postgres_regional_ingress_schema_matches_domain_and_is_reversible()
+ -> anyhow::Result<()> {
+ let _migration_guard = MIGRATION_TEST_LOCK.lock().await;
+ let database_url = std::env::var("GRASS_TEST_DATABASE_URL")
+ .expect("GRASS_TEST_DATABASE_URL must be set to run this ignored migration test");
+ let test_db = PostgresMigrationDatabase::start(&database_url).await?;
+
+ let verification = async {
+ Migrator::up(&test_db.db, Some(31)).await?;
+ assert_migration_tracking(&test_db.db, 31, 0).await?;
+ assert_regional_ingress_schema(&test_db.db).await?;
+
+ Migrator::down(&test_db.db, Some(1)).await?;
+ assert_migration_tracking(&test_db.db, 30, 1).await?;
+ assert_regional_ingress_lifecycle_absent(&test_db.db).await?;
+
+ Migrator::down(&test_db.db, Some(1)).await?;
+ assert_migration_tracking(&test_db.db, 29, 2).await?;
+ assert_regional_ingress_schema_absent(&test_db.db).await?;
+
+ Migrator::up(&test_db.db, Some(2)).await?;
+ assert_migration_tracking(&test_db.db, 31, 0).await?;
+ assert_regional_ingress_schema(&test_db.db).await
+ }
+ .await;
+ let cleanup = test_db.cleanup().await;
+
+ match (verification, cleanup) {
+ (Err(verification_error), Err(cleanup_error)) => Err(verification_error.context(
+ format!("disposable schema cleanup also failed: {cleanup_error:#}"),
+ )),
+ (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
+ (Ok(()), Ok(())) => Ok(()),
+ }
+ }
+
#[tokio::test]
#[ignore = "requires GRASS_TEST_DATABASE_URL"]
async fn postgres_registration_allowlist_schema_matches_domain_and_is_reversible()
@@ -1657,10 +1714,13 @@ VALUES (
async fn assert_migration_tracking(
db: &DatabaseConnection,
applied_count: usize,
- pending_count: usize,
+ _phase_pending_count: usize,
) -> anyhow::Result<()> {
let applied = Migrator::get_applied_migrations(db).await?;
let pending = Migrator::get_pending_migrations(db).await?;
+ // Historical shape tests stop at their target migration. Later migrations
+ // remain pending even when that historical phase is fully applied.
+ let pending_count = Migrator::migrations().len() - applied_count;
ensure!(
applied.len() == applied_count,
@@ -1736,24 +1796,6 @@ ORDER BY table_name
Ok(())
}
- async fn assert_avatar_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> {
- let row = db
- .query_one_raw(Statement::from_string(
- DatabaseBackend::Postgres,
- r#"
-SELECT count(*)::bigint AS count
-FROM information_schema.columns
-WHERE table_schema = current_schema()
- AND column_name = 'avatar_version'
- AND table_name IN ('teams', 'users')
-"#,
- ))
- .await?
- .context("avatar absence query returned no row")?;
- ensure!(row.try_get::("", "count")? == 0);
- Ok(())
- }
-
async fn assert_screenshot_schema(db: &DatabaseConnection) -> anyhow::Result<()> {
let enum_rows = db
.query_all_raw(Statement::from_string(
@@ -1891,39 +1933,6 @@ WHERE schemaname = current_schema()
Ok(())
}
- async fn assert_screenshot_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> {
- let row = db
- .query_one_raw(Statement::from_string(
- DatabaseBackend::Postgres,
- r#"
-SELECT
- to_regclass('deployment_screenshot_jobs') IS NULL AS jobs_absent,
- NOT EXISTS (
- SELECT 1
- FROM pg_type t
- JOIN pg_namespace n ON n.oid = t.typnamespace
- WHERE n.nspname = current_schema()
- AND t.typname = 'deployment_screenshot_status'
- ) AS status_absent,
- NOT EXISTS (
- SELECT 1
- FROM pg_type t
- JOIN pg_enum e ON e.enumtypid = t.oid
- JOIN pg_namespace n ON n.oid = t.typnamespace
- WHERE n.nspname = current_schema()
- AND t.typname = 'deployment_artifact_kind'
- AND e.enumlabel = 'screenshot'
- ) AS artifact_value_absent
-"#,
- ))
- .await?
- .context("screenshot schema absence query returned no row")?;
- ensure!(row.try_get::("", "jobs_absent")?);
- ensure!(row.try_get::("", "status_absent")?);
- ensure!(row.try_get::("", "artifact_value_absent")?);
- Ok(())
- }
-
async fn assert_object_storage_schema(db: &DatabaseConnection) -> anyhow::Result<()> {
let table_count = object_count(
db,
@@ -2017,6 +2026,229 @@ WHERE schemaname = current_schema()
Ok(())
}
+ async fn assert_regional_ingress_schema(db: &DatabaseConnection) -> anyhow::Result<()> {
+ let columns = query_column_shapes(
+ db,
+ r#"
+SELECT column_name, udt_name, is_nullable, column_default
+FROM information_schema.columns
+WHERE table_schema = current_schema()
+ AND table_name = 'regional_ingresses'
+ORDER BY ordinal_position
+"#,
+ )
+ .await?;
+ ensure!(
+ columns
+ == vec![
+ column("id", "uuid", "NO", None),
+ column("region", "text", "NO", None),
+ column("hostname", "varchar", "NO", None),
+ column("enabled", "bool", "NO", Some("true")),
+ column("health_check_path", "text", "NO", Some("'/health'::text")),
+ column("health_check_interval_seconds", "int4", "NO", Some("30")),
+ column("origin_host_preservation", "bool", "NO", Some("true")),
+ column("tls_enabled", "bool", "NO", Some("true")),
+ column(
+ "certificate_issuer",
+ "text",
+ "NO",
+ Some("'letsencrypt'::text")
+ ),
+ column("certificate_auto_renew", "bool", "NO", Some("true")),
+ column("certificate_status", "text", "NO", Some("'pending'::text")),
+ column("certificate_expires_at", "timestamptz", "YES", None),
+ column("certificate_error", "text", "YES", None),
+ column("dns_challenge_provider", "text", "YES", None),
+ column("dns_challenge_config", "jsonb", "NO", Some("'{}'::jsonb")),
+ column(
+ "dns_challenge_status",
+ "text",
+ "NO",
+ Some("'not_configured'::text")
+ ),
+ column("dns_challenge_record_name", "text", "YES", None),
+ column("dns_challenge_record_value", "text", "YES", None),
+ column("deleted_at", "timestamptz", "YES", None),
+ column("created_at", "timestamptz", "NO", None),
+ column("updated_at", "timestamptz", "NO", None),
+ column("acme_account", "jsonb", "YES", None),
+ column("certificate_bundle", "jsonb", "YES", None),
+ column("certificate_issued_at", "timestamptz", "YES", None),
+ ],
+ "unexpected regional_ingresses column shapes: {columns:#?}"
+ );
+
+ let constraints = db
+ .query_all_raw(Statement::from_string(
+ DatabaseBackend::Postgres,
+ r#"
+SELECT conname, pg_get_constraintdef(oid) AS definition
+FROM pg_constraint
+WHERE conrelid = 'regional_ingresses'::regclass
+ AND conname LIKE 'ck_regional_ingresses_%'
+ORDER BY conname
+"#,
+ ))
+ .await?
+ .into_iter()
+ .map(|row| {
+ Ok((
+ row.try_get::("", "conname")?,
+ row.try_get::("", "definition")?,
+ ))
+ })
+ .collect::, sea_orm::DbErr>>()?;
+ ensure!(
+ constraints.len() == 7,
+ "unexpected ingress constraints: {constraints:#?}"
+ );
+ ensure!(constraints["ck_regional_ingresses_region_nonempty"].contains("char_length"));
+ ensure!(constraints["ck_regional_ingresses_hostname_nonempty"].contains("char_length"));
+ // PostgreSQL deparses LIKE and BETWEEN into their underlying operators.
+ ensure!(
+ constraints["ck_regional_ingresses_health_path"]
+ .contains("health_check_path ~~ '/%'::text")
+ );
+ ensure!(
+ constraints["ck_regional_ingresses_health_interval"]
+ .contains("health_check_interval_seconds >= 5")
+ && constraints["ck_regional_ingresses_health_interval"]
+ .contains("health_check_interval_seconds <= 3600")
+ );
+ ensure!(constraints["ck_regional_ingresses_certificate_issuer"].contains("letsencrypt"));
+ ensure!(
+ constraints["ck_regional_ingresses_certificate_status"].contains("certificate_status")
+ );
+ ensure!(
+ constraints["ck_regional_ingresses_dns_challenge_status"].contains("not_configured")
+ );
+
+ let indexes = db
+ .query_all_raw(Statement::from_string(
+ DatabaseBackend::Postgres,
+ r#"
+SELECT indexname, indexdef
+FROM pg_indexes
+WHERE schemaname = current_schema()
+ AND indexname IN (
+ 'ux_regional_ingresses_region_active',
+ 'ux_regional_ingresses_hostname_active',
+ 'ix_regional_ingresses_enabled',
+ 'ix_host_sources_region',
+ 'ix_project_host_bindings_region'
+ )
+ORDER BY indexname
+"#,
+ ))
+ .await?
+ .into_iter()
+ .map(|row| {
+ Ok((
+ row.try_get::("", "indexname")?,
+ row.try_get::("", "indexdef")?,
+ ))
+ })
+ .collect::, sea_orm::DbErr>>()?;
+ ensure!(
+ indexes.len() == 5,
+ "unexpected ingress indexes: {indexes:#?}"
+ );
+ ensure!(indexes["ux_regional_ingresses_region_active"].contains("UNIQUE"));
+ ensure!(indexes["ux_regional_ingresses_region_active"].contains("deleted_at IS NULL"));
+ ensure!(indexes["ux_regional_ingresses_hostname_active"].contains("UNIQUE"));
+ ensure!(indexes["ix_regional_ingresses_enabled"].contains("(region, enabled)"));
+ ensure!(indexes["ix_host_sources_region"].contains("(region)"));
+ ensure!(indexes["ix_project_host_bindings_region"].contains("(region)"));
+ ensure!(
+ object_count(
+ db,
+ "SELECT count(*)::bigint AS count FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'regional_ingress_health'",
+ )
+ .await?
+ == 1,
+ "regional ingress health table is missing"
+ );
+ Ok(())
+ }
+
+ async fn assert_regional_ingress_lifecycle_absent(
+ db: &DatabaseConnection,
+ ) -> anyhow::Result<()> {
+ ensure!(
+ object_count(
+ db,
+ "SELECT count(*)::bigint AS count FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'regional_ingress_health'",
+ )
+ .await?
+ == 0,
+ "regional ingress health table remained after lifecycle down migration"
+ );
+ ensure!(
+ object_count(
+ db,
+ "SELECT count(*)::bigint AS count FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'regional_ingresses' AND column_name IN ('acme_account', 'certificate_bundle', 'certificate_issued_at')",
+ )
+ .await?
+ == 0,
+ "regional ingress lifecycle columns remained after lifecycle down migration"
+ );
+ Ok(())
+ }
+
+ async fn assert_regional_ingress_schema_absent(db: &DatabaseConnection) -> anyhow::Result<()> {
+ ensure!(
+ object_count(
+ db,
+ r#"
+SELECT count(*)::bigint AS count
+FROM information_schema.tables
+WHERE table_schema = current_schema()
+ AND table_name = 'regional_ingresses'
+"#,
+ )
+ .await?
+ == 0,
+ "regional_ingresses table remained after down migration"
+ );
+ ensure!(
+ object_count(
+ db,
+ r#"
+SELECT count(*)::bigint AS count
+FROM information_schema.columns
+WHERE table_schema = current_schema()
+ AND table_name IN ('host_sources', 'project_host_bindings')
+ AND column_name = 'region'
+"#,
+ )
+ .await?
+ == 0,
+ "regional host columns remained after down migration"
+ );
+ ensure!(
+ object_count(
+ db,
+ r#"
+SELECT count(*)::bigint AS count
+FROM pg_indexes
+WHERE schemaname = current_schema()
+ AND indexname IN (
+ 'ux_regional_ingresses_region_active',
+ 'ux_regional_ingresses_hostname_active',
+ 'ix_regional_ingresses_enabled',
+ 'ix_host_sources_region',
+ 'ix_project_host_bindings_region'
+ )
+"#,
+ )
+ .await?
+ == 0,
+ "regional ingress indexes remained after down migration"
+ );
+ Ok(())
+ }
+
async fn assert_registration_allowlist_schema(db: &DatabaseConnection) -> anyhow::Result<()> {
let columns = query_column_shapes(
db,
diff --git a/apps/control-api/src/infra/database/migration/m20260908_000029_regional_routing.rs b/apps/control-api/src/infra/database/migration/m20260908_000029_regional_routing.rs
new file mode 100644
index 0000000..18670a7
--- /dev/null
+++ b/apps/control-api/src/infra/database/migration/m20260908_000029_regional_routing.rs
@@ -0,0 +1,58 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+pub(crate) const UP_SQL: &str = r#"
+ALTER TABLE nodes
+ ADD COLUMN region TEXT NOT NULL DEFAULT 'default',
+ ADD CONSTRAINT ck_nodes_region_nonempty CHECK (char_length(btrim(region)) > 0);
+
+ALTER TABLE deployments
+ ADD COLUMN region TEXT NOT NULL DEFAULT 'default',
+ ADD CONSTRAINT ck_deployments_region_nonempty CHECK (char_length(btrim(region)) > 0);
+
+CREATE INDEX ix_nodes_region_health
+ ON nodes (region, status, last_heartbeat_at)
+ WHERE deleted_at IS NULL AND serve_enabled = TRUE;
+
+CREATE INDEX ix_deployments_region_status
+ ON deployments (region, serve_status)
+ WHERE deleted_at IS NULL;
+"#;
+
+pub(crate) const DOWN_SQL: &str = r#"
+DROP INDEX IF EXISTS ix_deployments_region_status;
+DROP INDEX IF EXISTS ix_nodes_region_health;
+ALTER TABLE deployments DROP CONSTRAINT ck_deployments_region_nonempty, DROP COLUMN region;
+ALTER TABLE nodes DROP CONSTRAINT ck_nodes_region_nonempty, DROP COLUMN region;
+"#;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+ async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ manager.get_connection().execute_unprepared(UP_SQL).await?;
+ Ok(())
+ }
+
+ async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ manager
+ .get_connection()
+ .execute_unprepared(DOWN_SQL)
+ .await?;
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn migration_adds_defaulted_regions_and_indexes() {
+ assert!(UP_SQL.contains("ADD COLUMN region TEXT NOT NULL DEFAULT 'default'"));
+ assert!(UP_SQL.contains("ix_nodes_region_health"));
+ assert!(UP_SQL.contains("ix_deployments_region_status"));
+ assert!(DOWN_SQL.contains("DROP COLUMN region"));
+ }
+}
diff --git a/apps/control-api/src/infra/database/migration/m20260908_000030_regional_ingress.rs b/apps/control-api/src/infra/database/migration/m20260908_000030_regional_ingress.rs
new file mode 100644
index 0000000..172f8eb
--- /dev/null
+++ b/apps/control-api/src/infra/database/migration/m20260908_000030_regional_ingress.rs
@@ -0,0 +1,117 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+pub(crate) const UP_SQL: &str = r#"
+CREATE TABLE regional_ingresses (
+ id UUID PRIMARY KEY,
+ region TEXT NOT NULL,
+ hostname VARCHAR(253) NOT NULL,
+ enabled BOOLEAN NOT NULL DEFAULT TRUE,
+ health_check_path TEXT NOT NULL DEFAULT '/health',
+ health_check_interval_seconds INTEGER NOT NULL DEFAULT 30,
+ origin_host_preservation BOOLEAN NOT NULL DEFAULT TRUE,
+ tls_enabled BOOLEAN NOT NULL DEFAULT TRUE,
+ certificate_issuer TEXT NOT NULL DEFAULT 'letsencrypt',
+ certificate_auto_renew BOOLEAN NOT NULL DEFAULT TRUE,
+ certificate_status TEXT NOT NULL DEFAULT 'pending',
+ certificate_expires_at TIMESTAMPTZ NULL,
+ certificate_error TEXT NULL,
+ dns_challenge_provider TEXT NULL,
+ dns_challenge_config JSONB NOT NULL DEFAULT '{}',
+ dns_challenge_status TEXT NOT NULL DEFAULT 'not_configured',
+ dns_challenge_record_name TEXT NULL,
+ dns_challenge_record_value TEXT NULL,
+ deleted_at TIMESTAMPTZ NULL,
+ created_at TIMESTAMPTZ NOT NULL,
+ updated_at TIMESTAMPTZ NOT NULL,
+ CONSTRAINT ck_regional_ingresses_region_nonempty
+ CHECK (char_length(btrim(region)) > 0),
+ CONSTRAINT ck_regional_ingresses_hostname_nonempty
+ CHECK (char_length(btrim(hostname)) > 0),
+ CONSTRAINT ck_regional_ingresses_health_path
+ CHECK (health_check_path LIKE '/%'),
+ CONSTRAINT ck_regional_ingresses_health_interval
+ CHECK (health_check_interval_seconds BETWEEN 5 AND 3600),
+ CONSTRAINT ck_regional_ingresses_certificate_issuer
+ CHECK (certificate_issuer IN ('letsencrypt', 'zerossl', 'manual')),
+ CONSTRAINT ck_regional_ingresses_certificate_status
+ CHECK (certificate_status IN ('pending', 'issuing', 'active', 'expiring', 'failed', 'disabled')),
+ CONSTRAINT ck_regional_ingresses_dns_challenge_status
+ CHECK (dns_challenge_status IN ('not_configured', 'pending', 'valid', 'failed'))
+);
+
+ALTER TABLE project_host_bindings
+ ADD COLUMN region TEXT NOT NULL DEFAULT 'default',
+ ADD CONSTRAINT ck_project_host_bindings_region_nonempty
+ CHECK (char_length(btrim(region)) > 0);
+CREATE INDEX ix_project_host_bindings_region
+ ON project_host_bindings (region)
+ WHERE deleted_at IS NULL;
+
+ALTER TABLE host_sources
+ ADD COLUMN region TEXT NOT NULL DEFAULT 'default',
+ ADD CONSTRAINT ck_host_sources_region_nonempty
+ CHECK (char_length(btrim(region)) > 0);
+CREATE INDEX ix_host_sources_region
+ ON host_sources (region)
+ WHERE deleted_at IS NULL;
+
+CREATE UNIQUE INDEX ux_regional_ingresses_region_active
+ ON regional_ingresses (region)
+ WHERE deleted_at IS NULL;
+CREATE UNIQUE INDEX ux_regional_ingresses_hostname_active
+ ON regional_ingresses (hostname)
+ WHERE deleted_at IS NULL;
+CREATE INDEX ix_regional_ingresses_enabled
+ ON regional_ingresses (region, enabled)
+ WHERE deleted_at IS NULL;
+"#;
+
+pub(crate) const DOWN_SQL: &str = r#"
+DROP INDEX IF EXISTS ix_regional_ingresses_enabled;
+DROP INDEX IF EXISTS ux_regional_ingresses_hostname_active;
+DROP INDEX IF EXISTS ux_regional_ingresses_region_active;
+DROP TABLE regional_ingresses;
+DROP INDEX IF EXISTS ix_project_host_bindings_region;
+ALTER TABLE project_host_bindings
+ DROP CONSTRAINT ck_project_host_bindings_region_nonempty,
+ DROP COLUMN region;
+DROP INDEX IF EXISTS ix_host_sources_region;
+ALTER TABLE host_sources
+ DROP CONSTRAINT ck_host_sources_region_nonempty,
+ DROP COLUMN region;
+"#;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+ async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ manager.get_connection().execute_unprepared(UP_SQL).await?;
+ Ok(())
+ }
+
+ async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ manager
+ .get_connection()
+ .execute_unprepared(DOWN_SQL)
+ .await?;
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn migration_records_regional_ingress_and_certificate_controls() {
+ assert!(UP_SQL.contains("CREATE TABLE regional_ingresses"));
+ assert!(UP_SQL.contains("certificate_issuer TEXT NOT NULL DEFAULT 'letsencrypt'"));
+ assert!(UP_SQL.contains("certificate_status TEXT NOT NULL DEFAULT 'pending'"));
+ assert!(UP_SQL.contains("dns_challenge_config JSONB NOT NULL DEFAULT '{}'"));
+ assert!(UP_SQL.contains("dns_challenge_status TEXT NOT NULL DEFAULT 'not_configured'"));
+ assert!(UP_SQL.contains("ux_regional_ingresses_region_active"));
+ assert!(DOWN_SQL.contains("DROP TABLE regional_ingresses"));
+ }
+}
diff --git a/apps/control-api/src/infra/database/migration/m20260909_000031_regional_ingress_lifecycle.rs b/apps/control-api/src/infra/database/migration/m20260909_000031_regional_ingress_lifecycle.rs
new file mode 100644
index 0000000..5b53074
--- /dev/null
+++ b/apps/control-api/src/infra/database/migration/m20260909_000031_regional_ingress_lifecycle.rs
@@ -0,0 +1,83 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+pub(crate) const UP_SQL: &str = r#"
+ALTER TABLE project_host_bindings
+ ADD COLUMN ownership_status TEXT NOT NULL DEFAULT 'not_required',
+ ADD COLUMN ownership_checked_at TIMESTAMPTZ NULL,
+ ADD COLUMN ownership_error TEXT NULL;
+
+UPDATE project_host_bindings
+SET ownership_status = 'pending'
+WHERE host_source_id IS NULL AND deleted_at IS NULL;
+
+ALTER TABLE project_host_bindings
+ ADD CONSTRAINT ck_project_host_bindings_ownership_status
+ CHECK (ownership_status IN ('pending', 'verified', 'failed', 'not_required'));
+
+ALTER TABLE regional_ingresses
+ ADD COLUMN acme_account JSONB NULL,
+ ADD COLUMN certificate_bundle JSONB NULL,
+ ADD COLUMN certificate_issued_at TIMESTAMPTZ NULL;
+
+CREATE TABLE regional_ingress_health (
+ ingress_id UUID NOT NULL REFERENCES regional_ingresses(id) ON DELETE CASCADE,
+ node_id UUID NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
+ status TEXT NOT NULL DEFAULT 'unknown',
+ checked_at TIMESTAMPTZ NULL,
+ latency_ms INTEGER NULL,
+ error TEXT NULL,
+ PRIMARY KEY (ingress_id, node_id),
+ CONSTRAINT ck_regional_ingress_health_status
+ CHECK (status IN ('unknown', 'healthy', 'unhealthy')),
+ CONSTRAINT ck_regional_ingress_health_latency
+ CHECK (latency_ms IS NULL OR latency_ms >= 0)
+);
+CREATE INDEX ix_regional_ingress_health_lookup
+ ON regional_ingress_health (ingress_id, status, checked_at);
+"#;
+
+pub(crate) const DOWN_SQL: &str = r#"
+DROP INDEX IF EXISTS ix_regional_ingress_health_lookup;
+DROP TABLE regional_ingress_health;
+ALTER TABLE regional_ingresses
+ DROP COLUMN certificate_issued_at,
+ DROP COLUMN certificate_bundle,
+ DROP COLUMN acme_account;
+ALTER TABLE project_host_bindings
+ DROP CONSTRAINT ck_project_host_bindings_ownership_status,
+ DROP COLUMN ownership_error,
+ DROP COLUMN ownership_checked_at,
+ DROP COLUMN ownership_status;
+"#;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+ async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ manager.get_connection().execute_unprepared(UP_SQL).await?;
+ Ok(())
+ }
+
+ async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ manager
+ .get_connection()
+ .execute_unprepared(DOWN_SQL)
+ .await?;
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn migration_adds_lifecycle_state_and_health_table() {
+ assert!(UP_SQL.contains("ownership_status TEXT NOT NULL"));
+ assert!(UP_SQL.contains("CREATE TABLE regional_ingress_health"));
+ assert!(UP_SQL.contains("certificate_bundle JSONB"));
+ assert!(DOWN_SQL.contains("DROP TABLE regional_ingress_health"));
+ }
+}
diff --git a/apps/control-api/src/infra/database/migration/m20260910_000032_managed_certificates.rs b/apps/control-api/src/infra/database/migration/m20260910_000032_managed_certificates.rs
new file mode 100644
index 0000000..710a010
--- /dev/null
+++ b/apps/control-api/src/infra/database/migration/m20260910_000032_managed_certificates.rs
@@ -0,0 +1,172 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+pub(crate) const UP_SQL: &str = r#"
+ALTER TABLE regional_ingresses ALTER COLUMN health_check_path SET DEFAULT '/_grass/health';
+UPDATE regional_ingresses SET health_check_path = '/_grass/health' WHERE health_check_path = '/health';
+CREATE TABLE managed_certificates (
+ id UUID PRIMARY KEY,
+ ingress_id UUID NOT NULL REFERENCES regional_ingresses(id) ON DELETE CASCADE,
+ host_binding_id UUID NULL UNIQUE REFERENCES project_host_bindings(id) ON DELETE CASCADE,
+ hostname VARCHAR(253) NOT NULL,
+ issuer TEXT NOT NULL CHECK (issuer IN ('letsencrypt', 'zerossl', 'manual')),
+ challenge_method TEXT NOT NULL DEFAULT 'http01' CHECK (challenge_method IN ('http01', 'dns01')),
+ auto_renew BOOLEAN NOT NULL DEFAULT TRUE,
+ status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','issuing','active','failed','disabled')),
+ error TEXT NULL,
+ bundle JSONB NULL,
+ acme_account JSONB NULL,
+ revision TEXT NOT NULL DEFAULT '',
+ issued_at TIMESTAMPTZ NULL,
+ expires_at TIMESTAMPTZ NULL,
+ retry_at TIMESTAMPTZ NULL,
+ failure_count INTEGER NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
+ lease_until TIMESTAMPTZ NULL,
+ generation UUID NOT NULL,
+ challenge_token TEXT NULL,
+ challenge_value TEXT NULL,
+ challenge_expires_at TIMESTAMPTZ NULL,
+ dns_record_name TEXT NULL,
+ dns_record_value TEXT NULL,
+ dns_cleanup JSONB NULL,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ CHECK ((challenge_token IS NULL AND challenge_value IS NULL AND challenge_expires_at IS NULL)
+ OR (challenge_token IS NOT NULL AND challenge_value IS NOT NULL AND challenge_expires_at IS NOT NULL))
+);
+CREATE UNIQUE INDEX ux_managed_certificates_regional ON managed_certificates(ingress_id) WHERE host_binding_id IS NULL;
+CREATE INDEX ix_managed_certificates_retry ON managed_certificates(retry_at, expires_at);
+CREATE TABLE node_ingress_status (
+ node_id UUID PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
+ certificates JSONB NOT NULL DEFAULT '[]',
+ challenge_revision TEXT NOT NULL DEFAULT '',
+ tls_ready BOOLEAN NOT NULL DEFAULT FALSE,
+ checked_at TIMESTAMPTZ NOT NULL
+);
+"#;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+ async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ manager.get_connection().execute_unprepared(UP_SQL).await?;
+ Ok(())
+ }
+ async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ manager.get_connection().execute_unprepared("DROP TABLE node_ingress_status; DROP TABLE managed_certificates; ALTER TABLE regional_ingresses ALTER COLUMN health_check_path SET DEFAULT '/health';").await?;
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use anyhow::{Context, ensure};
+ use sea_orm::{Database, DatabaseBackend, Statement};
+ use sea_orm_migration::MigratorTrait;
+
+ /// Read-only inspection of an already migrated, operator-provided database.
+ #[tokio::test]
+ #[ignore = "requires GRASS_TEST_DATABASE_URL pointing to a database migrated through managed certificates"]
+ async fn managed_certificate_schema_matches_signed_lifecycle_and_ack_protocol()
+ -> anyhow::Result<()> {
+ let url = std::env::var("GRASS_TEST_DATABASE_URL")
+ .context("test database runtime configuration required")?;
+ let db = Database::connect(url).await?;
+ let applied =
+ crate::infra::database::migrate::Migrator::get_applied_migrations(&db).await?;
+ ensure!(
+ applied
+ .iter()
+ .any(|m| m.name() == "m20260910_000032_managed_certificates"),
+ "managed certificate migration was not recorded"
+ );
+ ensure!(
+ crate::infra::database::migrate::Migrator::get_pending_migrations(&db)
+ .await?
+ .is_empty(),
+ "pending migrations remain"
+ );
+ let columns=db.query_all_raw(Statement::from_string(DatabaseBackend::Postgres,"SELECT table_name,column_name,udt_name,is_nullable,column_default FROM information_schema.columns WHERE table_schema=current_schema() AND table_name IN ('managed_certificates','node_ingress_status')")).await?;
+ let mut found = std::collections::BTreeMap::new();
+ for column in columns {
+ let table: String = column.try_get("", "table_name")?;
+ let name: String = column.try_get("", "column_name")?;
+ found.insert(
+ (table, name),
+ (
+ column.try_get::("", "udt_name")?,
+ column.try_get::("", "is_nullable")?,
+ column.try_get::>("", "column_default")?,
+ ),
+ );
+ }
+ for name in [
+ "issued_at",
+ "expires_at",
+ "retry_at",
+ "lease_until",
+ "challenge_expires_at",
+ ] {
+ ensure!(
+ found.get(&("managed_certificates".to_owned(), name.to_owned()))
+ == Some(&("timestamptz".to_owned(), "YES".to_owned(), None)),
+ "nullable lifecycle timestamp {name} has wrong type or default"
+ );
+ }
+ for (table, name, kind, nullable) in [
+ ("managed_certificates", "bundle", "jsonb", "YES"),
+ ("managed_certificates", "dns_cleanup", "jsonb", "YES"),
+ ("managed_certificates", "generation", "uuid", "NO"),
+ ("managed_certificates", "host_binding_id", "uuid", "YES"),
+ ("node_ingress_status", "certificates", "jsonb", "NO"),
+ ("node_ingress_status", "tls_ready", "bool", "NO"),
+ ("node_ingress_status", "checked_at", "timestamptz", "NO"),
+ ] {
+ let column = found
+ .get(&(table.to_owned(), name.to_owned()))
+ .context("required certificate column missing")?;
+ ensure!(
+ column.0 == kind && column.1 == nullable,
+ "certificate column shape mismatch"
+ );
+ }
+ let indexes=db.query_all_raw(Statement::from_string(DatabaseBackend::Postgres,"SELECT indexdef FROM pg_indexes WHERE schemaname=current_schema() AND tablename='managed_certificates'")).await?;
+ let indexes = indexes
+ .iter()
+ .map(|r| r.try_get::("", "indexdef"))
+ .collect::, _>>()?
+ .join("\n");
+ ensure!(
+ indexes.contains("UNIQUE INDEX ux_managed_certificates_regional")
+ && indexes.contains("host_binding_id IS NULL")
+ && indexes.contains("ix_managed_certificates_retry"),
+ "certificate uniqueness/retry indexes missing"
+ );
+ let constraints=db.query_all_raw(Statement::from_string(DatabaseBackend::Postgres,"SELECT pg_get_constraintdef(oid) AS definition FROM pg_constraint WHERE conrelid IN ('managed_certificates'::regclass,'node_ingress_status'::regclass)")).await?;
+ let constraints = constraints
+ .iter()
+ .map(|r| r.try_get::("", "definition"))
+ .collect::, _>>()?
+ .join("\n");
+ for required in [
+ "REFERENCES regional_ingresses(id)",
+ "REFERENCES project_host_bindings(id)",
+ "REFERENCES nodes(id)",
+ "http01",
+ "dns01",
+ "letsencrypt",
+ "zerossl",
+ "manual",
+ "failure_count >= 0",
+ "challenge_expires_at IS NOT NULL",
+ ] {
+ ensure!(
+ constraints.contains(required),
+ "certificate constraint missing: {required}"
+ );
+ }
+ db.close().await?;
+ Ok(())
+ }
+}
diff --git a/apps/control-api/src/infra/database/migration/mod.rs b/apps/control-api/src/infra/database/migration/mod.rs
index 92afd63..9223a1d 100644
--- a/apps/control-api/src/infra/database/migration/mod.rs
+++ b/apps/control-api/src/infra/database/migration/mod.rs
@@ -26,3 +26,7 @@ pub mod m20260806_000025_registration_allowlist;
pub mod m20260807_000026_avatars;
pub mod m20260807_000027_deployment_screenshots;
pub mod m20260808_000028_object_storage;
+pub mod m20260908_000029_regional_routing;
+pub mod m20260908_000030_regional_ingress;
+pub mod m20260909_000031_regional_ingress_lifecycle;
+pub mod m20260910_000032_managed_certificates;
diff --git a/apps/control-api/src/infra/host_provision/cloudflare.rs b/apps/control-api/src/infra/host_provision/cloudflare.rs
index 25dba2b..d09339d 100644
--- a/apps/control-api/src/infra/host_provision/cloudflare.rs
+++ b/apps/control-api/src/infra/host_provision/cloudflare.rs
@@ -22,7 +22,7 @@ const DEFAULT_BASE_URL: &str = "https://api.cloudflare.com/client/v4";
const RECORD_EXISTS_CODES: [i64; 2] = [81_057, 81_053];
/// Parsed view of a Cloudflare host source `config` object.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct CloudflareConfig {
pub api_token: String,
pub zone_id: String,
@@ -40,6 +40,14 @@ impl CloudflareConfig {
Self::from_json(&source.config)
}
+ pub fn for_txt(&self, value: &str) -> Self {
+ let mut config = self.clone();
+ config.record_type = "TXT".to_owned();
+ config.record_value = value.to_owned();
+ config.proxied = false;
+ config
+ }
+
pub fn from_json(config: &serde_json::Value) -> Result {
let object = config
.as_object()
@@ -196,6 +204,24 @@ impl CloudflareDns {
}
}
+ pub async fn ensure_txt_record(
+ &self,
+ config: &CloudflareConfig,
+ name: &str,
+ value: &str,
+ ) -> Result {
+ self.ensure_record(&config.for_txt(value), name).await
+ }
+
+ pub async fn remove_txt_record(
+ &self,
+ config: &CloudflareConfig,
+ name: &str,
+ value: &str,
+ ) -> Result, HostProvisionError> {
+ self.remove_record(&config.for_txt(value), name).await
+ }
+
async fn parse(
response: reqwest::Response,
) -> Result, HostProvisionError> {
diff --git a/apps/control-api/src/infra/host_provision/mod.rs b/apps/control-api/src/infra/host_provision/mod.rs
index fb1a353..9abf6b3 100644
--- a/apps/control-api/src/infra/host_provision/mod.rs
+++ b/apps/control-api/src/infra/host_provision/mod.rs
@@ -115,6 +115,58 @@ impl DnsProviderHostProvisioner {
}
}
+ pub async fn ensure_txt_record(
+ &self,
+ provider: &str,
+ config: &serde_json::Value,
+ zone: &str,
+ name: &str,
+ value: &str,
+ ) -> Result {
+ let _ = zone;
+ match provider.trim().to_ascii_lowercase().as_str() {
+ cloudflare::PROVIDER_NAME => {
+ let mut object = config.as_object().cloned().unwrap_or_default();
+ object.insert("record_type".to_owned(), serde_json::json!("CNAME"));
+ object.insert("record_value".to_owned(), serde_json::json!(value));
+ let parsed =
+ cloudflare::CloudflareConfig::from_json(&serde_json::Value::Object(object))
+ .map_err(HostProvisionError::Provider)?;
+ Ok(self
+ .cloudflare
+ .ensure_txt_record(&parsed, name, value)
+ .await?
+ .id)
+ }
+ other => Err(Self::unsupported(Some(other))),
+ }
+ }
+
+ pub async fn remove_txt_record(
+ &self,
+ provider: &str,
+ config: &serde_json::Value,
+ zone: &str,
+ name: &str,
+ value: &str,
+ ) -> Result, HostProvisionError> {
+ let _ = zone;
+ match provider.trim().to_ascii_lowercase().as_str() {
+ cloudflare::PROVIDER_NAME => {
+ let mut object = config.as_object().cloned().unwrap_or_default();
+ object.insert("record_type".to_owned(), serde_json::json!("CNAME"));
+ object.insert("record_value".to_owned(), serde_json::json!(value));
+ let parsed =
+ cloudflare::CloudflareConfig::from_json(&serde_json::Value::Object(object))
+ .map_err(HostProvisionError::Provider)?;
+ self.cloudflare
+ .remove_txt_record(&parsed, name, value)
+ .await
+ }
+ other => Err(Self::unsupported(Some(other))),
+ }
+ }
+
fn unsupported(provider: Option<&str>) -> HostProvisionError {
HostProvisionError::UnsupportedSource(format!(
"dns provider '{}' is not supported (supported: {})",
@@ -245,6 +297,7 @@ mod tests {
kind,
label: "test".to_owned(),
base_domain: "grass.test".to_owned(),
+ region: "default".to_owned(),
enabled: true,
allows_auto_assign: true,
is_default: true,
diff --git a/apps/control-api/src/infra/host_provision/service.rs b/apps/control-api/src/infra/host_provision/service.rs
index c121287..40143c2 100644
--- a/apps/control-api/src/infra/host_provision/service.rs
+++ b/apps/control-api/src/infra/host_provision/service.rs
@@ -30,6 +30,7 @@ pub struct BindHostRequest<'a> {
pub team: &'a team::Model,
pub source: Option<&'a host_source::Model>,
pub host: String,
+ pub region: String,
pub kind: HostBindingKind,
pub environment: HostBindingEnvironment,
pub is_primary: bool,
@@ -43,12 +44,14 @@ pub enum DeprovisionOutcome {
Failed,
}
-fn custom_binding_status(review_status: &HostReviewStatus) -> HostBindingStatus {
- match review_status {
- HostReviewStatus::Approved => HostBindingStatus::Active,
- HostReviewStatus::NotRequired | HostReviewStatus::Pending | HostReviewStatus::Rejected => {
- HostBindingStatus::Pending
- }
+fn custom_binding_status(
+ review_status: &HostReviewStatus,
+ ownership_status: &str,
+) -> HostBindingStatus {
+ match (review_status, ownership_status) {
+ (HostReviewStatus::Approved, "verified") => HostBindingStatus::Active,
+ (HostReviewStatus::Rejected, _) => HostBindingStatus::Disabled,
+ _ => HostBindingStatus::Pending,
}
}
@@ -104,6 +107,7 @@ impl<'a> HostBindingService<'a> {
team_id: request.team.id,
host_source_id: request.source.map(|source| source.id),
host: request.host.clone(),
+ region: request.region.clone(),
kind: request.kind,
environment: request.environment,
status: HostBindingStatus::Pending,
@@ -137,7 +141,8 @@ impl<'a> HostBindingService<'a> {
// Custom-host DNS is user-managed, but manual review still gates
// whether the binding can become serving/active.
None => {
- let status = custom_binding_status(&binding.review_status);
+ let status =
+ custom_binding_status(&binding.review_status, &binding.ownership_status);
hosts::update_binding_status(self.db, binding, status, None)
.await
.map_err(|source| AppError::Infrastructure { op, source })?
@@ -266,16 +271,16 @@ mod tests {
#[test]
fn custom_binding_only_activates_after_automatic_approval() {
assert_eq!(
- custom_binding_status(&HostReviewStatus::Approved),
+ custom_binding_status(&HostReviewStatus::Approved, "verified"),
HostBindingStatus::Active
);
assert_eq!(
- custom_binding_status(&HostReviewStatus::Pending),
+ custom_binding_status(&HostReviewStatus::Pending, "pending"),
HostBindingStatus::Pending
);
assert_eq!(
- custom_binding_status(&HostReviewStatus::Rejected),
- HostBindingStatus::Pending
+ custom_binding_status(&HostReviewStatus::Rejected, "pending"),
+ HostBindingStatus::Disabled
);
}
@@ -287,6 +292,7 @@ mod tests {
kind: HostSourceKind::DnsProvider,
label: "unsupported provider".to_owned(),
base_domain: "example.invalid".to_owned(),
+ region: "default".to_owned(),
enabled: true,
allows_auto_assign: true,
is_default: true,
@@ -302,6 +308,7 @@ mod tests {
team_id: Uuid::now_v7(),
host_source_id: Some(source.id),
host: "site.example.invalid".to_owned(),
+ region: "default".to_owned(),
kind: HostBindingKind::Platform,
environment: HostBindingEnvironment::Production,
status: HostBindingStatus::Active,
@@ -311,6 +318,9 @@ mod tests {
reviewed_by_user_id: None,
reviewed_at: None,
review_reason: None,
+ ownership_status: "pending".to_owned(),
+ ownership_checked_at: None,
+ ownership_error: None,
deleted_at: Some(now),
created_at: now,
updated_at: now,
diff --git a/apps/control-api/src/infra/http/middlewares/audit.rs b/apps/control-api/src/infra/http/middlewares/audit.rs
index d1db0c6..a6ee6f9 100644
--- a/apps/control-api/src/infra/http/middlewares/audit.rs
+++ b/apps/control-api/src/infra/http/middlewares/audit.rs
@@ -529,6 +529,7 @@ mod tests {
id: deployment_id,
project_id,
team_id,
+ region: "default".to_owned(),
build_node_id: None,
serve_node_id: None,
environment: DeploymentEnvironment::Production,
diff --git a/apps/control-api/src/infra/http/middlewares/node_auth.rs b/apps/control-api/src/infra/http/middlewares/node_auth.rs
index 870ed3e..b4943b1 100644
--- a/apps/control-api/src/infra/http/middlewares/node_auth.rs
+++ b/apps/control-api/src/infra/http/middlewares/node_auth.rs
@@ -154,6 +154,7 @@ mod tests {
let node = crate::infra::database::entity::node::Model {
id: uuid::Uuid::now_v7(),
name: "disabled-node".to_owned(),
+ region: "default".to_owned(),
token_hash: grass_token::hash_token(token),
status: NodeStatus::Disabled,
build_enabled: true,
diff --git a/apps/control-api/src/infra/node_manager/config_file.rs b/apps/control-api/src/infra/node_manager/config_file.rs
index 73da322..ccfd02e 100644
--- a/apps/control-api/src/infra/node_manager/config_file.rs
+++ b/apps/control-api/src/infra/node_manager/config_file.rs
@@ -14,6 +14,7 @@ const GENERATED_BY: &str = "grass-control-api";
pub struct GenerateParams<'a> {
pub node_name: &'a str,
+ pub region: &'a str,
pub node_token: &'a str,
pub control_api_url: String,
pub storage_root: &'a str,
@@ -30,6 +31,7 @@ struct GeneratedConfig {
#[derive(Serialize)]
struct NodeSection {
id: String,
+ region: String,
control_api: String,
node_token: String,
work_root: String,
@@ -85,6 +87,7 @@ pub fn generate(path: &str, params: &GenerateParams<'_>) -> anyhow::Result anyhow::Result<()> {
+ let _ = rustls::crypto::ring::default_provider().install_default();
let cli = Cli::parse();
let mut config = init::config(cli.config_path())?;
apply_cli(&mut config, &cli);
@@ -32,6 +33,8 @@ async fn main() -> anyhow::Result<()> {
init::storage(&state).await?;
init::cache(&state).await?;
spawn_node_health_sweep(state.clone());
+ spawn_regional_ingress_health_sweep(state.clone());
+ spawn_regional_ingress_certificate_sweep(state.clone());
spawn_audit_retention_sweep(state.clone());
spawn_artifact_retention_sweep(state.clone());
spawn_storage_migration_sweep(state.clone());
@@ -355,6 +358,61 @@ async fn shutdown_signal() {
}
}
+fn spawn_regional_ingress_health_sweep(state: ControlApiState) {
+ tokio::spawn(async move {
+ let mut interval = tokio::time::interval(std::time::Duration::from_secs(15));
+ interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ loop {
+ interval.tick().await;
+ let Some(db) = state.try_database() else {
+ continue;
+ };
+ let ingresses = match domain::ingress::list(db).await {
+ Ok(items) => items
+ .into_iter()
+ .filter(|item| item.enabled)
+ .collect::>(),
+ Err(error) => {
+ tracing::warn!(operation = "control_api.ingress_health_list", %error, "failed to load regional ingresses");
+ continue;
+ }
+ };
+ for ingress in ingresses {
+ if let Err(error) = domain::ingress::probe_regional_ingress(
+ db,
+ &ingress,
+ time::OffsetDateTime::now_utc(),
+ )
+ .await
+ {
+ tracing::warn!(operation = "control_api.ingress_health_probe", ingress_id = %ingress.id, %error, "regional ingress health probe failed");
+ }
+ }
+ }
+ });
+}
+
+fn spawn_regional_ingress_certificate_sweep(state: ControlApiState) {
+ tokio::spawn(async move {
+ let mut interval = tokio::time::interval(std::time::Duration::from_secs(15));
+ interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ loop {
+ interval.tick().await;
+ let Some(db) = state.try_database() else {
+ continue;
+ };
+ let secret = state.config.read().unwrap().secrets.secret_key.clone();
+ if let Err(error) = domain::acme::sweep(db, &secret).await {
+ tracing::warn!(
+ operation = "control_api.acme.sweep",
+ %error,
+ "regional ingress certificate sweep failed"
+ );
+ }
+ }
+ });
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/apps/node/Cargo.toml b/apps/node/Cargo.toml
index f34548e..12f1c39 100644
--- a/apps/node/Cargo.toml
+++ b/apps/node/Cargo.toml
@@ -29,6 +29,7 @@ toml.workspace = true
url.workspace = true
walkdir.workspace = true
reqwest.workspace = true
+tempfile = "3"
serde.workspace = true
serde_json.workspace = true
tokio-tungstenite.workspace = true
diff --git a/apps/node/src/client/mod.rs b/apps/node/src/client/mod.rs
index 2e91b96..a240426 100644
--- a/apps/node/src/client/mod.rs
+++ b/apps/node/src/client/mod.rs
@@ -5,14 +5,15 @@ use std::{path::Path, time::Duration};
use anyhow::Context;
use futures_util::StreamExt;
use grass_node_protocol::{
- AppendBuildLogRequest, AppendBuildLogResponse, ClaimRequest, ClaimResponse,
- ExchangePreviewCodeRequest, ExchangePreviewCodeResponse, HeartbeatRequest, HeartbeatResponse,
- ObserveSshHostKeyRequest, ObserveSshHostKeyResponse, RedeemGitCredentialRequest,
- RedeemGitCredentialResponse, RegisterRequest, RegisterResponse, ReportServeStatusRequest,
- ReportServeStatusResponse, ResolveHostResponse, RouteSnapshotResponse, ServeAssignment,
- ServeAssignmentsResponse, SsrLeaseResponse, StageRequest, StageResponse,
- StartPreviewAuthorizationRequest, StartPreviewAuthorizationResponse, UploadArtifactResponse,
- VerifyPreviewGrantRequest, VerifyPreviewGrantResponse, artifact_headers,
+ AppendBuildLogRequest, AppendBuildLogResponse, CertificateBundlesResponse, ClaimRequest,
+ ClaimResponse, ExchangePreviewCodeRequest, ExchangePreviewCodeResponse, HeartbeatRequest,
+ HeartbeatResponse, ObserveSshHostKeyRequest, ObserveSshHostKeyResponse,
+ RedeemGitCredentialRequest, RedeemGitCredentialResponse, RegisterRequest, RegisterResponse,
+ ReportServeStatusRequest, ReportServeStatusResponse, ResolveHostResponse,
+ RouteSnapshotResponse, ServeAssignment, ServeAssignmentsResponse, SsrLeaseResponse,
+ StageRequest, StageResponse, StartPreviewAuthorizationRequest,
+ StartPreviewAuthorizationResponse, UploadArtifactResponse, VerifyPreviewGrantRequest,
+ VerifyPreviewGrantResponse, artifact_headers,
};
use serde::Deserialize;
use serde::de::DeserializeOwned;
@@ -458,6 +459,17 @@ impl ControlApiClient {
.map_err(RouteSnapshotError::Infrastructure)
}
+ pub async fn certificate_bundles(&self) -> anyhow::Result {
+ let response = self
+ .http
+ .get(self.url("/serve/certificates"))
+ .bearer_auth(&self.token)
+ .send()
+ .await
+ .context("serve.certificates: request failed")?;
+ Self::unwrap_envelope(response, "serve.certificates").await
+ }
+
#[allow(dead_code)] // Wired by the serve resolver in Milestone 10.
pub async fn resolve_host(&self, host: &str) -> anyhow::Result> {
let response = self
diff --git a/apps/node/src/config.rs b/apps/node/src/config.rs
index ee6f2be..0cb000b 100644
--- a/apps/node/src/config.rs
+++ b/apps/node/src/config.rs
@@ -51,6 +51,8 @@ pub struct NodeIdentityConfig {
pub control_api: String,
#[serde(default = "default_node_token")]
pub node_token: String,
+ #[serde(default = "default_node_region")]
+ pub region: String,
#[serde(default = "default_node_work_root")]
pub work_root: String,
#[serde(default)]
@@ -63,6 +65,7 @@ impl Default for NodeIdentityConfig {
id: default_node_id(),
control_api: default_control_api(),
node_token: default_node_token(),
+ region: default_node_region(),
work_root: default_node_work_root(),
capabilities: NodeCapabilitiesConfig::default(),
}
@@ -306,6 +309,7 @@ impl NodeConfig {
id: self.node.id.clone(),
control_api: self.node.control_api.clone(),
work_root: self.node.work_root.clone(),
+ region: self.node.region.clone(),
capabilities: NodeCapabilities {
build: self.node.capabilities.build,
serve: self.node.capabilities.serve,
@@ -373,6 +377,7 @@ impl NodeConfig {
self.node.id.clone_from(&desired.node.id);
self.node.control_api.clone_from(&desired.node.control_api);
self.node.work_root.clone_from(&desired.node.work_root);
+ self.node.region.clone_from(&desired.node.region);
self.node.capabilities.build = desired.node.capabilities.build;
self.node.capabilities.serve = desired.node.capabilities.serve;
self.build.concurrency = desired.build.concurrency;
@@ -465,6 +470,8 @@ impl NodeConfig {
pub fn validate(&self) -> anyhow::Result<()> {
let capabilities = &self.node.capabilities;
+ grass_validator::normalize_region(&self.node.region)
+ .map_err(|error| anyhow::anyhow!("invalid node region: {error}"))?;
if !capabilities.build && !capabilities.serve {
anyhow::bail!("node must enable build or serve");
}
@@ -486,6 +493,7 @@ fn apply_env(config: &mut NodeConfig) -> Result<(), ConfigError> {
overlay_string("GWNODE_ID", &mut config.node.id);
overlay_string("GWNODE_CONTROL_API", &mut config.node.control_api);
overlay_string("GWNODE_NODE_TOKEN", &mut config.node.node_token);
+ overlay_string("GWNODE_REGION", &mut config.node.region);
overlay_string("GWNODE_WORK_ROOT", &mut config.node.work_root);
overlay_u16("GWNODE_BUILD_CONCURRENCY", &mut config.build.concurrency)?;
overlay_u64(
@@ -537,6 +545,10 @@ fn default_node_token() -> String {
"change-me".to_owned()
}
+fn default_node_region() -> String {
+ "default".to_owned()
+}
+
fn default_node_work_root() -> String {
"/data/node".to_owned()
}
diff --git a/apps/node/src/lifecycle.rs b/apps/node/src/lifecycle.rs
index ee6e675..3ff8a0d 100644
--- a/apps/node/src/lifecycle.rs
+++ b/apps/node/src/lifecycle.rs
@@ -53,6 +53,7 @@ pub fn registration_request(
build: config.node.capabilities.build,
serve: config.node.capabilities.serve,
},
+ region: config.node.region.clone(),
build_concurrency: if config.node.capabilities.build {
config.build.concurrency
} else {
diff --git a/apps/node/src/main.rs b/apps/node/src/main.rs
index cf124d2..3d77dd3 100644
--- a/apps/node/src/main.rs
+++ b/apps/node/src/main.rs
@@ -95,45 +95,48 @@ async fn main() -> anyhow::Result<()> {
});
let mut ssr_manager_for_shutdown = None;
- let (artifact_sync, route_refresh, ssr_reaper, serve_task) = if config.node.capabilities.serve {
- let ssr_manager = Arc::new(serve::ssr::SsrManager::with_client(
- runtime,
- node_id,
- &config,
- client.clone(),
- ));
- ssr_manager_for_shutdown = Some(ssr_manager.clone());
- let ssr_reaper = ssr_manager.clone().spawn_reaper();
- let route_table = Arc::new(serve::routes::RouteTable::default());
- let route_refresh = serve::routes::spawn(
- client.clone(),
- route_table.clone(),
- node_id,
- ssr_manager.clone(),
- );
- let serve_state = Arc::new(serve::ServeState::new(
- client.clone(),
- node_id,
- gateway_token
- .clone()
- .expect("Serve registration requires a gateway token"),
- route_table,
- &config,
- ssr_manager,
- ));
- let artifact_sync = serve::sync::spawn(
- client.clone(),
- std::path::PathBuf::from(&config.serve.artifact_cache_root),
- );
- (
- Some(artifact_sync),
- Some(route_refresh),
- Some(ssr_reaper),
- Some(serve::spawn(serve_state, &config)),
- )
- } else {
- (None, None, None, None)
- };
+ let (artifact_sync, route_refresh, certificate_sync, ssr_reaper, serve_task) =
+ if config.node.capabilities.serve {
+ let ssr_manager = Arc::new(serve::ssr::SsrManager::with_client(
+ runtime,
+ node_id,
+ &config,
+ client.clone(),
+ ));
+ ssr_manager_for_shutdown = Some(ssr_manager.clone());
+ let ssr_reaper = ssr_manager.clone().spawn_reaper();
+ let route_table = Arc::new(serve::routes::RouteTable::default());
+ let route_refresh = serve::routes::spawn(
+ client.clone(),
+ route_table.clone(),
+ node_id,
+ ssr_manager.clone(),
+ );
+ let serve_state = Arc::new(serve::ServeState::new(
+ client.clone(),
+ node_id,
+ gateway_token
+ .clone()
+ .expect("Serve registration requires a gateway token"),
+ route_table,
+ &config,
+ ssr_manager,
+ ));
+ let artifact_sync = serve::sync::spawn(
+ client.clone(),
+ std::path::PathBuf::from(&config.serve.artifact_cache_root),
+ );
+ let certificate_sync = serve::certificates::spawn(client.clone(), config.clone());
+ (
+ Some(artifact_sync),
+ Some(route_refresh),
+ Some(certificate_sync),
+ Some(ssr_reaper),
+ Some(serve::spawn(serve_state, &config)),
+ )
+ } else {
+ (None, None, None, None, None)
+ };
wait_for_shutdown().await;
@@ -146,6 +149,9 @@ async fn main() -> anyhow::Result<()> {
if let Some(route_refresh) = route_refresh {
route_refresh.abort();
}
+ if let Some(certificate_sync) = certificate_sync {
+ certificate_sync.abort();
+ }
if let Some(ssr_reaper) = ssr_reaper {
ssr_reaper.abort();
}
diff --git a/apps/node/src/serve/certificates.rs b/apps/node/src/serve/certificates.rs
new file mode 100644
index 0000000..f2d73cf
--- /dev/null
+++ b/apps/node/src/serve/certificates.rs
@@ -0,0 +1,145 @@
+//! Certificate bundle synchronization for Serve Nodes.
+
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use grass_node_protocol::CertificateBundle;
+use tokio::io::AsyncWriteExt;
+use tracing::{info, warn};
+
+use crate::{client::ControlApiClient, config::NodeConfig};
+
+const CERTIFICATE_REFRESH_INTERVAL: Duration = Duration::from_secs(60);
+
+pub fn spawn(client: ControlApiClient, config: NodeConfig) -> tokio::task::JoinHandle<()> {
+ tokio::spawn(async move {
+ let mut interval = tokio::time::interval(CERTIFICATE_REFRESH_INTERVAL);
+ interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ loop {
+ interval.tick().await;
+ match client.certificate_bundles().await {
+ Ok(response) => {
+ for bundle in response.bundles {
+ if let Err(error) =
+ install_bundle(Path::new(&config.serve.artifact_cache_root), &bundle)
+ .await
+ {
+ warn!(
+ operation = "node.serve.certificates.install_failed",
+ ingress_id = %bundle.ingress_id,
+ %error,
+ "failed to install certificate bundle"
+ );
+ }
+ }
+ }
+ Err(error) => warn!(
+ operation = "node.serve.certificates.refresh_failed",
+ %error,
+ "failed to refresh regional ingress certificates"
+ ),
+ }
+ }
+ })
+}
+
+pub async fn install_bundle(cache_root: &Path, bundle: &CertificateBundle) -> anyhow::Result<()> {
+ let hostname = safe_hostname(&bundle.hostname)?;
+ let directory = cache_root.join("certificates");
+ tokio::fs::create_dir_all(&directory).await?;
+ let certificate_path = directory.join(format!("{hostname}.fullchain.pem"));
+ let private_key_path = directory.join(format!("{hostname}.key.pem"));
+ atomic_write(&certificate_path, bundle.certificate_pem.as_bytes()).await?;
+ if let Err(error) = atomic_write(&private_key_path, bundle.private_key_pem.as_bytes()).await {
+ let _ = tokio::fs::remove_file(&certificate_path).await;
+ return Err(error);
+ }
+ info!(
+ operation = "node.serve.certificates.installed",
+ ingress_id = %bundle.ingress_id,
+ hostname = %bundle.hostname,
+ "regional ingress certificate installed"
+ );
+ Ok(())
+}
+
+fn safe_hostname(hostname: &str) -> anyhow::Result {
+ let normalized = grass_validator::normalize_host(hostname)
+ .map_err(|error| anyhow::anyhow!("invalid certificate hostname: {error}"))?;
+ if normalized.len() > 253
+ || !normalized
+ .bytes()
+ .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-'))
+ {
+ anyhow::bail!("certificate hostname contains unsafe path characters");
+ }
+ Ok(normalized)
+}
+
+async fn atomic_write(path: &Path, bytes: &[u8]) -> anyhow::Result<()> {
+ let temp_path = PathBuf::from(format!(
+ "{}.tmp-{}",
+ path.display(),
+ uuid::Uuid::now_v7().simple()
+ ));
+ let result = async {
+ let mut file = tokio::fs::File::create(&temp_path).await?;
+ file.write_all(bytes).await?;
+ file.sync_all().await?;
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt;
+ file.set_permissions(std::fs::Permissions::from_mode(0o600))
+ .await?;
+ }
+ drop(file);
+ tokio::fs::rename(&temp_path, path).await?;
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt;
+ tokio::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).await?;
+ }
+ Ok::<(), anyhow::Error>(())
+ }
+ .await;
+ if result.is_err() {
+ let _ = tokio::fs::remove_file(&temp_path).await;
+ }
+ result
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::os::unix::fs::PermissionsExt;
+
+ #[tokio::test]
+ async fn installs_bundles_atomically_with_private_permissions() {
+ let root = tempfile::tempdir().unwrap();
+ install_bundle(
+ root.path(),
+ &CertificateBundle {
+ ingress_id: uuid::Uuid::now_v7(),
+ hostname: "edge.example.com".to_owned(),
+ certificate_pem: "CERT".to_owned(),
+ private_key_pem: "KEY".to_owned(),
+ issued_at_unix: None,
+ expires_at_unix: None,
+ revision: String::new(),
+ },
+ )
+ .await
+ .unwrap();
+ let key = root.path().join("certificates/edge.example.com.key.pem");
+ assert_eq!(tokio::fs::read_to_string(&key).await.unwrap(), "KEY");
+ assert_eq!(
+ tokio::fs::metadata(key).await.unwrap().permissions().mode() & 0o777,
+ 0o600
+ );
+ }
+
+ #[test]
+ fn rejects_unsafe_certificate_hostnames() {
+ assert!(safe_hostname("../secret").is_err());
+ }
+}
diff --git a/apps/node/src/serve/mod.rs b/apps/node/src/serve/mod.rs
index 1d86e05..cea6736 100644
--- a/apps/node/src/serve/mod.rs
+++ b/apps/node/src/serve/mod.rs
@@ -6,6 +6,7 @@
//! outputs use strict path normalization, while SSR outputs are proxied to a
//! deployment service container started on demand by [`ssr::SsrManager`].
+pub mod certificates;
pub mod routes;
pub mod ssr;
pub mod static_files;
@@ -1404,6 +1405,7 @@ mod tests {
revision: "before-withdrawal".to_owned(),
routes: vec![ServeRoute {
host: "app.example.com".to_owned(),
+ region: "default".to_owned(),
deployment_id,
target_node_id: Uuid::now_v7(),
target_base_url: format!("http://{authority_address}"),
diff --git a/apps/node/src/serve/release_smoke.rs b/apps/node/src/serve/release_smoke.rs
index a184129..9449b57 100644
--- a/apps/node/src/serve/release_smoke.rs
+++ b/apps/node/src/serve/release_smoke.rs
@@ -332,6 +332,7 @@ async fn checks_out_builds_packages_stages_and_serves_vite() -> anyhow::Result<(
revision: "release-smoke-v1".to_owned(),
routes: vec![ServeRoute {
host: FIXTURE_HOST.to_owned(),
+ region: "default".to_owned(),
deployment_id,
target_node_id: node_id,
target_base_url: "http://127.0.0.1:1".to_owned(),
diff --git a/apps/node/src/serve/routes.rs b/apps/node/src/serve/routes.rs
index f460c5b..eba5e2f 100644
--- a/apps/node/src/serve/routes.rs
+++ b/apps/node/src/serve/routes.rs
@@ -168,6 +168,7 @@ mod tests {
fn route(host: &str, deployment_id: Uuid) -> ServeRoute {
ServeRoute {
host: host.to_owned(),
+ region: "default".to_owned(),
deployment_id,
target_node_id: Uuid::now_v7(),
target_base_url: "http://node-a:8080".to_owned(),
diff --git a/crates/node-protocol/src/lib.rs b/crates/node-protocol/src/lib.rs
index 4e2f13a..62e808f 100644
--- a/crates/node-protocol/src/lib.rs
+++ b/crates/node-protocol/src/lib.rs
@@ -51,6 +51,8 @@ pub struct NodeIdentityConfiguration {
pub id: String,
pub control_api: String,
pub work_root: String,
+ #[serde(default = "default_region")]
+ pub region: String,
pub capabilities: NodeCapabilities,
}
@@ -144,6 +146,8 @@ pub struct RegisterRequest {
pub name: String,
pub version: String,
pub capabilities: NodeCapabilities,
+ #[serde(default = "default_region")]
+ pub region: String,
pub build_concurrency: u16,
/// Public base URL of the Node serve listener, when known.
#[serde(default)]
@@ -434,6 +438,8 @@ pub struct SsrLeaseResponse {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ServeRoute {
pub host: String,
+ #[serde(default = "default_region")]
+ pub region: String,
pub deployment_id: Uuid,
pub target_node_id: Uuid,
pub target_base_url: String,
@@ -447,6 +453,53 @@ pub struct RouteSnapshotResponse {
pub routes: Vec,
}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CertificateBundle {
+ pub ingress_id: Uuid,
+ pub hostname: String,
+ pub certificate_pem: String,
+ pub private_key_pem: String,
+ pub issued_at_unix: Option,
+ #[serde(default)]
+ pub revision: String,
+ #[serde(default)]
+ pub expires_at_unix: Option,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CertificateBundlesResponse {
+ pub bundles: Vec,
+ #[serde(default)]
+ pub challenges: Vec,
+ #[serde(default)]
+ pub challenge_revision: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HttpChallenge {
+ pub hostname: String,
+ pub token: String,
+ pub key_authorization: String,
+ pub expires_at_unix: i64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct InstalledCertificate {
+ pub ingress_id: Uuid,
+ pub revision: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ReportIngressStatusRequest {
+ pub certificates: Vec,
+ pub challenge_revision: String,
+ pub tls_ready: bool,
+}
+
+fn default_region() -> String {
+ "default".to_owned()
+}
+
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResolveHostResponse {
pub deployment_id: Uuid,
@@ -715,6 +768,7 @@ mod tests {
let route = ServeRoute {
host: "app.example.com".to_owned(),
+ region: "eu-west".to_owned(),
deployment_id: Uuid::nil(),
target_node_id: Uuid::nil(),
target_base_url: "http://node-1:8080".to_owned(),
@@ -724,6 +778,7 @@ mod tests {
let parsed: ServeRoute =
serde_json::from_slice(&serde_json::to_vec(&route).unwrap()).unwrap();
assert_eq!(parsed.host, "app.example.com");
+ assert_eq!(parsed.region, "eu-west");
assert_eq!(parsed.resources, resources);
assert_eq!(parsed.access, ServeAccess::TeamOrPlatformAdmin);
}
diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs
index cca2086..ca6b7bb 100644
--- a/crates/validator/src/lib.rs
+++ b/crates/validator/src/lib.rs
@@ -87,6 +87,14 @@ pub fn normalize_slug(value: &str) -> Result {
Ok(normalized)
}
+pub fn normalize_region(value: &str) -> Result {
+ let normalized = normalize_slug(value)?;
+ if normalized.len() > 64 {
+ return Err(SlugError::TooLong);
+ }
+ Ok(normalized)
+}
+
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum HostError {
Empty,
@@ -194,6 +202,12 @@ mod tests {
);
}
+ #[test]
+ fn normalizes_regions_with_a_stable_short_slug() {
+ assert_eq!(normalize_region(" EU_West 1 ").unwrap(), "eu-west-1");
+ assert_eq!(normalize_region(&"a".repeat(65)), Err(SlugError::TooLong));
+ }
+
#[test]
fn rejects_empty_invalid_and_oversized_slugs() {
assert_eq!(normalize_slug("---"), Err(SlugError::Empty));
diff --git a/docs/self-hosting.md b/docs/self-hosting.md
index 4e1eff8..4fe40d7 100644
--- a/docs/self-hosting.md
+++ b/docs/self-hosting.md
@@ -266,6 +266,12 @@ socket = "unix:///run/user/1000/podman/podman.sock"
default_build_image = "docker.io/library/node:22"
```
+Nodes are assigned to the `default` region unless `node.region` (or
+`GWNODE_REGION`) is set to another region slug. Deployments can request a
+region and the scheduler will only place them on healthy Serve Nodes in that
+region. Keep the same region on every Node that should accept those
+deployments.
+
```sh
just run node # or: grass-node --config node.toml
```
@@ -395,6 +401,32 @@ editing a source only overwrites the fields you fill in.
A **Manual** source assigns domains without touching DNS; bindings stay
`pending` until an operator creates the record and re-runs provisioning.
+### Regional CNAME ingress and certificates
+
+Platform administrators can create one ingress hostname per region under
+**Administration → Regional ingresses**. A project custom domain then shows a
+CNAME target for that region and a TXT ownership record named
+`_grass.`. Publish both records before serving traffic. The Control
+API derives the TXT value from its secret key and the binding id, so it is
+stable for retries but cannot be guessed from the hostname alone.
+
+Ingress guidance also lists the same-region Serve Nodes with a fresh heartbeat,
+the configured health-check path and interval, and whether the original Host
+header is preserved when traffic reaches the deployment. Disabled or stale
+nodes are omitted; keep at least two healthy Serve Nodes in a production
+region when failover is required.
+
+Regional ingress records expose certificate controls for Let's Encrypt,
+ZeroSSL, or manual certificates. The `certificate_status`, expiry, and error
+fields are the source of truth for the configured ACME/DNS-01 reconciler. For
+Let's Encrypt or ZeroSSL, configure a supported DNS-01 provider and grant the
+Control API only the zone permissions needed to create and remove the
+challenge TXT record. The ingress hostname remains an ordinary HTTP(S)
+endpoint in this release: terminate TLS at the regional proxy or load
+balancer, and use the status fields and TXT record guidance to coordinate
+certificate issuance and renewal there. Do not mark a certificate active until
+the proxy has installed the renewed certificate.
+
Custom domains use the Domain Review Policy configured under Administration
settings. The platform default is `auto`; a Team Group can override it with
`auto`, `manual`, or inherit the platform default. Manual review keeps a new
diff --git a/node.toml.example b/node.toml.example
index 3edbb8f..b050e90 100644
--- a/node.toml.example
+++ b/node.toml.example
@@ -2,6 +2,7 @@
id = "local-node-1"
control_api = "http://127.0.0.1:7817"
node_token = "change-me"
+region = "default"
work_root = "/data/node"
[node.capabilities]