feat: run preview cobalt on fly through alchemy

.github/workflows/preview.yaml

@@ -15,7 +15,7 @@ env:
   STAGE: pr-${{ github.event.pull_request.number }}
   CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
   CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
-  COBALT_API_KEY: ${{ secrets.COBALT_API_KEY }}
+  FLY_API_TOKEN: ${{ secrets.FLY_PREVIEW_API_TOKEN }}
 
 jobs:
   deploy:
@@ -25,7 +25,7 @@ jobs:
       github.event.action != 'closed' &&
       github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     permissions:
       contents: read
       pull-requests: write
@@ -43,6 +43,9 @@ jobs:
 
       - run: vp install
 
+      - name: Log in to the fly registry
+        run: echo "$FLY_API_TOKEN" | docker login registry.fly.io --username x --password-stdin
+
       - name: Deploy
         run: bun run alchemy:deploy --stage "$STAGE" --yes
 

Dockerfile.cobalt.dockerignore

@@ -0,0 +1,4 @@
+*
+!cobalt-machine-proxy.mjs
+!cobalt-soundcloud.mjs
+!cobalt-soundcloud-helpers.mjs

README.md

@@ -52,15 +52,20 @@ Deploys go through Cloudflare Workers Builds; `bun run deploy:preview` and
 `deploy:production` exist for manual uploads.
 
 Isolated preview stages are declared in `alchemy.run.ts`. Each stage gets its own Worker, D1
-database, and R2 bucket on a workers.dev URL. With `CLOUDFLARE_API_TOKEN`,
-`CLOUDFLARE_ACCOUNT_ID`, and `COBALT_API_KEY` set:
+database, and R2 bucket on a workers.dev URL, plus its own Cobalt Fly app,
+`tagium-cobalt-<stage>`, with a per-stage API key. Stage names must fit that 30-character Fly app
+name. Deploying needs Bun 1.4.2, Docker, `CLOUDFLARE_API_TOKEN`, `CLOUDFLARE_ACCOUNT_ID`, and an
+org-scoped `FLY_API_TOKEN`, whose org owns the Fly apps. The Cobalt image builds from production's
+pinned base in `registry.fly.io`, so log Docker in first:
 
 ```sh
+echo "$FLY_API_TOKEN" | docker login registry.fly.io -u x --password-stdin
 bun run alchemy:deploy --stage dev-yourname
 bun run alchemy:destroy --stage dev-yourname
 ```
 
-Pull requests deploy to `pr-<number>` once the `ALCHEMY_PREVIEWS` repository variable is `true`.
+Pull requests deploy to `pr-<number>` once the `ALCHEMY_PREVIEWS` repository variable is `true`,
+using the `CLOUDFLARE_API_TOKEN`, `CLOUDFLARE_ACCOUNT_ID`, and `FLY_PREVIEW_API_TOKEN` secrets.
 The `prod` and `production` stages are refused.
 
 ## License

alchemy.run.ts

@@ -1,27 +1,149 @@
 import * as Alchemy from "alchemy";
 import * as Cloudflare from "alchemy/Cloudflare";
+import * as Docker from "alchemy/Docker";
+import * as Fly from "alchemy/Fly";
+import * as Output from "alchemy/Output";
 import * as Config from "effect/Config";
 import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import { Buffer } from "node:buffer";
 
 const productionStages = new Set(["prod", "production"]);
 const shareArtworkRetentionSeconds = 90 * 24 * 60 * 60;
+const flyRegistry = "registry.fly.io";
+const cobaltAppNamePattern = /^tagium-cobalt-[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/;
+const cobaltProxyPort = 9000;
+
+const cobaltEnv = {
+  CUSTOM_INNERTUBE_CLIENT: "TV_SIMPLY",
+  YOUTUBE_GENERATE_PO_TOKENS: "1",
+  RATELIMIT_WINDOW: "60",
+  RATELIMIT_MAX: "1000",
+  TUNNEL_RATELIMIT_WINDOW: "60",
+  TUNNEL_RATELIMIT_MAX: "2000",
+  PROXY_MAX_QUEUED_RESOLVE: "96",
+  PROXY_MAX_QUEUED_TUNNEL: "192",
+  PROXY_MAX_QUEUE_WAIT_MS: "30000",
+  PROXY_DRAIN_TIMEOUT_MS: "270000",
+  API_AUTH_REQUIRED: "1",
+};
 
 const rateLimit = (name: string, namespaceId: string, limit: number) =>
   Cloudflare.RateLimit(name, { namespaceId, simple: { limit, period: 60 } });
 
+const cobaltAppName = (stage: string) =>
+  Effect.gen(function* () {
+    const name = `tagium-cobalt-${stage.toLowerCase()}`;
+    if (!cobaltAppNamePattern.test(name) || name.length > 30) {
+      return yield* Effect.die(
+        new Error(
+          `stage ${stage} cannot name a fly app: ${name} must be at most 30 lowercase letters, digits, or hyphens`,
+        ),
+      );
+    }
+    return name;
+  });
+
+const toUuid = (hex: string) =>
+  [hex.slice(0, 8), hex.slice(8, 12), hex.slice(12, 16), hex.slice(16, 20), hex.slice(20, 32)].join(
+    "-",
+  );
+
+const toCobaltKeysUrl = (apiKey: Redacted.Redacted<string>) =>
+  `data:application/json;base64,${Buffer.from(
+    JSON.stringify({ [Redacted.value(apiKey)]: { name: "tagium-worker" } }),
+  ).toString("base64")}`;
+
+const requireDigest = (repoDigest: string | undefined) => {
+  if (!repoDigest) {
+    throw new Error("the cobalt image push did not report a registry digest");
+  }
+  return repoDigest;
+};
+
 export default Alchemy.Stack(
   "tagium",
-  { providers: Cloudflare.providers(), state: Cloudflare.state() },
+  {
+    providers: Layer.mergeAll(Cloudflare.providers(), Fly.providers(), Docker.providers()),
+    state: Cloudflare.state(),
+  },
   Effect.gen(function* () {
     const stage = yield* Alchemy.Stage;
     if (productionStages.has(stage.toLowerCase())) {
       return yield* Effect.die(
         new Error(
-          `refusing stage ${stage}: production still deploys through wrangler and workers builds`,
+          `refusing stage ${stage}: production still deploys through wrangler, workers builds, and flyctl`,
         ),
       );
     }
 
+    const cobaltApp = yield* Fly.App("Cobalt", { name: yield* cobaltAppName(stage) });
+    yield* Fly.IpAssignment("CobaltSharedIpv4", { app: cobaltApp, type: "shared_v4" });
+    yield* Fly.IpAssignment("CobaltIpv6", { app: cobaltApp, type: "v6" });
+
+    const cobaltImage = yield* Docker.Image("CobaltImage", {
+      name: Output.interpolate`${flyRegistry}/${cobaltApp.appName}`,
+      registry: {
+        server: flyRegistry,
+        username: "x",
+        password: Config.Redacted("FLY_API_TOKEN"),
+      },
+      build: {
+        context: ".",
+        dockerfile: "Dockerfile.cobalt",
+        platform: "linux/amd64",
+        options: ["--provenance=false"],
+      },
+    });
+
+    const cobaltApiKey = Output.map(
+      yield* Alchemy.makeRandom("CobaltApiKey", { bytes: 16 }),
+      (seed) => Redacted.make(toUuid(Redacted.value(seed))),
+    );
+    const cobaltUrl = Output.interpolate`${cobaltApp.url}/`;
+
+    yield* Fly.Machine("CobaltMachine", {
+      app: cobaltApp,
+      region: "lax",
+      count: Config.Int("COBALT_MACHINE_COUNT").pipe(Config.withDefault(1)),
+      image: Output.map(cobaltImage.repoDigest, requireDigest),
+      init: { cmd: ["node", "/app/cobalt-machine-proxy.mjs"] },
+      guest: { cpuKind: "shared", cpus: 1, memoryMb: 1024 },
+      env: {
+        ...cobaltEnv,
+        API_URL: cobaltUrl,
+        API_KEY_URL: Output.map(cobaltApiKey, toCobaltKeysUrl),
+      },
+      services: [
+        {
+          protocol: "tcp",
+          internalPort: cobaltProxyPort,
+          ports: [
+            { port: 80, handlers: ["http"], forceHttps: true },
+            { port: 443, handlers: ["tls", "http"] },
+          ],
+          autostart: true,
+          autostop: "suspend",
+          minMachinesRunning: 0,
+          checks: [
+            {
+              type: "http",
+              port: cobaltProxyPort,
+              method: "GET",
+              path: "/readyz",
+              gracePeriod: "30s",
+              interval: "15s",
+              timeout: "2s",
+              headers: [{ name: "X-Forwarded-Proto", values: ["https"] }],
+            },
+          ],
+        },
+      ],
+      deploy: { strategy: "bluegreen", healthTimeout: "2 minutes" },
+      shutdown: { signal: "SIGTERM", timeout: "300 seconds" },
+    });
+
     const shareManifests = yield* Cloudflare.D1.Database("ShareManifests", {
       migrations: "migrations",
     });
@@ -52,8 +174,8 @@ export default Alchemy.Stack(
         traces: { enabled: false, persist: true, headSamplingRate: 1 },
       },
       env: {
-        COBALT_API_URL: "https://tagium-cobalt.fly.dev/",
-        COBALT_API_KEY: Config.Redacted("COBALT_API_KEY"),
+        COBALT_API_URL: cobaltUrl,
+        COBALT_API_KEY: cobaltApiKey,
         COBALT_MACHINE_AFFINITY_SECRET: cobaltMachineAffinitySecret,
         TAGIUM_DEPLOY_ENV: "preview",
         SHARE_MANIFESTS: shareManifests,
@@ -67,6 +189,6 @@ export default Alchemy.Stack(
       },
     });
 
-    return { url: app.url };
+    return { url: app.url, cobaltUrl: cobaltApp.url };
   }),
 );