diff --git a/apps/api/src/api/routes/admin-prompts.ts b/apps/api/src/api/routes/admin-prompts.ts index d8424e6766..83ddf8ed7f 100644 --- a/apps/api/src/api/routes/admin-prompts.ts +++ b/apps/api/src/api/routes/admin-prompts.ts @@ -15,12 +15,14 @@ * * Mounted by `admin.ts`, therefore already behind `requireDeploymentAdmin`. */ +import { repoRefFromOwnerName } from "@decocms/shared/git-providers"; import { Hono } from "hono"; import type { Env } from "@/api/hono-env"; +import { contentClientWithToken } from "@/git-providers/content"; import { - createGitDataClient, - type GitDataClient, -} from "@/decofile/github-git-data"; + type RepoContentClient, + requireBranchHead, +} from "@/git-providers/content/types"; import { githubConnectionAccessToken } from "@/oauth/github-mint"; import { resolveGithubConnection } from "@/tools/task-board/prs-get"; import { @@ -116,7 +118,7 @@ async function resolveActorOrg( /** A GitHub client on the acting admin's own connection. */ async function clientForActor( ctx: Ctx, -): Promise<{ gh: GitDataClient; org: { slug: string; name: string } }> { +): Promise<{ gh: RepoContentClient; org: { slug: string; name: string } }> { const org = await resolveActorOrg(ctx); // The mint path for a repo-scoped child connection reads `ctx.organization`, // which is exactly what this route doesn't have — bind the resolved org so @@ -137,7 +139,10 @@ async function clientForActor( throw new PromptEditorError("Reconnect GitHub to edit prompts", 400); } return { - gh: createGitDataClient({ ...PROMPT_REPO, accessToken }), + gh: contentClientWithToken( + repoRefFromOwnerName(PROMPT_REPO.owner, PROMPT_REPO.repo), + accessToken, + ), org: { slug: org.slug, name: org.name }, }; } @@ -173,12 +178,12 @@ export function applyPromptEdits( /** Every distinct source file the registry touches, read once at `ref`. */ async function readSources( - gh: GitDataClient, + gh: RepoContentClient, ref: string, ): Promise> { const paths = [...new Set(PROMPTS.map((p) => p.path))]; const texts = await Promise.all( - paths.map((path) => gh.getFileTextAtRef(ref, path)), + paths.map((path) => gh.readFileAtRef(ref, path)), ); const sources = new Map(); paths.forEach((path, i) => { @@ -200,7 +205,7 @@ export function createAdminPromptRoutes(): Hono { // Pin the read to the commit, not the branch: the sha goes back to the // client and is what a save is written against, so a push landing between // the two can be reported as a conflict instead of silently reverted. - const baseSha = await gh.getHeadSha(branch); + const baseSha = await requireBranchHead(gh, branch); const sources = await readSources(gh, baseSha); return c.json({ @@ -245,7 +250,7 @@ export function createAdminPromptRoutes(): Hono { const { gh } = await clientForActor(c.get("studioContext")); const base = await gh.getDefaultBranch(); - const baseSha = await gh.getHeadSha(base); + const baseSha = await requireBranchHead(gh, base); if (typeof body.baseSha === "string" && body.baseSha !== baseSha) { // The editor loaded an older HEAD; committing its text would revert // whatever landed since. The client reloads and the operator re-applies. @@ -266,29 +271,20 @@ export function createAdminPromptRoutes(): Hono { const changedPaths = [ ...new Set(edits.map((e) => PROMPTS.find((p) => p.id === e.id)!.path)), ]; - const entries = await Promise.all( - changedPaths.map(async (path) => ({ + const branch = `admin/prompts-${Date.now().toString(36)}`; + await gh.createBranch(branch, baseSha); + await gh.commitFiles({ + branch, + message: title, + expectedHead: baseSha, + changes: changedPaths.map((path) => ({ path, - mode: "100644", - type: "blob" as const, - sha: await gh.createBlob(sources.get(path)!), + content: sources.get(path)!, })), - ); - - const treeSha = await gh.createTree( - await gh.getCommitTreeSha(baseSha), - entries, - ); - const commitSha = await gh.createCommit({ - message: title, - treeSha, - parentShas: [baseSha], }); - const branch = `admin/prompts-${Date.now().toString(36)}`; - await gh.createRef(branch, commitSha); - const pr = await gh.createPullRequest({ base, head: branch, title }); + const pr = await gh.createChangeRequest({ base, head: branch, title }); - return c.json({ number: pr.number, url: pr.html_url, branch }); + return c.json({ number: pr.number, url: pr.url, branch }); }); app.onError((error, c) => { diff --git a/apps/api/src/api/routes/decofile.ts b/apps/api/src/api/routes/decofile.ts index f30c9960f1..3fc63e57e3 100644 --- a/apps/api/src/api/routes/decofile.ts +++ b/apps/api/src/api/routes/decofile.ts @@ -32,14 +32,18 @@ import { createMiddleware } from "hono/factory"; import { z } from "zod"; import { coAuthorFromStudioContext } from "@/lib/co-author-identity"; import { parseGithubRepoFromMetadata } from "@/tools/sandbox/sync-git-credentials"; -import { gitDataClientForRepo } from "@/decofile/client-for-repo"; +import { contentClientForProjectRepo } from "@/git-providers/content"; import { enqueueDecofilePatch, type DecofilePatch, } from "@/decofile/commit-coalescer"; import { signDraftToken, verifyDraftToken } from "@/decofile/draft-token"; -import { githubGitRebase } from "@/decofile/git-compat"; -import { GitHubApiError, type GitDataClient } from "@/decofile/github-git-data"; +import { repoGitRebase } from "@/decofile/git-compat"; +import { + type RepoContentClient, + repoErrorStatus, + RepoWriteConflict, +} from "@/git-providers/content/types"; import { readDecofileSnapshot } from "@/decofile/read-decofile"; import type { Env } from "../hono-env"; @@ -204,11 +208,11 @@ function requestApiHost(c: Context): string { return c.req.header("x-forwarded-host") ?? new URL(c.req.url).host; } -async function gitDataClientForScope( +async function contentClientForScope( c: Context, -): Promise { +): Promise { const scope = c.get("decofileScope"); - return gitDataClientForRepo( + return contentClientForProjectRepo( c.var.studioContext, scope.organizationId, scope.githubRepo, @@ -216,20 +220,20 @@ async function gitDataClientForScope( } function errorResponse(c: Context, err: unknown) { - if (err instanceof GitHubApiError) { - // 401/403 from GitHub means our credential, not the caller's — map to 502 - // so the client doesn't treat it as a Studio auth failure. + const message = err instanceof Error ? err.message : String(err); + if (err instanceof RepoWriteConflict) return c.json({ error: message }, 409); + const providerStatus = repoErrorStatus(err); + if (providerStatus !== null) { + /** 401/403 from the provider means OUR credential, not the caller's — map + * it to 502 so the client doesn't treat it as a Studio auth failure. */ const status = - err.status === 404 + providerStatus === 404 ? 404 - : err.status === 409 + : providerStatus === 409 || providerStatus === 422 ? 409 - : err.status === 422 - ? 409 - : 502; - return c.json({ error: err.message }, status); + : 502; + return c.json({ error: message }, status); } - const message = err instanceof Error ? err.message : String(err); return c.json({ error: message }, 500); } @@ -242,7 +246,7 @@ export function createDecofileRoutes() { app.get("/:virtualMcpId/:branch", async (c) => { const scope = c.get("decofileScope"); try { - const client = await gitDataClientForScope(c); + const client = await contentClientForScope(c); const snapshot = await readDecofileSnapshot( client, scope.branch, @@ -297,16 +301,16 @@ export function createDecofileRoutes() { ? `${scope.packagePath}/.deco/meta.gen.json` : ".deco/meta.gen.json"; try { - const client = await gitDataClientForScope(c); + const client = await contentClientForScope(c); /** The thread branch may not be materialized on GitHub yet (it forks * from default at first CMS touch), so fall back to the default branch — * meta.gen.json is a code artifact the CMS never edits, so the default's * copy is the schema the forked branch would carry anyway. */ - let text = await client.getFileTextAtRef(scope.branch, metaPath); + let text = await client.readFileAtRef(scope.branch, metaPath); if (text === null) { const defaultBranch = await client.getDefaultBranch(); if (defaultBranch !== scope.branch) { - text = await client.getFileTextAtRef(defaultBranch, metaPath); + text = await client.readFileAtRef(defaultBranch, metaPath); } } if (text === null) { @@ -364,7 +368,7 @@ export function createDecofileRoutes() { } try { - const client = await gitDataClientForScope(c); + const client = await contentClientForScope(c); const sha = await enqueueDecofilePatch( `${scope.organizationId}/${scope.virtualMcpId}/${scope.branch}`, { @@ -389,7 +393,7 @@ export function createDecofileRoutes() { app.post("/:virtualMcpId/:branch/publish", async (c) => { const scope = c.get("decofileScope"); try { - const client = await gitDataClientForScope(c); + const client = await contentClientForScope(c); const baseBranch = await client.getDefaultBranch(); if (baseBranch === scope.branch) { return c.json({ error: "Branch is already the default branch" }, 400); @@ -398,29 +402,29 @@ export function createDecofileRoutes() { try { let sha: string | null; try { - sha = await client.mergeBranch(baseBranch, scope.branch, message); + sha = await client.mergeBranches(baseBranch, scope.branch, message); } catch (err) { - if (!(err instanceof GitHubApiError && err.status === 409)) throw err; + if (repoErrorStatus(err) !== 409) throw err; // Sync branch-wins first; the branch then sits on base and this FFs. - await githubGitRebase(client, scope.branch, baseBranch); - sha = await client.mergeBranch(baseBranch, scope.branch, message); + await repoGitRebase(client, scope.branch, baseBranch); + sha = await client.mergeBranches(baseBranch, scope.branch, message); } return sha ? c.json({ result: "merged", sha }) : c.json({ result: "up-to-date" }); } catch (err) { - if (err instanceof GitHubApiError && err.status === 409) { + if (repoErrorStatus(err) === 409) { return c.json({ error: "merge-conflict" }, 409); } // 405 = merge blocked (protected base branch) → fall back to a PR. - if (err instanceof GitHubApiError && err.status === 405) { - const existing = await client.findOpenPullRequest( + if (repoErrorStatus(err) === 405) { + const existing = await client.findOpenChangeRequest( baseBranch, scope.branch, ); const pr = existing ?? - (await client.createPullRequest({ + (await client.createChangeRequest({ base: baseBranch, head: scope.branch, title: `Publish ${scope.branch}`, @@ -428,7 +432,7 @@ export function createDecofileRoutes() { return c.json({ result: "pull-request", number: pr.number, - url: pr.html_url, + url: pr.url, }); } throw err; @@ -441,7 +445,7 @@ export function createDecofileRoutes() { app.get("/:virtualMcpId/:branch/status", async (c) => { const scope = c.get("decofileScope"); try { - const client = await gitDataClientForScope(c); + const client = await contentClientForScope(c); const baseBranch = await client.getDefaultBranch(); // Null lastCommitAt == "no age, never auto-switch off this branch". if (baseBranch === scope.branch) { @@ -455,17 +459,17 @@ export function createDecofileRoutes() { try { const [{ aheadBy, behindBy }, head] = await Promise.all([ client.compare(baseBranch, scope.branch), - client.getBranchHead(scope.branch), + client.getBranch(scope.branch), ]); return c.json({ baseBranch, aheadBy, behindBy, - lastCommitAt: head.committedAt, + lastCommitAt: head?.committedAt ?? null, }); } catch (err) { // A thread-minted branch not materialized yet has no drift and no age. - if (err instanceof GitHubApiError && err.status === 404) { + if (repoErrorStatus(err) === 404) { return c.json({ baseBranch, aheadBy: 0, diff --git a/apps/api/src/api/routes/sandbox-proxy.ts b/apps/api/src/api/routes/sandbox-proxy.ts index 0d95543e69..d42ca2c6e0 100644 --- a/apps/api/src/api/routes/sandbox-proxy.ts +++ b/apps/api/src/api/routes/sandbox-proxy.ts @@ -52,16 +52,18 @@ import { suggestCommitMessageWithLlm, } from "../../lib/suggest-commit-message"; import { judgeRequiresReviewWithLlm } from "../../lib/judge-requires-review"; -import { gitDataClientForRepo } from "../../decofile/client-for-repo"; +import { contentClientForProjectRepo } from "../../git-providers/content"; import { - GitHubApiError, - GitHubRateLimitError, -} from "../../decofile/github-git-data"; + repoErrorStatus, + repoRateLimitRetryAfterMs, + RepoWriteConflict, +} from "../../git-providers/content/types"; +import { GitProviderError } from "../../git-providers/types"; import { - githubGitDiff, - githubGitDiscard, - githubGitRebase, - githubGitStatus, + repoGitDiff, + repoGitDiscard, + repoGitRebase, + repoGitStatus, } from "../../decofile/git-compat"; import { buildLoaderInvokeUrl, @@ -382,37 +384,42 @@ async function fastPreviewGitClient(c: Context) { connectionIds, ); if (!githubRepo) { - throw new GitHubApiError( - 404, - "AUTH", - "repo", - "Project has no GitHub repository", - ); + throw new GitProviderError({ + provider: "github", + status: 404, + message: "Project has no GitHub repository", + }); } const organization = requireOrganization(ctx); - return gitDataClientForRepo(ctx, organization.id, githubRepo); + return contentClientForProjectRepo(ctx, organization.id, githubRepo); } function fastPreviewGitError(c: Context, err: unknown): Response { - /** 429 with GitHub's own wait, so the client backs off instead of retrying. */ - if (err instanceof GitHubRateLimitError) { - return c.json({ error: err.message }, 429, { + const message = err instanceof Error ? err.message : String(err); + /** 429 with the provider's own wait, so the client backs off instead of + * retrying immediately. */ + const retryAfterMs = repoRateLimitRetryAfterMs(err); + if (retryAfterMs !== undefined) { + return c.json({ error: message }, 429, { ...SANDBOX_PROXY_CACHE_HEADERS, - ...(err.retryAfterMs === null + ...(retryAfterMs === null ? {} - : { "Retry-After": String(Math.ceil(err.retryAfterMs / 1000)) }), + : { "Retry-After": String(Math.ceil(retryAfterMs / 1000)) }), }); } - if (err instanceof GitHubApiError) { + if (err instanceof RepoWriteConflict) { + return c.json({ error: message }, 409, SANDBOX_PROXY_CACHE_HEADERS); + } + const providerStatus = repoErrorStatus(err); + if (providerStatus !== null) { const status = - err.status === 404 + providerStatus === 404 ? 404 - : err.status === 409 || err.status === 422 + : providerStatus === 409 || providerStatus === 422 ? 409 : 502; - return c.json({ error: err.message }, status, SANDBOX_PROXY_CACHE_HEADERS); + return c.json({ error: message }, status, SANDBOX_PROXY_CACHE_HEADERS); } - const message = err instanceof Error ? err.message : String(err); return c.json({ error: message }, 502, SANDBOX_PROXY_CACHE_HEADERS); } @@ -453,7 +460,7 @@ async function proxyPreviewUpstream( async function fastPreviewGitStatus(c: Context): Promise { try { const client = await fastPreviewGitClient(c); - const status = await githubGitStatus(client, c.get("vmClaim").branch); + const status = await repoGitStatus(client, c.get("vmClaim").branch); return c.json(status, 200, SANDBOX_PROXY_CACHE_HEADERS); } catch (err) { return fastPreviewGitError(c, err); @@ -950,7 +957,7 @@ export const createSandboxRoutes = () => { base?: string; }; const client = await fastPreviewGitClient(c); - const diff = await githubGitDiff(client, claim.branch, body.base); + const diff = await repoGitDiff(client, claim.branch, body.base); return c.json(diff, 200, SANDBOX_PROXY_CACHE_HEADERS); } catch (err) { return fastPreviewGitError(c, err); @@ -1035,11 +1042,13 @@ export const createSandboxRoutes = () => { } try { const client = await fastPreviewGitClient(c); - await githubGitDiscard(client, claim.branch, filepaths); + await repoGitDiscard(client, claim.branch, filepaths); return c.json({ ok: true }, 200, SANDBOX_PROXY_CACHE_HEADERS); } catch (err) { const status = - err instanceof GitHubApiError && err.status === 409 ? 409 : 502; + repoErrorStatus(err) === 409 || err instanceof RepoWriteConflict + ? 409 + : 502; const message = err instanceof Error ? err.message : String(err); return c.json( { error: message }, @@ -1066,7 +1075,7 @@ export const createSandboxRoutes = () => { }; const client = await fastPreviewGitClient(c); const base = body.base ?? (await client.getDefaultBranch()); - await githubGitRebase(client, claim.branch, base); + await repoGitRebase(client, claim.branch, base); return c.json({ ok: true }, 200, SANDBOX_PROXY_CACHE_HEADERS); } catch (err) { return fastPreviewGitError(c, err); @@ -1156,8 +1165,8 @@ export const createSandboxRoutes = () => { await (async () => { const client = await fastPreviewGitClient(c); return Promise.all([ - githubGitStatus(client, claim.branch), - githubGitDiff(client, claim.branch), + repoGitStatus(client, claim.branch), + repoGitDiff(client, claim.branch), ]); })(); const suggestion = await suggestCommitMessageWithLlm(ctx, status, diff); @@ -1245,8 +1254,8 @@ export const createSandboxRoutes = () => { : await (async () => { const client = await fastPreviewGitClient(c); return Promise.all([ - githubGitStatus(client, claim.branch), - githubGitDiff(client, claim.branch), + repoGitStatus(client, claim.branch), + repoGitDiff(client, claim.branch), ]); })(); const verdict = await judgeRequiresReviewWithLlm( diff --git a/apps/api/src/decofile/client-for-repo.ts b/apps/api/src/decofile/client-for-repo.ts deleted file mode 100644 index 30ff3e311f..0000000000 --- a/apps/api/src/decofile/client-for-repo.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { GithubRepo } from "@decocms/shared/sdk/types"; -import type { StudioContext } from "@/core/studio-context"; -import { githubConnectionAccessToken } from "@/oauth/github-mint"; -import { RECONNECT_ERROR } from "@/oauth/token-refresh"; -import { createGitDataClient, GitHubApiError } from "./github-git-data"; -import type { GitDataClient } from "./github-git-data"; - -/** - * Git Data client for a project's linked repo: resolves the recorded - * connection (org-scoped) and mints/reuses the repo-scoped installation - * token. Shared by the decofile routes and the sandbox-less `/git/*` compat - * handlers so credential resolution cannot drift between them. - */ -export async function gitDataClientForRepo( - ctx: StudioContext, - organizationId: string, - githubRepo: GithubRepo, -): Promise { - if (!githubRepo.connectionId) { - throw new GitHubApiError( - 401, - "AUTH", - "connection", - "Project's GitHub connection is missing — reconnect GitHub", - ); - } - const connection = await ctx.storage.connections.findById( - githubRepo.connectionId, - organizationId, - ); - if (!connection) { - throw new GitHubApiError( - 401, - "AUTH", - "connection", - "Project's GitHub connection is missing — reconnect GitHub", - ); - } - const accessToken = await githubConnectionAccessToken(ctx, connection); - if (!accessToken) { - throw new GitHubApiError(401, "AUTH", "connection", RECONNECT_ERROR); - } - return createGitDataClient({ - owner: githubRepo.owner, - repo: githubRepo.name, - accessToken, - }); -} diff --git a/apps/api/src/decofile/commit-coalescer.ts b/apps/api/src/decofile/commit-coalescer.ts index dfad203935..0bd627f69f 100644 --- a/apps/api/src/decofile/commit-coalescer.ts +++ b/apps/api/src/decofile/commit-coalescer.ts @@ -3,9 +3,13 @@ import { type CoAuthorIdentity, } from "@decocms/sandbox/shared"; import { blockKeyToFileStem, mergeBlocks } from "@decocms/shared/decofile"; +import { repoIdentityKey } from "@decocms/shared/git-providers"; import { exponentialBackoffWithJitter, sleep } from "@decocms/shared/std"; -import type { GitDataClient, TreeWriteEntry } from "./github-git-data"; -import { GitHubApiError } from "./github-git-data"; +import { + type FileChange, + type RepoContentClient, + RepoWriteConflict, +} from "@/git-providers/content/types"; import { aliasPathsForKey, blockEntriesInTree, @@ -21,9 +25,10 @@ import { * commit is in flight merge into ONE pending batch, and the next flush lands * them all in a single commit. * - * Multi-replica safety comes from GitHub itself: `updateRef` is non-forced, so - * a concurrent writer (another replica, a dev pushing code) makes it fail - * non-fast-forward and the flush rebuilds on the fresh head and retries. + * Multi-replica safety comes from the provider itself: `commitFiles` guards on + * the head the batch was built against, so a concurrent writer (another + * replica, a dev pushing code) makes it fail with `RepoWriteConflict` and the + * flush rebuilds on the fresh head and retries. */ const MAX_CAS_ATTEMPTS = 3; @@ -34,7 +39,7 @@ export interface DecofilePatch { } export interface CommitDeps { - client: GitDataClient; + client: RepoContentClient; branch: string; packagePath: string | null; /** Acting user, appended as a `Co-authored-by:` trailer when present. */ @@ -123,43 +128,41 @@ async function commitBatch(batch: Batch): Promise { // Writes are session-only, so first-touch of a thread-minted branch may // materialize it here (a save can race ahead of the editor's first read). const headSha = await resolveOrCreateHead(client, branch); - const baseTreeSha = await client.getCommitTreeSha(headSha); - const tree = await client.getDecofileTree(baseTreeSha, packagePath); + const tree = await client.listDecofileEntries(headSha, packagePath); const entries = blockEntriesInTree(tree, packagePath); - const writes: TreeWriteEntry[] = []; + const writes: FileChange[] = []; // Post-patch view of the blocks dir, for blocks.gen.json regeneration. const nextBlocks = new Map< string, - { stem: string; sha: string | null; content?: string } + { stem: string; sha: string } | { stem: string; content: string } >(entries.map((e) => [e.path, { stem: e.stem, sha: e.sha }])); for (const [key, value] of batch.set) { const aliases = aliasPathsForKey(entries, key); const content = `${JSON.stringify(value, null, 2)}\n`; - const blobSha = await client.createBlob(content); // Write-through to the disk store (fail-open) so the read after this // save never re-fetches content the replica already has in hand. - await primeBlobCache(client.owner, client.repo, blobSha, content); + await primeBlobCache(client.repo, content); // Land on the existing on-disk spelling (a differently-encoded stem would // otherwise become a duplicate sibling); collapse extra aliases. const target = aliases[0] ?? `${blocksDirPath(packagePath)}/${blockKeyToFileStem(key)}.json`; - writes.push({ path: target, mode: "100644", type: "blob", sha: blobSha }); + writes.push({ path: target, content }); const targetStem = target.slice( target.lastIndexOf("/") + 1, -".json".length, ); - nextBlocks.set(target, { stem: targetStem, sha: blobSha, content }); + nextBlocks.set(target, { stem: targetStem, content }); for (const extra of aliases.slice(1)) { - writes.push({ path: extra, mode: "100644", type: "blob", sha: null }); + writes.push({ path: extra, deleted: true }); nextBlocks.delete(extra); } } for (const key of batch.del) { for (const alias of aliasPathsForKey(entries, key)) { - writes.push({ path: alias, mode: "100644", type: "blob", sha: null }); + writes.push({ path: alias, deleted: true }); nextBlocks.delete(alias); } } @@ -175,41 +178,37 @@ async function commitBatch(batch: Batch): Promise { const files = await Promise.all( [...nextBlocks.values()].map(async (b) => ({ stem: b.stem, - content: b.content ?? (await client.getBlobText(b.sha as string)), + content: "content" in b ? b.content : await client.readBlob(b.sha), })), ); const { decofile: genContent, skipped } = mergeBlocks(files); if (skipped.length > 0) { console.warn("decofile gen: dropped blocks that were not valid JSON", { - repo: `${client.owner}/${client.repo}`, + repo: repoIdentityKey(client.repo), branch, packagePath, blocks: skipped.map((s) => s.key), }); } - const genBlobSha = await client.createBlob(genContent); - writes.push({ - path: genPath, - mode: "100644", - type: "blob", - sha: genBlobSha, - }); + writes.push({ path: genPath, content: genContent }); } - const newTreeSha = await client.createTree(baseTreeSha, writes); - const commitSha = await client.createCommit({ - message: commitMessage(batch), - treeSha: newTreeSha, - parentShas: [headSha], - }); try { - await client.updateRef(branch, commitSha); - return commitSha; + const { sha } = await client.commitFiles({ + branch, + message: commitMessage(batch), + expectedHead: headSha, + changes: writes, + }); + return sha; } catch (err) { - // Non-fast-forward: someone else advanced the branch between our head - // read and the ref update. Rebuild on the fresh head and try again. - const isCas = err instanceof GitHubApiError && err.status === 422; - if (!isCas || attempt >= MAX_CAS_ATTEMPTS - 1) throw err; + // Someone else advanced the branch between our head read and the write. + if ( + !(err instanceof RepoWriteConflict) || + attempt >= MAX_CAS_ATTEMPTS - 1 + ) { + throw err; + } await sleep(exponentialBackoffWithJitter(2_000, 200, attempt, 2, 0.5)); } } diff --git a/apps/api/src/decofile/git-compat.test.ts b/apps/api/src/decofile/git-compat.test.ts index b2329ff2be..f814e1f6f9 100644 --- a/apps/api/src/decofile/git-compat.test.ts +++ b/apps/api/src/decofile/git-compat.test.ts @@ -1,37 +1,29 @@ import { describe, expect, it } from "bun:test"; +import type { TreeEntry } from "@/git-providers/content/types"; import { - buildDiscardTreeEntries, - buildMergeTreeEntries, + buildDiscardPlan, + buildMergeReplayPlan, buildPublishStatus, normalizeCompareStatus, } from "./git-compat"; -import type { TreeEntry } from "./github-git-data"; -describe("buildMergeTreeEntries", () => { +describe("buildMergeReplayPlan", () => { it("writes a rename as create-destination + delete-source", () => { - const entries = buildMergeTreeEntries( + const plan = buildMergeReplayPlan( [ { filename: "Header.json", status: "renamed", - sha: "b1", previousFilename: "Hero.json", }, ], - new Map([["Header.json", { sha: "b1" }]]), + "merged-sha", ); - expect(entries).toContainEqual({ + expect(plan).toContainEqual({ path: "Header.json", - mode: "100644", - type: "blob", - sha: "b1", - }); - expect(entries).toContainEqual({ - path: "Hero.json", - mode: "100644", - type: "blob", - sha: null, + copyFromRef: "merged-sha", }); + expect(plan).toContainEqual({ path: "Hero.json", deleted: true }); }); it("keeps a path's new content when it is both a rename destination and another file's rename source, regardless of diff order", () => { @@ -39,87 +31,83 @@ describe("buildMergeTreeEntries", () => { { filename: "Header.json", status: "renamed", - sha: "b1", previousFilename: "Hero.json", }, { filename: "Hero.json", status: "renamed", - sha: "b2", previousFilename: "Banner.json", }, ]; - const branchBlobByPath = new Map([ - ["Header.json", { sha: "b1" }], - ["Hero.json", { sha: "b2" }], - ]); for (const ordered of [files, [...files].reverse()]) { - const entries = buildMergeTreeEntries(ordered, branchBlobByPath); - const heroEntry = entries.find((e) => e.path === "Hero.json"); - expect(heroEntry).toEqual({ + const plan = buildMergeReplayPlan(ordered, "merged-sha"); + expect(plan.find((e) => e.path === "Hero.json")).toEqual({ path: "Hero.json", - mode: "100644", - type: "blob", - sha: "b2", - }); - expect(entries).toContainEqual({ - path: "Banner.json", - mode: "100644", - type: "blob", - sha: null, + copyFromRef: "merged-sha", }); + expect(plan).toContainEqual({ path: "Banner.json", deleted: true }); } }); - it("deletes a removed file", () => { - const entries = buildMergeTreeEntries( - [{ filename: "Old.json", status: "removed", sha: "b1" }], - new Map(), + it("deletes a removed file rather than replaying it from the source ref", () => { + const plan = buildMergeReplayPlan( + [{ filename: "Old.json", status: "removed" }], + "merged-sha", ); - expect(entries).toEqual([ - { path: "Old.json", mode: "100644", type: "blob", sha: null }, + expect(plan).toEqual([{ path: "Old.json", deleted: true }]); + }); + + it("replays every surviving path from the one source ref, uploading nothing", () => { + const plan = buildMergeReplayPlan( + [ + { filename: "A.json", status: "modified" }, + { filename: "B.json", status: "added" }, + ], + "branch-head", + ); + expect(plan).toEqual([ + { path: "A.json", copyFromRef: "branch-head" }, + { path: "B.json", copyFromRef: "branch-head" }, ]); }); }); -describe("buildDiscardTreeEntries", () => { - const blob = (sha: string, mode: string): TreeEntry => ({ +describe("buildDiscardPlan", () => { + const blob = (sha: string): TreeEntry => ({ path: "irrelevant", // overwritten by the map key at lookup time - mode, type: "blob", sha, }); - it("restores the base blob's mode, not a hardcoded 100644", () => { - const entries = buildDiscardTreeEntries( + it("restores the path from the base ref itself, so an executable stays executable", () => { + const plan = buildDiscardPlan( ["run.sh"], - new Map([["run.sh", blob("base-sha", "100755")]]), - new Map([["run.sh", blob("head-sha", "100755")]]), + new Map([["run.sh", blob("base-sha")]]), + new Map([["run.sh", blob("head-sha")]]), + "merge-base", ); - expect(entries).toEqual([ - { path: "run.sh", mode: "100755", type: "blob", sha: "base-sha" }, - ]); + expect(plan).toEqual([{ path: "run.sh", copyFromRef: "merge-base" }]); }); it("deletes a path the base doesn't have", () => { - const entries = buildDiscardTreeEntries( + const plan = buildDiscardPlan( ["New.json"], new Map(), - new Map([["New.json", blob("head-sha", "100644")]]), + new Map([["New.json", blob("head-sha")]]), + "merge-base", ); - expect(entries).toEqual([ - { path: "New.json", mode: "100644", type: "blob", sha: null }, - ]); + expect(plan).toEqual([{ path: "New.json", deleted: true }]); }); it("is a no-op when base and head already match, or both are absent", () => { - const entries = buildDiscardTreeEntries( + const plan = buildDiscardPlan( ["Same.json", "Never.json"], - new Map([["Same.json", blob("s", "100644")]]), - new Map([["Same.json", blob("s", "100644")]]), + new Map([["Same.json", blob("s")]]), + new Map([["Same.json", blob("s")]]), + "merge-base", ); - expect(entries).toEqual([]); + expect(plan).toEqual([]); }); }); diff --git a/apps/api/src/decofile/git-compat.ts b/apps/api/src/decofile/git-compat.ts index d619cdbae9..26bff32351 100644 --- a/apps/api/src/decofile/git-compat.ts +++ b/apps/api/src/decofile/git-compat.ts @@ -1,9 +1,10 @@ /** - * GitHub-backed implementations of the sandbox `/git/*` route contracts for + * Provider-backed implementations of the sandbox `/git/*` route contracts for * sandbox-less Fast Preview projects. The web's publish dialog and header * cluster speak the daemon's JSON shapes (`GitStatus`, `GitDiffResult` — see - * apps/web .../sandbox-git-api.ts); serving the same shapes from the GitHub - * API lets those components work UNCHANGED with no working tree behind them. + * apps/web .../sandbox-git-api.ts); serving the same shapes from a + * `RepoContentClient` lets those components work UNCHANGED with no working + * tree behind them. * * Invariants of sandbox-less mode that shape these payloads: * - There is no working tree: every save is already a commit on the branch, @@ -13,11 +14,14 @@ */ import { - GitHubApiError, - type GitDataClient, + type FileChange, + type RepoContentClient, + repoErrorStatus, + RepoWriteConflict, + requireBranchHead, type TreeEntry, - type TreeWriteEntry, -} from "./github-git-data"; +} from "@/git-providers/content/types"; +import { GitProviderError } from "@/git-providers/types"; import { mapBounded, resolveOrCreateHead } from "./read-decofile"; const DIFF_MAX_FILES = 200; @@ -55,11 +59,12 @@ export interface CompatGitStatus { headSha: string; unpushed: 0; /** - * The base…head changed-path manifest — every path {@link githubGitDiff} - * would return a body for, without paying for the bodies. Free: GitHub's - * compare response already carries `files`, and asking for it costs the SAME - * request the drift check already makes (identical URL, one ETag entry). - * Sliced to {@link DIFF_MAX_FILES} so manifest and diff agree key-for-key. + * The base…head changed-path manifest — every path {@link repoGitDiff} + * would return a body for, without paying for the bodies. Free: the + * provider's compare response already carries `files`, and asking for it + * costs the SAME request the drift check already makes (identical URL, one + * ETag entry). Sliced to {@link DIFF_MAX_FILES} so manifest and diff agree + * key-for-key. */ changedFiles: CompatChangedFile[]; /** Changed-path count BEFORE the cap — what "N changes" must actually say. */ @@ -69,8 +74,8 @@ export interface CompatGitStatus { } /** - * GitHub's compare `status` vocabulary is wider than the three states the CMS - * renders. `copied`/`changed`/`unchanged` fold into the nearest state the + * A provider's compare `status` vocabulary is wider than the three states the + * CMS renders. `copied`/`changed`/`unchanged` fold into the nearest state the * publish surface can draw, rather than leaking an unhandled string into it. */ export function normalizeCompareStatus(raw: string): CompatChangeStatus { @@ -132,7 +137,7 @@ const NO_DRIFT: CompareDetail = { aheadBy: 0, behindBy: 0, files: [] }; * concurrent {@link resolveOrCreateHead} that mints it — before asking again. */ async function compareAfterMaterializing( - client: GitDataClient, + client: RepoContentClient, base: string, branch: string, materialized: Promise, @@ -140,15 +145,15 @@ async function compareAfterMaterializing( try { return await client.compareDetailed(base, branch); } catch (err) { - if (!(err instanceof GitHubApiError) || err.status !== 404) throw err; + if (repoErrorStatus(err) !== 404) throw err; await materialized; return client.compareDetailed(base, branch); } } /** The manifest rides free on the drift check: same URL, same ETag entry. */ -export async function githubGitStatus( - client: GitDataClient, +export async function repoGitStatus( + client: RepoContentClient, branch: string, ): Promise { const base = await client.getDefaultBranch(); @@ -174,10 +179,10 @@ export interface CompatGitDiff { /** * Base…head diff with full file contents: head-side blobs come off the compare * entries, base-side contents are read by path at the merge base (see - * {@link GitDataClient.getFileTextAtRef}). Capped at {@link DIFF_MAX_FILES}. + * {@link RepoContentClient.readFileAtRef}). Capped at {@link DIFF_MAX_FILES}. */ -export async function githubGitDiff( - client: GitDataClient, +export async function repoGitDiff( + client: RepoContentClient, branch: string, base?: string, ): Promise { @@ -195,8 +200,8 @@ export async function githubGitDiff( const [from, to] = await Promise.all([ f.status === "added" ? null - : client.getFileTextAtRef(detailed.mergeBaseSha, basePath), - f.status === "removed" ? null : client.getBlobText(f.sha), + : client.readFileAtRef(detailed.mergeBaseSha, basePath), + f.status === "removed" ? null : client.readBlob(f.sha), ]); return [f.filename, { from, to }] as const; }); @@ -207,25 +212,33 @@ export async function githubGitDiff( }; } +/** One entry of a provider compare's changed-file list. */ +type ComparedFile = Awaited< + ReturnType +>["files"][number]; + /** * Bring the branch up to date with `base`, leaving it as ONE commit whose only * parent is the base head — a squash-rebase, not a merge, because a merge's - * resolution lives only in its tree and `git rebase` drops merge commits. Clean - * merges take GitHub's 3-way tree; conflicts always take - * {@link buildBranchWinsTree} and the cost documented there. + * resolution lives only in its tree and `git rebase` drops merge commits. + * Clean merges replay the merged content; a conflict replays the branch's own + * blobs (whole file wins) for every path it touched. * - * The forced ref update has no compare-and-swap (GitHub takes no `If-Match` on - * refs); re-reading the head right before it is the closest available lease. + * The squash is a single `commitFiles({ rewriteFrom: baseHead })`, so the + * commit exists before the branch is moved onto it and a crash mid-sync cannot + * leave the branch reset with its own head unreferenced. What is left to undo + * is the merge commit `mergeBranches` put on the branch on its way here + * ({@link restoreAbandonedSync}). */ -export async function githubGitRebase( - client: GitDataClient, +export async function repoGitRebase( + client: RepoContentClient, branch: string, base: string, ): Promise { for (let attempt = 1; ; attempt++) { - // Read first: the pre-force check re-reads it, catching any later autosave. + // Read first: the write's own guard re-reads it, catching a later autosave. const [branchHead, compared] = await Promise.all([ - client.getHeadSha(branch), + requireBranchHead(client, branch), client.compareDetailed(base, branch), ]); @@ -233,114 +246,149 @@ export async function githubGitRebase( if (compared.behindBy === 0 && compared.aheadBy <= 1) return; // Not hoisted into the pair above: pure waste on that early return. - const baseHead = await client.getHeadSha(base); + const baseHead = await requireBranchHead(client, base); - // No commits of its own: a fast-forward, which needs no force. + // No commits of its own: a fast-forward, so nothing of the branch is lost. if (compared.aheadBy === 0) { - try { - await client.updateRef(branch, baseHead); - return; - } catch (err) { - // 422 = an autosave landed since the compare; rebuild on the new head. - if (!(err instanceof GitHubApiError && err.status === 422)) throw err; - if (attempt >= MERGE_CAS_ATTEMPTS) throw err; - continue; + if ((await requireBranchHead(client, branch)) !== branchHead) { + if (attempt < MERGE_CAS_ATTEMPTS) continue; + throw keptChanging(client, branch, base); } + await client.forceBranchHead(branch, baseHead); + return; } - let treeSha: string; + let replayed: ComparedFile[]; let branchWon = false; - // `mergeBranch` moves the ref onto the merge commit it creates. + // `mergeBranches` moves the ref onto the merge commit it creates. let mergedSha: string | null = null; try { - const mergeSha = await client.mergeBranch( + mergedSha = await client.mergeBranches( branch, base, `chore(decofile): merge ${base} into ${branch}`, ); - if (mergeSha === null) { - // Branch already contains base but is >1 commit ahead: flatten as-is. - treeSha = await client.getCommitTreeSha(branchHead); - } else { - treeSha = await client.getCommitTreeSha(mergeSha); - mergedSha = mergeSha; - } + /** A null merge means the branch already contains base and only needs + * flattening, so its own three-dot diff already names the content to + * replay; otherwise the merge commit's does. */ + replayed = + mergedSha === null + ? compared.files + : (await client.compareDetailed(base, branch)).files; } catch (err) { - if (!(err instanceof GitHubApiError && err.status === 409)) throw err; - treeSha = await buildBranchWinsTree( - client, - compared.files, - branchHead, - baseHead, - ); + if (repoErrorStatus(err) !== 409) throw err; + replayed = compared.files; branchWon = true; } + // Raised here, not inside the try: a 409 there means the merge conflicted. + assertReplayable(client, replayed); + const changes = buildMergeReplayPlan(replayed, mergedSha ?? branchHead); const expectedHead = mergedSha ?? branchHead; try { - const commitSha = await client.createCommit({ + /** Nothing differs from base: the branch's tree already IS base's, so + * the reset alone is the sync and a commit would only carry a message. */ + if (changes.length === 0) { + if ((await requireBranchHead(client, branch)) !== expectedHead) { + if (attempt < MERGE_CAS_ATTEMPTS) continue; + throw keptChanging(client, branch, base); + } + await client.forceBranchHead(branch, baseHead); + return; + } + await client.commitFiles({ + branch, message: squashMessage( branch, base, branchWon, compared.commitMessages, ), - treeSha, - parentShas: [baseHead], + expectedHead, + rewriteFrom: baseHead, + changes, }); - - if ((await client.getHeadSha(branch)) !== expectedHead) { - // An autosave landed mid-sync: rebuild on the new head, never force over it. - if (attempt < MERGE_CAS_ATTEMPTS) continue; - throw new GitHubApiError( - 409, - "PATCH", - `git/refs/heads/${branch}`, - `${branch} kept changing while syncing with ${base}; try again`, - ); - } - await client.updateRef(branch, commitSha, { force: true }); return; } catch (err) { - await undoAbandonedMerge(client, branch, mergedSha, branchHead); - throw err; + // An autosave landed mid-sync: rebuild on the new head, never over it. + if (err instanceof RepoWriteConflict && attempt < MERGE_CAS_ATTEMPTS) { + continue; + } + await restoreAbandonedSync(client, branch, mergedSha, branchHead); + throw err instanceof RepoWriteConflict + ? keptChanging(client, branch, base) + : err; } } } +/** + * The replay names one path per changed file, and the compare it is built from + * enumerates at most this many (GitHub truncates there, and every provider + * caps a diff somewhere) — so at the cap the list may be partial, and a partial + * replay would silently revert the paths it never saw. Hence `>=`, and hence a + * refusal rather than a best effort: a branch that far from base is a merge to + * resolve on the provider. + */ const MERGE_REPLAY_FILE_CAP = 300; const MERGE_CAS_ATTEMPTS = 3; +function assertReplayable( + client: RepoContentClient, + files: ComparedFile[], +): void { + if (files.length < MERGE_REPLAY_FILE_CAP) return; + throw new GitProviderError({ + provider: client.repo.provider, + status: 409, + message: `Too many changed files on the branch to auto-merge (${files.length}); resolve it on ${client.repo.host}`, + }); +} + +function keptChanging( + client: RepoContentClient, + branch: string, + base: string, +): GitProviderError { + return new GitProviderError({ + provider: client.repo.provider, + status: 409, + message: `${branch} kept changing while syncing with ${base}; try again`, + }); +} + /** - * Tree write entries for the branch-wins replay, deduped by path. A path can - * be BOTH a rename destination for one file and a rename source for another - * in the same diff (e.g. `Hero.json`→`Header.json` while `Banner.json`→ - * `Hero.json`) — GitHub's tree write applies the LAST entry for a duplicate + * The replay plan for a sync, deduped by path: every surviving path takes the + * version `sourceRef` has, which is the resolution the merge (or the branch) + * already computed — addressed by ref rather than read, so the write reuses + * blobs the repository holds and each path keeps its own file mode. + * + * A path can be BOTH a rename destination for one file and a rename source for + * another in the same diff (e.g. `Hero.json`→`Header.json` while + * `Banner.json`→`Hero.json`) — the write applies the LAST entry for a duplicate * path, so pushing both a create and a stale-source delete for the same path * let iteration order decide whether the renamed-in content survived. A * destination entry always describes the path's real final content, so it - * always wins; a source delete only applies when nothing else claims that - * path as its destination. + * always wins; a source delete only applies when nothing else claims that path + * as its destination. */ -export function buildMergeTreeEntries( +export function buildMergeReplayPlan( files: Array<{ filename: string; status: string; - sha: string; previousFilename?: string; }>, - branchBlobByPath: Map, -): TreeWriteEntry[] { - const byPath = new Map(); + sourceRef: string, +): FileChange[] { + const byPath = new Map(); for (const f of files) { - const blob = branchBlobByPath.get(f.filename); - byPath.set(f.filename, { - path: f.filename, - mode: blob?.mode ?? "100644", - type: "blob", - sha: f.status === "removed" || !blob ? null : blob.sha, - }); + byPath.set( + f.filename, + f.status === "removed" + ? { path: f.filename, deleted: true } + : { path: f.filename, copyFromRef: sourceRef }, + ); } for (const f of files) { if ( @@ -350,9 +398,7 @@ export function buildMergeTreeEntries( ) { byPath.set(f.previousFilename, { path: f.previousFilename, - mode: "100644", - type: "blob", - sha: null, + deleted: true, }); } } @@ -360,30 +406,28 @@ export function buildMergeTreeEntries( } /** - * Tree write entries for a discard: reset each path to its blob (and MODE — a - * script's executable bit is real content, not metadata a discard should - * silently drop) at the merge base, or delete it when the base doesn't have - * it either. Pure, so it's unit-tested without a `GitDataClient`. + * The replay plan for a discard: take each path back from `baseRef`, or delete + * it when the base doesn't have it either. Addressed by ref, so a discarded + * executable comes back executable rather than as a plain file. Pure, so it's + * unit-tested without a `RepoContentClient`. */ -export function buildDiscardTreeEntries( +export function buildDiscardPlan( filepaths: string[], baseBlobByPath: Map, headBlobByPath: Map, -): TreeWriteEntry[] { - const entries: TreeWriteEntry[] = []; + baseRef: string, +): FileChange[] { + const entries: FileChange[] = []; for (const path of filepaths) { const baseBlob = baseBlobByPath.get(path); const headBlob = headBlobByPath.get(path); // Already at the base content (or absent on both sides): nothing to do. if (baseBlob?.sha === headBlob?.sha) continue; - // Deleting a path the head tree doesn't have is a 422, not a no-op. + // Neither side has it: there is nothing to reset and nothing to delete. if (!baseBlob && !headBlob) continue; - entries.push({ - path, - mode: baseBlob?.mode ?? "100644", - type: "blob", - sha: baseBlob?.sha ?? null, - }); + entries.push( + baseBlob ? { path, copyFromRef: baseRef } : { path, deleted: true }, + ); } return entries; } @@ -393,56 +437,43 @@ export function buildDiscardTreeEntries( * resets each path to its content at the merge base with the default branch * (or deletes it when it did not exist there). The sandbox-less equivalent of * the daemon's working-tree discard — in Fast Preview every edit is already a - * commit, so undoing one is a commit too. CAS-retried like the merge above so - * an autosave landing mid-discard is never clobbered. + * commit, so undoing one is a commit too. Guarded on the head it was built + * against, so an autosave landing mid-discard is never clobbered. */ -export async function githubGitDiscard( - client: GitDataClient, +export async function repoGitDiscard( + client: RepoContentClient, branch: string, filepaths: string[], ): Promise { if (filepaths.length === 0) return; const base = await client.getDefaultBranch(); for (let attempt = 1; ; attempt++) { - const branchHead = await client.getHeadSha(branch); + const branchHead = await requireBranchHead(client, branch); const { mergeBaseSha } = await client.compareDetailed(base, branch); // Scoped to `filepaths`, not a whole-repo recursive read. const [baseBlobByPath, headBlobByPath] = await Promise.all([ - client.getBlobsAtPaths( - await client.getCommitTreeSha(mergeBaseSha), - filepaths, - ), - client.getBlobsAtPaths( - await client.getCommitTreeSha(branchHead), - filepaths, - ), + client.getEntriesAtPaths(mergeBaseSha, filepaths), + client.getEntriesAtPaths(branchHead, filepaths), ]); - const entries = buildDiscardTreeEntries( + const plan = buildDiscardPlan( filepaths, baseBlobByPath, headBlobByPath, + mergeBaseSha, ); - if (entries.length === 0) return; + if (plan.length === 0) return; - const treeSha = await client.createTree( - await client.getCommitTreeSha(branchHead), - entries, - ); - const commitSha = await client.createCommit({ - message: `chore(decofile): discard changes to ${entries.length} file(s)`, - treeSha, - parentShas: [branchHead], - }); try { - await client.updateRef(branch, commitSha); + await client.commitFiles({ + branch, + message: `chore(decofile): discard changes to ${plan.length} file(s)`, + expectedHead: branchHead, + changes: plan, + }); return; } catch (err) { - if ( - err instanceof GitHubApiError && - err.status === 422 && - attempt < MERGE_CAS_ATTEMPTS - ) { + if (err instanceof RepoWriteConflict && attempt < MERGE_CAS_ATTEMPTS) { continue; } throw err; @@ -450,59 +481,22 @@ export async function githubGitDiscard( } } -type ComparedFile = Awaited< - ReturnType ->["files"][number]; - -/** - * The branch-wins tree: base's tree with every path the branch changed since - * the merge base (the three-dot compare set) replaced by the branch's version, - * whole file. The same resolution the merge commit used to carry — only where - * it lands has changed. - */ -async function buildBranchWinsTree( - client: GitDataClient, - files: ComparedFile[], - branchHead: string, - baseHead: string, -): Promise { - if (files.length >= MERGE_REPLAY_FILE_CAP) { - throw new GitHubApiError( - 409, - "POST", - "merges", - `Too many changed files on the branch to auto-merge (${files.length}); resolve on GitHub`, - ); - } - - // Blob shas AND modes, scoped to `files` rather than a whole-repo recursive read. - const branchBlobByPath = await client.getBlobsAtPaths( - await client.getCommitTreeSha(branchHead), - files.map((f) => f.filename), - ); - - return client.createTree( - await client.getCommitTreeSha(baseHead), - buildMergeTreeEntries(files, branchBlobByPath), - ); -} - /** - * `mergeBranch` advances the branch ref before the squash can replace it. When - * the squash then fails, roll the ref back so a sync that reported failure + * `mergeBranches` advances the branch ref before the squash can replace it. + * When the squash then fails, roll the ref back so a sync that reported failure * leaves no merge commit behind — but only while nothing else has landed on it, * since a rollback is itself a force. */ -async function undoAbandonedMerge( - client: GitDataClient, +async function restoreAbandonedSync( + client: RepoContentClient, branch: string, mergedSha: string | null, branchHead: string, ): Promise { if (mergedSha === null) return; try { - if ((await client.getHeadSha(branch)) !== mergedSha) return; - await client.updateRef(branch, branchHead, { force: true }); + if ((await requireBranchHead(client, branch)) !== mergedSha) return; + await client.forceBranchHead(branch, branchHead); } catch { // Best effort: the caller needs to see the original failure, not this one. } @@ -510,8 +504,8 @@ async function undoAbandonedMerge( /** * Collapsing N commits into one would drop the `Co-authored-by:` trailers the - * coalescer stamps per editor, and those trailers are how GitHub attributes the - * work. Re-emit the distinct ones on the squash. + * coalescer stamps per editor, and those trailers are how the provider + * attributes the work. Re-emit the distinct ones on the squash. */ function squashMessage( branch: string, diff --git a/apps/api/src/decofile/github-git-data.test.ts b/apps/api/src/decofile/github-git-data.test.ts deleted file mode 100644 index 55722939c3..0000000000 --- a/apps/api/src/decofile/github-git-data.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { describe, expect, it } from "bun:test"; -import { resolveBlobsAtPaths, type TreeEntry } from "./github-git-data"; - -/** An in-memory repo tree, keyed by directory path ("" for root). */ -function fakeOps(dirs: Record) { - return { - resolveSubtreeSha: (_root: string, segments: string[]) => { - const dir = segments.join("/"); - return Promise.resolve(dir in dirs ? `tree:${dir}` : null); - }, - treeShallow: (treeSha: string) => { - const dir = treeSha.slice("tree:".length); - return Promise.resolve(dirs[dir] ?? []); - }, - }; -} - -const blob = (path: string, sha: string): TreeEntry => ({ - path, - mode: "100644", - type: "blob", - sha, -}); - -describe("resolveBlobsAtPaths", () => { - it("resolves blobs at the root and in nested directories", async () => { - const ops = fakeOps({ - "": [ - blob("Header.json", "b1"), - { ...blob("blocks", "t1"), type: "tree" }, - ], - blocks: [blob("Footer.json", "b2")], - }); - - const result = await resolveBlobsAtPaths( - "root", - ["Header.json", "blocks/Footer.json"], - ops, - ); - - expect(result.get("Header.json")?.sha).toBe("b1"); - expect(result.get("blocks/Footer.json")?.sha).toBe("b2"); - }); - - it("omits a path missing at this commit instead of throwing", async () => { - const ops = fakeOps({ "": [blob("Header.json", "b1")] }); - - const result = await resolveBlobsAtPaths( - "root", - ["Header.json", "Deleted.json"], - ops, - ); - - expect(result.has("Header.json")).toBe(true); - expect(result.has("Deleted.json")).toBe(false); - }); - - it("omits every path under a directory that does not exist", async () => { - const ops = fakeOps({ "": [] }); - - const result = await resolveBlobsAtPaths( - "root", - ["missing/dir/File.json"], - ops, - ); - - expect(result.size).toBe(0); - }); - - it("lists a shared directory once for paths in the same directory", async () => { - let listings = 0; - const dirs = { "": [blob("A.json", "a"), blob("B.json", "b")] }; - const ops = { - resolveSubtreeSha: fakeOps(dirs).resolveSubtreeSha, - treeShallow: (treeSha: string) => { - listings++; - return fakeOps(dirs).treeShallow(treeSha); - }, - }; - - await resolveBlobsAtPaths("root", ["A.json", "B.json"], ops); - - expect(listings).toBe(1); - }); -}); diff --git a/apps/api/src/decofile/github-git-data.ts b/apps/api/src/decofile/github-git-data.ts deleted file mode 100644 index 7f06d4d70d..0000000000 --- a/apps/api/src/decofile/github-git-data.ts +++ /dev/null @@ -1,720 +0,0 @@ -/** - * Minimal GitHub Git Data (+ merges/pulls/compare) client for the sandbox-less - * decofile API. Raw fetch, no SDK — mirrors the header/UA conventions of - * `shared/github-runtime-detect.ts`, but errors PROPAGATE (a failed write must - * surface, not degrade to null). - * - * The base URL is overridable so the e2e suite can point the whole client at a - * local stub — tests must never reach api.github.com. - */ - -import { - countGithubRateLimited, - githubRetryAfterMs, - isGithubRateLimited, - recordGithubRateLimit, -} from "@/observability/github-rate-limit"; - -const DEFAULT_TIMEOUT_MS = 15_000; - -/** e2e seam: set GITHUB_API_BASE_URL to a local stub. Read per call site so a - * long-lived process (dev server) and the test webServer agree on one value. */ -function githubApiBaseUrl(): string { - return process.env.GITHUB_API_BASE_URL ?? "https://api.github.com"; -} - -/** - * `default_branch` per repo, cached across requests — a client instance lives - * for ONE of them (see `client-for-repo.ts`), so its own memo never survives. - * - * Ten minutes, not an hour: a renamed default branch compares against a branch - * that no longer exists until this expires. Keyed by full URL, so the e2e stub - * and api.github.com can never share an entry. - */ -const DEFAULT_BRANCH_TTL_MS = 10 * 60_000; -const defaultBranchCache = new Map(); - -function cachedDefaultBranch(url: string): string | null { - const hit = defaultBranchCache.get(url); - if (!hit) return null; - if (Date.now() - hit.at > DEFAULT_BRANCH_TTL_MS) { - defaultBranchCache.delete(url); - return null; - } - return hit.value; -} - -/** - * Conditional-request cache: url -> { etag, body }. GitHub serves 304s for - * matching `If-None-Match` and those do NOT count against the primary rate - * limit, so re-reads of hot mutable endpoints (ref resolves, compares, repo - * meta) become rate-limit-free when nothing changed. Module-level because a - * client instance lives for one HTTP request. - * - * Content-addressed payloads (blobs, trees, tarballs) are excluded: they are - * immutable per sha and already cached at the read layer, so an ETag entry - * would never be re-requested — it would only duplicate large bodies here. - * - * Keyed by URL alone, NOT by token: a 304 is only ever served after GitHub - * authorized THIS request's token, so a cached body is never revealed to a - * caller GitHub itself would refuse. - */ -const ETAG_CACHE_MAX = 512; -/** Byte budget across all stored bodies (approximated as the JSON.stringify - * length, computed once at insert). Primary bound; entry count is secondary. */ -const ETAG_CACHE_MAX_BYTES = 8 * 1024 * 1024; -/** Bodies over this are not cached at all — one huge response must not evict - * the whole working set. */ -const ETAG_CACHE_MAX_BODY_BYTES = 256 * 1024; -const etagCache = new Map< - string, - { etag: string; body: unknown; bytes: number } ->(); -let etagCacheBytes = 0; - -function etagCacheable(method: string, path: string): boolean { - return ( - method === "GET" && - !path.includes("/git/blobs/") && - !path.includes("/git/trees/") && - // `/contents/{path}?ref=` is sha-pinned, hence content-addressed too. - !path.includes("/contents/") && - !path.includes("/tarball/") - ); -} - -function etagCacheDelete(url: string): void { - const existing = etagCache.get(url); - if (!existing) return; - etagCache.delete(url); - etagCacheBytes -= existing.bytes; -} - -function etagCachePut(url: string, etag: string, body: unknown): void { - let bytes: number; - try { - bytes = JSON.stringify(body)?.length ?? 0; - } catch { - return; // unserializable body — don't cache - } - etagCacheDelete(url); - if (bytes > ETAG_CACHE_MAX_BODY_BYTES) return; // stale entry already dropped - etagCache.set(url, { etag, body, bytes }); - etagCacheBytes += bytes; - while ( - etagCacheBytes > ETAG_CACHE_MAX_BYTES || - etagCache.size > ETAG_CACHE_MAX - ) { - const oldest = etagCache.keys().next().value; - if (oldest === undefined) break; - etagCacheDelete(oldest); - } -} - -export class GitHubApiError extends Error { - constructor( - readonly status: number, - readonly method: string, - readonly path: string, - message: string, - ) { - super(`GitHub ${method} ${path} failed (${status}): ${message}`); - this.name = "GitHubApiError"; - } -} - -/** - * GitHub refused the call for rate reasons. NOT retriable: retrying a secondary - * limit is the burst being limited. `retryAfterMs` tells the caller when to come - * back; it is never slept on inside a request. - */ -export class GitHubRateLimitError extends GitHubApiError { - constructor( - status: number, - method: string, - path: string, - readonly kind: "primary" | "secondary", - readonly retryAfterMs: number | null, - ) { - super( - status, - method, - path, - `GitHub ${kind} rate limit reached${ - retryAfterMs === null - ? "" - : `; retry in ${Math.ceil(retryAfterMs / 1000)}s` - }`, - ); - this.name = "GitHubRateLimitError"; - } -} - -export interface TreeEntry { - path: string; - mode: string; - type: "blob" | "tree" | "commit"; - sha: string; - /** Blob byte size — GitHub returns it for blob entries (absent for - * trees/commits). Feeds the cold-read tarball pre-validation. */ - size?: number; -} - -/** Tree write entry — `sha: null` deletes the path (Git Data API contract). */ -export interface TreeWriteEntry { - path: string; - /** Git file mode: "100644" for a new file, else the source entry's own. */ - mode: string; - type: "blob"; - sha: string | null; -} - -export interface PullRequestInfo { - number: number; - html_url: string; -} - -export interface GitDataClient { - readonly owner: string; - readonly repo: string; - getDefaultBranch(): Promise; - getHeadSha(branch: string): Promise; - /** - * Branch head sha + the head commit's committer date, in one call. Git stores - * no ref-creation time, so this date is the branch's last-activity signal: a - * branch that was edited carries its last edit's date; a branch cut from the - * default branch and never touched carries the base commit's date (old the - * moment it lags behind an advancing default branch). The CMS staleness check - * reads it. Throws GitHubApiError(404) for a branch not yet on GitHub. - */ - getBranchHead(branch: string): Promise<{ sha: string; committedAt: string }>; - /** - * Gzipped tar of the repo at `ref`, as a stream — ONE request for every - * file, vs one blob request per block. The preferred cold-read path. The - * body is NEVER buffered here (golden rule 1: the archive must not touch - * the JS heap); the caller pipes it straight into a `tar` subprocess. - * Callers fall back to tree+blob reads when the server doesn't support it - * (the e2e stub) or the download/extraction fails. - */ - getTarballStream(ref: string): Promise>; - getCommitTreeSha(commitSha: string): Promise; - /** - * Block-dir view of a commit's tree WITHOUT a whole-repo recursive read. - * A whole-repo recursive read on the root tree of a large repo trips GitHub's - * 100k-entry / 7MB recursive cap (`truncated: true` → hard 502), which broke - * every decofile read/write on big storefronts. Instead this walks - * `/.deco/` non-recursively and returns only the block sources - * (`/.deco/blocks/*.json`) plus the merged artifact - * (`/.deco/blocks.gen.json`) when the repo commits it — the whole set the - * decofile read/merge and the commit coalescer consume. Paths are full and - * repo-relative, so `blockEntriesInTree` and the gen-file check operate on the - * result unchanged. Empty when the project has no `.deco/` yet. - */ - getDecofileTree( - commitTreeSha: string, - packagePath: string | null, - ): Promise; - /** - * Blob entries for a bounded, caller-known set of paths — the same - * 502-on-large-repo cap `getDecofileTree` dodges for decofile reads, but for - * callers that already know exactly which paths they need (discard's - * `filepaths`, the branch-wins merge's changed-file list) instead of a - * whole subdirectory. Walks only the directories those paths live in, so - * cost scales with the path set, never with repo size. A path absent at - * this commit (deleted, or never existed) is omitted from the map. - */ - getBlobsAtPaths( - commitTreeSha: string, - paths: string[], - ): Promise>; - getBlobText(blobSha: string): Promise; - /** - * One file's text at a ref, addressed by PATH rather than blob sha — null when - * the path does not exist there. The base side of a diff needs this: the - * compare response's `files[].sha` is the HEAD blob, never the base one. - */ - getFileTextAtRef(ref: string, filePath: string): Promise; - createBlob(content: string): Promise; - createTree(baseTreeSha: string, entries: TreeWriteEntry[]): Promise; - createCommit(params: { - message: string; - treeSha: string; - /** One parent for a plain commit; two (ours, theirs) for a merge commit. */ - parentShas: string[]; - }): Promise; - /** - * Ref update, non-forced by default: a concurrent writer makes it fail - * non-fast-forward (GitHubApiError 422) instead of being clobbered, which is - * the commit coalescer's multi-replica safety property. - * - * `force: true` rewrites history and there is NO compare-and-swap available: - * GitHub's ref API accepts no `If-Match`, so a forced caller must re-read the - * head itself immediately before forcing (see `githubGitRebase`). - */ - updateRef( - branch: string, - sha: string, - opts?: { force?: boolean }, - ): Promise; - /** Create `refs/heads/` at `sha`; GitHubApiError(422) if it exists. */ - createRef(branch: string, sha: string): Promise; - /** Returns the merge commit sha, or null when base already contains head. */ - mergeBranch( - base: string, - head: string, - message: string, - ): Promise; - createPullRequest(params: { - base: string; - head: string; - title: string; - }): Promise; - findOpenPullRequest( - base: string, - head: string, - ): Promise; - compare( - base: string, - head: string, - ): Promise<{ aheadBy: number; behindBy: number }>; - /** Compare with per-file detail — feeds the sandbox-less `/git/*` compat. */ - compareDetailed( - base: string, - head: string, - ): Promise<{ - aheadBy: number; - behindBy: number; - mergeBaseSha: string; - files: Array<{ - filename: string; - status: string; - sha: string; - previousFilename?: string; - }>; - /** - * Messages of the commits on `head` that `base` lacks — the set a - * squash collapses. GitHub caps this list at 250 entries; a longer branch - * is truncated, so it feeds attribution, never correctness. - */ - commitMessages: string[]; - }>; -} - -/** - * `getBlobsAtPaths`'s directory-scoped walk, factored out for testing: given - * how to list a subtree's direct children, resolve each path to its blob - * entry by visiting only the directories the paths actually live in (cached - * per directory, so siblings share one listing call). - */ -export async function resolveBlobsAtPaths( - rootTreeSha: string, - paths: string[], - ops: { - resolveSubtreeSha: ( - rootTreeSha: string, - segments: string[], - ) => Promise; - treeShallow: (treeSha: string) => Promise; - }, -): Promise> { - const result = new Map(); - const dirListings = new Map(); - for (const path of paths) { - const slash = path.lastIndexOf("/"); - const dir = slash === -1 ? "" : path.slice(0, slash); - const base = slash === -1 ? path : path.slice(slash + 1); - let listing = dirListings.get(dir); - if (listing === undefined) { - const subtreeSha = await ops.resolveSubtreeSha( - rootTreeSha, - dir === "" ? [] : dir.split("/"), - ); - listing = subtreeSha === null ? null : await ops.treeShallow(subtreeSha); - dirListings.set(dir, listing); - } - const entry = listing?.find((e) => e.type === "blob" && e.path === base); - if (entry) result.set(path, { ...entry, path }); - } - return result; -} - -export function createGitDataClient(params: { - owner: string; - repo: string; - accessToken: string; -}): GitDataClient { - const { owner, repo, accessToken } = params; - const repoBase = `/repos/${owner}/${repo}`; - let defaultBranch: string | null = null; - - async function call( - method: string, - path: string, - body?: unknown, - opts?: { allow?: number[] }, - ): Promise<{ status: number; json: T }> { - const url = `${githubApiBaseUrl()}${path}`; - const conditional = etagCacheable(method, path) - ? etagCache.get(url) - : undefined; - const res = await fetch(url, { - method, - headers: { - Accept: "application/vnd.github+json", - "User-Agent": "studio-decofile", - Authorization: `token ${accessToken}`, - ...(conditional ? { "If-None-Match": conditional.etag } : {}), - ...(body !== undefined ? { "content-type": "application/json" } : {}), - }, - body: body !== undefined ? JSON.stringify(body) : undefined, - signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS), - }); - recordGithubRateLimit(res.headers, { lane: "rest", operation: method }); - - if (conditional && res.status === 304) { - return { status: 200, json: conditional.body as T }; - } - if (isGithubRateLimited(res)) { - const kind = - res.headers.get("retry-after") !== null ? "secondary" : "primary"; - countGithubRateLimited({ lane: "rest", operation: method, kind }); - await res.body?.cancel().catch(() => {}); - throw new GitHubRateLimitError( - res.status, - method, - path, - kind, - githubRetryAfterMs(res.headers), - ); - } - if (!res.ok && !opts?.allow?.includes(res.status)) { - const text = await res.text().catch(() => ""); - let message = text; - try { - message = (JSON.parse(text) as { message?: string }).message ?? text; - } catch { - // keep raw text - } - throw new GitHubApiError(res.status, method, path, message); - } - const json = - res.status === 204 ? (undefined as T) : ((await res.json()) as T); - if (res.status === 200 && etagCacheable(method, path)) { - const etag = res.headers.get("etag"); - if (etag) etagCachePut(url, etag, json); - } - return { status: res.status, json }; - } - - /** Blob text by sha via the Blob API (base64, up to 100MB). Shared by - * `getBlobText` and by `getFileTextAtRef`'s large-file fallback. */ - async function blobText(blobSha: string): Promise { - const { json } = await call<{ content: string; encoding: string }>( - "GET", - `${repoBase}/git/blobs/${blobSha}`, - ); - if (json.encoding !== "base64") { - throw new GitHubApiError( - 502, - "GET", - `${repoBase}/git/blobs/${blobSha}`, - `unexpected blob encoding ${json.encoding}`, - ); - } - return Buffer.from(json.content, "base64").toString("utf-8"); - } - - /** One tree's DIRECT children (non-recursive). GitHub returns subdirectories - * as `type: "tree"` entries carrying their own sha; the entry `path` is the - * bare child name, not a repo-relative path. */ - async function treeShallow(treeSha: string): Promise { - const { json } = await call<{ tree: TreeEntry[]; truncated: boolean }>( - "GET", - `${repoBase}/git/trees/${treeSha}`, - ); - if (json.truncated) { - // One directory over the cap: unreachable once scoped, but fail loud. - throw new GitHubApiError( - 502, - "GET", - `${repoBase}/git/trees/${treeSha}`, - "tree listing truncated by GitHub; directory too large", - ); - } - return json.tree; - } - - /** Walk `segments` from `rootTreeSha`, returning the final subtree's sha, or - * null when any segment is missing or is not a directory. */ - async function resolveSubtreeSha( - rootTreeSha: string, - segments: string[], - ): Promise { - let sha = rootTreeSha; - for (const segment of segments) { - const child = (await treeShallow(sha)).find( - (e) => e.type === "tree" && e.path === segment, - ); - if (!child) return null; - sha = child.sha; - } - return sha; - } - - return { - owner, - repo, - - async getDefaultBranch() { - if (defaultBranch) return defaultBranch; - const url = `${githubApiBaseUrl()}${repoBase}`; - const shared = cachedDefaultBranch(url); - if (shared) { - defaultBranch = shared; - return shared; - } - const { json } = await call<{ default_branch: string }>("GET", repoBase); - defaultBranch = json.default_branch; - defaultBranchCache.set(url, { value: defaultBranch, at: Date.now() }); - return defaultBranch; - }, - - async getHeadSha(branch) { - const { json } = await call<{ object: { sha: string } }>( - "GET", - `${repoBase}/git/ref/heads/${encodeRefPath(branch)}`, - ); - return json.object.sha; - }, - - async getBranchHead(branch) { - const { json } = await call<{ - commit: { sha: string; commit: { committer: { date: string } } }; - }>("GET", `${repoBase}/branches/${encodeRefPath(branch)}`); - return { - sha: json.commit.sha, - committedAt: json.commit.commit.committer.date, - }; - }, - - async getTarballStream(ref) { - // Redirects to codeload; fetch follows them. Longer deadline — this is - // one multi-MB download instead of hundreds of small calls. The signal - // covers the whole body, so a stalled download aborts the stream and - // the consumer's tar pipeline fails over to per-blob fetches. - const path = `${repoBase}/tarball/${encodeRefPath(ref)}`; - const res = await fetch(`${githubApiBaseUrl()}${path}`, { - headers: { - "User-Agent": "studio-decofile", - Authorization: `token ${accessToken}`, - }, - signal: AbortSignal.timeout(60_000), - }); - if (!res.ok) { - const text = await res.text().catch(() => ""); - throw new GitHubApiError(res.status, "GET", path, text.slice(0, 300)); - } - if (!res.body) { - throw new GitHubApiError(502, "GET", path, "tarball had no body"); - } - return res.body; - }, - - async getCommitTreeSha(commitSha) { - const { json } = await call<{ tree: { sha: string } }>( - "GET", - `${repoBase}/git/commits/${commitSha}`, - ); - return json.tree.sha; - }, - - async getDecofileTree(commitTreeSha, packagePath) { - const decoSegments = [ - ...(packagePath ? packagePath.split("/") : []), - ".deco", - ]; - const decoTreeSha = await resolveSubtreeSha(commitTreeSha, decoSegments); - if (decoTreeSha === null) return []; // no `.deco/` in this project yet - const decoDir = packagePath ? `${packagePath}/.deco` : ".deco"; - const decoChildren = await treeShallow(decoTreeSha); - const out: TreeEntry[] = []; - // The merged artifact, only when this repo commits it (usually gitignored). - const gen = decoChildren.find( - (e) => e.type === "blob" && e.path === "blocks.gen.json", - ); - if (gen) out.push({ ...gen, path: `${decoDir}/blocks.gen.json` }); - const blocksDir = decoChildren.find( - (e) => e.type === "tree" && e.path === "blocks", - ); - if (blocksDir) { - for (const child of await treeShallow(blocksDir.sha)) { - if (child.type !== "blob") continue; - out.push({ ...child, path: `${decoDir}/blocks/${child.path}` }); - } - } - return out; - }, - - getBlobsAtPaths(commitTreeSha, paths) { - return resolveBlobsAtPaths(commitTreeSha, paths, { - resolveSubtreeSha, - treeShallow, - }); - }, - - getBlobText(blobSha) { - return blobText(blobSha); - }, - - async getFileTextAtRef(ref, filePath) { - const { status, json } = await call<{ - content?: string; - encoding?: string; - sha?: string; - }>( - "GET", - `${repoBase}/contents/${encodeRefPath(filePath)}?ref=${encodeURIComponent(ref)}`, - undefined, - { allow: [404] }, - ); - if (status === 404 || json?.content === undefined) return null; - /** Files over the Contents API's 1MB limit come back with - * `encoding: "none"` and empty content, but the blob sha is still there — - * fetch the blob directly (the Blob API serves up to 100MB). A large - * `.deco/meta.gen.json` routinely crosses that line. */ - if (json.encoding === "none" && json.sha) { - return blobText(json.sha); - } - if (json.encoding !== "base64") { - throw new GitHubApiError( - 502, - "GET", - `${repoBase}/contents/${filePath}`, - `unexpected content encoding ${json.encoding}`, - ); - } - return Buffer.from(json.content, "base64").toString("utf-8"); - }, - - async createBlob(content) { - const { json } = await call<{ sha: string }>( - "POST", - `${repoBase}/git/blobs`, - { content, encoding: "utf-8" }, - ); - return json.sha; - }, - - async createTree(baseTreeSha, entries) { - const { json } = await call<{ sha: string }>( - "POST", - `${repoBase}/git/trees`, - { base_tree: baseTreeSha, tree: entries }, - ); - return json.sha; - }, - - async createCommit({ message, treeSha, parentShas }) { - const { json } = await call<{ sha: string }>( - "POST", - `${repoBase}/git/commits`, - { message, tree: treeSha, parents: parentShas }, - ); - return json.sha; - }, - - async updateRef(branch, sha, opts) { - await call( - "PATCH", - `${repoBase}/git/refs/heads/${encodeRefPath(branch)}`, - { - sha, - force: opts?.force === true, - }, - ); - }, - - async createRef(branch, sha) { - await call("POST", `${repoBase}/git/refs`, { - ref: `refs/heads/${branch}`, - sha, - }); - }, - - async mergeBranch(base, head, message) { - const { status, json } = await call<{ sha?: string } | undefined>( - "POST", - `${repoBase}/merges`, - { base, head, commit_message: message }, - { allow: [204] }, - ); - // 204 = base already contains head; 201 carries the merge commit. - return status === 204 ? null : (json?.sha ?? null); - }, - - async createPullRequest({ base, head, title }) { - const { json } = await call( - "POST", - `${repoBase}/pulls`, - { base, head, title }, - ); - return { number: json.number, html_url: json.html_url }; - }, - - async findOpenPullRequest(base, head) { - const { json } = await call( - "GET", - `${repoBase}/pulls?state=open&base=${encodeURIComponent(base)}&head=${encodeURIComponent(`${owner}:${head}`)}`, - ); - const first = json[0]; - return first ? { number: first.number, html_url: first.html_url } : null; - }, - - async compare(base, head) { - const { json } = await call<{ ahead_by: number; behind_by: number }>( - "GET", - `${repoBase}/compare/${encodeRefPath(base)}...${encodeRefPath(head)}`, - ); - return { aheadBy: json.ahead_by, behindBy: json.behind_by }; - }, - - async compareDetailed(base, head) { - const { json } = await call<{ - ahead_by: number; - behind_by: number; - merge_base_commit: { sha: string }; - files?: Array<{ - filename: string; - status: string; - sha: string; - previous_filename?: string; - }>; - commits?: Array<{ commit?: { message?: string } }>; - }>( - "GET", - `${repoBase}/compare/${encodeRefPath(base)}...${encodeRefPath(head)}`, - ); - return { - aheadBy: json.ahead_by, - behindBy: json.behind_by, - mergeBaseSha: json.merge_base_commit.sha, - files: (json.files ?? []).map((f) => ({ - filename: f.filename, - status: f.status, - sha: f.sha, - ...(f.previous_filename - ? { previousFilename: f.previous_filename } - : {}), - })), - commitMessages: (json.commits ?? []) - .map((c) => c.commit?.message ?? "") - .filter((m) => m.length > 0), - }; - }, - }; -} - -/** Encode a branch name for a ref path segment, preserving `/` separators. */ -function encodeRefPath(branch: string): string { - return branch.split("/").map(encodeURIComponent).join("/"); -} diff --git a/apps/api/src/decofile/read-decofile.test.ts b/apps/api/src/decofile/read-decofile.test.ts index fae7189e66..b876fc8391 100644 --- a/apps/api/src/decofile/read-decofile.test.ts +++ b/apps/api/src/decofile/read-decofile.test.ts @@ -1,13 +1,14 @@ import { describe, expect, it } from "bun:test"; -import type { TreeEntry } from "./github-git-data"; +import type { TreeEntry } from "@/git-providers/content/types"; import { aliasPathsForKey, blockEntriesInTree, blocksDirPath, + gitBlobSha, } from "./read-decofile"; function blob(path: string): TreeEntry { - return { path, mode: "100644", type: "blob", sha: `sha-${path}` }; + return { path, type: "blob", sha: `sha-${path}` }; } describe("blocksDirPath", () => { @@ -26,7 +27,7 @@ describe("blockEntriesInTree", () => { blob(".deco/blocks/readme.md"), blob(".deco/blocks.gen.json"), blob("src/index.ts"), - { path: ".deco/blocks", mode: "040000", type: "tree", sha: "t" }, + { path: ".deco/blocks", type: "tree", sha: "t" }, ]; const stems = blockEntriesInTree(tree, null).map((e) => e.stem); expect(stems.sort()).toEqual(["Header", "Upper"]); @@ -67,3 +68,19 @@ describe("aliasPathsForKey", () => { expect(aliasPathsForKey(entries, "missing")).toEqual([]); }); }); + +describe("gitBlobSha", () => { + /** The object ids git itself produces — the cache is keyed by the sha the + * provider's tree listing reports, so a mismatch is a silent cache miss. */ + it("matches git's blob hashing, including for an empty file", () => { + expect(gitBlobSha("")).toBe("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"); + expect(gitBlobSha("hello\n")).toBe( + "ce013625030ba8dba906f756967f9e9ca394464a", + ); + }); + + it("hashes byte length, not code-point length", () => { + expect(gitBlobSha("é")).toBe(gitBlobSha("\u00e9")); + expect(gitBlobSha("é")).not.toBe(gitBlobSha("e")); + }); +}); diff --git a/apps/api/src/decofile/read-decofile.ts b/apps/api/src/decofile/read-decofile.ts index e6e99bdad0..2a55502eb5 100644 --- a/apps/api/src/decofile/read-decofile.ts +++ b/apps/api/src/decofile/read-decofile.ts @@ -4,7 +4,14 @@ import { decoBlockKeyFromFileStem, mergeBlocks, } from "@decocms/shared/decofile"; +import { repoIdentityKey, type RepoRef } from "@decocms/shared/git-providers"; import type { Counter } from "@opentelemetry/api"; +import { + type RepoContentClient, + requireBranchHead, + RepoWriteConflict, + type TreeEntry, +} from "@/git-providers/content/types"; import { meter } from "../observability"; import { getBlob, @@ -14,34 +21,38 @@ import { putMerged, removeScratchDir, } from "./disk-cache"; -import type { GitDataClient, TreeEntry } from "./github-git-data"; -import { GitHubApiError } from "./github-git-data"; import { createSingleFlight } from "./single-flight"; import { extractBlocksFromTarball } from "./tar-extract"; +/** Disk-cache and single-flight namespace for a repo, host-qualified so two + * providers' identically-named repos can never share an entry. */ +function cacheScope(repo: RepoRef): [owner: string, name: string] { + return [repo.host, repo.path]; +} + /** * Branch head sha, creating the branch from the default branch's head when it * doesn't exist yet. Thread-scoped branches are minted client-side and only - * materialize on GitHub at first CMS touch — the sandbox flow forks locally at - * clone time, and this is the sandbox-less equivalent. A 422 on create means a - * concurrent first-touch won the race; re-read the ref it created. + * materialize on the provider at first CMS touch — the sandbox flow forks + * locally at clone time, and this is the sandbox-less equivalent. A conflict on + * create means a concurrent first-touch won the race; re-read the ref it made. */ export async function resolveOrCreateHead( - client: GitDataClient, + client: RepoContentClient, branch: string, ): Promise { + const existing = await client.getBranch(branch); + if (existing) return existing.sha; + const baseSha = await requireBranchHead( + client, + await client.getDefaultBranch(), + ); try { - return await client.getHeadSha(branch); - } catch (err) { - if (!(err instanceof GitHubApiError) || err.status !== 404) throw err; - } - const baseSha = await client.getHeadSha(await client.getDefaultBranch()); - try { - await client.createRef(branch, baseSha); + await client.createBranch(branch, baseSha); return baseSha; } catch (err) { - if (err instanceof GitHubApiError && err.status === 422) { - return client.getHeadSha(branch); + if (err instanceof RepoWriteConflict) { + return requireBranchHead(client, branch); } throw err; } @@ -176,16 +187,25 @@ export function aliasPathsForKey( .sort(); } -/** Let writers prime blobs they just created, so the read after a save never +/** + * A blob's git object id: sha1 over `blob \0`. Computed + * locally rather than read off a write response, because the intent-level write + * (`commitFiles`) never reports the object ids it created — and it needn't, + * since git's hashing is the same everywhere the cache is keyed by it. + */ +export function gitBlobSha(content: string): string { + const bytes = Buffer.from(content, "utf-8"); + return createHash("sha1") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex"); +} + +/** Let writers prime blobs they just wrote, so the read after a save never * re-fetches content this replica already has in hand. Write-through to the * disk store; fail-open (a store problem is a no-op, never an error). */ -export function primeBlobCache( - owner: string, - repo: string, - blobSha: string, - content: string, -): Promise { - return putBlob(owner, repo, blobSha, content); +export function primeBlobCache(repo: RepoRef, content: string): Promise { + return putBlob(...cacheScope(repo), gitBlobSha(content), content); } export interface DecofileSnapshot { @@ -219,7 +239,7 @@ function mergedDocSha(sha: string, packagePath: string | null): string { const snapshotFlight = createSingleFlight(); export async function readDecofileSnapshot( - client: GitDataClient, + client: RepoContentClient, branch: string, packagePath: string | null, options?: { @@ -231,19 +251,19 @@ export async function readDecofileSnapshot( ): Promise { const sha = options?.createBranchIfMissing ? await resolveOrCreateHead(client, branch) - : await client.getHeadSha(branch); + : await requireBranchHead(client, branch); return snapshotFlight.run( - `${client.owner}/${client.repo}@${sha}:${packagePath ?? ""}`, + `${repoIdentityKey(client.repo)}@${sha}:${packagePath ?? ""}`, () => resolveSnapshot(client, sha, packagePath), ); } async function resolveSnapshot( - client: GitDataClient, + client: RepoContentClient, sha: string, packagePath: string | null, ): Promise { - const { owner, repo } = client; + const [owner, repo] = cacheScope(client.repo); const docSha = mergedDocSha(sha, packagePath); // Merged disk hit: serve the stored string as-is — no JSON parse, no @@ -251,8 +271,7 @@ async function resolveSnapshot( const cachedMerged = await getMerged(owner, repo, docSha); if (cachedMerged !== null) return { sha, decofile: cachedMerged }; - const treeSha = await client.getCommitTreeSha(sha); - const tree = await client.getDecofileTree(treeSha, packagePath); + const tree = await client.listDecofileEntries(sha, packagePath); const entries = blockEntriesInTree(tree, packagePath); // Per-blob disk hits first; only what's left goes to GitHub. @@ -284,7 +303,7 @@ async function resolveSnapshot( async (e) => { const hit = known.get(e.path); if (hit !== undefined) return { stem: e.stem, content: hit }; - const content = await client.getBlobText(e.sha); + const content = await client.readBlob(e.sha); await putBlob(owner, repo, e.sha, content); return { stem: e.stem, content }; }, @@ -315,12 +334,13 @@ async function resolveSnapshot( * blob fetches instead of a wrong document. */ async function tryTarballIngest( - client: GitDataClient, + client: RepoContentClient, sha: string, packagePath: string | null, entries: Array<{ stem: string; sha: string; path: string; size?: number }>, ): Promise | null> { - const repoKey = `${client.owner}/${client.repo}`; + const [owner, repo] = cacheScope(client.repo); + const repoKey = repoIdentityKey(client.repo); // Pre-validate BEFORE downloading (golden rule 3 beats golden rule 2): the // extracted files are only "small" in aggregate if the tree says so. A tree @@ -350,7 +370,8 @@ async function tryTarballIngest( const scratchDir = await makeScratchDir(); if (scratchDir === null) return null; try { - const body = await client.getTarballStream(sha); + const body = await client.getArchive(sha); + if (body === null) return null; // provider serves no archive (the e2e stub) const files = await extractBlocksFromTarball( body, scratchDir, @@ -364,7 +385,7 @@ async function tryTarballIngest( // same ref): skip rather than cache under a wrong sha. if (blobSha === undefined) continue; const content = await readFile(f.diskPath, "utf8"); - await putBlob(client.owner, client.repo, blobSha, content); + await putBlob(owner, repo, blobSha, content); out.set(f.path, content); } return out; diff --git a/apps/api/src/git-providers/content/github.test.ts b/apps/api/src/git-providers/content/github.test.ts new file mode 100644 index 0000000000..e039630baa --- /dev/null +++ b/apps/api/src/git-providers/content/github.test.ts @@ -0,0 +1,247 @@ +import { describe, expect, it } from "bun:test"; +import { + type ChangeSources, + mapGithubPull, + resolveEntriesAtPaths, + treeWriteEntries, +} from "./github"; +import type { TreeEntry } from "./types"; + +/** An in-memory repo tree, keyed by directory path ("" for root). */ +function fakeOps(dirs: Record) { + return { + resolveSubtreeSha: (_root: string, segments: string[]) => { + const dir = segments.join("/"); + return Promise.resolve(dir in dirs ? `tree:${dir}` : null); + }, + treeShallow: (treeSha: string) => { + const dir = treeSha.slice("tree:".length); + return Promise.resolve(dirs[dir] ?? []); + }, + }; +} + +const blob = (path: string, sha: string): TreeEntry => ({ + path, + type: "blob", + sha, +}); + +describe("resolveEntriesAtPaths", () => { + it("resolves blobs at the root and in nested directories", async () => { + const ops = fakeOps({ + "": [ + blob("Header.json", "b1"), + { ...blob("blocks", "t1"), type: "tree" }, + ], + blocks: [blob("Footer.json", "b2")], + }); + + const result = await resolveEntriesAtPaths( + "root", + ["Header.json", "blocks/Footer.json"], + ops, + ); + + expect(result.get("Header.json")?.sha).toBe("b1"); + expect(result.get("blocks/Footer.json")?.sha).toBe("b2"); + }); + + it("omits a path missing at this commit instead of throwing", async () => { + const ops = fakeOps({ "": [blob("Header.json", "b1")] }); + + const result = await resolveEntriesAtPaths( + "root", + ["Header.json", "Deleted.json"], + ops, + ); + + expect(result.has("Header.json")).toBe(true); + expect(result.has("Deleted.json")).toBe(false); + }); + + it("omits every path under a directory that does not exist", async () => { + const ops = fakeOps({ "": [] }); + + const result = await resolveEntriesAtPaths( + "root", + ["missing/dir/File.json"], + ops, + ); + + expect(result.size).toBe(0); + }); + + it("lists a shared directory once for paths in the same directory", async () => { + let listings = 0; + const dirs = { "": [blob("A.json", "a"), blob("B.json", "b")] }; + const ops = { + resolveSubtreeSha: fakeOps(dirs).resolveSubtreeSha, + treeShallow: (treeSha: string) => { + listings++; + return fakeOps(dirs).treeShallow(treeSha); + }, + }; + + await resolveEntriesAtPaths("root", ["A.json", "B.json"], ops); + + expect(listings).toBe(1); + }); +}); + +describe("treeWriteEntries", () => { + /** Nothing exists at any ref unless a test says so. */ + const uploads: ChangeSources = { + blobSha: (change) => `blob:${change.path}`, + copySource: () => null, + }; + + it("maps a write to its blob and a deletion to a null sha", () => { + const entries = treeWriteEntries( + [ + { path: ".deco/blocks/Hero.json", content: "{}\n" }, + { path: ".deco/blocks/Old.json", deleted: true }, + ], + uploads, + ); + + expect(entries).toEqual([ + { + path: ".deco/blocks/Hero.json", + mode: "100644", + type: "blob", + sha: "blob:.deco/blocks/Hero.json", + }, + { + path: ".deco/blocks/Old.json", + mode: "100644", + type: "blob", + sha: null, + }, + ]); + }); + + it("keeps the caller's order, since the last entry for a path wins", () => { + const entries = treeWriteEntries( + [ + { path: "A.json", deleted: true }, + { path: "A.json", content: "{}\n" }, + ], + { ...uploads, blobSha: () => "b1" }, + ); + + expect(entries.map((e) => e.sha)).toEqual([null, "b1"]); + }); + + it("asks for a blob only for the paths that carry content", () => { + const asked: string[] = []; + treeWriteEntries( + [ + { path: "A.json", deleted: true }, + { path: "B.json", content: "{}\n" }, + { path: "C.json", copyFromRef: "base" }, + ], + { + blobSha: (change) => { + asked.push(change.path); + return "b1"; + }, + copySource: () => ({ sha: "c1", mode: "100644" }), + }, + ); + + expect(asked).toEqual(["B.json"]); + }); + + it("points a copy at the sha already in the source ref's tree", () => { + const entries = treeWriteEntries( + [{ path: "A.json", copyFromRef: "merge-base" }], + { + ...uploads, + copySource: (ref, path) => + ref === "merge-base" && path === "A.json" + ? { sha: "existing", mode: "100644" } + : null, + }, + ); + + expect(entries).toEqual([ + { path: "A.json", mode: "100644", type: "blob", sha: "existing" }, + ]); + }); + + it("carries the source entry's mode through a copy, so an executable file stays executable", () => { + const entries = treeWriteEntries( + [{ path: "run.sh", copyFromRef: "merge-base" }], + { ...uploads, copySource: () => ({ sha: "s", mode: "100755" }) }, + ); + + expect(entries[0]?.mode).toBe("100755"); + }); + + it("keeps a symlink a symlink rather than folding it into a regular file", () => { + const entries = treeWriteEntries([{ path: "link", copyFromRef: "r" }], { + ...uploads, + copySource: () => ({ sha: "s", mode: "120000" }), + }); + + expect(entries[0]?.mode).toBe("120000"); + }); + + it("takes the change's mode over the source's when it states one", () => { + const entries = treeWriteEntries( + [ + { path: "run.sh", content: "#!/bin/sh\n", mode: "100755" }, + { path: "copied.sh", copyFromRef: "r", mode: "100644" }, + ], + { ...uploads, copySource: () => ({ sha: "s", mode: "100755" }) }, + ); + + expect(entries.map((e) => e.mode)).toEqual(["100755", "100644"]); + }); + + it("fails loud when a copy source is not there, rather than deleting the path", () => { + expect(() => + treeWriteEntries([{ path: "Gone.json", copyFromRef: "r" }], uploads), + ).toThrow(/does not exist at r/); + }); +}); + +describe("mapGithubPull", () => { + it("reports a merged pull as merged, not closed", () => { + expect( + mapGithubPull({ + number: 7, + html_url: "https://github.example/o/r/pull/7", + title: "Publish draft", + state: "closed", + merged: true, + }), + ).toEqual({ + number: 7, + url: "https://github.example/o/r/pull/7", + title: "Publish draft", + state: "merged", + }); + }); + + it("defaults to open for a response that omits the state", () => { + const pull = mapGithubPull({ + number: 1, + html_url: "https://github.example/o/r/pull/1", + }); + expect(pull.state).toBe("open"); + expect(pull.title).toBe(""); + }); + + it("keeps a closed-but-unmerged pull closed", () => { + expect( + mapGithubPull({ + number: 2, + html_url: "u", + state: "closed", + merged: false, + }).state, + ).toBe("closed"); + }); +}); diff --git a/apps/api/src/git-providers/content/github.ts b/apps/api/src/git-providers/content/github.ts new file mode 100644 index 0000000000..aa10c5f12a --- /dev/null +++ b/apps/api/src/git-providers/content/github.ts @@ -0,0 +1,920 @@ +/** + * `RepoContentClient` over GitHub's REST API (Git Data + contents + merges + + * pulls + compare). Raw fetch, no SDK — mirrors the header/UA conventions of + * `shared/github-runtime-detect.ts`, but errors PROPAGATE (a failed write must + * surface, not degrade to null). + * + * Ported from `decofile/github-git-data.ts`, whose four-call write sequence + * (blob → tree → commit → ref) is now folded into `commitFiles` — the only + * shape GitLab can also implement. Everything else is the same wire traffic. + * + * The base URL is overridable so the e2e suite can point the whole client at a + * local stub — tests must never reach api.github.com. + */ + +import { + apiBaseUrlFor, + type RepoRef, + splitOwnerName, +} from "@decocms/shared/git-providers"; +import { + countGithubRateLimited, + githubRetryAfterMs, + isGithubRateLimited, + recordGithubRateLimit, +} from "@/observability/github-rate-limit"; +import type { TokenSource } from "../types"; +import { + type ChangeRequestInfo, + type FileChange, + type RepoContentClient, + RepoWriteConflict, + type TreeEntry, +} from "./types"; + +/** + * A tree entry with the `mode` GitHub reports for it. The neutral `TreeEntry` + * has no use for a Git file mode, but a tree WRITE does: a `copyFromRef` + * change reproduces the source entry's mode verbatim, which is how an + * executable file (or a symlink) survives a replay. + */ +type GithubEntry = TreeEntry & { mode: string }; + +const DEFAULT_TIMEOUT_MS = 15_000; + +/** A whole-repo archive is a download, not a REST call — it needs room to stream. */ +const ARCHIVE_TIMEOUT_MS = 60_000; + +/** e2e seam: set GITHUB_API_BASE_URL to a local stub. Read per call site so a + * long-lived process (dev server) and the test webServer agree on one value. */ +function githubApiBaseUrl(host: string): string { + return process.env.GITHUB_API_BASE_URL ?? apiBaseUrlFor("github", host); +} + +/** + * `default_branch` per repo, cached across requests — a client instance lives + * for ONE of them (see `content/index.ts`), so its own memo never survives. + * + * Ten minutes, not an hour: a renamed default branch compares against a branch + * that no longer exists until this expires. Keyed by full URL, so the e2e stub + * and api.github.com can never share an entry. + */ +const DEFAULT_BRANCH_TTL_MS = 10 * 60_000; +const defaultBranchCache = new Map(); + +function cachedDefaultBranch(url: string): string | null { + const hit = defaultBranchCache.get(url); + if (!hit) return null; + if (Date.now() - hit.at > DEFAULT_BRANCH_TTL_MS) { + defaultBranchCache.delete(url); + return null; + } + return hit.value; +} + +/** + * Conditional-request cache: url -> { etag, body }. GitHub serves 304s for + * matching `If-None-Match` and those do NOT count against the primary rate + * limit, so re-reads of hot mutable endpoints (ref resolves, compares, repo + * meta) become rate-limit-free when nothing changed. Module-level because a + * client instance lives for one HTTP request. + * + * Content-addressed payloads (blobs, trees, tarballs) are excluded: they are + * immutable per sha and already cached at the read layer, so an ETag entry + * would never be re-requested — it would only duplicate large bodies here. + * + * Keyed by URL alone, NOT by token: a 304 is only ever served after GitHub + * authorized THIS request's token, so a cached body is never revealed to a + * caller GitHub itself would refuse. + */ +const ETAG_CACHE_MAX = 512; +/** Byte budget across all stored bodies (approximated as the JSON.stringify + * length, computed once at insert). Primary bound; entry count is secondary. */ +const ETAG_CACHE_MAX_BYTES = 8 * 1024 * 1024; +/** Bodies over this are not cached at all — one huge response must not evict + * the whole working set. */ +const ETAG_CACHE_MAX_BODY_BYTES = 256 * 1024; +const etagCache = new Map< + string, + { etag: string; body: unknown; bytes: number } +>(); +let etagCacheBytes = 0; + +function etagCacheable(method: string, path: string): boolean { + return ( + method === "GET" && + !path.includes("/git/blobs/") && + !path.includes("/git/trees/") && + // `/contents/{path}?ref=` is sha-pinned, hence content-addressed too. + !path.includes("/contents/") && + !path.includes("/tarball/") + ); +} + +function etagCacheDelete(url: string): void { + const existing = etagCache.get(url); + if (!existing) return; + etagCache.delete(url); + etagCacheBytes -= existing.bytes; +} + +function etagCachePut(url: string, etag: string, body: unknown): void { + let bytes: number; + try { + bytes = JSON.stringify(body)?.length ?? 0; + } catch { + return; // unserializable body — don't cache + } + etagCacheDelete(url); + if (bytes > ETAG_CACHE_MAX_BODY_BYTES) return; // stale entry already dropped + etagCache.set(url, { etag, body, bytes }); + etagCacheBytes += bytes; + while ( + etagCacheBytes > ETAG_CACHE_MAX_BYTES || + etagCache.size > ETAG_CACHE_MAX + ) { + const oldest = etagCache.keys().next().value; + if (oldest === undefined) break; + etagCacheDelete(oldest); + } +} + +class GitHubApiError extends Error { + constructor( + readonly status: number, + readonly method: string, + readonly path: string, + message: string, + ) { + super(`GitHub ${method} ${path} failed (${status}): ${message}`); + this.name = "GitHubApiError"; + } +} + +/** + * GitHub refused the call for rate reasons. NOT retriable: retrying a secondary + * limit is the burst being limited. `retryAfterMs` tells the caller when to come + * back; it is never slept on inside a request. + */ +class GitHubRateLimitError extends GitHubApiError { + /** Duck-typed by `repoRateLimitRetryAfterMs`, so a 403 primary limit is + * recognisable as a rate refusal without importing this class. */ + readonly isRateLimited = true; + + constructor( + status: number, + method: string, + path: string, + readonly kind: "primary" | "secondary", + readonly retryAfterMs: number | null, + ) { + super( + status, + method, + path, + `GitHub ${kind} rate limit reached${ + retryAfterMs === null + ? "" + : `; retry in ${Math.ceil(retryAfterMs / 1000)}s` + }`, + ); + this.name = "GitHubRateLimitError"; + } +} + +/** GitHub's own tree entry shape — `mode` and submodule entries included. */ +interface GithubTreeEntry { + path: string; + mode: string; + type: "blob" | "tree" | "commit"; + sha: string; + size?: number; +} + +/** Tree write entry — `sha: null` deletes the path (Git Data API contract). */ +export interface TreeWriteEntry { + path: string; + /** + * Git file mode: the change's own (defaulting to a regular file), or for a + * `copyFromRef` change the source entry's mode passed through unchanged — + * so a `100755` script stays executable and a `120000` symlink stays a + * symlink instead of becoming a text file holding a path. + */ + mode: string; + type: "blob"; + sha: string | null; +} + +const BLOB_MODE = "100644"; + +/** What one change points the tree at: a blob sha, or null to delete. */ +export interface ChangeSources { + /** Sha of the blob uploaded for a change that carries content. */ + blobSha: (change: { path: string; content: string }) => string; + /** The entry `path` has at `ref`, or null when it is absent there. */ + copySource: ( + ref: string, + path: string, + ) => { sha: string; mode: string } | null; +} + +/** + * The tree write for one commit's changes: a deletion is `sha: null`, a + * content write the sha of the blob created for it, and a `copyFromRef` the + * sha ALREADY in that ref's tree — no blob is uploaded and no body is read, + * which is what keeps a whole-branch replay the cost of one tree write. Order + * is preserved (GitHub applies the last entry for a duplicated path), and the + * mapping is pure so the deletion and mode contracts are asserted without a + * network. + */ +export function treeWriteEntries( + changes: readonly FileChange[], + sources: ChangeSources, +): TreeWriteEntry[] { + return changes.map((change) => { + if ("deleted" in change) { + return { path: change.path, mode: BLOB_MODE, type: "blob", sha: null }; + } + if ("copyFromRef" in change) { + const source = sources.copySource(change.copyFromRef, change.path); + if (!source) { + throw new GitHubApiError( + 404, + "POST", + "git/trees", + `${change.path} does not exist at ${change.copyFromRef}`, + ); + } + return { + path: change.path, + mode: change.mode ?? source.mode, + type: "blob", + sha: source.sha, + }; + } + return { + path: change.path, + mode: change.mode ?? BLOB_MODE, + type: "blob", + sha: sources.blobSha(change), + }; + }); +} + +/** GitHub tree entry → the neutral one (plus its mode, which the write side + * reads back). Submodules are not addressable content. */ +function neutralEntry(entry: GithubTreeEntry): GithubEntry | null { + if (entry.type !== "blob" && entry.type !== "tree") return null; + return { + path: entry.path, + sha: entry.sha, + type: entry.type, + mode: entry.mode, + ...(entry.size === undefined ? {} : { size: entry.size }), + }; +} + +/** + * `getEntriesAtPaths`'s directory-scoped walk, factored out for testing: given + * how to list a subtree's direct children, resolve each path to its entry by + * visiting only the directories the paths actually live in (cached per + * directory, so siblings share one listing call). + */ +export async function resolveEntriesAtPaths( + rootTreeSha: string, + paths: string[], + ops: { + resolveSubtreeSha: ( + rootTreeSha: string, + segments: string[], + ) => Promise; + treeShallow: (treeSha: string) => Promise; + }, +): Promise> { + const result = new Map(); + const dirListings = new Map(); + for (const path of paths) { + const slash = path.lastIndexOf("/"); + const dir = slash === -1 ? "" : path.slice(0, slash); + const base = slash === -1 ? path : path.slice(slash + 1); + let listing = dirListings.get(dir); + if (listing === undefined) { + const subtreeSha = await ops.resolveSubtreeSha( + rootTreeSha, + dir === "" ? [] : dir.split("/"), + ); + listing = subtreeSha === null ? null : await ops.treeShallow(subtreeSha); + dirListings.set(dir, listing); + } + const entry = listing?.find((e) => e.type === "blob" && e.path === base); + if (entry) result.set(path, { ...entry, path }); + } + return result; +} + +export interface GithubContentClientOptions { + repo: RepoRef; + tokenSource: TokenSource; +} + +export class GithubContentClient implements RepoContentClient { + readonly repo: RepoRef; + private readonly tokenSource: TokenSource; + private readonly apiBaseUrl: string; + private readonly repoBase: string; + private readonly owner: string; + private defaultBranch: string | null = null; + /** commit sha -> its tree sha. A commit is immutable, and one request can + * ask for the same commit's tree several times (compare + read + write). */ + private readonly treeShaOfCommit = new Map(); + /** tree sha -> its direct children. Also immutable, and also asked for + * twice by one request: a discard resolves a directory to plan the write, + * then again to resolve the blobs that write copies. */ + private readonly treeListings = new Map(); + + constructor(options: GithubContentClientOptions) { + this.repo = options.repo; + this.tokenSource = options.tokenSource; + this.apiBaseUrl = githubApiBaseUrl(options.repo.host); + const { owner, name } = splitOwnerName(options.repo); + this.owner = owner; + this.repoBase = `/repos/${owner}/${name}`; + } + + private async accessToken(): Promise { + const token = await this.tokenSource.get(); + if (!token) { + throw new GitHubApiError( + 401, + "AUTH", + this.repoBase, + `No usable GitHub token for ${this.repo.path}; reconnect the account`, + ); + } + return token.token; + } + + private async call( + method: string, + path: string, + body?: unknown, + opts?: { allow?: number[] }, + ): Promise<{ status: number; json: T }> { + const url = `${this.apiBaseUrl}${path}`; + const accessToken = await this.accessToken(); + const conditional = etagCacheable(method, path) + ? etagCache.get(url) + : undefined; + const res = await fetch(url, { + method, + headers: { + Accept: "application/vnd.github+json", + "User-Agent": "studio-decofile", + Authorization: `token ${accessToken}`, + ...(conditional ? { "If-None-Match": conditional.etag } : {}), + ...(body !== undefined ? { "content-type": "application/json" } : {}), + }, + body: body !== undefined ? JSON.stringify(body) : undefined, + signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS), + }); + recordGithubRateLimit(res.headers, { lane: "rest", operation: method }); + + if (conditional && res.status === 304) { + return { status: 200, json: conditional.body as T }; + } + if (isGithubRateLimited(res)) { + const kind = + res.headers.get("retry-after") !== null ? "secondary" : "primary"; + countGithubRateLimited({ lane: "rest", operation: method, kind }); + await res.body?.cancel().catch(() => {}); + throw new GitHubRateLimitError( + res.status, + method, + path, + kind, + githubRetryAfterMs(res.headers), + ); + } + if (!res.ok && !opts?.allow?.includes(res.status)) { + const text = await res.text().catch(() => ""); + let message = text; + try { + message = (JSON.parse(text) as { message?: string }).message ?? text; + } catch { + // keep raw text + } + throw new GitHubApiError(res.status, method, path, message); + } + const json = + res.status === 204 ? (undefined as T) : ((await res.json()) as T); + if (res.status === 200 && etagCacheable(method, path)) { + const etag = res.headers.get("etag"); + if (etag) etagCachePut(url, etag, json); + } + return { status: res.status, json }; + } + + /** One tree's DIRECT children (non-recursive). GitHub returns subdirectories + * as `type: "tree"` entries carrying their own sha; the entry `path` is the + * bare child name, not a repo-relative path. */ + private async treeShallow(treeSha: string): Promise { + const memo = this.treeListings.get(treeSha); + if (memo) return memo; + const { json } = await this.call<{ + tree: GithubTreeEntry[]; + truncated: boolean; + }>("GET", `${this.repoBase}/git/trees/${treeSha}`); + if (json.truncated) { + // One directory over the cap: unreachable once scoped, but fail loud. + throw new GitHubApiError( + 502, + "GET", + `${this.repoBase}/git/trees/${treeSha}`, + "tree listing truncated by GitHub; directory too large", + ); + } + const entries = json.tree.flatMap((e) => neutralEntry(e) ?? []); + this.treeListings.set(treeSha, entries); + return entries; + } + + /** Walk `segments` from `rootTreeSha`, returning the final subtree's sha, or + * null when any segment is missing or is not a directory. */ + private async resolveSubtreeSha( + rootTreeSha: string, + segments: string[], + ): Promise { + let sha = rootTreeSha; + for (const segment of segments) { + const child = (await this.treeShallow(sha)).find( + (e) => e.type === "tree" && e.path === segment, + ); + if (!child) return null; + sha = child.sha; + } + return sha; + } + + /** + * The tree sha a tree-ish addresses. Callers hold commit shas (a branch head, + * a merge base), which GitHub's tree endpoints do NOT accept, so the commit is + * resolved first; a sha that is already a tree answers 404/422 there and is + * used as-is. + */ + private async treeShaFor(treeish: string): Promise { + const memo = this.treeShaOfCommit.get(treeish); + if (memo) return memo; + const { status, json } = await this.call<{ tree?: { sha: string } }>( + "GET", + `${this.repoBase}/git/commits/${treeish}`, + undefined, + { allow: [404, 422] }, + ); + const treeSha = status === 200 && json?.tree?.sha ? json.tree.sha : treeish; + this.treeShaOfCommit.set(treeish, treeSha); + return treeSha; + } + + /** Blob text by sha via the Blob API (base64, up to 100MB). */ + private async blobText(blobSha: string): Promise { + const { json } = await this.call<{ content: string; encoding: string }>( + "GET", + `${this.repoBase}/git/blobs/${blobSha}`, + ); + if (json.encoding !== "base64") { + throw new GitHubApiError( + 502, + "GET", + `${this.repoBase}/git/blobs/${blobSha}`, + `unexpected blob encoding ${json.encoding}`, + ); + } + return Buffer.from(json.content, "base64").toString("utf-8"); + } + + async getDefaultBranch(): Promise { + if (this.defaultBranch) return this.defaultBranch; + const url = `${this.apiBaseUrl}${this.repoBase}`; + const shared = cachedDefaultBranch(url); + if (shared) { + this.defaultBranch = shared; + return shared; + } + const { json } = await this.call<{ default_branch: string }>( + "GET", + this.repoBase, + ); + this.defaultBranch = json.default_branch; + defaultBranchCache.set(url, { value: this.defaultBranch, at: Date.now() }); + return this.defaultBranch; + } + + async getBranch( + branch: string, + ): Promise<{ sha: string; committedAt: string } | null> { + const { status, json } = await this.call<{ + commit?: { sha: string; commit: { committer: { date: string } } }; + }>("GET", `${this.repoBase}/branches/${encodeRefPath(branch)}`, undefined, { + allow: [404], + }); + if (status === 404 || !json?.commit) return null; + return { + sha: json.commit.sha, + committedAt: json.commit.commit.committer.date, + }; + } + + /** The branch head, as a hard requirement — a missing branch is a 404. */ + private async requireBranchSha(branch: string): Promise { + const head = await this.getBranch(branch); + if (!head) { + throw new GitHubApiError( + 404, + "GET", + `${this.repoBase}/branches/${branch}`, + `Branch ${branch} does not exist`, + ); + } + return head.sha; + } + + /** + * Redirects to codeload; fetch follows them. The signal covers the whole + * body, so a stalled download aborts the stream and the consumer's tar + * pipeline fails over to per-blob fetches. + */ + async getArchive(ref: string): Promise | null> { + const path = `${this.repoBase}/tarball/${encodeRefPath(ref)}`; + const res = await fetch(`${this.apiBaseUrl}${path}`, { + headers: { + "User-Agent": "studio-decofile", + Authorization: `token ${await this.accessToken()}`, + }, + signal: AbortSignal.timeout(ARCHIVE_TIMEOUT_MS), + }); + if (res.status === 404) { + await res.body?.cancel().catch(() => {}); + return null; + } + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new GitHubApiError(res.status, "GET", path, text.slice(0, 300)); + } + if (!res.body) { + throw new GitHubApiError(502, "GET", path, "tarball had no body"); + } + return res.body; + } + + async listDecofileEntries( + treeish: string, + packagePath: string | null, + ): Promise { + const rootTreeSha = await this.treeShaFor(treeish); + const decoSegments = [ + ...(packagePath ? packagePath.split("/") : []), + ".deco", + ]; + const decoTreeSha = await this.resolveSubtreeSha(rootTreeSha, decoSegments); + if (decoTreeSha === null) return []; // no `.deco/` in this project yet + const decoDir = packagePath ? `${packagePath}/.deco` : ".deco"; + const decoChildren = await this.treeShallow(decoTreeSha); + const out: TreeEntry[] = []; + // The merged artifact, only when this repo commits it (usually gitignored). + const gen = decoChildren.find( + (e) => e.type === "blob" && e.path === "blocks.gen.json", + ); + if (gen) out.push({ ...gen, path: `${decoDir}/blocks.gen.json` }); + const blocksDir = decoChildren.find( + (e) => e.type === "tree" && e.path === "blocks", + ); + if (blocksDir) { + for (const child of await this.treeShallow(blocksDir.sha)) { + if (child.type !== "blob") continue; + out.push({ ...child, path: `${decoDir}/blocks/${child.path}` }); + } + } + return out; + } + + getEntriesAtPaths( + treeish: string, + paths: string[], + ): Promise> { + return this.entriesWithModeAt(treeish, paths); + } + + /** As `getEntriesAtPaths`, keeping the mode the tree write needs. */ + private async entriesWithModeAt( + treeish: string, + paths: string[], + ): Promise> { + return resolveEntriesAtPaths(await this.treeShaFor(treeish), paths, { + resolveSubtreeSha: (root, segments) => + this.resolveSubtreeSha(root, segments), + treeShallow: (treeSha) => this.treeShallow(treeSha), + }); + } + + readBlob(sha: string): Promise { + return this.blobText(sha); + } + + async readFileAtRef(ref: string, path: string): Promise { + const { status, json } = await this.call<{ + content?: string; + encoding?: string; + sha?: string; + }>( + "GET", + `${this.repoBase}/contents/${encodeRefPath(path)}?ref=${encodeURIComponent(ref)}`, + undefined, + { allow: [404] }, + ); + if (status === 404 || json?.content === undefined) return null; + /** Files over the Contents API's 1MB limit come back with + * `encoding: "none"` and empty content, but the blob sha is still there — + * fetch the blob directly (the Blob API serves up to 100MB). A large + * `.deco/meta.gen.json` routinely crosses that line. */ + if (json.encoding === "none" && json.sha) { + return this.blobText(json.sha); + } + if (json.encoding !== "base64") { + throw new GitHubApiError( + 502, + "GET", + `${this.repoBase}/contents/${path}`, + `unexpected content encoding ${json.encoding}`, + ); + } + return Buffer.from(json.content, "base64").toString("utf-8"); + } + + /** + * Blob → tree → commit → ref, in that order. The ref moves LAST, which is + * what makes a `rewriteFrom` commit crash-safe: every object exists before + * anything is repointed, so an interrupted rewrite loses a commit nobody + * referenced yet rather than the branch's own history. + */ + async commitFiles(params: { + branch: string; + message: string; + expectedHead: string | null; + rewriteFrom?: string; + changes: FileChange[]; + }): Promise<{ sha: string }> { + const { branch, message, changes, rewriteFrom, expectedHead } = params; + const parent = + rewriteFrom ?? expectedHead ?? (await this.requireBranchSha(branch)); + const baseTreeSha = await this.treeShaFor(parent); + + const copySources = await this.resolveCopySources(changes); + const blobShas = new Map(); + for (const change of changes) { + if ("content" in change) { + blobShas.set(change.path, await this.createBlob(change.content)); + } + } + const entries = treeWriteEntries(changes, { + blobSha: (change) => blobShas.get(change.path) as string, + copySource: (ref, path) => copySources.get(ref)?.get(path) ?? null, + }); + + const { json: tree } = await this.call<{ sha: string }>( + "POST", + `${this.repoBase}/git/trees`, + { base_tree: baseTreeSha, tree: entries }, + ); + const { json: commit } = await this.call<{ sha: string }>( + "POST", + `${this.repoBase}/git/commits`, + { message, tree: tree.sha, parents: [parent] }, + ); + if (rewriteFrom !== undefined) { + /** A forced ref update takes no `If-Match`, so the guard is a re-read + * here, after the commit exists — the latest point at which the lease + * can still be checked, and the shortest window available. */ + if ( + expectedHead !== null && + (await this.requireBranchSha(branch)) !== expectedHead + ) { + throw new RepoWriteConflict( + `${branch} moved while rewriting ${this.repo.path}`, + ); + } + await this.updateRef(branch, commit.sha, true); + return { sha: commit.sha }; + } + try { + await this.updateRef(branch, commit.sha, false); + } catch (err) { + // Non-fast-forward: the branch moved past `parent` while we built this. + if (err instanceof GitHubApiError && err.status === 422) { + throw new RepoWriteConflict( + `${branch} moved while committing to ${this.repo.path}`, + { cause: err }, + ); + } + throw err; + } + return { sha: commit.sha }; + } + + /** + * The blobs a commit's `copyFromRef` changes point at, one directory-scoped + * resolve per distinct ref. Nothing is downloaded: the tree write reuses the + * sha (and the mode) the entry already has there. + */ + private async resolveCopySources( + changes: readonly FileChange[], + ): Promise>> { + const pathsByRef = new Map(); + for (const change of changes) { + if (!("copyFromRef" in change)) continue; + const bucket = pathsByRef.get(change.copyFromRef); + if (bucket) bucket.push(change.path); + else pathsByRef.set(change.copyFromRef, [change.path]); + } + const byRef = new Map>(); + for (const [ref, paths] of pathsByRef) { + byRef.set(ref, await this.entriesWithModeAt(ref, paths)); + } + return byRef; + } + + /** Blob for `content`; GitHub is content-addressed, so this is idempotent. */ + private async createBlob(content: string): Promise { + const { json } = await this.call<{ sha: string }>( + "POST", + `${this.repoBase}/git/blobs`, + { content, encoding: "utf-8" }, + ); + return json.sha; + } + + private async updateRef( + branch: string, + sha: string, + force: boolean, + ): Promise { + await this.call( + "PATCH", + `${this.repoBase}/git/refs/heads/${encodeRefPath(branch)}`, + { sha, force }, + ); + } + + async createBranch(branch: string, sha: string): Promise { + try { + await this.call("POST", `${this.repoBase}/git/refs`, { + ref: `refs/heads/${branch}`, + sha, + }); + } catch (err) { + if (err instanceof GitHubApiError && err.status === 422) { + throw new RepoWriteConflict( + `${branch} already exists on ${this.repo.path}`, + { cause: err }, + ); + } + throw err; + } + } + + forceBranchHead(branch: string, sha: string): Promise { + return this.updateRef(branch, sha, true); + } + + async mergeBranches( + base: string, + head: string, + message: string, + ): Promise { + const { status, json } = await this.call<{ sha?: string } | undefined>( + "POST", + `${this.repoBase}/merges`, + { base, head, commit_message: message }, + { allow: [204] }, + ); + // 204 = base already contains head; 201 carries the merge commit. + return status === 204 ? null : (json?.sha ?? null); + } + + async createChangeRequest(params: { + base: string; + head: string; + title: string; + }): Promise { + const { json } = await this.call( + "POST", + `${this.repoBase}/pulls`, + params, + ); + return mapGithubPull(json); + } + + async findOpenChangeRequest( + base: string, + head: string, + ): Promise { + const { json } = await this.call( + "GET", + `${this.repoBase}/pulls?state=open&base=${encodeURIComponent(base)}&head=${encodeURIComponent(`${this.owner}:${head}`)}`, + ); + const first = json[0]; + return first ? mapGithubPull(first) : null; + } + + async compare( + base: string, + head: string, + ): Promise<{ aheadBy: number; behindBy: number }> { + const { json } = await this.call<{ ahead_by: number; behind_by: number }>( + "GET", + `${this.repoBase}/compare/${encodeRefPath(base)}...${encodeRefPath(head)}`, + ); + return { aheadBy: json.ahead_by, behindBy: json.behind_by }; + } + + async compareDetailed( + base: string, + head: string, + ): Promise<{ + aheadBy: number; + behindBy: number; + mergeBaseSha: string; + files: Array<{ + filename: string; + status: string; + sha: string; + previousFilename?: string; + }>; + commitMessages: string[]; + }> { + const { json } = await this.call<{ + ahead_by: number; + behind_by: number; + merge_base_commit: { sha: string }; + files?: Array<{ + filename: string; + status: string; + sha: string; + previous_filename?: string; + }>; + commits?: Array<{ commit?: { message?: string } }>; + }>( + "GET", + `${this.repoBase}/compare/${encodeRefPath(base)}...${encodeRefPath(head)}`, + ); + return { + aheadBy: json.ahead_by, + behindBy: json.behind_by, + mergeBaseSha: json.merge_base_commit.sha, + files: (json.files ?? []).map((f) => ({ + filename: f.filename, + status: f.status, + sha: f.sha, + ...(f.previous_filename + ? { previousFilename: f.previous_filename } + : {}), + })), + /** + * Messages of the commits on `head` that `base` lacks — the set a squash + * collapses. GitHub caps this list at 250 entries; a longer branch is + * truncated, so it feeds attribution, never correctness. + */ + commitMessages: (json.commits ?? []) + .map((c) => c.commit?.message ?? "") + .filter((m) => m.length > 0), + }; + } +} + +interface GithubPullJson { + number: number; + html_url: string; + title?: string; + state?: string; + merged?: boolean; +} + +/** Pure: GitHub's pull request object → the neutral change request. */ +export function mapGithubPull(json: GithubPullJson): ChangeRequestInfo { + const state = + json.merged === true + ? "merged" + : json.state === "closed" + ? "closed" + : "open"; + return { + number: json.number, + url: json.html_url, + title: json.title ?? "", + state, + }; +} + +/** Encode a branch name for a ref path segment, preserving `/` separators. */ +function encodeRefPath(branch: string): string { + return branch.split("/").map(encodeURIComponent).join("/"); +} diff --git a/apps/api/src/git-providers/content/gitlab.test.ts b/apps/api/src/git-providers/content/gitlab.test.ts new file mode 100644 index 0000000000..8212759a0d --- /dev/null +++ b/apps/api/src/git-providers/content/gitlab.test.ts @@ -0,0 +1,471 @@ +import { describe, expect, test } from "bun:test"; +import { + isProtectedBranch, + buildCommitActions, + decoDirFor, + directoryOf, + gitlabErrorMessage, + groupPathsByDirectory, + isBranchExistsConflict, + isCommitConflict, + mapCompareDiff, + mapMergeRequest, + mapMergeRequestState, + pooledMap, + type ResolvedChange, +} from "./gitlab"; + +describe("buildCommitActions", () => { + test("an existing file is an update guarded by its last commit", () => { + const changes: ResolvedChange[] = [{ path: "a.json", content: "{}" }]; + expect(buildCommitActions(changes, new Map([["a.json", "c1"]]))).toEqual([ + { + action: "update", + file_path: "a.json", + content: "{}", + last_commit_id: "c1", + }, + ]); + }); + + test("an absent file is an unguarded create", () => { + const changes: ResolvedChange[] = [{ path: "new.json", content: "1" }]; + expect(buildCommitActions(changes, new Map())).toEqual([ + { action: "create", file_path: "new.json", content: "1" }, + ]); + expect(buildCommitActions(changes, new Map([["new.json", null]]))).toEqual([ + { action: "create", file_path: "new.json", content: "1" }, + ]); + }); + + test("a delete carries the guard and never the content field", () => { + const actions = buildCommitActions( + [{ path: "gone.json", deleted: true }], + new Map([["gone.json", "c9"]]), + ); + expect(actions).toEqual([ + { action: "delete", file_path: "gone.json", last_commit_id: "c9" }, + ]); + expect(actions[0]).not.toHaveProperty("content"); + }); + + test("deleting a path that is not there is dropped, not sent", () => { + expect( + buildCommitActions([{ path: "never.json", deleted: true }], new Map()), + ).toEqual([]); + expect( + buildCommitActions( + [ + { path: "never.json", deleted: true }, + { path: "here.json", deleted: true }, + ], + new Map([["here.json", "c2"]]), + ), + ).toEqual([ + { action: "delete", file_path: "here.json", last_commit_id: "c2" }, + ]); + }); + + test("mixed batch keeps input order and maps each action independently", () => { + const actions = buildCommitActions( + [ + { path: "keep.json", content: "k" }, + { path: "drop.json", deleted: true }, + { path: "add.json", content: "a" }, + ], + new Map([ + ["keep.json", "c1"], + ["drop.json", "c2"], + ]), + ); + expect(actions.map((a) => [a.action, a.file_path])).toEqual([ + ["update", "keep.json"], + ["delete", "drop.json"], + ["create", "add.json"], + ]); + }); + + test("two changes to one path collapse to the last, in the first's slot", () => { + const actions = buildCommitActions( + [ + { path: "a.json", content: "first" }, + { path: "b.json", content: "b" }, + { path: "a.json", deleted: true }, + ], + new Map([ + ["a.json", "c1"], + ["b.json", "c2"], + ]), + ); + expect(actions).toEqual([ + { action: "delete", file_path: "a.json", last_commit_id: "c1" }, + { + action: "update", + file_path: "b.json", + content: "b", + last_commit_id: "c2", + }, + ]); + }); + + test("no changes and no-op deletes both yield an empty action array", () => { + expect(buildCommitActions([], new Map())).toEqual([]); + }); + + test("an executable file asks for the exec bit, on a create and on an update", () => { + expect( + buildCommitActions( + [ + { path: "new.sh", content: "#!/bin/sh\n", mode: "100755" }, + { path: "old.sh", content: "#!/bin/sh\n", mode: "100755" }, + ], + new Map([["old.sh", "c1"]]), + ), + ).toEqual([ + { + action: "create", + file_path: "new.sh", + content: "#!/bin/sh\n", + execute_filemode: true, + }, + { + action: "update", + file_path: "old.sh", + content: "#!/bin/sh\n", + execute_filemode: true, + last_commit_id: "c1", + }, + ]); + }); + + test("a regular file clears the exec bit, and a change without a mode never mentions it", () => { + const [stated, silent] = buildCommitActions( + [ + { path: "plain.json", content: "{}", mode: "100644" }, + { path: "whatever.json", content: "{}" }, + ], + new Map(), + ); + expect(stated?.execute_filemode).toBe(false); + expect(silent).not.toHaveProperty("execute_filemode"); + }); +}); + +describe("isCommitConflict", () => { + test("the per-file guard firing is a conflict", () => { + expect( + isCommitConflict( + 400, + "GitLab API 400: The file has changed since you started editing it: .deco/blocks/page.json", + ), + ).toBe(true); + }); + + test("a create that lost the race is a conflict", () => { + expect(isCommitConflict(400, "A file with this name already exists")).toBe( + true, + ); + }); + + test("an update whose file vanished is a conflict", () => { + expect(isCommitConflict(400, "A file with this name doesn't exist")).toBe( + true, + ); + }); + + test("other 400s are not conflicts", () => { + expect(isCommitConflict(400, "invalid parameters")).toBe(false); + expect( + isCommitConflict( + 400, + "You can only create or edit files when you are on a branch", + ), + ).toBe(false); + }); + + test("other statuses are never conflicts, whatever the message says", () => { + expect( + isCommitConflict( + 403, + "The file has changed since you started editing it", + ), + ).toBe(false); + expect(isCommitConflict(500, "already exists")).toBe(false); + expect( + isCommitConflict(0, "The file has changed since you started editing it"), + ).toBe(false); + }); +}); + +describe("isBranchExistsConflict", () => { + test("400 Branch already exists", () => { + expect(isBranchExistsConflict(400, "Branch already exists")).toBe(true); + }); + test("a 400 for anything else is a plain failure", () => { + expect(isBranchExistsConflict(400, "Invalid reference name: nope")).toBe( + false, + ); + }); + test("a 403 on a protected branch is not a conflict", () => { + expect(isBranchExistsConflict(403, "already exists")).toBe(false); + }); +}); + +describe("mapMergeRequestState", () => { + test("opened is the interface's open", () => { + expect(mapMergeRequestState("opened")).toBe("open"); + }); + test("merged passes through", () => { + expect(mapMergeRequestState("merged")).toBe("merged"); + }); + test("closed and locked are both closed", () => { + expect(mapMergeRequestState("closed")).toBe("closed"); + expect(mapMergeRequestState("locked")).toBe("closed"); + }); + test("an unknown or missing state is closed, never open", () => { + expect(mapMergeRequestState("something_new")).toBe("closed"); + expect(mapMergeRequestState(null)).toBe("closed"); + expect(mapMergeRequestState(undefined)).toBe("closed"); + }); +}); + +describe("mapMergeRequest", () => { + test("iid is the number and web_url is the url", () => { + expect( + mapMergeRequest({ + iid: 7, + web_url: "https://gitlab.com/group/project/-/merge_requests/7", + title: "Publish", + state: "opened", + }), + ).toEqual({ + number: 7, + url: "https://gitlab.com/group/project/-/merge_requests/7", + title: "Publish", + state: "open", + }); + }); + + test("a payload missing the optional strings still maps", () => { + expect(mapMergeRequest({ iid: 1 })).toEqual({ + number: 1, + url: "", + title: "", + state: "closed", + }); + }); +}); + +describe("directoryOf", () => { + test("a nested path", () => { + expect(directoryOf(".deco/blocks/page.json")).toBe(".deco/blocks"); + }); + test("a root file has the empty directory", () => { + expect(directoryOf("README.md")).toBe(""); + }); + test("a leading slash addresses the same file", () => { + expect(directoryOf("/src/app.ts")).toBe("src"); + }); +}); + +describe("groupPathsByDirectory", () => { + test("siblings share one bucket", () => { + expect( + groupPathsByDirectory([ + ".deco/blocks/a.json", + ".deco/blocks/b.json", + "apps/site/.deco/blocks/c.json", + ]), + ).toEqual( + new Map([ + [".deco/blocks", [".deco/blocks/a.json", ".deco/blocks/b.json"]], + ["apps/site/.deco/blocks", ["apps/site/.deco/blocks/c.json"]], + ]), + ); + }); + + test("duplicates are listed once", () => { + expect(groupPathsByDirectory(["a/b.json", "a/b.json"])).toEqual( + new Map([["a", ["a/b.json"]]]), + ); + }); + + test("root files bucket under the empty directory", () => { + expect(groupPathsByDirectory(["deno.json", "a/b.json"])).toEqual( + new Map([ + ["", ["deno.json"]], + ["a", ["a/b.json"]], + ]), + ); + }); + + test("empty and slash-only input yields no buckets", () => { + expect(groupPathsByDirectory([])).toEqual(new Map()); + expect(groupPathsByDirectory(["", "/"])).toEqual(new Map()); + }); +}); + +describe("decoDirFor", () => { + test("a repo-root project", () => { + expect(decoDirFor(null)).toBe(".deco"); + }); + test("a nested project", () => { + expect(decoDirFor("apps/site")).toBe("apps/site/.deco"); + }); +}); + +describe("mapCompareDiff", () => { + const shas = new Map([["a.json", "sha-a"]]); + + test("new_file is added and picks up the head sha", () => { + expect( + mapCompareDiff({ new_path: "a.json", new_file: true }, shas), + ).toEqual({ filename: "a.json", status: "added", sha: "sha-a" }); + }); + + test("deleted_file is removed and has no sha at head", () => { + expect( + mapCompareDiff( + { new_path: "gone.json", old_path: "gone.json", deleted_file: true }, + shas, + ), + ).toEqual({ filename: "gone.json", status: "removed", sha: "" }); + }); + + test("renamed_file carries previousFilename", () => { + expect( + mapCompareDiff( + { new_path: "a.json", old_path: "old.json", renamed_file: true }, + shas, + ), + ).toEqual({ + filename: "a.json", + status: "renamed", + sha: "sha-a", + previousFilename: "old.json", + }); + }); + + test("no flag set is a modification and never sets previousFilename", () => { + const mapped = mapCompareDiff( + { new_path: "a.json", old_path: "a.json" }, + shas, + ); + expect(mapped).toEqual({ + filename: "a.json", + status: "modified", + sha: "sha-a", + }); + expect(mapped).not.toHaveProperty("previousFilename"); + }); + + test("an unresolved path degrades to an empty sha, not undefined", () => { + expect(mapCompareDiff({ new_path: "other.json" }, shas).sha).toBe(""); + }); +}); + +describe("gitlabErrorMessage", () => { + test("the common string form", () => { + expect( + gitlabErrorMessage( + '{"message":"The file has changed since you started editing it: a.json"}', + ), + ).toBe("The file has changed since you started editing it: a.json"); + }); + + test("the merge-request array form, so the classifier still sees the text", () => { + expect( + gitlabErrorMessage( + '{"message":["Another open merge request already exists for this source branch: !3"]}', + ), + ).toBe( + "Another open merge request already exists for this source branch: !3", + ); + }); + + test("the validation-object form is flattened", () => { + expect( + gitlabErrorMessage('{"message":{"base":["Branch already exists"]}}'), + ).toBe("Branch already exists"); + }); + + test("the error form", () => { + expect(gitlabErrorMessage('{"error":"insufficient_scope"}')).toBe( + "insufficient_scope", + ); + }); + + test("a non-JSON body is truncated raw", () => { + expect(gitlabErrorMessage("502 Bad Gateway")).toBe( + "502 Bad Gateway", + ); + expect(gitlabErrorMessage("x".repeat(500))).toHaveLength(300); + }); + + test("JSON without a usable message falls back to the body", () => { + expect(gitlabErrorMessage('{"other":1}')).toBe('{"other":1}'); + expect(gitlabErrorMessage('{"message":[]}')).toBe('{"message":[]}'); + }); +}); + +describe("pooledMap", () => { + test("results keep the input order regardless of completion order", async () => { + const out = await pooledMap([3, 1, 2], 2, async (n) => { + await Promise.resolve(); + return n * 10; + }); + expect(out).toEqual([30, 10, 20]); + }); + + test("never exceeds the concurrency limit", async () => { + let inFlight = 0; + let peak = 0; + await pooledMap( + Array.from({ length: 20 }, (_, i) => i), + 4, + async (n) => { + inFlight++; + peak = Math.max(peak, inFlight); + await Promise.resolve(); + await Promise.resolve(); + inFlight--; + return n; + }, + ); + expect(peak).toBeLessThanOrEqual(4); + }); + + test("an empty input runs nothing", async () => { + let calls = 0; + expect( + await pooledMap([], 4, async () => { + calls++; + return 1; + }), + ).toEqual([]); + expect(calls).toBe(0); + }); +}); + +describe("isProtectedBranch", () => { + /** The one rewrite failure with a slower answer rather than a fatal one: + * a forced push is refused, so the branch has to be replaced instead. */ + test("recognises GitLab's forced-push refusal", () => { + expect( + isProtectedBranch( + "You are not allowed to force push code to a protected branch on this project.", + ), + ).toBe(true); + expect(isProtectedBranch("protected branch")).toBe(true); + }); + + test("does not swallow an unrelated failure", () => { + expect(isProtectedBranch("A branch called 'main' already exists.")).toBe( + false, + ); + expect( + isProtectedBranch("The file has changed since you started editing it: a"), + ).toBe(false); + expect(isProtectedBranch("404 Project Not Found")).toBe(false); + }); +}); diff --git a/apps/api/src/git-providers/content/gitlab.ts b/apps/api/src/git-providers/content/gitlab.ts new file mode 100644 index 0000000000..2cbcbf41ef --- /dev/null +++ b/apps/api/src/git-providers/content/gitlab.ts @@ -0,0 +1,1161 @@ +/** + * `RepoContentClient` over GitLab's REST v4 API. + * + * GitLab exposes no Git Data plumbing (no create-blob / create-tree / + * create-commit / move-ref), which is exactly why the interface is stated as + * intentions: what GitHub does in four calls, GitLab does in one + * `POST /repository/commits` carrying an action per file. The read side maps + * almost one-to-one — GitLab's tree entries already carry full repo-relative + * paths and their object sha (`id`), and a commit sha is accepted wherever a + * ref is. + * + * Where GitLab cannot express the interface's wording, the gap is documented + * on the method: branch-level compare-and-swap (`commitFiles`), moving a + * branch to an arbitrary sha (`forceBranchHead`), rewriting a branch's history + * (`rewriteBranch`), pointing a commit at a blob that already exists + * (`resolveCopies`), merging without a merge request (`mergeBranches`), and + * blob sizes in a tree listing (`listDecofileEntries`). + * + * The request plumbing mirrors `gitlab/client.ts` (bearer auth, 15s timeout, + * 404 → null, every other non-2xx → `GitProviderError` with a rate-limit + * hint). It is re-stated here rather than imported because that module's + * `gitlabRequest` is module-private and GET-only; the write side needs POST, + * PUT and DELETE. Hoisting both into a `gitlab/http.ts` (as `github/http.ts` + * already does) is the obvious follow-up. + */ + +import { apiBaseUrlFor, type RepoRef } from "@decocms/shared/git-providers"; +import { retry, RetryError } from "@decocms/shared/std"; +import { + encodeFilePath, + encodeProjectPath, + gitlabRetryAfterMs, + GitlabProviderClient, +} from "../gitlab/client"; +import { GitProviderError, type TokenSource } from "../types"; +import { + type ChangeRequestInfo, + type FileChange, + type FileMode, + type RepoContentClient, + RepoWriteConflict, + type TreeEntry, +} from "./types"; + +/** Matches `gitlab/client.ts`: one REST call, not a download. */ +const REQUEST_TIMEOUT_MS = 15_000; +const TREE_PAGE_SIZE = 100; +/** 100 pages × 100 entries — a directory past that is a bug, not a repo. */ +const MAX_TREE_PAGES = 100; +/** + * Per-file metadata reads (`last_commit_id`) and per-directory tree listings + * fan out; this bounds the burst so a 300-block commit does not trip GitLab's + * rate limiter. + */ +const FANOUT_CONCURRENCY = 8; + +/** + * A forced rewrite refused because the branch is protected — the one failure + * that has a slower answer (replace the branch) rather than being fatal. + */ +export function isProtectedBranch(message: string): boolean { + return /not allowed to force push|protected branch/i.test(message); +} + +/** GitLab's tree listing row. `id` IS the object sha; `path` is repo-relative. */ +interface GitlabTreeRow { + id: string; + name: string; + type: "blob" | "tree"; + path: string; +} + +interface GitlabMergeRequestRow { + iid: number; + web_url?: string | null; + title?: string | null; + state?: string | null; + sha?: string | null; + merge_commit_sha?: string | null; +} + +/** One entry of `POST /repository/commits`'s `actions` array. */ +export interface GitlabCommitAction { + action: "create" | "update" | "delete"; + file_path: string; + content?: string; + /** + * The exec bit, sent only when the caller stated a mode — omitted, GitLab + * keeps whatever the path already had, which is what every write that does + * not care about the mode wants. + */ + execute_filemode?: boolean; + /** + * The last commit that touched this file, as read just before the write. + * GitLab rejects the whole commit when the file moved since — see + * `commitFiles` for why both this and the branch-head check are needed. + */ + last_commit_id?: string; +} + +/** + * A change with its bytes in hand. GitLab cannot point a commit at a blob + * that already exists, so a `copyFromRef` change is read into `content` + * (carrying the source's mode) before the action array is built. + */ +export type ResolvedChange = + | { path: string; content: string; mode?: FileMode } + | { path: string; deleted: true }; + +/** + * The `actions` array for a set of changes, given what each path looked like + * at the base commit: a `string` is the file's `last_commit_id` (it exists), a + * `null`/absent entry means it does not exist there. + * + * - existing + content → `update`, guarded by `last_commit_id` + * - absent + content → `create`, unguarded (there is nothing to guard against; + * GitLab rejects a `create` on a path that appeared meanwhile, which the + * conflict classifier turns back into a `RepoWriteConflict`) + * - existing + deleted → `delete`, guarded + * - absent + deleted → omitted; deleting what is not there is a no-op, and + * GitLab would fail the entire atomic commit over it + * + * Two changes to one path collapse to the last one, keeping the first + * occurrence's position so the action array is deterministic. + */ +export function buildCommitActions( + changes: readonly ResolvedChange[], + lastCommitIdByPath: ReadonlyMap, +): GitlabCommitAction[] { + const collapsed = new Map(); + for (const change of changes) collapsed.set(change.path, change); + + const actions: GitlabCommitAction[] = []; + for (const [path, change] of collapsed) { + const lastCommitId = lastCommitIdByPath.get(path) ?? null; + if ("deleted" in change) { + if (lastCommitId === null) continue; + actions.push({ + action: "delete", + file_path: path, + last_commit_id: lastCommitId, + }); + continue; + } + const mode = + change.mode === undefined + ? {} + : { execute_filemode: change.mode === "100755" }; + actions.push( + lastCommitId === null + ? { + action: "create", + file_path: path, + content: change.content, + ...mode, + } + : { + action: "update", + file_path: path, + content: change.content, + ...mode, + last_commit_id: lastCommitId, + }, + ); + } + return actions; +} + +/** + * Whether a refused commit means "someone else wrote first" rather than "this + * request was wrong". GitLab answers 400 for all three shapes of a lost race: + * the per-file guard fired, a `create` found the path already taken, or an + * `update`/`delete` found it gone. + */ +export function isCommitConflict(status: number, message: string): boolean { + if (status !== 400 && status !== 409) return false; + return ( + /has changed since you started editing it/i.test(message) || + /file with this name already exists/i.test(message) || + /file with this name doesn'?t exist/i.test(message) + ); +} + +/** `POST /repository/branches` on a name that is taken answers 400. */ +export function isBranchExistsConflict( + status: number, + message: string, +): boolean { + return status === 400 && /already exists/i.test(message); +} + +/** + * GitLab's merge-request states, in the interface's vocabulary. `locked` is a + * merge in progress that GitLab has already taken out of the open set, and an + * unrecognised state is reported as closed rather than as actionable. + */ +export function mapMergeRequestState( + state: string | null | undefined, +): ChangeRequestInfo["state"] { + if (state === "opened") return "open"; + if (state === "merged") return "merged"; + return "closed"; +} + +export function mapMergeRequest(mr: GitlabMergeRequestRow): ChangeRequestInfo { + return { + number: mr.iid, + url: mr.web_url ?? "", + title: mr.title ?? "", + state: mapMergeRequestState(mr.state), + }; +} + +/** Repo-relative directory of `path`; `""` for a file at the repo root. */ +export function directoryOf(path: string): string { + const normalized = path.replace(/^\/+/, ""); + const slash = normalized.lastIndexOf("/"); + return slash === -1 ? "" : normalized.slice(0, slash); +} + +/** + * Paths bucketed by the directory they live in, deduplicated, insertion + * ordered. One bucket is one tree listing, which is what keeps + * `getEntriesAtPaths` scaling with the path set instead of with repo size. + */ +export function groupPathsByDirectory( + paths: readonly string[], +): Map { + const byDir = new Map(); + const seen = new Set(); + for (const raw of paths) { + const path = raw.replace(/^\/+/, ""); + if (path === "" || seen.has(path)) continue; + seen.add(path); + const dir = directoryOf(path); + const bucket = byDir.get(dir); + if (bucket) bucket.push(path); + else byDir.set(dir, [path]); + } + return byDir; +} + +/** `/.deco`, or `.deco` for a single-project repo. */ +export function decoDirFor(packagePath: string | null): string { + return packagePath ? `${packagePath}/.deco` : ".deco"; +} + +interface GitlabCompareDiff { + new_path: string; + old_path?: string | null; + new_file?: boolean; + deleted_file?: boolean; + renamed_file?: boolean; +} + +/** + * One compare diff in the interface's file shape. GitLab reports the change + * as three booleans instead of a status word, and reports no per-file blob + * sha at all — `shaByPath` carries the shas resolved from the tree at `head`, + * and a deleted path (absent there by definition) keeps `sha: ""`. + */ +export function mapCompareDiff( + diff: GitlabCompareDiff, + shaByPath: ReadonlyMap, +): { + filename: string; + status: string; + sha: string; + previousFilename?: string; +} { + const status = diff.new_file + ? "added" + : diff.deleted_file + ? "removed" + : diff.renamed_file + ? "renamed" + : "modified"; + return { + filename: diff.new_path, + status, + sha: shaByPath.get(diff.new_path) ?? "", + ...(diff.renamed_file && diff.old_path + ? { previousFilename: diff.old_path } + : {}), + }; +} + +/** + * `fn` over `items`, at most `concurrency` in flight, results in input order. + * Pure scheduling — the caller supplies the I/O. + */ +export async function pooledMap( + items: readonly T[], + concurrency: number, + fn: (item: T) => Promise, +): Promise { + const out = new Array(items.length); + let next = 0; + const worker = async (): Promise => { + while (true) { + const index = next++; + if (index >= items.length) return; + out[index] = await fn(items[index] as T); + } + }; + const workers = Math.max(1, Math.min(concurrency, items.length)); + await Promise.all(Array.from({ length: workers }, worker)); + return out; +} + +/** + * The human-readable half of a GitLab error body. GitLab is inconsistent: + * `{"message": "..."}` for most refusals, `{"message": ["..."]}` for merge + * requests, `{"message": {"base": ["..."]}}` for model validation, and + * `{"error": "..."}` for OAuth-ish failures. The conflict classifiers match on + * this text, so flattening every shape is load-bearing, not cosmetic. + */ +export function gitlabErrorMessage(bodyText: string): string { + const flatten = (value: unknown): string[] => { + if (typeof value === "string") return value.length > 0 ? [value] : []; + if (Array.isArray(value)) return value.flatMap(flatten); + if (value !== null && typeof value === "object") { + return Object.values(value).flatMap(flatten); + } + return []; + }; + try { + const parsed: unknown = JSON.parse(bodyText); + if (parsed !== null && typeof parsed === "object") { + const record = parsed as Record; + const parts = [ + ...flatten(record.message), + ...flatten(record.error), + ...flatten(record.error_description), + ]; + if (parts.length > 0) return parts.join("; "); + } + } catch { + // Not JSON: an HTML error page or an empty body. Fall through to the text. + } + return bodyText.slice(0, 300); +} + +async function gitlabFailure(res: Response): Promise { + const text = await res.text().catch(() => ""); + const detail = text ? gitlabErrorMessage(text) : res.statusText; + return new GitProviderError({ + provider: "gitlab", + status: res.status, + message: `GitLab API ${res.status}: ${detail}`, + retryAfterMs: res.status === 429 ? gitlabRetryAfterMs(res.headers) : null, + }); +} + +interface CallInit { + method?: "GET" | "POST" | "PUT" | "DELETE"; + body?: unknown; + accept?: string; +} + +export class GitlabContentClient implements RepoContentClient { + readonly repo: RepoRef; + private readonly apiBase: string; + private readonly projectBase: string; + /** Owns token minting and the archive download; both are already solved there. */ + private readonly provider: GitlabProviderClient; + private defaultBranch: string | null = null; + + constructor(params: { repo: RepoRef; tokenSource: TokenSource }) { + this.repo = params.repo; + this.apiBase = apiBaseUrlFor("gitlab", params.repo.host); + this.projectBase = `/projects/${encodeProjectPath(params.repo.path)}`; + this.provider = new GitlabProviderClient({ + host: params.repo.host, + tokenSource: params.tokenSource, + }); + } + + async getDefaultBranch(): Promise { + if (this.defaultBranch) return this.defaultBranch; + const project = await this.json<{ default_branch?: string | null }>( + this.projectBase, + ); + if (project === null) { + throw new GitProviderError({ + provider: "gitlab", + status: 404, + message: `GitLab project ${this.repo.path} not found`, + }); + } + if (!project.default_branch) { + throw new GitProviderError({ + provider: "gitlab", + status: 409, + message: `GitLab project ${this.repo.path} has no default branch (empty repository)`, + }); + } + this.defaultBranch = project.default_branch; + return this.defaultBranch; + } + + async getBranch( + branch: string, + ): Promise<{ sha: string; committedAt: string } | null> { + const json = await this.json<{ + commit?: { id?: string; committed_date?: string }; + }>(`${this.projectBase}/repository/branches/${encodeRef(branch)}`); + const commit = json?.commit; + if (!commit?.id) return null; + if (!commit.committed_date) { + throw new GitProviderError({ + provider: "gitlab", + status: 502, + message: `GitLab branch ${branch} came back without a commit date`, + }); + } + return { sha: commit.id, committedAt: commit.committed_date }; + } + + getArchive(ref: string): Promise | null> { + return this.provider.archiveTarball(this.repo, ref); + } + + /** + * GitLab addresses subtrees by path, so this is at most two listings: the + * `.deco/` directory (which reveals whether the merged artifact is + * committed) and `.deco/blocks/`. Direct blob children are returned + * unfiltered, exactly as the GitHub implementation does — `blockEntriesInTree` + * downstream applies the `*.json` rule, and diverging here would make the two + * providers disagree about what a decofile tree contains. + */ + async listDecofileEntries( + treeish: string, + packagePath: string | null, + ): Promise { + const decoDir = decoDirFor(packagePath); + const children = await this.listTree(treeish, decoDir); + const out: TreeEntry[] = []; + const gen = children.find( + (entry) => + entry.type === "blob" && entry.path === `${decoDir}/blocks.gen.json`, + ); + if (gen) out.push(gen); + const hasBlocksDir = children.some( + (entry) => entry.type === "tree" && entry.path === `${decoDir}/blocks`, + ); + if (!hasBlocksDir) return out; + for (const child of await this.listTree(treeish, `${decoDir}/blocks`)) { + if (child.type === "blob") out.push(child); + } + return out; + } + + async getEntriesAtPaths( + treeish: string, + paths: string[], + ): Promise> { + const byDir = groupPathsByDirectory(paths); + const listings = await pooledMap( + [...byDir.keys()], + FANOUT_CONCURRENCY, + (dir) => this.listTree(treeish, dir), + ); + const result = new Map(); + let index = 0; + for (const wanted of byDir.values()) { + const byPath = new Map( + (listings[index++] ?? []).map((entry) => [entry.path, entry]), + ); + for (const path of wanted) { + const entry = byPath.get(path); + if (entry?.type === "blob") result.set(path, entry); + } + } + return result; + } + + async readBlob(sha: string): Promise { + const res = await this.call( + `${this.projectBase}/repository/blobs/${encodeURIComponent(sha)}/raw`, + { accept: "text/plain, */*" }, + ); + if (res === null) { + throw new GitProviderError({ + provider: "gitlab", + status: 404, + message: `GitLab blob ${sha} not found in ${this.repo.path}`, + }); + } + return res.text(); + } + + async readFileAtRef(ref: string, path: string): Promise { + const res = await this.call( + `${this.projectBase}/repository/files/${encodeFilePath(path)}/raw?ref=${encodeURIComponent(ref)}`, + { accept: "text/plain, */*" }, + ); + return res === null ? null : res.text(); + } + + /** + * One atomic `POST /repository/commits`, guarded twice. `branch` must + * already exist — creating one as a side effect of a write (GitLab's + * `start_branch`) is `createBranch`'s job, not this one's. + * + * `expectedHead` is a branch-level compare-and-swap and GitLab has none: its + * only optimistic lock is per file, `actions[].last_commit_id`. So both + * layers run. The head read catches the common case for the price of one + * request, before any content is uploaded, and covers changes to files this + * commit does not touch (a rebase, someone else's block). The per-file guards + * close the window the head read leaves open — between reading the head and + * GitLab applying the commit — for exactly the files being written, and are + * enforced inside GitLab's own transaction, which no check of ours can be. + * Neither is sufficient alone; a 400 naming the changed file is translated + * back into `RepoWriteConflict` so the coalescer rebuilds and retries rather + * than clobbering the other writer. + * + * `rewriteFrom` cannot be a ref move here at all — see {@link rewriteBranch}. + * Either way a commit costs one metadata read per path, plus one content read + * per `copyFromRef` path: GitLab has no tree write, so every byte of every + * changed file travels, however little of it changed. + */ + async commitFiles(params: { + branch: string; + message: string; + expectedHead: string | null; + rewriteFrom?: string; + changes: FileChange[]; + }): Promise<{ sha: string }> { + const { branch, message, expectedHead, rewriteFrom } = params; + const head = await this.getBranch(branch); + if (expectedHead !== null && head?.sha !== expectedHead) { + throw new RepoWriteConflict( + `GitLab branch ${branch} is at ${head?.sha ?? "(absent)"}, expected ${expectedHead}`, + ); + } + const changes = await this.resolveCopies(params.changes); + if (rewriteFrom !== undefined) { + return this.rewriteBranch({ + branch, + message, + expectedHead, + rewriteFrom, + changes, + }); + } + /** Pin the metadata reads to the sha the guard just approved, not to the + * branch name, so a push landing mid-fanout cannot feed us its ids. */ + const base = head?.sha ?? branch; + const sha = await this.commitOnto({ branch, base, message, changes }); + if (sha === null) { + if (head === null) { + throw new GitProviderError({ + provider: "gitlab", + status: 404, + message: `GitLab branch ${branch} does not exist in ${this.repo.path}`, + }); + } + return { sha: head.sha }; + } + return { sha }; + } + + /** + * A history rewrite. + * + * `POST /repository/commits` does it in ONE atomic call given `start_sha` + + * `force: true` — measured against gitlab.com: 201, one commit parented on + * `start_sha`, the branch's former content gone. That is the path taken, and + * it has no window at all. + * + * A PROTECTED branch refuses it ("You are not allowed to force push code to + * a protected branch"), and `main` is protected by default. Only then does + * this fall back to replacing the branch: build the commit on a throwaway + * first so the content exists before the target is touched, then delete and + * recreate the target at it. That fallback is what costs a window — the + * branch briefly does not exist, and a crash there leaves it missing — and + * it fails too if the protection also blocks deletion. + * + * Without `force`, committing onto an existing branch from an earlier + * `start_sha` is refused outright: 400 "A branch called 'x' already exists." + */ + private async rewriteBranch(params: { + branch: string; + message: string; + expectedHead: string | null; + rewriteFrom: string; + changes: readonly ResolvedChange[]; + }): Promise<{ sha: string }> { + const { branch, message, expectedHead, rewriteFrom, changes } = params; + + await this.assertHeadIs(branch, expectedHead); + const forced = await this.commitOnto({ + branch, + base: rewriteFrom, + message, + changes, + startSha: rewriteFrom, + force: true, + }).catch((cause: unknown) => { + if ( + cause instanceof GitProviderError && + isProtectedBranch(cause.message) + ) { + return undefined; + } + throw cause; + }); + if (forced !== undefined) return { sha: forced ?? rewriteFrom }; + + const staging = `studio-rewrite-${crypto.randomUUID().replaceAll("-", "").slice(0, 12)}`; + await this.createBranch(staging, rewriteFrom); + let sha: string; + try { + sha = + (await this.commitOnto({ + branch: staging, + base: rewriteFrom, + message, + changes, + })) ?? rewriteFrom; + /** Re-read as late as possible: the guard has to hold when the target is + * replaced, not when this call started. */ + const current = await this.getBranch(branch); + if (expectedHead !== null && current?.sha !== expectedHead) { + throw new RepoWriteConflict( + `GitLab branch ${branch} moved to ${current?.sha ?? "(absent)"} while rewriting it`, + ); + } + } catch (cause) { + await this.deleteBranch(staging).catch(() => {}); + throw cause; + } + await this.deleteBranch(branch); + await this.createBranchAt(branch, sha); + await this.deleteBranch(staging).catch(() => {}); + return { sha }; + } + + /** + * The branch-level half of the guard, read as late as the caller allows. + * Cheap, and it covers the files a commit does not touch — the per-file + * `last_commit_id` closes the rest. + */ + private async assertHeadIs( + branch: string, + expectedHead: string | null, + ): Promise { + if (expectedHead === null) return; + const current = await this.getBranch(branch); + if (current?.sha !== expectedHead) { + throw new RepoWriteConflict( + `GitLab branch ${branch} is at ${current?.sha ?? "(absent)"}, not ${expectedHead}`, + ); + } + } + + /** + * One atomic `POST /repository/commits` on `branch`, with the per-file + * guards read at `base`. Null when the changes reduce to nothing (deletes + * for paths that are already gone) — there is no commit to make, and GitLab + * refuses an empty action array. + */ + private async commitOnto(params: { + branch: string; + base: string; + message: string; + changes: readonly ResolvedChange[]; + /** Rewrites `branch` onto this sha instead of appending to its head. */ + startSha?: string; + force?: boolean; + }): Promise { + const { branch, base, message, changes, startSha, force } = params; + const paths = [...new Set(changes.map((change) => change.path))]; + const lastCommitIds = new Map( + await pooledMap( + paths, + FANOUT_CONCURRENCY, + async (path) => + [path, await this.lastCommitIdFor(base, path)] as [ + string, + string | null, + ], + ), + ); + const actions = buildCommitActions(changes, lastCommitIds); + if (actions.length === 0) return null; + + let json: { id?: string } | null; + try { + json = await this.json<{ id?: string }>( + `${this.projectBase}/repository/commits`, + { + method: "POST", + body: { + branch, + commit_message: message, + actions, + ...(startSha ? { start_sha: startSha } : {}), + ...(force ? { force: true } : {}), + }, + }, + ); + } catch (cause) { + if ( + cause instanceof GitProviderError && + isCommitConflict(cause.status, cause.message) + ) { + throw new RepoWriteConflict( + `GitLab refused the commit on ${branch}: ${cause.message}`, + { cause }, + ); + } + throw cause; + } + if (!json?.id) { + throw new GitProviderError({ + provider: "gitlab", + status: 502, + message: `GitLab accepted the commit on ${branch} without returning a sha`, + }); + } + return json.id; + } + + /** + * `copyFromRef` changes, read into content. One call per path serves both + * halves of the copy: GitLab's file endpoint carries the base64 body AND the + * exec bit, so the source's mode rides along instead of costing a tree read. + */ + private resolveCopies( + changes: readonly FileChange[], + ): Promise { + return pooledMap( + changes, + FANOUT_CONCURRENCY, + async (change): Promise => { + if (!("copyFromRef" in change)) return change; + const source = await this.fileAtRef(change.copyFromRef, change.path); + if (source === null) { + throw new GitProviderError({ + provider: "gitlab", + status: 404, + message: `${change.path} does not exist at ${change.copyFromRef} in ${this.repo.path}`, + }); + } + return { + path: change.path, + content: source.content, + mode: change.mode ?? source.mode, + }; + }, + ); + } + + async createBranch(branch: string, sha: string): Promise { + try { + await this.createBranchAt(branch, sha); + } catch (cause) { + if ( + cause instanceof GitProviderError && + isBranchExistsConflict(cause.status, cause.message) + ) { + throw new RepoWriteConflict( + `GitLab branch ${branch} already exists in ${this.repo.path}`, + { cause }, + ); + } + throw cause; + } + } + + /** + * GitLab has no ref-update endpoint at all: a branch can only be created or + * deleted, never repointed. So this deletes and recreates, which is NOT + * atomic — between the two calls the branch does not exist, and a concurrent + * reader sees a 404 rather than the old sha. Two consequences worth knowing: + * a protected branch refuses the delete (403, with GitLab's own message + * surfaced), and a failure after the delete leaves the branch missing rather + * than stale. The interface already declares this method unguarded, so the + * caller re-reads the head before calling either way. + */ + async forceBranchHead(branch: string, sha: string): Promise { + await this.deleteBranch(branch); + await this.createBranchAt(branch, sha); + } + + /** + * GitLab has no merge endpoint on the repository — merging means opening a + * merge request and merging it, so this leaves an MR behind where GitHub + * leaves nothing. An open MR for the same source/target is reused instead of + * creating a duplicate (GitLab rejects the second one anyway). + */ + async mergeBranches( + base: string, + head: string, + message: string, + ): Promise { + const ahead = await this.compareRaw(base, head); + if (ahead.commits.length === 0) return null; + const title = message.split("\n")[0] || message; + const mr = + (await this.findOpenChangeRequest(base, head)) ?? + (await this.createChangeRequest({ base, head, title })); + return this.mergeChangeRequest(mr.number, base, message); + } + + async createChangeRequest(params: { + base: string; + head: string; + title: string; + }): Promise { + const json = await this.json( + `${this.projectBase}/merge_requests`, + { + method: "POST", + body: { + source_branch: params.head, + target_branch: params.base, + title: params.title, + }, + }, + ); + if (!json) { + throw new GitProviderError({ + provider: "gitlab", + status: 404, + message: `GitLab project ${this.repo.path} not found`, + }); + } + return mapMergeRequest(json); + } + + async findOpenChangeRequest( + base: string, + head: string, + ): Promise { + const query = new URLSearchParams({ + state: "opened", + source_branch: head, + target_branch: base, + per_page: "1", + }); + const json = await this.json( + `${this.projectBase}/merge_requests?${query}`, + ); + const first = Array.isArray(json) ? json[0] : undefined; + return first ? mapMergeRequest(first) : null; + } + + /** + * GitLab's compare is one-directional, so "how far apart" costs two calls — + * `base..head` for ahead, `head..base` for behind. + */ + async compare( + base: string, + head: string, + ): Promise<{ aheadBy: number; behindBy: number }> { + const [ahead, behind] = await Promise.all([ + this.compareRaw(base, head), + this.compareRaw(head, base), + ]); + return { + aheadBy: ahead.commits.length, + behindBy: behind.commits.length, + }; + } + + async compareDetailed( + base: string, + head: string, + ): Promise<{ + aheadBy: number; + behindBy: number; + mergeBaseSha: string; + files: Array<{ + filename: string; + status: string; + sha: string; + previousFilename?: string; + }>; + commitMessages: string[]; + }> { + const [ahead, behind, mergeBaseSha] = await Promise.all([ + this.compareRaw(base, head), + this.compareRaw(head, base), + this.mergeBaseSha(base, head), + ]); + /** GitLab's diffs carry no blob sha, so resolve the surviving paths from + * the tree at `head` — one listing per distinct directory. */ + const survivors = ahead.diffs + .filter((diff) => !diff.deleted_file) + .map((diff) => diff.new_path); + const entries = await this.getEntriesAtPaths(head, survivors); + const shaByPath = new Map( + [...entries].map(([path, entry]) => [path, entry.sha]), + ); + return { + aheadBy: ahead.commits.length, + behindBy: behind.commits.length, + mergeBaseSha, + files: ahead.diffs.map((diff) => mapCompareDiff(diff, shaByPath)), + commitMessages: ahead.commits + .map((commit) => commit.title ?? "") + .filter((title) => title.length > 0), + }; + } + + private async createBranchAt(branch: string, sha: string): Promise { + const query = new URLSearchParams({ branch, ref: sha }); + const res = await this.call( + `${this.projectBase}/repository/branches?${query}`, + { method: "POST" }, + ); + if (res === null) { + throw new GitProviderError({ + provider: "gitlab", + status: 404, + message: `GitLab could not branch ${branch} from ${sha}: project or ref not found`, + }); + } + } + + /** + * GitLab computes mergeability asynchronously and answers 405 while it is + * still deciding, so a merge issued right after the MR was created loses a + * race it would win a second later. Bounded retry on 405 only; a genuine + * conflict exhausts the attempts and surfaces GitLab's own message. + */ + private async mergeChangeRequest( + iid: number, + base: string, + message: string, + ): Promise { + const attempt = async (): Promise => { + const json = await this.json( + `${this.projectBase}/merge_requests/${iid}/merge`, + { method: "PUT", body: { merge_commit_message: message } }, + ); + if (!json) { + throw new GitProviderError({ + provider: "gitlab", + status: 404, + message: `GitLab merge request !${iid} not found in ${this.repo.path}`, + }); + } + if (json.state !== "merged") { + throw new GitProviderError({ + provider: "gitlab", + status: 409, + message: `GitLab left merge request !${iid} in state ${json.state ?? "unknown"}`, + }); + } + /** A fast-forward or squash merge produces no merge commit; the target + * branch's new head is then the result the caller wants. */ + if (json.merge_commit_sha) return json.merge_commit_sha; + const merged = await this.getBranch(base); + const sha = merged?.sha ?? json.sha; + if (!sha) { + throw new GitProviderError({ + provider: "gitlab", + status: 502, + message: `GitLab merged !${iid} without reporting a commit sha`, + }); + } + return sha; + }; + try { + return await retry(attempt, { + maxAttempts: 4, + minTimeout: 500, + maxTimeout: 4_000, + jitter: 0.5, + isRetriable: (err) => + err instanceof GitProviderError && err.status === 405, + }); + } catch (err) { + if (err instanceof RetryError && err.cause instanceof GitProviderError) { + throw err.cause; + } + throw err; + } + } + + private async deleteBranch(branch: string): Promise { + await this.call( + `${this.projectBase}/repository/branches/${encodeRef(branch)}`, + { method: "DELETE" }, + ); + } + + private async lastCommitIdFor( + ref: string, + path: string, + ): Promise { + const json = await this.json<{ last_commit_id?: string }>( + `${this.projectBase}/repository/files/${encodeFilePath(path)}?ref=${encodeURIComponent(ref)}`, + ); + return json?.last_commit_id ?? null; + } + + /** + * One file's content and mode at a ref, or null when it is absent there. + * GitLab reports the exec bit only on this metadata endpoint (never in a + * tree listing or on `/raw`), and it is absent on GitLab versions that + * predate the field, which reads as a regular file. + */ + private async fileAtRef( + ref: string, + path: string, + ): Promise<{ content: string; mode: FileMode } | null> { + const json = await this.json<{ + content?: string; + encoding?: string; + execute_filemode?: boolean; + }>( + `${this.projectBase}/repository/files/${encodeFilePath(path)}?ref=${encodeURIComponent(ref)}`, + ); + if (!json || json.content === undefined) return null; + if (json.encoding !== "base64") { + throw new GitProviderError({ + provider: "gitlab", + status: 502, + message: `GitLab served ${path} at ${ref} with unexpected encoding ${json.encoding}`, + }); + } + return { + content: Buffer.from(json.content, "base64").toString("utf-8"), + mode: json.execute_filemode === true ? "100755" : "100644", + }; + } + + /** + * Direct children of `path` at `treeish`, following GitLab's pagination. + * A missing directory (or a missing ref) answers 404, which is the empty + * listing — callers treat "no `.deco/` yet" as normal. + */ + private async listTree(treeish: string, path: string): Promise { + const out: TreeEntry[] = []; + let page = 1; + for (let visited = 0; visited < MAX_TREE_PAGES; visited++) { + const query = new URLSearchParams({ + ref: treeish, + per_page: String(TREE_PAGE_SIZE), + page: String(page), + }); + if (path) query.set("path", path); + const res = await this.call( + `${this.projectBase}/repository/tree?${query}`, + ); + if (res === null) return out; + const rows: unknown = await res.json(); + if (!Array.isArray(rows)) { + throw new GitProviderError({ + provider: "gitlab", + status: 502, + message: "GitLab /repository/tree returned a non-array payload", + }); + } + for (const row of rows as GitlabTreeRow[]) { + out.push({ + path: row.path, + sha: row.id, + type: row.type === "tree" ? "tree" : "blob", + }); + } + const next = Number(res.headers.get("x-next-page") ?? ""); + if ( + rows.length < TREE_PAGE_SIZE || + !Number.isFinite(next) || + next <= page + ) + break; + page = next; + } + return out; + } + + private async compareRaw( + from: string, + to: string, + ): Promise<{ + commits: Array<{ title?: string }>; + diffs: GitlabCompareDiff[]; + }> { + const query = new URLSearchParams({ from, to, straight: "false" }); + const json = await this.json<{ + commits?: Array<{ title?: string }>; + diffs?: GitlabCompareDiff[]; + }>(`${this.projectBase}/repository/compare?${query}`); + if (!json) { + throw new GitProviderError({ + provider: "gitlab", + status: 404, + message: `GitLab cannot compare ${from}...${to} in ${this.repo.path}`, + }); + } + return { commits: json.commits ?? [], diffs: json.diffs ?? [] }; + } + + /** Empty string when the refs share no history — GitLab answers 404 there. */ + private async mergeBaseSha(base: string, head: string): Promise { + const query = new URLSearchParams(); + query.append("refs[]", base); + query.append("refs[]", head); + const json = await this.json<{ id?: string }>( + `${this.projectBase}/repository/merge_base?${query}`, + ); + return json?.id ?? ""; + } + + private async json( + pathAndQuery: string, + init?: CallInit, + ): Promise { + const res = await this.call(pathAndQuery, init); + if (res === null) return null; + return (await res.json()) as T; + } + + /** + * One authenticated REST call. 404 resolves to null so "absent" needs no + * try/catch; every other non-2xx becomes a `GitProviderError`, carrying a + * wait hint when GitLab rate-limited us. + */ + private async call( + pathAndQuery: string, + init: CallInit = {}, + ): Promise { + const { token } = await this.provider.tokenForRepo(this.repo); + const headers: Record = { + Authorization: `Bearer ${token}`, + Accept: init.accept ?? "application/json", + }; + if (init.body !== undefined) headers["Content-Type"] = "application/json"; + let res: Response; + try { + res = await fetch(`${this.apiBase}${pathAndQuery}`, { + method: init.method ?? "GET", + headers, + body: init.body === undefined ? undefined : JSON.stringify(init.body), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } catch (cause) { + throw new GitProviderError({ + provider: "gitlab", + status: 0, + message: `GitLab request failed: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, + }); + } + if (res.status === 404) return null; + if (!res.ok) throw await gitlabFailure(res); + return res; + } +} + +/** Branch and tag names are one path segment for GitLab: slashes encode too. */ +function encodeRef(ref: string): string { + return encodeURIComponent(ref); +} diff --git a/apps/api/src/git-providers/content/index.ts b/apps/api/src/git-providers/content/index.ts new file mode 100644 index 0000000000..0b9357d866 --- /dev/null +++ b/apps/api/src/git-providers/content/index.ts @@ -0,0 +1,179 @@ +/** + * From a repository (or a legacy `metadata.githubRepo` binding) to a + * `RepoContentClient`. + * + * Two credential paths, the same two the clone path has: + * - a first-class `repositories` row whose git provider account Studio can + * serve — the provider client mints the repo token (`clientForAccount`), and + * the row's `provider` picks the implementation; + * - the legacy `mcp-github` connection recorded on the binding, for orgs that + * have not been migrated to a repository row yet. + * + * A client instance is meant to live for ONE request: its default-branch memo + * and commit→tree memo assume no ref moves under it. + */ + +import type { GithubRepo } from "@decocms/shared/sdk/types"; +import { + repoRefFromOwnerName, + type RepoRef, +} from "@decocms/shared/git-providers"; +import type { StudioContext } from "@/core/studio-context"; +import { githubConnectionAccessToken } from "@/oauth/github-mint"; +import { RECONNECT_ERROR } from "@/oauth/token-refresh"; +import { GitProviderAccountStorage } from "@/storage/git-provider-accounts"; +import { type RepositoryRecord, repoRefOf } from "@/storage/repositories"; +import { + clientForAccount, + findRepositoryForLegacyBinding, + type GitProviderDeps, + repositoryUsesStudioCredentials, +} from "../credentials"; +import { + type GitProviderClient, + GitProviderError, + type GitTokenKind, + type TokenSource, +} from "../types"; +import { GithubContentClient } from "./github"; +import { GitlabContentClient } from "./gitlab"; +import type { RepoContentClient } from "./types"; + +/** The token kind an account's stored credential produces, before minting it. */ +function tokenKindOf(account: { authKind: string }): GitTokenKind { + if (account.authKind === "github_app") return "installation"; + return account.authKind === "token" ? "token" : "oauth"; +} + +/** A token source over one repository's push/read credential. */ +function repoTokenSource( + client: GitProviderClient, + repo: RepoRef, + kind: GitTokenKind, +): TokenSource { + return { kind, get: (opts) => client.tokenForRepo(repo, opts) }; +} + +/** A token source over an already-minted token — the legacy connection path. */ +function staticTokenSource(token: string, kind: GitTokenKind): TokenSource { + return { kind, get: () => Promise.resolve({ token, kind, expiresAt: null }) }; +} + +/** + * Content client for a repository whose token the caller already holds — the + * legacy connection paths, where the credential was minted before Studio knew + * which repository it was for. + */ +export function contentClientWithToken( + repo: RepoRef, + token: string, + kind: GitTokenKind = "installation", +): RepoContentClient { + return contentClientFor(repo, staticTokenSource(token, kind)); +} + +function contentClientFor( + repo: RepoRef, + tokenSource: TokenSource, +): RepoContentClient { + switch (repo.provider) { + case "github": + return new GithubContentClient({ repo, tokenSource }); + case "gitlab": + return new GitlabContentClient({ repo, tokenSource }); + } +} + +/** + * Content client for a first-class repository row, credentialed through its + * git provider account. Throws `GitProviderError` when the row is anonymous or + * its account cannot produce a token — reading a repository's contents is + * always authenticated here, even for a public one. + */ +async function contentClientForRepository( + deps: GitProviderDeps, + repository: RepositoryRecord, +): Promise { + const ref = repoRefOf(repository); + if (!repository.accountId) { + throw new GitProviderError({ + provider: repository.provider, + status: 401, + message: `${repository.path} is linked without an account; link it again to read and write its contents`, + }); + } + const account = await new GitProviderAccountStorage(deps.db).getUnscoped( + repository.accountId, + ); + if (!account || account.organizationId !== repository.organizationId) { + throw new GitProviderError({ + provider: repository.provider, + status: 404, + message: `The account backing ${repository.path} no longer exists. Link the repository again.`, + }); + } + const client = clientForAccount(deps, account); + return contentClientFor( + ref, + repoTokenSource(client, ref, tokenKindOf(account)), + ); +} + +/** + * Content client for a legacy `metadata.githubRepo` binding: the recorded + * `mcp-github` connection (org-scoped) plus the repo-scoped installation token + * it mints. Kept for orgs whose repos have no `repositories` row yet. + */ +async function contentClientForLegacyConnection( + ctx: StudioContext, + organizationId: string, + githubRepo: GithubRepo, +): Promise { + const missing = new GitProviderError({ + provider: "github", + status: 401, + message: "Project's GitHub connection is missing — reconnect GitHub", + }); + if (!githubRepo.connectionId) throw missing; + const connection = await ctx.storage.connections.findById( + githubRepo.connectionId, + organizationId, + ); + if (!connection) throw missing; + const accessToken = await githubConnectionAccessToken(ctx, connection); + if (!accessToken) { + throw new GitProviderError({ + provider: "github", + status: 401, + message: RECONNECT_ERROR, + }); + } + return contentClientWithToken( + repoRefFromOwnerName(githubRepo.owner, githubRepo.name), + accessToken, + ); +} + +/** + * Content client for a project's linked repo, taking whichever credential path + * the org is on. Shared by the decofile routes and the sandbox-less `/git/*` + * compat handlers so credential resolution cannot drift between them. + */ +export async function contentClientForProjectRepo( + ctx: StudioContext, + organizationId: string, + githubRepo: GithubRepo, +): Promise { + const repository = await findRepositoryForLegacyBinding( + ctx.storage, + organizationId, + githubRepo, + ); + if ( + repository && + (await repositoryUsesStudioCredentials(ctx.storage, repository)) + ) { + return contentClientForRepository(ctx, repository); + } + return contentClientForLegacyConnection(ctx, organizationId, githubRepo); +} diff --git a/apps/api/src/git-providers/content/types.test.ts b/apps/api/src/git-providers/content/types.test.ts new file mode 100644 index 0000000000..f19bab68a4 --- /dev/null +++ b/apps/api/src/git-providers/content/types.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "bun:test"; +import { GitProviderError } from "../types"; +import { + repoErrorStatus, + repoRateLimitRetryAfterMs, + RepoWriteConflict, +} from "./types"; + +describe("repoErrorStatus", () => { + it("reads the status off either provider's error", () => { + expect( + repoErrorStatus( + new GitProviderError({ + provider: "gitlab", + status: 409, + message: "conflict", + }), + ), + ).toBe(409); + expect(repoErrorStatus({ status: 404 })).toBe(404); + }); + + it("is null for anything that does not carry one", () => { + expect(repoErrorStatus(new Error("boom"))).toBeNull(); + expect(repoErrorStatus(new RepoWriteConflict("moved"))).toBeNull(); + expect(repoErrorStatus({ status: "409" })).toBeNull(); + expect(repoErrorStatus(null)).toBeNull(); + expect(repoErrorStatus(undefined)).toBeNull(); + }); +}); + +describe("repoRateLimitRetryAfterMs", () => { + it("distinguishes a rate refusal without a hint from a non-rate failure", () => { + expect( + repoRateLimitRetryAfterMs({ isRateLimited: true, retryAfterMs: 30_000 }), + ).toBe(30_000); + // A primary limit reports no wait; still a rate refusal. + expect( + repoRateLimitRetryAfterMs({ isRateLimited: true, retryAfterMs: null }), + ).toBeNull(); + expect(repoRateLimitRetryAfterMs({ status: 403 })).toBeUndefined(); + expect(repoRateLimitRetryAfterMs(new Error("boom"))).toBeUndefined(); + }); + + it("recognises a rate-limited GitProviderError", () => { + const err = new GitProviderError({ + provider: "github", + status: 429, + message: "slow down", + retryAfterMs: 1_000, + }); + expect(err.isRateLimited).toBe(true); + expect(repoRateLimitRetryAfterMs(err)).toBe(1_000); + }); +}); diff --git a/apps/api/src/git-providers/content/types.ts b/apps/api/src/git-providers/content/types.ts new file mode 100644 index 0000000000..19e922e6ac --- /dev/null +++ b/apps/api/src/git-providers/content/types.ts @@ -0,0 +1,231 @@ +/** + * Reading and writing a repository's contents, as an intention rather than as + * one provider's object model. + * + * The predecessor (`decofile/github-git-data.ts`) was shaped like GitHub's Git + * Data API — create a blob, then a tree, then a commit, then move a ref. GitLab + * exposes no such plumbing at all, so an interface in that shape is not + * implementable there. The write side is therefore stated as what the caller + * actually wants: put these files on this branch, atomically, unless the branch + * moved. GitHub does it in four calls, GitLab in one; neither leaks here. + * + * The read side needed no such lift: both providers address tree entries and + * blobs by object sha, and both accept a commit sha wherever a tree-ish is + * asked for. + */ + +import type { RepoRef } from "@decocms/shared/git-providers"; +import { GitProviderError } from "../types"; + +export interface TreeEntry { + path: string; + /** Object sha, as the provider's tree listing reports it. */ + sha: string; + type: "blob" | "tree"; + size?: number; +} + +/** + * The two file modes this interface writes. A path's executable bit is real + * content — a discard or a replay that rewrote `run.sh` as `100644` would be + * losing part of the file, not part of its metadata. + */ +export type FileMode = "100644" | "100755"; + +export type FileChange = + /** `mode` defaults to `100644`, as a new regular file. */ + | { path: string; content: string; mode?: FileMode } + /** + * "This path should end up with the content it has at `copyFromRef`" — the + * replay of a blob that is already in the repository. GitHub resolves it to + * the sha in that ref's tree and points the new tree at it, so nothing is + * read or uploaded and the source entry's mode (an executable, a symlink) + * survives verbatim; GitLab, which cannot reference an existing blob from a + * commit, reads the content and writes it back as one of the two + * {@link FileMode}s. `mode` overrides the source's; the path must exist at + * that ref (use the `deleted` variant when it should not). + */ + | { path: string; copyFromRef: string; mode?: FileMode } + | { path: string; deleted: true }; + +export interface ChangeRequestInfo { + number: number; + url: string; + title: string; + state: "open" | "closed" | "merged"; +} + +/** + * A write lost its race: the branch (or the file it guarded) moved between the + * read and the write. The coalescer's multi-replica safety is built on this + * being distinguishable from every other failure — it rebuilds on the fresh + * head and retries, rather than clobbering a concurrent writer. + */ +export class RepoWriteConflict extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "RepoWriteConflict"; + } +} + +/** + * The provider's HTTP status for a failed call, when the error carries one — + * `GitHubApiError` and `GitProviderError` both expose `status`, and the two + * codes consumers actually branch on (404 for "not there yet", 409 for "the + * merge does not apply") mean the same thing on both providers. + */ +export function repoErrorStatus(err: unknown): number | null { + const status = (err as { status?: unknown } | null)?.status; + return typeof status === "number" ? status : null; +} + +/** + * How long the provider asked us to wait, for an error that is a rate refusal: + * `null` retry-after still means "rate limited, no hint given", and a non-rate + * failure returns undefined so the caller can tell the two apart. + */ +export function repoRateLimitRetryAfterMs( + err: unknown, +): number | null | undefined { + const e = err as { isRateLimited?: unknown; retryAfterMs?: unknown } | null; + if (e?.isRateLimited !== true) return undefined; + return typeof e.retryAfterMs === "number" ? e.retryAfterMs : null; +} + +export interface RepoContentClient { + readonly repo: RepoRef; + + getDefaultBranch(): Promise; + /** + * Branch head sha + the head commit's date, or null when the branch does not + * exist. Git stores no ref-creation time, so that date is the branch's + * last-activity signal and what the CMS staleness check reads. + */ + getBranch( + branch: string, + ): Promise<{ sha: string; committedAt: string } | null>; + /** + * Gzipped tar at `ref`, streamed — one request for every file, versus one + * blob request per block. The body is never buffered here; the caller pipes + * it into `tar`. Null when the provider cannot serve it. + */ + getArchive(ref: string): Promise | null>; + + /** + * Block sources under `/.deco/`, without a whole-repo recursive + * read — that trips GitHub's recursive cap on a large storefront and 502s. + * Walks only that subtree. Empty when the project has no `.deco/` yet. + */ + listDecofileEntries( + treeish: string, + packagePath: string | null, + ): Promise; + /** + * Entries for a caller-known set of paths, walking only the directories they + * live in — cost scales with the path set, never with repo size. A path + * absent at `treeish` is omitted. + */ + getEntriesAtPaths( + treeish: string, + paths: string[], + ): Promise>; + /** Blob content by object sha. */ + readBlob(sha: string): Promise; + /** One file's text at a ref, by path — null when absent there. */ + readFileAtRef(ref: string, path: string): Promise; + + /** + * Put `changes` on `branch` in one commit. + * + * `expectedHead` is the guard, and the reason this is one call rather than + * four: when the branch has moved past it the write must fail with + * `RepoWriteConflict` instead of winning the race. Pass null only for a + * caller that has already decided to overwrite. + * + * `rewriteFrom` makes the commit a history rewrite: it is the new commit's + * parent, and `branch` ends up at the commit even when that is not a + * fast-forward from where it points now (the squash-rebase). The commit is + * always created BEFORE the branch is moved, so a crash in between leaves + * the branch exactly where it was and the new commit merely unreferenced — + * never the reverse. `expectedHead` still guards a rewrite, as a re-read + * taken as late as possible (neither provider offers a compare-and-swap on + * a non-fast-forward ref move), and on GitLab the move is unavoidably a + * delete-and-recreate; see that implementation for the residual window. + */ + commitFiles(params: { + branch: string; + message: string; + expectedHead: string | null; + rewriteFrom?: string; + changes: FileChange[]; + }): Promise<{ sha: string }>; + + /** Create `branch` at `sha`; `RepoWriteConflict` when it already exists. */ + createBranch(branch: string, sha: string): Promise; + /** + * Move `branch` to `sha`, discarding whatever it pointed at — the rebase and + * reset paths. Deliberately unguarded: neither provider offers a + * compare-and-swap here, so a caller that cares must re-read the head itself + * immediately before calling. + */ + forceBranchHead(branch: string, sha: string): Promise; + + /** Merge commit sha, or null when `base` already contains `head`. */ + mergeBranches( + base: string, + head: string, + message: string, + ): Promise; + createChangeRequest(params: { + base: string; + head: string; + title: string; + }): Promise; + findOpenChangeRequest( + base: string, + head: string, + ): Promise; + + compare( + base: string, + head: string, + ): Promise<{ + aheadBy: number; + behindBy: number; + }>; + compareDetailed( + base: string, + head: string, + ): Promise<{ + aheadBy: number; + behindBy: number; + mergeBaseSha: string; + files: Array<{ + filename: string; + status: string; + sha: string; + previousFilename?: string; + }>; + /** Commit subjects on `head` that `base` lacks; feeds attribution only. */ + commitMessages: string[]; + }>; +} + +/** + * The branch head, as a hard requirement. Every caller that reads or writes a + * draft addresses it by branch and has nothing to fall back to when it is + * absent, so this is where "missing branch" becomes a 404 instead of a null + * threaded through each of them. + */ +export async function requireBranchHead( + client: RepoContentClient, + branch: string, +): Promise { + const head = await client.getBranch(branch); + if (head) return head.sha; + throw new GitProviderError({ + provider: client.repo.provider, + status: 404, + message: `Branch ${branch} does not exist in ${client.repo.path}`, + }); +} diff --git a/apps/api/src/tools/github/graphql.ts b/apps/api/src/tools/github/graphql.ts index da9540a1c3..e3933ebcf5 100644 --- a/apps/api/src/tools/github/graphql.ts +++ b/apps/api/src/tools/github/graphql.ts @@ -23,7 +23,7 @@ import { recordGithubRateLimit, } from "@/observability/github-rate-limit"; -/** e2e seam, mirroring `decofile/github-git-data.ts`. Read per call site so a +/** e2e seam, mirroring `git-providers/content/github.ts`. Read per call site so a * long-lived dev server and a test webServer agree on one value. */ function githubGraphqlUrl(): string { return process.env.GITHUB_GRAPHQL_URL ?? "https://api.github.com/graphql"; diff --git a/apps/api/src/tools/github/list-user-orgs.ts b/apps/api/src/tools/github/list-user-orgs.ts index be739be9c0..2426dcc08f 100644 --- a/apps/api/src/tools/github/list-user-orgs.ts +++ b/apps/api/src/tools/github/list-user-orgs.ts @@ -16,7 +16,7 @@ import { } from "@/observability/github-rate-limit"; const GITHUB_API = "https://api.github.com"; -/** Matches the Git Data client's per-attempt timeout in `decofile/github-git-data.ts`. */ +/** Matches the content client's per-attempt timeout in `git-providers/content/github.ts`. */ const GITHUB_TIMEOUT_MS = 15_000; interface InstallationsPage { diff --git a/packages/e2e/fixtures/fast-preview.ts b/packages/e2e/fixtures/fast-preview.ts index 3835a69542..032663b095 100644 --- a/packages/e2e/fixtures/fast-preview.ts +++ b/packages/e2e/fixtures/fast-preview.ts @@ -81,7 +81,7 @@ export interface FastPreviewProject { * Full Fast Preview project wiring: repo-scoped GitHub child + unexpired * downstream token + a virtual MCP carrying the Fast Preview gate. * `repoScopeMode` picks which real repo-child shape to seed; the two resolve - * credentials down different paths (see `client-for-repo`), and the default is + * credentials down different paths (see `git-providers/content`), and the default is * the one every repo imported since refreshable grants landed actually has. */ export async function createFastPreviewProject(