diff --git a/docs/scanner-imports/semgrep.md b/docs/scanner-imports/semgrep.md index 2777b204e..54f81cac8 100644 --- a/docs/scanner-imports/semgrep.md +++ b/docs/scanner-imports/semgrep.md @@ -77,8 +77,11 @@ required when publishing the self-contained report. For Cloud use `--to cloud` and a connected repository URL or ID. SDK callers can use `collectSemgrepPlatformFindings` and `normalizeSemgrepPlatformFindings`. -Pages start at zero and are read sequentially through the last page. Collection -errors do not yield partial reports. No time or status filters are applied; +The collector reads pages in order, starting at zero. On HTTP 429, it retries the +same GET once if `Retry-After` specifies a wait of 10 seconds or less, in whole +seconds or as an HTTP date. Zero or a past date triggers an immediate retry. +Cancellation stops the wait. The collector returns a report only after it reads +every page. It applies no time or status filters. Semgrep never returns removed findings from this endpoint. It is a paged view, not a transactional snapshot. Native numeric finding IDs are scoped to the deployment, project, ref and product. Integers outside JavaScript's safe range, diff --git a/docs/scanner-imports/snyk.md b/docs/scanner-imports/snyk.md index dd46f6ca7..17ce8f94f 100644 --- a/docs/scanner-imports/snyk.md +++ b/docs/scanner-imports/snyk.md @@ -87,9 +87,13 @@ the opaque `attributes.key` that Snyk defines across tests within a project. Keep a separate source key per selection. A duplicate key excludes **every** collision; unsupported issue types and malformed finding fields are reported by position. A completed empty response has zero records and still identifies -the mapped repository. A missing page, repeated cursor, scope mismatch, API -error (including rate limiting), or interrupted request is **not** an empty -result. Retry collection; do not publish partial output. +the mapped repository. + +On HTTP 429, the collector retries the same GET once if `Retry-After` specifies +a wait of 10 seconds or less, in whole seconds or as an HTTP date. Zero or a +past date triggers an immediate retry. Cancellation stops the wait. A missing +page, repeated cursor, scope mismatch or failed API request stops collection. +The collector returns a report only after it reads every page. - Open Source uses the documented [Issues field mapping](https://docs.snyk.io/developer-tools/snyk-api/api-end-of-life-eol-process-and-migration-guides/guides-to-migration/v1-reporting-issues-and-v1-aggregated-issues-apis-to-rest-issues-api-migration-guide): problems, weakness classes, dependency representations, effective severity diff --git a/sdk/typescript/src/hosted-scanner-http.ts b/sdk/typescript/src/hosted-scanner-http.ts index 4c377ae00..19a65193c 100644 --- a/sdk/typescript/src/hosted-scanner-http.ts +++ b/sdk/typescript/src/hosted-scanner-http.ts @@ -1,5 +1,8 @@ +import { waitForRetry } from "./deduplication/retry.js"; import { CodexSecurityError } from "./errors.js"; +const maxRetryAfterMs = 10_000; + export interface HostedScannerHttpOptions { environment?: NodeJS.ProcessEnv; fetch?: (url: URL, init: RequestInit) => Promise; @@ -13,22 +16,81 @@ export function combineHostedSignals( return first && second ? AbortSignal.any([first, second]) : (first ?? second); } +function httpDate(hint: string, now: number): number { + const normalized = hint.replace("23:59:60", "23:59:59"); + const leapSecond = normalized === hint ? 0 : 1000; + let canonical = normalized; + const asctime = + /^(\w{3}) (\w{3}) ( \d|\d{2}) (\d{2}:\d{2}:\d{2}) (\d{4})$/u.exec( + normalized, + ); + const rfc850 = + /^(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), (\d{2})-(\w{3})-(\d{2}) (\d{2}:\d{2}:\d{2}) GMT$/u.exec( + normalized, + ); + if (asctime) { + const [, weekday, month, day, time, year] = asctime; + canonical = `${weekday}, ${day!.replace(" ", "0")} ${month} ${year} ${time} GMT`; + } else if (rfc850) { + const [, weekday, day, month, year, time] = rfc850; + const cutoff = new Date(now); + cutoff.setUTCFullYear(cutoff.getUTCFullYear() + 50); + const century = Math.floor(cutoff.getUTCFullYear() / 100) * 100; + const dateIn = (century: number) => + `${weekday!.slice(0, 3)}, ${day} ${month} ${century + Number(year)} ${time} GMT`; + canonical = dateIn(century); + if (Date.parse(canonical) + leapSecond > cutoff.getTime()) + canonical = dateIn(century - 100); + } + const date = new Date(canonical); + if (leapSecond && new Date(date.getTime() + leapSecond).getUTCDate() !== 1) + return NaN; + return canonical.length === 29 && date.toUTCString() === canonical + ? date.getTime() + leapSecond + : NaN; +} + +function retryAfterMs(response: Response): number | undefined { + if (response.status !== 429) return; + const hint = response.headers.get("retry-after"); + if (!hint) return; + let delay: number; + if (/^\d+$/u.test(hint)) { + delay = Number(hint) * 1000; + } else { + const now = Date.now(); + const serverDate = httpDate(response.headers.get("date") ?? "", now); + const reference = Number.isNaN(serverDate) ? now : serverDate; + delay = Math.max(0, httpDate(hint, reference) - reference); + } + return delay <= maxRetryAfterMs ? delay : undefined; +} + export async function getHostedJson( url: URL, authorization: string, options: HostedScannerHttpOptions = {}, parse?: (body: string) => unknown, ): Promise { - options.signal?.throwIfAborted(); - const response = await (options.fetch ?? fetch)(url, { - method: "GET", - headers: { - accept: "application/vnd.api+json, application/json", - authorization, - }, - redirect: "error", - signal: options.signal, - }); + const get = () => { + options.signal?.throwIfAborted(); + return (options.fetch ?? fetch)(url, { + method: "GET", + headers: { + accept: "application/vnd.api+json, application/json", + authorization, + }, + redirect: "error", + signal: options.signal, + }); + }; + let response = await get(); + const delay = retryAfterMs(response); + if (delay !== undefined) { + void response.body?.cancel().catch(() => undefined); + if (delay > 0) await waitForRetry(delay, options.signal); + response = await get(); + } if (!response.ok) { const retry = response.headers.get("retry-after"); const detail = await response.text(); diff --git a/sdk/typescript/tests-ts/hosted-scanner-http.test.ts b/sdk/typescript/tests-ts/hosted-scanner-http.test.ts new file mode 100644 index 000000000..71a4256f9 --- /dev/null +++ b/sdk/typescript/tests-ts/hosted-scanner-http.test.ts @@ -0,0 +1,118 @@ +import { setImmediate } from "node:timers/promises"; +import { afterEach, expect, jest, mock, test } from "bun:test"; +import { getHostedJson } from "../src/hosted-scanner-http.js"; + +const url = new URL( + "https://semgrep.dev/api/v1/deployments/example/findings?page=1", +); +const now = Date.UTC(2026, 10, 6, 8, 49, 20, 250); +afterEach(() => jest.useRealTimers()); + +function limited(hint: string, cancel: () => void) { + return new Response(new ReadableStream({ cancel }), { + status: 429, + headers: { "Retry-After": hint }, + }); +} + +test.each([ + ["010", 10_000, null], + ["Fri, 06 Nov 2026 08:48:05 GMT", 5_000, "Fri, 06 Nov 2026 08:48:00 GMT"], + ["Fri, 06 Nov 2026 08:49:30 GMT", 9_750, null], + ["Friday, 06-Nov-26 08:49:30 GMT", 9_750, "unusable date"], + ["Fri Nov 6 08:49:30 2026", 9_750, null], + ["Sun, 06 Nov 1994 08:49:37 GMT", 0, null], + ["Sunday, 06-Nov-94 08:49:37 GMT", 0, null], + ["Sat, 31 Dec 2016 23:59:60 GMT", 0, null], +] as const)( + "releases the 429 body and waits for Retry-After %s", + async (hint, delay, serverDate) => { + jest.useFakeTimers({ now }); + const released = Promise.withResolvers(); + const fetch = mock(async () => Response.json({ complete: true })); + const response = limited(hint, released.resolve); + if (serverDate) response.headers.set("Date", serverDate); + fetch.mockResolvedValueOnce(response); + const result = getHostedJson(url, "Bearer synthetic", { fetch }); + await released.promise; + if (delay > 0) { + jest.advanceTimersByTime(delay - 1); + await setImmediate(); + expect(fetch).toHaveBeenCalledTimes(1); + jest.advanceTimersByTime(1); + } + expect(await result).toEqual({ complete: true }); + expect(fetch).toHaveBeenCalledTimes(2); + }, +); + +test.each(["cancel", "timeout"])( + "a %s during Retry-After prevents the second GET", + async (mode) => { + jest.useFakeTimers({ now }); + const controller = new AbortController(); + const signal = + mode === "timeout" ? AbortSignal.timeout(500) : controller.signal; + const released = Promise.withResolvers(); + const fetch = mock(async () => limited("10", released.resolve)); + const result = getHostedJson(url, "Bearer synthetic", { + fetch, + signal, + }).catch((error: unknown) => error); + await released.promise; + if (mode === "timeout") jest.advanceTimersByTime(500); + else controller.abort(new Error("collection stopped")); + expect(await result).toBe(signal.reason); + jest.advanceTimersByTime(10_000); + expect(fetch).toHaveBeenCalledTimes(1); + }, +); + +test("unusable or long hints and other HTTP errors fail on the first response with raw details", async () => { + jest.useFakeTimers({ now }); + const detail = "vendor detail: synthetic-token\n\u001b[31mretry later"; + for (const [status, hint] of [ + [429, null], + [429, ""], + [429, "-1"], + [429, "1.5"], + [429, "1e0"], + [429, "0, 1"], + [429, "11"], + [429, "9".repeat(400)], + [429, "Fri, 06 Nov 2026 08:49:31 GMT"], + [429, "Tuesday, 06-Nov-74 08:49:30 GMT"], + [429, "2026-11-06"], + [429, "Fri, 31 Feb 2023 08:49:30 GMT"], + [429, "Fri, 30 Dec 2016 23:59:60 GMT"], + [401, "0"], + [403, "0"], + [503, "0"], + [307, "0"], + ] as const) { + const fetch = mock( + async () => + new Response(detail, { + status, + headers: hint === null ? {} : { "Retry-After": hint }, + }), + ); + await expect( + getHostedJson(url, "Bearer synthetic", { fetch }), + ).rejects.toThrow( + `GET ${url.origin}${url.pathname} failed (HTTP ${status})${hint ? `; Retry-After: ${hint}` : ""}: ${detail}`, + ); + expect(fetch).toHaveBeenCalledTimes(1); + } +}); + +test("does not retry a fetch rejection", async () => { + const error = new TypeError("connection reset"); + const fetch = mock(async () => { + throw error; + }); + await expect(getHostedJson(url, "Bearer synthetic", { fetch })).rejects.toBe( + error, + ); + expect(fetch).toHaveBeenCalledTimes(1); +}); diff --git a/sdk/typescript/tests-ts/semgrep-platform-findings.test.ts b/sdk/typescript/tests-ts/semgrep-platform-findings.test.ts index 0555ed05b..78649d72d 100644 --- a/sdk/typescript/tests-ts/semgrep-platform-findings.test.ts +++ b/sdk/typescript/tests-ts/semgrep-platform-findings.test.ts @@ -23,7 +23,7 @@ const first = Array.from({ length: 100 }, (_, index) => platformFinding(index + 1), ); -test("collects consecutive authenticated pages for exactly one project, ref and product", async () => { +test("retries only the throttled page for exactly one Semgrep project, ref and product", async () => { const pages: string[] = []; const report = await collectSemgrepPlatformFindings(options, { environment, @@ -33,6 +33,7 @@ test("collects consecutive authenticated pages for exactly one project, ref and "Bearer synthetic-web-token", ); expect(init.redirect).toBe("error"); + expect(init.method).toBe("GET"); if (url.pathname.includes("/projects/")) { expect(decodeURIComponent(url.pathname)).toBe( "/api/v1/deployments/example/projects/example/project", @@ -48,13 +49,18 @@ test("collects consecutive authenticated pages for exactly one project, ref and page_size: "100", }); pages.push(query["page"]!); + if (pages.length === 2) + return new Response("try again", { + status: 429, + headers: { "Retry-After": "0" }, + }); return Response.json({ findings: query["page"] === "0" ? first : [platformFinding(101)], }); }, }); - expect(pages).toEqual(["0", "1"]); - expect(report.findings).toHaveLength(101); + expect(pages).toEqual(["0", "1", "1"]); + expect(report.findings).toEqual([...first, platformFinding(101)]); expect(JSON.stringify(report)).not.toContain("synthetic-web-token"); const parsed = normalizeSemgrepPlatformFindings(report); expect(parsed.findings[0]?.evidence).toMatchObject({ @@ -111,29 +117,31 @@ test("fails before acquisition without opt-in, honors cancellation, and never re }, ), ).rejects.toThrow("dependency canceled"); - for (const mode of ["repeat", "rate-limit", "malformed"]) { + for (const [mode, message] of [ + ["repeat", "repeated"], + ["malformed", "must be an array"], + ["throttled", "Retry-After: 0: try later"], + ] as const) { + const pages: string[] = []; const collect = collectSemgrepPlatformFindings(options, { environment, fetch: async (url) => { if (url.pathname.includes("/projects/")) return Response.json({ project: platformProject }); - if (url.searchParams.get("page") === "0" || mode === "repeat") + const page = url.searchParams.get("page")!; + pages.push(page); + if (page === "0" || mode === "repeat") return Response.json({ findings: first }); - return mode === "rate-limit" - ? new Response("try later", { - status: 429, - headers: { "retry-after": "11" }, - }) - : Response.json({ error: "not a findings page" }); + if (mode === "malformed") + return Response.json({ error: "not a findings page" }); + return new Response("try later", { + status: 429, + headers: { "Retry-After": "0" }, + }); }, }); - await expect(collect).rejects.toThrow( - mode === "repeat" - ? "repeated" - : mode === "rate-limit" - ? "Retry-After: 11" - : "must be an array", - ); + await expect(collect).rejects.toThrow(message); + expect(pages).toEqual(mode === "throttled" ? ["0", "1", "1"] : ["0", "1"]); } }); diff --git a/sdk/typescript/tests-ts/snyk-rest-findings.test.ts b/sdk/typescript/tests-ts/snyk-rest-findings.test.ts index bc39fa005..92042f3d1 100644 --- a/sdk/typescript/tests-ts/snyk-rest-findings.test.ts +++ b/sdk/typescript/tests-ts/snyk-rest-findings.test.ts @@ -58,7 +58,7 @@ function transport(...responses: unknown[]) { dependencies: { environment: env, fetch: async (url: URL, init: RequestInit) => { - requests.push({ url, init }); + requests.push({ url: new URL(url), init }); if (!responses.length) throw new Error("Unexpected request."); const response = responses.shift(); return response instanceof Response @@ -81,7 +81,7 @@ function envelope(data: unknown[] = [issue()]) { }; } -test("collects only one region, org, project and Open Source issue type across JSON:API pages", async () => { +test("retries only the throttled JSON:API page within the selected Snyk scope", async () => { const primary = issue({ description: "Original description", ignored: true, @@ -122,6 +122,10 @@ test("collects only one region, org, project and Open Source issue type across J document([primary], { href: `/orgs/${organizationId}/issues?version=${version}&starting_after=${encodeURIComponent(cursor)}`, }), + new Response("try again", { + status: 429, + headers: { "Retry-After": "0", Location: "https://other.invalid/issues" }, + }), document([secondary], null), ); const signal = new AbortController().signal; @@ -131,6 +135,7 @@ test("collects only one region, org, project and Open Source issue type across J `/rest/orgs/${organizationId}/projects/${projectId}`, path, path, + path, ]); for (const { url, init } of requests) { expect(url.origin).toBe(apiOrigin); @@ -154,6 +159,7 @@ test("collects only one region, org, project and Open Source issue type across J } expect(requests[1]!.url.searchParams.has("starting_after")).toBe(false); expect(requests[2]!.url.searchParams.get("starting_after")).toBe(cursor); + expect(requests[3]!.url.href).toBe(requests[2]!.url.href); expect(report.pages.map((page) => page.startingAfter)).toEqual([ null, cursor, @@ -723,15 +729,32 @@ test("HTTP, JSON, and second-page failures are errors, never successful partial { errors: [{ status: "400", detail: "issue error" }] }, { data: null }, ]) { - const { dependencies } = transport( + const { dependencies, requests } = transport( document(project()), document([issue()], "?starting_after=more"), response, ); await expect(collectSnykIssues(options, dependencies)).rejects.toThrow(); + expect(requests).toHaveLength(3); } }); +test("reports the last HTTP error when the retried Snyk cursor is still throttled", async () => { + const limited = (detail: string) => + new Response(detail, { status: 429, headers: { "Retry-After": "0" } }); + const { dependencies, requests } = transport( + document(project()), + document([issue()], "?starting_after=more"), + limited("first failure"), + limited("still throttled"), + ); + await expect(collectSnykIssues(options, dependencies)).rejects.toThrow( + `GET ${apiOrigin}${path} failed (HTTP 429); Retry-After: 0: still throttled`, + ); + expect(requests).toHaveLength(4); + expect(requests[3]!.url.href).toBe(requests[2]!.url.href); +}); + test.each(["options", "dependencies"] as const)( "aborting the %s signal with the other signal live cancels the request and prevents new requests", async (caller) => {