fix(imports): honor bounded Retry-After on hosted scanner reads

docs/scanner-imports/semgrep.md

@@ -77,8 +77,11 @@ required when publishing the self-contained report. For Cloud use `--to cloud`
 and a connected repository URL or ID. SDK callers can use
 `collectSemgrepPlatformFindings` and `normalizeSemgrepPlatformFindings`.
 
-Pages start at zero and are read sequentially through the last page. Collection
-errors do not yield partial reports. No time or status filters are applied;
+The collector reads pages in order, starting at zero. On HTTP 429, it retries the
+same GET once if `Retry-After` specifies a wait of 10 seconds or less, in whole
+seconds or as an HTTP date. Zero or a past date triggers an immediate retry.
+Cancellation stops the wait. The collector returns a report only after it reads
+every page. It applies no time or status filters.
 Semgrep never returns removed findings from this endpoint. It is a paged view,
 not a transactional snapshot. Native numeric finding IDs are scoped to the
 deployment, project, ref and product. Integers outside JavaScript's safe range,

docs/scanner-imports/snyk.md

@@ -87,9 +87,13 @@ the opaque `attributes.key` that Snyk defines across tests within a project.
 Keep a separate source key per selection. A duplicate key excludes **every**
 collision; unsupported issue types and malformed finding fields are reported
 by position. A completed empty response has zero records and still identifies
-the mapped repository. A missing page, repeated cursor, scope mismatch, API
-error (including rate limiting), or interrupted request is **not** an empty
-result. Retry collection; do not publish partial output.
+the mapped repository.
+
+On HTTP 429, the collector retries the same GET once if `Retry-After` specifies
+a wait of 10 seconds or less, in whole seconds or as an HTTP date. Zero or a
+past date triggers an immediate retry. Cancellation stops the wait. A missing
+page, repeated cursor, scope mismatch or failed API request stops collection.
+The collector returns a report only after it reads every page.
 
 - Open Source uses the documented [Issues field mapping](https://docs.snyk.io/developer-tools/snyk-api/api-end-of-life-eol-process-and-migration-guides/guides-to-migration/v1-reporting-issues-and-v1-aggregated-issues-apis-to-rest-issues-api-migration-guide):
   problems, weakness classes, dependency representations, effective severity

sdk/typescript/src/hosted-scanner-http.ts

@@ -1,5 +1,8 @@
+import { waitForRetry } from "./deduplication/retry.js";
 import { CodexSecurityError } from "./errors.js";
 
+const maxRetryAfterMs = 10_000;
+
 export interface HostedScannerHttpOptions {
   environment?: NodeJS.ProcessEnv;
   fetch?: (url: URL, init: RequestInit) => Promise<Response>;
@@ -13,22 +16,81 @@ export function combineHostedSignals(
   return first && second ? AbortSignal.any([first, second]) : (first ?? second);
 }
 
+function httpDate(hint: string, now: number): number {
+  const normalized = hint.replace("23:59:60", "23:59:59");
+  const leapSecond = normalized === hint ? 0 : 1000;
+  let canonical = normalized;
+  const asctime =
+    /^(\w{3}) (\w{3}) ( \d|\d{2}) (\d{2}:\d{2}:\d{2}) (\d{4})$/u.exec(
+      normalized,
+    );
+  const rfc850 =
+    /^(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), (\d{2})-(\w{3})-(\d{2}) (\d{2}:\d{2}:\d{2}) GMT$/u.exec(
+      normalized,
+    );
+  if (asctime) {
+    const [, weekday, month, day, time, year] = asctime;
+    canonical = `${weekday}, ${day!.replace(" ", "0")} ${month} ${year} ${time} GMT`;
+  } else if (rfc850) {
+    const [, weekday, day, month, year, time] = rfc850;
+    const cutoff = new Date(now);
+    cutoff.setUTCFullYear(cutoff.getUTCFullYear() + 50);
+    const century = Math.floor(cutoff.getUTCFullYear() / 100) * 100;
+    const dateIn = (century: number) =>
+      `${weekday!.slice(0, 3)}, ${day} ${month} ${century + Number(year)} ${time} GMT`;
+    canonical = dateIn(century);
+    if (Date.parse(canonical) + leapSecond > cutoff.getTime())
+      canonical = dateIn(century - 100);
+  }
+  const date = new Date(canonical);
+  if (leapSecond && new Date(date.getTime() + leapSecond).getUTCDate() !== 1)
+    return NaN;
+  return canonical.length === 29 && date.toUTCString() === canonical
+    ? date.getTime() + leapSecond
+    : NaN;
+}
+
+function retryAfterMs(response: Response): number | undefined {
+  if (response.status !== 429) return;
+  const hint = response.headers.get("retry-after");
+  if (!hint) return;
+  let delay: number;
+  if (/^\d+$/u.test(hint)) {
+    delay = Number(hint) * 1000;
+  } else {
+    const now = Date.now();
+    const serverDate = httpDate(response.headers.get("date") ?? "", now);
+    const reference = Number.isNaN(serverDate) ? now : serverDate;
+    delay = Math.max(0, httpDate(hint, reference) - reference);
+  }
+  return delay <= maxRetryAfterMs ? delay : undefined;
+}
+
 export async function getHostedJson(
   url: URL,
   authorization: string,
   options: HostedScannerHttpOptions = {},
   parse?: (body: string) => unknown,
 ): Promise<unknown> {
-  options.signal?.throwIfAborted();
-  const response = await (options.fetch ?? fetch)(url, {
-    method: "GET",
-    headers: {
-      accept: "application/vnd.api+json, application/json",
-      authorization,
-    },
-    redirect: "error",
-    signal: options.signal,
-  });
+  const get = () => {
+    options.signal?.throwIfAborted();
+    return (options.fetch ?? fetch)(url, {
+      method: "GET",
+      headers: {
+        accept: "application/vnd.api+json, application/json",
+        authorization,
+      },
+      redirect: "error",
+      signal: options.signal,
+    });
+  };
+  let response = await get();
+  const delay = retryAfterMs(response);
+  if (delay !== undefined) {
+    void response.body?.cancel().catch(() => undefined);
+    if (delay > 0) await waitForRetry(delay, options.signal);
+    response = await get();
+  }
   if (!response.ok) {
     const retry = response.headers.get("retry-after");
     const detail = await response.text();

sdk/typescript/tests-ts/hosted-scanner-http.test.ts

@@ -0,0 +1,118 @@
+import { setImmediate } from "node:timers/promises";
+import { afterEach, expect, jest, mock, test } from "bun:test";
+import { getHostedJson } from "../src/hosted-scanner-http.js";
+
+const url = new URL(
+  "https://semgrep.dev/api/v1/deployments/example/findings?page=1",
+);
+const now = Date.UTC(2026, 10, 6, 8, 49, 20, 250);
+afterEach(() => jest.useRealTimers());
+
+function limited(hint: string, cancel: () => void) {
+  return new Response(new ReadableStream({ cancel }), {
+    status: 429,
+    headers: { "Retry-After": hint },
+  });
+}
+
+test.each([
+  ["010", 10_000, null],
+  ["Fri, 06 Nov 2026 08:48:05 GMT", 5_000, "Fri, 06 Nov 2026 08:48:00 GMT"],
+  ["Fri, 06 Nov 2026 08:49:30 GMT", 9_750, null],
+  ["Friday, 06-Nov-26 08:49:30 GMT", 9_750, "unusable date"],
+  ["Fri Nov  6 08:49:30 2026", 9_750, null],
+  ["Sun, 06 Nov 1994 08:49:37 GMT", 0, null],
+  ["Sunday, 06-Nov-94 08:49:37 GMT", 0, null],
+  ["Sat, 31 Dec 2016 23:59:60 GMT", 0, null],
+] as const)(
+  "releases the 429 body and waits for Retry-After %s",
+  async (hint, delay, serverDate) => {
+    jest.useFakeTimers({ now });
+    const released = Promise.withResolvers<void>();
+    const fetch = mock(async () => Response.json({ complete: true }));
+    const response = limited(hint, released.resolve);
+    if (serverDate) response.headers.set("Date", serverDate);
+    fetch.mockResolvedValueOnce(response);
+    const result = getHostedJson(url, "Bearer synthetic", { fetch });
+    await released.promise;
+    if (delay > 0) {
+      jest.advanceTimersByTime(delay - 1);
+      await setImmediate();
+      expect(fetch).toHaveBeenCalledTimes(1);
+      jest.advanceTimersByTime(1);
+    }
+    expect(await result).toEqual({ complete: true });
+    expect(fetch).toHaveBeenCalledTimes(2);
+  },
+);
+
+test.each(["cancel", "timeout"])(
+  "a %s during Retry-After prevents the second GET",
+  async (mode) => {
+    jest.useFakeTimers({ now });
+    const controller = new AbortController();
+    const signal =
+      mode === "timeout" ? AbortSignal.timeout(500) : controller.signal;
+    const released = Promise.withResolvers<void>();
+    const fetch = mock(async () => limited("10", released.resolve));
+    const result = getHostedJson(url, "Bearer synthetic", {
+      fetch,
+      signal,
+    }).catch((error: unknown) => error);
+    await released.promise;
+    if (mode === "timeout") jest.advanceTimersByTime(500);
+    else controller.abort(new Error("collection stopped"));
+    expect(await result).toBe(signal.reason);
+    jest.advanceTimersByTime(10_000);
+    expect(fetch).toHaveBeenCalledTimes(1);
+  },
+);
+
+test("unusable or long hints and other HTTP errors fail on the first response with raw details", async () => {
+  jest.useFakeTimers({ now });
+  const detail = "vendor detail: synthetic-token\n\u001b[31mretry later";
+  for (const [status, hint] of [
+    [429, null],
+    [429, ""],
+    [429, "-1"],
+    [429, "1.5"],
+    [429, "1e0"],
+    [429, "0, 1"],
+    [429, "11"],
+    [429, "9".repeat(400)],
+    [429, "Fri, 06 Nov 2026 08:49:31 GMT"],
+    [429, "Tuesday, 06-Nov-74 08:49:30 GMT"],
+    [429, "2026-11-06"],
+    [429, "Fri, 31 Feb 2023 08:49:30 GMT"],
+    [429, "Fri, 30 Dec 2016 23:59:60 GMT"],
+    [401, "0"],
+    [403, "0"],
+    [503, "0"],
+    [307, "0"],
+  ] as const) {
+    const fetch = mock(
+      async () =>
+        new Response(detail, {
+          status,
+          headers: hint === null ? {} : { "Retry-After": hint },
+        }),
+    );
+    await expect(
+      getHostedJson(url, "Bearer synthetic", { fetch }),
+    ).rejects.toThrow(
+      `GET ${url.origin}${url.pathname} failed (HTTP ${status})${hint ? `; Retry-After: ${hint}` : ""}: ${detail}`,
+    );
+    expect(fetch).toHaveBeenCalledTimes(1);
+  }
+});
+
+test("does not retry a fetch rejection", async () => {
+  const error = new TypeError("connection reset");
+  const fetch = mock(async () => {
+    throw error;
+  });
+  await expect(getHostedJson(url, "Bearer synthetic", { fetch })).rejects.toBe(
+    error,
+  );
+  expect(fetch).toHaveBeenCalledTimes(1);
+});

sdk/typescript/tests-ts/semgrep-platform-findings.test.ts

@@ -23,7 +23,7 @@ const first = Array.from({ length: 100 }, (_, index) =>
   platformFinding(index + 1),
 );
 
-test("collects consecutive authenticated pages for exactly one project, ref and product", async () => {
+test("retries only the throttled page for exactly one Semgrep project, ref and product", async () => {
   const pages: string[] = [];
   const report = await collectSemgrepPlatformFindings(options, {
     environment,
@@ -33,6 +33,7 @@ test("collects consecutive authenticated pages for exactly one project, ref and
         "Bearer synthetic-web-token",
       );
       expect(init.redirect).toBe("error");
+      expect(init.method).toBe("GET");
       if (url.pathname.includes("/projects/")) {
         expect(decodeURIComponent(url.pathname)).toBe(
           "/api/v1/deployments/example/projects/example/project",
@@ -48,13 +49,18 @@ test("collects consecutive authenticated pages for exactly one project, ref and
         page_size: "100",
       });
       pages.push(query["page"]!);
+      if (pages.length === 2)
+        return new Response("try again", {
+          status: 429,
+          headers: { "Retry-After": "0" },
+        });
       return Response.json({
         findings: query["page"] === "0" ? first : [platformFinding(101)],
       });
     },
   });
-  expect(pages).toEqual(["0", "1"]);
-  expect(report.findings).toHaveLength(101);
+  expect(pages).toEqual(["0", "1", "1"]);
+  expect(report.findings).toEqual([...first, platformFinding(101)]);
   expect(JSON.stringify(report)).not.toContain("synthetic-web-token");
   const parsed = normalizeSemgrepPlatformFindings(report);
   expect(parsed.findings[0]?.evidence).toMatchObject({
@@ -111,29 +117,31 @@ test("fails before acquisition without opt-in, honors cancellation, and never re
       },
     ),
   ).rejects.toThrow("dependency canceled");
-  for (const mode of ["repeat", "rate-limit", "malformed"]) {
+  for (const [mode, message] of [
+    ["repeat", "repeated"],
+    ["malformed", "must be an array"],
+    ["throttled", "Retry-After: 0: try later"],
+  ] as const) {
+    const pages: string[] = [];
     const collect = collectSemgrepPlatformFindings(options, {
       environment,
       fetch: async (url) => {
         if (url.pathname.includes("/projects/"))
           return Response.json({ project: platformProject });
-        if (url.searchParams.get("page") === "0" || mode === "repeat")
+        const page = url.searchParams.get("page")!;
+        pages.push(page);
+        if (page === "0" || mode === "repeat")
           return Response.json({ findings: first });
-        return mode === "rate-limit"
-          ? new Response("try later", {
-              status: 429,
-              headers: { "retry-after": "11" },
-            })
-          : Response.json({ error: "not a findings page" });
+        if (mode === "malformed")
+          return Response.json({ error: "not a findings page" });
+        return new Response("try later", {
+          status: 429,
+          headers: { "Retry-After": "0" },
+        });
       },
     });
-    await expect(collect).rejects.toThrow(
-      mode === "repeat"
-        ? "repeated"
-        : mode === "rate-limit"
-          ? "Retry-After: 11"
-          : "must be an array",
-    );
+    await expect(collect).rejects.toThrow(message);
+    expect(pages).toEqual(mode === "throttled" ? ["0", "1", "1"] : ["0", "1"]);
   }
 });