feat(scanning): advisory npm vulnerability scanning with admin UI

AGENTS.md

@@ -23,6 +23,18 @@ Check if services are running: `bash dev/status.sh`
 
 Log files are in `dev/logs/` (dev-api.log, dev-ui.log, docker-compose.log).
 
+### Vulnerability scanner (osv-scanner)
+
+Dev config enables npm vulnerability scanning (`api/config/development.js`), which shells out to the **osv-scanner v2** binary. Install it once so it's on your PATH (Linux x86_64):
+
+    OSV_VERSION=v2.2.3
+    sudo curl -sSfL -o /usr/local/bin/osv-scanner \
+      "https://github.com/google/osv-scanner/releases/download/${OSV_VERSION}/osv-scanner_linux_amd64"
+    sudo chmod +x /usr/local/bin/osv-scanner
+    osv-scanner --version
+
+(Any directory on PATH works; drop `sudo` if you install into a user-writable dir.) The offline OSV database (~200 MB) downloads on first scan into `./data/osv-db` and is refreshed by the periodic rescan; extracted artefacts are cached under `./data/tmp/scan-cache`. If osv-scanner is not installed, scans fail gracefully (`scan.status: "error"`) and the rest of the app keeps working.
+
 ## Testing
 
 Tests use Playwright as a test runner with 3 project types: unit (*.unit.spec.ts), api (*.api.spec.ts), and e2e (*.e2e.spec.ts). State setup/teardown fixtures run before api and e2e tests.
@@ -37,6 +49,18 @@ Run a specific test file:
 
 Test users are defined in @dev/resources/users.json and organizations in @dev/resources/organizations.json.
 
+## Vulnerability scanning
+
+Advisory, admin-only vulnerability scanning of `npm` artefacts (bundled `node_modules` tarballs) via the bundled **osv-scanner v2** binary. It never blocks uploads or downloads, and scan data is stripped from responses for non-admin callers.
+
+- Controlled by the `scanning.*` config block (`api/config/type/schema.json`); **off by default** (`scanning.enabled`).
+- osv-scanner runs in **offline local-DB mode** (`scanning.dbDir`), refreshed on the rescan interval. Bundled, lockfile-less `node_modules` are detected via the `--experimental-plugins javascript/packagejson` extractor on `osv-scanner scan source`.
+- Scans run on three triggers: after an npm upload (async, non-blocking), on a periodic interval (`scanning.rescanIntervalHours`, also refreshes the DB), and on-demand via `POST /api/v1/artefacts/:id/scan` (admin).
+- A summary lives on the artefact `scan` field (admin-only); full findings are stored in the `artefact-scans` Mongo collection and served by `GET /api/v1/artefacts/:id/scan` (admin).
+- Module: `api/src/scanning/` (`operations.ts` pure mapping, `extract.ts` tarball extraction, `runner.ts` osv-scanner subprocess, `service.ts` orchestration, `router.ts` endpoints).
+- The osv-scanner binary is bundled in the Docker image (see `Dockerfile`). The mapper's test fixture is `tests/resources/osv-sample-output.json`.
+- Extracted artefacts are cached as a mirror under `<tmpDir>/scan-cache/` (config `tmpDir`, env `TMP_DIR`; mount as a k8s `emptyDir`). A scan reuses the cached extraction when the artefact's bytes are unchanged (keyed on `artefact.path`); `rescanAll` prunes slots for deleted artefacts.
+
 ## Code patterns
 
   - API route: @api/src/artefacts/router.ts

Dockerfile

@@ -17,6 +17,17 @@ ADD package.json package-lock.json ./
 RUN jq '.version="build"' package.json | sponge package.json
 RUN jq '.version="build"' package-lock.json | sponge package-lock.json
 
+##########################
+FROM base AS osv-scanner
+# Advisory npm vulnerability scanner (statically-linked Go binary; runs on musl).
+ARG OSV_SCANNER_VERSION=v2.2.3
+ARG TARGETARCH=amd64
+RUN apk add --no-cache curl && \
+    curl -sSfL -o /usr/local/bin/osv-scanner \
+      "https://github.com/google/osv-scanner/releases/download/${OSV_SCANNER_VERSION}/osv-scanner_linux_${TARGETARCH}" && \
+    chmod +x /usr/local/bin/osv-scanner && \
+    /usr/local/bin/osv-scanner --version
+
 ##########################
 FROM base AS installer
 
@@ -77,6 +88,7 @@ RUN mkdir -p /app/api/node_modules
 FROM native-deps AS main
 
 COPY --from=api-installer /app/node_modules node_modules
+COPY --from=osv-scanner /usr/local/bin/osv-scanner /usr/local/bin/osv-scanner
 ADD /api api
 ADD /shared shared
 COPY --from=types /app/api/types api/types

api/config/custom-environment-variables.js

@@ -15,6 +15,7 @@ export default {
   },
   upgradeRoot: 'UPGRADE_ROOT',
   dataDir: 'DATA_DIR',
+  tmpDir: 'TMP_DIR',
   maxUploadBytes: {
     __name: 'MAX_UPLOAD_BYTES',
     __format: 'number'
@@ -27,6 +28,26 @@ export default {
     __name: 'MAX_TAR_ENTRIES',
     __format: 'number'
   },
+  scanning: {
+    enabled: {
+      __name: 'SCANNING_ENABLED',
+      __format: 'boolean'
+    },
+    osvScannerPath: 'OSV_SCANNER_PATH',
+    dbDir: 'OSV_DB_DIR',
+    concurrency: {
+      __name: 'SCANNING_CONCURRENCY',
+      __format: 'number'
+    },
+    rescanIntervalHours: {
+      __name: 'SCANNING_RESCAN_INTERVAL_HOURS',
+      __format: 'number'
+    },
+    timeoutSeconds: {
+      __name: 'SCANNING_TIMEOUT_SECONDS',
+      __format: 'number'
+    }
+  },
   filesStorage: 'FILES_STORAGE',
   s3: {
     region: 'S3_REGION',

api/config/default.js

@@ -4,9 +4,18 @@ export default {
   mongoUrl: 'mongodb://localhost:27017/data-fair-registry',
   port: 8080,
   dataDir: '/data',
+  tmpDir: undefined,
   maxUploadBytes: 200 * 1024 * 1024 * 1024,
   maxDecompressedBytes: 1024 * 1024 * 1024,
   maxTarEntries: 100_000,
+  scanning: {
+    enabled: false,
+    osvScannerPath: 'osv-scanner',
+    dbDir: '/data/osv-db',
+    concurrency: 1,
+    rescanIntervalHours: 24,
+    timeoutSeconds: 300
+  },
   filesStorage: 'fs',
   s3: {
     region: '',

api/config/development.js

@@ -9,6 +9,15 @@ export default {
   privateEventsUrl: `http://localhost:${process.env.EVENTS_PORT}`,
   mongoUrl: `mongodb://localhost:${process.env.MONGO_PORT}/data-fair-registry-development`,
   dataDir: './data',
+  // Vulnerability scanning is enabled in dev. Requires the osv-scanner binary
+  // on PATH (see "Vulnerability scanner" in AGENTS.md for install). The offline
+  // OSV DB is downloaded under dataDir on first scan. If osv-scanner is missing,
+  // scans just record scan.status="error" and the rest of the app is unaffected.
+  scanning: {
+    enabled: true,
+    osvScannerPath: 'osv-scanner',
+    dbDir: './data/osv-db'
+  },
   observer: {
     active: false
   },

api/config/type/schema.json

@@ -27,6 +27,10 @@
     "dataDir": {
       "type": "string"
     },
+    "tmpDir": {
+      "type": "string",
+      "description": "Working directory for ephemeral scan extractions (the scan cache lives at <tmpDir>/scan-cache). Defaults to <dataDir>/tmp, or an OS temp dir. Mount as an emptyDir in k8s."
+    },
     "maxUploadBytes": {
       "type": "integer",
       "default": 524288000,
@@ -42,6 +46,46 @@
       "default": 100000,
       "description": "Maximum number of entries (files + directories) accepted in an uploaded npm tarball. Pre-installed node_modules can easily exceed the lower legacy default."
     },
+    "scanning": {
+      "type": "object",
+      "additionalProperties": false,
+      "default": {},
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "default": false,
+          "description": "Enable advisory vulnerability scanning of npm artefacts via osv-scanner. Off by default so deployments without the binary keep working."
+        },
+        "osvScannerPath": {
+          "type": "string",
+          "default": "osv-scanner",
+          "description": "Path or command name of the osv-scanner v2 binary."
+        },
+        "dbDir": {
+          "type": "string",
+          "default": "/data/osv-db",
+          "description": "Local directory holding the offline OSV vulnerability database."
+        },
+        "concurrency": {
+          "type": "integer",
+          "minimum": 1,
+          "default": 1,
+          "description": "Maximum number of artefact scans running at once in this process."
+        },
+        "rescanIntervalHours": {
+          "type": "integer",
+          "minimum": 1,
+          "default": 24,
+          "description": "Interval in hours between automatic rescans of all npm artefacts (also refreshes the OSV DB)."
+        },
+        "timeoutSeconds": {
+          "type": "integer",
+          "minimum": 30,
+          "default": 300,
+          "description": "Per-artefact scan timeout in seconds (extraction + osv-scanner)."
+        }
+      }
+    },
     "privateDirectoryUrl": {
       "type": "string",
       "pattern": "^https?://"

api/package.json

@@ -34,9 +34,10 @@
     "mongodb": "^6.8.0",
     "piscina": "^4.7.0",
     "prom-client": "^15.1.3",
-    "sharp": "^0.33.5",
     "resolve-path": "^1.4.0",
     "semver": "^7.6.0",
+    "sharp": "^0.33.5",
+    "tar": "^7.5.16",
     "tar-stream": "^3.1.0"
   }
 }

api/src/app.ts

@@ -44,6 +44,7 @@ if (process.env.NODE_ENV === 'development') {
   app.delete('/api/test-env', async (req, res) => {
     assertReqInternal(req)
     await mongo.artefacts.deleteMany({})
+    await mongo.artefactScans.deleteMany({})
     await mongo.apiKeys.deleteMany({})
     // Scoped delete: only grants for test accounts (test users/orgs are
     // `test*`-prefixed) are wiped, so manually-created dev grants survive.
@@ -64,6 +65,47 @@ if (process.env.NODE_ENV === 'development') {
     }
     res.send()
   })
+
+  app.put('/api/test-env/artefacts/:id/scan', async (req, res) => {
+    assertReqInternal(req)
+    const id = decodeURIComponent(req.params.id)
+    const { summary, findings } = req.body
+    await mongo.artefacts.updateOne(
+      { _id: id },
+      { $set: { scan: { status: 'success', finishedAt: new Date().toISOString(), summary } } }
+    )
+    await mongo.artefactScans.replaceOne(
+      { _id: id },
+      { scannedAt: new Date().toISOString(), scannerVersion: 'test', vulnerabilities: findings ?? [] },
+      { upsert: true }
+    )
+    res.send()
+  })
+
+  // Upsert an artefact doc directly with arbitrary fields — lets tests build a
+  // deterministic fleet (with controlled scan state) without the upload
+  // pipeline, whose auto-scan would otherwise race injected state.
+  app.put('/api/test-env/artefacts/:id/doc', async (req, res) => {
+    assertReqInternal(req)
+    const id = decodeURIComponent(req.params.id)
+    const now = new Date().toISOString()
+    await mongo.artefacts.replaceOne(
+      { _id: id },
+      {
+        _id: id,
+        name: id,
+        format: 'npm',
+        public: false,
+        privateAccess: [],
+        createdAt: now,
+        updatedAt: now,
+        dataUpdatedAt: now,
+        ...req.body
+      } as any,
+      { upsert: true }
+    )
+    res.send()
+  })
 }
 
 app.use('/api', (req, res) => res.status(404).send('unknown api endpoint'))

api/src/artefacts/router.ts

@@ -14,15 +14,29 @@ import { filesStorage } from '../files-storage/index.ts'
 import {
   listArtefacts, getArtefact, getArtefactById, patchArtefact, deleteArtefact,
   commitFileUpload, commitNpmUpload, extractStagedManifest, resolveDownload,
-  listGroupValues
+  listGroupValues, getScanSummary
 } from './service.ts'
 import * as patchReqBody from '#doc/artefacts/patch-req/index.ts'
 import { artefactThumbnailRouter } from '../thumbnails/router.ts'
+import scanRouter from '../scanning/router.ts'
+import type { Caller } from '../access.ts'
 
 const router = Router()
 export default router
 
 router.use('/:id/thumbnail', artefactThumbnailRouter)
+router.use('/:id/scan', scanRouter)
+
+// Drop the advisory, admin-only scan field unconditionally.
+const omitScan = <T extends { scan?: unknown }>(artefact: T): T => {
+  if (artefact.scan === undefined) return artefact
+  const { scan, ...rest } = artefact
+  return rest as T
+}
+
+// Scan results are advisory and admin-only; strip them for non-admin callers.
+const stripScan = <T extends { scan?: unknown }>(artefact: T, caller: Caller): T =>
+  caller.admin ? artefact : omitScan(artefact)
 
 const npmCategories = ['processing', 'catalog', 'application', 'other'] as const
 const fileCategories = ['tileset', 'maplibre-style', 'other'] as const
@@ -85,10 +99,18 @@ const tryInternalSecret = (req: import('express').Request): boolean => {
 // List artefacts (filtered by access)
 router.get('/', async (req, res, next) => {
   try {
-    const filter = artefactAccessFilter(await resolveCaller(req))
+    const caller = await resolveCaller(req)
+    const filter = artefactAccessFilter(caller)
     const skip = Math.max(0, Math.min(parseInt(req.query.skip as string) || 0, 100000))
     const size = Math.min(parseInt(req.query.size as string) || 10, 100)
-    const sort: Record<string, 1 | -1> = req.query.sort === 'name' ? { name: 1 } : { dataUpdatedAt: -1 }
+    // `vulnerabilities` is admin-only: scan data is stripped for non-admins, so
+    // ordering by it would leak. Non-admins silently get the default sort.
+    let sort: Record<string, 1 | -1> = { dataUpdatedAt: -1 }
+    if (req.query.sort === 'name') {
+      sort = { name: 1 }
+    } else if (req.query.sort === 'vulnerabilities' && caller.admin) {
+      sort = { 'scan.summary.critical': -1, 'scan.summary.high': -1, 'scan.summary.medium': -1, dataUpdatedAt: -1 }
+    }
 
     // Text search on name
     if (req.query.q) {
@@ -116,7 +138,7 @@ router.get('/', async (req, res, next) => {
     }
 
     const { results, count } = await listArtefacts(filter, { sort, skip, size })
-    res.json({ results, count })
+    res.json({ results: results.map(r => stripScan(r, caller)), count })
   } catch (err) { next(err) }
 })
 
@@ -136,14 +158,23 @@ router.get('/groups', async (req, res, next) => {
   } catch (err) { next(err) }
 })
 
+// Fleet-wide vulnerability roll-up for the admin dashboard. Registered before
+// GET /:id so the literal path isn't swallowed as an id. Admin-only.
+router.get('/scan-summary', async (req, res, next) => {
+  try {
+    await session.reqAdminMode(req)
+    res.json(await getScanSummary())
+  } catch (err) { next(err) }
+})
+
 // Get artefact detail
 // All formats carry their tarball references directly on the doc.
 router.get('/:id', async (req, res, next) => {
   try {
-    const filter = artefactAccessFilter(await resolveCaller(req))
-    const artefact = await getArtefact(req.params.id, filter)
+    const caller = await resolveCaller(req)
+    const artefact = await getArtefact(req.params.id, artefactAccessFilter(caller))
     if (!artefact) throw httpError(404, 'artefact not found')
-    res.json(artefact)
+    res.json(stripScan(artefact, caller))
   } catch (err) { next(err) }
 })
 
@@ -241,7 +272,8 @@ router.post('/file/:name', async (req, res, next) => {
         : { internal: true }
     })
     stagingStored = false
-    res.status(201).json({ artefact })
+    // Uploaders are never admin session callers; never leak scan data.
+    res.status(201).json({ artefact: omitScan(artefact) })
   } catch (err) {
     if (stagingStored) await filesStorage.delete(stagingPath).catch(() => {})
     next(err)
@@ -299,7 +331,8 @@ router.post('/npm/:id', async (req, res, next) => {
       existing
     })
     stagingStored = false
-    res.status(201).json({ artefact })
+    // Uploaders are never admin session callers; never leak scan data.
+    res.status(201).json({ artefact: omitScan(artefact) })
   } catch (err) {
     if (stagingStored) await filesStorage.delete(stagingPath).catch(() => {})
     next(err)