diff --git a/src/main.ts b/src/main.ts index 402cf4a..010b4e1 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,6 +4,7 @@ import { log } from 'crawlee'; import { createAndStartContentCrawler, createAndStartSearchCrawler } from './crawlers.js'; import { processInput, processStandbyInput } from './input.js'; import { getMiniActor } from './mini-actors.js'; +import { initRequestLogger } from './request-logger.js'; import { addTimeoutToAllResponses } from './responses.js'; import { handleSearchNormalMode } from './search.js'; import { createServer } from './server.js'; @@ -37,6 +38,8 @@ if (isActorStandby()) { contentScraperSettings ${JSON.stringify(contentScraperSettings)} `); + initRequestLogger(); + const app = createServer(); app.listen(port, async () => { diff --git a/src/request-logger.ts b/src/request-logger.ts new file mode 100644 index 0000000..f39b4ad --- /dev/null +++ b/src/request-logger.ts @@ -0,0 +1,65 @@ +import { Actor, ApifyClient } from 'apify'; + +import { Routes } from './const.js'; +import { getMiniActor } from './mini-actors.js'; + +let dataset: ReturnType | null = null; +let userId: string | null = null; +let enabled = false; + +// For these routes, the single allow-listed key alone doesn't warrant logging (see logRequest). +const REQUIRED_QUERY_PARAM: Record = { + [Routes.SEARCH]: 'query', + [Routes.FETCH]: 'url', +}; + +/** + * Initialize request logging (no-op if REQUEST_LOG_DATASET_ID is not set). Call once at startup. + */ +export function initRequestLogger(): void { + const datasetId = process.env.REQUEST_LOG_DATASET_ID; + if (!datasetId) return; + + try { + userId = Actor.getEnv().userId; + const client = new ApifyClient({ token: process.env.REQUEST_LOG_DATASET_TOKEN }); + dataset = client.dataset(datasetId); + enabled = true; + } catch { + // Silently disable logging on any failure (invalid dataset id, etc.) + dataset = null; + enabled = false; + } +} + +/** + * Fire-and-forget logging of an incoming request to the request-log dataset. Never throws, never blocks. + * @param pathname the request path (e.g. req.path), also used to look up the route's required query param + * @param query the raw req.query object + */ +export function logRequest(pathname: string, query: Record): void { + if (!enabled || !dataset) return; + + try { + const requiredParamKey = REQUIRED_QUERY_PARAM[pathname]; + let loggedQuery: Record = {}; + + if (requiredParamKey !== undefined) { + const keys = Object.keys(query).filter((key) => key !== requiredParamKey && key !== 'token'); + if (keys.length === 0) return; + loggedQuery = { ...(query as Record) }; + delete loggedQuery[requiredParamKey]; + delete loggedQuery.token; + } + + const item = { + miniActor: getMiniActor().name, + pathname, + timestamp: new Date().toISOString(), + userId, + query: loggedQuery, + }; + + dataset.pushItems(item).catch(() => { /**/ }); + } catch { /**/ } +} diff --git a/src/server.ts b/src/server.ts index d866740..8a38c27 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,6 +5,7 @@ import express, { type Request, type Response } from 'express'; import { Routes } from './const.js'; import { McpServer } from './mcp/server.js'; import { getMiniActor } from './mini-actors.js'; +import { logRequest } from './request-logger.js'; import { handleSearchRequest } from './search.js'; import { extractUserAuthorization } from './utils.js'; @@ -22,6 +23,12 @@ export function createServer(): express.Express { res.status(200).json({ message: `Actor is running in Standby mode. ${HELP_MESSAGE}` }); }); + // Logs every request to the routes below (never reached by '/', which responds without calling next()). + app.use((req, _res, next) => { + logRequest(req.path, req.query as Record); + next(); + }); + app.get(miniActor.route, async (req: Request, res: Response) => { log.info(`Received GET message at: ${req.url}`); await handleSearchRequest(req, res);