From 3db72f2ecd44f47d0b11f104db008801ebe65d9f Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Wed, 22 Jul 2026 16:49:53 +0100 Subject: [PATCH 1/2] chore: create a log for http requests --- src/main.ts | 4 +++ src/request-logger.ts | 67 +++++++++++++++++++++++++++++++++++++++++++ src/server.ts | 7 +++++ 3 files changed, 78 insertions(+) create mode 100644 src/request-logger.ts diff --git a/src/main.ts b/src/main.ts index 402cf4a..b8c9d1b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,6 +4,7 @@ import { log } from 'crawlee'; import { createAndStartContentCrawler, createAndStartSearchCrawler } from './crawlers.js'; import { processInput, processStandbyInput } from './input.js'; import { getMiniActor } from './mini-actors.js'; +import { initRequestLogger } from './request-logger.js'; import { addTimeoutToAllResponses } from './responses.js'; import { handleSearchNormalMode } from './search.js'; import { createServer } from './server.js'; @@ -37,6 +38,9 @@ if (isActorStandby()) { contentScraperSettings ${JSON.stringify(contentScraperSettings)} `); + // Fire-and-forget: never delay server startup/real traffic for this best-effort observability feature. + void initRequestLogger(); + const app = createServer(); app.listen(port, async () => { diff --git a/src/request-logger.ts b/src/request-logger.ts new file mode 100644 index 0000000..c82714d --- /dev/null +++ b/src/request-logger.ts @@ -0,0 +1,67 @@ +import { Actor, ApifyClient } from 'apify'; + +import { Routes } from './const.js'; +import { getMiniActor } from './mini-actors.js'; + +let dataset: ReturnType | null = null; +let userId: string | null = null; +let enabled = false; + +// For these routes, the single allow-listed key alone doesn't warrant logging (see logRequest). +const REQUIRED_QUERY_PARAM: Record = { + [Routes.SEARCH]: 'query', + [Routes.FETCH]: 'url', +}; + +/** + * Initialize request logging (no-op if REQUEST_LOG_DATASET_ID is not set). Call once at startup. + */ +export async function initRequestLogger(): Promise { + const datasetId = process.env.REQUEST_LOG_DATASET_ID; + if (!datasetId) return; + + try { + userId = Actor.getEnv().userId; + const client = new ApifyClient({ token: process.env.REQUEST_LOG_DATASET_TOKEN }); + dataset = client.dataset(datasetId); + enabled = true; + } catch { + // Silently disable logging on any failure (invalid dataset id, etc.) + dataset = null; + enabled = false; + } +} + +/** + * Fire-and-forget logging of an incoming request to the request-log dataset. Never throws, never blocks. + * @param pathname the request path (e.g. req.path), also used to look up the route's required query param + * @param query the raw req.query object + */ +export function logRequest(pathname: string, query: Record): void { + if (!enabled || !dataset) return; + + try { + const requiredParamKey = REQUIRED_QUERY_PARAM[pathname]; + let loggedQuery: Record = {}; + + if (requiredParamKey !== undefined) { + const keys = Object.keys(query).filter((key) => key !== requiredParamKey); + if (keys.length === 0) return; + loggedQuery = { ...(query as Record) }; + delete loggedQuery[requiredParamKey]; + } + + // Never persist the token query param (used for standby-mode auth) into the logged item. + delete loggedQuery.token; + + const item = { + miniActor: getMiniActor().name, + pathname, + timestamp: new Date().toISOString(), + userId, + query: loggedQuery, + }; + + dataset.pushItems(item).catch(() => { /**/ }); + } catch { /**/ } +} diff --git a/src/server.ts b/src/server.ts index d866740..8a38c27 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,6 +5,7 @@ import express, { type Request, type Response } from 'express'; import { Routes } from './const.js'; import { McpServer } from './mcp/server.js'; import { getMiniActor } from './mini-actors.js'; +import { logRequest } from './request-logger.js'; import { handleSearchRequest } from './search.js'; import { extractUserAuthorization } from './utils.js'; @@ -22,6 +23,12 @@ export function createServer(): express.Express { res.status(200).json({ message: `Actor is running in Standby mode. ${HELP_MESSAGE}` }); }); + // Logs every request to the routes below (never reached by '/', which responds without calling next()). + app.use((req, _res, next) => { + logRequest(req.path, req.query as Record); + next(); + }); + app.get(miniActor.route, async (req: Request, res: Response) => { log.info(`Received GET message at: ${req.url}`); await handleSearchRequest(req, res); From 7679c67b33712dda6924958012aeb162b1a2da5d Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 23 Jul 2026 13:34:00 +0100 Subject: [PATCH 2/2] review feedback --- src/main.ts | 3 +-- src/request-logger.ts | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main.ts b/src/main.ts index b8c9d1b..010b4e1 100644 --- a/src/main.ts +++ b/src/main.ts @@ -38,8 +38,7 @@ if (isActorStandby()) { contentScraperSettings ${JSON.stringify(contentScraperSettings)} `); - // Fire-and-forget: never delay server startup/real traffic for this best-effort observability feature. - void initRequestLogger(); + initRequestLogger(); const app = createServer(); diff --git a/src/request-logger.ts b/src/request-logger.ts index c82714d..b92ff5a 100644 --- a/src/request-logger.ts +++ b/src/request-logger.ts @@ -16,7 +16,7 @@ const REQUIRED_QUERY_PARAM: Record = { /** * Initialize request logging (no-op if REQUEST_LOG_DATASET_ID is not set). Call once at startup. */ -export async function initRequestLogger(): Promise { +export function initRequestLogger(): void { const datasetId = process.env.REQUEST_LOG_DATASET_ID; if (!datasetId) return; @@ -44,6 +44,8 @@ export function logRequest(pathname: string, query: Record): vo const requiredParamKey = REQUIRED_QUERY_PARAM[pathname]; let loggedQuery: Record = {}; + delete loggedQuery.token; + if (requiredParamKey !== undefined) { const keys = Object.keys(query).filter((key) => key !== requiredParamKey); if (keys.length === 0) return; @@ -51,9 +53,6 @@ export function logRequest(pathname: string, query: Record): vo delete loggedQuery[requiredParamKey]; } - // Never persist the token query param (used for standby-mode auth) into the logged item. - delete loggedQuery.token; - const item = { miniActor: getMiniActor().name, pathname,