chore: create a log for http requests

src/main.ts

@@ -4,6 +4,7 @@ import { log } from 'crawlee';
 import { createAndStartContentCrawler, createAndStartSearchCrawler } from './crawlers.js';
 import { processInput, processStandbyInput } from './input.js';
 import { getMiniActor } from './mini-actors.js';
+import { initRequestLogger } from './request-logger.js';
 import { addTimeoutToAllResponses } from './responses.js';
 import { handleSearchNormalMode } from './search.js';
 import { createServer } from './server.js';
@@ -37,6 +38,8 @@ if (isActorStandby()) {
         contentScraperSettings ${JSON.stringify(contentScraperSettings)}
     `);
 
+    initRequestLogger();
+
     const app = createServer();
 
     app.listen(port, async () => {

src/request-logger.ts

@@ -0,0 +1,66 @@
+import { Actor, ApifyClient } from 'apify';
+
+import { Routes } from './const.js';
+import { getMiniActor } from './mini-actors.js';
+
+let dataset: ReturnType<ApifyClient['dataset']> | null = null;
+let userId: string | null = null;
+let enabled = false;
+
+// For these routes, the single allow-listed key alone doesn't warrant logging (see logRequest).
+const REQUIRED_QUERY_PARAM: Record<string, string> = {
+    [Routes.SEARCH]: 'query',
+    [Routes.FETCH]: 'url',
+};
+
+/**
+ * Initialize request logging (no-op if REQUEST_LOG_DATASET_ID is not set). Call once at startup.
+ */
+export function initRequestLogger(): void {
+    const datasetId = process.env.REQUEST_LOG_DATASET_ID;
+    if (!datasetId) return;
+
+    try {
+        userId = Actor.getEnv().userId;
+        const client = new ApifyClient({ token: process.env.REQUEST_LOG_DATASET_TOKEN });
+        dataset = client.dataset(datasetId);
+        enabled = true;
+    } catch {
+        // Silently disable logging on any failure (invalid dataset id, etc.)
+        dataset = null;
+        enabled = false;
+    }
+}
+
+/**
+ * Fire-and-forget logging of an incoming request to the request-log dataset. Never throws, never blocks.
+ * @param pathname the request path (e.g. req.path), also used to look up the route's required query param
+ * @param query the raw req.query object
+ */
+export function logRequest(pathname: string, query: Record<string, unknown>): void {
+    if (!enabled || !dataset) return;
+
+    try {
+        const requiredParamKey = REQUIRED_QUERY_PARAM[pathname];
+        let loggedQuery: Record<string, string> = {};
+
+        delete loggedQuery.token;
+
+        if (requiredParamKey !== undefined) {
+            const keys = Object.keys(query).filter((key) => key !== requiredParamKey);
+            if (keys.length === 0) return;
+            loggedQuery = { ...(query as Record<string, string>) };
+            delete loggedQuery[requiredParamKey];
+        }
+
+        const item = {
+            miniActor: getMiniActor().name,
+            pathname,
+            timestamp: new Date().toISOString(),
+            userId,
+            query: loggedQuery,
+        };
+
+        dataset.pushItems(item).catch(() => { /**/ });
+    } catch { /**/ }
+}

src/server.ts

@@ -5,6 +5,7 @@ import express, { type Request, type Response } from 'express';
 import { Routes } from './const.js';
 import { McpServer } from './mcp/server.js';
 import { getMiniActor } from './mini-actors.js';
+import { logRequest } from './request-logger.js';
 import { handleSearchRequest } from './search.js';
 import { extractUserAuthorization } from './utils.js';
 
@@ -22,6 +23,12 @@ export function createServer(): express.Express {
         res.status(200).json({ message: `Actor is running in Standby mode. ${HELP_MESSAGE}` });
     });
 
+    // Logs every request to the routes below (never reached by '/', which responds without calling next()).
+    app.use((req, _res, next) => {
+        logRequest(req.path, req.query as Record<string, unknown>);
+        next();
+    });
+
     app.get(miniActor.route, async (req: Request, res: Response) => {
         log.info(`Received GET message at: ${req.url}`);
         await handleSearchRequest(req, res);