feat: add pagination support for item listings and enhance data fetching

Author: Mahcks

README.md

@@ -38,19 +38,26 @@ These endpoints let you list everything or get a specific item:
 
 #### Items
 ```bash
-GET /v1/items                  # List all item IDs (490 items)
-GET /v1/items?full=true        # Get ALL items with full data (1 request!)
-GET /v1/items/{item_id}        # Get a specific item
+GET /v1/items                           # List all item IDs (490 items)
+GET /v1/items?full=true                 # Get first 45 items with full data
+GET /v1/items?full=true&limit=20        # Get first 20 items with full data
+GET /v1/items?full=true&offset=45       # Get next 45 items (pagination)
+GET /v1/items/{item_id}                 # Get a specific item
 ```
 
 **Examples:**
 ```bash
 # Get just the list of IDs
 curl https://arcdata.mahcks.com/v1/items
 
-# Get all items with complete data in one request
+# Get first 45 items with complete data (respects Cloudflare limits)
 curl https://arcdata.mahcks.com/v1/items?full=true
 
+# Paginate through all items
+curl https://arcdata.mahcks.com/v1/items?full=true&offset=0&limit=45
+curl https://arcdata.mahcks.com/v1/items?full=true&offset=45&limit=45
+curl https://arcdata.mahcks.com/v1/items?full=true&offset=90&limit=45
+
 # Get a specific item
 curl https://arcdata.mahcks.com/v1/items/anvil_i
 ```
@@ -120,7 +127,7 @@ Returns the complete JSON data for whatever you requested.
 
 ## Performance Tips
 
-### Use `?full=true` to avoid multiple requests
+### Use `?full=true` with pagination to avoid multiple requests
 
 Instead of making hundreds of requests:
 ```javascript
@@ -131,15 +138,46 @@ const items = await Promise.all(
 );
 ```
 
-Use the `?full=true` parameter:
+Use the `?full=true` parameter with pagination:
 ```javascript
-// ✅ GOOD - Makes 1 request total
-const { items } = await fetch('/v1/items?full=true').then(r => r.json());
+// ✅ GOOD - Fetch all items in pages (11 requests for 490 items)
+async function getAllItems() {
+  let allItems = [];
+  let offset = 0;
+  const limit = 45;
+
+  while (true) {
+    const response = await fetch(`/v1/items?full=true&offset=${offset}&limit=${limit}`);
+    const data = await response.json();
+
+    allItems.push(...data.items);
+
+    // Check if there's more data
+    if (!data.next) break;
+    offset += limit;
+  }
+
+  return allItems;
+}
 ```
 
 **Performance difference:**
 - Without `?full=true`: **491 requests** (~5-10 seconds)
-- With `?full=true`: **1 request** (~500ms)
+- With `?full=true` + pagination: **11 requests** (~2-3 seconds)
+- Each paginated request is cached separately!
+
+**Response includes pagination URLs:**
+```json
+{
+  "type": "items",
+  "total": 490,
+  "count": 45,
+  "offset": 0,
+  "limit": 45,
+  "items": [...],
+  "next": "/v1/items?full=true&offset=45&limit=45"
+}
+```
 
 ## How Caching Works
 

src/index.js

@@ -52,7 +52,9 @@ export default {
 				if (COLLECTION_TYPES[type]) {
 					// Check if user wants full data via ?full=true
 					const full = url.searchParams.get('full') === 'true';
-					return await handleList(type, env, ctx, full);
+					const limit = parseInt(url.searchParams.get('limit')) || undefined;
+					const offset = parseInt(url.searchParams.get('offset')) || 0;
+					return await handleList(type, env, ctx, full, limit, offset);
 				}
 
 				return jsonResponse({ error: `Unknown data type: ${type}` }, 404);
@@ -191,8 +193,10 @@ async function handleGetItem(type, id, env, ctx) {
  * List all items of a type
  * Uses GitHub API to dynamically list directory contents
  * @param {boolean} full - If true, fetch and return full data for all items
+ * @param {number} limit - Max items to return (default: 45 for full, all for list)
+ * @param {number} offset - Skip this many items (for pagination)
  */
-async function handleList(type, env, ctx, full = false) {
+async function handleList(type, env, ctx, full = false, limit = undefined, offset = 0) {
 	if (!COLLECTION_TYPES[type]) {
 		return jsonResponse({ error: `Unknown collection type: ${type}` }, 404);
 	}
@@ -222,43 +226,60 @@ async function handleList(type, env, ctx, full = false) {
 		const files = await githubResponse.json();
 
 		// Filter to only .json files
-		const jsonFiles = files
+		const allJsonFiles = files
 			.filter((f) => f.type === 'file' && f.name.endsWith('.json') && !f.name.startsWith('_'))
 			.map((f) => f.name.replace('.json', ''))
 			.sort((a, b) => a.localeCompare(b));
 
+		const totalItems = allJsonFiles.length;
+
+		// Apply pagination
+		let jsonFiles = allJsonFiles;
+		if (offset > 0) {
+			jsonFiles = jsonFiles.slice(offset);
+		}
+
 		let data;
 
 		if (full) {
-			// Cloudflare Workers has a 50 subrequest limit, so batch the fetches
-			const BATCH_SIZE = 50;
-			const allItems = [];
-
-			for (let i = 0; i < jsonFiles.length; i += BATCH_SIZE) {
-				const batch = jsonFiles.slice(i, i + BATCH_SIZE);
-				const batchPromises = batch.map(async (id) => {
-					const itemUrl = `${GITHUB_BASE}/${dirPath}/${id}.json`;
-					const itemResponse = await fetch(itemUrl, {
-						headers: { 'User-Agent': 'ArcRaiders-API/1.0' },
-					});
-
-					if (itemResponse.ok) {
-						return await itemResponse.json();
-					}
-					return null;
+			// Cloudflare Workers free tier has 50 subrequest limit
+			// We already used 1 for the GitHub API call, so we have 49 left
+			const MAX_ITEMS = 45; // Leave some buffer
+			const effectiveLimit = limit && limit < MAX_ITEMS ? limit : MAX_ITEMS;
+			const limitedFiles = jsonFiles.slice(0, effectiveLimit);
+
+			const itemPromises = limitedFiles.map(async (id) => {
+				const itemUrl = `${GITHUB_BASE}/${dirPath}/${id}.json`;
+				const itemResponse = await fetch(itemUrl, {
+					headers: { 'User-Agent': 'ArcRaiders-API/1.0' },
 				});
 
-				const batchResults = await Promise.all(batchPromises);
-				allItems.push(...batchResults.filter(Boolean));
-			}
+				if (itemResponse.ok) {
+					return await itemResponse.json();
+				}
+				return null;
+			});
+
+			const itemsData = await Promise.all(itemPromises);
 
 			data = {
 				type,
-				count: allItems.length,
-				items: allItems,
+				total: totalItems,
+				count: itemsData.filter(Boolean).length,
+				offset,
+				limit: effectiveLimit,
+				items: itemsData.filter(Boolean),
 			};
+
+			// Add pagination hints
+			if (offset + effectiveLimit < totalItems) {
+				data.next = `/v1/${type}?full=true&offset=${offset + effectiveLimit}&limit=${effectiveLimit}`;
+			}
+			if (offset > 0) {
+				data.prev = `/v1/${type}?full=true&offset=${Math.max(0, offset - effectiveLimit)}&limit=${effectiveLimit}`;
+			}
 		} else {
-			// Just return IDs and URLs (original behavior)
+			// Just return IDs and URLs (original behavior, no limit)
 			const items = jsonFiles.map((id) => ({
 				id,
 				url: `/v1/${type}/${id}`,