@@ -19,6 +19,7 @@ const port = '4001'
 const host = `http://localhost:${port}`
 const version = EnterpriseServerReleases.oldestSupported
 const GH_PAGES_URL = `https://github.github.com/docs-ghes-${version}`
+const DRY_RUN_PAGES_PER_LANGUAGE = 3
 
 type PageList = Page[]
 type MapObj = { [key: string]: string }
@@ -32,7 +33,10 @@ program
     `output directory to place scraped HTML files and redirects. By default, this temp directory is named 'tmpArchivalDir_<VERSION_TO_DEPRECATE>'`,
   )
   .option('-l, --local-dev', 'Do not rewrite asset paths to enable testing scraped content locally')
-  .option('-d, --dry-run', 'only scrape the first 10 pages for testing purposes')
+  .option(
+    '-d, --dry-run',
+    `only scrape the first ${DRY_RUN_PAGES_PER_LANGUAGE} pages in each language for testing purposes`,
+  )
   .option(
     '-p, --page <PATH>',
     'Note: this option is only used to re-scrape a page after the version was deprecated. Redirects will not be re-created because most of the deprecated content is already removed. This option scrapes a specific page in all languages. Pass the relative path to the page without a version or language prefix. ex: /admin/release-notes',
@@ -44,14 +48,12 @@ const dryRun = program.opts().dryRun
 const singlePage = program.opts().page
 if (singlePage && dryRun) {
   console.log(
-    'A dry run cannot be performed when the --page/-p option is used because a dry run scrapes 10 pages at a time.',
+    'A dry run cannot be performed when the --page/-p option is used because a dry run scrapes a sample of pages.',
   )
   process.exit(1)
 }
 const localDev = program.opts().localDev
-const tmpArchivalDirectory = output
-  ? path.join(process.cwd(), output)
-  : path.join(process.cwd(), `tmpArchivalDir_${version}`)
+const tmpArchivalDirectory = path.resolve(output || `tmpArchivalDir_${version}`)
 // rimraf refused to remove a filesystem root. fs.rm does not.
 if (path.resolve(tmpArchivalDirectory) === path.parse(path.resolve(tmpArchivalDirectory)).root) {
   throw new Error(`Refusing to remove filesystem root: ${tmpArchivalDirectory}`)
@@ -75,10 +77,10 @@ async function main() {
     const permalinksPerVersion = Object.keys(pageMap)
       .filter((key) => key.includes(`/enterprise-server@${version}`))
       .map((href) => `${host}${href}`)
-    urls = dryRun ? permalinksPerVersion.slice(0, 10) : permalinksPerVersion
+    urls = dryRun ? sampleEachLanguage(permalinksPerVersion) : permalinksPerVersion
     if (dryRun) {
       console.log(
-        `\nThis is a dry run! Creating HTML for redirects and scraping the first 10 pages only:\n${urls.join('\n')}\n`,
+        `\nThis is a dry run! Creating HTML for redirects and scraping the first ${DRY_RUN_PAGES_PER_LANGUAGE} pages in each language only:\n${urls.join('\n')}\n`,
       )
     } else {
       console.log(`Found ${urls.length} pages for version ${version}`)
@@ -87,6 +89,7 @@ async function main() {
 
   await fs.promises.rm(tmpArchivalDirectory, { recursive: true, force: true })
 
+  const errorResponses: string[] = []
   const app = createApp()
   const server = http.createServer(app)
   server
@@ -103,7 +106,10 @@ async function main() {
           directory: tmpArchivalDirectory,
           filenameGenerator: 'bySiteStructure',
           requestConcurrency: 6,
-          plugins: [new RewriteAssetPathsPlugin(tmpArchivalDirectory, localDev, GH_PAGES_URL)],
+          plugins: [
+            new RewriteAssetPathsPlugin(tmpArchivalDirectory, localDev, GH_PAGES_URL),
+            new ErrorResponsesPlugin(errorResponses),
+          ],
         })
       } catch (err) {
         console.error('scraping error')
@@ -112,6 +118,16 @@ async function main() {
         return
       }
 
+      // website-scraper saves error pages like any other response, so fail loudly instead.
+      if (errorResponses.length) {
+        console.error(`\n\n${errorResponses.length} responses had an error status:`)
+        console.error(errorResponses.slice(0, 50).join('\n'))
+        if (errorResponses.length > 50) console.error(`...and ${errorResponses.length - 50} more`)
+        console.error('\nRe-run with DEBUG_MIDDLEWARE_TESTS=true to see server error details.')
+        server.close(() => process.exit(1))
+        return
+      }
+
       fs.renameSync(
         path.join(tmpArchivalDirectory, `/localhost_${port}`),
         path.join(tmpArchivalDirectory, version),
@@ -133,6 +149,39 @@ async function main() {
     })
 }
 
+function sampleEachLanguage(urls: string[]) {
+  const counts: Record<string, number> = {}
+  return urls.filter((url) => {
+    const language = new URL(url).pathname.split('/')[1]
+    counts[language] = (counts[language] || 0) + 1
+    return counts[language] <= DRY_RUN_PAGES_PER_LANGUAGE
+  })
+}
+
+type ScraperResponse = { statusCode: number; url: string }
+
+class ErrorResponsesPlugin {
+  errorResponses: string[]
+
+  constructor(errorResponses: string[]) {
+    this.errorResponses = errorResponses
+  }
+
+  apply(
+    registerAction: (
+      event: string,
+      callback: (args: { response: ScraperResponse }) => Promise<ScraperResponse>,
+    ) => void,
+  ) {
+    registerAction('afterResponse', async ({ response }) => {
+      if (response.statusCode >= 400) {
+        this.errorResponses.push(`${response.statusCode} ${response.url}`)
+      }
+      return response
+    })
+  }
+}
+
 async function createRedirectsFile(pageList: PageList, outputDirectory: string) {
   console.log('Creating redirects file...')
   const redirects = await loadRedirects(pageList)