feat: add metascraper-pdf

README.md

@@ -253,6 +253,7 @@ const metascraper = require('metascraper')([
 - [metascraper-logo](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-logo) – Get logo property from HTML markup.
 - [metascraper-manifest](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-manifest) –  Metascraper integration for detecting PWA Web app [manifests](https://developer.mozilla.org/en-US/docs/Web/Manifest).
 - [metascraper-media-provider](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-media-provider) – Get specific video provider url (Facebook/Twitter/Vimeo/etc).
+- [metascraper-pdf](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-pdf) – Get title, author, date, description, publisher, image, logo, and lang from a PDF document.
 - [metascraper-publisher](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-publisher) – Get publisher property from HTML markup.
 - [metascraper-readability](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-readability) – A Mozilla readability connector for metascraper.
 - [metascraper-title](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-title) – Get title property from HTML markup.

packages/metascraper-pdf/README.md

@@ -0,0 +1,124 @@
+<div align="center">
+  <br>
+  <img style="width: 500px; margin:3rem 0 1.5rem;" src="https://metascraper.js.org/static/logo-banner.png" alt="metascraper">
+  <br>
+  <br>
+  <p align="center"><strong>metascraper-pdf</strong>: Get title, author, date, description, publisher, image, logo, and lang out of a PDF document.</p>
+  <p align="center">See our <a href="https://metascraper.js.org" target='_blank' rel='noopener noreferrer'>website</a> for more information.</p>
+  <br>
+</div>
+
+## Install
+
+```bash
+$ npm install metascraper-pdf --save
+```
+
+## Usage
+
+The rules download the document at `url` when it looks like a PDF.
+
+```js
+const metascraper = require('metascraper')([require('metascraper-pdf')()])
+
+const metadata = await metascraper({
+  url: 'https://arxiv.org/pdf/1706.03762v7'
+})
+
+// {
+//   title: 'Attention Is All You Need',
+//   author: 'Ashish Vaswani',
+//   publisher: 'arXiv',
+//   date: '2017-06-01T00:00:00.000Z',
+//   description: 'The dominant sequence transduction models are based on…',
+//   lang: 'en',
+//   logo: 'https://www.google.com/s2/favicons?domain_url=…',
+//   image: null
+// }
+```
+
+The bundle is a no-op unless [`.test()`](#testprops) sees a PDF URL, so it is safe to mix with the HTML rules:
+
+```js
+const metascraper = require('metascraper')([
+  require('metascraper-pdf')(),
+  require('metascraper-title')(),
+  require('metascraper-author')()
+])
+
+const metadata = await metascraper({ url, html })
+```
+
+## How it reads a document
+
+The package fetches the URL, then reads the page the way a person does. Embedded PDF metadata is
+mostly unusable — arXiv ships an empty `Title`, LaTeX ships `pedregosa11a.dvi`, Word ships
+`Microsoft Word - draft.docx`, conference templates ship the venue as the `Subject`.
+
+- **title** — the largest type on the first page, skipping the banner publishers print above it
+  (`NBER WORKING PAPER SERIES`, `arXiv:2303.08774v6`, `REVIEW`) and any byline set in the same size.
+- **author** — the block under the title, stripped of emails, affiliation superscripts and
+  organisation names. Following metascraper's convention this returns a single name.
+- **description** — the abstract, or the first paragraph of body text when there is no abstract.
+- **publisher** — the venue in the running header or footer; otherwise a known host (`arXiv`,
+  `NBER`, `PLOS`) or the domain.
+- **date** — the identifier when the url encodes it (an arXiv id is a year-month; proceedings hosts
+  put the year in the path). Otherwise the markers on the page win over the PDF creation date.
+- **lang** — `dc:language` when present, then the host, then the words on the page.
+- **image** — a first-page figure encoded as a PNG data URI, when the PDF embeds one that is not
+  just a decoration.
+- **logo** — a first-page mark encoded as a PNG data URI, or the publisher favicon.
+
+## API
+
+### metascraper-pdf([options])
+
+#### options
+
+##### maxPages
+
+Type: `number`<br>
+Default: `2`
+
+How many pages to read text from. The title, author and publisher only ever come from the first
+page; the extra page feeds the description when a document has no abstract.
+
+##### gotOpts
+
+Type: `object`
+
+Any option provided here will be passed to [got#options](https://github.com/sindresorhus/got#options).
+
+##### keyvOpts
+
+Type: `object`
+
+Any option provided here will be passed to [@keyvhq/memoize#options](https://github.com/microlinkhq/keyv/tree/master/packages/memoize#keyvoptions).
+
+##### getPdf
+
+Type: `function`
+
+It will be called to get the PDF bytes behind `url`. Defaults to downloading the URL with `got`.
+
+### .test(props)
+
+Type: `function`<br>
+Returns: `boolean`
+
+`true` when `props.url` points at a PDF (`.pdf`, an `/pdf` path, or `type=printable`), which is how
+the bundle stays inert for HTML input.
+
+```js
+const { test: isPdf } = require('metascraper-pdf')
+
+isPdf({ url: 'https://arxiv.org/pdf/1706.03762v7' }) // => true
+isPdf({ url: 'https://example.com' }) // => false
+```
+
+## License
+
+**metascraper-pdf** © [microlink.io](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/metascraper/blob/master/LICENSE.md) License.<br>
+Authored and maintained by [Kiko Beats](https://kikobeats.com) with help from [contributors](https://github.com/microlinkhq/metascraper/contributors).
+
+> [microlink.io](https://microlink.io) · GitHub [microlink.io](https://github.com/microlinkhq) · X [@microlinkhq](https://x.com/microlinkhq)

packages/metascraper-pdf/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "metascraper-pdf",
+  "description": "Metascraper rules for extracting title, author, date, description, publisher, image, logo, and lang from PDF documents.",
+  "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-pdf",
+  "version": "5.57.0",
+  "types": "src/index.d.ts",
+  "main": "src/index.js",
+  "author": {
+    "email": "hello@microlink.io",
+    "name": "microlink.io",
+    "url": "https://microlink.io"
+  },
+  "repository": {
+    "directory": "packages/metascraper-pdf",
+    "type": "git",
+    "url": "git+https://github.com/microlinkhq/metascraper.git"
+  },
+  "bugs": {
+    "url": "https://github.com/microlinkhq/metascraper/issues"
+  },
+  "keywords": [
+    "document",
+    "extract",
+    "metadata",
+    "metascraper",
+    "paper",
+    "pdf",
+    "scraper"
+  ],
+  "dependencies": {
+    "@keyvhq/memoize": "~2.2.4",
+    "@metascraper/helpers": "workspace:*",
+    "async-memoize-one": "~1.2.1",
+    "got": "~11.8.6",
+    "unpdf": "~1.8.1"
+  },
+  "devDependencies": {
+    "async-listen": "~3.1.0",
+    "ava": "8",
+    "metascraper": "workspace:*"
+  },
+  "engines": {
+    "node": ">= 22"
+  },
+  "files": [
+    "src"
+  ],
+  "scripts": {
+    "test": "NODE_PATH=.. TZ=UTC ava --timeout 30s"
+  },
+  "license": "MIT",
+  "ava": {
+    "files": [
+      "test/**/*.js",
+      "!test/helpers/**"
+    ]
+  }
+}

packages/metascraper-pdf/src/author.js

@@ -0,0 +1,153 @@
+'use strict'
+
+const {
+  ORGANIZATION_WORDS,
+  PLACE_NAME,
+  flatten,
+  isBannerLine,
+  isInvertedName,
+  isPersonName,
+  splitNamePairs,
+  splitNames,
+  stripNoise,
+  tidy
+} = require('./text')
+
+const EDITOR_PREFIX =
+  /^(edited|reviewed|approved|submitted|received|accepted|published)\s+by\s*:?|^(editors?|reviewing editors?|action editors?)\s*:/i
+const SECTION_WORDS =
+  /^(abstract|summary|introduction|contents|table of contents|keywords|index|preface|foreword|version|draft)\b/i
+
+const AUTHOR_BLOCK_MARGIN = 4
+const EDITOR_BLOCK_LINES = 4
+const MAX_ORGANIZATION_WORDS = 4
+const MAX_AUTHORS = 10
+const NEIGHBOUR_OFFSETS = [1, 2, 3, 4]
+
+const isCapitalizedWord = word => /^[\p{Lu}]/u.test(word)
+
+const isOrganizationAuthor = text => {
+  if (/\S+@\S+/.test(text) || /\d/.test(text) || /^https?:/i.test(text)) {
+    return false
+  }
+  if (SECTION_WORDS.test(text) || PLACE_NAME.test(text)) return false
+  const words = text.split(/\s+/)
+  return (
+    words.length <= MAX_ORGANIZATION_WORDS && words.every(isCapitalizedWord)
+  )
+}
+
+const editorBlock = lines => {
+  const excluded = new Set()
+
+  for (const line of lines) {
+    if (!EDITOR_PREFIX.test(line.text)) continue
+    for (let offset = 0; offset <= EDITOR_BLOCK_LINES; offset++) {
+      excluded.add(line.index + offset)
+    }
+  }
+
+  return excluded
+}
+
+const toAuthor = (lines, indexes, options = {}) => {
+  const { organizationLimit = Infinity, allowOrganization = false } = options
+  const excluded = editorBlock(lines)
+  const usable = index => !excluded.has(index)
+
+  const names = indexes
+    .filter(usable)
+    .map(index => lines[index])
+    .filter(Boolean)
+    .map(line => line.text)
+    .filter(
+      text =>
+        !EDITOR_PREFIX.test(text) &&
+        !ORGANIZATION_WORDS.test(text) &&
+        !isBannerLine(text)
+    )
+    .flatMap(text => stripNoise(text).split(/;\s*/).flatMap(splitNames))
+    .map(tidy)
+    .filter(isPersonName)
+
+  const unique = [...new Set(names.map(flatten))].slice(0, MAX_AUTHORS)
+  if (unique.length > 1) return unique.join(', ')
+
+  const paired = indexes
+    .filter(usable)
+    .map(index => lines[index])
+    .filter(line => line && !ORGANIZATION_WORDS.test(line.text))
+    .flatMap(line => splitNamePairs(stripNoise(line.text)))
+    .filter(isPersonName)
+  if (paired.length > unique.length) {
+    return [...new Set(paired)].slice(0, MAX_AUTHORS).join(', ')
+  }
+  if (unique.length > 0) return unique.join(', ')
+
+  if (!allowOrganization) return null
+
+  const organization = indexes
+    .filter(index => index <= organizationLimit && usable(index))
+    .map(index => lines[index])
+    .filter(Boolean)
+    .map(line => flatten(stripNoise(line.text)))
+    .find(isOrganizationAuthor)
+
+  return organization || null
+}
+
+/**
+ * Bylines share a font size. Once one name is found, every line set in the same
+ * size around it belongs to the same block, which is what recovers the authors
+ * hidden between affiliation and email lines.
+ */
+const expandAuthorLines = (lines, indexes, { titleIndexes = [] } = {}) => {
+  const excludedTitle = new Set(titleIndexes)
+  const usable = indexes.filter(index => !excludedTitle.has(index))
+  const named = usable
+    .map(index => lines[index])
+    .filter(line => line && isPersonName(stripNoise(line.text)))
+
+  if (named.length === 0) return usable
+
+  const sizes = new Set(named.map(line => line.size))
+  const first = Math.min(...named.map(line => line.index)) - AUTHOR_BLOCK_MARGIN
+  const last = Math.max(...named.map(line => line.index)) + AUTHOR_BLOCK_MARGIN
+
+  return lines
+    .filter(
+      line =>
+        line.index >= first &&
+        line.index <= last &&
+        !excludedTitle.has(line.index)
+    )
+    .filter(line => sizes.has(line.size) && isPersonName(stripNoise(line.text)))
+    .map(line => line.index)
+}
+
+const nameCount = value => {
+  if (!value) return 0
+  return value.split(/\s*;\s*/).reduce((count, part) => {
+    const trimmed = part.trim()
+    if (!trimmed) return count
+    if (isInvertedName(trimmed)) return count + 1
+    return count + trimmed.split(/,|\s+and\s+/i).filter(Boolean).length
+  }, 0)
+}
+
+const getAuthor = (lines, { titleIndexes = [] } = {}) => {
+  const titleIndex =
+    titleIndexes.length > 0 ? titleIndexes[titleIndexes.length - 1] : 0
+  const neighbours = NEIGHBOUR_OFFSETS.map(offset => titleIndex + offset)
+  const organization = {
+    allowOrganization: true,
+    organizationLimit: titleIndex + AUTHOR_BLOCK_MARGIN
+  }
+
+  return (
+    toAuthor(lines, expandAuthorLines(lines, neighbours, { titleIndexes })) ||
+    toAuthor(lines, neighbours, organization)
+  )
+}
+
+module.exports = { expandAuthorLines, getAuthor, nameCount, toAuthor }